From 9ead3875f6af99f0dc56a2e4c4f81c070652ecd1 Mon Sep 17 00:00:00 2001 From: Andrey Golovanov Date: Fri, 21 Aug 2026 02:03:25 +0100 Subject: [PATCH] Correctness, performance, and documentation overhaul Multi-round review program over the analysis engine, model layer, DSL, and documentation. Findings were verified by runtime probes and, for the higher-risk ones, adversarially re-checked before being applied. Correctness (probe-confirmed, each with regression tests): - Weighted failure sampling computed Efraimidis-Spirakis keys as u**(1/w), which underflows for small weights: with per-hour failure rates (~1e-5) one entity was selected 97% of the time, biased by entity id rather than weight. Keys are now computed in the log domain. - Demand placement could report more flow than the network carries. An SPF-cached demand and a FlowPolicy-based demand sharing (source, destination, priority) produced colliding FlowIndex values whose flows silently merged; a plain-YAML scenario reported 15 units placed across a 10-unit min cut. - Demand expansion could fuse distinct demands' pseudo endpoints, because composed ids concatenate demand ids and group labels and both may contain "|". Duplicate ids and colliding pseudo endpoints are now rejected. - A risk group sharing a name with a node was misclassified under failure: the node was excluded and the group's members were not. - expand_groups expanded differently depending on whether the failure came from an entity rule or a risk_group rule, and did not reach members of nested groups. - Seeded Monte Carlo was not reproducible across identical scenario rebuilds: uuid-suffixed link ids sorted differently per rebuild, remapping seeded draws onto different parallel links. Link ids are now deterministic. - Boundary guards replace silent corruption: link capacities at or above the internal pseudo-edge capacity, accumulated path costs that overflow the core's int64 arithmetic, bound-context/argument mismatches, scalar in/not_in values, and unhashable group_by values now raise instead of producing wrong numbers. Performance: - Weighted-choice selection precomputes per-rule weight splits and selects via a heap (~4x faster on 100k-candidate pools). - MaximumSupportedDemand probes share one SPF DAG cache, cutting SPF runs from probes x sources to sources. - Membership rules flatten entity attributes once instead of once per rule. Structure: - Selector evaluation moved to ngraph.model.selectors; the DSL package keeps YAML-facing parsing and re-exports the moved names. - FailureManager pre-builds per-run inputs through a prepare_inputs hook on analysis functions, replacing kwarg-name sniffing in the engine. - Repeated logic consolidated behind shared helpers; dead and production-orphaned code removed; the package layering policy is documented. Documentation: - Every curated document was verified by execution: all Python examples, 46 DSL YAML blocks, and the bundled scenarios run against this tree, and documented outputs match. The design reference's algorithm sections were checked line-by-line against the NetGraph-Core C++ sources, which corrected unsupported complexity bounds and a mischaracterization of reverse residual arcs. - A prose pass tightened docstrings and docs, followed by independent verification that restored facts the tightening had dropped. Suite grows to 1186 tests; coverage 91.65%. make check-ci, make validate, and make docs are green. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 100 + Makefile | 2 +- README.md | 2 +- dev/generate_api_docs.py | 10 +- docs/assets/diagrams/system_pipeline.dot | 2 +- docs/assets/diagrams/system_pipeline.dot.svg | 2 +- docs/examples/basic.md | 6 +- docs/examples/bundled-scenarios.md | 4 +- docs/examples/clos-fabric.md | 16 +- docs/getting-started/installation.md | 5 +- docs/getting-started/tutorial.md | 4 +- docs/index.md | 16 +- docs/reference/api-full.md | 1642 +++++++++-------- docs/reference/api.md | 76 +- docs/reference/cli.md | 70 +- docs/reference/design.md | 189 +- docs/reference/dsl.md | 225 ++- docs/reference/schemas.md | 6 +- docs/reference/workflow.md | 35 +- ngraph/__init__.py | 9 +- ngraph/analysis/__init__.py | 11 +- ngraph/analysis/context.py | 1215 +++++++----- ngraph/analysis/demand.py | 145 +- ngraph/analysis/failure_manager.py | 546 +++--- ngraph/analysis/functions.py | 247 ++- ngraph/analysis/placement.py | 99 +- ngraph/cli.py | 193 +- ngraph/dsl/__init__.py | 3 +- ngraph/dsl/blueprints/__init__.py | 4 +- ngraph/dsl/blueprints/expand.py | 464 ++--- ngraph/dsl/blueprints/parser.py | 4 +- ngraph/dsl/expansion/__init__.py | 7 +- ngraph/dsl/expansion/brackets.py | 49 +- ngraph/dsl/expansion/schema.py | 5 +- ngraph/dsl/expansion/variables.py | 61 +- ngraph/dsl/loader.py | 6 +- ngraph/dsl/selectors/__init__.py | 32 +- ngraph/dsl/selectors/normalize.py | 97 +- ngraph/explorer.py | 143 +- ngraph/lib/__init__.py | 5 +- ngraph/lib/nx.py | 60 +- ngraph/logging.py | 57 +- ngraph/model/__init__.py | 6 +- ngraph/model/components.py | 88 +- ngraph/model/demand/__init__.py | 3 +- ngraph/model/demand/builder.py | 30 +- ngraph/model/demand/matrix.py | 36 +- ngraph/model/demand/spec.py | 52 +- ngraph/model/failure/generate.py | 47 +- ngraph/model/failure/membership.py | 186 +- ngraph/model/failure/parser.py | 75 +- ngraph/model/failure/policy.py | 548 ++++-- ngraph/model/failure/policy_set.py | 9 +- ngraph/model/failure/validation.py | 14 +- ngraph/model/flow/__init__.py | 4 +- ngraph/model/flow/policy_config.py | 17 +- ngraph/model/network.py | 122 +- ngraph/model/path.py | 16 +- ngraph/model/selectors/__init__.py | 53 + ngraph/{dsl => model}/selectors/conditions.py | 12 +- ngraph/model/selectors/parse.py | 77 + ngraph/{dsl => model}/selectors/schema.py | 59 +- ngraph/{dsl => model}/selectors/select.py | 140 +- ngraph/profiling/__init__.py | 2 +- ngraph/profiling/profiler.py | 27 +- ngraph/results/artifacts.py | 101 +- ngraph/results/flow.py | 12 +- ngraph/results/snapshot.py | 66 +- ngraph/results/store.py | 47 +- ngraph/scenario.py | 195 +- ngraph/schemas/scenario.json | 77 +- ngraph/types/__init__.py | 10 +- ngraph/types/base.py | 27 +- ngraph/types/dto.py | 6 +- ngraph/utils/ids.py | 9 +- ngraph/utils/output_paths.py | 7 +- ngraph/utils/seed_manager.py | 4 +- ngraph/utils/yaml_utils.py | 15 +- ngraph/workflow/base.py | 130 +- ngraph/workflow/build_graph.py | 54 +- ngraph/workflow/cost_power.py | 28 +- ngraph/workflow/max_flow_step.py | 62 +- .../workflow/maximum_supported_demand_step.py | 131 +- ngraph/workflow/network_stats.py | 51 +- .../workflow/traffic_matrix_placement_step.py | 124 +- pyproject.toml | 2 +- scenarios/backbone_clos.yml | 53 +- scenarios/nsfnet.yaml | 1 - scenarios/square_mesh.yaml | 2 - tests/analysis/test_context_review_fixes.py | 364 ++++ .../test_demand_expansion_semantics.py | 319 ++++ tests/analysis/test_failure_manager.py | 20 +- tests/analysis/test_failure_manager_fixes.py | 549 ++++++ tests/analysis/test_functions.py | 84 +- tests/analysis/test_functions_details.py | 1 - .../test_functions_mode_validation.py | 65 + tests/analysis/test_maxflow_api.py | 42 +- tests/analysis/test_paths.py | 182 ++ tests/analysis/test_placement.py | 106 +- tests/analysis/test_profile_env_restore.py | 31 + tests/cli/test_cli.py | 48 +- tests/cli/test_cli_helpers.py | 6 +- tests/cli/test_cli_inspect_fixes.py | 158 ++ tests/cli/test_cli_profile_hook.py | 69 + tests/cli/test_package_layering.py | 22 + tests/dsl/test_dsl_features_validation.py | 17 +- tests/dsl/test_examples.py | 2 - tests/dsl/test_expand_review_fixes.py | 481 +++++ tests/dsl/test_expansion.py | 110 +- tests/dsl/test_native_substitution_guards.py | 93 + tests/dsl/test_selectors.py | 15 - tests/dsl/test_skill_examples_validation.py | 16 +- tests/explorer/test_explorer_review_fixes.py | 184 ++ tests/lib/test_nx.py | 15 +- tests/lib/test_nx_regressions.py | 160 ++ tests/logging/test_library_logging_pattern.py | 107 ++ tests/logging/test_logging.py | 8 +- .../test_components_yaml_edge_cases.py | 73 + tests/model/demand/test_builder.py | 101 +- tests/model/demand/test_spec.py | 12 +- tests/model/failure/test_conditions_unit.py | 2 +- tests/model/failure/test_failure_trace.py | 26 +- tests/model/failure/test_policy.py | 17 +- tests/model/failure/test_policy_expansion.py | 62 +- .../failure/test_policy_random_selection.py | 99 + .../test_policy_serialization_roundtrip.py | 104 ++ .../failure/test_policy_zero_weight_modes.py | 91 + tests/model/failure/test_risk_group_parser.py | 210 +++ tests/model/test_layering.py | 106 ++ tests/model/test_network_basics.py | 33 + tests/model/test_types_base.py | 34 + tests/profiling/test_worker_profile_merge.py | 87 + tests/results/test_capacity_envelope_unit.py | 59 + tests/results/test_result.py | 16 +- tests/results/test_store_deep_convert.py | 65 + tests/scenario/test_scenario.py | 61 +- .../test_scenario_disabled_risk_groups.py | 144 ++ tests/scenario/test_scenario_run_hook.py | 141 ++ tests/scenario/test_schema_validation.py | 5 +- .../workflow/test_alpha_resolution_errors.py | 61 + tests/workflow/test_build_graph_attrs.py | 123 ++ .../test_capacity_envelope_analysis.py | 2 +- tests/workflow/test_cost_power.py | 59 + .../workflow/test_maximum_supported_demand.py | 48 +- tests/workflow/test_msd_perf_safety.py | 1 + .../test_placement_rounds_deprecated.py | 101 + tests/workflow/test_seed_provenance.py | 68 + tests/workflow/test_step_name_collision.py | 62 + .../workflow/test_traffic_matrix_placement.py | 99 +- 149 files changed, 9958 insertions(+), 4199 deletions(-) create mode 100644 ngraph/model/selectors/__init__.py rename ngraph/{dsl => model}/selectors/conditions.py (93%) create mode 100644 ngraph/model/selectors/parse.py rename ngraph/{dsl => model}/selectors/schema.py (77%) rename ngraph/{dsl => model}/selectors/select.py (63%) create mode 100644 tests/analysis/test_context_review_fixes.py create mode 100644 tests/analysis/test_demand_expansion_semantics.py create mode 100644 tests/analysis/test_failure_manager_fixes.py create mode 100644 tests/analysis/test_functions_mode_validation.py create mode 100644 tests/analysis/test_profile_env_restore.py create mode 100644 tests/cli/test_cli_inspect_fixes.py create mode 100644 tests/cli/test_cli_profile_hook.py create mode 100644 tests/cli/test_package_layering.py create mode 100644 tests/dsl/test_expand_review_fixes.py create mode 100644 tests/dsl/test_native_substitution_guards.py create mode 100644 tests/explorer/test_explorer_review_fixes.py create mode 100644 tests/lib/test_nx_regressions.py create mode 100644 tests/logging/test_library_logging_pattern.py create mode 100644 tests/model/components/test_components_yaml_edge_cases.py create mode 100644 tests/model/failure/test_policy_random_selection.py create mode 100644 tests/model/failure/test_policy_serialization_roundtrip.py create mode 100644 tests/model/failure/test_policy_zero_weight_modes.py create mode 100644 tests/model/failure/test_risk_group_parser.py create mode 100644 tests/model/test_layering.py create mode 100644 tests/model/test_types_base.py create mode 100644 tests/profiling/test_worker_profile_merge.py create mode 100644 tests/results/test_capacity_envelope_unit.py create mode 100644 tests/results/test_store_deep_convert.py create mode 100644 tests/scenario/test_scenario_disabled_risk_groups.py create mode 100644 tests/scenario/test_scenario_run_hook.py create mode 100644 tests/workflow/test_alpha_resolution_errors.py create mode 100644 tests/workflow/test_build_graph_attrs.py create mode 100644 tests/workflow/test_placement_rounds_deprecated.py create mode 100644 tests/workflow/test_seed_provenance.py create mode 100644 tests/workflow/test_step_name_collision.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3893e2e..825f4fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,106 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Combine-mode demand expansion with overlapping source/target selections (including `per_group` and `group_pairwise`) no longer routes volume through a zero-cost pseudo-node bypass: shared nodes are excluded from the target side, so `demand_placement_analysis`, `TrafficMatrixPlacement`, and `MaximumSupportedDemand` report placement bounded by real network capacity. A source group left with no targets is skipped; a fully-overlapping `flatten`/`combine` demand fails with `No demands could be expanded` +- Fractional link and augmentation costs now raise `ValueError` naming the offending links when the analysis graph is built, instead of being silently truncated to int64 (which corrupted SPF, KSP, cost distributions, and max-flow results); `from_networkx` applies the same check to edge costs +- **BREAKING**: `MaxFlowResult.min_cut` (`max_flow_detailed` with `include_min_cut=True`) now returns a true minimum cut whose capacity equals the max flow, instead of all saturated edges; saturated-edge analysis remains available via `sensitivity()` +- Monte Carlo analysis with `seed=None` now falls back to the failure policy's own seed, restoring per-iteration variation (a seeded policy previously produced one identical failure pattern every iteration) +- Demand-placement Monte Carlo no longer crashes with `KeyError` when demand configs lack `id`; ids are derived deterministically from source/target/position +- `group_mode: per_group` now matches documented semantics: `combine` creates one demand per source group, `pairwise` pairs nodes within each same-label group, and volume is split evenly across groups so totals are conserved — except for the share of a skipped group (combine-mode overlap exclusion can empty a group's targets, and a pairwise label with no non-self pairs is skipped) +- Risk-group failure expansion is transitive and independent of which rule kind failed: exclusions now include members of nested (grandchild) risk groups, and members of a rule-failed group seed `expand_groups` the same way whether the failure came from an entity rule or a `risk_group` rule +- The disable cascade for `disabled: true` risk groups now runs after membership rules and `generate` blocks, so entities assigned to risk groups by those mechanisms are correctly disabled +- `membership`, `disabled`, and `generate` keys on nested risk-group `children` entries are now rejected (by the JSON schema at load and by the parser with `ValueError`) instead of being silently ignored; only top-level groups are registered in `network.risk_groups`, so define such groups at top level and reference them by name as children +- Link `expand:` blocks now substitute variables in every string field (nested `attrs`, `risk_groups`, selector `match` values), matching documented behavior, and match-only selectors with expand blocks are no longer silently ignored. A variable used as a whole-string value (e.g. a match condition `value: "${t}"`) keeps its native type, so selectors match numeric node/link attributes with `==`/`!=`/`in`/`not_in`; placeholders embedded in longer strings still interpolate as text, and a bare placeholder bound to a non-string variable used as a path selector raises `ValueError` instead of being silently stringified +- Blueprint instantiation paths are regex-escaped when building blueprint link selectors, so group names containing regex metacharacters no longer link sibling subtrees or silently drop links +- Blueprint `params` overrides with unknown subgroup prefixes or malformed keys raise `ValueError` instead of being silently ignored (nested blueprint params use the dict-valued `.params` form), and subgroup names containing dots (e.g. `rack.a.count` for subgroup `rack.a`) now resolve by longest-`.`-prefix match against literal blueprint subgroup names instead of failing with a contradictory "matches no node group" error; when one subgroup name is a dotted extension of another, the longest match wins deterministically +- `link_rules` entries now require `source` and `target` (enforced by schema and expansion), and non-list `node_rules`/`link_rules` sections raise `ValueError` instead of being silently skipped +- Failure modes with zero weight are never applied, and scenario loading rejects policies whose modes all have zero weight +- Unbound `AnalysisContext` flow methods (`max_flow`, `max_flow_detailed`, `sensitivity`) now honor custom augmentations passed to `analyze()` +- `k_shortest_paths` between multi-node groups merges results across all source/sink node pairs instead of returning KSP for only the single best pair, and returns a deterministic selection when equal-cost paths exceed `max_k` (structural tie-breaking, independent of `PYTHONHASHSEED`); `shortest_paths` output ordering is deterministic for equal-cost paths +- Bound pairwise analysis results for overlapping/empty pairs (`sensitivity`, `max_flow_detailed`, `sensitivity_with_flow`) no longer share a single mutable default object across pairs +- FailureManager's prepared-policy match cache verifies policy identity to avoid stale results after `id()` address reuse +- A `FailureRule` with a mistyped `scope` (e.g. `nodes`) now fails at construction instead of silently matching nothing; a bare-string `risk_groups:` value anywhere in the DSL now raises `ValueError` instead of silently expanding per character +- Weighted failure sampling (`weight_by`) computes selection keys in the log domain: very small weights (e.g. per-hour failure rates around 1e-5) previously underflowed to zero and silently inverted the selection bias toward lexicographically larger entity IDs +- Demand placement no longer reports phantom flow: an SPF-cached demand and a FlowPolicy-based demand sharing (source, destination, priority) produced colliding flow ids whose flows silently merged, so reported placement could exceed the network's min cut. Cached flow ids now live in a disjoint range, and two policy-based demands with the same triple are rejected with `ValueError` +- Demand-expansion pseudo endpoints can no longer merge silently: `group_pairwise` composed demand ids are injective, `expand_demands` rejects duplicate `TrafficDemand` ids, and a post-expansion check rejects any two expansions claiming the same pseudo endpoint (ids and group labels containing `|` can compose identically). Each such collision previously fused distinct demands' pseudo nodes, resurrecting the zero-cost bypass +- A risk group sharing its name with a node or link is excluded correctly under failure: `compute_exclusions` classifies failed entities by rule scope (via the new `apply_failures_typed`) instead of probing merged IDs against network collections +- Seeded Monte Carlo results are reproducible across identical scenario rebuilds: link IDs use a deterministic per-(source, target) sequence (`A|B|0`) instead of a random uuid suffix, whose rebuild-dependent sort order re-mapped seeded draws onto different parallel links; `add_link` now raises on id collisions (possible when node names contain `|`) and on re-adding a link, instead of silently overwriting +- `MaximumSupportedDemand` probes `alpha_min` before declaring infeasibility on downward bracket exhaustion (a large `alpha_start` could previously raise a false "No feasible alpha found") +- `NetworkExplorer` paths no longer truncate at hierarchy segments literally named `root`; the walk stops at the synthetic tree root by identity +- Loud errors replace silent corruption at validation boundaries: the analysis graph build rejects link capacities at or above the internal pseudo-edge capacity (1e15) and cost totals reaching 2^62 (accumulated path costs overflow the core's int64 arithmetic, verified against the C++ implementation); `max_flow_analysis`/`sensitivity_analysis` reject a bound context whose binding differs from the call arguments; `in`/`not_in` conditions require list values at parse time; `generate` blocks report unhashable `group_by` values and same-name renders from distinct values (e.g. int `1` vs str `"1"`) as `ValueError` naming the attribute; and a demand endpoint missing from a supplied analysis context reports the likely config/context mismatch instead of a raw `KeyError` +- Inline dict `flow_policy` values are rejected at scenario build time with a `ValueError` listing valid presets, instead of being accepted and crashing later in analysis; boolean `flow_policy` values (`true`/`false`) are likewise rejected instead of silently coercing to integer presets 1/0 +- Duplicate effective workflow step names raise `ValueError` before execution instead of silently overwriting results; `Scenario.run()` validates step names up front +- Workflow metadata `seed_source`/`active_seed` now report the seed a step actually uses; steps constructed without their own seed record `seed_source: none` +- `TrafficMatrixPlacement` with `alpha_from_step` naming a missing or not-yet-run producer step raises a clear error pointing at the step +- `flatten_node_attrs`/`flatten_link_attrs` expose `risk_groups` as a sorted list, making `group_by` labels and comparisons deterministic +- `BuildGraph` no longer crashes with `TypeError` when node/link attrs collide with reserved keys (`disabled`, `id`, `capacity`, `cost`); reserved keys take precedence in the exported node-link graph +- `ngraph inspect --detail`: the "Top demands (by offered volume)" table is now actually sorted by demand volume +- `NetworkExplorer.get_bom_map` honors `include_root`, and a custom `root_label` no longer duplicates the root BOM under the empty-string key +- `ComponentsLibrary.from_yaml` treats an empty or null `components:` mapping as an empty library, and a warning is logged when a component definition contains an unrecognized `cost` key (the parser reads `capex`) +- `ngraph run --profile`: per-worker `.pstats` profiles are merged into step profiles again (glob fixed from `*_worker_*` to `*_thread_*`; the Workers column previously always showed `-`), and `NGRAPH_PROFILE_DIR` no longer leaks into the process environment — it is restored (or removed) after the run, preventing stale worker-thread profiling in later in-process runs +- `ngraph run --stdout` now emits pure JSON on stdout: status banners, the `--profile` performance report, and run error messages are written to stderr, so output is safe to pipe to `jq` +- `scenarios/backbone_clos.yml` failure rules now nest conditions/logic under `match:`; `make validate` also covers `*.yml` scenario files (previously silently skipped) +- Docs: accuracy pass over the reference and getting-started documentation — every Python example and YAML snippet executes against the current code, and the design reference's algorithm sections (SPF/max-flow/KSP pseudocode, complexity bounds, residual-arc semantics, constants) were verified line-by-line against the NetGraph-Core C++ sources. Corrected: an inverted `require_capacity` description; results file locations and export shapes; link-id format; membership/generate contracts; YAML anchor-merge behavior; invalid YAML in condition examples; a "circular hierarchy" example that did not actually fail; a NetworkX write-back example that could not run and mislabeled edges; unsupported max-flow complexity bounds; node/link rule `risk_groups` documented as additive when they replace; a `one_to_one` self-pair example that creates no links; a YAML selector whose escape sequence made it invalid; workflow step `parallelism` and the CLI profiling notes, which now say "worker threads" (not processes) to match thread-based execution; and the `Scenario.from_yaml` docstring, which now reflects the actual risk-group processing order (membership resolution before `generate` blocks, disabled-member cascade after both) + +### Changed + +- **BREAKING**: `from_networkx` `bidirectional` now defaults to `None` and is inferred from the graph type: undirected inputs produce antiparallel arc pairs per edge (preserving connectivity) instead of a single arbitrary arc; pass `bidirectional=False` to restore the old conversion +- Monte Carlo execution is thread-based throughout: nothing is pickled, so analysis functions defined in `__main__` or a notebook run at full parallelism instead of being forced serial +- `FailurePolicy.to_dict` now emits the scenario YAML failure-policy format (conditions/logic nested under `match`, `weight_by` included when set, no derived `seed` key) and round-trips through `build_failure_policy`; the exported `scenario.failures` snapshot shape changed accordingly +- Scenario results snapshot serializes demand `flow_policy` as the preset name string (e.g. `SHORTEST_PATHS_ECMP`) instead of a bare integer +- `Results.to_dict` recursively converts nested output: dict keys are stringified and tuples emitted as lists throughout, making exports JSON-safe without `default=str` fallbacks +- Invalid `mode` strings in `max_flow_analysis`, `sensitivity_analysis`, and `build_maxflow_context` raise `ValueError` instead of silently selecting pairwise; parsing is case-insensitive +- Selector evaluation (schema types, condition evaluation, node selection, attribute flattening) and `parse_match_spec` moved from `ngraph.dsl.selectors` to `ngraph.model.selectors`; the DSL module keeps YAML-facing parsing and re-exports all moved names, and the model layer no longer imports the DSL package at import time or runtime +- Validation is unified at construction and parse boundaries: failure-rule `match` blocks go through the shared `parse_match_spec`; `FailureRule` and `TrafficDemand` validate their field vocabularies in `__post_init__`, replacing scattered, partly missing checks during expansion and selection; workflow steps validate `parallelism`/`mode` through the shared helpers; and malformed DSL values (e.g. non-dict child `attrs`) raise `ValueError` consistently across blueprint and nested node groups +- FailureManager pre-builds per-run analysis inputs through a `prepare_inputs` hook carried by the built-in analysis functions, replacing kwarg-name sniffing in the engine; custom analysis functions opt in by setting the attribute, and passing `context` explicitly skips the hook +- `run_monte_carlo_analysis` metadata includes `occurrence_counts` aligned with `results`, so custom result types get correct pattern multiplicities; docs clarify that deduplication assumes deterministic analysis functions and that failure traces describe each pattern's representative iteration +- Demand placement semantics are documented and pinned: `SHORTEST_PATHS_*` presets admit flow onto the cost-only shortest paths of the base topology and drop overflow (IGP semantics), while `TE_*` presets reroute remaining volume onto residual-capacity paths +- `TrafficDemand.to_dict()` is the canonical serialized demand form used by the results snapshot and step `base_demands` outputs (now consistently including `group_mode` and `attrs`), and demand configs round-trip it faithfully: `flow_policy` preset names/ints are coerced to the enum and config defaults match `TrafficDemand`'s (a config without `mode` now means `combine`, previously `pairwise`) +- `AnalysisContext.build_node_mask`/`build_edge_mask` are public methods for custom analysis functions calling Core primitives directly, replacing both the private methods and the unused module-level wrappers +- `ngraph inspect` quiets the `ngraph` package logger (not just the CLI logger) while printing the network hierarchy, so explorer INFO logs no longer interleave with the table output +- Importing `ngraph` no longer installs a stdout stream handler or eagerly imports `ngraph.cli`: the library attaches only a `logging.NullHandler` (importing `ngraph.logging` has no other side effects), the CLI configures logging explicitly in `main()`, and console logs go to stderr +- `CostPower` no longer constructs a `NetworkExplorer` internally; it always completes and reports aggregated capex/power even on networks that strict hardware validation would reject (hardware validation remains available via NetworkStats/inspect), and an O(nodes x links) validation scan is gone +- Performance: work that used to repeat per iteration now happens once — bound flow calls precompute expected pair keys at bind time; demand expansion, node-ID resolution, the Monte Carlo dedup-key kwargs component, and blueprint `params` resolution are computed once per run; and FailureManager builds the risk-group expansion index once per manager. PAIRWISE pseudo-attachment edges are created once per group member (O(G x members) instead of O(G^2 x members)), and plain unbound contexts build the Core graph lazily on first use +- Performance: `mode: random` failure selection uses a binomial draw plus uniform sample on Python 3.12+ (O(failures) instead of O(matched)); seeded RNG streams for `mode: random` differ from previous versions on Python 3.12+ (distribution unchanged) +- Performance: NetworkExplorer hardware validation and `ngraph inspect` aggregation use single-pass O(V+E)/O(E) scans instead of per-entity link loops; CapacityEnvelope frequency aggregation uses `collections.Counter` (O(n) instead of O(n^2)) +- Performance: `k_shortest_paths` between multi-node groups prunes per-pair KSP runs — pairs are processed in ascending shortest-cost order with early termination once the top-k result cannot change, instead of running KSP for every reachable node pair +- Performance: weighted-choice failure selection precomputes per-rule weight splits and selects via a heap (about 4x faster on 100k-candidate pools); `MaximumSupportedDemand` probes share one SPF DAG cache (SPF runs drop from probes x sources to sources); membership rules flatten entity attributes once instead of per rule; pairwise overlap checks reuse per-group sets; and `group_by` selection resolves attributes directly +- Internal: repeated logic across analysis, DSL, and workflow layers is consolidated behind shared helpers (selector resolution, sensitivity decoding, Monte Carlo result serialization, blueprint parent merging, membership matching), defensive `getattr`/log-swallowing patterns on typed objects are removed, and function-local imports are reserved for optional dependencies (the package layering policy is now documented in the design reference) + +### Added + +- `Scenario.run(step_hook=...)`: optional callable returning a context manager entered around each workflow step's execution; the CLI `--profile` path now drives execution through it instead of a hand-rolled loop +- `AnalysisContext.sensitivity_with_flow`: computes max flow and edge sensitivity per group pair in a single pass; `sensitivity_analysis` and the sensitivity Monte Carlo hot path use it (results unchanged) +- `build_demand_placement_inputs` helper, exported from `ngraph.analysis`; `demand_placement_analysis` accepts precomputed `expansion=`/`resolved_ids=` +- `FailurePolicy.build_risk_group_index` static method and the `prepared_rg_index` parameter on `apply_failures` for reuse across Monte Carlo iterations +- `Mode.from_string` in `ngraph.types` for validated, case-insensitive parsing of analysis mode strings +- `TrafficDemand.to_dict()` returning the canonical serialized demand form (flow policy as preset name) +- `link_path_key` in `ngraph.model.selectors`: the canonical "source|target" key used when path-matching links +- `netgraph-core` dependency floor raised to 0.7.0, the API family this release is developed and tested against; the previous 0.3.0 floor predated APIs the library now uses +- `FailurePolicy.apply_failures_typed` (scope-typed failure sets; `apply_failures` remains as a merged-list wrapper) and `FailurePolicy.prepare_weights` for reusing weighted-selection splits across Monte Carlo iterations + +### Removed + +- **BREAKING**: failure-policy `expand_children` flag (field, parser, schema, serialization); cascading a failed risk group to its children is inherent and always applied, and scenarios using the key must drop it (previously the flag crashed when enabled and had no effect when disabled) +- **BREAKING**: inline-object `flow_policy` form from the scenario schema; use a preset name string +- `placement_rounds` parameter from `demand_placement_analysis` and `FailureManager.run_demand_placement_monte_carlo` (it never affected placement; the core engine handles optimization internally) +- Dead `TrafficDemand.volume_placed` and `TrafficDemand.flow_policy_obj` fields; routing is selected solely via the `flow_policy` preset +- Unused `FailurePatternResult`, `DemandSet.to_dict`, `ngraph.types.MIN_CAP`/`MIN_FLOW` constants, and the scenario schema's unused `linkProperties` definition +- `NetworkExplorer.get_node_utilization` no-op `include_disabled` parameter and the always-False `NodeUtilization.disabled` field +- `cli` and `logging` from `ngraph.__all__` (both remain importable as explicit submodules) +- `build_demand_context` from `ngraph.analysis` (use `build_demand_placement_inputs`, which also returns the expansion and resolved node IDs) +- `select_nodes` `excluded_nodes` parameter (no production caller; analysis exclusions are applied via Core masks) and dict-input support from `flatten_risk_group_attrs` (all callers pass `RiskGroup` objects) +- Production-orphaned public API: `Scenario.seed_manager` property and `expand_templates` from `ngraph.dsl.expansion` (`expand_block` covers its use case) +- Dead code across the analysis, DSL, workflow, and CLI layers: write-only fields, unused parameters and branches, and test-only helpers that had leaked into production modules + +### Deprecated + +- `placement_rounds` on `MaximumSupportedDemand` and `TrafficMatrixPlacement` workflow steps: still accepted in YAML for backward compatibility but logs a warning, has no effect, and is no longer exported in result contexts + ## [0.21.0] - 2026-03-26 ### Fixed diff --git a/Makefile b/Makefile index aa6819d..4392c7a 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ perf: validate: @echo "📋 Validating YAML schemas..." @if $(PYTHON) -c "import jsonschema" >/dev/null 2>&1; then \ - $(PYTHON) -c "import json, yaml, jsonschema, pathlib; from importlib import resources as res; f=res.files('ngraph.schemas').joinpath('scenario.json').open('r', encoding='utf-8'); schema=json.load(f); f.close(); scenario_files=list(pathlib.Path('scenarios').rglob('*.yaml')); integration_files=list(pathlib.Path('tests/integration').glob('*.yaml')); all_files=scenario_files+integration_files; [jsonschema.validate(yaml.safe_load(open(fp)), schema) for fp in all_files]; print(f'✅ Validated {len(all_files)} YAML files against schema ({len(scenario_files)} scenarios, {len(integration_files)} integration tests)')"; \ + $(PYTHON) -c "import json, yaml, jsonschema, pathlib; from importlib import resources as res; f=res.files('ngraph.schemas').joinpath('scenario.json').open('r', encoding='utf-8'); schema=json.load(f); f.close(); scenario_files=sorted(set(pathlib.Path('scenarios').rglob('*.yaml')) | set(pathlib.Path('scenarios').rglob('*.yml'))); integration_files=sorted(set(pathlib.Path('tests/integration').glob('*.yaml')) | set(pathlib.Path('tests/integration').glob('*.yml'))); all_files=scenario_files+integration_files; [jsonschema.validate(yaml.safe_load(open(fp)), schema) for fp in all_files]; print(f'✅ Validated {len(all_files)} YAML files against schema ({len(scenario_files)} scenarios, {len(integration_files)} integration tests)')"; \ else \ echo "⚠️ jsonschema not installed. Skipping schema validation"; \ fi diff --git a/README.md b/README.md index 5d47989..5dfaedc 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ print(result) # {('^A$', '^C$'): 10.0} ## Scenario DSL -For reproducible analysis workflows, define topology, traffic, demands, and failure policies in YAML: +For reproducible analysis workflows, define topology, demands, and failure policies in YAML: ```yaml seed: 42 diff --git a/dev/generate_api_docs.py b/dev/generate_api_docs.py index 36d84b0..24e5832 100755 --- a/dev/generate_api_docs.py +++ b/dev/generate_api_docs.py @@ -279,11 +279,19 @@ def get_class_info(cls): else: default_val = None + default_str = str(default_val) if default_val is not None else None + # Drop non-reproducible reprs such as + # "": the address changes + # on every run, so the documented value is meaningless to a reader + # and makes the generated file differ between runs. + if default_str is not None and " object at 0x" in default_str: + default_str = None + info["attributes"].append( { "name": field_name, "type": field_type, - "default": str(default_val) if default_val is not None else None, + "default": default_str, } ) diff --git a/docs/assets/diagrams/system_pipeline.dot b/docs/assets/diagrams/system_pipeline.dot index add6cc3..0837d1d 100644 --- a/docs/assets/diagrams/system_pipeline.dot +++ b/docs/assets/diagrams/system_pipeline.dot @@ -19,7 +19,7 @@ digraph SystemPipeline { subgraph cluster_model { label=""; style=invis; - scenario [label="Scenario\n·Network\n·FailurePolicySet\n·TrafficMatrixSet\n·Workflow", shape=box]; + scenario [label="Scenario\n·Network\n·FailurePolicySet\n·DemandSet\n·Workflow", shape=box]; } subgraph cluster_exec { diff --git a/docs/assets/diagrams/system_pipeline.dot.svg b/docs/assets/diagrams/system_pipeline.dot.svg index 61b9717..9027fc9 100644 --- a/docs/assets/diagrams/system_pipeline.dot.svg +++ b/docs/assets/diagrams/system_pipeline.dot.svg @@ -78,7 +78,7 @@ Scenario ·Network ·FailurePolicySet -·TrafficMatrixSet +·DemandSet ·Workflow diff --git a/docs/examples/basic.md b/docs/examples/basic.md index da14997..4a91e8d 100644 --- a/docs/examples/basic.md +++ b/docs/examples/basic.md @@ -1,6 +1,6 @@ # Basic Example -This example builds a tiny topology inline to show APIs. For real analysis, prefer running a provided scenario and generating metrics via the CLI. +A tiny topology defined inline, used here to walk through the analysis APIs. For real analysis, run a bundled scenario through the CLI and generate metrics from that. See [Tutorial](../getting-started/tutorial.md) for CLI usage and bundled scenarios. @@ -19,7 +19,7 @@ See [Tutorial](../getting-started/tutorial.md) for CLI usage and bundled scenari They have the same metric of 1 but different capacities (1 and 2). ``` -Let's create this network by using NetGraph's scenario system: +Build it with the scenario system: ```python from ngraph.scenario import Scenario @@ -77,7 +77,7 @@ scenario = Scenario.from_yaml(scenario_yaml) network = scenario.network ``` -Note that here we used a simple `nodes` and `links` structure to directly define the network topology. The optional `seed` parameter ensures reproducible results when using randomized workflow steps. In more complex scenarios, you would typically use node groups with `count` and `template` to define groups of nodes and link rules to define their connections, or even leverage the `blueprints` to create reusable components. This advanced functionality is explained in the [DSL Reference](../reference/dsl.md) and used in the [Clos Fabric Analysis](clos-fabric.md) example. +This spells out every node and link individually. The optional `seed` makes randomized workflow steps reproducible. Larger topologies instead use node groups (`count` plus `template`) with link rules connecting them, or `blueprints` for reusable components - see the [DSL Reference](../reference/dsl.md) and the [Clos Fabric Analysis](clos-fabric.md) example. ### Flow Analysis Variants diff --git a/docs/examples/bundled-scenarios.md b/docs/examples/bundled-scenarios.md index 0592554..0e32d0f 100644 --- a/docs/examples/bundled-scenarios.md +++ b/docs/examples/bundled-scenarios.md @@ -10,7 +10,7 @@ Inspect first, then run: # Inspect (structure, steps, demands, failure policies) ngraph inspect scenarios/backbone_clos.yml --detail -# Run and write JSON results next to the scenario (or under --output) +# Run and write JSON results in the current directory (or under --output) ngraph run scenarios/backbone_clos.yml --output out ``` @@ -76,4 +76,4 @@ ngraph run scenarios/nsfnet.yaml --keys node_to_node_capacity_matrix_1 --stdout ## Notes on results -All runs emit a consistent JSON shape with `workflow`, `steps`, and `scenario` sections. Steps like `MaxFlow` and `TrafficMatrixPlacement` store per-iteration lists under `data.flow_results` with `summary` and optional `cost_distribution` or `min_cut` fields. See Reference -> Workflow for the exact schema. +All runs emit a consistent JSON shape with `workflow`, `steps`, and `scenario` sections. Steps like `MaxFlow` and `TrafficMatrixPlacement` store a list under `data.flow_results` with one entry per unique failure pattern - patterns are deduplicated across iterations, so the list holds at most `iterations` entries and usually far fewer - alongside a single unfailed entry under `data.baseline`; with no `failure_policy`, `flow_results` is empty. Each entry carries a `summary` and per-flow `flows` entries whose `cost_distribution` is populated when `include_flow_details` is set (and `{}` otherwise), and with `include_min_cut` the min-cut edges appear under a flow entry's `data` (`edges` plus `edges_kind: "min_cut"`). See Reference -> Workflow for the exact schema. diff --git a/docs/examples/clos-fabric.md b/docs/examples/clos-fabric.md index 143d22f..15dbfc2 100644 --- a/docs/examples/clos-fabric.md +++ b/docs/examples/clos-fabric.md @@ -1,16 +1,12 @@ # Clos Fabric Analysis -This example demonstrates analysis of a 3-tier Clos fabric. For production use, run the bundled scenario and generate metrics via CLI, then iterate in Python if needed. +Analysis of a 3-tier Clos fabric. For production use, run the bundled scenario and generate metrics via CLI, then iterate in Python if needed. Refer to [Tutorial](../getting-started/tutorial.md) for running bundled scenarios via CLI. ## Scenario Overview -We'll create two separate 3-tier Clos networks and analyze the maximum flow capacity between them. This scenario showcases: - -- Hierarchical blueprint composition -- Complex link patterns -- Flow analysis with different placement policies +Two separate 3-tier Clos networks, with maximum flow capacity measured between them. The scenario nests blueprints inside blueprints, wires the tiers with `mesh` and `one_to_one` link patterns, and compares flow placement policies. ## Programmatic scenario @@ -112,14 +108,14 @@ NetGraph supports different flow placement policies: Combined with the path selection settings (shortest_path=True|False), we can achieve different flow placement policies emulating ECMP, WCMP, and TE behavior in IP/MPLS networks. -In this example, we use the `FlowPlacement.EQUAL_BALANCED` policy and `shortest_path=True` to emulate ECMP behavior and we will compare it with WCMP `FlowPlacement.PROPORTIONAL` (capacity-weighted split across equal-cost paths) under two conditions: +The example above pairs `FlowPlacement.EQUAL_BALANCED` with `shortest_path=True` to emulate ECMP. Compare it against `FlowPlacement.PROPORTIONAL` (WCMP) under two conditions: - Baseline: symmetric parallel inter-spine links -> ECMP = WCMP (256.0). -- Uneven links: make capacities within each equal-cost bundle different -> WCMP +- Uneven links: capacities differ within each equal-cost bundle -> WCMP achieves higher throughput than ECMP, which is limited by equal splitting. -We emulate partial inter-spine degradation by making capacities uneven across the -4 parallel spine-to-spine links per pair while keeping equal costs. This isolates +Partial inter-spine degradation is emulated by making capacities uneven across the +4 parallel spine-to-spine links per pair while keeping costs equal, which isolates the effect of the splitting policy. ```python diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index c15ffab..0412397 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -54,12 +54,15 @@ pip install -e . # Install NetGraph cd ../NetGraph pip install -e . + +# Or, with development tooling (tests, linters, docs): +pip install -e '.[dev]' ``` ## Platform Notes **Pre-built wheels**: Available for Linux (x86_64, aarch64), macOS (x86_64, arm64), and Windows (x86_64). -**Building from source**: Requires CMake 3.15+. Builds automatically during `pip install` if no compatible wheel is available. +**Building from source**: Requires CMake 3.23+ and a C++20 compiler (per netgraph-core's build configuration). Builds automatically during `pip install` if no compatible wheel is available. **Next**: See [Tutorial](tutorial.md) for running scenarios and programmatic usage examples. diff --git a/docs/getting-started/tutorial.md b/docs/getting-started/tutorial.md index d2568a3..b42c862 100644 --- a/docs/getting-started/tutorial.md +++ b/docs/getting-started/tutorial.md @@ -1,6 +1,6 @@ # Tutorial -This guide shows the fastest way to run a scenario from the CLI and a minimal programmatic example. See the Examples section for detailed scenarios and policies. +The fastest way to run a scenario from the CLI, plus a minimal programmatic example. See the Examples section for fuller scenarios and for flow placement and failure policies. ## CLI: run and inspect @@ -8,7 +8,7 @@ This guide shows the fastest way to run a scenario from the CLI and a minimal pr # Inspect (validate and preview structure, steps, demands) ngraph inspect scenarios/square_mesh.yaml --detail -# Run and store results (JSON) next to the scenario or under --output +# Run and store results (JSON) in the current directory or under --output ngraph run scenarios/square_mesh.yaml --output out # Filter exported results by workflow step names diff --git a/docs/index.md b/docs/index.md index 0c1ab24..6377f87 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,18 +2,18 @@ [![Python-test](https://github.com/networmix/NetGraph/actions/workflows/python-test.yml/badge.svg?branch=main)](https://github.com/networmix/NetGraph/actions/workflows/python-test.yml) -Scenario-driven network modeling and analysis framework combining Python's flexibility with high-performance C++ algorithms. +Scenario-driven network modeling and analysis framework combining Python with C++ graph algorithms. ## Overview -NetGraph enables declarative modeling of network topologies, traffic matrices, and failure scenarios. It delegates computationally intensive graph algorithms to [NetGraph-Core](https://github.com/networmix/NetGraph-Core) while providing a rich Python API and CLI for orchestration. +Model network topologies, traffic matrices, and failure scenarios declaratively. Graph algorithms are delegated to [NetGraph-Core](https://github.com/networmix/NetGraph-Core); NetGraph itself provides the Python API and CLI that orchestrate them. ## Architecture -NetGraph employs a **hybrid Python+C++ architecture**: +NetGraph uses a hybrid Python+C++ architecture, split across two layers: - **Python layer (NetGraph)**: Scenario DSL parsing, workflow orchestration, result aggregation, and high-level APIs. -- **C++ layer (NetGraph-Core)**: Performance-critical graph algorithms (SPF, KSP, Max-Flow) executing in optimized C++ with the GIL released. +- **C++ layer (NetGraph-Core)**: The compute-intensive graph algorithms (SPF, KSP, Max-Flow), executed in C++ with the GIL released. ## Key Features @@ -31,15 +31,15 @@ NetGraph employs a **hybrid Python+C++ architecture**: ### Traffic Engineering -- **Routing Modes**: Unified modeling of **IP Routing** (static costs, oblivious to congestion) and **Traffic Engineering** (dynamic residuals, congestion-aware). -- **Flow Placement**: Strategies for **ECMP** (Equal-Cost Multi-Path) and **WCMP** (Weighted Cost Multi-Path). +- **Routing Modes**: Unified modeling of IP routing (static costs, oblivious to congestion) and traffic engineering (dynamic residuals, congestion-aware). +- **Flow Placement**: Strategies for ECMP (Equal-Cost Multi-Path) and WCMP (Weighted Cost Multi-Path). - **Capacity Analysis**: Compute max-flow envelopes and demand allocation with configurable placement policies. ### Workflow & Integration - **Structured Results**: Export analysis artifacts to JSON for downstream processing. -- **CLI**: Comprehensive command-line interface for validation and execution. -- **Python API**: Full programmatic access to all modeling and solving capabilities. +- **CLI**: Validate, inspect, and run scenarios from the command line. +- **Python API**: Programmatic access to the same modeling and solving entry points. ## Getting Started diff --git a/docs/reference/api-full.md b/docs/reference/api-full.md index 24d1b09..e08a475 100644 --- a/docs/reference/api-full.md +++ b/docs/reference/api-full.md @@ -12,9 +12,9 @@ Quick links: - [CLI Reference](cli.md) - [DSL Reference](dsl.md) -Generated from source code on: February 08, 2026 at 18:38 UTC +Generated from source code on: August 21, 2026 at 00:49 UTC -Modules auto-discovered: 52 +Modules auto-discovered: 53 --- @@ -34,11 +34,15 @@ Args: ## ngraph.explorer -NetworkExplorer class for analyzing network hierarchy and structure. +Hierarchical exploration of a Network. + +Builds a tree of the node-name hierarchy and aggregates per-subtree +statistics — node and link counts, capacity, capex/power, and hardware +bills of materials — in two modes: all nodes, and enabled nodes only. ### ExternalLinkBreakdown -Holds stats for external links to a particular other subtree. +Stats for external links to one other subtree. Attributes: link_count (int): Number of links to that other subtree. @@ -51,7 +55,7 @@ Attributes: ### LinkCapacityIssue -Represents a link capacity constraint violation in active topology. +A link capacity constraint violation in active topology. Attributes: source: Source node name. @@ -70,8 +74,10 @@ Attributes: ### NetworkExplorer -Provides hierarchical exploration of a Network, computing statistics in two modes: -'all' (ignores disabled) and 'active' (only enabled). +Hierarchical view of a Network with per-subtree statistics. + +Statistics are computed in two modes: 'all' (ignores disabled) and +'active' (only enabled). **Methods:** @@ -80,7 +86,7 @@ Provides hierarchical exploration of a Network, computing statistics in two mode - `get_bom_by_path(self, path: 'str', include_disabled: 'bool' = True) -> 'Dict[str, float]'` - Return the hardware BOM for a specific hierarchy path. - `get_bom_map(self, include_disabled: 'bool' = True, include_root: 'bool' = True, root_label: 'str' = '') -> 'Dict[str, Dict[str, float]]'` - Return a mapping from hierarchy path to BOM for each subtree. - `get_link_issues(self) -> 'List[LinkCapacityIssue]'` - Return recorded link capacity issues discovered in non-strict mode. -- `get_node_utilization(self, include_disabled: 'bool' = True) -> 'List[NodeUtilization]'` - Return hardware utilization per node based on active topology. +- `get_node_utilization(self) -> 'List[NodeUtilization]'` - Return hardware utilization per node based on active topology. - `print_tree(self, node: 'Optional[TreeNode]' = None, indent: 'int' = 0, max_depth: 'Optional[int]' = None, skip_leaves: 'bool' = False, detailed: 'bool' = False, include_disabled: 'bool' = True, max_external_lines: 'Optional[int]' = None, line_prefix: 'str' = '') -> 'None'` - Print the hierarchy from 'node' down (default: root). ### NodeUtilization @@ -101,7 +107,6 @@ Attributes: ports_utilization: Ratio of used to available ports (0.0 when N/A). capacity_violation: True if attached capacity exceeds supported capacity. ports_violation: True if used ports exceed available ports. - disabled: True if the node itself is disabled. **Attributes:** @@ -116,11 +121,10 @@ Attributes: - `ports_utilization` (float) - `capacity_violation` (bool) - `ports_violation` (bool) -- `disabled` (bool) ### TreeNode -Represents a node in the hierarchical tree. +A node in the hierarchical tree. Attributes: name (str): Name/label of this node. @@ -139,8 +143,8 @@ Attributes: - `children` (Dict[str, TreeNode]) = {} - `subtree_nodes` (Set[str]) = set() - `active_subtree_nodes` (Set[str]) = set() -- `stats` (TreeStats) = TreeStats(node_count=0, internal_link_count=0, internal_link_capacity=0.0, external_link_count=0, external_link_capacity=0.0, external_link_details={}, total_capex=0.0, total_power=0.0, bom={}, active_bom={}) -- `active_stats` (TreeStats) = TreeStats(node_count=0, internal_link_count=0, internal_link_capacity=0.0, external_link_count=0, external_link_capacity=0.0, external_link_details={}, total_capex=0.0, total_power=0.0, bom={}, active_bom={}) +- `stats` (TreeStats) = TreeStats(node_count=0, internal_link_count=0, internal_link_capacity=0.0, external_link_count=0, external_link_capacity=0.0, external_link_details={}, total_capex=0.0, total_power=0.0, bom={}) +- `active_stats` (TreeStats) = TreeStats(node_count=0, internal_link_count=0, internal_link_capacity=0.0, external_link_count=0, external_link_capacity=0.0, external_link_details={}, total_capex=0.0, total_power=0.0, bom={}) - `raw_nodes` (List[Node]) = [] **Methods:** @@ -173,7 +177,6 @@ Attributes: - `total_capex` (float) = 0.0 - `total_power` (float) = 0.0 - `bom` (Dict[str, float]) = {} -- `active_bom` (Dict[str, float]) = {} --- @@ -181,6 +184,14 @@ Attributes: Centralized logging configuration for NetGraph. +Follows the standard library pattern: importing the package attaches only a +``logging.NullHandler`` to the root ``ngraph`` logger and never installs +stream handlers or sets levels. Applications opt into console output by +calling ``setup_root_logger()`` explicitly, or implicitly via +``set_global_log_level()``. The CLI does both in ``main()``: it calls +``setup_root_logger()`` first, then sets the level from +``--verbose``/``--quiet``. + ### disable_debug_logging() -> None Disable debug logging, set to INFO level. @@ -191,16 +202,18 @@ Enable debug logging for the entire package. ### get_logger(name: str) -> logging.Logger -Get a logger with NetGraph's standard configuration. +Get a logger under NetGraph's logging hierarchy. -This is the main function that should be used throughout the package. -All loggers will inherit from the root 'ngraph' logger configuration. +Use this everywhere in the package. It configures nothing: handlers and +levels are inherited from the root 'ngraph' logger, which is configured +only when an application calls setup_root_logger() (directly or via +set_global_log_level()). Args: name: Logger name (typically __name__ from calling module). Returns: - Configured logger instance. + Logger instance inheriting from the root ngraph logger. ### reset_logging() -> None @@ -210,19 +223,23 @@ Reset logging configuration (mainly for testing). Set the log level for all NetGraph loggers. +Installs the default console handler via setup_root_logger() if logging +has not been configured yet. Intended for applications (e.g. the CLI); +library code never calls this implicitly. + Args: level: Logging level (e.g., logging.DEBUG, logging.INFO). -### setup_root_logger(level: int = 20, format_string: Optional[str] = None, handler: Optional[logging.Handler] = None) -> None +### setup_root_logger(level: int = 20, format_string: str | None = None, handler: logging.Handler | None = None) -> None Set up the root NetGraph logger with a single handler. -This should only be called once to avoid duplicate handlers. +Subsequent calls are no-ops until ``reset_logging()`` is called. Args: level: Logging level (default: INFO). format_string: Custom format string (optional). - handler: Custom handler (optional, defaults to StreamHandler). + handler: Custom handler (optional, defaults to StreamHandler(sys.stderr)). --- @@ -232,13 +249,13 @@ Scenario class for defining network analysis workflows from YAML. ### Scenario -Represents a complete scenario for building and executing network workflows. +A complete scenario for building and executing network workflows. -This scenario includes: +Holds: - A network (nodes/links), constructed via blueprint expansion. - A failure policy set (one or more named failure policies). -- A traffic matrix set containing one or more named traffic matrices. +- A demand set containing one or more named demand collections. - A list of workflow steps to execute. - A results container for storing outputs. - A components_library for hardware/optics definitions. @@ -263,8 +280,8 @@ Typical usage example: **Methods:** -- `from_yaml(yaml_str: 'str', default_components: 'Optional[ComponentsLibrary]' = None) -> 'Scenario'` - Constructs a Scenario from a YAML string, optionally merging -- `run(self) -> 'None'` - Executes the scenario's workflow steps in order. +- `from_yaml(yaml_str: 'str', default_components: 'Optional[ComponentsLibrary]' = None) -> 'Scenario'` - Construct a Scenario from a YAML string, merging in a default +- `run(self, step_hook: 'Optional[Callable[[WorkflowStep], ContextManager[None]]]' = None) -> 'None'` - Execute the scenario's workflow steps in order. --- @@ -308,10 +325,10 @@ Attributes: **Methods:** - `as_dict(self, include_children: 'bool' = True) -> 'Dict[str, Any]'` - Returns a dictionary containing all properties of this component. -- `total_capacity(self) -> 'float'` - Computes the total (recursive) capacity of this component, +- `total_capacity(self) -> 'float'` - Computes capacity for this component and all descendants. - `total_capex(self) -> 'float'` - Computes total capex including children, multiplied by count. -- `total_power(self) -> 'float'` - Computes the total *typical* (recursive) power usage of this component, -- `total_power_max(self) -> 'float'` - Computes the total *peak* (recursive) power usage of this component, +- `total_power(self) -> 'float'` - Computes *typical* power for this component and all descendants. +- `total_power_max(self) -> 'float'` - Computes *peak* power for this component and all descendants. ### ComponentsLibrary @@ -322,19 +339,19 @@ Example (YAML-like): components: BigSwitch: component_type: chassis - cost: 20000 + capex: 20000 power_watts: 1750 capacity: 25600 children: PIM16Q-16x200G: component_type: linecard - cost: 1000 + capex: 1000 power_watts: 10 ports: 16 count: 8 200G-FR4: component_type: optic - cost: 2000 + capex: 2000 power_watts: 6 power_watts_max: 6.5 @@ -345,32 +362,26 @@ Example (YAML-like): **Methods:** - `clone(self) -> 'ComponentsLibrary'` - Creates a deep copy of this ComponentsLibrary. -- `from_dict(data: 'Dict[str, Any]') -> 'ComponentsLibrary'` - Constructs a ComponentsLibrary from a dictionary of raw component definitions. +- `from_dict(data: 'Dict[str, Any]') -> 'ComponentsLibrary'` - Constructs a ComponentsLibrary from raw component definitions. - `from_yaml(yaml_str: 'str') -> 'ComponentsLibrary'` - Constructs a ComponentsLibrary from a YAML string. If the YAML contains - `get(self, name: 'str') -> 'Optional[Component]'` - Retrieves a Component by its name from the library. -- `merge(self, other: 'ComponentsLibrary', override: 'bool' = True) -> 'ComponentsLibrary'` - Merges another ComponentsLibrary into this one. By default (override=True), +- `merge(self, other: 'ComponentsLibrary', override: 'bool' = True) -> 'ComponentsLibrary'` - Merges another ComponentsLibrary into this one. ### resolve_link_end_components(attrs: 'Dict[str, Any]', library: 'ComponentsLibrary') -> 'tuple[tuple[Optional[Component], float, bool], tuple[Optional[Component], float, bool], bool]' Resolve per-end hardware components for a link. -Input format inside ``link.attrs``: - -Structured mapping under ``hardware`` key only: +Input format inside ``link.attrs`` is a structured mapping under the +``hardware`` key only: ``{"hardware": {"source": {"component": NAME, "count": N}, "target": {"component": NAME, "count": N}}}`` +An optional ``exclusive: true`` per end indicates unsharable usage; for +exclusive ends, validation and BOM counting round counts up to integers. Args: attrs: Link attributes mapping. library: Components library for lookups. -Exclusive usage: - -- Optional ``exclusive: true`` per end indicates unsharable usage. - - For exclusive ends, validation and BOM counting should round-up counts - to integers. - Returns: ((src_comp, src_count, src_exclusive), (dst_comp, dst_count, dst_exclusive), per_end_specified) where components may be ``None`` if name is absent/unknown. ``per_end_specified`` @@ -425,23 +436,37 @@ Raises: ValueError: If ``raw`` is not a mapping of name -> list[dict], or if required fields are missing. +### coerce_flow_policy(value: 'Any') -> 'Optional[FlowPolicyPreset]' + +Return a FlowPolicyPreset from various user-friendly forms. + +Accepts: + +- None: returns None +- FlowPolicyPreset: returned as-is +- int: mapped by value (e.g., 1 -> SHORTEST_PATHS_ECMP); bools are + + rejected (True/False are not presets 1/0) + +- str: name of enum (case-insensitive); numeric strings are allowed + +Raises: + ValueError: If the value is not one of the accepted forms (including + bool and dict/object configs, which are not supported). + --- ## ngraph.model.demand.matrix Demand set containers. -Provides `DemandSet`, a named collection of `TrafficDemand` lists -used as input to demand expansion and placement. This module contains input -containers, not analysis results. +`DemandSet` holds named `TrafficDemand` lists as input to demand expansion and +placement. These are input containers, not analysis results. ### DemandSet Named collection of TrafficDemand lists. -This mutable container maps set names to lists of TrafficDemand objects, -allowing management of multiple demand sets for analysis. - Attributes: sets: Dictionary mapping set names to TrafficDemand lists. @@ -451,11 +476,10 @@ Attributes: **Methods:** -- `add(self, name: 'str', demands: 'list[TrafficDemand]') -> 'None'` - Add a demand list to the collection. +- `add(self, name: 'str', demands: 'list[TrafficDemand]') -> 'None'` - Add a demand list, replacing any set already stored under `name`. - `get_all_demands(self) -> 'list[TrafficDemand]'` - Get all traffic demands from all sets combined. - `get_default_set(self) -> 'list[TrafficDemand]'` - Get default demand set. - `get_set(self, name: 'str') -> 'list[TrafficDemand]'` - Get a specific demand set by name. -- `to_dict(self) -> 'dict[str, Any]'` - Convert to dictionary for JSON serialization. --- @@ -464,8 +488,7 @@ Attributes: Traffic demand specification. Defines `TrafficDemand`, a user-facing specification used by demand expansion -and placement. It can carry either a concrete `FlowPolicy` instance or a -`FlowPolicyPreset` enum to construct one. +and placement. Routing behavior is selected via an optional `FlowPolicyPreset`. ### TrafficDemand @@ -475,13 +498,11 @@ Attributes: source: Source node selector (string path or selector dict). target: Target node selector (string path or selector dict). volume: Total demand volume. - volume_placed: Portion of this demand placed so far. priority: Priority class (lower = higher priority). mode: Node pairing mode ("combine" or "pairwise"). group_mode: How grouped nodes produce demands ("flatten", "per_group", "group_pairwise"). flow_policy: Policy preset for routing. - flow_policy_obj: Concrete policy instance (overrides flow_policy). attrs: Arbitrary user metadata. id: Unique identifier. Auto-generated if empty. @@ -490,23 +511,24 @@ Attributes: - `source` (Union) - `target` (Union) - `volume` (float) = 0.0 -- `volume_placed` (float) = 0.0 - `priority` (int) = 0 - `mode` (str) = combine - `group_mode` (str) = flatten -- `flow_policy` (Optional) -- `flow_policy_obj` (Optional) +- `flow_policy` (Union) - `attrs` (Dict) = {} - `id` (str) +**Methods:** + +- `to_dict(self) -> Dict[str, Any]` - Return the canonical serialized form (results output, snapshots). + --- ## ngraph.model.failure.generate Dynamic risk group generation from entity attributes. -Provides functionality to auto-generate risk groups based on unique -attribute values from nodes or links. +Creates one risk group per unique value of a chosen node or link attribute. ### GenerateSpec @@ -542,8 +564,12 @@ Args: Returns: List of newly created RiskGroup objects. +Raises: + ValueError: If `group_by` resolves to an unhashable value, or if the + name template renders the same group name for two distinct values. + Note: - This function modifies entity risk_groups sets in place. + Modifies entity risk_groups sets in place. ### parse_generate_spec(raw: 'Dict[str, Any]') -> 'GenerateSpec' @@ -556,7 +582,9 @@ Returns: Parsed GenerateSpec. Raises: - ValueError: If required fields are missing or invalid. + ValueError: If 'scope' is missing or is neither 'node' nor 'link', if + 'group_by' or 'name' is missing, or if 'name' omits the '${value}' + placeholder. --- @@ -564,9 +592,8 @@ Raises: Risk group membership rule resolution. -Provides functionality to resolve policy-based membership rules that -auto-assign entities (nodes, links, risk groups) to risk groups based -on attribute conditions. +Resolves policy-based membership rules that auto-assign entities (nodes, +links, risk groups) to risk groups based on attribute conditions. ### MembershipSpec @@ -574,7 +601,9 @@ Parsed membership rule specification. Attributes: scope: Type of entities to match ("node", "link", or "risk_group"). - path: Optional regex pattern to filter entities by name. + path: Optional regex pattern. For node and risk_group scope it is + matched against the entity ID; for link scope it is matched + against the "source|target" key, not the link ID. match: Match specification with conditions. **Attributes:** @@ -601,8 +630,8 @@ Args: network: Network with risk_groups, nodes, and links populated. Note: - This function modifies entities in place. It should be called after - all risk groups are registered but before validation. + Modifies entities in place. Call after all risk groups are registered + but before validation. --- @@ -614,12 +643,9 @@ Parsers for FailurePolicySet and related failure modeling structures. Build a FailurePolicy from a raw configuration dictionary. -Parses modes, rules, and conditions from the policy definition and -constructs a fully initialized FailurePolicy object. - Args: fp_data: Policy definition dict with keys: modes (required), attrs, - expand_groups, expand_children. Each mode contains weight and rules. + expand_groups. Each mode contains weight and rules. policy_name: Name identifier for this policy (used for seed derivation). derive_seed: Callable to derive deterministic seeds from component names. @@ -627,7 +653,8 @@ Returns: FailurePolicy: Configured policy with parsed modes and rules. Raises: - ValueError: If modes is empty or malformed, or if rules are invalid. + ValueError: If modes is empty or malformed, if rules are invalid, or + if no mode has positive weight. ### build_failure_policy_set(raw: 'Dict[str, Any]', *, derive_seed: 'Callable[[str], Optional[int]]') -> 'FailurePolicySet' @@ -654,6 +681,11 @@ Supports: - Children are also expanded recursively - Generate blocks: {generate: {...}} for dynamic group creation +'membership', 'disabled', and 'generate' are only honored on top-level +entries: only top-level groups are registered in network.risk_groups, so +these keys would be silently inert on nested children. Child entries +carrying them are rejected with ValueError. + Args: rg_data: List of risk group definitions (strings or dicts). @@ -673,8 +705,9 @@ Defines `FailureRule` and `FailurePolicy` for expressing how nodes, links, and risk groups fail in analyses. Conditions match on top-level attributes with simple operators; rules select matches using "all", probabilistic "random" (with `probability`), or fixed-size "choice" (with `count`). -Policies can optionally expand failures by shared risk groups or by -risk-group children. +Policies can optionally expand failures by shared risk groups. Failed risk +groups always cascade to their children downstream (the hierarchy is +inherent), so no policy flag controls that behavior. ### FailureMode @@ -700,37 +733,39 @@ Attributes: A container for failure modes plus optional metadata in `attrs`. -The main entry point is `apply_failures`, which: - 1) Build a single RNG for the entire call (from `seed` or `self.seed`). - 2) Select a mode based on weights (one RNG draw). - 3) For each rule in the mode, gather relevant entities. - 4) Match based on rule conditions using 'and' or 'or' logic. - 5) Apply the selection strategy (all, random, or choice) drawing - from the same RNG, ensuring statistical independence across rules. - 6) Collect the union of all failed entities across all rules. - 7) Optionally expand failures by shared-risk groups or sub-risks. +The main entry point is `apply_failures_typed`, which: + 1) Builds a single RNG for the entire call (from `seed` or `self.seed`). + 2) Selects a mode based on weights (one RNG draw). + 3) Gathers the relevant entities for each rule in that mode. + 4) Matches them against the rule conditions using 'and' or 'or' logic. + 5) Applies the selection strategy (all, random, or choice), drawing + from the same RNG, which keeps rules statistically independent. + 6) Collects the union of all failed entities across all rules. + 7) Optionally expands failures by shared-risk groups. Attributes: attrs: Arbitrary metadata about this policy. expand_groups: If True, expand failures among entities sharing risk groups with failed entities. - expand_children: If True, expand failed risk groups to include - their children recursively. seed: Default seed for reproducible random operations. Overridden - by the ``seed`` parameter on ``apply_failures`` when provided. + by the ``seed`` parameter on ``apply_failures_typed`` when + provided. modes: List of weighted failure modes. **Attributes:** - `attrs` (Dict[str, Any]) = {} - `expand_groups` (bool) = False -- `expand_children` (bool) = False - `seed` (Optional[int]) - `modes` (List[FailureMode]) = [] **Methods:** -- `apply_failures(self, network_nodes: 'Dict[str, Any]', network_links: 'Dict[str, Any]', network_risk_groups: 'Dict[str, Any] | None' = None, *, seed: 'Optional[int]' = None, failure_trace: 'Optional[Dict[str, Any]]' = None) -> 'List[str]'` - Identify which entities fail for this iteration. +- `apply_failures(self, network_nodes: 'Dict[str, Any]', network_links: 'Dict[str, Any]', network_risk_groups: 'Dict[str, Any] | None' = None, *, seed: 'Optional[int]' = None, failure_trace: 'Optional[Dict[str, Any]]' = None, prepared_matches: 'Optional[Dict[int, tuple[str, ...]]]' = None, prepared_weights: 'Optional[Dict[int, Tuple[Dict[str, float], Tuple[str, ...]]]]' = None, prepared_rg_index: 'Optional[Dict[str, Set[str]]]' = None, prepared_rg_members: 'Optional[Dict[str, Tuple[frozenset, frozenset]]]' = None) -> 'List[str]'` - Identify which entities fail for this iteration. +- `apply_failures_typed(self, network_nodes: 'Dict[str, Any]', network_links: 'Dict[str, Any]', network_risk_groups: 'Dict[str, Any] | None' = None, *, seed: 'Optional[int]' = None, failure_trace: 'Optional[Dict[str, Any]]' = None, prepared_matches: 'Optional[Dict[int, tuple[str, ...]]]' = None, prepared_weights: 'Optional[Dict[int, Tuple[Dict[str, float], Tuple[str, ...]]]]' = None, prepared_rg_index: 'Optional[Dict[str, Set[str]]]' = None, prepared_rg_members: 'Optional[Dict[str, Tuple[frozenset, frozenset]]]' = None) -> 'Tuple[Set[str], Set[str], Set[str]]'` - Identify which entities fail for this iteration, typed by scope. +- `build_risk_group_index(network_nodes: 'Dict[str, Any]', network_links: 'Dict[str, Any]') -> 'Dict[str, Set[str]]'` - Build a risk-group -> entity-ID index for risk-group expansion. +- `prepare_matches(self, network_nodes: 'Dict[str, Any]', network_links: 'Dict[str, Any]', network_risk_groups: 'Dict[str, Any] | None' = None) -> 'Dict[int, tuple[str, ...]]'` - Prepare stable ordered candidate pools for all rules in this policy. +- `prepare_weights(self, prepared_matches: 'Dict[int, tuple[str, ...]]', network_nodes: 'Dict[str, Any]', network_links: 'Dict[str, Any]', network_risk_groups: 'Dict[str, Any] | None' = None) -> 'Dict[int, Tuple[Dict[str, float], Tuple[str, ...]]]'` - Precompute per-rule weight splits for weighted-choice rules. - `to_dict(self) -> 'Dict[str, Any]'` - Convert to dictionary for JSON serialization. ### FailureRule @@ -744,14 +779,20 @@ Attributes: logic: "and" (all must be true) or "or" (any must be true, default). mode: The selection strategy among the matched set: -- "random": each matched entity is chosen with probability. +- "random": each matched entity fails independently with + + `probability`. + - "choice": pick exactly `count` items (random sample). - "all": select every matched entity. probability: Probability in [0,1], used if mode="random". count: Number of entities to pick if mode="choice". weight_by: Optional attribute for weighted sampling in choice mode. - path: Optional regex pattern to filter entities by name. + path: Optional regex pattern applied after condition matching. For + node and risk_group scope it is matched against the entity ID; + for link scope it is matched against the "source|target" key, + not the link ID. **Attributes:** @@ -778,9 +819,6 @@ containers, not analysis results. Named collection of FailurePolicy objects. -This mutable container maps failure policy names to FailurePolicy objects, -allowing management of multiple failure policies for analysis. - Attributes: policies: Dictionary mapping failure policy names to FailurePolicy objects. @@ -790,7 +828,7 @@ Attributes: **Methods:** -- `add(self, name: 'str', policy: 'FailurePolicy') -> 'None'` - Add a failure policy to the collection. +- `add(self, name: 'str', policy: 'FailurePolicy') -> 'None'` - Add a policy, replacing any policy already stored under `name`. - `get_all_policies(self) -> 'list[FailurePolicy]'` - Get all failure policies from the collection. - `get_policy(self, name: 'str') -> 'FailurePolicy'` - Get a specific failure policy by name. - `to_dict(self) -> 'dict[str, Any]'` - Convert to dictionary for JSON serialization. @@ -810,9 +848,8 @@ Also provides cycle detection for risk group hierarchies. Detect circular references in risk group parent-child relationships. -Uses DFS-based cycle detection to find any risk group that is part of -a cycle in the children hierarchy. This can happen when membership rules -with scope='risk_group' create mutual parent-child relationships. +Cycles arise when membership rules with scope='risk_group' create mutual +parent-child relationships. Detection is a DFS over the children hierarchy. Args: network: Network with risk_groups populated (after membership resolution). @@ -822,11 +859,10 @@ Raises: ### validate_risk_group_references(network: "'Network'") -> 'None' -Ensure all risk group references resolve to defined groups. +Ensure every risk group named by a node or link is defined. -Checks that every risk group name referenced by nodes and links -exists in network.risk_groups. This catches typos and missing -definitions that would otherwise cause silent failures in simulations. +Names are checked against network.risk_groups; typos and missing +definitions would otherwise cause silent failures in simulations. Args: network: Network with nodes, links, and risk_groups populated. @@ -842,8 +878,8 @@ Raises: Flow policy preset configurations for NetGraph. -Provides convenient factory functions to create common FlowPolicy configurations -using NetGraph-Core's FlowPolicy and FlowPolicyConfig. +Named routing presets and the factory that materializes them as NetGraph-Core +FlowPolicy objects built from a FlowPolicyConfig. ### FlowPolicyPreset @@ -859,7 +895,8 @@ Create a FlowPolicy instance from a preset configuration. Args: algorithms: NetGraph-Core Algorithms instance. graph: NetGraph-Core Graph handle. - preset: FlowPolicyPreset enum value specifying the desired policy. + preset: Preset whose path algorithm, placement, edge selection, and + flow-count bounds to apply. node_mask: Optional numpy bool array for node exclusions (True = include). edge_mask: Optional numpy bool array for edge exclusions (True = include). @@ -879,14 +916,14 @@ Example: Serialize a FlowPolicyPreset to its string name for JSON storage. -Handles FlowPolicyPreset enum values, integer enum values, and string inputs. -Returns None for None input. - Args: - cfg: FlowPolicyPreset enum, integer, or other value to serialize. + cfg: FlowPolicyPreset enum, an integer coercible to one, or any other + value. Returns: - String name of the preset (e.g., "SHORTEST_PATHS_ECMP"), or None if input is None. + Preset name (e.g. "SHORTEST_PATHS_ECMP"); None when ``cfg`` is None. + Values that do not map to a preset are logged at debug level and + returned as ``str(cfg)``. --- @@ -894,17 +931,16 @@ Returns: Network topology modeling with Node, Link, RiskGroup, and Network classes. -This module provides the core network model classes (Node, Link, RiskGroup, Network) -that can be used independently. +These classes carry no analysis machinery and can be used on their own. ### Link Represents one directed link between two nodes. -The model stores a single direction (``source`` -> ``target``). When building -the working graph for analysis, a reverse edge is added by default to provide -bidirectional connectivity. Disable with ``add_reverse=False`` in -``Network.to_strict_multidigraph``. +The model stores a single direction (``source`` -> ``target``). When the +analysis graph is built (via ``AnalysisContext`` / netgraph-core), a reverse +edge is added automatically for each link to provide bidirectional +connectivity. Attributes: source (str): Name of the source node. @@ -914,7 +950,10 @@ Attributes: disabled (bool): Whether the link is disabled. risk_groups (Set[str]): Set of risk group names this link belongs to. attrs (Dict[str, Any]): Additional metadata (e.g., distance). - id (str): Auto-generated unique identifier: "{source}|{target}|". + id (str): Unique identifier. ``Network.add_link`` assigns the + deterministic form "{source}|{target}|", where is a + per-(source, target) insertion sequence number; links never added + to a Network keep a provisional uuid-suffixed id. **Attributes:** @@ -949,21 +988,22 @@ Attributes: - `risk_groups` (Dict[str, RiskGroup]) = {} - `attrs` (Dict[str, Any]) = {} - `_selection_cache` (Dict[str, Dict[str, List[Node]]]) = {} +- `_link_seq` (Dict[tuple, int]) = {} **Methods:** -- `add_link(self, link: 'Link') -> 'None'` - Add a link to the network (keyed by the link's auto-generated ID). +- `add_link(self, link: 'Link') -> 'None'` - Add a link to the network, assigning its deterministic ID. - `add_node(self, node: 'Node') -> 'None'` - Add a node to the network (keyed by node.name). - `disable_all(self) -> 'None'` - Mark all nodes and links as disabled. - `disable_link(self, link_id: 'str') -> 'None'` - Mark a link as disabled. - `disable_node(self, node_name: 'str') -> 'None'` - Mark a node as disabled. -- `disable_risk_group(self, name: 'str', recursive: 'bool' = True) -> 'None'` - Disable all nodes/links that have 'name' in their risk_groups. +- `disable_risk_group(self, name: 'str', recursive: 'bool' = True) -> 'None'` - Disable every node/link that has 'name' in its risk_groups. - `enable_all(self) -> 'None'` - Mark all nodes and links as enabled. - `enable_link(self, link_id: 'str') -> 'None'` - Mark a link as enabled. - `enable_node(self, node_name: 'str') -> 'None'` - Mark a node as enabled. -- `enable_risk_group(self, name: 'str', recursive: 'bool' = True) -> 'None'` - Enable all nodes/links that have 'name' in their risk_groups. -- `find_links(self, source_regex: 'Optional[str]' = None, target_regex: 'Optional[str]' = None, any_direction: 'bool' = False) -> 'List[Link]'` - Search for links using optional regex patterns for source or target node names. -- `get_links_between(self, source: 'str', target: 'str') -> 'List[str]'` - Retrieve all link IDs that connect the specified source node +- `enable_risk_group(self, name: 'str', recursive: 'bool' = True) -> 'None'` - Enable every node/link that has 'name' in its risk_groups. +- `find_links(self, source_regex: 'Optional[str]' = None, target_regex: 'Optional[str]' = None, any_direction: 'bool' = False) -> 'List[Link]'` - Search for links by regex on source and/or target node names. +- `get_links_between(self, source: 'str', target: 'str') -> 'List[str]'` - Retrieve the IDs of all direct links from source to target. - `select_node_groups_by_path(self, path: 'str') -> 'Dict[str, List[Node]]'` - Select and group nodes by regex pattern on node name. ### Node @@ -1022,10 +1062,9 @@ Attributes: Lightweight representation of a single routing path. -The ``Path`` dataclass stores a node-and-parallel-edges sequence and a numeric -cost. Cached properties expose derived sequences for nodes and edges, and -helpers provide equality, ordering by cost, and sub-path extraction with cost -recalculation. +``Path`` stores a sequence of (node, parallel edges) elements plus a numeric +cost. Paths sort by cost, compare by structure and cost, and support sub-path +extraction, which leaves the cost for the caller to recompute. ### Path @@ -1053,6 +1092,261 @@ Attributes: --- +## ngraph.model.selectors.conditions + +Condition evaluation for node/entity filtering. + +Evaluates the attribute conditions used by selectors and failure policies. +Operators: ==, !=, <, <=, >, >=, contains, not_contains, in, not_in, exists, +not_exists. + +Attribute names support dot-notation for nested access (e.g. "hardware.vendor"). + +### evaluate_condition(attrs: 'Dict[str, Any]', cond: "'Condition'") -> 'bool' + +Evaluate a single condition against an attribute dict. + +Supports dot-notation for nested attribute access (e.g., "hardware.vendor"). + +Args: + attrs: Mapping of entity attributes (may contain nested dicts). + cond: Condition to evaluate. + +Returns: + True if condition passes, False otherwise. + +Raises: + ValueError: If operator is unknown or value type is invalid. + +### evaluate_conditions(attrs: 'Dict[str, Any]', conditions: "Iterable['Condition']", logic: 'str' = 'or') -> 'bool' + +Evaluate multiple conditions with AND/OR logic. + +Args: + attrs: Flat mapping of entity attributes. + conditions: Iterable of Condition objects. + logic: "and" (all must match) or "or" (any must match). + +Returns: + True if combined predicate passes. + +Raises: + ValueError: If logic is not "and" or "or". + +### resolve_attr_path(attrs: 'Dict[str, Any]', path: 'str') -> 'Tuple[bool, Any]' + +Resolve a dot-notation attribute path. + +Supports nested attribute access like "hardware.vendor" which resolves +to attrs["hardware"]["vendor"]. + +Args: + attrs: Attribute dict (may contain nested dicts). + path: Attribute path, optionally with dots for nesting. + +Returns: + Tuple of (found, value). If found is False, value is None. + +Examples: + >>> resolve_attr_path({"role": "spine"}, "role") + (True, 'spine') + >>> resolve_attr_path({"hardware": {"vendor": "Acme"}}, "hardware.vendor") + (True, 'Acme') + >>> resolve_attr_path({"role": "spine"}, "missing") + (False, None) + +--- + +## ngraph.model.selectors.parse + +Parsing of match specifications from plain dicts. + +Builds model schema types (`Condition`, `MatchSpec`) from raw dict input. +Lives in the model layer so failure-policy and membership parsing can use +it without a runtime dependency on the DSL package. + +### parse_match_spec(raw: 'Dict[str, Any]', *, default_logic: "Literal['and', 'or']" = 'or', require_conditions: 'bool' = False, context: 'str' = 'match') -> 'MatchSpec' + +Parse a match specification from raw dict. + +Shared by adjacency, demands, membership rules, and failure policies. + +Args: + raw: Dict with 'conditions' list and optional 'logic'. Both keys are + optional; a missing 'conditions' yields an empty condition list. + default_logic: Used when 'logic' is absent. + require_conditions: If True, raise when conditions list is empty. + context: Name of the enclosing construct, quoted in error messages. + +Returns: + Parsed MatchSpec. + +Raises: + ValueError: If 'logic' is not 'and'/'or', 'conditions' is not a list, + a condition is not a dict or lacks 'attr'/'op', 'in'/'not_in' is + given a non-list value, or conditions are required but empty. + +--- + +## ngraph.model.selectors.schema + +Schema definitions for unified node selection. + +Dataclasses shared by network rules, demands, and workflow steps. + +### Condition + +A single attribute condition for filtering. + +Supports dot-notation for nested attribute access (e.g., "hardware.vendor" +resolves to attrs["hardware"]["vendor"]). + +Attributes: + attr: Attribute name to match (supports dot-notation for nested attrs). + op: Comparison operator. + value: Right-hand operand (unused for exists/not_exists). + +**Attributes:** + +- `attr` (str) +- `op` (ConditionOp) +- `value` (Any) + +### MatchSpec + +Specification for filtering nodes by attribute conditions. + +Attributes: + conditions: List of conditions to evaluate. + logic: How to combine conditions ("and" = all, "or" = any). + +**Attributes:** + +- `conditions` (List[Condition]) = [] +- `logic` (Literal['and', 'or']) = or + +### NodeSelector + +Unified node selection specification. + +Evaluation order: + +1. Select nodes matching `path` regex (default ".*" if omitted) +2. Filter by `match` conditions +3. Filter by `active_only` flag +4. Group by `group_by` attribute (if specified) + +At least one of path, group_by, or match must be specified. + +Attributes: + path: Regex pattern on node.name. + group_by: Attribute name to group nodes by. + match: Attribute-based filtering conditions. + active_only: Whether to exclude disabled nodes. None uses context default. + +**Attributes:** + +- `path` (Optional[str]) +- `group_by` (Optional[str]) +- `match` (Optional[MatchSpec]) +- `active_only` (Optional[bool]) + +--- + +## ngraph.model.selectors.select + +Node selection and evaluation. + +`select_nodes()` combines regex matching, attribute filtering, active-only +filtering, and grouping; the flatten helpers build the attribute dicts that +condition evaluation runs against. + +### flatten_link_attrs(link: "'Link'", link_id: 'str') -> 'Dict[str, Any]' + +Build flat attribute dict for condition evaluation on links. + +Merges link's top-level fields with link.attrs. Top-level fields +take precedence on key conflicts. + +Args: + link: Link object to flatten. + link_id: The link's ID in the network. + +Returns: + Flat dict suitable for condition evaluation. + +### flatten_node_attrs(node: "'Node'") -> 'Dict[str, Any]' + +Build flat attribute dict for condition evaluation. + +Merges node's top-level fields (name, disabled, risk_groups) with +node.attrs. Top-level fields take precedence on key conflicts. + +Args: + node: Node object to flatten. + +Returns: + Flat dict suitable for condition evaluation. + +### flatten_risk_group_attrs(rg: "'RiskGroup'") -> 'Dict[str, Any]' + +Build flat attribute dict for condition evaluation on risk groups. + +Merges risk group's top-level fields (name, disabled, children) with +rg.attrs. Top-level fields take precedence on key conflicts. + +Args: + rg: RiskGroup object. + +Returns: + Flat dict suitable for condition evaluation. + +### link_path_key(attrs: 'Dict[str, Any]') -> 'str' + +Return the "source|target" key used when path-matching links. + +Links have no name of their own, so path regexes match against this +canonical endpoint-pair form of the flattened link attributes. + +### match_entity_ids(entity_attrs: 'Dict[str, Dict[str, Any]]', conditions: 'List[Condition]', logic: 'str' = 'or') -> 'Set[str]' + +Match entity IDs by attribute conditions. + +General primitive for condition-based entity selection. Works with +any entity type as long as attributes are pre-flattened. + +Args: + entity_attrs: Mapping of {entity_id: flattened_attrs_dict} + conditions: List of conditions to evaluate + logic: "and" (all must match) or "or" (any must match) + +Returns: + Set of matching entity IDs. Returns all IDs if conditions is empty. + +### select_nodes(network: "'Network'", selector: 'NodeSelector', default_active_only: 'bool') -> "Dict[str, List['Node']]" + +Unified entry point for node selection. + +Evaluation order: + +1. Select nodes matching `path` regex (or all nodes if path is None) +2. Filter by `match` conditions +3. Filter by `active_only` flag +4. Group by `group_by` attribute (overrides regex capture grouping) + +Args: + network: Network whose nodes are searched. + selector: Node selection specification. + default_active_only: Used when the selector leaves `active_only` + unset. Required rather than defaulted so callers cannot silently + inherit the wrong policy. + +Returns: + Dict mapping group labels to lists of nodes. Groups that filter down + to nothing are dropped. + +--- + ## ngraph.workflow.base Base classes for workflow automation. @@ -1066,9 +1360,9 @@ re-raised. Base class for all workflow steps. -All workflow steps are automatically logged with execution timing information. -All workflow steps support seeding for reproducible random operations. -Workflow metadata is automatically stored in scenario.results for analysis. +Every step is logged with execution timing, supports seeding for +reproducible random operations, and has its metadata stored in +scenario.results for analysis. YAML Configuration: ```yaml @@ -1082,7 +1376,8 @@ YAML Configuration: Attributes: name: Optional custom identifier for this workflow step instance, - used for logging and result storage purposes. + used for logging and result storage. When empty, the class name + is used instead. seed: Optional seed for reproducible random operations. If None, random operations will be non-deterministic. @@ -1109,24 +1404,57 @@ Returns: ### resolve_parallelism(parallelism: 'Union[int, str]') -> 'int' -Resolve parallelism setting to a concrete worker count. +Validate and resolve a parallelism setting to a concrete worker count. Args: - parallelism: Either an integer worker count or "auto" for CPU count. + parallelism: Either a positive integer worker count or "auto" for + the CPU count. Returns: Positive integer worker count (minimum 1). +Raises: + ValueError: If parallelism is a string other than "auto", or an + integer < 1. + +### serialize_monte_carlo_results(raw: 'Dict[str, Any]') -> 'tuple[Any, list[dict]]' + +Convert FailureManager Monte Carlo output into JSON-safe dicts. + +Args: + raw: Dict with optional "baseline" entry and "results" list, whose + items expose to_dict() (e.g. FlowIterationResult) or are already + plain dicts. + +Returns: + Tuple of (baseline_dict, flow_results): the baseline iteration (or + None) and the failure iterations, converted via to_dict() when + available. + +### validate_unique_step_names(workflow: "'list[WorkflowStep]'") -> 'None' + +Validate that effective step names in a workflow list are unique. + +Effective names follow the same rule used for results storage: +``step.name`` or the step class name when no name is set. Duplicate +effective names would silently overwrite each other's namespace in the +results store. + +Args: + workflow: Workflow step list. + +Raises: + ValueError: If two or more steps share the same effective name. + --- ## ngraph.workflow.build_graph Graph building workflow component. -Validates and exports network topology as a node-link representation using NetworkX. -Actual graph building for analysis happens in analysis functions; this step -primarily validates the network and stores a serializable representation for -inspection. +Validates the network topology and exports it as a NetworkX node-link +representation for inspection. Graph building for analysis happens in the +analysis functions, not here. YAML Configuration Example: ```yaml @@ -1137,9 +1465,9 @@ YAML Configuration Example: add_reverse: true # Optional: Add reverse edges (default: true) ``` -The `add_reverse` parameter controls whether reverse edges are added for each link. -When `True` (default), each Link(A→B) gets both forward(A→B) and reverse(B→A) edges -for bidirectional connectivity. Set to `False` for directed-only graphs. +With `add_reverse: true` (the default), each Link(A→B) gets both a forward +(A→B) and a reverse (B→A) edge for bidirectional connectivity. Set it to +`false` for directed-only graphs. Results stored in `scenario.results` under the step name as two keys: @@ -1150,9 +1478,8 @@ Results stored in `scenario.results` under the step name as two keys: Validates network topology and stores node-link representation. -This step validates the network structure and stores a JSON-serializable -node-link representation using NetworkX. Core graph building happens in -analysis functions as needed. +The stored representation is JSON-serializable NetworkX node-link data. +Core graph building for analysis happens in analysis functions as needed. Attributes: add_reverse: If True, adds reverse edges for bidirectional connectivity. @@ -1176,9 +1503,8 @@ Attributes: CostPower workflow step: collect capex and power by hierarchy level. -This step aggregates capex and power from the network hardware inventory without -performing any normalization or reporting. It separates contributions into two -categories: +Aggregates capex and power from the network hardware inventory, with no +normalization or reporting. Contributions are split into two categories: - platform_*: node hardware (e.g., chassis, linecards) resolved from node attrs - optics_*: per-end link hardware (e.g., optics) resolved from link attrs @@ -1231,7 +1557,8 @@ Collect platform and optics capex/power by aggregation level. Attributes: include_disabled: If True, include disabled nodes and links. - aggregation_level: Inclusive depth for aggregation. 0=root only. + aggregation_level: Inclusive depth for aggregation; 0 = root only. + Must be >= 0. **Attributes:** @@ -1255,8 +1582,8 @@ MaxFlow workflow step. Monte Carlo analysis of maximum flow capacity between node groups using FailureManager. Produces unified `flow_results` per iteration under `data.flow_results`. -Baseline (no failures) is always run first as a separate reference. The `iterations` -parameter specifies how many failure scenarios to run. +Baseline (no failures) always runs first as a separate reference; `iterations` +counts failure scenarios only. YAML Configuration Example: @@ -1283,24 +1610,26 @@ YAML Configuration Example: Maximum flow Monte Carlo workflow step. -Baseline (no failures) is always run first as a separate reference. Results are -returned with baseline in a separate field. The flow_results list contains unique -failure patterns (deduplicated); each result has occurrence_count indicating how -many iterations matched that pattern. +Baseline (no failures) always runs first and is returned in a separate field. +The flow_results list holds unique failure patterns (deduplicated); each result +carries an occurrence_count of how many iterations matched that pattern. Attributes: source: Source node selector (string path or selector dict). target: Target node selector (string path or selector dict). mode: Flow analysis mode ("combine" or "pairwise"). failure_policy: Name of failure policy in scenario.failure_policy_set. - iterations: Number of failure iterations to run. - parallelism: Number of parallel worker processes. - shortest_path: Whether to use shortest paths only. + If None, no failure policy is applied. + iterations: Number of failure iterations to run; must be >= 0. + parallelism: Worker thread count, or "auto" for the CPU count. + shortest_path: Restrict flow to lowest-cost paths (IP/IGP mode). require_capacity: If True (default), path selection considers capacity. If False, path selection is cost-only (true IP/IGP semantics). flow_placement: Flow placement strategy. seed: Optional seed for reproducible results. - store_failure_patterns: Whether to store failure patterns in results. + store_failure_patterns: Record the failure trace on each result. + Iterations are deduplicated, so a trace describes the first + iteration of its pattern, not every matching iteration. include_flow_details: Whether to collect cost distribution per flow. include_min_cut: Whether to include min-cut edges per flow. @@ -1361,20 +1690,22 @@ YAML Configuration Example: Finds the maximum uniform traffic multiplier that is fully placeable. -Uses binary search to find alpha_star, the maximum multiplier for all -demands in the set that can still be fully placed on the network. +Binary search yields alpha_star: the largest multiplier at which every +demand in the set still places fully on the network. Attributes: demand_set: Name of the demand set to analyze. - acceptance_rule: Currently only "hard" is implemented. + acceptance_rule: Currently only "hard" is implemented; anything else + raises ValueError at run time. alpha_start: Starting multiplier for binary search. - growth_factor: Factor for bracket expansion. + growth_factor: Factor for bracket expansion; must be > 1.0. alpha_min: Minimum allowed alpha value. alpha_max: Maximum allowed alpha value. - resolution: Convergence threshold for binary search. + resolution: Convergence threshold for binary search; must be positive. max_bracket_iters: Maximum iterations for bracketing phase. max_bisect_iters: Maximum iterations for bisection phase. - placement_rounds: Placement optimization rounds. + placement_rounds: Deprecated; accepted for backward compatibility but + has no effect (placement optimization is handled by the core engine). **Attributes:** @@ -1404,8 +1735,9 @@ Attributes: Workflow step for basic node and link statistics. Computes and stores network statistics including node/link counts, -capacity distributions, cost distributions, and degree distributions. Supports -optional exclusion simulation and disabled entity handling. +capacity distributions, cost distributions, and degree distributions. Excluded +entities are filtered out without modifying the base network; disabled nodes +and links are excluded too unless `include_disabled` is set. YAML Configuration Example: ```yaml @@ -1482,8 +1814,8 @@ TrafficMatrixPlacement workflow step. Runs Monte Carlo demand placement using a named demand set and produces unified `flow_results` per iteration under `data.flow_results`. -Baseline (no failures) is always run first as a separate reference. The `iterations` -parameter specifies how many failure scenarios to run. +Baseline (no failures) always runs first as a separate reference; `iterations` +counts failure scenarios only. YAML Configuration Example: ```yaml @@ -1494,7 +1826,7 @@ YAML Configuration Example: demand_set: "default" failure_policy: "single_link" # Optional: failure policy name iterations: 100 # Number of failure scenarios - parallelism: 4 # Worker processes (or "auto") + parallelism: 4 # Worker threads (or "auto") alpha: 1.0 # Demand volume multiplier include_flow_details: true # Include cost distribution per flow ``` @@ -1503,23 +1835,29 @@ YAML Configuration Example: Monte Carlo demand placement using a named demand set. -Baseline (no failures) is always run first as a separate reference. Results are -returned with baseline in a separate field. The flow_results list contains unique -failure patterns (deduplicated); each result has occurrence_count indicating how -many iterations matched that pattern. +Baseline (no failures) always runs first and is returned in a separate field. +The flow_results list holds unique failure patterns (deduplicated); each result +carries an occurrence_count of how many iterations matched that pattern. Attributes: - demand_set: Name of the demand set to analyze. - failure_policy: Optional failure policy name in scenario.failure_policy_set. - iterations: Number of failure iterations to run. - parallelism: Number of parallel worker processes. - placement_rounds: Placement optimization rounds (int or "auto"). + demand_set: Name of the demand set to analyze. Required; an empty + value raises ValueError. + failure_policy: Failure policy name in scenario.failure_policy_set. + If None, no failure policy is applied. + iterations: Number of failure iterations to run; must be >= 0. + parallelism: Worker thread count, or "auto" for the CPU count. + placement_rounds: Deprecated; accepted for backward compatibility but + has no effect (placement optimization is handled by the core engine). seed: Optional seed for reproducibility. - store_failure_patterns: Whether to store failure pattern results. + store_failure_patterns: Record the failure trace on each result. + Iterations are deduplicated, so a trace describes the first + iteration of its pattern, not every matching iteration. include_flow_details: When True, include cost_distribution per flow. include_used_edges: When True, include set of used edges per demand in entry data. - alpha: Numeric scale for demands in the set. - alpha_from_step: Optional producer step name to read alpha from. + alpha: Numeric scale for demands in the set; must be > 0.0. Ignored + when alpha_from_step is set. + alpha_from_step: Optional producer step name to read alpha from; it + must run before this step. alpha_from_field: Dotted field path in producer step (default: "data.alpha_star"). **Attributes:** @@ -1548,11 +1886,16 @@ Attributes: ## ngraph.dsl.blueprints.expand -Network topology blueprints and generation. +Blueprint and network DSL expansion. + +Turns the `blueprints:` and `network:` sections into concrete Node and Link +objects: resolves blueprint instantiation and parameter overrides, expands +bracket patterns and `expand:` variable blocks, applies node and link rules, +and materializes `mesh`/`one_to_one` link patterns. ### Blueprint -Represents a reusable blueprint for hierarchical sub-topologies. +Reusable blueprint for hierarchical sub-topologies. A blueprint may contain multiple node definitions (each can have count and template), plus link definitions describing how those nodes connect. @@ -1611,480 +1954,238 @@ Field validation rules: - Link properties are flat (capacity, cost, etc. at link level). - For node definitions: count, template, attrs, disabled, risk_groups, - or blueprint for blueprint-based nodes. - -Args: - data: The YAML-parsed dictionary containing optional "blueprints" + "network". - -Returns: - The expanded Network object with all nodes and links. - ---- - -## ngraph.dsl.blueprints.parser - -Parsing helpers for the network DSL. - -This module factors out pure parsing/validation helpers from the expansion -module so they can be tested independently and reused. - -### check_link_keys(link_def: 'Dict[str, Any]', context: 'str') -> 'None' - -Ensure link definitions only contain recognized keys. - -### check_no_extra_keys(data_dict: 'Dict[str, Any]', allowed: 'set[str]', context: 'str') -> 'None' - -Raise if ``data_dict`` contains keys outside ``allowed``. - -Args: - data_dict: The dict to check. - allowed: Set of recognized keys. - context: Short description used in error messages. - -### join_paths(parent_path: 'str', rel_path: 'str') -> 'str' - -Join two path segments according to DSL conventions. - -The DSL has no concept of absolute paths. All paths are relative to the -current context (parent_path). A leading "/" on rel_path is stripped and -has no functional effect - it serves only as a visual indicator that the -path starts from the current scope's root. - -Behavior: - -- Leading "/" on rel_path is stripped (not treated as filesystem root) -- Result is always: "{parent_path}/{stripped_rel_path}" if parent_path is non-empty -- Examples: - - join_paths("", "/leaf") -> "leaf" - join_paths("pod1", "/leaf") -> "pod1/leaf" - join_paths("pod1", "leaf") -> "pod1/leaf" (same result) - -Args: - parent_path: Parent path prefix (e.g., "pod1" when expanding a blueprint). - rel_path: Path to join. Leading "/" is stripped if present. - -Returns: - Combined path string. - ---- - -## ngraph.dsl.expansion.brackets - -Bracket expansion for name patterns. - -Provides expand_name_patterns() for expanding bracket expressions -like "fa[1-3]" into ["fa1", "fa2", "fa3"]. - -### expand_name_patterns(name: 'str') -> 'List[str]' - -Expand bracket expressions in a group name. - -Supports: - -- Ranges: [1-3] -> 1, 2, 3 -- Lists: [a,b,c] -> a, b, c -- Mixed: [1,3,5-7] -> 1, 3, 5, 6, 7 -- Multiple brackets: Cartesian product - -Args: - name: Name pattern with optional bracket expressions. - -Returns: - List of expanded names. - -Examples: - >>> expand_name_patterns("fa[1-3]") - ["fa1", "fa2", "fa3"] - >>> expand_name_patterns("dc[1,3,5-6]") - ["dc1", "dc3", "dc5", "dc6"] - >>> expand_name_patterns("fa[1-2]_plane[5-6]") - ["fa1_plane5", "fa1_plane6", "fa2_plane5", "fa2_plane6"] - -### expand_risk_group_refs(rg_list: 'Iterable[str]') -> 'Set[str]' - -Expand bracket patterns in a list of risk group references. - -Takes an iterable of risk group names (possibly containing bracket -expressions) and returns a set of all expanded names. - -Args: - rg_list: Iterable of risk group name patterns. - -Returns: - Set of expanded risk group names. - -Examples: - >>> expand_risk_group_refs(["RG1"]) - {"RG1"} - >>> expand_risk_group_refs(["RG[1-3]"]) - {"RG1", "RG2", "RG3"} - >>> expand_risk_group_refs(["A[1-2]", "B[a,b]"]) - {"A1", "A2", "Ba", "Bb"} - ---- - -## ngraph.dsl.expansion.schema - -Schema definitions for variable expansion. - -Provides dataclasses for template expansion configuration. - -### ExpansionSpec - -Specification for variable-based expansion. - -Attributes: - vars: Mapping of variable names to lists of values. - mode: How to combine variable values. - -- "cartesian": All combinations (default) -- "zip": Pair values by position - -**Attributes:** - -- `vars` (Dict[str, List[Any]]) = {} -- `mode` (Literal['cartesian', 'zip']) = cartesian - -**Methods:** - -- `from_dict(data: 'Dict[str, Any]') -> "Optional['ExpansionSpec']"` - Extract expand: block from dict. -- `is_empty(self) -> 'bool'` - Check if no variables are defined. - ---- - -## ngraph.dsl.expansion.variables - -Variable expansion for templates. - -Provides substitution of $var and ${var} placeholders in strings, -with recursive substitution in nested structures. - -### expand_block(block: 'Dict[str, Any]', spec: "Optional['ExpansionSpec']") -> 'Iterator[Dict[str, Any]]' - -Expand a DSL block, yielding one dict per variable combination. - -If no expand spec is provided or it has no vars, yields the original block. -Otherwise, yields a deep copy with all strings substituted for each -variable combination. - -Args: - block: DSL block (dict) that may contain template strings. - spec: Optional expansion specification. - -Yields: - Dict with variable substitutions applied. - -### expand_templates(templates: 'Dict[str, str]', spec: "'ExpansionSpec'") -> 'Iterator[Dict[str, str]]' - -Expand template strings with variable substitution. - -Uses $var or ${var} syntax only. - -Args: - templates: Dict of template strings. - spec: Expansion specification with variables and mode. - -Yields: - Dicts with same keys as templates, values substituted. - -Raises: - ValueError: If zip mode has mismatched list lengths or expansion exceeds limit. - KeyError: If a template references an undefined variable. - -### substitute_vars(obj: 'Any', var_dict: 'Dict[str, Any]') -> 'Any' - -Recursively substitute ${var} in all strings within obj. - -Args: - obj: Any value (string, dict, list, or primitive). - var_dict: Mapping of variable names to values. - -Returns: - Object with all string values having variables substituted. - ---- - -## ngraph.dsl.loader - -YAML loader + schema validation for Scenario DSL. - -Provides a single entrypoint to parse a YAML string, normalize keys where -needed, validate against the packaged JSON schema, and return a canonical -dictionary suitable for downstream expansion/parsing. - -### load_scenario_yaml(yaml_str: 'str') -> 'Dict[str, Any]' - -Load, normalize, and validate a Scenario YAML string. - -Returns a canonical dictionary representation that downstream parsers can -consume without worrying about YAML-specific quirks (e.g., boolean-like -keys) and with schema shape already enforced. - ---- - -## ngraph.dsl.selectors.conditions - -Condition evaluation for node/entity filtering. - -Provides evaluation logic for attribute conditions used in selectors -and failure policies. Supports operators: ==, !=, <, <=, >, >=, -contains, not_contains, in, not_in, exists, not_exists. - -Supports dot-notation for nested attribute access (e.g., "hardware.vendor"). - -### evaluate_condition(attrs: 'Dict[str, Any]', cond: "'Condition'") -> 'bool' - -Evaluate a single condition against an attribute dict. - -Supports dot-notation for nested attribute access (e.g., "hardware.vendor"). - -Args: - attrs: Mapping of entity attributes (may contain nested dicts). - cond: Condition to evaluate. - -Returns: - True if condition passes, False otherwise. - -Raises: - ValueError: If operator is unknown or value type is invalid. - -### evaluate_conditions(attrs: 'Dict[str, Any]', conditions: "Iterable['Condition']", logic: 'str' = 'or') -> 'bool' - -Evaluate multiple conditions with AND/OR logic. - -Args: - attrs: Flat mapping of entity attributes. - conditions: Iterable of Condition objects. - logic: "and" (all must match) or "or" (any must match). - -Returns: - True if combined predicate passes. - -Raises: - ValueError: If logic is not "and" or "or". - -### resolve_attr_path(attrs: 'Dict[str, Any]', path: 'str') -> 'Tuple[bool, Any]' - -Resolve a dot-notation attribute path. - -Supports nested attribute access like "hardware.vendor" which resolves -to attrs["hardware"]["vendor"]. + or blueprint for blueprint-based nodes. Args: - attrs: Attribute dict (may contain nested dicts). - path: Attribute path, optionally with dots for nesting. + data: The YAML-parsed dictionary containing optional "blueprints" + "network". Returns: - Tuple of (found, value). If found is False, value is None. - -Examples: - >>> resolve_attr_path({"role": "spine"}, "role") - (True, "spine") - >>> resolve_attr_path({"hardware": {"vendor": "Acme"}}, "hardware.vendor") - (True, "Acme") - >>> resolve_attr_path({"role": "spine"}, "missing") - (False, None) + The expanded Network object with all nodes and links. --- -## ngraph.dsl.selectors.normalize +## ngraph.dsl.blueprints.parser -Selector parsing and normalization. +Parsing helpers for the network DSL. -Provides the single entry point for converting raw selector values -(strings or dicts) into NodeSelector objects. +Pure parsing/validation helpers, kept separate from the expansion module so +they can be tested independently and reused. -### normalize_selector(raw: 'Union[str, Dict[str, Any], NodeSelector]', context: 'str') -> 'NodeSelector' +### check_link_keys(link_def: 'Dict[str, Any]', context: 'str') -> 'None' -Normalize a raw selector (string or dict) to a NodeSelector. +Ensure link definitions only contain recognized keys. -This is the single entry point for all selector parsing. All downstream -code works with NodeSelector objects only. +### check_no_extra_keys(data_dict: 'Dict[str, Any]', allowed: 'set[str]', context: 'str') -> 'None' + +Raise if ``data_dict`` contains keys outside ``allowed``. Args: - raw: Either a regex string, selector dict, or existing NodeSelector. - context: Usage context ("adjacency", "demand", "override", "workflow"). - Determines the default for active_only. + data_dict: The dict to check. + allowed: Set of recognized keys. + context: Short description used in error messages. -Returns: - Normalized NodeSelector instance. +### join_paths(parent_path: 'str', rel_path: 'str') -> 'str' -Raises: - ValueError: If selector format is invalid or context is unknown. +Join two path segments according to DSL conventions. -### parse_match_spec(raw: 'Dict[str, Any]', *, default_logic: "Literal['and', 'or']" = 'or', require_conditions: 'bool' = False, context: 'str' = 'match') -> 'MatchSpec' +The DSL has no concept of absolute paths. All paths are relative to the +current context (parent_path). A leading "/" on rel_path is stripped and +has no functional effect - it serves only as a visual indicator that the +path starts from the current scope's root. -Parse a match specification from raw dict. +Behavior: + +- Leading "/" on rel_path is stripped (not treated as filesystem root) +- Result is always: "{parent_path}/{stripped_rel_path}" if parent_path is non-empty +- Examples: -Unified match specification parser for use across adjacency, demands, -membership rules, and failure policies. + join_paths("", "/leaf") -> "leaf" + join_paths("pod1", "/leaf") -> "pod1/leaf" + join_paths("pod1", "leaf") -> "pod1/leaf" (same result) Args: - raw: Dict with 'conditions' list and optional 'logic'. - default_logic: Default when 'logic' not specified. - require_conditions: If True, raise when conditions list is empty. - context: Used in error messages. + parent_path: Parent path prefix (e.g., "pod1" when expanding a blueprint). + rel_path: Path to join. Leading "/" is stripped if present. Returns: - Parsed MatchSpec. - -Raises: - ValueError: If validation fails. + Combined path string. --- -## ngraph.dsl.selectors.schema +## ngraph.dsl.expansion.brackets -Schema definitions for unified node selection. +Bracket expansion for name patterns. -Provides dataclasses for node selection configuration used across -network rules, demands, and workflow steps. +`expand_name_patterns()` turns bracket expressions like "fa[1-3]" into +["fa1", "fa2", "fa3"]. -### Condition +### expand_name_patterns(name: 'str') -> 'List[str]' -A single attribute condition for filtering. +Expand bracket expressions in a group name. -Supports dot-notation for nested attribute access (e.g., "hardware.vendor" -resolves to attrs["hardware"]["vendor"]). +Supports: -Attributes: - attr: Attribute name to match (supports dot-notation for nested attrs). - op: Comparison operator. - value: Right-hand operand (unused for exists/not_exists). +- Ranges: [1-3] -> 1, 2, 3 +- Lists: [a,b,c] -> a, b, c +- Mixed: [1,3,5-7] -> 1, 3, 5, 6, 7 +- Multiple brackets: Cartesian product -**Attributes:** +Args: + name: Name pattern with optional bracket expressions. -- `attr` (str) -- `op` (Literal['==', '!=', '<', '<=', '>', '>=', 'contains', 'not_contains', 'in', 'not_in', 'exists', 'not_exists']) -- `value` (Any) +Returns: + List of expanded names. -### MatchSpec +Examples: + >>> expand_name_patterns("fa[1-3]") + ['fa1', 'fa2', 'fa3'] + >>> expand_name_patterns("dc[1,3,5-6]") + ['dc1', 'dc3', 'dc5', 'dc6'] + >>> expand_name_patterns("fa[1-2]_plane[5-6]") + ['fa1_plane5', 'fa1_plane6', 'fa2_plane5', 'fa2_plane6'] -Specification for filtering nodes by attribute conditions. +### expand_risk_group_refs(rg_list: 'Union[List[str], Set[str], Tuple[str, ...]]') -> 'Set[str]' -Attributes: - conditions: List of conditions to evaluate. - logic: How to combine conditions ("and" = all, "or" = any). +Expand bracket patterns in a list of risk group references. -**Attributes:** +Takes a list, set, or tuple of risk group names (possibly containing +bracket expressions) and returns a set of all expanded names. -- `conditions` (List[Condition]) = [] -- `logic` (Literal['and', 'or']) = or +Args: + rg_list: List, set, or tuple of risk group name patterns. Other + iterables (including bare strings and generators) are rejected. -### NodeSelector +Returns: + Set of expanded risk group names. -Unified node selection specification. +Raises: + ValueError: If the container is not a list/set/tuple (a bare string + would silently expand per character), or if an entry is not a + string (e.g. a variable expansion substituted a non-string value). -Evaluation order: +Examples: + >>> sorted(expand_risk_group_refs(["RG1"])) + ['RG1'] + >>> sorted(expand_risk_group_refs(["RG[1-3]"])) + ['RG1', 'RG2', 'RG3'] + >>> sorted(expand_risk_group_refs(["A[1-2]", "B[a,b]"])) + ['A1', 'A2', 'Ba', 'Bb'] -1. Select nodes matching `path` regex (default ".*" if omitted) -2. Filter by `match` conditions -3. Filter by `active_only` flag -4. Group by `group_by` attribute (if specified) +--- -At least one of path, group_by, or match must be specified. +## ngraph.dsl.expansion.schema + +Dataclasses describing template expansion configuration. + +### ExpansionSpec + +Specification for variable-based expansion. Attributes: - path: Regex pattern on node.name. - group_by: Attribute name to group nodes by. - match: Attribute-based filtering conditions. - active_only: Whether to exclude disabled nodes. None uses context default. + vars: Mapping of variable names to lists of values. + mode: How to combine variable values. + +- "cartesian": All combinations (default) +- "zip": Pair values by position **Attributes:** -- `path` (Optional[str]) -- `group_by` (Optional[str]) -- `match` (Optional[MatchSpec]) -- `active_only` (Optional[bool]) +- `vars` (Dict[str, List[Any]]) = {} +- `mode` (Literal['cartesian', 'zip']) = cartesian + +**Methods:** + +- `from_dict(data: 'Dict[str, Any]') -> "Optional['ExpansionSpec']"` - Extract expand: block from dict. +- `is_empty(self) -> 'bool'` - Check if no variables are defined. --- -## ngraph.dsl.selectors.select +## ngraph.dsl.expansion.variables -Node selection and evaluation. +Variable expansion for templates. -Provides the unified select_nodes() function that handles regex matching, -attribute filtering, active-only filtering, and grouping. +Substitutes $var and ${var} placeholders in strings, recursing into nested +structures. -### flatten_link_attrs(link: "'Link'", link_id: 'str') -> 'Dict[str, Any]' +### expand_block(block: 'Dict[str, Any]', spec: "Optional['ExpansionSpec']") -> 'Iterator[Dict[str, Any]]' -Build flat attribute dict for condition evaluation on links. +Expand a DSL block, yielding one dict per variable combination. -Merges link's top-level fields with link.attrs. Top-level fields -take precedence on key conflicts. +If no expand spec is provided or it has no vars, yields the original block. +Otherwise, yields a deep copy with all strings substituted for each +variable combination; the 'expand' key itself is removed from each copy. Args: - link: Link object to flatten. - link_id: The link's ID in the network. + block: DSL block (dict) that may contain template strings. + spec: Optional expansion specification. -Returns: - Flat dict suitable for condition evaluation. +Yields: + Dict with variable substitutions applied. -### flatten_node_attrs(node: "'Node'") -> 'Dict[str, Any]' +### substitute_vars(obj: 'Any', var_dict: 'Dict[str, Any]') -> 'Any' -Build flat attribute dict for condition evaluation. +Recursively substitute ${var} in all strings within obj. -Merges node's top-level fields (name, disabled, risk_groups) with -node.attrs. Top-level fields take precedence on key conflicts. +A string consisting of exactly one placeholder (e.g. "${t}") is replaced +by the variable's native value, preserving its type. This keeps match +condition values comparable to non-string attributes (e.g. int tiers). +Placeholders embedded in longer strings (e.g. "dc${dc}_internal") are +interpolated as text, so the result is a string. Args: - node: Node object to flatten. + obj: Any value (string, dict, list, or primitive). + var_dict: Mapping of variable names to values. Returns: - Flat dict suitable for condition evaluation. + Object with variables substituted: whole-placeholder strings replaced + by the variable's native value, other strings interpolated as text. -### flatten_risk_group_attrs(rg: "Union['RiskGroup', Dict[str, Any]]") -> 'Dict[str, Any]' +Raises: + KeyError: If a placeholder names a variable absent from var_dict. -Build flat attribute dict for condition evaluation on risk groups. +--- -Merges risk group's top-level fields (name, disabled, children) with -rg.attrs. Top-level fields take precedence on key conflicts. +## ngraph.dsl.loader -Supports both RiskGroup objects and dict representations (for flexibility -in failure policy matching). +YAML loader + schema validation for Scenario DSL. -Args: - rg: RiskGroup object or dict representation. +A single entrypoint parses a YAML string, normalizes keys where needed, +validates against the packaged JSON schema, and returns a canonical +dictionary suitable for downstream expansion/parsing. -Returns: - Flat dict suitable for condition evaluation. +### load_scenario_yaml(yaml_str: 'str') -> 'Dict[str, Any]' -### match_entity_ids(entity_attrs: 'Dict[str, Dict[str, Any]]', conditions: 'List[Condition]', logic: 'str' = 'or') -> 'Set[str]' +Load, normalize, and validate a Scenario YAML string. -Match entity IDs by attribute conditions. +Returns a canonical dictionary representation that downstream parsers can +consume without worrying about YAML-specific quirks (e.g., boolean-like +keys) and with schema shape already enforced. -General primitive for condition-based entity selection. Works with -any entity type as long as attributes are pre-flattened. +--- -Args: - entity_attrs: Mapping of {entity_id: flattened_attrs_dict} - conditions: List of conditions to evaluate - logic: "and" (all must match) or "or" (any must match) +## ngraph.dsl.selectors.normalize -Returns: - Set of matching entity IDs. Returns all IDs if conditions is empty. +Selector parsing and normalization. -### select_nodes(network: "'Network'", selector: 'NodeSelector', default_active_only: 'bool', excluded_nodes: 'Optional[Set[str]]' = None) -> "Dict[str, List['Node']]" +Single entry point for converting raw selector values (strings or dicts) +into NodeSelector objects. -Unified entry point for node selection. +### normalize_selector(raw: 'Union[str, Dict[str, Any], NodeSelector]', context: 'str') -> 'NodeSelector' -Evaluation order: +Normalize a raw selector (string or dict) to a NodeSelector. -1. Select nodes matching `path` regex (or all nodes if path is None) -2. Filter by `match` conditions -3. Filter by `active_only` flag and excluded_nodes -4. Group by `group_by` attribute (overrides regex capture grouping) +All downstream code works with NodeSelector objects only. Args: - network: The network graph. - selector: Node selection specification. - default_active_only: Context-aware default for active_only flag. - Required parameter to prevent silent bugs. - excluded_nodes: Additional node names to exclude. + raw: Either a regex string, selector dict, or existing NodeSelector. + context: Usage context ("adjacency", "demand", "override", "workflow"). + Determines the default for active_only. Returns: - Dict mapping group labels to lists of nodes. + Normalized NodeSelector instance. + +Raises: + ValueError: If selector format is invalid or context is unknown. --- @@ -2092,21 +2193,16 @@ Returns: Serializable result artifacts for analysis workflows. -This module defines dataclasses that capture outputs from analyses and -simulations in a JSON-serializable form: - -- `CapacityEnvelope`: frequency-based capacity distributions and optional - - aggregated flow statistics - -- `FailurePatternResult`: capacity results for specific failure patterns +`CapacityEnvelope` captures a frequency-based capacity distribution, plus +optional aggregated flow statistics, in JSON-serializable form. ### CapacityEnvelope -Frequency-based capacity envelope that stores capacity values as frequencies. +Capacity distribution stored as a value -> occurrence-count map. -This approach is memory-efficient for Monte Carlo analysis where we care -about statistical distributions rather than individual sample order. +Monte Carlo runs repeat the same capacity values many times, so counting +them keeps memory proportional to the number of distinct values. Individual +sample order is not preserved. Attributes: source_pattern: Regex pattern used to select source nodes. @@ -2142,31 +2238,6 @@ Attributes: - `get_percentile(self, percentile: 'float') -> 'float'` - Calculate percentile from frequency distribution. - `to_dict(self) -> 'Dict[str, Any]'` - Convert to dictionary for JSON serialization. -### FailurePatternResult - -Result for a unique failure pattern with associated capacity matrix. - -Attributes: - excluded_nodes: List of failed node IDs. - excluded_links: List of failed link IDs. - capacity_matrix: Dictionary mapping flow keys to capacity values. - count: Number of times this pattern occurred. - is_baseline: Whether this represents the baseline (no failures) case. - -**Attributes:** - -- `excluded_nodes` (List[str]) -- `excluded_links` (List[str]) -- `capacity_matrix` (Dict[str, float]) -- `count` (int) -- `is_baseline` (bool) = False -- `_pattern_key_cache` (str) - -**Methods:** - -- `from_dict(data: 'Dict[str, Any]') -> "'FailurePatternResult'"` - Construct FailurePatternResult from a dictionary. -- `to_dict(self) -> 'Dict[str, Any]'` - Convert to dictionary for JSON serialization. - --- ## ngraph.results.flow @@ -2182,13 +2253,12 @@ arbitrary `data` payloads are sanitized. These dicts are written under `data.flow_results` by steps. Utilities: - _fmt_float_key: Formats floats as stable string keys for JSON serialization. - Uses fixed-point notation with trailing zeros stripped for human-readable, - canonical representations of numeric keys like cost distributions. + _fmt_float_key: Formats floats as stable string keys for JSON serialization, + in fixed-point notation with trailing zeros stripped. ### FlowEntry -Represents a single source→destination flow outcome within an iteration. +One source→destination flow outcome within an iteration. Fields are unit-agnostic. Callers can interpret numbers as needed for presentation (e.g., Gbit/s). @@ -2285,10 +2355,6 @@ export into results without keeping heavy domain objects. Build a concise dictionary snapshot of the scenario state. -Creates a serializable representation of the scenario's failure policies -and demand sets, suitable for export into results without keeping heavy -domain objects. - Args: seed: Scenario-level seed for reproducibility, or None if unseeded. failure_policy_set: FailurePolicySet containing named failure policies. @@ -2329,7 +2395,7 @@ Structure: - `_store` (Dict) = {} - `_metadata` (Dict) = {} -- `_active_step` (Optional) +- `_active_step` (Union) - `_scenario` (Dict) = {} **Methods:** @@ -2339,10 +2405,10 @@ Structure: - `get(self, key: str, default: Any = None) -> Any` - Get a value from the active step scope. - `get_all_step_metadata(self) -> Dict[str, ngraph.results.store.WorkflowStepMetadata]` - Get metadata for all workflow steps. - `get_step(self, step_name: str) -> Dict[str, Any]` - Return the raw dict for a given step name (for cross-step reads). -- `get_step_metadata(self, step_name: str) -> Optional[ngraph.results.store.WorkflowStepMetadata]` - Get metadata for a workflow step. +- `get_step_metadata(self, step_name: str) -> ngraph.results.store.WorkflowStepMetadata | None` - Get metadata for a workflow step. - `get_steps_by_execution_order(self) -> list[str]` - Get step names ordered by their execution order. - `put(self, key: str, value: Any) -> None` - Store a value in the active step under an allowed key. -- `put_step_metadata(self, step_name: str, step_type: str, execution_order: int, *, scenario_seed: Optional[int] = None, step_seed: Optional[int] = None, seed_source: str = 'none', active_seed: Optional[int] = None) -> None` - Store metadata for a workflow step. +- `put_step_metadata(self, step_name: str, step_type: str, execution_order: int, *, scenario_seed: int | None = None, step_seed: int | None = None, seed_source: str = 'none', active_seed: int | None = None) -> None` - Store metadata for a workflow step. - `set_scenario_snapshot(self, snapshot: Dict[str, Any]) -> None` - Attach a normalized scenario snapshot for export. - `to_dict(self) -> Dict[str, Any]` - Return exported results with shape: {workflow, steps, scenario}. @@ -2351,7 +2417,7 @@ Structure: Metadata for a workflow step execution. Attributes: - step_type: The workflow step class name (e.g., 'CapacityEnvelopeAnalysis'). + step_type: The workflow step class name (e.g., 'NetworkStats'). step_name: The instance name of the step. execution_order: Order in which this step was executed (0-based). scenario_seed: Scenario-level seed provided in the YAML (if any). @@ -2371,10 +2437,10 @@ Attributes: - `step_type` (str) - `step_name` (str) - `execution_order` (int) -- `scenario_seed` (Optional) -- `step_seed` (Optional) +- `scenario_seed` (Union) +- `step_seed` (Union) - `seed_source` (str) = none -- `active_seed` (Optional) +- `active_seed` (Union) --- @@ -2390,7 +2456,8 @@ summaries and identifies time-dominant steps (bottlenecks). CPU profiler for NetGraph workflow execution. -Profiles workflow steps using cProfile and identifies bottlenecks. +Profiles each workflow step with cProfile and flags steps that take more +than 10% of total wall time as bottlenecks. **Methods:** @@ -2404,10 +2471,9 @@ Profiles workflow steps using cProfile and identifies bottlenecks. ### PerformanceReporter -Format and render performance profiling results. +Render profiling results as a plain-text report. -Generates plain-text reports with timing analysis, bottleneck identification, -and practical performance tuning suggestions. +Covers per-step timing, bottleneck identification, and tuning suggestions. **Methods:** @@ -2494,8 +2560,8 @@ Defines immutable summary containers for algorithm outputs. Reference to a directed edge via scenario link_id and direction. -Provides stable, scenario-native edge identification across Core reorderings -using the link's unique ID rather than node name tuples. +Identifying an edge by the link's unique ID rather than by a node-name +tuple keeps the reference valid across Core edge reorderings. Attributes: link_id: Scenario link identifier (matches Network.links keys) @@ -2515,7 +2581,7 @@ Captures total flow, cost distribution, and optionally min-cut edges. Attributes: total_flow: Maximum flow value achieved. cost_distribution: Mapping of path cost to flow volume placed at that cost. - min_cut: Saturated edges forming the min-cut (None if not computed). + min_cut: Edges forming a minimum cut (None if not computed). **Attributes:** @@ -2531,13 +2597,12 @@ Attributes: Return a 22-character URL-safe Base64-encoded UUID without padding. -The function generates a random version 4 UUID, encodes the 16 raw bytes -using URL-safe Base64, removes the two trailing padding characters, and -decodes to ASCII. The resulting string length is 22 characters. +The 16 raw bytes of a random version 4 UUID are encoded with URL-safe +Base64; the two trailing padding characters are dropped, leaving 22 ASCII +characters. Returns: - A 22-character URL-safe Base64 representation of a UUID4 without - padding. + A 22-character URL-safe Base64 representation of a UUID4, unpadded. --- @@ -2545,10 +2610,9 @@ Returns: Utilities for building CLI artifact output paths. -This module centralizes logic for composing file and directory paths for -artifacts produced by the NetGraph CLI. Paths are built from an optional -output directory, a prefix (usually derived from the scenario file or -results file), and a per-artifact suffix. +Every artifact path the NetGraph CLI writes is composed here, from an optional +output directory, a prefix (usually derived from the scenario file or results +file), and a per-artifact suffix. ### build_artifact_path(output_dir: 'Optional[Path]', prefix: 'str', suffix: 'str') -> 'Path' @@ -2659,9 +2723,8 @@ Utilities for handling YAML parsing quirks and common operations. Normalize dictionary keys from YAML parsing to ensure consistent string keys. -YAML 1.1 boolean keys (e.g., true, false, yes, no, on, off) get converted to -Python True/False boolean values. This function converts them to predictable -string representations ("True"/"False") and ensures all keys are strings. +YAML 1.1 parses true/false/yes/no/on/off keys as Python booleans. Those +become "True"/"False"; every other key is coerced with str(). Args: data: Dictionary that may contain boolean or other non-string keys from YAML parsing @@ -2680,35 +2743,38 @@ Examples: ## ngraph.analysis.context -AnalysisContext: Prepared state for efficient network analysis. +AnalysisContext: prepared graph state for repeated network analysis. -This module provides the primary API for network analysis in NetGraph. -AnalysisContext encapsulates Core graph infrastructure and provides -methods for max-flow, shortest paths, and sensitivity analysis. +AnalysisContext holds the Core graph infrastructure and exposes max-flow, +shortest-path, and sensitivity analysis over it. Usage: # One-off analysis from ngraph import analyze flow = analyze(network).max_flow("^A$", "^B$") - # Efficient repeated analysis (bound context) + # Repeated analysis over one prepared graph (bound context) ctx = analyze(network, source="^A$", sink="^B$") baseline = ctx.max_flow() degraded = ctx.max_flow(excluded_links=failed_links) ### AnalysisContext -Prepared state for efficient network analysis. +Prepared graph state for repeated network analysis. -Encapsulates Core graph infrastructure. Supports two usage patterns: +Wraps the Core graph infrastructure. Two usage patterns: -**Unbound** - flexible, specify source/sink per-call: +**Unbound** - source/sink given per call: ctx = AnalysisContext.from_network(network) cost = ctx.shortest_path_cost("A", "B") - flow = ctx.max_flow("A", "B") # Builds pseudo-nodes each call + flow = ctx.max_flow("A", "B") -**Bound** - optimized for repeated analysis with same groups: +Every flow call on an unbound context builds a full temporary bound +context, which rebuilds the graph from scratch; bind the context instead +for repeated flow analysis. + +**Bound** - source/sink fixed at construction, reused across calls: ctx = AnalysisContext.from_network( network, @@ -2718,6 +2784,9 @@ Encapsulates Core graph infrastructure. Supports two usage patterns: baseline = ctx.max_flow() # Uses pre-built pseudo-nodes degraded = ctx.max_flow(excluded_links=failed) +Selectors, here and in every method taking one, are either a regex path +string or a dict with ``path``/``group_by``/``match``. + Thread Safety: Immutable after creation. Safe for concurrent analysis calls with different exclusion sets. @@ -2729,26 +2798,24 @@ Attributes: **Attributes:** - `_network` ('Network') -- `_handle` (netgraph_core.Graph) -- `_multidigraph` (netgraph_core.StrictMultiDiGraph) -- `_node_mapper` (_NodeMapper) -- `_edge_mapper` (_EdgeMapper) -- `_algorithms` (netgraph_core.Algorithms) -- `_disabled_node_ids` (FrozenSet[int]) -- `_disabled_link_ids` (FrozenSet[str]) -- `_link_id_to_edge_indices` (Mapping[str, Tuple[int, ...]]) +- `_core` (Optional[_GraphBuildResult]) - `_source` (Optional[Union[str, Dict[str, Any]]]) - `_sink` (Optional[Union[str, Dict[str, Any]]]) - `_mode` (Optional[Mode]) - `_pseudo_context` (Optional[_PseudoNodeContext]) +- `_augmentations` (Tuple[AugmentationEdge, ...]) = () +- `_core_lock` (threading.Lock) **Methods:** +- `build_edge_mask(self, excluded_links: 'Optional[Set[str]]' = None) -> 'np.ndarray'` - Build an edge inclusion mask for Core algorithms. +- `build_node_mask(self, excluded_nodes: 'Optional[Set[str]]' = None) -> 'np.ndarray'` - Build a node inclusion mask for Core algorithms. - `from_network(network: "'Network'", *, source: 'Optional[Union[str, Dict[str, Any]]]' = None, sink: 'Optional[Union[str, Dict[str, Any]]]' = None, mode: 'Mode' = , augmentations: 'Optional[List[AugmentationEdge]]' = None) -> "'AnalysisContext'"` - Create analysis context from network. - `k_shortest_paths(self, source: 'Optional[Union[str, Dict[str, Any]]]' = None, sink: 'Optional[Union[str, Dict[str, Any]]]' = None, *, mode: 'Mode' = , max_k: 'int' = 3, edge_select: 'EdgeSelect' = , max_path_cost: 'float' = inf, max_path_cost_factor: 'Optional[float]' = None, split_parallel_edges: 'bool' = False, excluded_nodes: 'Optional[Set[str]]' = None, excluded_links: 'Optional[Set[str]]' = None) -> 'Dict[Tuple[str, str], List[Path]]'` - Compute up to K shortest paths per group pair. - `max_flow(self, source: 'Optional[Union[str, Dict[str, Any]]]' = None, sink: 'Optional[Union[str, Dict[str, Any]]]' = None, *, mode: 'Mode' = , shortest_path: 'bool' = False, require_capacity: 'bool' = True, flow_placement: 'FlowPlacement' = , excluded_nodes: 'Optional[Set[str]]' = None, excluded_links: 'Optional[Set[str]]' = None) -> 'Dict[Tuple[str, str], float]'` - Compute maximum flow between node groups. - `max_flow_detailed(self, source: 'Optional[Union[str, Dict[str, Any]]]' = None, sink: 'Optional[Union[str, Dict[str, Any]]]' = None, *, mode: 'Mode' = , shortest_path: 'bool' = False, require_capacity: 'bool' = True, flow_placement: 'FlowPlacement' = , excluded_nodes: 'Optional[Set[str]]' = None, excluded_links: 'Optional[Set[str]]' = None, include_min_cut: 'bool' = False) -> 'Dict[Tuple[str, str], MaxFlowResult]'` - Compute max flow with detailed results including cost distribution. - `sensitivity(self, source: 'Optional[Union[str, Dict[str, Any]]]' = None, sink: 'Optional[Union[str, Dict[str, Any]]]' = None, *, mode: 'Mode' = , shortest_path: 'bool' = False, require_capacity: 'bool' = True, flow_placement: 'FlowPlacement' = , excluded_nodes: 'Optional[Set[str]]' = None, excluded_links: 'Optional[Set[str]]' = None) -> 'Dict[Tuple[str, str], Dict[str, float]]'` - Analyze sensitivity of max flow to edge failures. +- `sensitivity_with_flow(self, source: 'Optional[Union[str, Dict[str, Any]]]' = None, sink: 'Optional[Union[str, Dict[str, Any]]]' = None, *, mode: 'Mode' = , shortest_path: 'bool' = False, require_capacity: 'bool' = True, flow_placement: 'FlowPlacement' = , excluded_nodes: 'Optional[Set[str]]' = None, excluded_links: 'Optional[Set[str]]' = None) -> 'Dict[Tuple[str, str], Tuple[float, Dict[str, float]]]'` - Compute max flow and edge sensitivity together per group pair. - `shortest_path_cost(self, source: 'Optional[Union[str, Dict[str, Any]]]' = None, sink: 'Optional[Union[str, Dict[str, Any]]]' = None, *, mode: 'Mode' = , edge_select: 'EdgeSelect' = , excluded_nodes: 'Optional[Set[str]]' = None, excluded_links: 'Optional[Set[str]]' = None) -> 'Dict[Tuple[str, str], float]'` - Compute shortest path costs between node groups. - `shortest_paths(self, source: 'Optional[Union[str, Dict[str, Any]]]' = None, sink: 'Optional[Union[str, Dict[str, Any]]]' = None, *, mode: 'Mode' = , edge_select: 'EdgeSelect' = , split_parallel_edges: 'bool' = False, excluded_nodes: 'Optional[Set[str]]' = None, excluded_links: 'Optional[Set[str]]' = None) -> 'Dict[Tuple[str, str], List[Path]]'` - Compute concrete shortest paths between node groups. @@ -2764,19 +2831,19 @@ Attributes: source: Source node name (real or pseudo) target: Target node name (real or pseudo) capacity: Edge capacity - cost: Edge cost (converted to int64 for Core) + cost: Edge cost (must be an integer value; Core uses int64 costs) ### analyze(network: "'Network'", *, source: 'Optional[Union[str, Dict[str, Any]]]' = None, sink: 'Optional[Union[str, Dict[str, Any]]]' = None, mode: 'Mode' = , augmentations: 'Optional[List[AugmentationEdge]]' = None) -> 'AnalysisContext' Create an analysis context for the network. -This is THE primary entry point for network analysis in NetGraph. +Primary entry point for network analysis in NetGraph. Args: network: Network topology to analyze. source: Optional source node selector (string path or selector dict). - If provided with sink, creates bound context with pre-built - pseudo-nodes for efficient repeated flow analysis. + If provided with sink, creates a bound context whose pseudo + nodes are pre-built once and reused by every flow call. sink: Optional sink node selector (string path or selector dict). mode: Group mode (COMBINE or PAIRWISE). Only used if bound. augmentations: Optional custom augmentation edges. @@ -2784,13 +2851,30 @@ Args: Returns: AnalysisContext ready for analysis calls. +Raises: + ValueError: If only one of source/sink is provided, or if a bound + selector matches no nodes. + ValueError: If any link capacity is at or above LARGE_CAPACITY (1e15, + the internal pseudo-edge capacity), since such a link would be + silently clamped by the pseudo attachment edges in combine-mode + flows. + ValueError: If any link or augmentation cost is negative or + non-integer, or if the total of all edge costs reaches 2**62. + Core's int64 cost arithmetic would overflow and silently corrupt + SPF and flow results. + +Note: + The capacity and cost checks run during the graph build, which happens + here for bound contexts and for contexts with custom augmentations, + and on first use otherwise. + Examples: One-off analysis (unbound context): flow = analyze(network).max_flow("^A$", "^B$") paths = analyze(network).shortest_paths("^A$", "^B$") - Efficient repeated analysis (bound context): + Repeated analysis over one prepared graph (bound context): ctx = analyze(network, source="^dc/", sink="^edge/") baseline = ctx.max_flow() @@ -2802,42 +2886,15 @@ Examples: for scenario in failure_scenarios: result = ctx.max_flow(excluded_links=scenario) -### build_edge_mask(ctx: 'AnalysisContext', excluded_links: 'Optional[Set[str]]' = None) -> 'np.ndarray' - -Build an edge mask array for Core algorithms. - -Uses O(|excluded| + |disabled|) time complexity. -Core semantics: True = include, False = exclude. - -Args: - ctx: AnalysisContext with pre-computed edge index mapping. - excluded_links: Optional set of link IDs to exclude. - -Returns: - Boolean numpy array of shape (num_edges,) where True means included. - -### build_node_mask(ctx: 'AnalysisContext', excluded_nodes: 'Optional[Set[str]]' = None) -> 'np.ndarray' - -Build a node mask array for Core algorithms. - -Uses O(|excluded| + |disabled|) time complexity. -Core semantics: True = include, False = exclude. - -Args: - ctx: AnalysisContext with pre-computed disabled node IDs. - excluded_nodes: Optional set of node names to exclude. - -Returns: - Boolean numpy array of shape (num_nodes,) where True means included. - --- ## ngraph.analysis.demand Demand expansion: converts TrafficDemand specs into concrete placement demands. -Supports both pairwise and combine modes through augmentation-based pseudo nodes. -Uses unified selectors for node selection. +Combine mode aggregates each side behind augmentation-based pseudo nodes; +pairwise mode emits one demand per (source, target) pair. Endpoints are +resolved through the shared selector layer. ### DemandExpansion @@ -2865,7 +2922,6 @@ Attributes: volume: Traffic volume to place. priority: Priority class (lower is higher priority). policy_preset: FlowPolicy configuration preset. - demand_id: Parent TrafficDemand ID for tracking. **Attributes:** @@ -2874,7 +2930,6 @@ Attributes: - `volume` (float) - `priority` (int) - `policy_preset` (FlowPolicyPreset) -- `demand_id` (str) ### expand_demands(network: 'Network', traffic_demands: 'List[TrafficDemand]', default_policy_preset: 'FlowPolicyPreset' = ) -> 'DemandExpansion' @@ -2902,7 +2957,9 @@ Returns: DemandExpansion with demands and augmentations. Raises: - ValueError: If no demands could be expanded or unsupported mode. + ValueError: If no demands could be expanded, or if two demands share + an id (pseudo node names embed the id, so duplicates would merge + distinct demands' attachment edges into one endpoint). --- @@ -2910,15 +2967,16 @@ Raises: FailureManager for Monte Carlo failure analysis. -Provides the failure analysis engine for NetGraph. Supports parallel -processing, graph caching, and failure policy handling for workflow steps -and direct programmatic use. +Runs an analysis function over many failure scenarios, handling failure policy +application, graph caching, and parallel execution. Used by workflow steps and +directly from user code. Performance characteristics: Time complexity: O(S + I * A / P), where S is one-time graph setup cost, I is iteration count, A is per-iteration analysis cost, and P is parallelism. -Graph caching amortizes expensive graph construction across all iterations, -and O(|excluded|) mask building replaces O(V+E) iteration. +Graph caching amortizes graph construction across all iterations: each +iteration applies its exclusions as an O(|excluded|) mask update instead of +rebuilding the graph or re-scanning all O(V+E) nodes and edges. Space complexity: O(V + E + I * R), where V and E are node and link counts, and R is result size per iteration. The pre-built graph is shared across @@ -2938,15 +2996,14 @@ parameters, returning results of any type. ### FailureManager -Failure analysis engine with Monte Carlo capabilities. +Run an analysis function across Monte Carlo failure scenarios. -This is the component for failure analysis in NetGraph. -Provides parallel processing, worker caching, and failure -policy handling for workflow steps and direct notebook usage. +Applies a failure policy, deduplicates identical failure patterns, and +runs iterations in parallel. Used by workflow steps and directly from user code. -The FailureManager can execute any analysis function that takes a Network -with exclusion sets and returns results, making it generic for different -types of failure analysis (capacity, traffic, connectivity, etc.). +Executes any analysis function that takes a Network plus exclusion sets and +returns results, so the same engine covers capacity, traffic, connectivity, +and custom analyses. Attributes: network: The underlying network (not modified during analysis). @@ -2957,11 +3014,11 @@ Attributes: - `compute_exclusions(self, policy: "'FailurePolicy | None'" = None, seed_offset: 'int | None' = None, failure_trace: 'Optional[Dict[str, Any]]' = None) -> 'tuple[set[str], set[str]]'` - Compute set of nodes and links to exclude for a failure iteration. - `get_failure_policy(self) -> "'FailurePolicy | None'"` - Get failure policy for analysis. -- `run_demand_placement_monte_carlo(self, demands_config: 'list[dict[str, Any]] | Any', iterations: 'int' = 100, parallelism: 'int' = 1, placement_rounds: 'int | str' = 'auto', seed: 'int | None' = None, store_failure_patterns: 'bool' = False, include_flow_details: 'bool' = False, include_used_edges: 'bool' = False) -> 'Any'` - Analyze traffic demand placement success under failures. -- `run_max_flow_monte_carlo(self, source: 'str | dict[str, Any]', target: 'str | dict[str, Any]', mode: 'str' = 'combine', iterations: 'int' = 100, parallelism: 'int' = 1, shortest_path: 'bool' = False, require_capacity: 'bool' = True, flow_placement: 'FlowPlacement | str' = , seed: 'int | None' = None, store_failure_patterns: 'bool' = False, include_flow_summary: 'bool' = False, include_min_cut: 'bool' = False) -> 'Any'` - Analyze maximum flow capacity envelopes between node groups under failures. +- `run_demand_placement_monte_carlo(self, demands_config: 'list[dict[str, Any]] | Any', iterations: 'int' = 100, parallelism: 'int' = 1, seed: 'int | None' = None, store_failure_patterns: 'bool' = False, include_flow_details: 'bool' = False, include_used_edges: 'bool' = False) -> 'Any'` - Analyze traffic demand placement success under failures. +- `run_max_flow_monte_carlo(self, source: 'str | dict[str, Any]', target: 'str | dict[str, Any]', mode: 'str' = 'combine', iterations: 'int' = 100, parallelism: 'int' = 1, shortest_path: 'bool' = False, require_capacity: 'bool' = True, flow_placement: 'FlowPlacement | str' = , seed: 'int | None' = None, store_failure_patterns: 'bool' = False, include_flow_summary: 'bool' = False, include_min_cut: 'bool' = False) -> 'Any'` - Compute max-flow capacity envelopes between node groups under failures. - `run_monte_carlo_analysis(self, analysis_func: 'AnalysisFunction', iterations: 'int' = 1, parallelism: 'int' = 1, seed: 'int | None' = None, store_failure_patterns: 'bool' = False, **analysis_kwargs) -> 'dict[str, Any]'` - Run Monte Carlo failure analysis with any analysis function. - `run_sensitivity_monte_carlo(self, source: 'str | dict[str, Any]', target: 'str | dict[str, Any]', mode: 'str' = 'combine', iterations: 'int' = 100, parallelism: 'int' = 1, shortest_path: 'bool' = False, flow_placement: 'FlowPlacement | str' = , seed: 'int | None' = None, store_failure_patterns: 'bool' = False) -> 'dict[str, Any]'` - Analyze component criticality for flow capacity under failures. -- `run_single_failure_scenario(self, analysis_func: 'AnalysisFunction', **kwargs) -> 'Any'` - Run a single failure scenario for convenience. +- `run_single_failure_scenario(self, analysis_func: 'AnalysisFunction', **kwargs) -> 'Any'` - Run one failure iteration, for quick analysis or debugging. --- @@ -2973,29 +3030,35 @@ These functions are designed for use with FailureManager. Each analysis function takes a Network, exclusion sets, and analysis-specific parameters, returning results of type FlowIterationResult. -Parameters should ideally be hashable for efficient caching in FailureManager; -non-hashable objects are identified by memory address for cache key generation. +Parameters should ideally be hashable so FailureManager can deduplicate +identical failure patterns before dispatch; non-hashable objects are keyed +by memory address. -Graph caching enables efficient repeated analysis with different exclusion -sets by building the graph once and using O(|excluded|) masks for exclusions. +Graph caching builds the graph once and applies each exclusion set as an +O(|excluded|) mask instead of rebuilding. -SPF caching enables efficient demand placement by computing shortest paths once -per unique source node rather than once per demand. For networks with many demands -sharing the same sources, this can reduce SPF computations by an order of magnitude. +SPF caching computes shortest paths once per unique source node rather than +once per demand. For networks with many demands sharing the same sources, this +can reduce SPF computations by an order of magnitude. -### build_demand_context(network: "'Network'", demands_config: 'list[dict[str, Any]]') -> 'AnalysisContext' +### build_demand_placement_inputs(network: "'Network'", demands_config: 'list[dict[str, Any]]') -> 'tuple[AnalysisContext, DemandExpansion, list[tuple[int, int]]]' -Build an AnalysisContext for repeated demand placement analysis. +Build context, expansion, and resolved node IDs for demand placement. -Pre-computes the graph with augmentations (pseudo source/target nodes) for -efficient repeated analysis with different exclusion sets. +Reconstructs and expands demands once so repeated calls to +demand_placement_analysis (e.g., Monte Carlo iterations) can skip the +per-iteration expansion and node-ID resolution work. Building the +expansion and context together guarantees that pseudo node names +(derived from demand ids) match the context's graph. Args: network: Network instance. - demands_config: List of demand configurations (same format as demand_placement_analysis). + demands_config: List of demand configurations (same format as + demand_placement_analysis). Returns: - AnalysisContext ready for use with demand_placement_analysis. + Tuple of (context, expansion, resolved_ids) where resolved_ids holds + (src_id, dst_id) pairs aligned with expansion.demands. ### build_maxflow_context(network: "'Network'", source: 'str | dict[str, Any]', target: 'str | dict[str, Any]', mode: 'str' = 'combine') -> 'AnalysisContext' @@ -3013,17 +3076,28 @@ Args: Returns: AnalysisContext ready for use with max_flow_analysis or sensitivity_analysis. -### demand_placement_analysis(network: "'Network'", excluded_nodes: 'Set[str]', excluded_links: 'Set[str]', demands_config: 'list[dict[str, Any]]', placement_rounds: 'int | str' = 'auto', include_flow_details: 'bool' = False, include_used_edges: 'bool' = False, context: 'Optional[AnalysisContext]' = None) -> 'FlowIterationResult' +### demand_placement_analysis(network: "'Network'", excluded_nodes: 'Set[str]', excluded_links: 'Set[str]', demands_config: 'list[dict[str, Any]]', include_flow_details: 'bool' = False, include_used_edges: 'bool' = False, context: 'Optional[AnalysisContext]' = None, expansion: 'Optional[DemandExpansion]' = None, resolved_ids: 'Optional[Sequence[tuple[int, int]]]' = None) -> 'FlowIterationResult' Analyze traffic demand placement success rates using Core directly. -This function: +Steps: + +1. Build Core infrastructure (graph, algorithms, flow_graph), or reuse the + + pre-built ``context`` + +2. Expand demands into concrete (src, dst, volume) tuples (or use a + + pre-computed expansion) + +3. Place each demand using SPF caching for cacheable policies. + + SHORTEST_PATHS_* presets admit flow onto the cost-only shortest paths + of the base topology and drop overflow (IGP semantics); TE_* presets + reroute remaining volume onto residual-capacity paths. -1. Builds Core infrastructure (graph, algorithms, flow_graph) or uses cached -2. Expands demands into concrete (src, dst, volume) tuples -3. Places each demand using SPF caching for cacheable policies -4. Uses FlowPolicy for complex multi-flow policies -5. Aggregates results into FlowIterationResult +4. Fall back to FlowPolicy for presets outside CACHEABLE_PRESETS +5. Aggregate results into FlowIterationResult SPF Caching Optimization: For cacheable policies (ECMP, WCMP, TE_WCMP_UNLIM), SPF results are @@ -3036,10 +3110,18 @@ Args: excluded_nodes: Set of node names to exclude temporarily. excluded_links: Set of link IDs to exclude temporarily. demands_config: List of demand configurations (serializable dicts). - placement_rounds: Number of placement optimization rounds (unused - Core handles internally). include_flow_details: When True, include cost_distribution per flow. include_used_edges: When True, include set of used edges per demand in entry data. - context: Pre-built AnalysisContext for fast repeated analysis. + context: Pre-built AnalysisContext, reused across calls. Must be built + from this same demands_config - pseudo node names embed demand + ids, so a context built from a different config raises ValueError + during endpoint resolution. See build_demand_placement_inputs. + expansion: Pre-computed DemandExpansion matching demands_config. When + provided, per-call demand reconstruction and expansion are skipped. + Must be built together with ``context`` (pseudo node names embed + demand ids) - see build_demand_placement_inputs. + resolved_ids: Pre-resolved (src_id, dst_id) pairs aligned with + expansion.demands. Only valid together with ``context``. Returns: FlowIterationResult describing this iteration. @@ -3055,13 +3137,15 @@ Args: source: Source node selector (string path or selector dict). target: Target node selector (string path or selector dict). mode: Flow analysis mode ("combine" or "pairwise"). - shortest_path: Whether to use shortest paths only. + shortest_path: If True, use single-tier shortest-path flow (IP/IGP + mode) instead of full iterative max-flow. require_capacity: If True (default), path selection considers available capacity. If False, path selection is cost-only (true IP/IGP semantics). - flow_placement: Flow placement strategy. + flow_placement: PROPORTIONAL (WCMP) or EQUAL_BALANCED (ECMP). include_flow_details: Whether to collect cost distribution and similar details. include_min_cut: Whether to include min-cut edge list in entry data. - context: Pre-built AnalysisContext for efficient repeated analysis. + context: Pre-built AnalysisContext reused across calls. Must be + unbound or bound to these same source/target/mode arguments. Returns: FlowIterationResult describing this iteration. @@ -3088,8 +3172,9 @@ Args: shortest_path: If True, use single-tier shortest-path flow (IP/IGP mode). Reports only edges used under ECMP routing. If False (default), use full iterative max-flow (SDN/TE mode) and report all saturated edges. - flow_placement: Flow placement strategy. - context: Pre-built AnalysisContext for efficient repeated analysis. + flow_placement: PROPORTIONAL (WCMP) or EQUAL_BALANCED (ECMP). + context: Pre-built AnalysisContext reused across calls. Must be + unbound or bound to these same source/target/mode arguments. Returns: FlowIterationResult with sensitivity data in each FlowEntry.data. @@ -3116,7 +3201,12 @@ Single demand placement result. ### PlacementResult -Complete placement result. +Result of one `place_demands` call. + +Attributes: + summary: Aggregated demand and placed totals. + entries: Per-demand results, or None unless the call passed + ``collect_entries=True``. **Attributes:** @@ -3132,33 +3222,52 @@ Aggregated placement totals. - `total_demand` (float) - `total_placed` (float) -### place_demands(demands: "Sequence['ExpandedDemand']", volumes: 'Sequence[float]', flow_graph: 'netgraph_core.FlowGraph', ctx: "'AnalysisContext'", node_mask: 'np.ndarray', edge_mask: 'np.ndarray', *, resolved_ids: 'Sequence[tuple[int, int]] | None' = None, collect_entries: 'bool' = False, include_cost_distribution: 'bool' = False, include_used_edges: 'bool' = False) -> 'PlacementResult' +### place_demands(demands: "Sequence['ExpandedDemand']", volumes: 'Sequence[float]', flow_graph: 'netgraph_core.FlowGraph', ctx: "'AnalysisContext'", node_mask: 'np.ndarray', edge_mask: 'np.ndarray', *, resolved_ids: 'Sequence[tuple[int, int]] | None' = None, collect_entries: 'bool' = False, include_cost_distribution: 'bool' = False, include_used_edges: 'bool' = False, dag_cache: 'dict[tuple[int, bool], tuple[np.ndarray, Any]] | None' = None) -> 'PlacementResult' Place demands on a flow graph with SPF caching. Args: demands: Expanded demands (policy_preset, priority, names). - volumes: Demand volumes (allows scaling without modifying demands). - flow_graph: Target FlowGraph. - ctx: AnalysisContext with graph infrastructure. - node_mask: Node inclusion mask. - edge_mask: Edge inclusion mask. - resolved_ids: Pre-resolved (src_id, dst_id) pairs. Computed if None. + volumes: Volume per demand, positionally aligned with `demands`; + passed separately so callers can scale without rebuilding demands. + flow_graph: Target FlowGraph; placed flow accumulates here. + ctx: AnalysisContext holding the built graph and Core algorithms. + node_mask: Node inclusion mask (True = include), as built by + ctx.build_node_mask. + edge_mask: Edge inclusion mask (True = include), as built by + ctx.build_edge_mask. + resolved_ids: Pre-resolved (src_id, dst_id) pairs. Computed from the + demand names if None. collect_entries: If True, populate result.entries. include_cost_distribution: Include cost distribution in entries. include_used_edges: Include used edges in entries. + dag_cache: Optional persistent SPF DAG cache keyed by + (src_id, uses_capacity_aware_selection). Base DAGs depend only on + the static graph and masks, so repeated calls with the same + context and masks (e.g. MSD probes) can share one cache. Returns: PlacementResult with summary and optional entries. +Raises: + ValueError: If a demand endpoint is not present in ``ctx``'s graph + (checked only when ``resolved_ids`` is not supplied). Pseudo node + names embed demand ids, so this usually means the context was + built from a different demands_config. + ValueError: If two policy-based demands (presets outside + CACHEABLE_PRESETS) share the same (src, dst, priority): their + FlowIndex values would collide and silently merge in FlowGraph. + ValueError: If ``demands``, ``volumes``, and ``resolved_ids`` are not + all the same length. + --- ## ngraph.lib.nx NetworkX graph conversion utilities. -This module provides functions to convert between NetworkX graphs and the -internal graph representation used by ngraph for high-performance algorithms. +Convert between NetworkX graphs and the internal graph representation that +ngraph's algorithms run on. Example: >>> import networkx as nx @@ -3192,11 +3301,13 @@ Attributes: Example: >>> graph, node_map, edge_map = from_networkx(G) - >>> # After running algorithms, map flow results back to original edges - >>> for ext_id, flow in enumerate(flow_state.edge_flow_view()): + >>> # edge_flow_view() is indexed by internal Core edge index, so + >>> # translate through ext_edge_ids_view() before using to_ref. + >>> ext_edge_ids = graph.ext_edge_ids_view() + >>> for edge_idx, flow in enumerate(flow_state.edge_flow_view()): ... if flow > 0: - ... u, v, key = edge_map.to_ref[ext_id] - ... G.edges[u, v, key]["flow"] = flow + ... u, v, key = edge_map.to_ref[int(ext_edge_ids[edge_idx])] + ... G.edges[u, v, key]["flow"] = flow # G.edges[u, v] for a DiGraph **Attributes:** @@ -3232,7 +3343,7 @@ Example: - `from_names(names: 'List[Hashable]') -> "'NodeMap'"` - Create a NodeMap from a list of node names. -### from_networkx(G: 'NxGraph', *, capacity_attr: 'str' = 'capacity', cost_attr: 'str' = 'cost', default_capacity: 'float' = 1.0, default_cost: 'int' = 1, bidirectional: 'bool' = False) -> 'Tuple[netgraph_core.StrictMultiDiGraph, NodeMap, EdgeMap]' +### from_networkx(G: 'NxGraph', *, capacity_attr: 'str' = 'capacity', cost_attr: 'str' = 'cost', default_capacity: 'float' = 1.0, default_cost: 'int' = 1, bidirectional: 'Optional[bool]' = None) -> 'Tuple[netgraph_core.StrictMultiDiGraph, NodeMap, EdgeMap]' Convert a NetworkX graph to ngraph's internal graph format. @@ -3243,11 +3354,17 @@ the returned NodeMap and EdgeMap preserve mappings for result interpretation. Args: G: NetworkX graph (DiGraph, MultiDiGraph, Graph, or MultiGraph) capacity_attr: Edge attribute name for capacity (default: "capacity") - cost_attr: Edge attribute name for cost (default: "cost") + cost_attr: Edge attribute name for cost (default: "cost"). Cost values + must be integers (netgraph_core requires int64 costs); fractional + values raise ValueError. default_capacity: Capacity value when attribute is missing (default: 1.0) - default_cost: Cost value when attribute is missing (default: 1) - bidirectional: If True, add reverse edge for each edge. Useful for - undirected connectivity analysis. (default: False) + default_cost: Cost value when attribute is missing (default: 1). + Must be an integer value. + bidirectional: If True, add a reverse edge for each edge. If None + (default), inferred from the graph type: directed inputs get one + arc per edge, undirected inputs get antiparallel arc pairs (the + standard undirected-to-directed reduction for max-flow and + reachability). Pass an explicit True or False to override. Returns: Tuple of (graph, node_map, edge_map) where: @@ -3258,7 +3375,8 @@ Returns: Raises: TypeError: If G is not a NetworkX graph - ValueError: If graph has no nodes + ValueError: If graph has no nodes, or an edge cost is not an integer + value Example: >>> import networkx as nx @@ -3267,10 +3385,10 @@ Example: >>> graph, node_map, edge_map = from_networkx(G) >>> graph.num_nodes() 2 - >>> node_map.to_index["src"] - 0 - >>> edge_map.to_ref[0] # First edge - ('dst', 'src', 0) # sorted node order: dst < src + >>> node_map.to_index # node indices assigned in sorted-name order + {'dst': 0, 'src': 1} + >>> edge_map.to_ref[0] # edge refs preserve original (u, v, key) + ('src', 'dst', 0) ### to_networkx(graph: 'netgraph_core.StrictMultiDiGraph', node_map: 'Optional[NodeMap]' = None, *, capacity_attr: 'str' = 'capacity', cost_attr: 'str' = 'cost') -> "'nx.MultiDiGraph'" diff --git a/docs/reference/api.md b/docs/reference/api.md index 0740f77..3149710 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -8,7 +8,7 @@ Quick links: - [CLI Reference](cli.md) -- command-line tools for running scenarios - [Auto-Generated API Reference](api-full.md) -- complete class and method documentation -This section provides a curated guide to NetGraph's Python API, organized by typical usage patterns. +A curated guide to NetGraph's Python API, organized by typical usage patterns. ## 1. Programmatic Quickstart @@ -45,13 +45,11 @@ print("baseline", baseline, "degraded", degraded) ## 2. Fundamentals -The core components that form the foundation of most NetGraph programs. +The three types most NetGraph programs are built from. ### Scenario -**Purpose:** Coordinates network topology, workflow execution, and result storage for complete analysis pipelines. - -**When to use:** Entry point for analysis workflows - load from YAML for declarative scenarios or construct programmatically for direct API access. +The entry point for analysis workflows: it owns the network, runs the workflow steps in order, and stores their results. Load one from YAML for declarative scenarios, or construct one programmatically. ```python from pathlib import Path @@ -72,15 +70,13 @@ print(exported["workflow"].keys()) **Key Methods:** - `from_yaml(yaml_str, default_components=None)` - Parse scenario from YAML string (use `Path.read_text()` for file loading) -- `run()` - Execute workflow steps in sequence +- `run(step_hook=None)` - Execute workflow steps in sequence. The optional `step_hook` is a callable that receives each `WorkflowStep` and returns a context manager entered around that step's execution (used by the CLI for per-step profiling) -**Integration:** Scenario coordinates Network topology, workflow execution, and Results collection. Components can also be used independently for direct programmatic access. +Network, workflow, and Results can also be used independently of Scenario for direct programmatic access. ### Network -**Purpose:** Represents network topology. - -**When to use:** Core component for representing network structure. Used directly for programmatic topology creation or accessed via `scenario.network`. +The topology itself: nodes, links, and risk groups. Construct one directly for programmatic use, or read the YAML-built topology from `scenario.network`. ```python from ngraph import Network, Node, Link, analyze @@ -109,9 +105,7 @@ print(flow_result) # {("^n1$", "^n2$"): 100.0} ### Results -**Purpose:** Centralized container for storing and retrieving analysis results from workflow steps. - -**When to use:** Managed by Scenario; stores workflow step outputs with metadata. Access via `scenario.results` for result retrieval and custom step implementation. +Holds each workflow step's output, with metadata, under that step's name; every step writes here. Managed by Scenario - access it via `scenario.results` to read results or to write from a custom step. ```python # Access results from scenario @@ -130,8 +124,6 @@ print(list(all_data["steps"].keys())) - `get_step(step_name)` - Retrieve complete step dict for cross-step reads - `to_dict()` - Export results with shape `{workflow, steps, scenario}` (JSON-serializable) -**Integration:** Used by all workflow steps for result storage. Provides consistent access pattern for analysis outputs. - ## 3. NetworkX Integration Convert between NetworkX graphs and the internal graph format for algorithm execution. @@ -161,7 +153,7 @@ handle = algorithms.build_graph(graph) src_idx = node_map.to_index["A"] dst_idx = node_map.to_index["C"] dists, _ = algorithms.spf(handle, src=src_idx, dst=dst_idx) -print(f"Shortest path cost A->C: {dists[dst_idx]}") # 15 (via B) +print(f"Shortest path cost A->C: {dists[dst_idx]}") # 15.0 (via B) ``` **Key Functions:** @@ -180,9 +172,9 @@ print(f"Shortest path cost A->C: {dists[dst_idx]}") # 15 (via B) **Options:** -- `bidirectional=True` - Add reverse edge for each edge (for undirected analysis) +- `bidirectional` - Direction handling. `None` (default) infers from the graph type: directed inputs get one arc per edge; undirected inputs get antiparallel arc pairs (the standard undirected-to-directed reduction for max-flow/reachability). Pass an explicit `True`/`False` to override. - `capacity_attr` / `cost_attr` - Custom attribute names for capacity and cost -- `default_capacity` / `default_cost` - Default values when attributes missing +- `default_capacity` / `default_cost` - Default values when attributes missing. Cost values must be integers — `from_networkx` raises `ValueError` on fractional costs because the core engine requires int64 costs; pre-scale fractional costs (e.g., multiply by 10 or 100) before conversion. ### Writing Results Back @@ -191,25 +183,27 @@ print(f"Shortest path cost A->C: {dists[dst_idx]}") # 15 (via B) flow_state = netgraph_core.FlowState(graph) # ... place flow ... -# Use edge_map to update original NetworkX graph -edge_flows = flow_state.edge_flow_view() -for edge_id, flow in enumerate(edge_flows): +# edge_flow_view() is indexed by internal Core edge index; translate through +# ext_edge_ids_view() to the external edge IDs that edge_map is keyed by. +ext_edge_ids = graph.ext_edge_ids_view() +for edge_idx, flow in enumerate(flow_state.edge_flow_view()): if flow > 0: - u, v, key = edge_map.to_ref[edge_id] - G.edges[u, v, key]["flow"] = float(flow) + u, v, key = edge_map.to_ref[int(ext_edge_ids[edge_idx])] + # G is a DiGraph here; for a MultiDiGraph source graph use G.edges[u, v, key] + G.edges[u, v]["flow"] = float(flow) ``` ## 4. Basic Analysis -Essential analysis capabilities for network evaluation. +Max-flow, shortest paths, and edge sensitivity. ### Flow Analysis with `analyze()` -**Purpose:** Calculate network flows between source and sink groups with various policies and constraints. +**Purpose:** Calculate network flows between source and sink groups. -**When to use:** Compute network capacity between source and sink groups. Supports multiple flow placement policies and failure scenarios. +**When to use:** Measuring capacity between source and sink groups, under a choice of flow placement policy and with nodes or links excluded to model failures. -**Performance:** Max-flow computation executes in C++ with the GIL released for concurrent execution. Algorithm uses successive shortest paths with blocking flow augmentation; complexity is O(V^2 E log V) worst-case. +**Performance:** Max-flow computation executes in C++ with the GIL released for concurrent execution. The algorithm uses successive shortest paths on the residual graph, pushing a blocking flow across the full ECMP/WCMP shortest-path DAG at each augmentation step until no augmenting path remains. Worst case is `O(E * (V^2 E + (V+E) log V))`; in practice the phase count equals the small number of cost tiers actually used, and the `kMinFlow` tolerance caps phases at `F / kMinFlow` for total flow `F`. See [Design](design.md) for the derivation. ```python from ngraph import analyze, Mode, FlowPlacement @@ -235,8 +229,9 @@ print(summary.cost_distribution) # Dict[float, float] mapping cost to flow volu - `analyze(network, *, source=None, sink=None, mode=Mode.COMBINE)` - Create analysis context - `ctx.max_flow(source, sink, *, mode, shortest_path, require_capacity, flow_placement, excluded_nodes, excluded_links)` - Maximum flow -- `ctx.max_flow_detailed(..., include_min_cut=False)` - Maximum flow with cost distribution and optional min-cut +- `ctx.max_flow_detailed(..., include_min_cut=False)` - Maximum flow with cost distribution and optional min-cut; the min-cut is a true minimum cut (its capacity equals the max flow), not the set of saturated edges - `ctx.sensitivity(...)` - Identify critical edges and their impact on flow +- `ctx.sensitivity_with_flow(...)` - Compute max flow and edge sensitivity together per group pair in a single pass (used by the sensitivity Monte Carlo hot path) - `ctx.shortest_path_cost(source, sink, *, mode, edge_select=ALL_MIN_COST, excluded_nodes, excluded_links)` - Shortest path cost - `ctx.shortest_paths(source, sink, *, mode, edge_select, split_parallel_edges)` - Full Path objects @@ -248,12 +243,12 @@ print(summary.cost_distribution) # Dict[float, float] mapping cost to flow volu - **FlowPlacement.EQUAL_BALANCED (ECMP):** Equal split across parallel paths - **shortest_path=True:** Restricts flow to lowest-cost paths only (IP/IGP routing semantics) - **shortest_path=False:** Uses all paths progressively (TE/SDN semantics) -- **require_capacity=True:** Flow cannot exceed link capacity (default) -- **require_capacity=False:** Unconstrained flow for capacity-free analysis +- **require_capacity=True:** Path selection considers available capacity; flow moves to next-cheapest paths as cheaper ones saturate (default) +- **require_capacity=False:** Path selection is cost-only; saturated paths are not bypassed (true IP/IGP semantics; pair with shortest_path=True for IP simulation) ### Efficient Repeated Analysis (Bound Context) -For efficient repeated analysis with the same source/sink groups: +Bind source and sink groups once, then reuse the context across many calls: ```python from ngraph import analyze, Mode @@ -273,9 +268,11 @@ for failed_links in failure_scenarios: **Benefits of Bound Context:** - Graph infrastructure built once at context creation -- Each analysis call only builds O(|excluded|) masks +- Each analysis call rebuilds only the node and edge masks — a full-length array fill (Theta(V) for nodes, Theta(E) for edges) plus O(|excluded| + |disabled|) updates — instead of rebuilding the Core graph - Thread-safe: can run concurrent analysis calls with different exclusions +**Unbound vs. bound construction:** Unbound flow calls (`max_flow`, `max_flow_detailed`, `sensitivity`) construct a full temporary bound context per call, so repeated analysis should use a bound context. A plain unbound context builds its Core graph lazily on first use; bound contexts (and contexts with augmentations) build eagerly at creation. + ### Shortest Paths ```python @@ -346,12 +343,15 @@ from ngraph import Network, Node, Link, FailureManager from ngraph.model.failure.policy import FailurePolicy, FailureMode, FailureRule from ngraph.model.failure.policy_set import FailurePolicySet -# Build a simple network +# Build a network with two disjoint A->C paths, so a single link failure +# degrades capacity rather than disconnecting the pair entirely. network = Network() -for name in ["A", "B", "C"]: +for name in ["A", "B", "C", "D"]: network.add_node(Node(name=name)) network.add_link(Link("A", "B", capacity=100.0)) network.add_link(Link("B", "C", capacity=100.0)) +network.add_link(Link("A", "D", capacity=60.0)) +network.add_link(Link("D", "C", capacity=60.0)) # Define failure policy: randomly choose 1 link to fail rule = FailureRule(scope="link", mode="choice", count=1) # scope can be "node", "link", or "risk_group" @@ -376,9 +376,10 @@ results = fm.run_max_flow_monte_carlo( seed=42 # For reproducibility ) -# Access results +# Access results: one entry per unique failure pattern (iterations are +# deduplicated; occurrence_count gives how many iterations matched) for iter_result in results["results"]: - print(f"Flow: {iter_result.summary.total_placed:.1f}") + print(f"Flow: {iter_result.summary.total_placed:.1f} (x{iter_result.occurrence_count})") ``` **Key Methods:** @@ -418,7 +419,6 @@ workflow: failure_policy: "dual_link_failures" iterations: 100 parallelism: auto - placement_rounds: auto ``` ### MaximumSupportedDemand Step @@ -481,7 +481,7 @@ from ngraph import MaxFlowResult, FlowEntry, FlowSummary, FlowIterationResult # MaxFlowResult - Detailed max-flow result result.total_flow # Total flow placed result.cost_distribution # Dict[cost, flow_volume] -result.min_cut # Optional tuple of EdgeRef (saturated edges) +result.min_cut # Optional tuple of EdgeRef (edges forming a minimum cut) # FlowEntry - Single flow entry entry.source # Source label diff --git a/docs/reference/cli.md b/docs/reference/cli.md index a58993c..6d6ddc2 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -8,11 +8,11 @@ Quick links: - [API Reference](api.md) — Python API for programmatic scenario creation - [Auto-Generated API Reference](api-full.md) — complete class and method documentation -NetGraph provides a command-line interface for inspecting, running, and analyzing scenarios from the terminal. +The `ngraph` command inspects, runs, and analyzes scenarios from the terminal. ## Basic Usage -The CLI provides two primary commands: +Two commands: - `inspect`: Analyze and validate scenario files without running them - `run`: Execute scenario files and generate results @@ -52,20 +52,20 @@ ngraph [--verbose|--quiet] inspect [options] **Options:** - `--detail`, `-d`: Show detailed information including complete node/link tables and step parameters -- `--output`, `-o`: Output directory for generated artifacts (e.g., profiles) +- `--output`, `-o`: Output directory for generated artifacts (accepted for CLI consistency; `inspect` itself writes no files) **What it does:** -The `inspect` command loads and validates a scenario file, then provides information about: +Loads and validates the scenario file, then reports: -- **Scenario metadata**: Seed configuration and deterministic behavior -- **Network structure**: Node/link counts, enabled/disabled breakdown, hierarchy analysis -- **Capacity statistics**: Link and node capacity analysis with min/max/mean/total values -- **Risk groups**: Network resilience groupings and their status -- **Components library**: Available components for network modeling -- **Failure policies**: Configured failure scenarios and their rules -- **Traffic matrices**: Demand patterns and traffic flows -- **Workflow steps**: Analysis pipeline and step-by-step execution plan +- **Scenario metadata**: seed, and whether the run is reproducible +- **Network structure**: node/link counts, enabled vs. disabled, hierarchy +- **Capacity statistics**: link and node capacity min/max/mean/median/total +- **Risk groups**: defined groups, each enabled or disabled +- **Components library**: components available to the scenario +- **Failure policies**: each policy's mode count (modes and rules in detail mode) +- **Demand sets**: demand patterns and volumes, capacity-vs-demand summary +- **Workflow steps**: the steps that would run, in order In detail mode (`--detail`), shows complete tables for all nodes and links with capacity and connectivity information. @@ -105,9 +105,9 @@ ngraph [--verbose|--quiet] run [options] **Options:** -- `--results`, `-r`: Path to export results as JSON (default: `.results.json`) +- `--results`, `-r`: Path to export results as JSON (default: `.results.json`; relative paths are placed under `--output` when provided) - `--no-results`: Disable results file generation -- `--stdout`: Print results to stdout in addition to saving file +- `--stdout`: Print results to stdout in addition to saving file. Log output, status banners, the `--profile` performance report, and run error messages all go to stderr, so stdout contains only the JSON results (safe to pipe to `jq`) - `--keys`, `-k`: Space-separated list of workflow step names to include in output - `--profile`: Enable performance profiling with CPU analysis and bottleneck detection - `--profile-memory`: Also track peak memory per step @@ -118,39 +118,22 @@ ngraph [--verbose|--quiet] run [options] ### Basic Execution ```bash -# Run a scenario (creates square_mesh.results.json by default) +# Default output file (square_mesh.results.json) ngraph run scenarios/square_mesh.yaml -# Run a scenario and save results to custom file -ngraph run scenarios/backbone_clos.yml --results clos_analysis.json - -# Run a scenario without creating any files -ngraph run scenarios/nsfnet.yaml --no-results -``` - -### Save Results to File - -```bash -# Save results to a custom JSON file +# Custom results path ngraph run scenarios/backbone_clos.yml --results analysis.json -# Save to file AND print to stdout +# Save and also print JSON to stdout ngraph run scenarios/backbone_clos.yml --results analysis.json --stdout -# Use default filename and also print to stdout -ngraph run scenarios/square_mesh.yaml --stdout -``` - -### Running Test Scenarios - -```bash -# Run one of the provided scenarios with results export -ngraph run scenarios/backbone_clos.yml --results results.json +# Run without writing any files +ngraph run scenarios/nsfnet.yaml --no-results ``` ### Filtering Results by Step Names -You can filter the output to include only specific workflow steps using the `--keys` option: +`--keys` restricts the `steps` section to the named workflow steps; the `workflow` metadata section still lists every step that ran: ```bash # Only include results from the MSD step @@ -214,14 +197,14 @@ The CLI outputs results as JSON with a fixed top-level shape: "steps": { "network_statistics": { "metadata": {}, "data": { "node_count": 42, "link_count": 84 } }, "msd_baseline": { "metadata": {}, "data": { "alpha_star": 1.23, "context": { "demand_set": "baseline_traffic_matrix" } } }, - "tm_placement": { "metadata": { "iterations": 1000 }, "data": { "flow_results": [ { "flows": [], "summary": {} } ], "context": { "demand_set": "baseline_traffic_matrix" } } } + "tm_placement": { "metadata": { "iterations": 1000 }, "data": { "baseline": { "flows": [], "summary": {} }, "flow_results": [ { "flows": [], "summary": {} } ], "context": { "demand_set": "baseline_traffic_matrix" } } } }, "scenario": { "seed": 42, "failures": { }, "demands": { } } } ``` - **BuildGraph**: stores `data.graph` in node-link JSON format -- **MaxFlow** and **TrafficMatrixPlacement**: store `data.flow_results` as lists of per-iteration results (flows + summary) +- **MaxFlow** and **TrafficMatrixPlacement**: store the no-failure reference under `data.baseline` and unique failure patterns (deduplicated, each with `occurrence_count`, flows + summary) under `data.flow_results` - **NetworkStats**: stores capacity and degree statistics under `data` ## Output Behavior @@ -246,7 +229,7 @@ ngraph run scenarios/square_mesh.yaml ngraph run scenarios/square_mesh.yaml --results my_analysis.json ``` -- Creates specified JSON file instead of results.json +- Creates specified JSON file instead of the default `.results.json` - Useful for organizing multiple analysis runs ### Print to Terminal @@ -255,7 +238,7 @@ ngraph run scenarios/square_mesh.yaml --results my_analysis.json ngraph run scenarios/square_mesh.yaml --stdout ``` -- Creates results.json AND prints JSON to stdout +- Creates `.results.json` AND prints JSON to stdout - Useful for viewing results immediately while also saving them ### Combined Output @@ -265,7 +248,6 @@ ngraph run scenarios/square_mesh.yaml --results analysis.json --stdout ``` - Creates custom JSON file AND prints to stdout -- Provides flexibility for different workflows ### Disable File Generation (Edge Cases) @@ -279,7 +261,7 @@ ngraph run scenarios/square_mesh.yaml --no-results ## Integration with Workflows -The CLI executes the complete workflow defined in your scenario file, running all steps in sequence and accumulating results. This runs complex network analysis tasks without manual intervention. +`ngraph run` executes every step of the workflow defined in the scenario file, in sequence, accumulating results as it goes. ### Recommended Workflow @@ -309,7 +291,7 @@ ngraph --verbose inspect scenarios/backbone_clos.yml --detail ngraph inspect scenarios/backbone_clos.yml --detail | grep -A 5 "WORKFLOW STEPS" ``` -The `inspect` command will catch common issues like: +`inspect` catches common issues: - Invalid YAML syntax - Missing blueprint references diff --git a/docs/reference/design.md b/docs/reference/design.md index 8f26589..ba3b5ef 100644 --- a/docs/reference/design.md +++ b/docs/reference/design.md @@ -1,6 +1,6 @@ # NetGraph Design and Implementation -This document describes NetGraph's internal design: scenario DSL, data models, execution flow, algorithms, manager components, and result handling. It focuses on architecture and key implementation details. +NetGraph's internal design: scenario DSL, data models, execution flow, algorithms, manager components, and result handling. ## Overview @@ -32,7 +32,7 @@ NetGraph is a network scenario analysis engine using a **hybrid Python+C++ archi ```text ngraph/ ├── analysis/ # AnalysisContext, FailureManager, placement -├── model/ # Network, Node, Link, demand/, failure/, flow/ +├── model/ # Network, Node, Link, demand/, failure/, flow/, selectors/ ├── dsl/ # YAML parsing (blueprints/, selectors/, expansion/) ├── workflow/ # WorkflowStep implementations ├── results/ # Results store and flow result types @@ -46,16 +46,33 @@ ngraph/ └── cli.py # Command-line interface ``` +### Package Layering + +Packages import strictly downward in this order; a lower layer never imports a higher one: + +```text +types, utils -> model -> dsl -> analysis -> workflow -> scenario -> cli +``` + +Two deliberate exceptions: + +- `model` may use the dependency-free string-expansion helpers in `ngraph.dsl.expansion` (bracket patterns in risk-group references and demand `expand:` blocks). No other `model -> dsl` import is allowed; selector schema types and evaluation live in `ngraph.model.selectors`, and `ngraph.dsl.selectors` re-exports them for backward compatibility. Enforced by `tests/model/test_layering.py`. +- `workflow` references `Scenario` only under `TYPE_CHECKING` (workflow steps execute against a `Scenario`); the runtime import goes downward, from `scenario` to `workflow`. + +The package root also does not eagerly import `ngraph.cli` (enforced by `tests/cli/test_package_layering.py`); the console entry point and `python -m ngraph` import it explicitly. + +Deferred (function-local) imports are used for optional dependencies (networkx, jsonschema), opt-in profiling, and a few narrow internal cases (`ngraph.model.failure.parser` defers `ngraph.dsl.expansion`; `FailureManager._process_sensitivity_results` defers `ngraph.results.flow`) — not as a general layering workaround; outside those cases, if a module needs a lower layer, it imports it at module level. + ### Integration Points The Python layer uses the `analyze()` function and `AnalysisContext` class (`ngraph.analysis`) to: 1. Build Core graphs from Network instances with optional pseudo-nodes for source/sink groups 2. Map node names (str) to NodeId (int32) and link IDs (str) to EdgeId/ext_edge_id (int64) -3. Execute analysis methods (max_flow, shortest_paths, sensitivity) with efficient masking +3. Execute analysis methods (max_flow, shortest_paths, sensitivity) with boolean masking 4. Translate results (costs, flows, paths) back to scenario-level objects -Core algorithms release the GIL during execution, enabling concurrent Python threads to execute analysis in parallel with minimal Python-level overhead. +Core algorithms release the GIL during execution, so concurrent Python threads run analysis in parallel with minimal Python-level overhead. **Primary API:** @@ -71,17 +88,17 @@ baseline = ctx.max_flow() degraded = ctx.max_flow(excluded_links=failed_links) ``` -The `AnalysisContext` encapsulates all graph building and provides properties for advanced use by workflow steps and the FailureManager. +`AnalysisContext` encapsulates all graph building and exposes the underlying graph components as properties for workflow steps and the FailureManager. ### Execution Flow -The diagram below shows the architecture and end-to-end execution flow from scenario input through both Python and C++ layers to final results. The Python layer handles scenario loading, workflow orchestration, and result aggregation, while compute-intensive graph algorithms execute in C++ with the GIL released for parallel execution. +The diagram below traces a scenario from input through both layers to final results: the Python layer loads the scenario, orchestrates the workflow, and aggregates results, while compute-intensive graph algorithms execute in C++ with the GIL released. ![NetGraph execution flow](../assets/diagrams/system_pipeline.dot.svg) ## Scenario DSL and Input Expansion -NetGraph scenarios are defined in YAML using a declarative DSL (see [DSL Reference](dsl.md)). The DSL allows concise specification of network topologies, traffic demands, failure policies, and analysis workflows. Before execution, scenario files are validated against a JSON Schema to catch errors early (unknown keys, type mismatches), enforcing strict definitions. +NetGraph scenarios are defined in YAML using a declarative DSL (see [DSL Reference](dsl.md)) covering network topologies, traffic demands, failure policies, and analysis workflows. Before execution, scenario files are validated against a JSON Schema, so unknown keys and type mismatches fail early. Key elements of the DSL include: @@ -91,7 +108,7 @@ Key elements of the DSL include: - **Node Groups**: Definitions of node groups in the topology, either explicitly or via patterns. Groups can use a blueprint (`blueprint`) with parameters (`params`), or define a number of nodes (`count`) with a naming template (`template`). -- **Links**: Rules to generate links between node groups. Instead of enumerating every link, a link rule specifies source and target selectors (by path pattern), a wiring pattern (e.g. mesh for full mesh or one_to_one for paired links), number of parallel links (`count`), and link properties (capacity, cost, attributes like distance, hardware, risk group tags, etc.). Link properties are specified at the top level, not inside a wrapper. Advanced matching allows filtering nodes by attributes with logical conditions (AND/OR) to apply link rules to selected nodes only. A single rule can thus expand into many concrete links. +- **Links**: Rules to generate links between node groups. Instead of enumerating every link, a link rule specifies source and target selectors (by path pattern), a wiring pattern (e.g. mesh for full mesh or one_to_one for paired links), number of parallel links (`count`), and link properties (capacity, cost, attributes like distance, hardware, risk group tags, etc.). Link properties are specified at the top level, not inside a wrapper. Matching can also filter nodes by attributes with logical conditions (AND/OR) so a rule applies to selected nodes only. A single rule can thus expand into many concrete links. - **Rules**: Optional modifications applied after the initial expansion. `node_rules` or `link_rules` can match specific nodes or links (by path or endpoints) and change their attributes or disable them. This allows fine-tuning or simulating removals without changing the base definitions. @@ -105,11 +122,11 @@ Key elements of the DSL include: ### DSL Expansion Process -The loader validates and expands DSL definitions into concrete nodes and links. Unknown fields or schema violations cause an immediate error before any expansion. After schema validation, blueprints are resolved (each blueprint group becomes actual Node objects), group name patterns are expanded into individual names, and adjacency rules are iterated over matching source-target node sets to create Link objects. All nodes and links are then validated in runtime to ensure they are valid (e.g., no duplicate node names, all link endpoints exist). +The loader validates and expands DSL definitions into concrete nodes and links. Unknown fields or schema violations cause an immediate error before any expansion. After schema validation, blueprints are resolved (each blueprint group becomes actual Node objects), group name patterns are expanded into individual names, and adjacency rules are iterated over matching source-target node sets to create Link objects. The resulting nodes and links are then checked at runtime for duplicate node names and missing link endpoints. ## Data Model -Once the scenario is parsed and expanded, NetGraph represents the network with a set of core model classes. These define the in-memory representation of the scenario topology and enforce structural invariants (unique node names, valid link endpoints). +Once the scenario is parsed and expanded, NetGraph holds it in a set of core model classes. They are the in-memory representation of the scenario topology and enforce its structural invariants: unique node names, valid link endpoints. ### Node @@ -139,9 +156,9 @@ A Link represents a directed link between a source and target node. Each link ha - attrs dict for metadata (e.g. distance_km, fiber type), and -- an auto-generated unique id +- a unique id assigned when the link is added to a Network -The id is constructed as "source|target|", ensuring each link has a distinct identifier. The model stores each link as directed (source -> target). When the analysis graph is built, a reverse edge is added by default so algorithms see bidirectional connectivity. +The id is deterministic: `Network.add_link` assigns "source|target|", where is a per-(source, target) insertion sequence number, so ids and their sort order are stable across identical scenario builds (a provisional uuid-suffixed id exists only on links never added to a Network). The model stores each link as directed (source -> target). When the analysis graph is built, a reverse edge is added by default so algorithms see bidirectional connectivity. ### RiskGroup @@ -155,7 +172,7 @@ A RiskGroup represents a named failure domain or shared-risk link group (SRLG). - an attrs dict for any metadata -Hierarchical risk groups allow, for example, defining a large domain composed of smaller sub-domains. A failure event could disable an entire group, implicitly affecting all its descendants. +Hierarchy lets a large domain be composed of smaller sub-domains: a failure event that disables a group implicitly affects all its descendants. ### Network @@ -167,11 +184,11 @@ A Network is the container class that holds all nodes, links, and top-level risk - risk_groups: Dict[name, RiskGroup], -Network is the container for scenario topology. It enforces invariants during construction: adding a link validates that source and target nodes exist; adding a node rejects duplicates by name. Components are never removed from the Network; the `disabled` flag marks them inactive. The Network also maintains a selection cache for `select_node_groups_by_path` to avoid repeated regex/attribute queries. +Network enforces invariants during construction: adding a link validates that source and target nodes exist; adding a node rejects duplicates by name. Components are never removed from the Network; the `disabled` flag marks them inactive. The Network also maintains a selection cache for `select_node_groups_by_path` to avoid repeated regex queries; cached results are copied on return (fresh dict and lists, shared Node objects), and the cache is invalidated when nodes are added. ### Node and Link Selection -The model supports selecting groups of nodes via a unified selector system used by algorithms to choose source/sink sets matching on structured names or attributes. +A single selector system picks groups of nodes by structured name or by attribute; algorithms use it to choose source/sink sets. Selector evaluation (schema types, condition evaluation, node selection, attribute flattening) lives in `ngraph.model.selectors`; `ngraph.dsl.selectors` provides YAML-facing parsing and re-exports the evaluation names for backward compatibility. **Selector Forms:** @@ -182,7 +199,7 @@ Selectors can be specified as: **String Pattern Behavior:** -When using a regex pattern, if the regex contains capturing groups, the concatenated capture groups form the group label; otherwise, the entire pattern string is used as the label. For instance, the pattern `r"(\w+)-(\d+)"` on node names could produce group labels like "metroA-1" etc. +When using a regex pattern, if the regex contains capturing groups, the non-None captures joined with "|" form the group label; otherwise, the entire pattern string is used as the label. For instance, the pattern `r"(\w+)-(\d+)"` on node name "metroA-1" produces the group label "metroA|1". **Attribute-based Grouping:** @@ -209,11 +226,11 @@ source: value: "leaf" ``` -This selection mechanism allows workflow steps and API calls to refer to nodes flexibly (using human-readable patterns instead of explicit lists), which is particularly useful in large topologies. +Workflow steps and API calls therefore refer to nodes by readable pattern instead of by explicit list, which matters most in large topologies. ### Disabled Elements -Nodes or links marked as disabled=True represent elements present in the design but out of service for the analysis. The base model keeps them in the collection but analysis functions filter them out when selecting active nodes. This design preserves topology information (e.g., you know a link exists but is just turned off) and allows easily enabling it later if needed. +Nodes or links marked as disabled=True represent elements present in the design but out of service for the analysis. The base model keeps them in the collection but analysis functions filter them out when selecting active nodes. This preserves topology information — the link still exists, it is just turned off — and re-enabling it is a flag change. ### Filtered Analysis (Exclusions) @@ -231,9 +248,9 @@ results = analyze(network).max_flow( This approach avoids mutating the base graph when simulating failures (e.g., deleting nodes or toggling flags). It separates the static scenario (base network) from dynamic conditions (exclusions), enabling thread-safe parallel analyses and eliminating deep copies for each failure scenario. -**Implementation:** For repeated analysis (Monte Carlo, FailureManager), exclusions are applied via boolean masks passed to Core algorithms. The graph is built once without exclusions, and masks disable specific elements at algorithm execution time. This enables O(|excluded|) mask updates rather than O(V+E) graph rebuilding. For one-off solver calls, exclusions may be applied during graph construction for simplicity. +**Implementation:** Exclusions are applied via boolean masks passed to Core algorithms. The graph is built once without exclusions, and masks disable specific elements at algorithm execution time. For repeated analysis (Monte Carlo, FailureManager) this enables O(|excluded|) mask updates rather than O(V+E) graph rebuilding. One-off calls on an unbound context build a temporary bound context per call and apply exclusions the same way. -Multiple concurrent analyses can run on the same base network with different exclusion sets. This is important for performing parallel simulations (e.g., analyzing many failure combinations in a Monte Carlo) efficiently. +Multiple concurrent analyses can run on the same base network with different exclusion sets, which is what makes parallel Monte Carlo over many failure combinations practical. ### Graph Construction @@ -241,9 +258,9 @@ NetGraph builds graphs through `AnalysisContext` which translates from the Pytho **Python Side (`ngraph.analysis.AnalysisContext`):** -A single construction method is provided: +There is one construction method: -- `AnalysisContext.from_network()`: Constructs an immutable context with pre-built Core graph, mappers, algorithms instance, and pre-computed disabled topology. Exclusions are applied at algorithm call time via boolean masks rather than during graph construction. +- `AnalysisContext.from_network()`: Constructs an immutable context with Core graph, mappers, algorithms instance, and pre-computed disabled topology. Bound contexts and contexts with custom augmentations build the Core graph eagerly (pseudo node IDs must be resolved at bind time); plain unbound contexts defer the Core build until first use. Exclusions are applied at algorithm call time via boolean masks rather than during graph construction. **Graph Construction Steps:** @@ -251,6 +268,7 @@ A single construction method is provided: - Assigns stable node IDs (sorted by name for determinism) - Encodes link_id + direction as ext_edge_id (packed int64) - Constructs NumPy arrays (src, dst, capacity, cost, ext_edge_ids) +- Validates inputs: link capacities must be below the internal pseudo-edge capacity `LARGE_CAPACITY` (1e15), and costs must be non-negative integers whose total across all edges stays below 2^62. Core SPF accumulates path costs in int64 with INT64_MAX as the unreachable sentinel, so accumulated path costs — not just per-edge values — must stay in range; bounding the total of all edge costs bounds every path. Violations raise `ValueError` instead of silently corrupting results - Supports augmentation edges (e.g., pseudo-source/sink for multi-source max-flow) **AnalysisContext Internals:** @@ -264,7 +282,7 @@ A single construction method is provided: - `_link_id_to_edge_indices`: Pre-computed mapping for O(|excluded|) mask building - `_pseudo_context`: Optional context for pseudo source/sink node mappings -When analyzing many failure scenarios, the graph is built once via `AnalysisContext.from_network()` and exclusions are applied via boolean masks. The mask builders automatically include disabled nodes/links, ensuring disabled topology is always excluded. This avoids rebuilding the graph for each iteration, providing significant speedup for Monte Carlo simulations. +When analyzing many failure scenarios, the graph is built once via `AnalysisContext.from_network()` and exclusions are applied via boolean masks. The public mask builders (`build_node_mask`/`build_edge_mask`, also usable by custom analysis functions that call Core primitives directly) automatically include disabled nodes/links, ensuring disabled topology is always excluded. Nothing is rebuilt per iteration, which is where the Monte Carlo speedup comes from. **Disabled Topology Handling:** @@ -277,7 +295,7 @@ Disabled nodes and links from the Network are pre-computed during `AnalysisConte - Each edge stores capacity (float64), cost (int64), and ext_edge_id (int64) - Edges sorted by (cost, src, dst) for deterministic algorithm behavior - Zero-copy NumPy views for array access (capacities, costs, ext_edge_ids) -- Efficient neighbor iteration via CSR structure +- Neighbor iteration walks a contiguous CSR range **Edge Direction Handling:** @@ -297,17 +315,15 @@ are not mapped back to scenario links in results. ### Analysis Algorithms -NetGraph's core algorithms execute in C++ via NetGraph-Core. Algorithms operate on the immutable StrictMultiDiGraph and support masking (runtime exclusions via boolean arrays) for efficient repeated analysis under different failure scenarios without graph reconstruction. +NetGraph's core algorithms execute in C++ via NetGraph-Core. They operate on the immutable StrictMultiDiGraph and support masking (runtime exclusions via boolean arrays), so repeated analysis under different failure scenarios needs no graph reconstruction. -All Core algorithms release the Python GIL during execution, enabling concurrent execution across multiple Python threads without GIL contention. +All Core algorithms release the Python GIL during execution, so multiple Python threads run them concurrently without GIL contention. ### Shortest-Path First (SPF) Algorithm Implemented in C++ (`netgraph::core::shortest_paths`), using Dijkstra's algorithm with configurable edge selection and optional multipath predecessor recording. -**Core Features:** - **Edge Selection Policies:** The algorithm evaluates parallel edges per neighbor using `EdgeSelection` configuration: @@ -316,8 +332,8 @@ The algorithm evaluates parallel edges per neighbor using `EdgeSelection` config - `multi_edge=false`: Select single edge per (u,v) pair using tie-breaking: - `PreferHigherResidual`: Choose edge with highest residual capacity (secondary: lowest edge ID) - `Deterministic`: Choose edge with lowest edge ID for reproducibility -- `require_capacity=true`: Only consider edges with residual capacity > kMinCap (used in max-flow) -- `require_capacity=false` (default): Consider all edges regardless of residual capacity +- `require_capacity=true`: Only consider edges with residual capacity ≥ kMinCap (used in max-flow) +- `require_capacity=false` (default): Consider all edges regardless of residual capacity (supplying a residual view to SPF implicitly enables the same capacity filter) **Capacity-Aware Tie-Breaking:** @@ -343,7 +359,7 @@ This optimization reduces work when only source-to-sink distances are needed. **Masking:** Optional `node_mask` and `edge_mask` boolean arrays enable runtime exclusions without -rebuilding the graph. Used by FailureManager for efficient Monte Carlo analysis. +rebuilding the graph. Used by FailureManager for Monte Carlo analysis. **Complexity:** @@ -410,8 +426,10 @@ function SPF(graph, src, dst=None, multipath=True, edge_selection): max_edge_res = max(residual[e] for e in selected_edges) path_residual = min(min_residual_to_node[u], max_edge_res) - # Relaxation: found shorter path - if new_cost < costs[v]: + # Relaxation: found shorter path, or (single-path mode) an + # equal-cost path with higher bottleneck capacity + if new_cost < costs[v] or (not multipath and new_cost == costs[v] + and path_residual > min_residual_to_node[v] + epsilon): costs[v] = new_cost min_residual_to_node[v] = path_residual pred[v] = { u: selected_edges } @@ -473,31 +491,21 @@ See "Routing Semantics: IP/IGP vs SDN/TE" section for detailed explanation. The residual network is maintained via `FlowState`, which tracks per-edge flow and computes residual capacities on demand. For each edge u→v: - Forward residual capacity: `capacity(u,v) - flow(u,v)` -- Reverse residual capacity (for flow cancellation): `flow(u,v)` +- Reverse residual capacity: `flow(u,v)` (used for residual reachability when computing the min-cut and reachable set) SPF operates over the residual graph by requesting edges with `require_capacity=true`, which filters to edges with positive residual capacity. The `FlowState` provides a residual capacity view without graph mutation. -Note: Reverse residual arcs for flow cancellation are distinct from physical reverse edges added via `add_reverse=True` during graph construction. Physical reverse edges model bidirectional links with independent capacity; residual reverse arcs enable flow augmentation/cancellation. +Note: Reverse residual arcs are distinct from physical reverse edges added via `add_reverse=True` during graph construction. Physical reverse edges model bidirectional links with independent capacity; reverse residual arcs are bookkeeping over a single edge's flow. The augmenting SPF search traverses forward residual edges only — placed flow is never cancelled across tiers; reverse residual arcs are traversed only for reachability when computing the min-cut and reachable set, while Dinic-style reverse edges allow redistribution within a single tier's placement. The core loop finds augmenting paths using the cost-aware SPF described above: -Run SPF from source to sink with `multi_edge=true` and `require_capacity=true` (filters to edges with positive residual capacity). This computes shortest-path distances and a predecessor DAG over forward residual edges. The edge cost can represent distance, latency, or preference; SPF selects paths minimizing cumulative cost. - -If the pseudo-sink is not reached (i.e., no augmenting path exists), stop: the max flow is achieved. - -Otherwise, determine how much flow can be sent along the found paths: - -Using the predecessor DAG from SPF, `FlowState.place_on_dag` computes blocking flow considering parallel edges and the splitting policy. For PROPORTIONAL: builds reversed residual graph, assigns BFS levels, uses DFS to push flow with capacity-proportional splits. For EQUAL_BALANCED: performs topological traversal with equal splits, computes global scale factor to prevent oversubscription. - -This yields flow amount `f` and per-edge flow assignments tracking which edges carry flow and their utilization. +Run SPF from source to sink with `multi_edge=true`, `tie_break=Deterministic`, and the configured `require_capacity` (when true, SPF receives the residual view and filters to edges with residual capacity ≥ kMinCap). This computes shortest-path distances and a predecessor DAG over forward residual edges. The edge cost can represent distance, latency, or preference; SPF selects paths minimizing cumulative cost. -The algorithm then augments the flow: `FlowState` increases each edge's flow by its assigned portion. Per-edge flows and residual capacities are updated for the next iteration. +If the sink is not reached (i.e., no augmenting path exists), stop: the max flow is achieved. -Add f to the total flow counter. +Otherwise, `FlowState.place_on_dag` computes a blocking flow over the predecessor DAG from SPF, considering parallel edges and the splitting policy. For PROPORTIONAL: builds reversed residual graph, assigns BFS levels, uses DFS to push flow with capacity-proportional splits. For EQUAL_BALANCED: performs topological traversal with equal splits, computes global scale factor to prevent oversubscription. This yields flow amount `f` and per-edge flow assignments tracking which edges carry flow and their utilization. -If `f` is below tolerance `kMinFlow` (negligible flow placed due to numerical limits or exhausted capacity), terminate iteration. - -Repeat to find the next augmenting path (back to step 1). +`FlowState` then increases each edge's flow by its assigned portion, updating per-edge flows and residual capacities for the next iteration, and `f` is added to the total flow counter. If `f` is below tolerance `kMinFlow` (negligible flow placed due to numerical limits or exhausted capacity), iteration terminates; otherwise the loop repeats from the SPF step to find the next augmenting path. If `shortest_path=True`, the algorithm performs only one augmentation pass and returns (useful when the goal is a single cheapest augmentation rather than maximum flow). @@ -513,9 +521,9 @@ After the loop, the C++ algorithm computes a FlowSummary which includes: - min_cut: the list of edges that are saturated and go from reachable to non-reachable (these form the minimum cut) -- cost_distribution: flow volume placed at each path cost tier. Core returns parallel arrays (`costs`, `flows`); AnalysisContext converts these to `Dict[Cost, Flow]` mapping in `FlowSummary.cost_distribution`. +- cost_distribution: flow volume placed at each path cost tier. Core returns parallel arrays (`costs`, `flows`); AnalysisContext converts these to the `Dict[Cost, Flow]` mapping in `MaxFlowResult.cost_distribution`. -This is returned along with the total flow value. +The summary is returned along with the total flow value. ### Routing Semantics: IP/IGP vs SDN/TE @@ -563,11 +571,11 @@ Beyond routing semantics, NetGraph controls how flow splits across equal-cost pa - Single-pass admission: computes one global scale factor to avoid oversubscription - For IP ECMP simulation: use with `require_capacity=false` + `shortest_path=true` -`FlowState.place_on_dag` implements single-pass placement over a fixed SPF DAG: +`FlowState.place_on_dag` implements placement over a fixed SPF DAG (the DAG never changes within a call): - **PROPORTIONAL**: Constructs reversed residual graph from predecessor DAG. Uses Dinic-style BFS leveling and DFS push from sink to source. Within each edge group (parallel edges between node pair), splits flow proportionally to residual capacity. Distributes pushed flow back to underlying edges maintaining proportional ratios. Can be called iteratively on updated residuals. -- **EQUAL_BALANCED**: Performs topological traversal (Kahn's algorithm) from source to sink over forward DAG. Assigns equal splits across all outgoing parallel edges from each node. Computes global scale factor as `min(edge_capacity / edge_assignment)` across all edges to prevent oversubscription. Applies scale uniformly and stops. This models single-pass ECMP admission where the forwarding DAG doesn't change mid-flow. +- **EQUAL_BALANCED**: Performs topological traversal (Kahn's algorithm) from source to sink over forward DAG. Assigns equal splits across all outgoing parallel edges from each node. Computes global scale factor as `min(edge_residual / edge_assignment)` across all edges to prevent oversubscription. Applies scale uniformly and stops. This models single-pass ECMP admission where the forwarding DAG doesn't change mid-flow. **Configuration Examples:** @@ -591,11 +599,9 @@ analyze(network).max_flow(src, dst, require_capacity=False) # Fixed paths regardless of utilization ``` -These configurations enable realistic modeling of diverse forwarding behaviors: from traditional IP networks with best-effort delivery to modern SDN deployments with capacity-aware traffic engineering. - ### Flow Policy Presets -For traffic matrix placement, NetGraph provides `FlowPolicyPreset` values that bundle the routing semantics described above into convenient configurations. These presets map to real-world network behaviors: +For traffic matrix placement, `FlowPolicyPreset` values bundle the routing semantics above into named configurations that map to real-world network behaviors: | Preset | Behavior | Use Case | | -------- | ---------- | ---------- | @@ -645,10 +651,11 @@ demands: ### Pseudocode (simplified max-flow loop) ```text -function MAX_FLOW(graph, S, T, placement=PROPORTIONAL, require_capacity=True): +function MAX_FLOW(graph, S, T, placement=PROPORTIONAL, require_capacity=True, + shortest_path=False): flow_state = FlowState(graph) # Tracks per-edge flow and residuals total_flow = 0 - cost_distribution = [] + cost_distribution = {} # path_cost -> flow while True: # Configure edge selection for SPF @@ -676,7 +683,10 @@ function MAX_FLOW(graph, S, T, placement=PROPORTIONAL, require_capacity=True): break total_flow += placed - cost_distribution.append((path_cost, placed)) + cost_distribution[path_cost] += placed # merged by exact cost + + if shortest_path: # Single augmentation pass (IP/IGP mode) + break # Compute min-cut, reachability, cost distribution min_cut = flow_state.compute_min_cut(S, node_mask, edge_mask) @@ -691,38 +701,41 @@ function MAX_FLOW(graph, S, T, placement=PROPORTIONAL, require_capacity=True): ) ``` -The flow tolerance constant `kMinFlow` (default 1/4096 ≈ 2.4e-4) determines when flow placement is considered negligible and iteration terminates. +The flow tolerance constant `kMinFlow` (1/4096 ≈ 2.4e-4) determines when flow placement is considered negligible and iteration terminates. -Each augmentation phase performs one SPF \(O((V+E) \\log V)\) and one blocking-flow computation \(O(V+E)\) over the predecessor DAG. With blocking flow augmentation, the shortest path distance (in hops) increases with each phase, bounding the number of phases by \(O(V)\). This yields an overall complexity of \(O(V \\cdot (V+E) \\log V)\) = \(O(V^2 E \\log V)\) for sparse graphs where \(E = O(V)\). +Each augmentation phase performs one SPF \(O((V+E) \log V)\) and one placement pass over the tier's predecessor DAG. For EQUAL_BALANCED the placement is a single topological pass \(O(V+E)\); for PROPORTIONAL it is a complete Dinic max-flow over the tier DAG (repeated BFS level construction, level-restricted blocking-flow DFS, and a group rebuild from the updated residual), worst case \(O(V^2 E)\). Placed flow is never removed from an edge, so each phase permanently saturates at least one edge before the next SPF runs, bounding the number of phases by \(O(E)\); with PROPORTIONAL placement the tier's path cost also strictly increases between phases, so phases are further bounded by the number of distinct path-cost values. The resulting loose worst-case bound is \(O(E \cdot (V^2 E + (V+E) \log V))\). -Practical performance is significantly better than worst-case bounds due to early termination when residual capacity exhausts. For integer capacities, the bound becomes \(O(F \\cdot (V+E) \\log V)\) where \(F\) is the max-flow value, which dominates when \(F \ll V\). +Practical performance is significantly better than these worst-case bounds: iteration stops as soon as the residual network disconnects source from sink, the phase count in practice equals the small number of cost tiers actually used, and the `kMinFlow` threshold additionally caps the number of phases at \(F / k_{MinFlow}\) for total flow \(F\). ### Managers and Workflow Orchestration Managers handle scenario dynamics and prepare inputs for algorithmic steps. -**Demand Expansion** (`ngraph.model.demand.builder`): Builds demand sets from DSL definitions, expanding source/target patterns into concrete node groups. +**Demand Expansion** (`ngraph.analysis.demand`): Expands `TrafficDemand` specs (built from DSL definitions by `ngraph.model.demand.builder`) into concrete placement demands, resolving source/target selectors into node groups. -- Deterministic expansion: source/target node lists sorted alphabetically; no randomization -- Supports `combine` mode (aggregate via pseudo nodes) and `pairwise` mode (individual (src,dst) pairs with volume split) +- Deterministic expansion: node selection follows the network's stable node ordering; no randomization +- Supports `combine` mode (aggregate via pseudo source/sink nodes attached with large-capacity, zero-cost augmentation edges) and `pairwise` mode (individual (src,dst) pairs, self-pairs excluded, volume split evenly across pairs) +- `group_mode` controls grouping: `flatten` (default, merge all groups then apply mode), `per_group`, and `group_pairwise`; volume is split evenly across groups or group pairs +- In combine mode, nodes selected on both sides are excluded from the target set (prevents a zero-cost pseudo-node bypass); a demand or group whose target set empties is skipped +- Validation: duplicate demand ids and pseudo-endpoint collisions raise `ValueError` (either would silently merge distinct demands' attachment edges) - Demands sorted by ascending priority before placement (lower value = higher priority) - Placement uses SPF caching for simple policies (ECMP, WCMP, TE_WCMP_UNLIM), FlowPolicy for complex multi-flow policies - Non-mutating: operates on Core flow graphs with exclusions; Network remains unmodified **Failure Manager** (`ngraph.analysis.failure_manager`): Applies a `FailurePolicy` to compute exclusion sets and runs analyses with those exclusions. -- Parallel execution via `ThreadPoolExecutor` with zero-copy network sharing across worker threads -- Deterministic results when seed is provided (each iteration derives `seed + iteration_index`) -- Optional baseline execution (no failures) for comparing degraded vs. intact capacity -- Automatic parallelism adjustment: Forces serial execution when analysis function defined in `__main__` (notebook context) to avoid pickling failures +- Parallel execution via `ThreadPoolExecutor` with zero-copy network sharing across worker threads; nothing is pickled, so functions defined in `__main__` or notebooks run at full parallelism +- Deterministic results when seed is provided (each iteration derives `seed + iteration_index`); with `seed=None`, the failure policy's own seed is used as a fallback when present +- Baseline execution: a no-failure baseline is always run first as a separate reference for comparing degraded vs. intact capacity +- Deduplication: identical exclusion patterns execute once and are weighted by multiplicity (`occurrence_count` on results; `metadata["occurrence_counts"]` aligned with the results list). Stored failure traces describe each pattern's representative (first) iteration; this weighting assumes deterministic analysis functions (the built-ins are) - Thread-safe analysis: Network shared by reference; exclusion sets passed per-iteration -- Automatic graph pre-building: Before parallel iterations, builds `AnalysisContext` to amortize graph construction cost; per-iteration exclusions applied via O(|excluded|) mask operations +- Automatic graph pre-building: Before parallel iterations, the engine calls the analysis function's `prepare_inputs(network, kwargs)` hook (carried by all built-in analysis functions) once per run and merges the returned kwargs — typically a pre-built `AnalysisContext`, plus the precomputed demand expansion and resolved IDs for demand placement — into every iteration's call; per-iteration exclusions are applied via O(|excluded|) mask operations. Custom analysis functions opt in by setting a `prepare_inputs` attribute; functions without it run with their kwargs unchanged, and passing `context` explicitly skips the hook. Both the demand expansion logic and failure manager separate policy (how to expand demands or pick failures) from core algorithms. They prepare concrete inputs (expanded demands or exclusion sets) for each workflow iteration. ### Workflow Engine and Steps -NetGraph workflows (see Workflow Reference) are essentially recipes of analysis steps to run in sequence. Each step is typically a pure function: it takes the current model and possibly prior results, performs an analysis, and stores its outputs. The workflow engine coordinates these steps, using a Results store to record data. +A NetGraph workflow (see Workflow Reference) is an ordered recipe of analysis steps. Each step is a pure function: it takes the current model and possibly prior results, performs an analysis, and stores its outputs. The workflow engine runs the steps in sequence and records their data in a Results store. Common built-in steps: @@ -730,15 +743,15 @@ Common built-in steps: - NetworkStats: computes node/link counts, capacity statistics, cost statistics, and degree statistics. Supports optional `excluded_nodes`/`excluded_links` and `include_disabled`. -- TrafficMatrixPlacement: runs Monte Carlo placement using a named demand set and the Failure Manager. Supports `baseline`, `iterations`, `parallelism`, `placement_rounds`, `store_failure_patterns`, `include_flow_details`, `include_used_edges`, and `alpha` or `alpha_from_step` (default `data.alpha_star`). Produces `data.flow_results` per iteration. +- TrafficMatrixPlacement: runs Monte Carlo placement using a named demand set and the Failure Manager; a no-failure baseline always runs first. Supports `iterations`, `parallelism`, `store_failure_patterns`, `include_flow_details`, `include_used_edges`, and `alpha` or `alpha_from_step` (default field `data.alpha_star`). Produces `data.baseline` and `data.flow_results` (unique failure patterns with `occurrence_count`). (`placement_rounds` is deprecated and accepted only as a no-op for backward compatibility.) -- MaxFlow: runs Monte Carlo maximum-flow analysis between node groups using the Failure Manager. Supports `mode` (combine/pairwise), `baseline`, `iterations`, `parallelism`, `shortest_path`, `flow_placement`, and optional `include_flow_details`/`include_min_cut`. Produces `data.flow_results` per iteration. +- MaxFlow: runs Monte Carlo maximum-flow analysis between node groups using the Failure Manager; a no-failure baseline always runs first. Supports `mode` (combine/pairwise), `iterations`, `parallelism`, `shortest_path`, `require_capacity`, `flow_placement`, and optional `include_flow_details`/`include_min_cut`. Produces `data.baseline` and `data.flow_results` (unique failure patterns with `occurrence_count`). - MaximumSupportedDemand (MSD): uses bracketing and bisection on alpha to find the maximum multiplier such that alpha * volume is feasible. Stores `data.alpha_star`, `data.context`, `data.base_demands`, and `data.probes`. -- CostPower: aggregates platform and per-end optics capex/power by hierarchy level (0..N). Respects `include_disabled` and `aggregation_level`. Stores `data.levels` and `data.context`. +- CostPower: aggregates platform and per-end optics capex/power by hierarchy level (0..N). Respects `include_disabled` and `aggregation_level`. Stores `data.levels` and `data.context`. Performs no hardware capacity/ports validation and completes even on networks the explorer's strict validation would reject; hardware validation is available via `NetworkExplorer` (strict validation) or the `ngraph inspect` command. -Each step is implemented in the code (in ngraph.workflow module) and has a corresponding `type` name. Steps are pure functions that don't modify the Network. They take inputs, often including references to prior steps' results (the workflow engine allows one step to use another step's output). For instance, a placement step might need the value of alpha* from an MSD step; the workflow definition can specify that link. +Each step is implemented in the `ngraph.workflow` module and has a corresponding `type` name. Steps do not modify the Network. Their inputs often include references to prior steps' results: a placement step might need the value of alpha* from an MSD step, and the workflow definition names that link. ### Results storage @@ -756,7 +769,7 @@ NetGraph's design includes several features that differentiate it from tradition - Runtime Exclusions vs graph copying: Analysis-time exclusions avoid copying large structures for each scenario. The design separates static topology from dynamic failure states. -- Stable edge IDs: Links have auto-generated unique IDs (`source|target|`) that remain stable throughout analysis, simplifying correlation of results to original links. +- Deterministic link IDs: `Network.add_link` assigns each link a unique ID (`source|target|`, a per-endpoint-pair insertion sequence) that is stable across identical scenario builds and throughout analysis, simplifying correlation of results to original links and keeping seeded failure sampling reproducible. - Dual routing semantics: Models both IP/IGP (cost-only, fixed paths via `require_capacity=false` + `shortest_path=true`) and SDN/TE (capacity-aware, iterative via `require_capacity=true` + `shortest_path=false`) @@ -776,7 +789,7 @@ NetGraph's design includes several features that differentiate it from tradition - GIL released during algorithm execution, enabling concurrent analysis across Python threads - Zero-copy NumPy integration for array inputs/outputs (via buffer protocol) - Deterministic edge ordering for reproducible results -- Cache-friendly CSR representation for efficient neighbor traversal +- Cache-friendly CSR representation for neighbor traversal **Graph Building and Reuse:** @@ -785,9 +798,9 @@ For Monte Carlo analysis with many failure iterations, graph construction is amo - Context built once before iterations begin (includes all nodes and augmentation edges) - Per-iteration exclusions applied via boolean masks rather than graph rebuilding - Mask building is O(|excluded|) using pre-computed `link_id_to_edge_indices` mapping -- FailureManager automatically pre-builds the `AnalysisContext` before parallel execution +- FailureManager automatically pre-builds the `AnalysisContext` before parallel execution (via the analysis function's `prepare_inputs` hook) -This optimization is critical for performance: graph construction involves Python processing, NumPy array creation, and C++ object initialization. Building the graph once eliminates this overhead from the per-iteration critical path, enabling the GIL-releasing C++ algorithms to execute with minimal Python overhead. +Graph construction involves Python processing, NumPy array creation, and C++ object initialization. Building the graph once keeps that work off the per-iteration critical path, leaving the GIL-releasing C++ algorithms to run with minimal Python overhead. **SPF Caching for Demand Placement:** @@ -798,7 +811,7 @@ Both TrafficMatrixPlacement and MaximumSupportedDemand (MSD) use a unified place - Complex multi-flow policies (TE_ECMP_16_LSP, TE_ECMP_UP_TO_256_LSP) use FlowPolicy directly - MSD additionally pre-resolves node IDs once at cache build time and reuses them across all alpha probes -This reduces SPF computations from O(demands) to O(unique_sources) for workloads where many demands share the same source nodes. For MSD, the optimization is particularly significant since it evaluates many alpha values during binary search. +This reduces SPF computations from O(demands) to O(unique_sources) for workloads where many demands share the same source nodes. MSD gains the most, since it evaluates many alpha values during binary search. **Monte Carlo Deduplication:** @@ -809,9 +822,9 @@ common failure policies. **Complexity:** - SPF: \(O((V+E) \log V)\) using binary heap -- Max-flow: \(O(V^2 E \log V)\) worst-case for successive shortest paths with blocking flow - - Practical performance dominated by \(O(F \cdot (V+E) \log V)\) for integer capacities where \(F\) is max-flow value - - Early termination when residual capacity exhausts provides significant speedup in typical networks +- Max-flow: \(O(E \cdot (V^2 E + (V+E) \log V))\) worst case for the successive-shortest-paths scheme with blocking-flow placement (derived in "Maximum Flow Algorithm" above) + - Practical performance is far better: the number of augmentation phases equals the small number of cost tiers actually used, and the `kMinFlow` threshold caps phases at \(F / k_{MinFlow}\) for total flow \(F\) + - Early termination when the residual network disconnects source from sink provides significant speedup in typical networks **Scalability:** @@ -831,21 +844,21 @@ NetGraph's hybrid architecture combines: **C++ Layer:** -- High-performance graph algorithms (SPF, K-shortest paths, max-flow) +- Native C++ graph algorithms (SPF, K-shortest paths, max-flow) - Immutable StrictMultiDiGraph with CSR adjacency - Configurable flow placement policies (ECMP/WCMP simulation) -- Runtime masking for efficient repeated analysis +- Runtime masking for repeated analysis without graph rebuilds **Integration:** - `AnalysisContext` builds Core graphs, manages name/ID mapping, and bridges Python ↔ C++ - Stable node/edge ID mapping for result traceability -- NumPy array interface for efficient data transfer +- Zero-copy NumPy array interface for data transfer - GIL release during computation for concurrent thread execution This design adapts standard algorithms to network engineering use cases (flow splitting, -failure simulation, cost-aware routing) while achieving high performance through native -C++ execution and ergonomic interfaces through Python APIs. +failure simulation, cost-aware routing), running them in native C++ while keeping the +scenario, workflow, and result interfaces in Python. ## Cross-references diff --git a/docs/reference/dsl.md b/docs/reference/dsl.md index bc7d7a6..19fb5b4 100644 --- a/docs/reference/dsl.md +++ b/docs/reference/dsl.md @@ -8,7 +8,7 @@ Quick links: - [API Reference](api.md) — Python API for programmatic scenario creation - [Auto-Generated API Reference](api-full.md) — complete class and method documentation -This document describes the DSL for defining network scenarios in NetGraph. Scenarios are YAML files that describe network topology, traffic demands, and analysis workflows. +NetGraph scenarios are YAML files describing network topology, traffic demands, and analysis workflows. This document is the reference for that DSL. ## Overview @@ -18,7 +18,7 @@ A scenario file defines a complete network simulation including: - **Analysis configuration**: Traffic demands, failure policies, workflows - **Reusable components**: Blueprints, hardware definitions -The DSL enables both simple direct definitions and complex hierarchical structures with templates and parameters. +Every structure can be written out directly or generated from templates and parameters. ## Template Syntaxes @@ -32,7 +32,7 @@ The DSL uses three distinct template syntaxes in different contexts: **These syntaxes are not interchangeable.** Each works only in its designated context. -**Why different syntaxes?** Each serves a distinct purpose: +**Why different syntaxes?** | Syntax | Operation | Key Difference | |--------|-----------|----------------| @@ -44,12 +44,10 @@ Bracket expansion generates structure; variable expansion parameterizes rules; n ## Entity Creation Architecture -The DSL implements two fundamentally different selection patterns optimized for different use cases. Understanding these patterns is essential for effective scenario authoring. +The DSL has two selection patterns. Which one applies is fixed by the operation, not chosen by the author, so it is worth knowing which is which before writing selectors. ### Two Selection Models -The DSL uses distinct selection strategies depending on the operation: - **1. Path-Based Node Selection** (link rules, traffic demands, workflow steps) - Uses regex patterns on hierarchical node names @@ -62,9 +60,9 @@ The DSL uses distinct selection strategies depending on the operation: - Works on nodes, links, or risk_groups (`scope`) - Supports attribute-based filtering (`conditions`) -- Supports optional `path` regex filtering to narrow candidates before condition matching +- Supports optional `path` regex filtering (a pre-filter for membership and generate rules; applied after condition matching for failure rules) -These patterns share common primitives (condition evaluation, match specification) but serve different purposes and should not be confused. +Both build on the same primitives (condition evaluation, match specification), but they are not interchangeable. ### Link Creation Flow @@ -112,7 +110,7 @@ flowchart TD ### Traffic Demand Creation Flow -Traffic demands follow a similar pattern but with important differences: +Traffic demands follow a similar pattern, with these differences: ```mermaid flowchart TD @@ -161,8 +159,8 @@ flowchart TD Direct --> DirectDesc[Simply name the risk group
Entities reference it explicitly] Member --> MemberScope[Specify scope
node, link, or risk_group] - MemberScope --> MemberCond[Define match conditions
logic defaults to and] - MemberCond --> MemberExec[Scan ALL entities of that scope
Add matching entities to risk group] + MemberScope --> MemberCond[Define match conditions
logic defaults to and
optional path pre-filter] + MemberCond --> MemberExec[Scan entities of that scope
Add matching entities to risk group] Generate --> GenScope[Specify scope
node or link only] GenScope --> GenGroupBy[Specify group_by attribute] @@ -177,8 +175,8 @@ flowchart TD **Key Characteristics:** -- **No path patterns**: Operates on ALL entities of specified scope -- **Only attribute-based**: Uses `conditions` exclusively +- **Scope-wide scan**: Operates on all entities of the specified scope; an optional `path` regex narrows candidates by name (links match against their `source|target` form) +- **Attribute-based filtering**: Uses `conditions`; no capture-group grouping - **Logic defaults to "and"** for membership (stricter matching) - **Hierarchical support**: Risk groups can contain other risk groups as children @@ -190,7 +188,7 @@ flowchart TD | Regex Patterns | Yes | Yes | Yes (optional) | | Capture Groups | Yes | Yes | No | | `group_by` | Yes | Yes | Yes (generate only) | -| `match` Conditions | Yes | Yes | Yes (membership/generate) | +| `match` Conditions | Yes | Yes | Yes (membership only) | | `active_only` Default | False | True | N/A | | `match.logic` Default | "or" | "or" | "and" (membership) | | Variable Expansion | Yes | Yes | No | @@ -198,7 +196,7 @@ flowchart TD ### Shared Evaluation Primitives -All selection mechanisms share common evaluation primitives: +Every selection mechanism evaluates conditions the same way. **1. Condition Structure** @@ -215,18 +213,24 @@ conditions: | Operator | Description | Example | |----------|-------------|---------| -| `==` | Equals | `attr: "role", op: "==", value: "leaf"` | -| `!=` | Not equals | `attr: "tier", op: "!=", value: 1` | -| `<` | Less than (numeric) | `attr: "cost", op: "<", value: 100` | -| `<=` | Less than or equal | `attr: "priority", op: "<=", value: 5` | -| `>` | Greater than (numeric) | `attr: "capacity", op: ">", value: 1000` | -| `>=` | Greater than or equal | `attr: "tier", op: ">=", value: 2` | -| `contains` | String contains or collection includes | `attr: "name", op: "contains", value: "spine"` | -| `not_contains` | String/collection does not include | `attr: "tags", op: "not_contains", value: "deprecated"` | -| `in` | Value is in provided list | `attr: "role", op: "in", value: ["leaf", "spine"]` | -| `not_in` | Value is not in provided list | `attr: "dc", op: "not_in", value: ["dc3", "dc4"]` | -| `exists` | Attribute exists and is not null | `attr: "hardware.vendor", op: "exists"` | -| `not_exists` | Attribute missing or null | `attr: "deprecated", op: "not_exists"` | +| `==` | Equals | `{attr: "role", op: "==", value: "leaf"}` | +| `!=` | Not equals | `{attr: "tier", op: "!=", value: 1}` | +| `<` | Less than (numeric) | `{attr: "cost", op: "<", value: 100}` | +| `<=` | Less than or equal | `{attr: "priority", op: "<=", value: 5}` | +| `>` | Greater than (numeric) | `{attr: "capacity", op: ">", value: 1000}` | +| `>=` | Greater than or equal | `{attr: "tier", op: ">=", value: 2}` | +| `contains` | String contains or collection includes | `{attr: "name", op: "contains", value: "spine"}` | +| `not_contains` | String/collection does not include | `{attr: "tags", op: "not_contains", value: "deprecated"}` | +| `in` | Value is in provided list | `{attr: "role", op: "in", value: ["leaf", "spine"]}` | +| `not_in` | Value is not in provided list | `{attr: "dc", op: "not_in", value: ["dc3", "dc4"]}` | +| `exists` | Attribute exists and is not null | `{attr: "hardware.vendor", op: "exists"}` | +| `not_exists` | Attribute missing or null | `{attr: "deprecated", op: "not_exists"}` | + +Operator semantics: + +- Ordering operators (`<`, `<=`, `>`, `>=`) coerce both sides to float when possible, so `"10" > 5` is true; equality (`==`, `!=`) does **not** coerce, so `"10" == 10` is false. Keep attribute and condition value types consistent. +- For a missing or null attribute, every operator except `not_exists` returns false — including the negative ones (`!=`, `not_contains`, `not_in`). Use `not_exists` to match absent attributes. +- `in`/`not_in` require a list value. Link selectors, node/link rules, failure rules, and membership rules reject a scalar at scenario load; demand selectors reject it when the demand is first evaluated. **3. Condition Combining (`logic`)** @@ -237,15 +241,16 @@ conditions: match: logic: "and" conditions: - - attr: "role", op: "==", value: "leaf" - - attr: "tier", op: ">=", value: 2 + - {attr: "role", op: "==", value: "leaf"} + - {attr: "tier", op: ">=", value: 2} ``` **4. Attribute Access** Conditions evaluate against a flattened view of entity attributes: -- Top-level fields: `name`, `disabled`, `capacity`, `cost`, `risk_groups` -- Custom attributes from `attrs` block +- Node top-level fields: `name`, `disabled`, `risk_groups` +- Link top-level fields: `id`, `source`, `target`, `capacity`, `cost`, `disabled`, `risk_groups` +- Custom attributes from `attrs` block (top-level fields take precedence on key conflicts) **5. Dot-Notation for Nested Attributes** @@ -264,7 +269,7 @@ Template expansion (`$var`, `${var}`) is processed before condition evaluation ( ### Context-Aware Defaults -The DSL uses context-aware defaults to optimize for common use cases: +Defaults differ by context, chosen for the common case in each: | Context | Selection Type | Active Only | Match Logic | Rationale | |---------|---------------|-------------|-------------|-----------| @@ -276,25 +281,25 @@ The DSL uses context-aware defaults to optimize for common use cases: | Failure Rules | Condition-based | N/A | "or" | Inclusive matching for failure scenarios | | Generate Blocks | Condition-based | N/A | N/A | No conditions, groups by values | -These defaults ensure intuitive behavior while remaining overridable when needed. +The `active_only` and `match.logic` defaults can be set explicitly per selector. ## Top-Level Keys ```yaml -network: # Network topology (required) -blueprints: # Reusable network templates -components: # Hardware component library -risk_groups: # Failure correlation groups -vars: # YAML anchors and variables for reuse -demands: # Traffic demand definitions -failures: # Failure simulation policies -workflow: # Analysis execution steps -seed: # Master seed for reproducibility (integer) +network: {} # Network topology +blueprints: {} # Reusable network templates +components: {} # Hardware component library +risk_groups: [] # Failure correlation groups +vars: {} # YAML anchors and variables for reuse +demands: {} # Traffic demand definitions +failures: {} # Failure simulation policies +workflow: [] # Analysis execution steps +seed: 42 # Master seed for reproducibility (integer) ``` | Key | Required | Description | |-----|----------|-------------| -| `network` | **Yes** | Network topology: nodes, links, and rules | +| `network` | No | Network topology: nodes, links, and rules | | `blueprints` | No | Reusable topology templates | | `components` | No | Hardware component library for cost/power modeling | | `risk_groups` | No | Failure correlation groups for resilience analysis | @@ -304,11 +309,13 @@ seed: # Master seed for reproducibility (integer) | `workflow` | No | Analysis workflow steps to execute | | `seed` | No | Master seed (integer) for reproducible random operations | +All sections are optional. A scenario that omits `network` entirely, or sets it to an empty mapping (`network: {}`), builds with an empty topology — note that a bare `network:` with no value is a YAML null and fails schema validation; any unrecognized top-level key is rejected during JSON Schema validation with `jsonschema.ValidationError` (the schema sets `additionalProperties: false`). + **Seed:** When specified, the `seed` value is used to derive deterministic per-component seeds (via SHA-256 hashing) for failure sampling and workflow steps, ensuring reproducible results across runs. Each failure iteration creates a single isolated random number generator from its derived seed. Without a seed, results may vary between executions. ## `network` - Core Foundation -The only required section. Defines network topology through nodes and links. +Defines network topology through nodes and links. **Network metadata fields:** @@ -316,8 +323,8 @@ The only required section. Defines network topology through nodes and links. network: name: "my-network" # Optional network name (stored in network.attrs) version: "1.0" # Optional version (stored in network.attrs) - nodes: { ... } - links: [ ... ] + nodes: {} # Node definitions (see below) + links: [] # Link definitions (see below) ``` ### Direct Node and Link Definitions @@ -369,7 +376,7 @@ Recognized keys for each link entry: - `source`, `target`: node names (required) - `capacity`: link capacity (optional; default 1.0) -- `cost`: link cost (optional; default 1.0) +- `cost`: link cost (optional; default 1.0; must be an integer value — fractional costs are rejected with `ValueError` when the analysis graph is built, because the core engine requires int64 costs) - `disabled`: boolean (optional) - `risk_groups`: list of risk-group names (optional) - `attrs`: mapping of attributes (optional) @@ -426,7 +433,7 @@ network: Creates: `dc1/rack1/srv-1`, `dc1/rack1/srv-2`, ..., `dc1/tor/tor-1`, `dc1/tor/tor-2` -Nested nodes are useful for creating simple hierarchies without defining reusable blueprints. For complex or repeated structures, prefer blueprints. +Use nested nodes for one-off hierarchies. For structures repeated across the scenario, use blueprints. **Link Definitions:** @@ -440,7 +447,7 @@ network: cost: 1 - source: /spine target: /spine - pattern: "one_to_one" # Connect spines pairwise + pattern: "mesh" # Connect every spine to every other spine count: 2 # Create 2 parallel links per pair (optional) capacity: 1600 cost: 1 @@ -452,7 +459,7 @@ network: ### Attribute-filtered Links (selector objects) -You can filter the source or target node sets by attributes using the same condition syntax as failure policies. Replace a string `source`/`target` with an object that has `path` and optional `match`: +To filter the source or target node sets by attributes, replace a string `source`/`target` with an object that has `path` and optional `match`. The condition syntax is the same as in failure policies: ```yaml network: @@ -480,7 +487,7 @@ network: Notes: - `path` is a regex pattern matched against node names (anchored at start via Python `re.match`). -- `match.conditions` uses the shared condition operators: `==`, `!=`, `<`, `<=`, `>`, `>=`, `contains`, `not_contains`, `exists`, `not_exists`. +- `match.conditions` uses the shared condition operators: `==`, `!=`, `<`, `<=`, `>`, `>=`, `contains`, `not_contains`, `in`, `not_in`, `exists`, `not_exists`. - Conditions evaluate over a flat view of node attributes combining top-level fields (`name`, `disabled`, `risk_groups`) and `node.attrs`. - `logic` in the `match` block accepts "and" or "or" (default "or"). - Selectors filter node candidates before the link `pattern` is applied. @@ -523,7 +530,7 @@ network: **Connectivity Patterns:** - `mesh`: Full connectivity between all source and target nodes -- `one_to_one`: Pairwise connections. Compatible sizes means max(|S|,|T|) must be an integer multiple of min(|S|,|T|); mapping wraps modulo the smaller set (e.g., 4x2 and 6x3 valid; 3x2 invalid). +- `one_to_one`: Pairwise connections. Compatible sizes means max(|S|,|T|) must be an integer multiple of min(|S|,|T|); mapping wraps modulo the smaller set (e.g., 4x2 and 6x3 valid; 3x2 invalid). Self-pairs are skipped, so `one_to_one` between a group and itself creates no links — use `mesh` to interconnect a group with itself. ### Bracket Expansion @@ -551,11 +558,11 @@ dc[1-2]/rack[a,b]: # Creates: dc1/racka, dc1/rackb, dc2/racka, dc2/rackb **Scope:** Bracket expansion applies to: -- **Node group names** under `network.nodes` and `blueprints.*.nodes` +- **Node names** under `network.nodes` and `blueprints.*.nodes` — including direct single-node entries without count/template (`SEA[1-2]: {}` creates nodes `SEA1` and `SEA2`) - **Risk group names** in top-level `risk_groups` definitions (including children) -- **Risk group membership arrays** on nodes, links, and node groups +- **Risk group membership arrays** on nodes, links, node groups, and in node/link rules -Component names, direct node names (`network.nodes` without count/template), and other string fields treat brackets as literal characters. +Component names and other string fields treat brackets as literal characters. (Link `source`/`target` strings are regexes, where `[...]` is a character class, not bracket expansion.) **Risk Group Expansion Examples:** @@ -563,6 +570,7 @@ Component names, direct node names (`network.nodes` without count/template), and # Definition expansion - creates DC1_Power, DC2_Power, DC3_Power risk_groups: - name: "DC[1-3]_Power" + - name: "RG[1-3]" # Defines RG1, RG2, RG3 (referenced below) # Membership expansion - assigns to RG1, RG2, RG3 network: @@ -585,6 +593,8 @@ The range syntax `[start-end]` only supports integers. For letters, mixed sequen Use `$var` or `${var}` syntax with an `expand` block for template substitution. Variables are recursively substituted in all string fields within the block, including nested `attrs`. +**Type preservation:** A value consisting of exactly one placeholder (e.g. `value: "${t}"`) is replaced by the variable's native value, preserving its type — so `match` conditions compare correctly against numeric node/link attributes. Placeholders embedded in longer strings (e.g. `"dc${dc}_internal"`) interpolate as text and always produce strings. A bare placeholder bound to a non-string variable used where a path selector is required (e.g. `source: "${n}"` with `n: [1, 2]`) raises `ValueError` instead of being silently stringified; use an embedded form such as `"dc${n}/leaf"` for selector strings. + **Supported contexts:** - Link definitions (`network.links`) @@ -599,6 +609,8 @@ Use `$var` or `${var}` syntax with an `expand` block for template substitution. | `cartesian` (default) | All combinations of variable values | `p:[1,2]`, `r:[a,b]` → 4 expansions | | `zip` | Pair values by index (lists must have equal length) | `a:[1,2]`, `b:[x,y]` → 2 expansions | +**Warning — cartesian expansion and reversed pairs:** Each variable combination of an `expand` block is an independent link definition. Reversed-pair deduplication applies only within one combination, so cartesian expansion over symmetric variable lists (e.g. `vars: {a: [1, 2], b: [1, 2]}` with `source: "dc${a}/gw"`, `target: "dc${b}/gw"`) creates *both* orientations as separate parallel links, doubling capacity. To mesh one node set, prefer a single mesh definition with a regex selector (e.g. `source: "dc[0-9]+/gw"`, `target: "dc[0-9]+/gw"`, `pattern: mesh`), which deduplicates reversed pairs. + **Example in links:** ```yaml @@ -641,7 +653,7 @@ links: ## `blueprints` - Reusable Templates -Templates for network segments that can be instantiated multiple times: +Templates for network segments, instantiated as many times as needed: ```yaml blueprints: @@ -677,6 +689,38 @@ network: - Override parameters using dot notation during instantiation - Hierarchical naming: `pod1/leaf/leaf-1`, `pod2/spine/core-1` +**Parameter override rules:** + +- Override keys must be of the form `.`, where `` is a literal (unexpanded) subgroup name defined in the blueprint. Unknown group prefixes or bare keys (no dot) raise `ValueError` at scenario build time. +- Override keys are matched against literal blueprint subgroup names by the longest `.` prefix (not by splitting on the first dot), so subgroup names containing dots (e.g. `rack.a`) are addressable as `rack.a.count`. When one subgroup name is a dotted extension of another (e.g. `rack` and `rack.a`), the longest matching name receives the override. +- To override parameters of a nested blueprint, use a dict value under `.params`: + +```yaml +blueprints: + leaf_spine: + nodes: + leaf: + count: 4 + spine: + count: 2 + two_pod_dc: + nodes: + pod1: + blueprint: leaf_spine + pod2: + blueprint: leaf_spine + +network: + nodes: + dc1: + blueprint: two_pod_dc + params: + pod1.params: # Dict-valued pass-through to the nested blueprint + spine.count: 8 +``` + +The dotted form `pod1.params.spine.count` is rejected. + ## Node and Link Rules Modify specific nodes or links after initial creation. Rules run post-expansion and can override properties, add attributes, or disable elements. @@ -717,11 +761,11 @@ network: **Node rule fields:** -- `path`: Regex pattern matched against node names (required) +- `path`: Regex pattern matched against node names (optional; defaults to `.*`, matching all nodes) - `match`: Optional attribute conditions to filter matched nodes - `disabled`: Set node disabled state - `attrs`: Attributes to merge into matched nodes -- `risk_groups`: Risk groups to add to matched nodes +- `risk_groups`: Risk groups for matched nodes — **replaces** the node's existing `risk_groups` set (it does not add to it) - `expand`: Variable expansion block for templated rules ### Link Rules @@ -772,12 +816,12 @@ network: **Link rule fields:** -- `source`, `target`: Regex patterns or selector objects for endpoint matching +- `source`, `target`: Regex patterns or selector objects for endpoint matching (both required on every rule) - `bidirectional`: Match links in both directions (default: `true`) - `link_match`: Filter by link's own attributes (not endpoint attributes) -- `capacity`, `cost`, `disabled`: Override link properties +- `capacity`, `cost`, `disabled`: Override link properties (`cost` must be an integer value — fractional costs are rejected when the analysis graph is built) - `attrs`: Attributes to merge into matched links -- `risk_groups`: Risk groups to add to matched links +- `risk_groups`: Risk groups for matched links — **replaces** the link's existing `risk_groups` set (it does not add to it) - `expand`: Variable expansion block for templated rules **Execution order:** @@ -843,7 +887,7 @@ network: ## `risk_groups` - Risk Modeling -Define hierarchical failure correlation groups for modeling correlated failures. Risk groups can represent any failure correlation pattern: physical infrastructure, geographic regions, vendor dependencies, or custom domains. +Risk groups model correlated failures as hierarchies: physical infrastructure, geographic regions, vendor dependencies, or custom domains. ### Understanding Hierarchy @@ -871,6 +915,8 @@ for child in region.children: print(child.name) # Site_Seattle, Site_Portland ``` +**Top-level-only keys:** `membership`, `disabled`, and `generate` are honored only on top-level `risk_groups` entries. Nested `children` entries allow only `name`, `attrs`, and `children`; placing any other key on a child is rejected — by the JSON schema at scenario load and by the parser with `ValueError`. Define such groups at top level and reference them by name as children. + **Entity references:** Nodes and links reference risk groups by name. To reference a group, it must be defined at top level (children alone are not sufficient): ```yaml @@ -888,7 +934,7 @@ network: ### Common Use Cases -Risk groups can model any failure correlation pattern. Below are some common examples: +Common correlation patterns: **Physical Infrastructure** (fiber paths, power zones, cooling systems) **Geographic/Administrative** (regions, availability zones, maintenance windows) @@ -897,7 +943,7 @@ Risk groups can model any failure correlation pattern. Below are some common exa ### Example 1: Physical Infrastructure (Fiber Links) -One common use case is modeling physical infrastructure. For fiber links, you might use a hierarchy like Path -> Conduit -> Fiber Pair. +For fiber links, a hierarchy of Path -> Conduit -> Fiber Pair mirrors the physical plant. ```yaml risk_groups: @@ -936,7 +982,7 @@ network: ### Example 2: Physical Infrastructure (Data Center Nodes) -For data center nodes, you might model facility infrastructure with a hierarchy like Building -> Room -> Power Zone. +For data center nodes, a hierarchy of Building -> Room -> Power Zone mirrors the facility. ```yaml risk_groups: @@ -1019,7 +1065,9 @@ risk_groups: value: "DC1-R1-PZ-A" ``` -**Note:** Membership rules default to `logic: "and"` (stricter than link/demand selectors which default to `"or"`). This ensures precise entity matching for failure correlation. +**Note:** Membership rules default to `logic: "and"`, stricter than link/demand selectors, which default to `"or"`. + +A membership rule requires `scope` plus at least one of `path` or `match`; a `match` block must contain at least one condition. The optional `path` regex pre-filters candidates by name before conditions are evaluated (links match against their `source|target` form). ### Generated Risk Groups @@ -1044,6 +1092,8 @@ risk_groups: type: building ``` +A generate block requires `scope` (`node` or `link`), `group_by` (supports dot-notation), and `name`; `name` must contain the `${value}` placeholder. An optional `path` regex pre-filters entities by name (links match against their `source|target` form). Errors raised as `ValueError`: a `group_by` attribute resolving to an unhashable value (e.g. a list), a name template rendering the same group name for two distinct attribute values, and a generated name colliding with an existing risk group. Generate blocks run after membership resolution, so membership rules cannot match generated groups. + ### Validation Risk group references are validated at scenario load time: @@ -1061,6 +1111,8 @@ risk_groups: - name: "PowerZone_B" # Only PowerZone_B is defined ``` +**Child Key Restrictions:** Nested `children` entries accept only `name`, `attrs`, and `children`. `membership`, `disabled`, and `generate` on a child entry fail schema validation at load time (and the parser raises `ValueError`), since only top-level groups are registered in `network.risk_groups` and these keys would otherwise be silently inert. + **Circular Hierarchy Detection:** Parent-child relationships cannot form cycles: ```yaml @@ -1069,10 +1121,13 @@ risk_groups: - name: "GroupA" children: - name: "GroupB" - children: - - name: "GroupA" # Error: circular reference + - name: "GroupB" + children: + - name: "GroupA" # Error: circular reference ``` +Cycle detection runs over top-level groups (whose children are followed by name), including parent-child links added by membership rules with `scope: risk_group`. Detection walks only names that are registered as top-level risk groups: a direct child entry repeating its own parent's name is a self-cycle and *is* rejected, whereas a name repeated deeper than a direct child (nested under a child that is not itself a top-level group) is never followed and is not detected. + Validation errors list affected entities and undefined groups to aid debugging. ## `vars` - YAML Anchors @@ -1106,7 +1161,7 @@ network: - Anchors are resolved during YAML parsing, before schema validation - The `vars` section itself is ignored by NetGraph runtime logic - Anchors can be defined in any section, not just `vars` -- Merge operations follow YAML 1.1 semantics (later keys override earlier ones) +- Merge semantics (as parsed by PyYAML): explicit keys override merged keys regardless of position; with repeated `<<:` merge keys, later merges override earlier ones, while the sequence form `<<: [*a, *b]` gives earlier entries precedence ## `demands` - Traffic Analysis @@ -1193,11 +1248,12 @@ demands: | `priority` | integer | Priority class; lower = higher priority (default: 0) | | `mode` | string | Node pairing mode: `combine` or `pairwise` (default: `combine`) | | `group_mode` | string | How grouped nodes produce demands (default: `flatten`) | -| `flow_policy` | string | Routing policy preset name | -| `id` | string | Unique demand identifier (auto-generated if omitted) | +| `flow_policy` | string or integer | Routing policy preset name (case-insensitive, or its integer value); inline policy mappings fail schema validation at scenario load | | `attrs` | object | Arbitrary metadata | | `expand` | object | Variable expansion block | +Each demand receives an auto-generated unique `id`; an explicit `id` key is not accepted in scenario YAML. Duplicate ids can arise only when demands are constructed programmatically, and demand expansion rejects them with `ValueError`. + ### Selector Fields The `source` and `target` fields accept either: @@ -1212,13 +1268,15 @@ Controls how source and target node sets are paired: - `combine`: Aggregate all sources into one virtual source, all targets into one virtual target. Produces a single flow. - `pairwise`: Create individual flows between all source-target node pairs. Volume is distributed across pairs. +**Overlapping selections in `combine` mode:** Nodes selected by both `source` and `target` are excluded from the target side, so overlapping selections cannot route volume through a zero-cost pseudo-node bypass; placement is bounded by real network capacity. With `group_mode: flatten`, a demand whose source and target selections fully overlap leaves no targets after exclusion and expands to nothing; if no demand in the whole expansion produces anything, analysis fails with `No demands could be expanded`. + ### Group Modes (`group_mode`) When using `group_by` selectors, controls how grouped nodes produce demands: - `flatten` (default): Flatten all groups into a single source/target set, then apply `mode` -- `per_group`: Create separate demands for each group independently -- `group_pairwise`: Create demands between each source group and each target group pair +- `per_group`: Create separate demands for each group independently. With `mode: combine`, one demand per source group with all targets combined; each source group's combined target set excludes that group's own nodes, and a source group whose target set becomes empty after this exclusion is skipped (its even share of the volume is dropped). With `mode: pairwise`, node pairs within each group label present on both source and target sides (labels must match). Volume is split evenly across source groups (combine) or shared labels (pairwise); the total expanded volume equals the configured volume unless a group is skipped due to full overlap. +- `group_pairwise`: Create demands between each ordered (source_group, target_group) label pair; pairs with identical labels are skipped **Example with `group_mode`:** @@ -1232,18 +1290,20 @@ demands: group_by: "dc" volume: 1000 mode: "combine" - group_mode: "group_pairwise" # Creates dc1->dc2, dc1->dc3, dc2->dc3, etc. + group_mode: "group_pairwise" # Creates dc1->dc2, dc2->dc1, dc1->dc3, etc. ``` | `mode` | `group_mode` | Result | |--------|--------------|--------| | `combine` | `flatten` | Single aggregated demand across all nodes | | `combine` | `per_group` | One demand per source group (targets combined) | -| `combine` | `group_pairwise` | One demand per (source_group, target_group) pair | +| `combine` | `group_pairwise` | One demand per (source_group, target_group) pair with distinct labels | | `pairwise` | `flatten` | Individual demands for each (src_node, tgt_node) pair | | `pairwise` | `per_group` | Pairwise within each group | | `pairwise` | `group_pairwise` | Pairwise for each group pair combination | +In `per_group`, `group_pairwise`, and `pairwise` expansions the configured volume is split evenly at each expansion level (across groups or group pairs, then across node pairs within each), so total volume is conserved — except that a skipped expansion's share is dropped: in `combine` mode any group (or group pair) whose target set is empty after excluding shared source/target nodes is skipped, and in `pairwise` mode a group (or group pair) with no non-self node pairs is skipped. If no demands remain after exclusion, expansion fails with `No demands could be expanded`. + ### Flow Policies - `SHORTEST_PATHS_ECMP`: IP/IGP routing with hash-based ECMP; equal split across equal-cost paths @@ -1329,16 +1389,16 @@ failures: |-------|------|---------|-------------| | `modes` | array | required | List of weighted failure modes | | `attrs` | object | `{}` | Policy metadata (e.g., description) | -| `expand_groups` | boolean | `false` | When a risk group is selected, also fail its member nodes/links | -| `expand_children` | boolean | `false` | When a risk group is selected, recursively fail child risk groups | +| `expand_groups` | boolean | `false` | Also fail every entity sharing a risk group with a failed entity (members of a failed risk group are always excluded regardless; this flag adds shared-group correlation, applied identically whether the failure came from an entity rule or a risk_group rule) | + +A failed risk group always cascades to its child groups recursively; cascading is inherent to the risk-group hierarchy and is not controlled by a policy flag. **Risk group expansion example:** ```yaml failures: srlg_failures: - expand_groups: true # Fail all links in selected risk group - expand_children: true # Also fail nested child groups + expand_groups: true # Spread failures across shared risk-group memberships attrs: description: "Shared-risk link group failure simulation" modes: @@ -1363,13 +1423,14 @@ failures: | `mode` | string | Selection mode: `all`, `choice`, or `random` (default: `all`) | | `count` | integer | Number of entities to select (for `choice` mode) | | `probability` | number | Selection probability 0-1 (for `random` mode) | -| `path` | string | Regex filter on entity name | +| `path` | string | Regex filter on entity name (links match against their `source\|target` form) | | `match` | object | Attribute conditions block with `logic` and `conditions` | | `weight_by` | string | Attribute name for weighted sampling in `choice` mode | ### Notes - Policies are mode-based. Each mode has a non-negative `weight`. One mode is chosen per iteration with probability proportional to weights, then all rules in that mode are applied and their selections are unioned. +- At least one mode must have `weight > 0`; a policy whose modes all have zero weight is rejected at scenario load. Modes with zero weight are never selected. - Condition syntax uses the same operators as link/demand selectors. See [Condition Operators](#shared-evaluation-primitives) for the full reference. ## `workflow` - Execution Steps @@ -1403,7 +1464,7 @@ See [Workflow Reference](workflow.md) for detailed configuration. ## Node Selection -NetGraph provides a unified selector system for selecting and grouping nodes across links, demands, and workflow steps. +One selector syntax selects and groups nodes for links, demands, and workflow steps alike. ### Selector Forms @@ -1468,7 +1529,7 @@ source: Notes: -- `group_by` refers to a key in `node.attrs`. Nested keys are not supported. +- `group_by` refers to a top-level field (`name`, `disabled`, `risk_groups`) or a key in `node.attrs`. Nested (dot-notation) keys are not supported. - Nodes without the specified attribute are omitted. - Group labels are the string form of the attribute value. diff --git a/docs/reference/schemas.md b/docs/reference/schemas.md index 5885f53..cc0bdf9 100644 --- a/docs/reference/schemas.md +++ b/docs/reference/schemas.md @@ -26,11 +26,11 @@ The schema validates: - Top-level section organization - Basic constraint checking -Runtime: The schema is applied unconditionally during load in `ngraph.scenario.Scenario.from_yaml`. Additional business rules are enforced in code (e.g., blueprint expansion) and may still raise errors for semantically invalid inputs. +Runtime: The schema is applied unconditionally during load in `ngraph.scenario.Scenario.from_yaml` (via `ngraph.dsl.loader.load_scenario_yaml`). Additional business rules are enforced in code (e.g., blueprint expansion) and may still raise errors for semantically invalid inputs. ## IDE Integration (VS Code) -Automatic configuration via `.vscode/settings.json`: +Add to `.vscode/settings.json` (not committed to the repository): ```json { @@ -59,7 +59,7 @@ make check ### Integration Points -- Pre-commit hooks: Validates modified `scenarios/*.yaml` files +- Pre-commit hooks: Runs `make validate` when `scenarios/*.yaml` files change - CI pipeline: Validates scenarios on push/PR - Test suite: Validation exercised in integration tests diff --git a/docs/reference/workflow.md b/docs/reference/workflow.md index a8d233c..f68eb3e 100644 --- a/docs/reference/workflow.md +++ b/docs/reference/workflow.md @@ -8,7 +8,7 @@ Quick links: - [API Reference](api.md) — Python API for programmatic scenario creation - [Auto-Generated API Reference](api-full.md) — complete class and method documentation -This document describes NetGraph workflows – analysis execution pipelines that perform capacity analysis, demand placement, and statistics computation. +NetGraph workflows are analysis execution pipelines that perform capacity analysis, demand placement, and statistics computation. ## Overview @@ -32,7 +32,7 @@ workflow: - Steps run sequentially via `WorkflowStep.execute()`, which records timing and metadata and stores outputs under `{metadata, data}` for the step. - Monte Carlo steps (`MaxFlow`, `TrafficMatrixPlacement`) execute iterations using the Failure Manager. Each iteration analyzes the network with exclusion sets applied to mask failed nodes/links without mutating the base network. Workers are controlled by `parallelism: auto|int`. -- Seeding: a scenario-level `seed` derives per-step seeds unless a step sets an explicit `seed`. Metadata includes `scenario_seed`, `step_seed`, `seed_source`, and `active_seed`. +- Seeding: a scenario-level `seed` derives per-step seeds unless a step sets an explicit `seed`. Metadata includes `scenario_seed`, `step_seed`, `seed_source`, and `active_seed`. `seed_source`/`active_seed` reflect the seed the step actually uses: a step constructed without its own seed reports `seed_source: none` even when the scenario has a seed (YAML-loaded scenarios derive per-step seeds at parse time, so those report `scenario-derived`). ## Core Workflow Steps @@ -100,7 +100,6 @@ Monte Carlo placement of a named demand set with optional alpha scaling. Baselin failure_policy: random_failures # Optional: policy name in failures section iterations: 100 # Number of failure iterations parallelism: auto - placement_rounds: auto # or an integer include_flow_details: true # cost_distribution per flow include_used_edges: false # include per-demand used edge lists store_failure_patterns: false @@ -113,10 +112,13 @@ Monte Carlo placement of a named demand set with optional alpha scaling. Baselin Outputs: - metadata: iterations, parallelism, analysis_function, policy_name, - execution_time, unique_patterns -- data.context: demand_set, placement_rounds, include_flow_details, + execution_time, unique_patterns, occurrence_counts +- data.baseline and data.flow_results: see Results Export Shape below +- data.context: demand_set, include_flow_details, include_used_edges, base_demands, alpha, alpha_source +Note: `placement_rounds` is deprecated and has no effect. It is still accepted in YAML for backward compatibility and is not exported in `data.context`; setting it to any value other than `auto` also logs a deprecation warning. + ### MaximumSupportedDemand Search for the maximum uniform traffic multiplier `alpha_star` that is fully placeable. @@ -133,7 +135,6 @@ Search for the maximum uniform traffic multiplier `alpha_star` that is fully pla resolution: 0.01 # Convergence resolution for bisection max_bracket_iters: 32 # Maximum bracketing iterations max_bisect_iters: 32 # Maximum bisection iterations - placement_rounds: auto # Placement optimization rounds ``` Parameters: @@ -147,14 +148,14 @@ Parameters: - `resolution`: Convergence threshold for bisection. - `max_bracket_iters`: Maximum iterations for bracketing phase. - `max_bisect_iters`: Maximum iterations for bisection phase. -- `placement_rounds`: Number of placement optimization rounds (`int` or `"auto"`). +- `placement_rounds`: Deprecated; accepted for backward compatibility but has no effect (placement optimization is handled by the core engine). Outputs: - data.alpha_star: maximum uniform scaling factor - data.context: search parameters - data.base_demands: serialized base demands prior to scaling -- data.probes: bracket/bisect evaluations with feasibility and min ratios +- data.probes: bracket/bisect evaluations with feasibility and placement ratios ### CostPower @@ -173,14 +174,18 @@ Outputs: - data.levels: mapping level->list of {path, platform_capex, platform_power_watts, optics_capex, optics_power_watts, capex_total, power_total_watts} +CostPower performs no hardware capacity/ports validation and completes even on networks that strict hardware validation would reject; use `ngraph inspect` for hardware validation. + ## Node Selection Mechanism -Workflow steps use a unified selector system for node selection. Selectors can be specified as string patterns or selector objects. +Every workflow step selects nodes the same way: a selector is either a string pattern or a selector object. ### String Pattern Matching +String patterns are regular expressions matched against node names, anchored at the start (Python `re.match()`). + ```yaml -# Exact match +# Exact match (also matches names that continue past it, e.g. "spine-10") source: "spine-1" # Prefix match @@ -232,7 +237,7 @@ source: "(dc[1-3])/servers/.*" **Multiple Capturing Groups**: Group labels join captured values with `|`. ```yaml -source: "(dc[1-3])/(spine|leaf)/switch-(\d+)" +source: '(dc[1-3])/(spine|leaf)/switch-(\d+)' # Creates groups: "dc1|spine|1", "dc1|leaf|2", "dc2|spine|1", etc. ``` @@ -263,7 +268,7 @@ source: mode: combine # combine | pairwise (default: combine) iterations: 1000 # Failure iterations to run (default: 1) failure_policy: policy_name # Name in failures section (default: null) -parallelism: auto # Worker processes (default: auto) +parallelism: auto # Worker threads (default: auto) shortest_path: false # Restrict to shortest paths (default: false) require_capacity: true # Path selection considers capacity (default: true) # Set false for true IP/IGP semantics (cost-only routing) @@ -273,7 +278,7 @@ include_flow_details: false # Emit cost_distribution per flow include_min_cut: false # Emit min-cut edge list per flow ``` -Note: Baseline (no failures) is always run first as a separate reference. The `iterations` parameter specifies the number of failure scenarios to run. +Note: Baseline (no failures) is always run first as a separate reference; `iterations` counts failure scenarios only. ## Results Export Shape @@ -287,9 +292,9 @@ Exported results have a fixed top-level structure. Keys under `workflow` and `st "step_name": "network_statistics", "execution_order": 0, "scenario_seed": 42, - "step_seed": 42, + "step_seed": 1903777304, "seed_source": "scenario-derived", - "active_seed": 42 + "active_seed": 1903777304 } }, "steps": { diff --git a/ngraph/__init__.py b/ngraph/__init__.py index 5bf9443..b575351 100644 --- a/ngraph/__init__.py +++ b/ngraph/__init__.py @@ -1,7 +1,7 @@ """NetGraph: Network modeling and analysis library. -NetGraph provides interfaces for network topology modeling, traffic analysis, and -capacity planning using a hybrid Python+C++ architecture. +Network topology modeling, traffic analysis, and capacity planning on a hybrid +Python+C++ architecture. Primary API: analyze() - Create an analysis context for network queries @@ -32,8 +32,6 @@ from importlib.metadata import version -from ngraph import cli, logging - __version__ = version("ngraph") from ngraph.analysis import AnalysisContext, analyze from ngraph.analysis.failure_manager import FailureManager @@ -83,7 +81,4 @@ "NodeMap", "from_networkx", "to_networkx", - # Utilities - "cli", - "logging", ] diff --git a/ngraph/analysis/__init__.py b/ngraph/analysis/__init__.py index ccdb286..f27db7a 100644 --- a/ngraph/analysis/__init__.py +++ b/ngraph/analysis/__init__.py @@ -1,6 +1,7 @@ """Network analysis API. -This module provides the primary entry point for network analysis in NetGraph. +`analyze()` returns an AnalysisContext holding the prepared Core graph. Binding +source and sink reuses that graph across calls instead of rebuilding it. Usage: from ngraph import analyze @@ -8,7 +9,7 @@ # One-off analysis flow = analyze(network).max_flow("^A$", "^B$") - # Efficient repeated analysis (bound context) + # Repeated analysis over one prepared graph (bound context) ctx = analyze(network, source="^A$", sink="^B$") baseline = ctx.max_flow() degraded = ctx.max_flow(excluded_links=failed_links) @@ -22,8 +23,6 @@ AugmentationEdge, analyze, ) -from ngraph.analysis.context import build_edge_mask as build_edge_mask -from ngraph.analysis.context import build_node_mask as build_node_mask from ngraph.analysis.demand import ( DemandExpansion, ExpandedDemand, @@ -31,7 +30,7 @@ ) from ngraph.analysis.failure_manager import AnalysisFunction, FailureManager from ngraph.analysis.functions import ( - build_demand_context, + build_demand_placement_inputs, build_maxflow_context, demand_placement_analysis, max_flow_analysis, @@ -61,7 +60,7 @@ "ExpandedDemand", "expand_demands", # Analysis functions - "build_demand_context", + "build_demand_placement_inputs", "build_maxflow_context", "demand_placement_analysis", "max_flow_analysis", diff --git a/ngraph/analysis/context.py b/ngraph/analysis/context.py index 490eea0..9d2b2b0 100644 --- a/ngraph/analysis/context.py +++ b/ngraph/analysis/context.py @@ -1,15 +1,14 @@ -"""AnalysisContext: Prepared state for efficient network analysis. +"""AnalysisContext: prepared graph state for repeated network analysis. -This module provides the primary API for network analysis in NetGraph. -AnalysisContext encapsulates Core graph infrastructure and provides -methods for max-flow, shortest paths, and sensitivity analysis. +AnalysisContext holds the Core graph infrastructure and exposes max-flow, +shortest-path, and sensitivity analysis over it. Usage: # One-off analysis from ngraph import analyze flow = analyze(network).max_flow("^A$", "^B$") - # Efficient repeated analysis (bound context) + # Repeated analysis over one prepared graph (bound context) ctx = analyze(network, source="^A$", sink="^B$") baseline = ctx.max_flow() degraded = ctx.max_flow(excluded_links=failed_links) @@ -17,10 +16,12 @@ from __future__ import annotations +import threading from dataclasses import dataclass, field from typing import ( TYPE_CHECKING, Any, + Callable, Dict, FrozenSet, List, @@ -34,7 +35,9 @@ import netgraph_core import numpy as np +from ngraph.dsl.selectors import normalize_selector from ngraph.model.path import Path +from ngraph.model.selectors import select_nodes from ngraph.types.base import EdgeSelect, FlowPlacement, Mode from ngraph.types.dto import EdgeRef, MaxFlowResult @@ -57,7 +60,7 @@ class AugmentationEdge: source: Source node name (real or pseudo) target: Target node name (real or pseudo) capacity: Edge capacity - cost: Edge cost (converted to int64 for Core) + cost: Edge cost (must be an integer value; Core uses int64 costs) """ __slots__ = ("source", "target", "capacity", "cost") @@ -81,6 +84,41 @@ def _get_active_node_names( return [n.name for n in nodes if not n.disabled] +def _resolve_selector_groups( + network: "Network", + source: Union[str, Dict[str, Any]], + sink: Union[str, Dict[str, Any]], +) -> Tuple[Dict[str, List[Any]], Dict[str, List[Any]]]: + """Normalize source/sink selectors and resolve active node groups. + + Raises: + ValueError: If either selector matches no nodes. + """ + src_groups = select_nodes( + network, normalize_selector(source, "workflow"), default_active_only=True + ) + snk_groups = select_nodes( + network, normalize_selector(sink, "workflow"), default_active_only=True + ) + if not src_groups: + raise ValueError(f"No source nodes found matching '{source}'.") + if not snk_groups: + raise ValueError(f"No sink nodes found matching '{sink}'.") + return src_groups, snk_groups + + +def _combined_group_names( + groups: Dict[str, List[Any]], + excluded_nodes: Optional[Set[str]] = None, +) -> Tuple[str, List[str]]: + """Return the combined "a|b" COMBINE-mode label and active member names.""" + label = "|".join(sorted(groups.keys())) + names: List[str] = [] + for group_nodes in groups.values(): + names.extend(_get_active_node_names(group_nodes, excluded_nodes)) + return label, names + + class _NodeMapper: """Bidirectional mapping between node names (str) and Core NodeId (int).""" @@ -132,27 +170,38 @@ def to_name(self, ext_id: int) -> Optional[str]: @dataclass class _PseudoNodeContext: - """Context for pseudo nodes created during graph construction.""" + """Context for pseudo nodes created during graph construction. + + Attributes: + pairs: Mapping from (src_label, snk_label) to pseudo node IDs for + pairs that have pseudo nodes in the graph. + expected_pairs: All pair keys derived from the bound selectors at + build time, including pairs skipped for empty or overlapping + groups. Used to fill default results without re-running node + selection. + """ - source: Union[str, Dict[str, Any]] - sink: Union[str, Dict[str, Any]] - mode: Mode pairs: Dict[Tuple[str, str], Tuple[int, int]] + expected_pairs: Tuple[Tuple[str, str], ...] @dataclass class AnalysisContext: - """Prepared state for efficient network analysis. + """Prepared graph state for repeated network analysis. - Encapsulates Core graph infrastructure. Supports two usage patterns: + Wraps the Core graph infrastructure. Two usage patterns: - **Unbound** - flexible, specify source/sink per-call: + **Unbound** - source/sink given per call: ctx = AnalysisContext.from_network(network) cost = ctx.shortest_path_cost("A", "B") - flow = ctx.max_flow("A", "B") # Builds pseudo-nodes each call + flow = ctx.max_flow("A", "B") + + Every flow call on an unbound context builds a full temporary bound + context, which rebuilds the graph from scratch; bind the context instead + for repeated flow analysis. - **Bound** - optimized for repeated analysis with same groups: + **Bound** - source/sink fixed at construction, reused across calls: ctx = AnalysisContext.from_network( network, @@ -162,6 +211,9 @@ class AnalysisContext: baseline = ctx.max_flow() # Uses pre-built pseudo-nodes degraded = ctx.max_flow(excluded_links=failed) + Selectors, here and in every method taking one, are either a regex path + string or a dict with ``path``/``group_by``/``match``. + Thread Safety: Immutable after creation. Safe for concurrent analysis calls with different exclusion sets. @@ -174,15 +226,9 @@ class AnalysisContext: # Public read-only reference _network: "Network" - # Core infrastructure (internal) - _handle: netgraph_core.Graph = field(repr=False) - _multidigraph: netgraph_core.StrictMultiDiGraph = field(repr=False) - _node_mapper: _NodeMapper = field(repr=False) - _edge_mapper: _EdgeMapper = field(repr=False) - _algorithms: netgraph_core.Algorithms = field(repr=False) - _disabled_node_ids: FrozenSet[int] = field(repr=False) - _disabled_link_ids: FrozenSet[str] = field(repr=False) - _link_id_to_edge_indices: Mapping[str, Tuple[int, ...]] = field(repr=False) + # Core infrastructure (internal). Built eagerly for bound contexts and + # contexts with custom augmentations; lazily on first access otherwise. + _core: Optional[_GraphBuildResult] = field(default=None, repr=False) # Binding state (None if unbound) _source: Optional[Union[str, Dict[str, Any]]] = None @@ -190,6 +236,14 @@ class AnalysisContext: _mode: Optional[Mode] = None _pseudo_context: Optional[_PseudoNodeContext] = field(default=None, repr=False) + # User-supplied augmentations (excludes pseudo source/sink edges) + _augmentations: Tuple[AugmentationEdge, ...] = field(default=(), repr=False) + + # Guards the lazy Core graph build + _core_lock: threading.Lock = field( + default_factory=threading.Lock, repr=False, compare=False + ) + @property def network(self) -> "Network": """Reference to source network (read-only).""" @@ -225,6 +279,64 @@ def edge_count(self) -> int: """Number of edges in the graph (includes forward + reverse).""" return self._multidigraph.num_edges() + # ────────────────────────────────────────────────────────────── + # Lazy Core graph access (internal) + # ────────────────────────────────────────────────────────────── + + def _ensure_core(self) -> _GraphBuildResult: + """Return the Core graph build, constructing it lazily if needed. + + Bound contexts and contexts with custom augmentations build the + graph eagerly in from_network; plain unbound contexts defer the + build until a path-analysis method or Core accessor needs it. + """ + core = self._core + if core is None: + with self._core_lock: + core = self._core + if core is None: + core = _build_graph_core( + self._network, + add_reverse=True, + augmentations=( + list(self._augmentations) if self._augmentations else None + ), + ) + self._core = core + return core + + @property + def _handle(self) -> netgraph_core.Graph: + return self._ensure_core()._handle + + @property + def _multidigraph(self) -> netgraph_core.StrictMultiDiGraph: + return self._ensure_core()._multidigraph + + @property + def _node_mapper(self) -> _NodeMapper: + return self._ensure_core()._node_mapper + + @property + def _edge_mapper(self) -> _EdgeMapper: + return self._ensure_core()._edge_mapper + + @property + def _algorithms(self) -> netgraph_core.Algorithms: + return self._ensure_core()._algorithms + + @property + def _disabled_node_ids(self) -> FrozenSet[int]: + return self._ensure_core()._disabled_node_ids + + @property + def _disabled_link_ids(self) -> FrozenSet[str]: + return self._ensure_core()._disabled_link_ids + + @property + def _link_id_to_edge_indices(self) -> Mapping[str, Tuple[int, ...]]: + return self._ensure_core()._link_id_to_edge_indices + # ────────────────────────────────────────────────────────────── # Internal properties (not part of public API) # @@ -303,6 +415,19 @@ def from_network( Raises: ValueError: If only one of source/sink is provided. ValueError: If bound and no matching nodes found. + ValueError: If any link capacity is at or above LARGE_CAPACITY + (1e15, the internal pseudo-edge capacity), since such a link + would be silently clamped by the pseudo attachment edges in + combine-mode flows. + ValueError: If any link or augmentation cost is negative or + non-integer, or if the total of all edge costs reaches 2**62. + Core's int64 cost arithmetic would overflow and silently + corrupt SPF and flow results. + + Note: + The capacity and cost checks run during the graph build, which + happens here for bound contexts and for contexts with custom + augmentations, and on first use otherwise. """ if (source is None) != (sink is None): raise ValueError("source and sink must both be provided or both None") @@ -314,59 +439,83 @@ def from_network( # Build pseudo node augmentations if source/sink provided pseudo_pairs: Optional[Dict[Tuple[str, str], Tuple[str, str]]] = None + expected_pairs: Tuple[Tuple[str, str], ...] = () if source is not None and sink is not None: - pseudo_augmentations, pseudo_pairs = _build_pseudo_node_augmentations( - network, source, sink, mode + pseudo_augmentations, pseudo_pairs, expected_pairs = ( + _build_pseudo_node_augmentations(network, source, sink, mode) ) all_augmentations.extend(pseudo_augmentations) - # Build the core graph - ctx = _build_graph_core( - network, - add_reverse=True, - augmentations=all_augmentations if all_augmentations else None, - ) + # Build the core graph eagerly when bound (pseudo node IDs must be + # resolved now) or when custom augmentations are present. Plain + # unbound contexts build lazily on first use. + core: Optional[_GraphBuildResult] = None + if all_augmentations or source is not None: + core = _build_graph_core( + network, + add_reverse=True, + augmentations=all_augmentations if all_augmentations else None, + ) # Create pseudo context if bound pseudo_context: Optional[_PseudoNodeContext] = None if source is not None and sink is not None: + assert core is not None # Bound contexts always build eagerly resolved_pairs: Dict[Tuple[str, str], Tuple[int, int]] = {} if pseudo_pairs: for pair_key, ( pseudo_src_name, pseudo_snk_name, ) in pseudo_pairs.items(): - pseudo_src_id = ctx._node_mapper.to_id(pseudo_src_name) - pseudo_snk_id = ctx._node_mapper.to_id(pseudo_snk_name) + pseudo_src_id = core._node_mapper.to_id(pseudo_src_name) + pseudo_snk_id = core._node_mapper.to_id(pseudo_snk_name) resolved_pairs[pair_key] = (pseudo_src_id, pseudo_snk_id) pseudo_context = _PseudoNodeContext( - source=source, - sink=sink, - mode=mode, pairs=resolved_pairs, + expected_pairs=expected_pairs, ) return cls( _network=network, - _handle=ctx._handle, - _multidigraph=ctx._multidigraph, - _node_mapper=ctx._node_mapper, - _edge_mapper=ctx._edge_mapper, - _algorithms=ctx._algorithms, - _disabled_node_ids=ctx._disabled_node_ids, - _disabled_link_ids=ctx._disabled_link_ids, - _link_id_to_edge_indices=ctx._link_id_to_edge_indices, + _core=core, _source=source, _sink=sink, _mode=mode if source is not None else None, _pseudo_context=pseudo_context, + _augmentations=tuple(augmentations) if augmentations else (), ) # ────────────────────────────────────────────────────────────── # Flow analysis methods # ────────────────────────────────────────────────────────────── + def _dispatch_bound( + self, + source: Optional[Union[str, Dict[str, Any]]], + sink: Optional[Union[str, Dict[str, Any]]], + ) -> bool: + """Validate source/sink against the binding state of this context. + + Returns: + True when the context is bound (dispatch to the *_bound path); + False when unbound (source and sink are then non-None). + + Raises: + ValueError: If bound and source/sink are provided, or unbound + and source/sink are missing. + """ + if self.is_bound: + if source is not None or sink is not None: + raise ValueError( + "Bound context: source/sink already configured. " + "Create new context for different groups." + ) + return True + if source is None or sink is None: + raise ValueError("Unbound context: source and sink are required.") + return False + def max_flow( self, source: Optional[Union[str, Dict[str, Any]]] = None, @@ -381,14 +530,14 @@ def max_flow( ) -> Dict[Tuple[str, str], float]: """Compute maximum flow between node groups. - If context is bound (created with source/sink), uses pre-built - pseudo-nodes for efficiency. Otherwise builds them per-call. + If context is bound (created with source/sink), reuses its pre-built + pseudo-nodes. If unbound, each call rebuilds the graph (see "Unbound" + in the class docstring). Args: - source: Source node selector (required if unbound). Can be a - string pattern or a selector dict with path/group_by/match. - sink: Sink node selector (required if unbound). Can be a string - pattern or a selector dict with path/group_by/match. + source: Source node selector (required if unbound); see the class + docstring for the accepted selector forms. + sink: Sink node selector (required if unbound). mode: COMBINE or PAIRWISE (ignored if bound). shortest_path: If True, use only shortest paths (IP/IGP mode). require_capacity: If True (default), path selection considers @@ -407,12 +556,7 @@ def max_flow( ValueError: If unbound and source/sink not provided. ValueError: If bound and source/sink are provided. """ - if self.is_bound: - if source is not None or sink is not None: - raise ValueError( - "Bound context: source/sink already configured. " - "Create new context for different groups." - ) + if self._dispatch_bound(source, sink): return self._max_flow_bound( shortest_path=shortest_path, require_capacity=require_capacity, @@ -420,19 +564,17 @@ def max_flow( excluded_nodes=excluded_nodes, excluded_links=excluded_links, ) - else: - if source is None or sink is None: - raise ValueError("Unbound context: source and sink are required.") - return self._max_flow_unbound( - source=source, - sink=sink, - mode=mode, - shortest_path=shortest_path, - require_capacity=require_capacity, - flow_placement=flow_placement, - excluded_nodes=excluded_nodes, - excluded_links=excluded_links, - ) + assert source is not None and sink is not None + return self._max_flow_unbound( + source=source, + sink=sink, + mode=mode, + shortest_path=shortest_path, + require_capacity=require_capacity, + flow_placement=flow_placement, + excluded_nodes=excluded_nodes, + excluded_links=excluded_links, + ) def max_flow_detailed( self, @@ -449,11 +591,13 @@ def max_flow_detailed( ) -> Dict[Tuple[str, str], MaxFlowResult]: """Compute max flow with detailed results including cost distribution. + If unbound, each call rebuilds the graph (see "Unbound" in the class + docstring). + Args: - source: Source node selector (required if unbound). Can be a - string pattern or a selector dict with path/group_by/match. - sink: Sink node selector (required if unbound). Can be a string - pattern or a selector dict with path/group_by/match. + source: Source node selector (required if unbound); see the class + docstring for the accepted selector forms. + sink: Sink node selector (required if unbound). mode: COMBINE or PAIRWISE (ignored if bound). shortest_path: If True, restricts flow to shortest paths. require_capacity: If True (default), path selection considers @@ -466,9 +610,7 @@ def max_flow_detailed( Returns: Dict mapping (source_label, sink_label) to MaxFlowResult. """ - if self.is_bound: - if source is not None or sink is not None: - raise ValueError("Bound context: source/sink already configured.") + if self._dispatch_bound(source, sink): return self._max_flow_detailed_bound( shortest_path=shortest_path, require_capacity=require_capacity, @@ -477,20 +619,18 @@ def max_flow_detailed( excluded_links=excluded_links, include_min_cut=include_min_cut, ) - else: - if source is None or sink is None: - raise ValueError("Unbound context: source and sink are required.") - return self._max_flow_detailed_unbound( - source=source, - sink=sink, - mode=mode, - shortest_path=shortest_path, - require_capacity=require_capacity, - flow_placement=flow_placement, - excluded_nodes=excluded_nodes, - excluded_links=excluded_links, - include_min_cut=include_min_cut, - ) + assert source is not None and sink is not None + return self._max_flow_detailed_unbound( + source=source, + sink=sink, + mode=mode, + shortest_path=shortest_path, + require_capacity=require_capacity, + flow_placement=flow_placement, + excluded_nodes=excluded_nodes, + excluded_links=excluded_links, + include_min_cut=include_min_cut, + ) def sensitivity( self, @@ -509,11 +649,13 @@ def sensitivity( Identifies critical edges and computes the flow reduction caused by removing each one. + If unbound, each call rebuilds the graph (see "Unbound" in the class + docstring). + Args: - source: Source node selector (required if unbound). Can be a - string pattern or a selector dict with path/group_by/match. - sink: Sink node selector (required if unbound). Can be a string - pattern or a selector dict with path/group_by/match. + source: Source node selector (required if unbound); see the class + docstring for the accepted selector forms. + sink: Sink node selector (required if unbound). mode: COMBINE or PAIRWISE (ignored if bound). shortest_path: If True, use shortest-path-only flow (IP/IGP mode). require_capacity: If True (default), path selection considers @@ -525,9 +667,7 @@ def sensitivity( Returns: Dict mapping (source_label, sink_label) to {link_id:direction: flow_reduction}. """ - if self.is_bound: - if source is not None or sink is not None: - raise ValueError("Bound context: source/sink already configured.") + if self._dispatch_bound(source, sink): return self._sensitivity_bound( shortest_path=shortest_path, require_capacity=require_capacity, @@ -535,19 +675,79 @@ def sensitivity( excluded_nodes=excluded_nodes, excluded_links=excluded_links, ) - else: - if source is None or sink is None: - raise ValueError("Unbound context: source and sink are required.") - return self._sensitivity_unbound( - source=source, - sink=sink, - mode=mode, + assert source is not None and sink is not None + return self._sensitivity_unbound( + source=source, + sink=sink, + mode=mode, + shortest_path=shortest_path, + require_capacity=require_capacity, + flow_placement=flow_placement, + excluded_nodes=excluded_nodes, + excluded_links=excluded_links, + ) + + def sensitivity_with_flow( + self, + source: Optional[Union[str, Dict[str, Any]]] = None, + sink: Optional[Union[str, Dict[str, Any]]] = None, + *, + mode: Mode = Mode.COMBINE, + shortest_path: bool = False, + require_capacity: bool = True, + flow_placement: FlowPlacement = FlowPlacement.PROPORTIONAL, + excluded_nodes: Optional[Set[str]] = None, + excluded_links: Optional[Set[str]] = None, + ) -> Dict[Tuple[str, str], Tuple[float, Dict[str, float]]]: + """Compute max flow and edge sensitivity together per group pair. + + Produces the same values as calling max_flow and sensitivity with + identical arguments, but builds the node/edge masks once and walks + the group pairs once. Prefer this on repeated-analysis hot paths + (e.g., Monte Carlo iterations) when both results are needed. + + If unbound, each call rebuilds the graph (see "Unbound" in the class + docstring). + + Args: + source: Source node selector (required if unbound); see the class + docstring for the accepted selector forms. + sink: Sink node selector (required if unbound). + mode: COMBINE or PAIRWISE (ignored if bound). + shortest_path: If True, use shortest-path-only flow (IP/IGP mode). + require_capacity: If True (default), path selection considers + available capacity. If False, path selection is cost-only. + flow_placement: Flow placement strategy. + excluded_nodes: Nodes to exclude from this analysis. + excluded_links: Links to exclude from this analysis. + + Returns: + Dict mapping (source_label, sink_label) to a tuple of + (max flow value, {link_id:direction: flow_reduction}). + + Raises: + ValueError: If unbound and source/sink not provided. + ValueError: If bound and source/sink are provided. + """ + if self._dispatch_bound(source, sink): + return self._sensitivity_with_flow_bound( shortest_path=shortest_path, require_capacity=require_capacity, flow_placement=flow_placement, excluded_nodes=excluded_nodes, excluded_links=excluded_links, ) + assert source is not None and sink is not None + return self._sensitivity_with_flow_unbound( + source=source, + sink=sink, + mode=mode, + shortest_path=shortest_path, + require_capacity=require_capacity, + flow_placement=flow_placement, + excluded_nodes=excluded_nodes, + excluded_links=excluded_links, + ) # ────────────────────────────────────────────────────────────── # Path analysis methods @@ -569,10 +769,9 @@ def shortest_path_cost( groups. Otherwise source and sink arguments are required. Args: - source: Source node selector (required if unbound). Can be a - string pattern or a selector dict with path/group_by/match. - sink: Sink node selector (required if unbound). Can be a string - pattern or a selector dict with path/group_by/match. + source: Source node selector (required if unbound); see the class + docstring for the accepted selector forms. + sink: Sink node selector (required if unbound). mode: COMBINE or PAIRWISE (ignored if bound). edge_select: SPF edge selection strategy. excluded_nodes: Nodes to exclude from this analysis. @@ -616,10 +815,9 @@ def shortest_paths( groups. Otherwise source and sink arguments are required. Args: - source: Source node selector (required if unbound). Can be a - string pattern or a selector dict with path/group_by/match. - sink: Sink node selector (required if unbound). Can be a string - pattern or a selector dict with path/group_by/match. + source: Source node selector (required if unbound); see the class + docstring for the accepted selector forms. + sink: Sink node selector (required if unbound). mode: COMBINE or PAIRWISE (ignored if bound). edge_select: SPF edge selection strategy. split_parallel_edges: Expand parallel edges into distinct paths. @@ -666,13 +864,17 @@ def k_shortest_paths( groups. Otherwise source and sink arguments are required. Args: - source: Source node selector (required if unbound). Can be a - string pattern or a selector dict with path/group_by/match. - sink: Sink node selector (required if unbound). Can be a string - pattern or a selector dict with path/group_by/match. + source: Source node selector (required if unbound); see the class + docstring for the accepted selector forms. + sink: Sink node selector (required if unbound). mode: PAIRWISE (default) or COMBINE (ignored if bound). max_k: Maximum paths per pair. - edge_select: SPF/KSP edge selection strategy. + edge_select: SPF/KSP edge selection strategy. Note: it governs + only the pruning SPF pass; Core's KSP enumeration uses a + fixed internal selection (all parallel min-cost edges, + capacity-blind, deterministic tie-break), so SINGLE_MIN_COST + may yield one path per parallel edge where `shortest_paths` + returns one. max_path_cost: Absolute cost threshold. max_path_cost_factor: Relative threshold versus best path. split_parallel_edges: Expand parallel edges into distinct paths. @@ -727,21 +929,30 @@ def _resolve_source_sink( ValueError: If unbound and source/sink not provided. ValueError: If bound and source/sink are provided. """ - if self.is_bound: - if source is not None or sink is not None: - raise ValueError( - "Bound context: source/sink already configured. " - "Create new context for different groups." - ) + if self._dispatch_bound(source, sink): # Use bound values (can be str or dict) return self._source, self._sink, self._mode # type: ignore[return-value] - else: - if source is None or sink is None: - raise ValueError("Unbound context: source and sink are required.") - return source, sink, mode + assert source is not None and sink is not None + return source, sink, mode + + def build_node_mask(self, excluded_nodes: Optional[Set[str]] = None) -> np.ndarray: + """Build a node inclusion mask for Core algorithms. + + Core mask semantics: True includes the node, False excludes it. + Disabled nodes are always excluded, on top of any names passed in + ``excluded_nodes``. Building the mask costs an O(num_nodes) fill plus + O(|excluded| + |disabled|) updates, so it is cheap enough to redo per + failure iteration. Useful for custom analysis functions that call + Core primitives directly. + + Args: + excluded_nodes: Optional set of node names to exclude. Names not + present in the graph are ignored. - def _build_node_mask(self, excluded_nodes: Optional[Set[str]] = None) -> np.ndarray: - """Build node mask array for Core algorithms.""" + Returns: + Boolean numpy array of shape (num_nodes,) where True means + included. + """ num_nodes = len(self._node_mapper.node_names) mask = np.ones(num_nodes, dtype=bool) @@ -755,8 +966,26 @@ def _build_node_mask(self, excluded_nodes: Optional[Set[str]] = None) -> np.ndar return mask - def _build_edge_mask(self, excluded_links: Optional[Set[str]] = None) -> np.ndarray: - """Build edge mask array for Core algorithms.""" + def build_edge_mask(self, excluded_links: Optional[Set[str]] = None) -> np.ndarray: + """Build an edge inclusion mask for Core algorithms. + + Core mask semantics: True includes the edge, False excludes it. Edges + of disabled links are always excluded, on top of any IDs passed in + ``excluded_links``. Building the mask costs an O(num_edges) fill plus + O(|excluded| + |disabled|) updates, so it is cheap enough to redo per + failure iteration. Useful for custom analysis functions that call + Core primitives directly. + + Args: + excluded_links: Optional set of link IDs to exclude. IDs not + present in the graph are ignored. + + Returns: + Boolean numpy array of shape (num_edges,) where True means + included. There is one entry per Core edge - forward and reverse + direction of each link, plus any augmentation edges - not one + entry per link. + """ num_edges = self._multidigraph.num_edges() mask = np.ones(num_edges, dtype=bool) @@ -810,8 +1039,8 @@ def _max_flow_bound( ) -> Dict[Tuple[str, str], float]: """Max flow using pre-built pseudo nodes.""" core_flow_placement = self._map_flow_placement(flow_placement) - node_mask = self._build_node_mask(excluded_nodes) - edge_mask = self._build_edge_mask(excluded_links) + node_mask = self.build_node_mask(excluded_nodes) + edge_mask = self.build_edge_mask(excluded_links) pseudo_node_pairs = self._pseudo_context.pairs if self._pseudo_context else {} results: Dict[Tuple[str, str], float] = {} @@ -830,9 +1059,28 @@ def _max_flow_bound( results[pair_key] = flow_value # Fill missing pairs (overlapping src/snk) - self._fill_missing_pairs_bound(results, 0.0) + self._fill_missing_pairs_bound(results, lambda: 0.0) return results + def _bind_temp( + self, + source: Union[str, Dict[str, Any]], + sink: Union[str, Dict[str, Any]], + mode: Mode, + ) -> "AnalysisContext": + """Build a temporary bound context, preserving custom augmentations. + + Owns the re-binding semantics for all `_*_unbound` methods so state + that must survive re-binding (e.g. augmentations) is handled once. + """ + return AnalysisContext.from_network( + self._network, + source=source, + sink=sink, + mode=mode, + augmentations=list(self._augmentations) if self._augmentations else None, + ) + def _max_flow_unbound( self, *, @@ -846,11 +1094,7 @@ def _max_flow_unbound( excluded_links: Optional[Set[str]], ) -> Dict[Tuple[str, str], float]: """Max flow building pseudo nodes on demand.""" - # Build a temporary bound context - temp_ctx = AnalysisContext.from_network( - self._network, source=source, sink=sink, mode=mode - ) - return temp_ctx._max_flow_bound( + return self._bind_temp(source, sink, mode)._max_flow_bound( shortest_path=shortest_path, require_capacity=require_capacity, flow_placement=flow_placement, @@ -870,8 +1114,8 @@ def _max_flow_detailed_bound( ) -> Dict[Tuple[str, str], MaxFlowResult]: """Detailed max flow using pre-built pseudo nodes.""" core_flow_placement = self._map_flow_placement(flow_placement) - node_mask = self._build_node_mask(excluded_nodes) - edge_mask = self._build_edge_mask(excluded_links) + node_mask = self.build_node_mask(excluded_nodes) + edge_mask = self.build_edge_mask(excluded_links) ext_edge_ids = self._multidigraph.ext_edge_ids_view() pseudo_node_pairs = self._pseudo_context.pairs if self._pseudo_context else {} @@ -891,19 +1135,11 @@ def _max_flow_detailed_bound( min_cut_edges: Optional[Tuple[EdgeRef, ...]] = None if include_min_cut: - sens_results = self._algorithms.sensitivity_analysis( - self._handle, - pseudo_src_id, - pseudo_snk_id, - flow_placement=core_flow_placement, - shortest_path=shortest_path, - require_capacity=require_capacity, - node_mask=node_mask, - edge_mask=edge_mask, - ) + # core_summary.min_cut holds the true minimum cut; pseudo + # edges (ext id -1) are filtered out by decode_ext_id. edge_refs: List[EdgeRef] = [] - for edge_id, _delta in sens_results: - ext_id = ext_edge_ids[edge_id] + for edge_id in core_summary.min_cut.edges: + ext_id = ext_edge_ids[int(edge_id)] edge_ref = self._edge_mapper.decode_ext_id(int(ext_id)) if edge_ref is not None: edge_refs.append(edge_ref) @@ -914,7 +1150,7 @@ def _max_flow_detailed_bound( ) # Fill missing pairs - self._fill_missing_pairs_bound(results, _construct_max_flow_result(0.0)) + self._fill_missing_pairs_bound(results, lambda: _construct_max_flow_result(0.0)) return results def _max_flow_detailed_unbound( @@ -931,10 +1167,7 @@ def _max_flow_detailed_unbound( include_min_cut: bool, ) -> Dict[Tuple[str, str], MaxFlowResult]: """Detailed max flow building pseudo nodes on demand.""" - temp_ctx = AnalysisContext.from_network( - self._network, source=source, sink=sink, mode=mode - ) - return temp_ctx._max_flow_detailed_bound( + return self._bind_temp(source, sink, mode)._max_flow_detailed_bound( shortest_path=shortest_path, require_capacity=require_capacity, flow_placement=flow_placement, @@ -943,6 +1176,18 @@ def _max_flow_detailed_unbound( include_min_cut=include_min_cut, ) + def _decode_sensitivity_map( + self, sens_results: Any, ext_edge_ids: Any + ) -> Dict[str, float]: + """Decode core sensitivity results into a {"link_id:direction": delta} map.""" + sensitivity_map: Dict[str, float] = {} + for edge_id, delta in sens_results: + ext_id = ext_edge_ids[edge_id] + edge_ref = self._edge_mapper.decode_ext_id(int(ext_id)) + if edge_ref is not None: + sensitivity_map[f"{edge_ref.link_id}:{edge_ref.direction}"] = delta + return sensitivity_map + def _sensitivity_bound( self, *, @@ -954,8 +1199,8 @@ def _sensitivity_bound( ) -> Dict[Tuple[str, str], Dict[str, float]]: """Sensitivity analysis using pre-built pseudo nodes.""" core_flow_placement = self._map_flow_placement(flow_placement) - node_mask = self._build_node_mask(excluded_nodes) - edge_mask = self._build_edge_mask(excluded_links) + node_mask = self.build_node_mask(excluded_nodes) + edge_mask = self.build_edge_mask(excluded_links) ext_edge_ids = self._multidigraph.ext_edge_ids_view() pseudo_node_pairs = self._pseudo_context.pairs if self._pseudo_context else {} @@ -973,17 +1218,9 @@ def _sensitivity_bound( edge_mask=edge_mask, ) - sensitivity_map: Dict[str, float] = {} - for edge_id, delta in sens_results: - ext_id = ext_edge_ids[edge_id] - edge_ref = self._edge_mapper.decode_ext_id(int(ext_id)) - if edge_ref is not None: - key = f"{edge_ref.link_id}:{edge_ref.direction}" - sensitivity_map[key] = delta - - results[pair_key] = sensitivity_map + results[pair_key] = self._decode_sensitivity_map(sens_results, ext_edge_ids) - self._fill_missing_pairs_bound(results, {}) + self._fill_missing_pairs_bound(results, lambda: {}) return results def _sensitivity_unbound( @@ -999,10 +1236,7 @@ def _sensitivity_unbound( excluded_links: Optional[Set[str]], ) -> Dict[Tuple[str, str], Dict[str, float]]: """Sensitivity analysis building pseudo nodes on demand.""" - temp_ctx = AnalysisContext.from_network( - self._network, source=source, sink=sink, mode=mode - ) - return temp_ctx._sensitivity_bound( + return self._bind_temp(source, sink, mode)._sensitivity_bound( shortest_path=shortest_path, require_capacity=require_capacity, flow_placement=flow_placement, @@ -1010,29 +1244,98 @@ def _sensitivity_unbound( excluded_links=excluded_links, ) - def _fill_missing_pairs_bound(self, results: Dict, default_value) -> None: - """Fill results for pairs not in the graph (e.g., overlapping).""" - if not self._pseudo_context: - return + def _sensitivity_with_flow_bound( + self, + *, + shortest_path: bool, + require_capacity: bool, + flow_placement: FlowPlacement, + excluded_nodes: Optional[Set[str]], + excluded_links: Optional[Set[str]], + ) -> Dict[Tuple[str, str], Tuple[float, Dict[str, float]]]: + """Max flow plus sensitivity in one pass over pre-built pseudo nodes. + + Builds node/edge masks once and iterates the bound pairs once + instead of duplicating that work across separate max_flow and + sensitivity calls. Core still computes the baseline flow internally + for the sensitivity deltas; only the Python-side duplication + (masks and pair iteration) is removed here. + """ + core_flow_placement = self._map_flow_placement(flow_placement) + node_mask = self.build_node_mask(excluded_nodes) + edge_mask = self.build_edge_mask(excluded_links) + ext_edge_ids = self._multidigraph.ext_edge_ids_view() + + pseudo_node_pairs = self._pseudo_context.pairs if self._pseudo_context else {} + results: Dict[Tuple[str, str], Tuple[float, Dict[str, float]]] = {} - from ngraph.dsl.selectors import normalize_selector, select_nodes + for pair_key, (pseudo_src_id, pseudo_snk_id) in pseudo_node_pairs.items(): + flow_value, _ = self._algorithms.max_flow( + self._handle, + pseudo_src_id, + pseudo_snk_id, + flow_placement=core_flow_placement, + shortest_path=shortest_path, + require_capacity=require_capacity, + node_mask=node_mask, + edge_mask=edge_mask, + ) - src_selector = normalize_selector(self._source or "", "workflow") - snk_selector = normalize_selector(self._sink or "", "workflow") + sens_results = self._algorithms.sensitivity_analysis( + self._handle, + pseudo_src_id, + pseudo_snk_id, + flow_placement=core_flow_placement, + shortest_path=shortest_path, + require_capacity=require_capacity, + node_mask=node_mask, + edge_mask=edge_mask, + ) - src_groups = select_nodes(self._network, src_selector, default_active_only=True) - snk_groups = select_nodes(self._network, snk_selector, default_active_only=True) + results[pair_key] = ( + flow_value, + self._decode_sensitivity_map(sens_results, ext_edge_ids), + ) - if self._mode == Mode.COMBINE: - combined_src_label = "|".join(sorted(src_groups.keys())) - combined_snk_label = "|".join(sorted(snk_groups.keys())) - if (combined_src_label, combined_snk_label) not in results: - results[(combined_src_label, combined_snk_label)] = default_value - elif self._mode == Mode.PAIRWISE: - for src_label in src_groups: - for snk_label in snk_groups: - if (src_label, snk_label) not in results: - results[(src_label, snk_label)] = default_value + self._fill_missing_pairs_bound(results, lambda: (0.0, {})) + return results + + def _sensitivity_with_flow_unbound( + self, + *, + source: Union[str, Dict[str, Any]], + sink: Union[str, Dict[str, Any]], + mode: Mode, + shortest_path: bool, + require_capacity: bool, + flow_placement: FlowPlacement, + excluded_nodes: Optional[Set[str]], + excluded_links: Optional[Set[str]], + ) -> Dict[Tuple[str, str], Tuple[float, Dict[str, float]]]: + """Max flow plus sensitivity building pseudo nodes on demand.""" + return self._bind_temp(source, sink, mode)._sensitivity_with_flow_bound( + shortest_path=shortest_path, + require_capacity=require_capacity, + flow_placement=flow_placement, + excluded_nodes=excluded_nodes, + excluded_links=excluded_links, + ) + + def _fill_missing_pairs_bound( + self, results: Dict, default_factory: Callable[[], Any] + ) -> None: + """Fill results for pairs not in the graph (e.g., overlapping). + + Uses pair keys precomputed at bind time; no node selection is + re-run (the context is immutable after creation). The factory is + called once per missing pair so mutable defaults (dicts, result + objects) are never aliased across pairs. + """ + if not self._pseudo_context: + return + for pair_key in self._pseudo_context.expected_pairs: + if pair_key not in results: + results[pair_key] = default_factory() def _shortest_path_costs_impl( self, @@ -1045,88 +1348,54 @@ def _shortest_path_costs_impl( excluded_links: Optional[Set[str]], ) -> Dict[Tuple[str, str], float]: """Implementation of shortest_path_cost.""" - from ngraph.dsl.selectors import normalize_selector, select_nodes - - src_selector = normalize_selector(source, "workflow") - snk_selector = normalize_selector(sink, "workflow") - src_groups = select_nodes(self._network, src_selector, default_active_only=True) - snk_groups = select_nodes(self._network, snk_selector, default_active_only=True) + src_groups, snk_groups = _resolve_selector_groups(self._network, source, sink) - if not src_groups: - raise ValueError(f"No source nodes found matching '{source}'.") - if not snk_groups: - raise ValueError(f"No sink nodes found matching '{sink}'.") - - node_mask = self._build_node_mask(excluded_nodes) - edge_mask = self._build_edge_mask(excluded_links) + node_mask = self.build_node_mask(excluded_nodes) + edge_mask = self.build_edge_mask(excluded_links) core_edge_select = self._map_edge_select(edge_select) - if mode == Mode.COMBINE: - combined_src_label = "|".join(sorted(src_groups.keys())) - combined_snk_label = "|".join(sorted(snk_groups.keys())) - - combined_src_names = [] - for group_nodes in src_groups.values(): - combined_src_names.extend( - _get_active_node_names(group_nodes, excluded_nodes) - ) - combined_snk_names = [] - for group_nodes in snk_groups.values(): - combined_snk_names.extend( - _get_active_node_names(group_nodes, excluded_nodes) - ) - - if not combined_src_names or not combined_snk_names: - return {(combined_src_label, combined_snk_label): float("inf")} - if set(combined_src_names) & set(combined_snk_names): - return {(combined_src_label, combined_snk_label): float("inf")} + def _best_cost_for_groups(src_names: List[str], snk_names: List[str]) -> float: + if not src_names or not snk_names: + return float("inf") + if set(src_names) & set(snk_names): + return float("inf") best_cost = float("inf") - for src_name in combined_src_names: - src_id = self._node_mapper.to_id(src_name) + for src_name in src_names: dists, _ = self._algorithms.spf( self._handle, - src=src_id, + src=self._node_mapper.to_id(src_name), selection=core_edge_select, node_mask=node_mask, edge_mask=edge_mask, ) - for snk_name in combined_snk_names: - snk_id = self._node_mapper.to_id(snk_name) - cost = dists[snk_id] + for snk_name in snk_names: + cost = dists[self._node_mapper.to_id(snk_name)] if cost < best_cost: best_cost = cost - return {(combined_src_label, combined_snk_label): best_cost} + return best_cost + + if mode == Mode.COMBINE: + combined_src_label, combined_src_names = _combined_group_names( + src_groups, excluded_nodes + ) + combined_snk_label, combined_snk_names = _combined_group_names( + snk_groups, excluded_nodes + ) + return { + (combined_src_label, combined_snk_label): _best_cost_for_groups( + combined_src_names, combined_snk_names + ) + } if mode == Mode.PAIRWISE: results: Dict[Tuple[str, str], float] = {} for src_label, src_nodes in src_groups.items(): for snk_label, snk_nodes in snk_groups.items(): - active_src_names = _get_active_node_names(src_nodes, excluded_nodes) - active_snk_names = _get_active_node_names(snk_nodes, excluded_nodes) - if not active_src_names or not active_snk_names: - results[(src_label, snk_label)] = float("inf") - continue - if set(active_src_names) & set(active_snk_names): - results[(src_label, snk_label)] = float("inf") - continue - - best_cost = float("inf") - for src_name in active_src_names: - src_id = self._node_mapper.to_id(src_name) - dists, _ = self._algorithms.spf( - self._handle, - src=src_id, - selection=core_edge_select, - node_mask=node_mask, - edge_mask=edge_mask, - ) - for snk_name in active_snk_names: - snk_id = self._node_mapper.to_id(snk_name) - cost = dists[snk_id] - if cost < best_cost: - best_cost = cost - results[(src_label, snk_label)] = best_cost + results[(src_label, snk_label)] = _best_cost_for_groups( + _get_active_node_names(src_nodes, excluded_nodes), + _get_active_node_names(snk_nodes, excluded_nodes), + ) return results raise ValueError(f"Invalid mode '{mode}'.") @@ -1143,20 +1412,10 @@ def _shortest_paths_impl( excluded_links: Optional[Set[str]], ) -> Dict[Tuple[str, str], List[Path]]: """Implementation of shortest_paths.""" - from ngraph.dsl.selectors import normalize_selector, select_nodes + src_groups, snk_groups = _resolve_selector_groups(self._network, source, sink) - src_selector = normalize_selector(source, "workflow") - snk_selector = normalize_selector(sink, "workflow") - src_groups = select_nodes(self._network, src_selector, default_active_only=True) - snk_groups = select_nodes(self._network, snk_selector, default_active_only=True) - - if not src_groups: - raise ValueError(f"No source nodes found matching '{source}'.") - if not snk_groups: - raise ValueError(f"No sink nodes found matching '{sink}'.") - - node_mask = self._build_node_mask(excluded_nodes) - edge_mask = self._build_edge_mask(excluded_links) + node_mask = self.build_node_mask(excluded_nodes) + edge_mask = self.build_edge_mask(excluded_links) core_edge_select = self._map_edge_select(edge_select) def _best_paths_for_groups( @@ -1211,23 +1470,16 @@ def _best_paths_for_groups( ) if best_paths: - best_paths = sorted(set(best_paths)) + best_paths = sorted(set(best_paths), key=_path_sort_key) return best_paths if mode == Mode.COMBINE: - combined_src_label = "|".join(sorted(src_groups.keys())) - combined_snk_label = "|".join(sorted(snk_groups.keys())) - - combined_src_names = [] - for group_nodes in src_groups.values(): - combined_src_names.extend( - _get_active_node_names(group_nodes, excluded_nodes) - ) - combined_snk_names = [] - for group_nodes in snk_groups.values(): - combined_snk_names.extend( - _get_active_node_names(group_nodes, excluded_nodes) - ) + combined_src_label, combined_src_names = _combined_group_names( + src_groups, excluded_nodes + ) + combined_snk_label, combined_snk_names = _combined_group_names( + snk_groups, excluded_nodes + ) paths_list = _best_paths_for_groups(combined_src_names, combined_snk_names) return {(combined_src_label, combined_snk_label): paths_list} @@ -1260,20 +1512,10 @@ def _k_shortest_paths_impl( excluded_links: Optional[Set[str]], ) -> Dict[Tuple[str, str], List[Path]]: """Implementation of k_shortest_paths.""" - from ngraph.dsl.selectors import normalize_selector, select_nodes - - src_selector = normalize_selector(source, "workflow") - snk_selector = normalize_selector(sink, "workflow") - src_groups = select_nodes(self._network, src_selector, default_active_only=True) - snk_groups = select_nodes(self._network, snk_selector, default_active_only=True) - - if not src_groups: - raise ValueError(f"No source nodes found matching '{source}'.") - if not snk_groups: - raise ValueError(f"No sink nodes found matching '{sink}'.") + src_groups, snk_groups = _resolve_selector_groups(self._network, source, sink) - node_mask = self._build_node_mask(excluded_nodes) - edge_mask = self._build_edge_mask(excluded_links) + node_mask = self.build_node_mask(excluded_nodes) + edge_mask = self.build_edge_mask(excluded_links) core_edge_select = self._map_edge_select(edge_select) def _ksp_for_groups(src_names: List[str], snk_names: List[str]) -> List[Path]: @@ -1282,8 +1524,8 @@ def _ksp_for_groups(src_names: List[str], snk_names: List[str]) -> List[Path]: if set(src_names) & set(snk_names): return [] - # Find best pair - best_pair: Optional[Tuple[str, str]] = None + # SPF pass: per-pair shortest costs and the global best cost + pair_costs: Dict[Tuple[str, str], float] = {} best_cost = float("inf") for src_name in src_names: src_id = self._node_mapper.to_id(src_name) @@ -1297,67 +1539,86 @@ def _ksp_for_groups(src_names: List[str], snk_names: List[str]) -> List[Path]: for snk_name in snk_names: snk_id = self._node_mapper.to_id(snk_name) cost = dists[snk_id] + if cost == float("inf"): + continue + pair_costs[(src_name, snk_name)] = cost if cost < best_cost: best_cost = cost - best_pair = (src_name, snk_name) - if best_pair is None: + if not pair_costs: return [] - src_name, snk_name = best_pair - src_id = self._node_mapper.to_id(src_name) - snk_id = self._node_mapper.to_id(snk_name) - - results: List[Path] = [] - count = 0 - - for dists, pred_dag in self._algorithms.ksp( - self._handle, - src=src_id, - dst=snk_id, - k=max_k, - max_cost_factor=max_path_cost_factor, - node_mask=node_mask, - edge_mask=edge_mask, + # Absolute cost cap; max_path_cost_factor is relative to the + # best cost across ALL pairs, so per-pair KSP runs unbounded + # (max_cost_factor=None) and paths are filtered by this cap. + cost_cap = max_path_cost + if max_path_cost_factor is not None: + cost_cap = min(cost_cap, best_cost * max_path_cost_factor) + + # KSP per reachable pair within the cap, cheapest pairs first. + # Every path for a pair costs at least that pair's shortest + # cost, so once max_k unique paths are collected the loop stops + # as soon as the next pair's shortest cost strictly exceeds the + # current k-th best path cost: no remaining pair can contribute + # a path that survives the final top-k truncation. Pairs tied + # at the boundary are still explored so equal-cost truncation + # stays deterministic via _path_sort_key. + merged: Set[Path] = set() + kth_best_cost = float("inf") + for (src_name, snk_name), pair_cost in sorted( + pair_costs.items(), key=lambda kv: (kv[1], kv[0]) ): - cost = dists[snk_id] - if cost == float("inf") or cost > max_path_cost: - continue - for path in _extract_paths_from_pred_dag( - pred_dag, - src_name, - snk_name, - cost, - self._node_mapper, - self._edge_mapper, - self._multidigraph, - split_parallel_edges, + if pair_cost > cost_cap: + break # Pairs are cost-sorted; the rest exceed the cap too + if len(merged) >= max_k and pair_cost > kth_best_cost: + break + src_id = self._node_mapper.to_id(src_name) + snk_id = self._node_mapper.to_id(snk_name) + + count = 0 + for dists, pred_dag in self._algorithms.ksp( + self._handle, + src=src_id, + dst=snk_id, + k=max_k, + max_cost_factor=None, + node_mask=node_mask, + edge_mask=edge_mask, ): - results.append(path) - count += 1 + cost = dists[snk_id] + if cost == float("inf"): + continue + if cost > cost_cap: + break # KSP yields costs in non-decreasing order + for path in _extract_paths_from_pred_dag( + pred_dag, + src_name, + snk_name, + cost, + self._node_mapper, + self._edge_mapper, + self._multidigraph, + split_parallel_edges, + ): + merged.add(path) + count += 1 + if count >= max_k: + break if count >= max_k: break - if count >= max_k: - break - if results: - results = sorted(set(results))[:max_k] - return results + if len(merged) >= max_k: + kth_best_cost = sorted(p.cost for p in merged)[max_k - 1] - if mode == Mode.COMBINE: - combined_src_label = "|".join(sorted(src_groups.keys())) - combined_snk_label = "|".join(sorted(snk_groups.keys())) + return sorted(merged, key=_path_sort_key)[:max_k] - combined_src_names = [] - for group_nodes in src_groups.values(): - combined_src_names.extend( - _get_active_node_names(group_nodes, excluded_nodes) - ) - combined_snk_names = [] - for group_nodes in snk_groups.values(): - combined_snk_names.extend( - _get_active_node_names(group_nodes, excluded_nodes) - ) + if mode == Mode.COMBINE: + combined_src_label, combined_src_names = _combined_group_names( + src_groups, excluded_nodes + ) + combined_snk_label, combined_snk_names = _combined_group_names( + snk_groups, excluded_nodes + ) return { (combined_src_label, combined_snk_label): _ksp_for_groups( @@ -1384,45 +1645,55 @@ def _ksp_for_groups(src_names: List[str], snk_names: List[str]) -> List[Path]: # ────────────────────────────────────────────────────────────────────────────── +def _path_sort_key(path: Path) -> Tuple[Any, ...]: + """Deterministic total-order sort key for Path objects. + + Path.__lt__ compares cost only, so sorting equal-cost paths with the + default ordering preserves set-iteration order, which varies with + string-hash randomization (PYTHONHASHSEED). This key orders by cost, + then node sequence, then the structural edge sequence (link ids and + directions). Any two distinct paths differ in the key, so sorted + output and equal-cost truncation (e.g., k_shortest_paths max_k) are + independent of hash order. Paths that differ only in which parallel + link they traverse still order by link id, which embeds a build-time + UUID; node-sequence-level selection is stable across runs. + """ + return ( + path.cost, + path.nodes_seq, + tuple( + tuple((edge.link_id, edge.direction) for edge in edges) for _, edges in path + ), + ) + + def _build_pseudo_node_augmentations( network: "Network", source: Union[str, Dict[str, Any]], sink: Union[str, Dict[str, Any]], mode: Mode, -) -> Tuple[List[AugmentationEdge], Dict[Tuple[str, str], Tuple[str, str]]]: - """Build augmentation edges for pseudo source/sink nodes.""" - from ngraph.dsl.selectors import normalize_selector, select_nodes - - # Normalize selectors and select nodes - src_selector = normalize_selector(source, "workflow") - snk_selector = normalize_selector(sink, "workflow") - - # select_nodes returns Dict[str, List[Node]] with active_only=True by context - src_groups = select_nodes(network, src_selector, default_active_only=True) - snk_groups = select_nodes(network, snk_selector, default_active_only=True) - - if not src_groups: - raise ValueError(f"No source nodes found matching '{source}'.") - if not snk_groups: - raise ValueError(f"No sink nodes found matching '{sink}'.") +) -> Tuple[ + List[AugmentationEdge], + Dict[Tuple[str, str], Tuple[str, str]], + Tuple[Tuple[str, str], ...], +]: + """Build augmentation edges for pseudo source/sink nodes. - # Helper to get node names from groups - def _get_names(groups: Dict[str, List[Any]]) -> List[str]: - names: List[str] = [] - for nodes in groups.values(): - for node in nodes: - names.append(node.name) - return names + Returns: + Tuple of (augmentation edges, pair -> pseudo node names for pairs + materialized in the graph, all expected pair keys including pairs + skipped for empty or overlapping groups). + """ + src_groups, snk_groups = _resolve_selector_groups(network, source, sink) augmentations: List[AugmentationEdge] = [] pair_to_pseudo_names: Dict[Tuple[str, str], Tuple[str, str]] = {} + expected_pairs: Tuple[Tuple[str, str], ...] if mode == Mode.COMBINE: - combined_src_label = "|".join(sorted(src_groups.keys())) - combined_snk_label = "|".join(sorted(snk_groups.keys())) - - combined_src_names = _get_names(src_groups) - combined_snk_names = _get_names(snk_groups) + combined_src_label, combined_src_names = _combined_group_names(src_groups) + combined_snk_label, combined_snk_names = _combined_group_names(snk_groups) + expected_pairs = ((combined_src_label, combined_snk_label),) has_overlap = bool(set(combined_src_names) & set(combined_snk_names)) @@ -1445,34 +1716,64 @@ def _get_names(groups: Dict[str, List[Any]]) -> List[str]: ) elif mode == Mode.PAIRWISE: - for src_label, src_nodes in src_groups.items(): - for snk_label, snk_nodes in snk_groups.items(): - active_src_names = [n.name for n in src_nodes] - active_snk_names = [n.name for n in snk_nodes] + expected_pairs = tuple( + (src_label, snk_label) + for src_label in src_groups + for snk_label in snk_groups + ) - if set(active_src_names) & set(active_snk_names): + src_names_of = { + label: [n.name for n in nodes] for label, nodes in src_groups.items() + } + snk_names_of = { + label: [n.name for n in nodes] for label, nodes in snk_groups.items() + } + + # Hoisted out of the pair loop: one set per group, not one per pair. + src_name_sets = {label: set(names) for label, names in src_names_of.items()} + snk_name_sets = {label: set(names) for label, names in snk_names_of.items()} + + # Pass 1: determine valid pairs and which groups participate in + # at least one valid pair (no pseudo nodes for orphan groups). + participating_src: Set[str] = set() + participating_snk: Set[str] = set() + for src_label, src_names in src_names_of.items(): + for snk_label, snk_names in snk_names_of.items(): + if not src_names or not snk_names: continue - if not active_src_names or not active_snk_names: + if src_name_sets[src_label] & snk_name_sets[snk_label]: continue - pseudo_src = f"__PSEUDO_SRC_{src_label}__" - pseudo_snk = f"__PSEUDO_SNK_{snk_label}__" - - for src_name in active_src_names: - augmentations.append( - AugmentationEdge(pseudo_src, src_name, LARGE_CAPACITY, 0) - ) - for snk_name in active_snk_names: - augmentations.append( - AugmentationEdge(snk_name, pseudo_snk, LARGE_CAPACITY, 0) - ) - - pair_to_pseudo_names[(src_label, snk_label)] = (pseudo_src, pseudo_snk) + pair_to_pseudo_names[(src_label, snk_label)] = ( + f"__PSEUDO_SRC_{src_label}__", + f"__PSEUDO_SNK_{snk_label}__", + ) + participating_src.add(src_label) + participating_snk.add(snk_label) + + # Pass 2: emit attachment edges once per participating group + # member (not once per opposing group). + for src_label, src_names in src_names_of.items(): + if src_label not in participating_src: + continue + pseudo_src = f"__PSEUDO_SRC_{src_label}__" + for src_name in src_names: + augmentations.append( + AugmentationEdge(pseudo_src, src_name, LARGE_CAPACITY, 0) + ) + for snk_label, snk_names in snk_names_of.items(): + if snk_label not in participating_snk: + continue + pseudo_snk = f"__PSEUDO_SNK_{snk_label}__" + for snk_name in snk_names: + augmentations.append( + AugmentationEdge(snk_name, pseudo_snk, LARGE_CAPACITY, 0) + ) else: raise ValueError(f"Invalid mode '{mode}'.") - return augmentations, pair_to_pseudo_names + return augmentations, pair_to_pseudo_names, expected_pairs @dataclass @@ -1496,7 +1797,6 @@ def _build_graph_core( augmentations: Optional[List[AugmentationEdge]] = None, ) -> _GraphBuildResult: """Build Core graph infrastructure from Network.""" - # Identify real nodes real_node_names = set(network.nodes.keys()) # Infer pseudo nodes from augmentations @@ -1512,11 +1812,9 @@ def _build_graph_core( all_node_names = sorted(real_node_names) + sorted(pseudo_node_names) node_mapper = _NodeMapper(all_node_names) - # Build edge mapper link_ids = sorted(network.links.keys()) edge_mapper = _EdgeMapper(link_ids) - # Build edge arrays src_list: List[int] = [] dst_list: List[int] = [] capacity_list: List[float] = [] @@ -1554,14 +1852,60 @@ def _build_graph_core( cost_list.append(aug_edge.cost) ext_edge_id_list.append(-1) # Sentinel: not a network edge - # Convert to numpy arrays src_arr = np.array(src_list, dtype=np.int32) dst_arr = np.array(dst_list, dtype=np.int32) capacity_arr = np.array(capacity_list, dtype=np.float64) - cost_arr = np.array(cost_list, dtype=np.int64) + + # Pseudo attachment edges carry LARGE_CAPACITY; a real capacity at or + # above it would be silently clamped by those edges in combine-mode + # flows, so reject it loudly instead. + oversized = [ + link_id + for link_id in link_ids + if float(network.links[link_id].capacity) >= LARGE_CAPACITY + ] + if oversized: + raise ValueError( + f"Link capacities must be below {LARGE_CAPACITY:g} (the internal " + f"pseudo-edge capacity); offending links: {', '.join(oversized[:5])}" + ) + + # Core requires int64 costs; validate integrality instead of silently + # truncating (which would corrupt SPF/flow results). + cost_f = np.asarray(cost_list, dtype=np.float64) + # Core SPF uses int64 costs with INT64_MAX as the unreachable sentinel and + # computes d_u + edge_cost without overflow checks, so ACCUMULATED path + # costs must stay below 2**62 (a reverse-edge bounce can double a cost of + # 2**62 past INT64_MAX and wrap negative). Bounding the total of all edge + # costs bounds every simple path. + if np.any(cost_f < 0) or float(cost_f.sum()) >= 2**62: + raise ValueError( + "Link costs must be non-negative and their total must stay below " + "2**62: larger accumulated path costs overflow the core engine's " + "int64 cost arithmetic and silently corrupt results" + ) + if not np.array_equal(cost_f, np.trunc(cost_f)): + bad_indices = np.nonzero(cost_f != np.trunc(cost_f))[0] + edges_per_link = 2 if add_reverse else 1 + num_link_edges = len(link_ids) * edges_per_link + offenders: List[str] = [] + for idx in bad_indices: + if idx < num_link_edges: + link_id = link_ids[int(idx) // edges_per_link] + desc = f"link {link_id!r} (cost {network.links[link_id].cost})" + else: + aug = (augmentations or [])[int(idx) - num_link_edges] + desc = f"augmentation {aug.source!r}->{aug.target!r} (cost {aug.cost})" + if desc not in offenders: + offenders.append(desc) + raise ValueError( + "Non-integer link costs are not supported by the analysis engine " + f"(costs are int64): {', '.join(offenders)}" + ) + cost_arr = cost_f.astype(np.int64) + ext_edge_ids_arr = np.array(ext_edge_id_list, dtype=np.int64) - # Build StrictMultiDiGraph multidigraph = netgraph_core.StrictMultiDiGraph.from_arrays( num_nodes=len(all_node_names), src=src_arr, @@ -1571,7 +1915,6 @@ def _build_graph_core( ext_edge_ids=ext_edge_ids_arr, ) - # Build Core graph handle backend = netgraph_core.Backend.cpu() algorithms = netgraph_core.Algorithms(backend) handle = algorithms.build_graph(multidigraph) @@ -1598,7 +1941,6 @@ def _build_graph_core( if edge_ref: link_id_to_edge_indices.setdefault(edge_ref.link_id, []).append(edge_idx) - # Convert to immutable structures frozen_link_id_to_edge_indices = { k: tuple(v) for k, v in link_id_to_edge_indices.items() } @@ -1680,44 +2022,6 @@ def _extract_paths_from_pred_dag( # ────────────────────────────────────────────────────────────────────────────── -def build_node_mask( - ctx: AnalysisContext, - excluded_nodes: Optional[Set[str]] = None, -) -> np.ndarray: - """Build a node mask array for Core algorithms. - - Uses O(|excluded| + |disabled|) time complexity. - Core semantics: True = include, False = exclude. - - Args: - ctx: AnalysisContext with pre-computed disabled node IDs. - excluded_nodes: Optional set of node names to exclude. - - Returns: - Boolean numpy array of shape (num_nodes,) where True means included. - """ - return ctx._build_node_mask(excluded_nodes) - - -def build_edge_mask( - ctx: AnalysisContext, - excluded_links: Optional[Set[str]] = None, -) -> np.ndarray: - """Build an edge mask array for Core algorithms. - - Uses O(|excluded| + |disabled|) time complexity. - Core semantics: True = include, False = exclude. - - Args: - ctx: AnalysisContext with pre-computed edge index mapping. - excluded_links: Optional set of link IDs to exclude. - - Returns: - Boolean numpy array of shape (num_edges,) where True means included. - """ - return ctx._build_edge_mask(excluded_links) - - def analyze( network: "Network", *, @@ -1728,13 +2032,13 @@ def analyze( ) -> AnalysisContext: """Create an analysis context for the network. - This is THE primary entry point for network analysis in NetGraph. + Primary entry point for network analysis in NetGraph. Args: network: Network topology to analyze. source: Optional source node selector (string path or selector dict). - If provided with sink, creates bound context with pre-built - pseudo-nodes for efficient repeated flow analysis. + If provided with sink, creates a bound context whose pseudo + nodes are pre-built once and reused by every flow call. sink: Optional sink node selector (string path or selector dict). mode: Group mode (COMBINE or PAIRWISE). Only used if bound. augmentations: Optional custom augmentation edges. @@ -1742,13 +2046,30 @@ def analyze( Returns: AnalysisContext ready for analysis calls. + Raises: + ValueError: If only one of source/sink is provided, or if a bound + selector matches no nodes. + ValueError: If any link capacity is at or above LARGE_CAPACITY (1e15, + the internal pseudo-edge capacity), since such a link would be + silently clamped by the pseudo attachment edges in combine-mode + flows. + ValueError: If any link or augmentation cost is negative or + non-integer, or if the total of all edge costs reaches 2**62. + Core's int64 cost arithmetic would overflow and silently corrupt + SPF and flow results. + + Note: + The capacity and cost checks run during the graph build, which happens + here for bound contexts and for contexts with custom augmentations, + and on first use otherwise. + Examples: One-off analysis (unbound context): flow = analyze(network).max_flow("^A$", "^B$") paths = analyze(network).shortest_paths("^A$", "^B$") - Efficient repeated analysis (bound context): + Repeated analysis over one prepared graph (bound context): ctx = analyze(network, source="^dc/", sink="^edge/") baseline = ctx.max_flow() diff --git a/ngraph/analysis/demand.py b/ngraph/analysis/demand.py index bf0f841..3fe2702 100644 --- a/ngraph/analysis/demand.py +++ b/ngraph/analysis/demand.py @@ -1,7 +1,8 @@ """Demand expansion: converts TrafficDemand specs into concrete placement demands. -Supports both pairwise and combine modes through augmentation-based pseudo nodes. -Uses unified selectors for node selection. +Combine mode aggregates each side behind augmentation-based pseudo nodes; +pairwise mode emits one demand per (source, target) pair. Endpoints are +resolved through the shared selector layer. """ from __future__ import annotations @@ -10,10 +11,11 @@ from typing import Dict, List from ngraph.analysis.context import LARGE_CAPACITY, AugmentationEdge -from ngraph.dsl.selectors import normalize_selector, select_nodes +from ngraph.dsl.selectors import normalize_selector from ngraph.model.demand.spec import TrafficDemand from ngraph.model.flow.policy_config import FlowPolicyPreset from ngraph.model.network import Network, Node +from ngraph.model.selectors import select_nodes @dataclass @@ -29,7 +31,6 @@ class ExpandedDemand: volume: Traffic volume to place. priority: Priority class (lower is higher priority). policy_preset: FlowPolicy configuration preset. - demand_id: Parent TrafficDemand ID for tracking. """ src_name: str @@ -37,7 +38,6 @@ class ExpandedDemand: volume: float priority: int policy_preset: FlowPolicyPreset - demand_id: str @dataclass @@ -72,12 +72,24 @@ def _expand_combine( dst_groups: Dict[str, List[Node]], policy_preset: FlowPolicyPreset, ) -> tuple[list[ExpandedDemand], list[AugmentationEdge]]: - """Expand combine mode: aggregate sources/sinks through pseudo nodes.""" + """Expand combine mode: aggregate sources/sinks through pseudo nodes. + + Nodes selected on both sides are excluded from the target set. Without + this guard a shared node would be attached to both pseudo endpoints, + forming a zero-cost pseudo_src -> node -> pseudo_snk bypass over two + LARGE_CAPACITY augmentation edges that absorbs the entire demand + without touching the real network. This mirrors the overlap invariant + enforced in context._build_pseudo_node_augmentations. If the exclusion + empties the target set, nothing is expanded. + """ pseudo_src = f"_src_{td.id}" pseudo_snk = f"_snk_{td.id}" src_names = _flatten_group_names(src_groups) - dst_names = _flatten_group_names(dst_groups) + src_name_set = set(src_names) + dst_names = [ + name for name in _flatten_group_names(dst_groups) if name not in src_name_set + ] if not src_names or not dst_names: return [], [] @@ -99,7 +111,6 @@ def _expand_combine( volume=td.volume, priority=td.priority, policy_preset=policy_preset, - demand_id=td.id, ) return [expanded], augmentations @@ -133,7 +144,6 @@ def _expand_pairwise( volume=volume_per_pair, priority=td.priority, policy_preset=policy_preset, - demand_id=td.id, ) for src, dst in pairs ] @@ -150,48 +160,67 @@ def _expand_by_group_mode( """Expand demands based on group_mode. group_mode semantics: - - flatten: All nodes combined (default, current behavior) - - per_group: One demand per (src_group, dst_group) pair - - group_pairwise: Pairwise expansion within each group pair + - flatten: All groups merged into one node set, then mode applied. + - per_group: Each group expands independently. With mode=combine, one + demand per source group with all target groups combined (nodes in + the source group itself are excluded from the targets; a source + group whose targets become empty after exclusion is skipped); with + mode=pairwise, pairwise within each group label present on both + sides. td.volume is split evenly across groups; a skipped group's + share is not redistributed, so total expanded volume can be less + than td.volume when exclusion empties a group's targets or a label + has no non-self pairs. + - group_pairwise: One expansion per (src_group, dst_group) label pair + with distinct labels (same-label pairs are skipped), with td.volume + split evenly across those pairs; a pair that empties after + source/target overlap exclusion likewise drops its share. """ + # td.mode and td.group_mode are validated by TrafficDemand.__post_init__, + # so mode is "combine" or "pairwise" in every branch below. if td.group_mode == "flatten": # Standard behavior: flatten all groups, then apply mode if td.mode == "combine": return _expand_combine(td, src_groups, dst_groups, policy_preset) - elif td.mode == "pairwise": - return _expand_pairwise(td, src_groups, dst_groups, policy_preset) - else: - raise ValueError(f"Unknown demand mode: {td.mode}") + return _expand_pairwise(td, src_groups, dst_groups, policy_preset) elif td.group_mode == "per_group": - # One demand per (src_group, dst_group) pair all_demands: List[ExpandedDemand] = [] all_augmentations: List[AugmentationEdge] = [] - for src_label, src_nodes in src_groups.items(): - for dst_label, dst_nodes in dst_groups.items(): - if src_label == dst_label: - continue # Skip same-group pairs - - group_td = replace(td, id=f"{td.id}|{src_label}|{dst_label}") - single_src = {src_label: src_nodes} - single_dst = {dst_label: dst_nodes} - - if td.mode == "combine": - demands, augs = _expand_combine( - group_td, single_src, single_dst, policy_preset - ) - else: - demands, augs = _expand_pairwise( - group_td, single_src, single_dst, policy_preset - ) - + if td.mode == "combine": + # One demand per source group, all target groups combined + if not src_groups: + return [], [] + volume_per_group = td.volume / len(src_groups) + for src_label, src_nodes in src_groups.items(): + group_td = replace( + td, id=f"{td.id}|{src_label}", volume=volume_per_group + ) + demands, augs = _expand_combine( + group_td, {src_label: src_nodes}, dst_groups, policy_preset + ) + all_demands.extend(demands) + all_augmentations.extend(augs) + else: + # Pairwise within each group label present on both sides + shared_labels = [label for label in src_groups if label in dst_groups] + if not shared_labels: + return [], [] + volume_per_group = td.volume / len(shared_labels) + for label in shared_labels: + group_td = replace(td, id=f"{td.id}|{label}", volume=volume_per_group) + demands, augs = _expand_pairwise( + group_td, + {label: src_groups[label]}, + {label: dst_groups[label]}, + policy_preset, + ) all_demands.extend(demands) all_augmentations.extend(augs) return all_demands, all_augmentations - elif td.group_mode == "group_pairwise": + else: # group_pairwise # Pairwise between groups: each src group to each dst group all_demands: List[ExpandedDemand] = [] all_augmentations: List[AugmentationEdge] = [] @@ -209,10 +238,14 @@ def _expand_by_group_mode( # Divide volume among group pairs volume_per_group_pair = td.volume / len(group_pairs) - for src_label, dst_label in group_pairs: + for pair_index, (src_label, dst_label) in enumerate(group_pairs): + # Labels are '|'-joined regex captures and may themselves contain + # '|', so a purely label-composed id is ambiguous across pairs. + # The enumeration index makes the id injective; labels are kept + # for readability. group_td = replace( td, - id=f"{td.id}|{src_label}|{dst_label}", + id=f"{td.id}|{src_label}|{dst_label}#gp{pair_index}", volume=volume_per_group_pair, ) single_src = {src_label: src_groups[src_label]} @@ -232,9 +265,6 @@ def _expand_by_group_mode( return all_demands, all_augmentations - else: - raise ValueError(f"Unknown group_mode: {td.group_mode}") - def expand_demands( network: Network, @@ -264,8 +294,19 @@ def expand_demands( DemandExpansion with demands and augmentations. Raises: - ValueError: If no demands could be expanded or unsupported mode. + ValueError: If no demands could be expanded, or if two demands share + an id (pseudo node names embed the id, so duplicates would merge + distinct demands' attachment edges into one endpoint). """ + seen_ids: set[str] = set() + for td in traffic_demands: + if td.id in seen_ids: + raise ValueError( + f"Duplicate TrafficDemand id '{td.id}'. Demand ids must be " + "unique within one expansion." + ) + seen_ids.add(td.id) + all_demands: List[ExpandedDemand] = [] all_augmentations: List[AugmentationEdge] = [] @@ -299,6 +340,26 @@ def expand_demands( " - Source and target are identical (self-loops not allowed)" ) + # Pseudo endpoints must be unique across the whole expansion: composed + # ids concatenate demand ids and group labels, both of which may contain + # '|', so distinct demands can render to the same pseudo name (e.g. + # id "X" + label "Y|Z" vs id "X|Y" + label "Z"). A shared pseudo node + # would silently merge the demands' attachment edges, recreating the + # zero-cost bypass. + seen_endpoints: set[str] = set() + for d in all_demands: + for name in (d.src_name, d.dst_name): + if not name.startswith(("_src_", "_snk_")): + continue + if name in seen_endpoints: + raise ValueError( + f"Ambiguous demand expansion: pseudo endpoint '{name}' is " + "claimed by two different demand expansions. Demand ids " + "and group labels containing '|' can compose to the same " + "id; use distinct demand ids." + ) + seen_endpoints.add(name) + # Sort by priority (lower = higher priority) sorted_demands = sorted(all_demands, key=lambda d: d.priority) diff --git a/ngraph/analysis/failure_manager.py b/ngraph/analysis/failure_manager.py index 6157dfa..199a775 100644 --- a/ngraph/analysis/failure_manager.py +++ b/ngraph/analysis/failure_manager.py @@ -1,14 +1,15 @@ """FailureManager for Monte Carlo failure analysis. -Provides the failure analysis engine for NetGraph. Supports parallel -processing, graph caching, and failure policy handling for workflow steps -and direct programmatic use. +Runs an analysis function over many failure scenarios, handling failure policy +application, graph caching, and parallel execution. Used by workflow steps and +directly from user code. Performance characteristics: Time complexity: O(S + I * A / P), where S is one-time graph setup cost, I is iteration count, A is per-iteration analysis cost, and P is parallelism. -Graph caching amortizes expensive graph construction across all iterations, -and O(|excluded|) mask building replaces O(V+E) iteration. +Graph caching amortizes graph construction across all iterations: each +iteration applies its exclusions as an O(|excluded|) mask update instead of +rebuilding the graph or re-scanning all O(V+E) nodes and edges. Space complexity: O(V + E + I * R), where V and E are node and link counts, and R is result size per iteration. The pre-built graph is shared across @@ -28,30 +29,32 @@ from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Any, Dict, Optional, Protocol, Set -from ngraph.dsl.selectors import ( +from ngraph.logging import get_logger +from ngraph.model.failure.policy_set import FailurePolicySet +from ngraph.model.selectors import ( flatten_link_attrs, flatten_node_attrs, flatten_risk_group_attrs, ) -from ngraph.logging import get_logger -from ngraph.model.failure.policy_set import FailurePolicySet from ngraph.types.base import FlowPlacement if TYPE_CHECKING: import cProfile - from ngraph.model.network import Network + from ngraph.model.network import Network, RiskGroup +from ngraph.analysis.functions import ( + demand_placement_analysis, + max_flow_analysis, + sensitivity_analysis, +) from ngraph.model.failure.policy import FailurePolicy logger = get_logger(__name__) def _is_hashable(obj: Any) -> bool: - """Return True if obj is hashable, False otherwise. - - This avoids try/except TypeError patterns when checking hashability. - """ + """Return True if obj is hashable, False otherwise.""" try: hash(obj) return True @@ -59,29 +62,13 @@ def _is_hashable(obj: Any) -> bool: return False -def _create_cache_key( - excluded_nodes: Set[str], - excluded_links: Set[str], - analysis_name: str, - analysis_kwargs: Dict[str, Any], -) -> tuple: - """Create cache key from exclusions, analysis name, and parameters. - - Args: - excluded_nodes: Set of excluded node names. - excluded_links: Set of excluded link IDs. - analysis_name: Name of the analysis function. - analysis_kwargs: Analysis function arguments. +def _hashable_kwargs_key(analysis_kwargs: Dict[str, Any]) -> tuple: + """Build the hashable, order-independent kwargs component of a dedup key. - Returns: - Tuple suitable for use as a cache key. + This part of the key is invariant across Monte Carlo iterations, so + callers should compute it once and combine it with the per-iteration + exclusion sets via `_create_dedup_key`. """ - base_key = ( - tuple(sorted(excluded_nodes)), - tuple(sorted(excluded_links)), - analysis_name, - ) - hashable_kwargs = [] for key, value in sorted(analysis_kwargs.items()): if _is_hashable((key, value)): @@ -92,31 +79,36 @@ def _create_cache_key( # thousands of edges). id() is safe here because these objects # persist across calls within one FailureManager lifetime. hashable_kwargs.append((key, f"{type(value).__name__}_{id(value)}")) + return tuple(hashable_kwargs) - return base_key + (tuple(hashable_kwargs),) +def _create_dedup_key( + excluded_nodes: Set[str], + excluded_links: Set[str], + analysis_name: str, + kwargs_key: tuple, +) -> tuple: + """Create deduplication key from exclusions, analysis name, and parameters. -def _auto_adjust_parallelism(parallelism: int, analysis_func: Any) -> int: - """Adjust parallelism based on function characteristics. + The key identifies identical failure patterns before dispatch: iterations + sharing a key are executed once and fanned back out via occurrence_count. + This is pre-dispatch deduplication, not a result cache. Args: - parallelism: Requested parallelism level. - analysis_func: Analysis function to check. + excluded_nodes: Set of excluded node names. + excluded_links: Set of excluded link IDs. + analysis_name: Name of the analysis function. + kwargs_key: Precomputed `_hashable_kwargs_key(analysis_kwargs)`. Returns: - Adjusted parallelism level. + Tuple suitable for use as a deduplication key. """ - # Check if function is defined in __main__ (notebook context) - if hasattr(analysis_func, "__module__") and analysis_func.__module__ == "__main__": - if parallelism > 1: - logger.warning( - "Function defined in notebook/script (__main__) detected. " - "Forcing serial execution (parallelism=1) to avoid pickling issues. " - "Consider moving analysis function to a separate module for parallel execution." - ) - return 1 - - return parallelism + return ( + tuple(sorted(excluded_nodes)), + tuple(sorted(excluded_links)), + analysis_name, + kwargs_key, + ) class AnalysisFunction(Protocol): @@ -131,20 +123,21 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any: ... -def _generic_worker(args: tuple[Any, ...]) -> tuple[Any, int, bool, set[str], set[str]]: - """Execute analysis function with caching. +def _generic_worker(args: tuple[Any, ...]) -> Any: + """Execute the analysis function once for one exclusion pattern. - Caches analysis results based on exclusion patterns and analysis parameters - since many Monte Carlo iterations share the same exclusion sets. - Analysis computation is deterministic for identical inputs, making caching safe. + Unpacks the args tuple, optionally enables per-worker cProfile when + NGRAPH_PROFILE_DIR is set, and calls + analysis_func(network, excluded_nodes, excluded_links, **analysis_kwargs). + Duplicate exclusion patterns are deduplicated upstream in + run_monte_carlo_analysis, so each worker invocation is unique work. Args: args: Tuple containing (network, excluded_nodes, excluded_links, analysis_func, analysis_kwargs, iteration_index, is_baseline, analysis_name). Returns: - Tuple of (analysis_result, iteration_index, is_baseline, - excluded_nodes, excluded_links). + The analysis function's result. """ worker_logger = get_logger(f"{__name__}.worker") @@ -165,7 +158,6 @@ def _generic_worker(args: tuple[Any, ...]) -> tuple[Any, int, bool, set[str], se profiler: "cProfile.Profile | None" = None if collect_profile: - # Install per-worker profiler when requested import cProfile profiler = cProfile.Profile() @@ -184,7 +176,6 @@ def _generic_worker(args: tuple[Any, ...]) -> tuple[Any, int, bool, set[str], se f"excluded_nodes={len(excluded_nodes)}, excluded_links={len(excluded_links)}" ) - # Execute analysis function with network and exclusion sets worker_logger.debug(f"Worker {worker_id} executing {analysis_name}") result = analysis_func(network, excluded_nodes, excluded_links, **analysis_kwargs) worker_logger.debug(f"Worker {worker_id} completed analysis") @@ -193,7 +184,6 @@ def _generic_worker(args: tuple[Any, ...]) -> tuple[Any, int, bool, set[str], se if profiler is not None: profiler.disable() import pstats - import threading import uuid from pathlib import Path @@ -208,19 +198,18 @@ def _generic_worker(args: tuple[Any, ...]) -> tuple[Any, int, bool, set[str], se pstats.Stats(profiler).dump_stats(profile_path) worker_logger.debug("Saved worker profile to %s", profile_path.name) - return (result, iteration_index, is_baseline, excluded_nodes, excluded_links) + return result class FailureManager: - """Failure analysis engine with Monte Carlo capabilities. + """Run an analysis function across Monte Carlo failure scenarios. - This is the component for failure analysis in NetGraph. - Provides parallel processing, worker caching, and failure - policy handling for workflow steps and direct notebook usage. + Applies a failure policy, deduplicates identical failure patterns, and + runs iterations in parallel. Used by workflow steps and directly from user code. - The FailureManager can execute any analysis function that takes a Network - with exclusion sets and returns results, making it generic for different - types of failure analysis (capacity, traffic, connectivity, etc.). + Executes any analysis function that takes a Network plus exclusion sets and + returns results, so the same engine covers capacity, traffic, connectivity, + and custom analyses. Attributes: network: The underlying network (not modified during analysis). @@ -247,10 +236,25 @@ def __init__( self._merged_node_attrs: dict[str, dict[str, Any]] | None = None self._merged_link_attrs: dict[str, dict[str, Any]] | None = None self._merged_rg_attrs: dict[str, dict[str, Any]] | None = None - self._prepared_policy_matches: dict[int, dict[int, tuple[str, ...]]] = {} + # Keyed by id(policy); each entry pins the policy object so a freed + # policy's address cannot be reused and silently match a stale entry. + self._prepared_policy_matches: dict[ + int, tuple[FailurePolicy, dict[int, tuple[str, ...]]] + ] = {} + self._prepared_policy_weights: dict[ + int, + tuple[ + FailurePolicy, + dict[int, tuple[dict[str, float], tuple[str, ...]]], + ], + ] = {} self._risk_group_exclusions: ( dict[str, tuple[frozenset[str], frozenset[str]]] | None ) = None + # Risk-group -> entity-ID index used by policies with + # expand_groups=True. Depends only on the static network, so it is + # built once per manager and shared across policies and iterations. + self._prepared_rg_index: dict[str, set[str]] | None = None def get_failure_policy(self) -> "FailurePolicy | None": """Get failure policy for analysis. @@ -333,29 +337,44 @@ def compute_exclusions( prepared_matches = self._get_prepared_policy_matches( policy, node_map, link_map, rg_map ) + prepared_weights = self._get_prepared_policy_weights( + policy, prepared_matches, node_map, link_map, rg_map + ) + # getattr: compute_exclusions accepts duck-typed policy objects that + # may not declare expand_groups. + wants_expansion = getattr(policy, "expand_groups", False) + prepared_rg_index = self._get_prepared_rg_index() if wants_expansion else None + if wants_expansion: + if self._risk_group_exclusions is None: + self._risk_group_exclusions = self._build_risk_group_exclusions() + prepared_rg_members = self._risk_group_exclusions + else: + prepared_rg_members = None - # Apply failure policy with optional deterministic seed override - failed_ids = policy.apply_failures( + # Apply failure policy with optional deterministic seed override. + # The typed variant keeps entity kinds separate: probing merged IDs + # against network collections misclassifies a risk group that shares + # its name with a node or link. + failed_nodes, failed_links, failed_rgs = policy.apply_failures_typed( node_map, link_map, rg_map, seed=seed_offset, failure_trace=failure_trace, prepared_matches=prepared_matches, + prepared_weights=prepared_weights, + prepared_rg_index=prepared_rg_index, + prepared_rg_members=prepared_rg_members, ) - # Separate entity types for exclusion sets - for f_id in failed_ids: - if f_id in self.network.nodes: - excluded_nodes.add(f_id) - elif f_id in self.network.links: - excluded_links.add(f_id) - elif f_id in self.network.risk_groups: - risk_group_nodes, risk_group_links = self._get_risk_group_exclusions( - f_id - ) - excluded_nodes.update(risk_group_nodes) - excluded_links.update(risk_group_links) + excluded_nodes.update(failed_nodes) + excluded_links.update(failed_links) + for rg_name in failed_rgs: + risk_group_nodes, risk_group_links = self._get_risk_group_exclusions( + rg_name + ) + excluded_nodes.update(risk_group_nodes) + excluded_links.update(risk_group_links) return excluded_nodes, excluded_links @@ -392,34 +411,37 @@ def _build_risk_group_exclusions( expanded: dict[str, tuple[frozenset[str], frozenset[str]]] = {} - def expand_group(name: str) -> tuple[frozenset[str], frozenset[str]]: - cached = expanded.get(name) + # Recurse on RiskGroup objects directly: nested groups are not + # registered in network.risk_groups (only top-level groups are), so + # resolving children by name would silently drop grandchild members. + def expand_group( + group: "RiskGroup", + ) -> tuple[frozenset[str], frozenset[str]]: + cached = expanded.get(group.name) if cached is not None: return cached - if name in visiting: + if group.name in visiting: return ( - frozenset(direct_nodes.get(name, ())), - frozenset(direct_links.get(name, ())), + frozenset(direct_nodes.get(group.name, ())), + frozenset(direct_links.get(group.name, ())), ) - visiting.add(name) - nodes = set(direct_nodes.get(name, ())) - links = set(direct_links.get(name, ())) - risk_group = self.network.risk_groups.get(name) - if risk_group is not None: - for child in risk_group.children: - child_nodes, child_links = expand_group(child.name) - nodes.update(child_nodes) - links.update(child_links) + visiting.add(group.name) + nodes = set(direct_nodes.get(group.name, ())) + links = set(direct_links.get(group.name, ())) + for child in group.children: + child_nodes, child_links = expand_group(child) + nodes.update(child_nodes) + links.update(child_links) result = (frozenset(nodes), frozenset(links)) - visiting.remove(name) - expanded[name] = result + visiting.remove(group.name) + expanded[group.name] = result return result visiting: set[str] = set() - for risk_group_name in self.network.risk_groups: - expand_group(risk_group_name) + for risk_group in self.network.risk_groups.values(): + expand_group(risk_group) return expanded @@ -433,17 +455,60 @@ def _get_prepared_policy_matches( """Prepare stable ordered candidate pools for a policy once per manager.""" policy_key = id(policy) cached = self._prepared_policy_matches.get(policy_key) - if cached is not None: - return cached + if cached is not None and cached[0] is policy: + return cached[1] prepared = policy.prepare_matches( node_map, link_map, rg_map, ) - self._prepared_policy_matches[policy_key] = prepared + self._prepared_policy_matches[policy_key] = (policy, prepared) return prepared + def _get_prepared_policy_weights( + self, + policy: "FailurePolicy", + prepared_matches: dict[int, tuple[str, ...]], + node_map: dict[str, dict[str, Any]], + link_map: dict[str, dict[str, Any]], + rg_map: dict[str, dict[str, Any]], + ) -> dict[int, tuple[dict[str, float], tuple[str, ...]]]: + """Prepare per-rule weight splits for a policy once per manager.""" + policy_key = id(policy) + cached = self._prepared_policy_weights.get(policy_key) + if cached is not None and cached[0] is policy: + return cached[1] + + # Duck-typed policy objects may not implement the optional + # prepare_weights optimization; treat it as "no precomputed weights". + prepare = getattr(policy, "prepare_weights", None) + prepared = ( + prepare(prepared_matches, node_map, link_map, rg_map) if prepare else {} + ) + self._prepared_policy_weights[policy_key] = (policy, prepared) + return prepared + + def _get_prepared_rg_index(self) -> dict[str, set[str]]: + """Build the risk-group -> entity-ID expansion index once per manager. + + The index consumed by ``FailurePolicy.apply_failures`` (via + ``prepared_rg_index``) depends only on the static network, so it is + built once and reused across policies and Monte Carlo iterations + instead of being rebuilt on every ``apply_failures`` call. + """ + if self._prepared_rg_index is None: + self._ensure_flattened_maps() + assert ( + self._merged_node_attrs is not None + ) # guaranteed by _ensure_flattened_maps + assert self._merged_link_attrs is not None + self._prepared_rg_index = FailurePolicy.build_risk_group_index( + self._merged_node_attrs, + self._merged_link_attrs, + ) + return self._prepared_rg_index + def run_monte_carlo_analysis( self, analysis_func: AnalysisFunction, @@ -455,32 +520,55 @@ def run_monte_carlo_analysis( ) -> dict[str, Any]: """Run Monte Carlo failure analysis with any analysis function. - This is the main method for executing failure analysis. Handles - parallel processing, worker caching, and failure policy - application, while allowing flexibility in the analysis function. + The analysis function is arbitrary (see AnalysisFunction); this method + supplies the exclusion sets, the parallelism, and the deduplication. Baseline is always run first as a separate reference iteration (no failures). - The ``iterations`` parameter specifies the number of failure iterations to run. + + Analysis functions may carry a ``prepare_inputs`` attribute; the + built-in ones do, and custom functions can set + ``func.prepare_inputs = lambda network, kwargs: {...}``. It runs once + per run, and the kwargs it returns (typically a pre-built ``context``) + are merged into every iteration's call, so expensive graph + construction happens once. Functions without the attribute run with + their kwargs unchanged, and passing ``context`` explicitly skips the + hook. Args: analysis_func: Function that takes (network, excluded_nodes, excluded_links, **kwargs) - and returns results. Must be serializable for parallel execution. + and returns results. Executed concurrently via threads; + the network is shared by reference and must not be mutated. iterations: Number of failure iterations to run (baseline is always run separately). parallelism: Number of parallel worker threads to use. - seed: Optional seed for reproducible results across runs. + seed: Optional seed for reproducible results across runs. If None, + falls back to the policy's own seed when set, so iterations + still vary while remaining reproducible. store_failure_patterns: If True, populate failure_trace on each result. + Iterations are deduplicated by exclusion pattern, so the trace + describes the representative (first) iteration of each pattern; + different mode/rule draws that produced the same exclusions are + not individually recorded. **analysis_kwargs: Additional arguments passed to analysis_func. Returns: Dictionary containing: - 'baseline': FlowIterationResult for the baseline (no failures) - - 'results': List of unique FlowIterationResult objects (deduplicated patterns). - Each result has occurrence_count indicating how many iterations matched. - - 'metadata': Execution metadata (iterations, unique_patterns, execution_time, etc.) + - 'results': List of unique results (deduplicated patterns). + FlowIterationResult objects carry occurrence_count; for any + result type, metadata["occurrence_counts"] is aligned with + this list. + - 'metadata': Execution metadata (iterations, unique_patterns, + occurrence_counts, execution_time, etc.) + + Note: + Deduplication executes each unique exclusion pattern once and + weights it by occurrence count, which is statistically valid only + for deterministic analysis functions (the built-ins are). A + stochastic custom function should not rely on per-iteration + re-execution. """ policy = self.get_failure_policy() - # Check if policy has effective rules has_effective_rules = bool( policy and any(len(m.rules) > 0 for m in policy.modes) ) @@ -489,43 +577,25 @@ def run_monte_carlo_analysis( if not has_effective_rules: iterations = 0 - # Auto-adjust parallelism based on function characteristics - parallelism = _auto_adjust_parallelism(parallelism, analysis_func) - logger.info( f"Running baseline + {iterations} failure iterations" if iterations > 0 else "Running baseline only (no failure policy)" ) - # Pre-build context for analysis functions - # This amortizes expensive graph construction across all iterations - if "context" not in analysis_kwargs: - analysis_kwargs = dict(analysis_kwargs) # Don't mutate caller's dict + # Pre-build expensive per-run inputs when the analysis function + # carries a `prepare_inputs` hook (attached by the built-in analysis + # functions; third-party functions can set the attribute themselves) + # and the caller did not supply a pre-built context. This amortizes + # graph construction across all iterations. + prepare = getattr(analysis_func, "prepare_inputs", None) + if prepare is not None and "context" not in analysis_kwargs: cache_start = time.time() - - if "demands_config" in analysis_kwargs: - # Demand placement analysis - from ngraph.analysis.functions import build_demand_context - - logger.debug("Pre-building context for demand placement analysis") - analysis_kwargs["context"] = build_demand_context( - self.network, analysis_kwargs["demands_config"] - ) - logger.debug(f"Context built in {time.time() - cache_start:.3f}s") - - elif "source" in analysis_kwargs and "target" in analysis_kwargs: - # Max-flow analysis or sensitivity analysis - from ngraph.analysis.functions import build_maxflow_context - - logger.debug("Pre-building context for max-flow analysis") - analysis_kwargs["context"] = build_maxflow_context( - self.network, - analysis_kwargs["source"], - analysis_kwargs["target"], - mode=analysis_kwargs.get("mode", "combine"), - ) - logger.debug(f"Context built in {time.time() - cache_start:.3f}s") + analysis_kwargs = dict(analysis_kwargs) # Don't mutate caller's dict + analysis_kwargs.update(prepare(self.network, analysis_kwargs)) + logger.debug( + f"Pre-built analysis inputs in {time.time() - cache_start:.3f}s" + ) # Get function name safely (Protocol doesn't guarantee __name__) func_name = getattr(analysis_func, "__name__", "analysis_function") @@ -549,35 +619,41 @@ def run_monte_carlo_analysis( logger.debug("Pre-computing failure exclusions for all iterations") pre_compute_start = time.time() - worker_args: list[tuple] = [] key_to_first_arg: dict[tuple, tuple] = {} key_to_count: dict[tuple, int] = {} key_to_trace: dict[tuple, dict[str, Any]] = {} + # Fall back to the policy's own seed so iterations still vary: with + # seed_offset=None a seeded policy would rebuild the identical RNG + # (and failure pattern) on every iteration. + effective_seed = seed + if effective_seed is None and policy is not None: + effective_seed = policy.seed + + # Invariant across iterations; only the exclusion sets vary. + kwargs_key = _hashable_kwargs_key(analysis_kwargs) + for i in range(iterations): - seed_offset = seed + i if seed is not None else None + seed_offset = effective_seed + i if effective_seed is not None else None trace = {} if store_failure_patterns else None excluded_nodes, excluded_links = self.compute_exclusions( policy, seed_offset, failure_trace=trace ) - arg = ( - self.network, - excluded_nodes, - excluded_links, - analysis_func, - analysis_kwargs, - i, # iteration_index (0-based for failures) - False, # is_baseline - func_name, - ) - worker_args.append(arg) - - dedup_key = _create_cache_key( - excluded_nodes, excluded_links, func_name, analysis_kwargs + dedup_key = _create_dedup_key( + excluded_nodes, excluded_links, func_name, kwargs_key ) if dedup_key not in key_to_first_arg: - key_to_first_arg[dedup_key] = arg + key_to_first_arg[dedup_key] = ( + self.network, + excluded_nodes, + excluded_links, + analysis_func, + analysis_kwargs, + i, # iteration_index (0-based for failures) + False, # is_baseline + func_name, + ) key_to_count[dedup_key] = 1 if trace is not None: key_to_trace[dedup_key] = trace @@ -586,7 +662,7 @@ def run_monte_carlo_analysis( pre_compute_time = time.time() - pre_compute_start logger.debug( - f"Pre-computed {len(worker_args)} failure exclusion sets in {pre_compute_time:.2f}s" + f"Pre-computed {iterations} failure exclusion sets in {pre_compute_time:.2f}s" ) unique_worker_args: list[tuple] = list(key_to_first_arg.values()) @@ -625,8 +701,12 @@ def run_monte_carlo_analysis( elapsed_time = time.time() - start_time - # Enrich unique failure results with metadata and occurrence_count + # Enrich unique failure results with metadata and occurrence_count. + # metadata["occurrence_counts"] carries the per-pattern multiplicity + # aligned with `results`, so custom result types (which cannot be + # enriched in place) still get correct weights for aggregation. results: list[Any] = [] + occurrence_counts: list[int] = [] for dedup_key, rep_arg in key_to_first_arg.items(): result = key_to_result.get(dedup_key) if result is None: @@ -659,6 +739,7 @@ def run_monte_carlo_analysis( result.occurrence_count = key_to_count[dedup_key] results.append(result) + occurrence_counts.append(key_to_count[dedup_key]) return { "baseline": baseline_result, @@ -670,6 +751,7 @@ def run_monte_carlo_analysis( "policy_name": self.policy_name, "execution_time": elapsed_time, "unique_patterns": num_unique_tasks, + "occurrence_counts": occurrence_counts, }, } @@ -702,10 +784,6 @@ def _run_parallel( # Network is shared by reference (zero-copy) across threads logger.debug(f"Sharing network by reference across {workers} threads") - # Calculate optimal chunksize to minimize overhead - chunksize = max(1, total_tasks // (workers * 4)) - logger.debug(f"Using chunksize={chunksize} for parallel execution") - start_time = time.time() completed_tasks = 0 results = [] @@ -718,13 +796,7 @@ def _run_parallel( ) logger.info(f"Starting parallel execution of {total_tasks} iterations") - for ( - result, - _iteration_index, - _is_baseline, - _excluded_nodes, - _excluded_links, - ) in pool.map(_generic_worker, worker_args, chunksize=chunksize): + for result in pool.map(_generic_worker, worker_args): completed_tasks += 1 results.append(result) @@ -775,37 +847,37 @@ def _run_serial( "Temporarily disabled NGRAPH_PROFILE_DIR for serial execution to avoid nested profilers" ) - for i, args in enumerate(worker_args): - iter_start = time.time() - - is_baseline_arg = len(args) > 6 and args[6] # is_baseline flag - baseline_msg = " (baseline)" if is_baseline_arg else "" - logger.debug(f"Serial iteration {i + 1}/{len(worker_args)}{baseline_msg}") - - ( - result, - _iteration_index, - _is_baseline, - _excluded_nodes, - _excluded_links, - ) = _generic_worker(args) - - results.append(result) + try: + for i, args in enumerate(worker_args): + iter_start = time.time() - iter_time = time.time() - iter_start - if len(worker_args) <= 10: + is_baseline_arg = len(args) > 6 and args[6] # is_baseline flag + baseline_msg = " (baseline)" if is_baseline_arg else "" logger.debug( - f"Serial iteration {i + 1} completed in {iter_time:.3f} seconds" + f"Serial iteration {i + 1}/{len(worker_args)}{baseline_msg}" ) - if len(worker_args) > 1 and (i + 1) % max(1, len(worker_args) // 10) == 0: - logger.info( - f"Serial analysis progress: {i + 1}/{len(worker_args)} iterations completed" - ) + result = _generic_worker(args) + + results.append(result) - # Restore worker profiling env var if we changed it - if _restore_profile_env: - os.environ["NGRAPH_PROFILE_DIR"] = _saved_profile_dir or "" + iter_time = time.time() - iter_start + if len(worker_args) <= 10: + logger.debug( + f"Serial iteration {i + 1} completed in {iter_time:.3f} seconds" + ) + + if ( + len(worker_args) > 1 + and (i + 1) % max(1, len(worker_args) // 10) == 0 + ): + logger.info( + f"Serial analysis progress: {i + 1}/{len(worker_args)} iterations completed" + ) + finally: + # Restore worker profiling env var even if an analysis function raised + if _restore_profile_env and _saved_profile_dir is not None: + os.environ["NGRAPH_PROFILE_DIR"] = _saved_profile_dir elapsed_time = time.time() - start_time logger.info(f"Serial analysis completed in {elapsed_time:.2f} seconds") @@ -819,11 +891,9 @@ def _run_serial( def run_single_failure_scenario( self, analysis_func: AnalysisFunction, **kwargs ) -> Any: - """Run a single failure scenario for convenience. + """Run one failure iteration, for quick analysis or debugging. - This is a convenience method for running a single iteration, useful for - quick analysis or debugging. For full Monte Carlo analysis, use - run_monte_carlo_analysis(). + For full Monte Carlo analysis, use run_monte_carlo_analysis(). Args: analysis_func: Function that takes (network, excluded_nodes, excluded_links, **kwargs) @@ -837,7 +907,6 @@ def run_single_failure_scenario( result = self.run_monte_carlo_analysis( analysis_func=analysis_func, iterations=1, parallelism=1, **kwargs ) - # Return first failure result if available, otherwise baseline if result["results"]: return result["results"][0] return result["baseline"] @@ -859,11 +928,11 @@ def run_max_flow_monte_carlo( include_flow_summary: bool = False, include_min_cut: bool = False, ) -> Any: - """Analyze maximum flow capacity envelopes between node groups under failures. + """Compute max-flow capacity envelopes between node groups under failures. - Computes statistical distributions (envelopes) of maximum flow capacity between - source and target node groups across Monte Carlo failure scenarios. Results include - frequency-based capacity envelopes and optional failure pattern analysis. + Each iteration applies one failure pattern and re-solves; the + per-pattern results, weighted by occurrence count, form the + frequency-based envelope. Baseline (no failures) is always run first as a separate reference. @@ -872,12 +941,15 @@ def run_max_flow_monte_carlo( target: Target node selector (string path or selector dict). mode: "combine" (aggregate) or "pairwise" (individual flows). iterations: Number of failure scenarios to simulate. - parallelism: Number of parallel workers (auto-adjusted if needed). - shortest_path: Whether to use shortest paths only. + parallelism: Number of parallel worker threads. + shortest_path: If True, use single-tier shortest-path flow (IP/IGP + mode) instead of full iterative max-flow. require_capacity: If True (default), path selection considers available capacity. If False, path selection is cost-only (true IP/IGP semantics). - flow_placement: Flow placement strategy. - seed: Optional seed for reproducible results. + flow_placement: PROPORTIONAL (WCMP) or EQUAL_BALANCED (ECMP); + accepts the enum or its string name. + seed: Optional seed for reproducible results. If None, falls back + to the policy's own seed when set. store_failure_patterns: Whether to store failure trace on results. include_flow_summary: Whether to collect detailed flow summary data. include_min_cut: Whether to include min-cut edges in results. @@ -889,13 +961,9 @@ def run_max_flow_monte_carlo( Each result has occurrence_count indicating how many iterations matched. - 'metadata': Execution metadata (iterations, unique_patterns, execution_time, etc.) """ - from ngraph.analysis.functions import max_flow_analysis - - # Convert string flow_placement to enum if needed if isinstance(flow_placement, str): flow_placement = FlowPlacement.from_string(flow_placement) - # Run Monte Carlo analysis raw_results = self.run_monte_carlo_analysis( analysis_func=max_flow_analysis, iterations=iterations, @@ -976,7 +1044,6 @@ def run_demand_placement_monte_carlo( | Any, # List of demand configs or DemandSet iterations: int = 100, parallelism: int = 1, - placement_rounds: int | str = "auto", seed: int | None = None, store_failure_patterns: bool = False, include_flow_details: bool = False, @@ -992,9 +1059,9 @@ def run_demand_placement_monte_carlo( Args: demands_config: List of demand configs or DemandSet object. iterations: Number of failure scenarios to simulate. - parallelism: Number of parallel workers (auto-adjusted if needed). - placement_rounds: Optimization rounds for demand placement. - seed: Optional seed for reproducible results. + parallelism: Number of parallel worker threads. + seed: Optional seed for reproducible results. If None, falls back + to the policy's own seed when set. store_failure_patterns: Whether to store failure trace on results. include_flow_details: Whether to include cost distribution details. include_used_edges: Whether to include used edges in results. @@ -1006,31 +1073,18 @@ def run_demand_placement_monte_carlo( Each result has occurrence_count indicating how many iterations matched. - 'metadata': Execution metadata (iterations, unique_patterns, execution_time, etc.) """ - from ngraph.analysis.functions import demand_placement_analysis - # If caller passed a sequence of TrafficDemand objects, convert to dicts if not isinstance(demands_config, list): # Accept DemandSet or any container providing get_all_demands() serializable_demands: list[dict[str, Any]] = [] if hasattr(demands_config, "get_all_demands"): td_iter = demands_config.get_all_demands() # DemandSet helper - elif hasattr(demands_config, "demands"): - # Accept a mock object exposing 'demands' for tests - td_iter = demands_config.demands else: td_iter = [] for demand in td_iter: # type: ignore[assignment] + # Analysis wire format: canonical dict with the raw preset serializable_demands.append( - { - "id": getattr(demand, "id", None), - "source": getattr(demand, "source", ""), - "target": getattr(demand, "target", ""), - "volume": float(getattr(demand, "volume", 0.0)), - "mode": getattr(demand, "mode", "pairwise"), - "group_mode": getattr(demand, "group_mode", "flatten"), - "flow_policy": getattr(demand, "flow_policy", None), - "priority": int(getattr(demand, "priority", 0)), - } + {**demand.to_dict(), "flow_policy": demand.flow_policy} ) demands_config = serializable_demands @@ -1041,7 +1095,6 @@ def run_demand_placement_monte_carlo( seed=seed, store_failure_patterns=store_failure_patterns, demands_config=demands_config, - placement_rounds=placement_rounds, include_flow_details=include_flow_details, include_used_edges=include_used_edges, ) @@ -1062,8 +1115,7 @@ def run_sensitivity_monte_carlo( """Analyze component criticality for flow capacity under failures. Identifies critical network components by measuring their impact on flow - capacity across Monte Carlo failure scenarios. Returns aggregated sensitivity - scores showing which components have the greatest effect on network capacity. + capacity across Monte Carlo failure scenarios. Baseline (no failures) is always run first as a separate reference. @@ -1072,10 +1124,13 @@ def run_sensitivity_monte_carlo( target: Target node selector (string path or selector dict). mode: "combine" (aggregate) or "pairwise" (individual flows). iterations: Number of failure scenarios to simulate. - parallelism: Number of parallel workers (auto-adjusted if needed). - shortest_path: Whether to use shortest paths only. - flow_placement: Flow placement strategy. - seed: Optional seed for reproducible results. + parallelism: Number of parallel worker threads. + shortest_path: If True, report only edges used under ECMP routing + (IP/IGP mode); if False, report all saturated edges (SDN/TE). + flow_placement: PROPORTIONAL (WCMP) or EQUAL_BALANCED (ECMP); + accepts the enum or its string name. + seed: Optional seed for reproducible results. If None, falls back + to the policy's own seed when set. store_failure_patterns: Whether to store failure trace on results. Returns: @@ -1086,9 +1141,6 @@ def run_sensitivity_monte_carlo( - 'component_scores': aggregated statistics (mean, max, min, count) per component per flow - 'metadata': Execution metadata (iterations, unique_patterns, execution_time, etc.) """ - from ngraph.analysis.functions import sensitivity_analysis - - # Convert string flow_placement to enum if needed if isinstance(flow_placement, str): flow_placement = FlowPlacement.from_string(flow_placement) diff --git a/ngraph/analysis/functions.py b/ngraph/analysis/functions.py index 251e0ae..87dea1b 100644 --- a/ngraph/analysis/functions.py +++ b/ngraph/analysis/functions.py @@ -4,26 +4,28 @@ takes a Network, exclusion sets, and analysis-specific parameters, returning results of type FlowIterationResult. -Parameters should ideally be hashable for efficient caching in FailureManager; -non-hashable objects are identified by memory address for cache key generation. +Parameters should ideally be hashable so FailureManager can deduplicate +identical failure patterns before dispatch; non-hashable objects are keyed +by memory address. -Graph caching enables efficient repeated analysis with different exclusion -sets by building the graph once and using O(|excluded|) masks for exclusions. +Graph caching builds the graph once and applies each exclusion set as an +O(|excluded|) mask instead of rebuilding. -SPF caching enables efficient demand placement by computing shortest paths once -per unique source node rather than once per demand. For networks with many demands -sharing the same sources, this can reduce SPF computations by an order of magnitude. +SPF caching computes shortest paths once per unique source node rather than +once per demand. For networks with many demands sharing the same sources, this +can reduce SPF computations by an order of magnitude. """ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Set +from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence, Set import netgraph_core from ngraph.analysis.context import AnalysisContext, analyze -from ngraph.analysis.demand import expand_demands +from ngraph.analysis.demand import DemandExpansion, expand_demands from ngraph.analysis.placement import place_demands +from ngraph.model.demand.builder import coerce_flow_policy from ngraph.model.demand.spec import TrafficDemand from ngraph.model.flow.policy_config import FlowPolicyPreset from ngraph.results.flow import FlowEntry, FlowIterationResult, FlowSummary @@ -35,25 +37,37 @@ def _reconstruct_traffic_demands( ) -> list[TrafficDemand]: """Reconstruct TrafficDemand objects from serialized config. + Configs without an "id" receive a deterministic id derived from + source, target, and list position. This keeps pseudo node names + (which embed the demand id) stable across repeated reconstructions, + so a context pre-built from the same config list stays consistent. + + Field defaults match TrafficDemand's own defaults (mode="combine", + group_mode="flatten"), so a config produced by `TrafficDemand.to_dict` + round-trips faithfully. + Args: demands_config: List of demand configurations with fields: - source, target, volume, mode, group_mode, flow_policy, priority. + source, target, volume, mode, group_mode, flow_policy, + priority, attrs. Returns: - List of TrafficDemand objects with preserved IDs. + List of TrafficDemand objects with stable IDs. """ results = [] - for config in demands_config: + for i, config in enumerate(demands_config): results.append( TrafficDemand( - id=config.get("id") or "", + id=config.get("id") + or f"{config['source']}|{config.get('target', '')}|{i}", source=config["source"], target=config.get("target", ""), volume=config.get("volume", 0.0), - mode=config.get("mode", "pairwise"), + mode=config.get("mode", "combine"), group_mode=config.get("group_mode", "flatten"), - flow_policy=config.get("flow_policy"), + flow_policy=coerce_flow_policy(config.get("flow_policy")), priority=config.get("priority", 0), + attrs=config.get("attrs") or {}, ) ) return results @@ -63,6 +77,54 @@ def _reconstruct_traffic_demands( from ngraph.model.network import Network +def _with_prepare_inputs( + prepare: "Callable[[Network, dict[str, Any]], dict[str, Any]]", +): + """Attach a FailureManager pre-build hook to an analysis function. + + ``FailureManager.run_monte_carlo_analysis`` calls + ``func.prepare_inputs(network, analysis_kwargs)`` once per run (unless + the caller already supplied ``context``) and merges the returned extra + kwargs into every iteration's call. Third-party analysis functions can + opt in the same way by setting a ``prepare_inputs`` attribute. + """ + + def _attach(func): + func.prepare_inputs = prepare # type: ignore[attr-defined] + return func + + return _attach + + +def _prepare_demand_placement_inputs( + network: "Network", analysis_kwargs: dict[str, Any] +) -> dict[str, Any]: + """Pre-build context, expansion, and resolved node IDs once per MC run.""" + context, expansion, resolved_ids = build_demand_placement_inputs( + network, analysis_kwargs["demands_config"] + ) + return { + "context": context, + "expansion": expansion, + "resolved_ids": resolved_ids, + } + + +def _prepare_maxflow_inputs( + network: "Network", analysis_kwargs: dict[str, Any] +) -> dict[str, Any]: + """Pre-build the bound max-flow context once per MC run.""" + return { + "context": build_maxflow_context( + network, + analysis_kwargs["source"], + analysis_kwargs["target"], + mode=analysis_kwargs.get("mode", "combine"), + ) + } + + +@_with_prepare_inputs(_prepare_maxflow_inputs) def max_flow_analysis( network: "Network", excluded_nodes: Set[str], @@ -86,23 +148,38 @@ def max_flow_analysis( source: Source node selector (string path or selector dict). target: Target node selector (string path or selector dict). mode: Flow analysis mode ("combine" or "pairwise"). - shortest_path: Whether to use shortest paths only. + shortest_path: If True, use single-tier shortest-path flow (IP/IGP + mode) instead of full iterative max-flow. require_capacity: If True (default), path selection considers available capacity. If False, path selection is cost-only (true IP/IGP semantics). - flow_placement: Flow placement strategy. + flow_placement: PROPORTIONAL (WCMP) or EQUAL_BALANCED (ECMP). include_flow_details: Whether to collect cost distribution and similar details. include_min_cut: Whether to include min-cut edge list in entry data. - context: Pre-built AnalysisContext for efficient repeated analysis. + context: Pre-built AnalysisContext reused across calls. Must be + unbound or bound to these same source/target/mode arguments. Returns: FlowIterationResult describing this iteration. """ - # Convert string mode to Mode enum - mode_enum = Mode.COMBINE if mode == "combine" else Mode.PAIRWISE + # Convert string mode to Mode enum (raises on invalid values) + mode_enum = Mode.from_string(mode) - # Use provided context or create a new one + # Use provided context or create a new one. A bound context carries its + # own source/sink/mode; silently ignoring mismatched arguments would + # return results for the wrong pair, so reject the mismatch loudly. if context is not None: ctx = context + if ctx.is_bound and ( + ctx.bound_source != source + or ctx.bound_sink != target + or ctx.bound_mode != mode_enum + ): + raise ValueError( + "Provided context is bound to " + f"source={ctx.bound_source!r}, sink={ctx.bound_sink!r}, " + f"mode={ctx.bound_mode}, which differs from the analysis " + "arguments; rebuild the context or pass matching arguments." + ) else: ctx = analyze(network, source=source, sink=target, mode=mode_enum) @@ -181,24 +258,31 @@ def max_flow_analysis( return FlowIterationResult(flows=flow_entries, summary=summary) +@_with_prepare_inputs(_prepare_demand_placement_inputs) def demand_placement_analysis( network: "Network", excluded_nodes: Set[str], excluded_links: Set[str], demands_config: list[dict[str, Any]], - placement_rounds: int | str = "auto", include_flow_details: bool = False, include_used_edges: bool = False, context: Optional[AnalysisContext] = None, + expansion: Optional[DemandExpansion] = None, + resolved_ids: Optional[Sequence[tuple[int, int]]] = None, ) -> FlowIterationResult: """Analyze traffic demand placement success rates using Core directly. - This function: - 1. Builds Core infrastructure (graph, algorithms, flow_graph) or uses cached - 2. Expands demands into concrete (src, dst, volume) tuples - 3. Places each demand using SPF caching for cacheable policies - 4. Uses FlowPolicy for complex multi-flow policies - 5. Aggregates results into FlowIterationResult + Steps: + 1. Build Core infrastructure (graph, algorithms, flow_graph), or reuse the + pre-built ``context`` + 2. Expand demands into concrete (src, dst, volume) tuples (or use a + pre-computed expansion) + 3. Place each demand using SPF caching for cacheable policies. + SHORTEST_PATHS_* presets admit flow onto the cost-only shortest paths + of the base topology and drop overflow (IGP semantics); TE_* presets + reroute remaining volume onto residual-capacity paths. + 4. Fall back to FlowPolicy for presets outside CACHEABLE_PRESETS + 5. Aggregate results into FlowIterationResult SPF Caching Optimization: For cacheable policies (ECMP, WCMP, TE_WCMP_UNLIM), SPF results are @@ -211,22 +295,31 @@ def demand_placement_analysis( excluded_nodes: Set of node names to exclude temporarily. excluded_links: Set of link IDs to exclude temporarily. demands_config: List of demand configurations (serializable dicts). - placement_rounds: Number of placement optimization rounds (unused - Core handles internally). include_flow_details: When True, include cost_distribution per flow. include_used_edges: When True, include set of used edges per demand in entry data. - context: Pre-built AnalysisContext for fast repeated analysis. + context: Pre-built AnalysisContext, reused across calls. Must be built + from this same demands_config - pseudo node names embed demand + ids, so a context built from a different config raises ValueError + during endpoint resolution. See build_demand_placement_inputs. + expansion: Pre-computed DemandExpansion matching demands_config. When + provided, per-call demand reconstruction and expansion are skipped. + Must be built together with ``context`` (pseudo node names embed + demand ids) - see build_demand_placement_inputs. + resolved_ids: Pre-resolved (src_id, dst_id) pairs aligned with + expansion.demands. Only valid together with ``context``. Returns: FlowIterationResult describing this iteration. """ - traffic_demands = _reconstruct_traffic_demands(demands_config) - - # Phase 1: Expand demands (pure logic, returns names + augmentations) - expansion = expand_demands( - network, - traffic_demands, - default_policy_preset=FlowPolicyPreset.SHORTEST_PATHS_ECMP, - ) + if expansion is None: + traffic_demands = _reconstruct_traffic_demands(demands_config) + + # Phase 1: Expand demands (pure logic, returns names + augmentations) + expansion = expand_demands( + network, + traffic_demands, + default_policy_preset=FlowPolicyPreset.SHORTEST_PATHS_ECMP, + ) # Phase 2: Use cached context infrastructure or build fresh if context is not None: @@ -237,8 +330,8 @@ def demand_placement_analysis( network, augmentations=expansion.augmentations ) - node_mask = ctx._build_node_mask(excluded_nodes) - edge_mask = ctx._build_edge_mask(excluded_links) + node_mask = ctx.build_node_mask(excluded_nodes) + edge_mask = ctx.build_edge_mask(excluded_links) flow_graph = netgraph_core.FlowGraph(ctx.multidigraph) # Phase 3: Place demands using unified placement module @@ -249,6 +342,7 @@ def demand_placement_analysis( ctx, node_mask, edge_mask, + resolved_ids=resolved_ids, collect_entries=True, include_cost_distribution=include_flow_details, include_used_edges=include_used_edges, @@ -285,6 +379,7 @@ def demand_placement_analysis( return FlowIterationResult(flows=flow_entries, summary=summary, data={}) +@_with_prepare_inputs(_prepare_maxflow_inputs) def sensitivity_analysis( network: "Network", excluded_nodes: Set[str], @@ -315,31 +410,38 @@ def sensitivity_analysis( shortest_path: If True, use single-tier shortest-path flow (IP/IGP mode). Reports only edges used under ECMP routing. If False (default), use full iterative max-flow (SDN/TE mode) and report all saturated edges. - flow_placement: Flow placement strategy. - context: Pre-built AnalysisContext for efficient repeated analysis. + flow_placement: PROPORTIONAL (WCMP) or EQUAL_BALANCED (ECMP). + context: Pre-built AnalysisContext reused across calls. Must be + unbound or bound to these same source/target/mode arguments. Returns: FlowIterationResult with sensitivity data in each FlowEntry.data. """ - # Convert string mode to Mode enum - mode_enum = Mode.COMBINE if mode == "combine" else Mode.PAIRWISE + # Convert string mode to Mode enum (raises on invalid values) + mode_enum = Mode.from_string(mode) - # Use provided context or create a new one + # Use provided context or create a new one. A bound context carries its + # own source/sink/mode; silently ignoring mismatched arguments would + # return results for the wrong pair, so reject the mismatch loudly. if context is not None: ctx = context + if ctx.is_bound and ( + ctx.bound_source != source + or ctx.bound_sink != target + or ctx.bound_mode != mode_enum + ): + raise ValueError( + "Provided context is bound to " + f"source={ctx.bound_source!r}, sink={ctx.bound_sink!r}, " + f"mode={ctx.bound_mode}, which differs from the analysis " + "arguments; rebuild the context or pass matching arguments." + ) else: ctx = analyze(network, source=source, sink=target, mode=mode_enum) - # Get max flow values for each pair - flow_values = ctx.max_flow( - shortest_path=shortest_path, - flow_placement=flow_placement, - excluded_nodes=excluded_nodes, - excluded_links=excluded_links, - ) - - # Get sensitivity (critical edges) for each pair - sensitivity_results = ctx.sensitivity( + # Get max flow and sensitivity (critical edges) for each pair in a + # single pass: masks are built once and the pairs are walked once. + combined = ctx.sensitivity_with_flow( shortest_path=shortest_path, flow_placement=flow_placement, excluded_nodes=excluded_nodes, @@ -350,8 +452,7 @@ def sensitivity_analysis( flow_entries: list[FlowEntry] = [] total_flow = 0.0 - for (src, dst), flow_value in flow_values.items(): - sensitivity_map = sensitivity_results.get((src, dst), {}) + for (src, dst), (flow_value, sensitivity_map) in combined.items(): entry = FlowEntry( source=str(src), destination=str(dst), @@ -376,25 +477,30 @@ def sensitivity_analysis( return FlowIterationResult(flows=flow_entries, summary=summary) -def build_demand_context( +def build_demand_placement_inputs( network: "Network", demands_config: list[dict[str, Any]], -) -> AnalysisContext: - """Build an AnalysisContext for repeated demand placement analysis. +) -> tuple[AnalysisContext, DemandExpansion, list[tuple[int, int]]]: + """Build context, expansion, and resolved node IDs for demand placement. - Pre-computes the graph with augmentations (pseudo source/target nodes) for - efficient repeated analysis with different exclusion sets. + Reconstructs and expands demands once so repeated calls to + demand_placement_analysis (e.g., Monte Carlo iterations) can skip the + per-iteration expansion and node-ID resolution work. Building the + expansion and context together guarantees that pseudo node names + (derived from demand ids) match the context's graph. Args: network: Network instance. - demands_config: List of demand configurations (same format as demand_placement_analysis). + demands_config: List of demand configurations (same format as + demand_placement_analysis). Returns: - AnalysisContext ready for use with demand_placement_analysis. + Tuple of (context, expansion, resolved_ids) where resolved_ids holds + (src_id, dst_id) pairs aligned with expansion.demands. """ traffic_demands = _reconstruct_traffic_demands(demands_config) - # Expand demands to get augmentations + # Expand demands once to get augmentations and concrete demands expansion = expand_demands( network, traffic_demands, @@ -402,7 +508,14 @@ def build_demand_context( ) # Build context with augmentations - return analyze(network, augmentations=expansion.augmentations) + context = analyze(network, augmentations=expansion.augmentations) + + # Pre-resolve node IDs once + resolved_ids = [ + (context.node_mapper.to_id(d.src_name), context.node_mapper.to_id(d.dst_name)) + for d in expansion.demands + ] + return context, expansion, resolved_ids def build_maxflow_context( @@ -425,5 +538,5 @@ def build_maxflow_context( Returns: AnalysisContext ready for use with max_flow_analysis or sensitivity_analysis. """ - mode_enum = Mode.COMBINE if mode == "combine" else Mode.PAIRWISE + mode_enum = Mode.from_string(mode) return analyze(network, source=source, sink=target, mode=mode_enum) diff --git a/ngraph/analysis/placement.py b/ngraph/analysis/placement.py index 6dafac0..81f1f8e 100644 --- a/ngraph/analysis/placement.py +++ b/ngraph/analysis/placement.py @@ -28,8 +28,18 @@ } ) +# Threshold for recording a placed amount as a flow entry. The core engine +# itself never augments below kMinFlow = 1/4096 (see NetGraph-Core +# constants.hpp), so any nonzero amount it returns clears this comfortably. _MIN_FLOW = 1e-9 +# Cached-path FlowIndex ids start far above the ids Core's FlowPolicy assigns +# internally (0..max_flow_count, <= 256), so a cached demand and a +# policy-based demand sharing (src, dst, priority) can never produce the same +# FlowIndex. Duplicate FlowIndex values silently merge flows in FlowGraph, +# corrupting placement totals. +_CACHED_FLOW_ID_BASE = 1 << 20 + @dataclass(slots=True) class PlacementSummary: @@ -62,7 +72,13 @@ class PlacementEntry: @dataclass(slots=True) class PlacementResult: - """Complete placement result.""" + """Result of one `place_demands` call. + + Attributes: + summary: Aggregated demand and placed totals. + entries: Per-demand results, or None unless the call passed + ``collect_entries=True``. + """ summary: PlacementSummary entries: list[PlacementEntry] | None = None @@ -105,35 +121,68 @@ def place_demands( collect_entries: bool = False, include_cost_distribution: bool = False, include_used_edges: bool = False, + dag_cache: dict[tuple[int, bool], tuple[np.ndarray, Any]] | None = None, ) -> PlacementResult: """Place demands on a flow graph with SPF caching. Args: demands: Expanded demands (policy_preset, priority, names). - volumes: Demand volumes (allows scaling without modifying demands). - flow_graph: Target FlowGraph. - ctx: AnalysisContext with graph infrastructure. - node_mask: Node inclusion mask. - edge_mask: Edge inclusion mask. - resolved_ids: Pre-resolved (src_id, dst_id) pairs. Computed if None. + volumes: Volume per demand, positionally aligned with `demands`; + passed separately so callers can scale without rebuilding demands. + flow_graph: Target FlowGraph; placed flow accumulates here. + ctx: AnalysisContext holding the built graph and Core algorithms. + node_mask: Node inclusion mask (True = include), as built by + ctx.build_node_mask. + edge_mask: Edge inclusion mask (True = include), as built by + ctx.build_edge_mask. + resolved_ids: Pre-resolved (src_id, dst_id) pairs. Computed from the + demand names if None. collect_entries: If True, populate result.entries. include_cost_distribution: Include cost distribution in entries. include_used_edges: Include used edges in entries. + dag_cache: Optional persistent SPF DAG cache keyed by + (src_id, uses_capacity_aware_selection). Base DAGs depend only on + the static graph and masks, so repeated calls with the same + context and masks (e.g. MSD probes) can share one cache. Returns: PlacementResult with summary and optional entries. + + Raises: + ValueError: If a demand endpoint is not present in ``ctx``'s graph + (checked only when ``resolved_ids`` is not supplied). Pseudo node + names embed demand ids, so this usually means the context was + built from a different demands_config. + ValueError: If two policy-based demands (presets outside + CACHEABLE_PRESETS) share the same (src, dst, priority): their + FlowIndex values would collide and silently merge in FlowGraph. + ValueError: If ``demands``, ``volumes``, and ``resolved_ids`` are not + all the same length. """ if resolved_ids is None: - resolved_ids = [ - (ctx.node_mapper.to_id(d.src_name), ctx.node_mapper.to_id(d.dst_name)) - for d in demands - ] - - dag_cache: dict[tuple[int, FlowPolicyPreset], tuple[np.ndarray, Any]] = {} + try: + resolved_ids = [ + (ctx.node_mapper.to_id(d.src_name), ctx.node_mapper.to_id(d.dst_name)) + for d in demands + ] + except KeyError as exc: + raise ValueError( + f"Demand endpoint {exc.args[0]!r} is not present in the " + "analysis context graph. The context was likely built from a " + "different demands_config (pseudo node names embed demand " + "ids); rebuild the context from the same config." + ) from exc + + if dag_cache is None: + dag_cache = {} entries: list[PlacementEntry] | None = [] if collect_entries else None total_demand = 0.0 total_placed = 0.0 - flow_idx_counter = 0 + flow_idx_counter = _CACHED_FLOW_ID_BASE + # Core's FlowPolicy assigns flow ids internally per policy instance, so + # two policy-based demands sharing (src, dst, priority) would produce + # colliding FlowIndex values and silently merge/steal each other's flows. + policy_triples: set[tuple[int, int, int]] = set() for demand, volume, (src_id, dst_id) in zip( demands, volumes, resolved_ids, strict=True @@ -157,6 +206,15 @@ def place_demands( include_used_edges, ) else: + triple = (src_id, dst_id, demand.priority) + if triple in policy_triples: + raise ValueError( + f"Duplicate policy-based demand for source '{demand.src_name}', " + f"destination '{demand.dst_name}', priority {demand.priority}: " + "flow ids would collide and corrupt placement. Merge the " + "demand volumes or use distinct priorities." + ) + policy_triples.add(triple) placed, cost_dist, used_edges = _place_with_policy( src_id, dst_id, @@ -198,7 +256,7 @@ def _place_cached( volume: float, priority: int, preset: FlowPolicyPreset, - dag_cache: dict[tuple[int, FlowPolicyPreset], tuple[np.ndarray, Any]], + dag_cache: dict[tuple[int, bool], tuple[np.ndarray, Any]], ctx: "AnalysisContext", flow_graph: netgraph_core.FlowGraph, node_mask: np.ndarray, @@ -208,10 +266,13 @@ def _place_cached( include_used_edges: bool, ) -> tuple[float, dict[float, float], set[str], int]: """Place single demand with SPF caching.""" - cache_key = (src_id, preset) selection = _get_edge_selection(preset) placement = _get_flow_placement(preset) is_te = preset in _CACHEABLE_TE + # ECMP and WCMP share one EdgeSelection; TE presets share the other. + # Keying by selection family (not preset) lets mixed workloads reuse + # the same base SPF DAG. + cache_key = (src_id, is_te) flow_indices: list[netgraph_core.FlowIndex] = [] flow_costs: list[tuple[float, float]] = [] @@ -299,9 +360,10 @@ def _place_cached( used_edges: set[str] = set() if include_used_edges: + ext_ids = ctx.multidigraph.ext_edge_ids_view() for fidx in flow_indices: for edge_id, _ in flow_graph.get_flow_edges(fidx): - ref = ctx.edge_mapper.to_ref(edge_id, ctx.multidigraph) + ref = ctx.edge_mapper.decode_ext_id(int(ext_ids[edge_id])) if ref: used_edges.add(f"{ref.link_id}:{ref.direction}") @@ -335,6 +397,7 @@ def _place_with_policy( used_edges: set[str] = set() if include_cost_distribution or include_used_edges: + ext_ids = ctx.multidigraph.ext_edge_ids_view() for flow_key, flow_data in policy.flows.items(): if include_cost_distribution: cost, flow_vol = float(flow_data[2]), float(flow_data[3]) @@ -346,7 +409,7 @@ def _place_with_policy( flow_key[0], flow_key[1], flow_key[2], flow_key[3] ) for edge_id, _ in flow_graph.get_flow_edges(fidx): - ref = ctx.edge_mapper.to_ref(edge_id, ctx.multidigraph) + ref = ctx.edge_mapper.decode_ext_id(int(ext_ids[edge_id])) if ref: used_edges.add(f"{ref.link_id}:{ref.direction}") diff --git a/ngraph/cli.py b/ngraph/cli.py index 90561f2..59c5de0 100644 --- a/ngraph/cli.py +++ b/ngraph/cli.py @@ -7,13 +7,14 @@ import logging import os import sys +from contextlib import contextmanager from pathlib import Path from statistics import median from time import perf_counter -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterator, List, Optional from ngraph.explorer import NetworkExplorer -from ngraph.logging import get_logger, set_global_log_level +from ngraph.logging import get_logger, set_global_log_level, setup_root_logger from ngraph.profiling.profiler import PerformanceProfiler, PerformanceReporter from ngraph.scenario import Scenario from ngraph.utils.output_paths import ( @@ -21,6 +22,7 @@ profiles_dir_for_run, results_path_for_run, ) +from ngraph.workflow.base import WorkflowStep logger = get_logger(__name__) @@ -34,12 +36,15 @@ def _format_table( """Format data as a simple ASCII table. Args: - headers: Column headers - rows: Data rows - min_width: Minimum column width + headers: Column labels, one per column. + rows: One list of cell values per row, each the same length as + ``headers``. + min_width: Floor for every column width, in characters. + max_col_width: Clip longer cells to this width with a trailing "...". + None leaves cells unclipped. Returns: - Formatted table string + The rendered table, or "" when ``rows`` is empty. """ if not rows: return "" @@ -55,20 +60,18 @@ def clip(val: Any) -> str: clipped_headers = [clip(h) for h in headers] clipped_rows = [[clip(item) for item in row] for row in rows] - # Calculate column widths from clipped content + # Widths come from the clipped content, not the originals. all_data = [clipped_headers] + clipped_rows col_widths = [] for col_idx in range(len(clipped_headers)): max_width = max(len(str(row[col_idx])) for row in all_data) col_widths.append(max(max_width, min_width)) - # Format rows def format_row(row_data: List[str]) -> str: return " " + " | ".join( f"{str(item):<{col_widths[i]}}" for i, item in enumerate(row_data) ) - # Build table lines = [] lines.append(format_row(clipped_headers)) lines.append(" " + "-+-".join("-" * width for width in col_widths)) @@ -116,22 +119,6 @@ def _format_duration(seconds: float) -> str: return f"{minutes}m {rem:.1f}s" -def _plural(n: int, singular: str, plural: Optional[str] = None) -> str: - """Return grammatically correct unit for count n. - - Args: - n: Count. - singular: Singular form. - plural: Optional plural form; defaults to singular + 's' when None. - - Returns: - Appropriate unit string for the count. - """ - if n == 1: - return singular - return plural or (singular + "s") - - def _collect_step_path_fields(step: Any) -> list[tuple[str, str]]: """Return (field, pattern) pairs for fields that represent node selectors. @@ -217,6 +204,21 @@ def _print_network_structure( enabled_links = [link for link in links.values() if not link.disabled] disabled_links = [link for link in links.values() if link.disabled] + # Per-node attached capacity and enabled-link counts, computed in a single + # O(E) pass shared by the detail-mode node table and the statistics block. + # Self-loops contribute once per link, matching per-node match semantics. + cap_by_node: Dict[str, float] = {} + link_count_by_node: Dict[str, int] = {} + for link in enabled_links: + endpoints = ( + (link.source,) if link.source == link.target else (link.source, link.target) + ) + for endpoint in endpoints: + cap_by_node[endpoint] = cap_by_node.get(endpoint, 0.0) + float( + link.capacity + ) + link_count_by_node[endpoint] = link_count_by_node.get(endpoint, 0) + 1 + enabled_nodes_pct = (len(enabled_nodes) / len(nodes) * 100.0) if nodes else 0.0 print(f" Enabled Nodes: {len(enabled_nodes):,} ({enabled_nodes_pct:.1f}%)") if disabled_nodes: @@ -229,8 +231,9 @@ def _print_network_structure( # Network hierarchy analysis if nodes: - original_level = logger.level - logger.setLevel(logging.WARNING) + pkg_logger = logging.getLogger("ngraph") + original_level = pkg_logger.level + pkg_logger.setLevel(logging.WARNING) explorer = None try: # Use non-strict validation so hierarchy is printable even with issues @@ -253,12 +256,12 @@ def _print_network_structure( except Exception as e: print(f" Network Hierarchy: Unable to analyze ({e})") finally: - logger.setLevel(original_level) + pkg_logger.setLevel(original_level) # Hardware utilization and validation summary (non-fatal) try: if explorer is not None: - node_utils = explorer.get_node_utilization(include_disabled=False) + node_utils = explorer.get_node_utilization() link_issues = explorer.get_link_issues() # Node capacity violations @@ -378,14 +381,8 @@ def _print_network_structure( node = nodes[node_name] status = "disabled" if node.disabled else "enabled" - # Calculate total capacity and link count for this node - node_capacity = 0 - node_link_count = 0 - for link in links.values(): - if link.source == node_name or link.target == node_name: - if not link.disabled: - node_capacity += link.capacity - node_link_count += 1 + node_capacity = cap_by_node.get(node_name, 0.0) + node_link_count = link_count_by_node.get(node_name, 0) capacity_str = f"{node_capacity:,.0f}" if node_capacity > 0 else "0" @@ -447,14 +444,8 @@ def _print_network_structure( # Node capacity analysis if nodes and links: print("\n Node Capacity Statistics:") - node_capacities = [] - for node_name in nodes.keys(): - node_capacity = 0 - for link in enabled_links: - if link.source == node_name or link.target == node_name: - node_capacity += link.capacity - if node_capacity > 0: # Only include nodes with links - node_capacities.append(node_capacity) + # Only include nodes with enabled links attached + node_capacities = [cap_by_node[n] for n in nodes if cap_by_node.get(n, 0.0) > 0] if node_capacities: node_cap_table = _format_table( @@ -590,7 +581,7 @@ def _print_demand_sets( for demands in ds.sets.values(): grand_demand_count += len(demands) for d in demands: - grand_total_demand += float(getattr(d, "volume", 0.0)) + grand_total_demand += float(d.volume) print("\n Capacity vs Demand:") print(f" enabled link capacity: {total_enabled_link_capacity:,.1f}") @@ -610,7 +601,7 @@ def _print_demand_sets( set_items = list(ds.sets.items())[:5] for set_name, demands in set_items: demand_count = len(demands) - total_volume = sum(getattr(d, "volume", 0.0) for d in demands) + total_volume = sum(d.volume for d in demands) print( f" {set_name}: {demand_count} demand{'s' if demand_count != 1 else ''}" ) @@ -627,7 +618,7 @@ def _print_demand_sets( key = (src_key, tgt_key) stats = pair_counts.setdefault(key, {"count": 0, "volume": 0.0}) stats["count"] = int(stats["count"]) + 1 - stats["volume"] = float(stats["volume"]) + float(getattr(d, "volume", 0.0)) + stats["volume"] = float(stats["volume"]) + float(d.volume) if detail: print(f" total demand: {total_volume:,.0f}") @@ -682,7 +673,7 @@ def _print_demand_sets( top_n = 5 sorted_demands = sorted( demands, - key=lambda d: float(getattr(d, "demand", 0.0)), + key=lambda d: float(d.volume), reverse=True, )[:top_n] if sorted_demands: @@ -695,8 +686,8 @@ def _print_demand_sets( [ src, tgt, - f"{float(getattr(d, 'volume', 0.0)):,.1f}", - str(getattr(d, "priority", 0)), + f"{float(d.volume):,.1f}", + str(d.priority), ] ) top_table = _format_table( @@ -884,11 +875,7 @@ def _inspect_scenario(path: Path, detail: bool = False) -> None: "Scenario loaded: nodes=%d, links=%d, steps=%d, policies=%d, demand_sets=%d", len(getattr(scenario.network, "nodes", {})), len(getattr(scenario.network, "links", {})), - len( - getattr(scenario.workflow, "__iter__", []) - and list(scenario.workflow) - or [] - ), + len(scenario.workflow), len(getattr(scenario.failure_policy_set, "policies", {})), len(getattr(scenario.demand_set, "sets", {})), ) @@ -920,7 +907,7 @@ def _inspect_scenario(path: Path, detail: bool = False) -> None: for demands in ds.sets.values(): total_demands += len(demands) for d in demands: - total_demand_volume += float(getattr(d, "volume", 0.0)) + total_demand_volume += float(d.volume) util = ( (total_demand_volume / total_enabled_capacity) @@ -1047,14 +1034,15 @@ def _run_scenario( Args: path: Scenario YAML file. - output: Optional explicit path where JSON results should be written. When - ``None``, defaults to ``.results.json`` in the current directory, - or under ``--output`` if provided. + results_override: Optional explicit path for JSON results. When ``None``, + the path is derived from the scenario name under ``output_dir``. no_results: Whether to disable results file generation. stdout: Whether to also print results to stdout. keys: Optional list of workflow step names to include. When ``None`` all steps are exported. profile: Whether to enable performance profiling with CPU analysis. + profile_memory: Whether the profiler also tracks memory usage. + output_dir: Base directory for derived output paths (results, profiles). """ logger.info(f"Loading scenario from: {path}") _start_time = perf_counter() @@ -1065,35 +1053,45 @@ def _run_scenario( if profile: logger.info("Performance profiling enabled") - # Initialize detailed profiler profiler = PerformanceProfiler(track_memory=profile_memory) - - # Start scenario-level profiling profiler.start_scenario() logger.info("Starting scenario execution with profiling") - # Enable child-process profiling for parallel workflows + # Enable worker-thread profiling for parallel workflows child_profile_dir = profiles_dir_for_run(path, output_dir) child_profile_dir.mkdir(parents=True, exist_ok=True) + prev_profile_dir = os.environ.get("NGRAPH_PROFILE_DIR") os.environ["NGRAPH_PROFILE_DIR"] = str(child_profile_dir.resolve()) logger.info(f"Worker profiles will be saved to: {child_profile_dir}") - # Manual execution of workflow steps with profiling - for step in scenario.workflow: - step_name = step.name or step.__class__.__name__ - step_type = step.__class__.__name__ - - with profiler.profile_step(step_name, step_type): - step.execute(scenario) + @contextmanager + def _profile_step_hook(step: WorkflowStep) -> Iterator[None]: + """Wrap step execution with profiling and worker-profile merge. + Worker profiles are merged only after the profiled block exits + cleanly; exceptions raised by the step propagate unchanged and + skip the merge. + """ + step_name = step.name or step.__class__.__name__ + with profiler.profile_step(step_name, step.__class__.__name__): + yield # Merge any worker profiles generated by this step if child_profile_dir.exists(): profiler.merge_child_profiles(child_profile_dir, step_name) + try: + scenario.run(step_hook=_profile_step_hook) + finally: + # Restore the environment so later in-process runs do not + # inherit profiling into a stale directory. + if prev_profile_dir is None: + os.environ.pop("NGRAPH_PROFILE_DIR", None) + else: + os.environ["NGRAPH_PROFILE_DIR"] = prev_profile_dir + logger.info("Scenario execution completed successfully") - # End scenario profiling and analyze results profiler.end_scenario() profiler.analyze_performance() @@ -1117,19 +1115,20 @@ def _run_scenario( except Exception as exc: logger.debug("Failed to remove profiles dir: %s", exc) - # Generate and display performance report + # Report goes to stderr so machine-readable stdout (--stdout) + # stays pure JSON. reporter = PerformanceReporter(profiler.results) performance_report = reporter.generate_report() - print("\n" + performance_report) + print("\n" + performance_report, file=sys.stderr) else: logger.info("Starting scenario execution") scenario.run() logger.info("Scenario execution completed successfully") - print("✅ Scenario execution completed") + print("✅ Scenario execution completed", file=sys.stderr) # Export JSON results by default unless disabled - if not no_results: + if not no_results or stdout: logger.info("Serializing results to JSON") results_dict: Dict[str, Any] = scenario.results.to_dict() @@ -1143,32 +1142,22 @@ def _run_scenario( json_str = json.dumps(results_dict, indent=2, default=str) - # Derive default results file path using output directory policy - effective_output = results_path_for_run( - scenario_path=path, - output_dir=output_dir, - results_override=results_override, - ) + if not no_results: + # Derive default results file path using output directory policy + effective_output = results_path_for_run( + scenario_path=path, + output_dir=output_dir, + results_override=results_override, + ) - ensure_parent_dir(effective_output) - logger.info(f"Writing results to: {effective_output}") - effective_output.write_text(json_str) - logger.info("Results written successfully") - print(f"✅ Results written to: {effective_output}") + ensure_parent_dir(effective_output) + logger.info(f"Writing results to: {effective_output}") + effective_output.write_text(json_str) + logger.info("Results written successfully") + print(f"✅ Results written to: {effective_output}", file=sys.stderr) if stdout: print(json_str) - elif stdout: - # Print to stdout even without file export - results_dict: Dict[str, Any] = scenario.results.to_dict() - if keys: - steps_map = results_dict.get("steps", {}) - filtered_steps: Dict[str, Any] = { - step: steps_map[step] for step in keys if step in steps_map - } - results_dict["steps"] = filtered_steps - json_str = json.dumps(results_dict, indent=2, default=str) - print(json_str) # Final success duration log _elapsed = perf_counter() - _start_time @@ -1178,11 +1167,14 @@ def _run_scenario( except FileNotFoundError: logger.error(f"Scenario file not found: {path}") - print(f"❌ ERROR: Scenario file not found: {path}") + print(f"❌ ERROR: Scenario file not found: {path}", file=sys.stderr) sys.exit(1) except Exception as e: logger.error(f"Failed to run scenario: {type(e).__name__}: {e}") - print(f"❌ ERROR: Failed to run scenario: {type(e).__name__}: {e}") + print( + f"❌ ERROR: Failed to run scenario: {type(e).__name__}: {e}", + file=sys.stderr, + ) sys.exit(1) @@ -1290,6 +1282,7 @@ def main(argv: Optional[List[str]] = None) -> None: args = parser.parse_args(effective_args) # Configure logging based on arguments + setup_root_logger() if args.verbose: set_global_log_level(logging.DEBUG) logger.debug("Debug logging enabled") diff --git a/ngraph/dsl/__init__.py b/ngraph/dsl/__init__.py index 8b738fc..ecb287a 100644 --- a/ngraph/dsl/__init__.py +++ b/ngraph/dsl/__init__.py @@ -1,7 +1,6 @@ """Domain-specific language (DSL) for defining networks and reusable blueprints. -This package provides high-level constructs to describe network topologies in -YAML. Compile DSL definitions into a concrete +Describes network topologies in YAML. Compile DSL definitions into a concrete `ngraph.model.network.Network` with `ngraph.dsl.blueprints.expand.expand_network_dsl`. """ diff --git a/ngraph/dsl/blueprints/__init__.py b/ngraph/dsl/blueprints/__init__.py index d63dea5..480ff50 100644 --- a/ngraph/dsl/blueprints/__init__.py +++ b/ngraph/dsl/blueprints/__init__.py @@ -1,5 +1,5 @@ """Blueprint DSL types and expansion utilities. -This subpackage defines blueprint structures and expansion helpers that turn -group and adjacency patterns into a `ngraph.model.network.Network`. +Blueprint structures plus the expansion helpers that turn group and adjacency +patterns into a `ngraph.model.network.Network`. """ diff --git a/ngraph/dsl/blueprints/expand.py b/ngraph/dsl/blueprints/expand.py index 7f5ae03..7c399c4 100644 --- a/ngraph/dsl/blueprints/expand.py +++ b/ngraph/dsl/blueprints/expand.py @@ -1,31 +1,37 @@ -"""Network topology blueprints and generation.""" +"""Blueprint and network DSL expansion. + +Turns the `blueprints:` and `network:` sections into concrete Node and Link +objects: resolves blueprint instantiation and parameter overrides, expands +bracket patterns and `expand:` variable blocks, applies node and link rules, +and materializes `mesh`/`one_to_one` link patterns. +""" from __future__ import annotations import copy +import re from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Set +from typing import Any, Dict, Iterable, List, Optional, Set from ngraph.dsl.blueprints import parser as _bp_parse from ngraph.dsl.expansion import ( ExpansionSpec, expand_block, expand_risk_group_refs, - expand_templates, ) -from ngraph.dsl.selectors import ( +from ngraph.dsl.selectors import normalize_selector +from ngraph.model.network import Link, Network, Node +from ngraph.model.selectors import ( evaluate_conditions, flatten_link_attrs, - normalize_selector, parse_match_spec, select_nodes, ) -from ngraph.model.network import Link, Network, Node @dataclass class Blueprint: - """Represents a reusable blueprint for hierarchical sub-topologies. + """Reusable blueprint for hierarchical sub-topologies. A blueprint may contain multiple node definitions (each can have count and template), plus link definitions describing how those nodes connect. @@ -112,7 +118,7 @@ def expand_network_dsl(data: Dict[str, Any]) -> Network: raise ValueError("'network' must be a dictionary if present.") net = Network() - # Pull recognized top-level fields from network_data + # Reject unrecognized top-level fields before anything is expanded for key in network_data.keys(): if key not in ( "name", @@ -129,7 +135,6 @@ def expand_network_dsl(data: Dict[str, Any]) -> Network: if "version" in network_data: net.attrs["version"] = network_data["version"] - # Create a context ctx = DSLExpansionContext(blueprints=blueprint_map, network=net) # 3) Expand top-level node definitions @@ -145,7 +150,7 @@ def expand_network_dsl(data: Dict[str, Any]) -> Network: # 5) Expand deferred blueprint links for _link_def, _parent in ctx.pending_bp_links: - _expand_blueprint_link(ctx, _link_def, _parent) + _expand_link(ctx, _link_def, _parent, context="blueprint link") # 6) Expand top-level link definitions for link_def in network_data.get("links", []): @@ -159,6 +164,66 @@ def expand_network_dsl(data: Dict[str, Any]) -> Network: return net +def _parent_merge_context( + group_def: Dict[str, Any], + group_name: str, + inherited_risk_groups: Set[str], +) -> tuple[Dict[str, Any], bool, Set[str]]: + """Extract the parent attrs/disabled/risk_groups merged into child groups. + + Args: + group_def: Parent group definition. + group_name: Parent group name, used in error messages. + inherited_risk_groups: Risk groups inherited from a higher-level group. + + Returns: + Tuple of (deep-copied parent attrs, parent disabled flag, parent risk + groups unioned with the inherited ones). + + Raises: + ValueError: If 'attrs' on the parent group is not a dict, or + 'risk_groups' is not a list/set/tuple of names. + """ + parent_attrs = copy.deepcopy(group_def.get("attrs", {})) + if not isinstance(parent_attrs, dict): + raise ValueError(f"'attrs' must be a dict in node '{group_name}'.") + parent_disabled = bool(group_def.get("disabled", False)) + parent_risk_groups = set(inherited_risk_groups) + if "risk_groups" in group_def: + parent_risk_groups |= expand_risk_group_refs(group_def["risk_groups"]) + return parent_attrs, parent_disabled, parent_risk_groups + + +def _merge_parent_into_child( + merged_def: Dict[str, Any], + context: str, + parent_attrs: Dict[str, Any], + parent_disabled: bool, + parent_risk_groups: Set[str], +) -> None: + """Merge parent's disabled/attrs/risk_groups into a child definition in place. + + Args: + merged_def: Child definition, mutated in place. + context: Child description used in error messages. + parent_attrs: Parent attrs; child keys win on conflict. + parent_disabled: When True, forces the child disabled. + parent_risk_groups: Unioned with the child's own risk_groups. + + Raises: + ValueError: If the child's 'attrs' is not a dict, or its 'risk_groups' + is not a list/set/tuple of names. + """ + if parent_disabled: + merged_def["disabled"] = True + child_attrs = merged_def.get("attrs", {}) + if not isinstance(child_attrs, dict): + raise ValueError(f"Node {context} has non-dict 'attrs'.") + merged_def["attrs"] = {**parent_attrs, **child_attrs} + child_rgs = expand_risk_group_refs(merged_def.get("risk_groups", [])) + merged_def["risk_groups"] = parent_risk_groups | child_rgs + + def _expand_node_group( ctx: DSLExpansionContext, parent_path: str, @@ -172,13 +237,13 @@ def _expand_node_group( - A direct node group (with count, etc.), - Possibly replicating itself if group_name has bracket expansions. - If 'blueprint' is present, we expand that blueprint. If 'nodes' is present, - we recurse for nested groups. Otherwise, we create nodes directly. + A 'blueprint' key expands that blueprint; a 'nodes' key recurses for nested + groups; otherwise nodes are created directly. For blueprint usage: Allowed keys: {"blueprint", "params", "attrs", "disabled", "risk_groups"}. - We merge 'attrs', 'disabled', and 'risk_groups' from this parent - into each blueprint node definition. + The parent's 'attrs', 'disabled', and 'risk_groups' are merged into + each blueprint node definition. For nested nodes: Allowed keys: {"nodes", "attrs", "disabled", "risk_groups"}. @@ -227,46 +292,53 @@ def _expand_node_group( f"Node '{group_name}' references unknown blueprint '{blueprint_name}'." ) - parent_attrs = copy.deepcopy(group_def.get("attrs", {})) - if not isinstance(parent_attrs, dict): - raise ValueError(f"'attrs' must be a dict in node '{group_name}'.") - parent_disabled = bool(group_def.get("disabled", False)) - - # Merge parent's risk_groups - parent_risk_groups = set(inherited_risk_groups) - if "risk_groups" in group_def: - rg_val = group_def["risk_groups"] - if not isinstance(rg_val, (list, set)): - raise ValueError( - f"'risk_groups' must be list or set in node '{group_name}'." - ) - parent_risk_groups |= expand_risk_group_refs(rg_val) + parent_attrs, parent_disabled, parent_risk_groups = _parent_merge_context( + group_def, group_name, inherited_risk_groups + ) param_overrides: Dict[str, Any] = group_def.get("params", {}) if not isinstance(param_overrides, dict): raise ValueError(f"'params' must be a dict in node '{group_name}'.") - # For each node in the blueprint, apply param overrides and - # merge parent's attrs/disabled/risk_groups - for bp_sub_name, bp_sub_def in bp.nodes.items(): - merged_def = _apply_parameters(bp_sub_name, bp_sub_def, param_overrides) - merged_def = dict(merged_def) # ensure we can mutate - - # Force disabled if parent is disabled - if parent_disabled: - merged_def["disabled"] = True - - # Merge parent's attrs - child_attrs = merged_def.get("attrs", {}) - if not isinstance(child_attrs, dict): + # Validate override keys against the literal (unexpanded) blueprint + # subgroup names so typos and unsupported deep 'params.*' dotted + # paths fail loudly instead of silently expanding with defaults. + # Keys are matched by longest '.' prefix (not first dotted + # segment) so subgroup names containing dots remain addressable. + # The resolved key -> group mapping is reused by _apply_parameters. + override_to_group: Dict[str, str] = {} + for override_key in param_overrides: + if "." not in override_key or override_key in bp.nodes: + raise ValueError( + f"params override '{override_key}' in node '{group_name}' " + "must be of the form '.'. To override a " + "nested blueprint's params, use a dict value, e.g. " + "'.params': {'.': value}." + ) + matched_group = _longest_subgroup_prefix(override_key, bp.nodes) + if matched_group is None: + available = ", ".join(sorted(bp.nodes)) or "" raise ValueError( - f"Node '{bp_sub_name}' has non-dict 'attrs' inside blueprint '{blueprint_name}'." + f"params override '{override_key}' in node '{group_name}' " + f"matches no node group in blueprint '{blueprint_name}' " + f"(available: {available})." ) - merged_def["attrs"] = {**parent_attrs, **child_attrs} + override_to_group[override_key] = matched_group - # Merge parent's risk_groups with child's - child_rgs = expand_risk_group_refs(merged_def.get("risk_groups", [])) - merged_def["risk_groups"] = parent_risk_groups | child_rgs + # For each node in the blueprint, apply param overrides and + # merge parent's attrs/disabled/risk_groups + for bp_sub_name, bp_sub_def in bp.nodes.items(): + # _apply_parameters returns a deep copy, safe to mutate + merged_def = _apply_parameters( + bp_sub_name, bp_sub_def, param_overrides, override_to_group + ) + _merge_parent_into_child( + merged_def, + f"'{bp_sub_name}' inside blueprint '{blueprint_name}'", + parent_attrs, + parent_disabled, + parent_risk_groups, + ) # Recursively expand _expand_node_group( @@ -289,18 +361,9 @@ def _expand_node_group( context=f"nested node '{group_name}'", ) - parent_attrs = copy.deepcopy(group_def.get("attrs", {})) - parent_disabled = bool(group_def.get("disabled", False)) - - # Merge parent's risk_groups - parent_risk_groups = set(inherited_risk_groups) - if "risk_groups" in group_def: - rg_val = group_def["risk_groups"] - if not isinstance(rg_val, (list, set)): - raise ValueError( - f"'risk_groups' must be list or set in node '{group_name}'." - ) - parent_risk_groups |= expand_risk_group_refs(rg_val) + parent_attrs, parent_disabled, parent_risk_groups = _parent_merge_context( + group_def, group_name, inherited_risk_groups + ) # Recursively process nested nodes nested_nodes = group_def["nodes"] @@ -313,20 +376,13 @@ def _expand_node_group( f"Nested node definition for '{nested_name}' must be a dict." ) merged_def = dict(nested_def) - - # Force disabled if parent is disabled - if parent_disabled: - merged_def["disabled"] = True - - # Merge parent's attrs - child_attrs = merged_def.get("attrs", {}) - if not isinstance(child_attrs, dict): - child_attrs = {} - merged_def["attrs"] = {**parent_attrs, **child_attrs} - - # Merge parent's risk_groups with child's - child_rgs = expand_risk_group_refs(merged_def.get("risk_groups", [])) - merged_def["risk_groups"] = parent_risk_groups | child_rgs + _merge_parent_into_child( + merged_def, + f"'{nested_name}' in '{group_name}'", + parent_attrs, + parent_disabled, + parent_risk_groups, + ) _expand_node_group( ctx, @@ -343,15 +399,9 @@ def _expand_node_group( allowed={"count", "template", "attrs", "disabled", "risk_groups"}, context=f"node '{group_name}'", ) - combined_attrs = copy.deepcopy(group_def.get("attrs", {})) - if not isinstance(combined_attrs, dict): - raise ValueError(f"attrs must be a dict in node '{group_name}'.") - group_disabled = bool(group_def.get("disabled", False)) - - # Merge parent's risk groups - parent_risk_groups = set(inherited_risk_groups) - child_rgs = expand_risk_group_refs(group_def.get("risk_groups", [])) - final_risk_groups = parent_risk_groups | child_rgs + combined_attrs, group_disabled, final_risk_groups = _parent_merge_context( + group_def, group_name, inherited_risk_groups + ) # Check if this is a simple single node (no count, no template) has_count = "count" in group_def @@ -388,6 +438,26 @@ def _expand_node_group( ctx.network.add_node(node) +def _join_parent(base: str, rel: str) -> str: + """Join a literal parent path with a user-supplied relative regex. + + The parent path is a literal node-name prefix (e.g. a blueprint + instantiation path), not a regex, so its metacharacters (e.g. '.' in + group names like 'dc.1') must be escaped before the joined path is + compiled as a regex. re.escape("") == "" so top-level links (base="") + are unaffected, and "/" is never escaped so multi-level parents join + cleanly. + + Args: + base: Literal parent path prefix. + rel: User-supplied relative path regex. + + Returns: + Combined path with the parent prefix regex-escaped. + """ + return _bp_parse.join_paths(re.escape(base), rel) + + def _normalize_link_selector(sel: Any, base: str) -> Dict[str, Any]: """Normalize a source/target selector for link expansion. @@ -399,23 +469,16 @@ def _normalize_link_selector(sel: Any, base: str) -> Dict[str, Any]: Normalized selector dict. """ if isinstance(sel, str): - return {"path": _bp_parse.join_paths(base, sel)} + return {"path": _join_parent(base, sel)} if isinstance(sel, dict): + # NodeSelector.__post_init__ validates that at least one of path, + # group_by, or match is present when the selector is parsed. path = sel.get("path") - group_by = sel.get("group_by") - match = sel.get("match") - - # Validate: must have path, group_by, or match - if path is None and group_by is None and match is None: - raise ValueError( - "Selector object must contain 'path', 'group_by', or 'match'." - ) - out = dict(sel) if path is not None: if not isinstance(path, str): raise ValueError("Selector 'path' must be a string.") - out["path"] = _bp_parse.join_paths(base, path) + out["path"] = _join_parent(base, path) return out raise ValueError( "Link 'source'/'target' must be string or object with " @@ -423,52 +486,31 @@ def _normalize_link_selector(sel: Any, base: str) -> Dict[str, Any]: ) -def _expand_blueprint_link( +def _expand_link( ctx: DSLExpansionContext, link_def: Dict[str, Any], - parent_path: str, + parent_path: str = "", + context: str = "top-level link", ) -> None: - """Expands link definitions from within a blueprint, using parent_path - as the local root. Handles optional expand: block for repeated links. + """Expand a link definition into Link objects. - Args: - ctx: The context object with blueprint info and the network. - link_def: The link definition inside the blueprint. - parent_path: The path serving as the base for the blueprint's node paths. - """ - _bp_parse.check_link_keys(link_def, context="blueprint link") - - # Check for expand block - expand_spec = ExpansionSpec.from_dict(link_def) - if expand_spec and not expand_spec.is_empty(): - _expand_link_with_variables(ctx, link_def, parent_path) - return - - source_rel = link_def["source"] - target_rel = link_def["target"] - pattern = link_def.get("pattern", "mesh") - count = link_def.get("count", 1) - - src_sel = _normalize_link_selector(source_rel, parent_path) - tgt_sel = _normalize_link_selector(target_rel, parent_path) - - _expand_link_pattern(ctx, src_sel, tgt_sel, pattern, link_def, count) - - -def _expand_link(ctx: DSLExpansionContext, link_def: Dict[str, Any]) -> None: - """Expands a top-level link definition from 'network.links'. - If expand: block is provided, we expand the source/target as templates. + Used for both top-level 'network.links' entries (parent_path="") and + deferred blueprint links (parent_path is the blueprint instantiation + path). If an expand: block is provided, the definition is replicated + per variable combination. Args: ctx: The context containing the target network. link_def: The link definition dict. + parent_path: Prepended to source/target paths ("" for top-level + links, the blueprint base path for blueprint links). + context: Short description used in error messages. """ - _bp_parse.check_link_keys(link_def, context="top-level link") + _bp_parse.check_link_keys(link_def, context=context) - # Check for expand block expand_spec = ExpansionSpec.from_dict(link_def) if expand_spec and not expand_spec.is_empty(): - _expand_link_with_variables(ctx, link_def, parent_path="") + _expand_link_with_variables(ctx, link_def, expand_spec, parent_path) return source_raw = link_def["source"] @@ -476,88 +518,44 @@ def _expand_link(ctx: DSLExpansionContext, link_def: Dict[str, Any]) -> None: pattern = link_def.get("pattern", "mesh") count = link_def.get("count", 1) - src_sel = _normalize_link_selector(source_raw, "") - tgt_sel = _normalize_link_selector(target_raw, "") + src_sel = _normalize_link_selector(source_raw, parent_path) + tgt_sel = _normalize_link_selector(target_raw, parent_path) _expand_link_pattern(ctx, src_sel, tgt_sel, pattern, link_def, count) def _expand_link_with_variables( - ctx: DSLExpansionContext, link_def: Dict[str, Any], parent_path: str + ctx: DSLExpansionContext, + link_def: Dict[str, Any], + expand_spec: ExpansionSpec, + parent_path: str, ) -> None: - """Handles link expansions when 'expand' block is provided. + """Expand a link definition once per combination in its 'expand' block. - Substitutes variables into 'source' and 'target' templates using $var or ${var} - syntax to produce multiple link expansions. Supports both string paths - and dict selectors (with path/group_by). + Substitutes $var or ${var} variables in all string fields of the link + definition - source/target selectors (including match condition values), + attrs, and risk_groups - yielding one full link definition per variable + combination, consistent with node_rules/link_rules expansion. Args: ctx: The DSL expansion context. link_def: The link definition including expand block, source, target, etc. + expand_spec: Parsed expansion spec for link_def (computed by caller). parent_path: Prepended to source/target paths. """ - source_template = link_def["source"] - target_template = link_def["target"] - pattern = link_def.get("pattern", "mesh") - count = link_def.get("count", 1) - - # Get expansion spec from expand: block - expand_spec = ExpansionSpec.from_dict(link_def) - if expand_spec is None: - expand_spec = ExpansionSpec(vars={}, mode="cartesian") - - # Collect all string fields that need variable substitution - templates = _extract_selector_templates(source_template, "source") - templates.update(_extract_selector_templates(target_template, "target")) - - if not templates: - # No variables to expand - just process once - src_sel = _normalize_link_selector(source_template, parent_path) - tgt_sel = _normalize_link_selector(target_template, parent_path) - _expand_link_pattern(ctx, src_sel, tgt_sel, pattern, link_def, count) - return - - # Expand templates and rebuild selectors - for substituted in expand_templates(templates, expand_spec): - src_sel = _rebuild_selector(source_template, substituted, "source", parent_path) - tgt_sel = _rebuild_selector(target_template, substituted, "target", parent_path) - _expand_link_pattern(ctx, src_sel, tgt_sel, pattern, link_def, count) - - -def _extract_selector_templates(selector: Any, prefix: str) -> Dict[str, str]: - """Extract string fields from a selector that may contain variables.""" - templates: Dict[str, str] = {} - if isinstance(selector, str): - templates[prefix] = selector - elif isinstance(selector, dict): - if "path" in selector and isinstance(selector["path"], str): - templates[f"{prefix}.path"] = selector["path"] - if "group_by" in selector and isinstance(selector["group_by"], str): - templates[f"{prefix}.group_by"] = selector["group_by"] - return templates - - -def _rebuild_selector( - original: Any, substituted: Dict[str, str], prefix: str, parent_path: str -) -> Dict[str, Any]: - """Rebuild a selector with substituted values.""" - if isinstance(original, str): - path = substituted.get(prefix, original) - return {"path": _bp_parse.join_paths(parent_path, path)} - - if isinstance(original, dict): - result = dict(original) - if f"{prefix}.path" in substituted: - result["path"] = _bp_parse.join_paths( - parent_path, substituted[f"{prefix}.path"] - ) - elif "path" in result: - result["path"] = _bp_parse.join_paths(parent_path, result["path"]) - if f"{prefix}.group_by" in substituted: - result["group_by"] = substituted[f"{prefix}.group_by"] - return result - - raise ValueError(f"Selector must be string or dict, got {type(original)}") + # expand_block deep-copies, drops 'expand', and substitutes variables + # recursively in every string field of the definition. + for substituted in expand_block(link_def, expand_spec): + src_sel = _normalize_link_selector(substituted["source"], parent_path) + tgt_sel = _normalize_link_selector(substituted["target"], parent_path) + _expand_link_pattern( + ctx, + src_sel, + tgt_sel, + substituted.get("pattern", "mesh"), + substituted, + substituted.get("count", 1), + ) def _expand_link_pattern( @@ -571,13 +569,17 @@ def _expand_link_pattern( """Generates Link objects for the chosen link pattern among matched nodes. Supported Patterns: - * "mesh": Connect every source node to every target node - (no self-loops, deduplicate reversed pairs). + * "mesh": Connect every source node to every target node. * "one_to_one": Pair each source node with exactly one target node using wrap-around. The larger set size must be a multiple of the smaller set size. - Link properties are now flat in link_def (capacity, cost, disabled, + Both patterns skip self-pairs and deduplicate reversed pairs. That + deduplication is scoped to a single call: each variable combination of an + expand: block is an independent link definition, so overlapping selections + across combinations create parallel links. + + Link properties are flat in link_def (capacity, cost, disabled, risk_groups, attrs). Args: @@ -662,9 +664,9 @@ def _create_link( link_def: Dict[str, Any], count: int = 1, ) -> None: - """Creates and adds one or more Links to the network. + """Create and add one or more Links to the network. - Link properties are now flat in link_def (capacity, cost, disabled, + Link properties are flat in link_def (capacity, cost, disabled, risk_groups, attrs). Args: @@ -694,14 +696,14 @@ def _create_link( def _process_node_rules(net: Network, network_data: Dict[str, Any]) -> None: - """Processes the 'node_rules' section of the network DSL, updating + """Process the 'node_rules' section of the network DSL, updating existing nodes with new attributes in bulk. Rules are applied in order if multiple items match the same node. Each rule must have {"path"} plus optionally {"attrs", "disabled", "risk_groups"}. - - If "disabled" is present, we set node.disabled. - - If "risk_groups" is present, we *replace* the node's risk_groups. + - "disabled" sets node.disabled. + - "risk_groups" *replaces* the node's risk_groups. - Everything else merges into node.attrs. Args: @@ -710,7 +712,7 @@ def _process_node_rules(net: Network, network_data: Dict[str, Any]) -> None: """ node_rules = network_data.get("node_rules", []) if not isinstance(node_rules, list): - return + raise ValueError("'node_rules' must be a list.") for rule in node_rules: if not isinstance(rule, dict): @@ -721,7 +723,6 @@ def _process_node_rules(net: Network, network_data: Dict[str, Any]) -> None: context="node rule", ) - # Handle expand block expand_spec = ExpansionSpec.from_dict(rule) if expand_spec and not expand_spec.is_empty(): for expanded_rule in expand_block(rule, expand_spec): @@ -762,7 +763,7 @@ def _process_link_rules(net: Network, network_data: Dict[str, Any]) -> None: """ link_rules = network_data.get("link_rules", []) if not isinstance(link_rules, list): - return + raise ValueError("'link_rules' must be a list.") for link_rule in link_rules: if not isinstance(link_rule, dict): @@ -783,8 +784,9 @@ def _process_link_rules(net: Network, network_data: Dict[str, Any]) -> None: }, context="link rule", ) + if "source" not in link_rule or "target" not in link_rule: + raise ValueError("Each link_rule must include 'source' and 'target'.") - # Handle expand block expand_spec = ExpansionSpec.from_dict(link_rule) if expand_spec and not expand_spec.is_empty(): for expanded_rule in expand_block(link_rule, expand_spec): @@ -809,7 +811,7 @@ def _update_links( rule: Dict[str, Any], bidirectional: bool = True, ) -> None: - """Updates all Link objects between nodes matching source and target selectors + """Update all Link objects between nodes matching source and target selectors with new parameters (capacity, cost, disabled, risk_groups, attrs). If bidirectional=True, both (source->target) and (target->source) links @@ -824,7 +826,6 @@ def _update_links( rule: Rule dict with flat link properties. bidirectional: If True, also update reversed direction links. """ - # Use unified selector system for full selector support src_sel = normalize_selector(source, context="override") tgt_sel = normalize_selector(target, context="override") @@ -901,12 +902,10 @@ def _update_nodes( disabled_val: Boolean or None for disabling or enabling nodes. risk_groups_val: List or set or None for replacing node.risk_groups. """ - # Build selector dict with path and optional match selector_dict: Dict[str, Any] = {"path": path} if match_spec: selector_dict["match"] = match_spec - # Use unified selector system normalized = normalize_selector(selector_dict, context="override") node_groups = select_nodes(net, normalized, default_active_only=False) @@ -915,29 +914,51 @@ def _update_nodes( if disabled_val is not None: node.disabled = bool(disabled_val) if risk_groups_val is not None: - if not isinstance(risk_groups_val, (list, set)): - raise ValueError( - f"risk_groups override must be list or set, got {type(risk_groups_val)}." - ) node.risk_groups = expand_risk_group_refs(risk_groups_val) node.attrs.update(attrs) +def _longest_subgroup_prefix(key: str, group_names: Iterable[str]) -> Optional[str]: + """Return the longest group name prefixing key as '.'. + + Subgroup names may themselves contain dots, so override keys are matched + against literal group names by longest prefix rather than by splitting on + the first dot. Equal-length distinct names cannot both prefix the same + key, so the longest match is unique. + + Args: + key: Dotted params override key (e.g. 'rack.a.count'). + group_names: Literal blueprint subgroup names. + + Returns: + The longest matching group name, or None if no group name prefixes key. + """ + matches = [name for name in group_names if key.startswith(name + ".")] + return max(matches, key=len) if matches else None + + def _apply_parameters( - subgroup_name: str, subgroup_def: Dict[str, Any], params_overrides: Dict[str, Any] + subgroup_name: str, + subgroup_def: Dict[str, Any], + params_overrides: Dict[str, Any], + override_to_group: Dict[str, str], ) -> Dict[str, Any]: """Applies user-provided parameter overrides to a blueprint subgroup. Example: - If 'spine.node_count' = 6 is in params_overrides, - it sets 'node_count' = 6 for the 'spine' subgroup. + If 'spine.count' = 6 is in params_overrides, + it sets 'count' = 6 for the 'spine' subgroup. If 'spine.attrs.hw_type' = 'Dell', it sets subgroup_def['attrs']['hw_type'] = 'Dell'. Args: subgroup_name (str): Name of the subgroup in the blueprint. subgroup_def (Dict[str, Any]): The default definition of that subgroup. params_overrides (Dict[str, Any]): Overrides in the form of - {'spine.node_count': 6, 'spine.attrs.hw_type': 'Dell'}. + {'spine.count': 6, 'spine.attrs.hw_type': 'Dell'}. + override_to_group (Dict[str, str]): Override key -> subgroup name, + resolved once by the caller via `_longest_subgroup_prefix` so a + key applies to subgroup 'rack.a' rather than 'rack' when both + exist and the key starts with 'rack.a.'. Returns: Dict[str, Any]: A copy of subgroup_def with parameter overrides applied. @@ -945,9 +966,8 @@ def _apply_parameters( out = copy.deepcopy(subgroup_def) for key, val in params_overrides.items(): - parts = key.split(".") - if parts[0] == subgroup_name and len(parts) > 1: - subpath = parts[1:] + if override_to_group.get(key) == subgroup_name: + subpath = key[len(subgroup_name) + 1 :].split(".") _apply_nested_path(out, subpath, val) return out diff --git a/ngraph/dsl/blueprints/parser.py b/ngraph/dsl/blueprints/parser.py index 9d1e5a1..3779cba 100644 --- a/ngraph/dsl/blueprints/parser.py +++ b/ngraph/dsl/blueprints/parser.py @@ -1,7 +1,7 @@ """Parsing helpers for the network DSL. -This module factors out pure parsing/validation helpers from the expansion -module so they can be tested independently and reused. +Pure parsing/validation helpers, kept separate from the expansion module so +they can be tested independently and reused. """ from __future__ import annotations diff --git a/ngraph/dsl/expansion/__init__.py b/ngraph/dsl/expansion/__init__.py index 220b480..465cf22 100644 --- a/ngraph/dsl/expansion/__init__.py +++ b/ngraph/dsl/expansion/__init__.py @@ -1,7 +1,7 @@ """Variable and pattern expansion for NetGraph DSL. -This module provides template expansion with $var syntax and -bracket pattern expansion for name generation. +Template expansion with $var syntax, plus bracket pattern expansion for name +generation. Usage: from ngraph.dsl.expansion import expand_block, expand_name_patterns, ExpansionSpec @@ -17,13 +17,12 @@ from .brackets import expand_name_patterns, expand_risk_group_refs from .schema import ExpansionSpec -from .variables import expand_block, expand_templates, substitute_vars +from .variables import expand_block, substitute_vars __all__ = [ # Schema "ExpansionSpec", # Variable expansion - "expand_templates", "expand_block", "substitute_vars", # Bracket expansion diff --git a/ngraph/dsl/expansion/brackets.py b/ngraph/dsl/expansion/brackets.py index 4123ada..04d9c82 100644 --- a/ngraph/dsl/expansion/brackets.py +++ b/ngraph/dsl/expansion/brackets.py @@ -1,14 +1,14 @@ """Bracket expansion for name patterns. -Provides expand_name_patterns() for expanding bracket expressions -like "fa[1-3]" into ["fa1", "fa2", "fa3"]. +`expand_name_patterns()` turns bracket expressions like "fa[1-3]" into +["fa1", "fa2", "fa3"]. """ from __future__ import annotations import re from itertools import product -from typing import Iterable, List, Set +from typing import List, Set, Tuple, Union __all__ = [ "expand_name_patterns", @@ -35,11 +35,11 @@ def expand_name_patterns(name: str) -> List[str]: Examples: >>> expand_name_patterns("fa[1-3]") - ["fa1", "fa2", "fa3"] + ['fa1', 'fa2', 'fa3'] >>> expand_name_patterns("dc[1,3,5-6]") - ["dc1", "dc3", "dc5", "dc6"] + ['dc1', 'dc3', 'dc5', 'dc6'] >>> expand_name_patterns("fa[1-2]_plane[5-6]") - ["fa1_plane5", "fa1_plane6", "fa2_plane5", "fa2_plane6"] + ['fa1_plane5', 'fa1_plane6', 'fa2_plane5', 'fa2_plane6'] """ matches = list(_RANGE_REGEX.finditer(name)) if not matches: @@ -65,28 +65,45 @@ def expand_name_patterns(name: str) -> List[str]: return expanded_names -def expand_risk_group_refs(rg_list: Iterable[str]) -> Set[str]: +def expand_risk_group_refs( + rg_list: Union[List[str], Set[str], Tuple[str, ...]], +) -> Set[str]: """Expand bracket patterns in a list of risk group references. - Takes an iterable of risk group names (possibly containing bracket - expressions) and returns a set of all expanded names. + Takes a list, set, or tuple of risk group names (possibly containing + bracket expressions) and returns a set of all expanded names. Args: - rg_list: Iterable of risk group name patterns. + rg_list: List, set, or tuple of risk group name patterns. Other + iterables (including bare strings and generators) are rejected. Returns: Set of expanded risk group names. + Raises: + ValueError: If the container is not a list/set/tuple (a bare string + would silently expand per character), or if an entry is not a + string (e.g. a variable expansion substituted a non-string value). + Examples: - >>> expand_risk_group_refs(["RG1"]) - {"RG1"} - >>> expand_risk_group_refs(["RG[1-3]"]) - {"RG1", "RG2", "RG3"} - >>> expand_risk_group_refs(["A[1-2]", "B[a,b]"]) - {"A1", "A2", "Ba", "Bb"} + >>> sorted(expand_risk_group_refs(["RG1"])) + ['RG1'] + >>> sorted(expand_risk_group_refs(["RG[1-3]"])) + ['RG1', 'RG2', 'RG3'] + >>> sorted(expand_risk_group_refs(["A[1-2]", "B[a,b]"])) + ['A1', 'A2', 'Ba', 'Bb'] """ + if isinstance(rg_list, str) or not isinstance(rg_list, (list, set, tuple)): + raise ValueError( + "'risk_groups' must be a list or set of names, " + f"got {type(rg_list).__name__}: {rg_list!r}" + ) result: Set[str] = set() for rg in rg_list: + if not isinstance(rg, str): + raise ValueError( + f"Risk group reference must be a string, got {type(rg).__name__}: {rg!r}" + ) result.update(expand_name_patterns(rg)) return result diff --git a/ngraph/dsl/expansion/schema.py b/ngraph/dsl/expansion/schema.py index 776ebd9..f24a201 100644 --- a/ngraph/dsl/expansion/schema.py +++ b/ngraph/dsl/expansion/schema.py @@ -1,7 +1,4 @@ -"""Schema definitions for variable expansion. - -Provides dataclasses for template expansion configuration. -""" +"""Dataclasses describing template expansion configuration.""" from __future__ import annotations diff --git a/ngraph/dsl/expansion/variables.py b/ngraph/dsl/expansion/variables.py index b606ed1..abc2945 100644 --- a/ngraph/dsl/expansion/variables.py +++ b/ngraph/dsl/expansion/variables.py @@ -1,7 +1,7 @@ """Variable expansion for templates. -Provides substitution of $var and ${var} placeholders in strings, -with recursive substitution in nested structures. +Substitutes $var and ${var} placeholders in strings, recursing into nested +structures. """ from __future__ import annotations @@ -15,7 +15,6 @@ from .schema import ExpansionSpec __all__ = [ - "expand_templates", "substitute_vars", "expand_block", ] @@ -27,6 +26,13 @@ MAX_TEMPLATE_EXPANSIONS = 10_000 +def _lookup_var(var_name: str, var_dict: Dict[str, Any]) -> Any: + """Return the value bound to var_name, raising the standard KeyError.""" + if var_name not in var_dict: + raise KeyError(f"Variable '${var_name}' not found in expand.vars") + return var_dict[var_name] + + def _substitute_string(template: str, var_dict: Dict[str, Any]) -> str: """Substitute $var and ${var} placeholders in a template string. @@ -43,9 +49,7 @@ def _substitute_string(template: str, var_dict: Dict[str, Any]) -> str: def replace(match: re.Match[str]) -> str: var_name = match.group(1) or match.group(2) - if var_name not in var_dict: - raise KeyError(f"Variable '${var_name}' not found in expand.vars") - return str(var_dict[var_name]) + return str(_lookup_var(var_name, var_dict)) return _VAR_PATTERN.sub(replace, template) @@ -53,14 +57,27 @@ def replace(match: re.Match[str]) -> str: def substitute_vars(obj: Any, var_dict: Dict[str, Any]) -> Any: """Recursively substitute ${var} in all strings within obj. + A string consisting of exactly one placeholder (e.g. "${t}") is replaced + by the variable's native value, preserving its type. This keeps match + condition values comparable to non-string attributes (e.g. int tiers). + Placeholders embedded in longer strings (e.g. "dc${dc}_internal") are + interpolated as text, so the result is a string. + Args: obj: Any value (string, dict, list, or primitive). var_dict: Mapping of variable names to values. Returns: - Object with all string values having variables substituted. + Object with variables substituted: whole-placeholder strings replaced + by the variable's native value, other strings interpolated as text. + + Raises: + KeyError: If a placeholder names a variable absent from var_dict. """ if isinstance(obj, str): + whole = _VAR_PATTERN.fullmatch(obj) + if whole: + return _lookup_var(whole.group(1) or whole.group(2), var_dict) return _substitute_string(obj, var_dict) if isinstance(obj, dict): return {k: substitute_vars(v, var_dict) for k, v in obj.items()} @@ -122,7 +139,7 @@ def expand_block( If no expand spec is provided or it has no vars, yields the original block. Otherwise, yields a deep copy with all strings substituted for each - variable combination. + variable combination; the 'expand' key itself is removed from each copy. Args: block: DSL block (dict) that may contain template strings. @@ -137,33 +154,5 @@ def expand_block( for var_dict in _generate_combinations(spec.vars, spec.mode): expanded = copy.deepcopy(block) - # Remove the expand block from the result expanded.pop("expand", None) yield substitute_vars(expanded, var_dict) - - -def expand_templates( - templates: Dict[str, str], - spec: "ExpansionSpec", -) -> Iterator[Dict[str, str]]: - """Expand template strings with variable substitution. - - Uses $var or ${var} syntax only. - - Args: - templates: Dict of template strings. - spec: Expansion specification with variables and mode. - - Yields: - Dicts with same keys as templates, values substituted. - - Raises: - ValueError: If zip mode has mismatched list lengths or expansion exceeds limit. - KeyError: If a template references an undefined variable. - """ - if spec.is_empty(): - yield templates - return - - for var_dict in _generate_combinations(spec.vars, spec.mode): - yield {k: _substitute_string(v, var_dict) for k, v in templates.items()} diff --git a/ngraph/dsl/loader.py b/ngraph/dsl/loader.py index 3fde49c..1a77510 100644 --- a/ngraph/dsl/loader.py +++ b/ngraph/dsl/loader.py @@ -1,7 +1,7 @@ """YAML loader + schema validation for Scenario DSL. -Provides a single entrypoint to parse a YAML string, normalize keys where -needed, validate against the packaged JSON schema, and return a canonical +A single entrypoint parses a YAML string, normalizes keys where needed, +validates against the packaged JSON schema, and returns a canonical dictionary suitable for downstream expansion/parsing. """ @@ -35,7 +35,7 @@ def load_scenario_yaml(yaml_str: str) -> Dict[str, Any]: data["demands"] # type: ignore[arg-type] ) - # Early shape checks helpful for better error messages prior to schema validation + # Early shape checks give better error messages than schema validation would network_section = data.get("network") if isinstance(network_section, dict): if "nodes" in network_section and not isinstance( diff --git a/ngraph/dsl/selectors/__init__.py b/ngraph/dsl/selectors/__init__.py index fda41e7..c11794f 100644 --- a/ngraph/dsl/selectors/__init__.py +++ b/ngraph/dsl/selectors/__init__.py @@ -1,7 +1,9 @@ """Unified node selection for NetGraph DSL. -This module provides a single abstraction for node selection used across -adjacency, demands, overrides, and workflow steps. +Selector parsing for YAML configs: the single abstraction for node selection +used across adjacency, demands, overrides, and workflow steps. The schema +types and the runtime evaluation engine live in `ngraph.model.selectors`; +they are re-exported here for backward compatibility. Usage: from ngraph.dsl.selectors import normalize_selector, select_nodes, NodeSelector @@ -13,35 +15,45 @@ groups = select_nodes(network, selector, default_active_only=True) """ -from .conditions import evaluate_condition, evaluate_conditions, resolve_attr_path -from .normalize import normalize_selector, parse_match_spec -from .schema import Condition, EntityScope, MatchSpec, NodeSelector -from .select import ( +from ngraph.model.selectors import ( + VALID_OPERATORS, + Condition, + EntityScope, + MatchSpec, + NodeSelector, + evaluate_condition, + evaluate_conditions, flatten_link_attrs, flatten_node_attrs, flatten_risk_group_attrs, + link_path_key, match_entity_ids, + resolve_attr_path, select_nodes, ) +from .normalize import normalize_selector, parse_match_spec + __all__ = [ - # Schema + # Schema (re-exported from ngraph.model.selectors) "Condition", "EntityScope", "MatchSpec", "NodeSelector", + "VALID_OPERATORS", # Parsing "normalize_selector", "parse_match_spec", - # Evaluation + # Evaluation (re-exported from ngraph.model.selectors) "select_nodes", "evaluate_condition", "evaluate_conditions", "resolve_attr_path", - # Attribute flattening + # Attribute flattening (re-exported from ngraph.model.selectors) "flatten_node_attrs", "flatten_link_attrs", "flatten_risk_group_attrs", - # Entity matching + "link_path_key", + # Entity matching (re-exported from ngraph.model.selectors) "match_entity_ids", ] diff --git a/ngraph/dsl/selectors/normalize.py b/ngraph/dsl/selectors/normalize.py index d1a305b..0526f4c 100644 --- a/ngraph/dsl/selectors/normalize.py +++ b/ngraph/dsl/selectors/normalize.py @@ -1,15 +1,15 @@ """Selector parsing and normalization. -Provides the single entry point for converting raw selector values -(strings or dicts) into NodeSelector objects. +Single entry point for converting raw selector values (strings or dicts) +into NodeSelector objects. """ from __future__ import annotations from dataclasses import replace -from typing import Any, Dict, Literal, Union +from typing import Any, Dict, Union -from .schema import Condition, MatchSpec, NodeSelector +from ngraph.model.selectors import NodeSelector, parse_match_spec __all__ = [ "normalize_selector", @@ -31,8 +31,7 @@ def normalize_selector( ) -> NodeSelector: """Normalize a raw selector (string or dict) to a NodeSelector. - This is the single entry point for all selector parsing. All downstream - code works with NodeSelector objects only. + All downstream code works with NodeSelector objects only. Args: raw: Either a regex string, selector dict, or existing NodeSelector. @@ -67,86 +66,18 @@ def normalize_selector( def _parse_dict(raw: Dict[str, Any], default_active_only: bool) -> NodeSelector: - """Parse a selector dictionary into a NodeSelector.""" + """Parse a selector dictionary into a NodeSelector. + + NodeSelector.__post_init__ validates that at least one selection + mechanism (path, group_by, or match) is present. + """ match_spec = None if "match" in raw: - match_spec = _parse_match(raw["match"]) - - path = raw.get("path") - group_by = raw.get("group_by") - active_only = raw.get("active_only", default_active_only) - - # Validate at least one selection mechanism - if path is None and group_by is None and match_spec is None: - raise ValueError( - "Selector dict requires at least one of: path, group_by, or match" - ) + match_spec = parse_match_spec(raw["match"]) return NodeSelector( - path=path, - group_by=group_by, + path=raw.get("path"), + group_by=raw.get("group_by"), match=match_spec, - active_only=active_only, + active_only=raw.get("active_only", default_active_only), ) - - -def parse_match_spec( - raw: Dict[str, Any], - *, - default_logic: Literal["and", "or"] = "or", - require_conditions: bool = False, - context: str = "match", -) -> MatchSpec: - """Parse a match specification from raw dict. - - Unified match specification parser for use across adjacency, demands, - membership rules, and failure policies. - - Args: - raw: Dict with 'conditions' list and optional 'logic'. - default_logic: Default when 'logic' not specified. - require_conditions: If True, raise when conditions list is empty. - context: Used in error messages. - - Returns: - Parsed MatchSpec. - - Raises: - ValueError: If validation fails. - """ - logic = raw.get("logic", default_logic) - if logic not in ("and", "or"): - raise ValueError( - f"Invalid logic '{logic}' in {context}. Must be 'and' or 'or'." - ) - - conditions_raw = raw.get("conditions", []) - if require_conditions and not conditions_raw: - raise ValueError(f"{context} requires at least one condition") - - conditions = [] - for cond_dict in conditions_raw: - if not isinstance(cond_dict, dict): - raise ValueError( - f"Condition in {context} must be a dict, got {type(cond_dict).__name__}" - ) - if "attr" not in cond_dict or "op" not in cond_dict: - raise ValueError(f"Condition in {context} must have 'attr' and 'op'") - - conditions.append( - Condition( - attr=cond_dict["attr"], - op=cond_dict["op"], - value=cond_dict.get("value"), - ) - ) - - return MatchSpec(conditions=conditions, logic=logic) - - -def _parse_match(raw: Dict[str, Any]) -> MatchSpec: - """Parse a match specification dict (internal helper). - - Uses parse_match_spec with selector defaults (logic="or", conditions optional). - """ - return parse_match_spec(raw, default_logic="or", require_conditions=False) diff --git a/ngraph/explorer.py b/ngraph/explorer.py index 16c3dc6..f9ad53d 100644 --- a/ngraph/explorer.py +++ b/ngraph/explorer.py @@ -1,4 +1,9 @@ -"""NetworkExplorer class for analyzing network hierarchy and structure.""" +"""Hierarchical exploration of a Network. + +Builds a tree of the node-name hierarchy and aggregates per-subtree +statistics — node and link counts, capacity, capex/power, and hardware +bills of materials — in two modes: all nodes, and enabled nodes only. +""" from __future__ import annotations @@ -32,7 +37,7 @@ def _link_is_disabled(link: Link) -> bool: @dataclass class ExternalLinkBreakdown: - """Holds stats for external links to a particular other subtree. + """Stats for external links to one other subtree. Attributes: link_count (int): Number of links to that other subtree. @@ -74,7 +79,6 @@ class TreeStats: total_power: float = 0.0 # Hardware BOM aggregation bom: Dict[str, float] = field(default_factory=dict) - active_bom: Dict[str, float] = field(default_factory=dict) @dataclass @@ -95,7 +99,6 @@ class NodeUtilization: ports_utilization: Ratio of used to available ports (0.0 when N/A). capacity_violation: True if attached capacity exceeds supported capacity. ports_violation: True if used ports exceed available ports. - disabled: True if the node itself is disabled. """ node_name: str @@ -109,12 +112,11 @@ class NodeUtilization: ports_utilization: float capacity_violation: bool ports_violation: bool - disabled: bool @dataclass class LinkCapacityIssue: - """Represents a link capacity constraint violation in active topology. + """A link capacity constraint violation in active topology. Attributes: source: Source node name. @@ -133,7 +135,7 @@ class LinkCapacityIssue: @dataclass(eq=False) class TreeNode: - """Represents a node in the hierarchical tree. + """A node in the hierarchical tree. Attributes: name (str): Name/label of this node. @@ -177,8 +179,10 @@ def is_leaf(self) -> bool: class NetworkExplorer: - """Provides hierarchical exploration of a Network, computing statistics in two modes: - 'all' (ignores disabled) and 'active' (only enabled). + """Hierarchical view of a Network with per-subtree statistics. + + Statistics are computed in two modes: 'all' (ignores disabled) and + 'active' (only enabled). """ def __init__( @@ -196,6 +200,7 @@ def __init__( # For quick lookups: self._node_map: Dict[str, TreeNode] = {} # node_name -> deepest TreeNode self._path_map: Dict[str, TreeNode] = {} # path -> TreeNode + self._node_path_map: Dict[TreeNode, str] = {} # TreeNode -> path # Cache for ancestor sets: self._ancestors_cache: Dict[TreeNode, Set[TreeNode]] = {} @@ -282,8 +287,10 @@ def _compute_subtree_sets_active(self, node: TreeNode) -> Set[str]: return collected def _build_node_map(self, node: TreeNode) -> None: - """Assign each node's name to the *deepest* TreeNode that actually holds it. - We do a parent-first approach so children override if needed. + """Assign each node's name to the *deepest* TreeNode that holds it. + + Walks parents before children so a child TreeNode overrides its + parent's claim on a name. """ # Map the raw_nodes at this level for nd in node.raw_nodes: @@ -294,17 +301,22 @@ def _build_node_map(self, node: TreeNode) -> None: self._build_node_map(child) def _build_path_map(self, node: TreeNode) -> None: - """Build a path->TreeNode map for easy lookups. Skips "root" in path strings.""" + """Build path<->TreeNode maps, omitting the synthetic root from paths.""" path_str = self._compute_full_path(node) self._path_map[path_str] = node + self._node_path_map[node] = path_str for child in node.children.values(): self._build_path_map(child) def _compute_full_path(self, node: TreeNode) -> str: - """Return a '/'-joined path, omitting "root".""" + """Return a '/'-joined path, omitting the synthetic root node. + + The walk stops at the tree root by parent identity, so hierarchy + segments that happen to be named "root" are preserved in paths. + """ parts = [] current = node - while current and current.name != "root": + while current is not None and current.parent is not None: parts.append(current.name) current = current.parent return "/".join(reversed(parts)) @@ -366,6 +378,16 @@ def _compute_node_costs_and_utilization(self) -> Dict[str, bool]: Returns: Mapping from node name to whether it has hardware assigned. """ + # One O(E) pass: index enabled links by endpoint so per-node utilization + # validation iterates only that node's attached links instead of all links. + links_by_node: Dict[str, List[Link]] = {} + for lk in self.network.links.values(): + if _link_is_disabled(lk): + continue + links_by_node.setdefault(lk.source, []).append(lk) + if lk.target != lk.source: + links_by_node.setdefault(lk.target, []).append(lk) + node_has_hw: Dict[str, bool] = {} for nd in self.network.nodes.values(): comp, hw_count = resolve_node_hardware(nd.attrs, self.components_library) @@ -413,7 +435,13 @@ def _compute_node_costs_and_utilization(self) -> Dict[str, bool]: and node_comp_capacity > 0.0 and not _node_is_disabled(nd) ): - self._validate_node_utilization(nd, comp, hw_count, node_comp_capacity) + self._validate_node_utilization( + nd, + comp, + hw_count, + node_comp_capacity, + links_by_node.get(nd.name, []), + ) return node_has_hw @@ -423,39 +451,44 @@ def _validate_node_utilization( comp: "Component", hw_count: float, node_comp_capacity: float, + attached_links: List[Link], ) -> None: """Validate and record node hardware utilization. Checks attached link capacity and port usage against node hardware limits. Records utilization snapshot and raises if strict_validation is enabled. + + Args: + nd: Node being validated. + comp: Resolved hardware component for the node. + hw_count: Hardware multiplicity for the node. + node_comp_capacity: Total capacity supported by node hardware. + attached_links: Enabled links attached to this node. """ # Sum capacities of all enabled links attached to this node attached_capacity = 0.0 # Track optics usage in "equivalent optics" and ports tally used_ports = 0.0 - for lk in self.network.links.values(): - if _link_is_disabled(lk): + for lk in attached_links: + # If the opposite endpoint is disabled, skip in active view + other = lk.target if lk.source == nd.name else lk.source + other_node = self.network.nodes.get(other) + if other_node is not None and _node_is_disabled(other_node): continue - if lk.source == nd.name or lk.target == nd.name: - # If the opposite endpoint is disabled, skip in active view - other = lk.target if lk.source == nd.name else lk.source - other_node = self.network.nodes.get(other, Node(name=other)) - if _node_is_disabled(other_node): - continue - attached_capacity += float(lk.capacity) - - # Compute optics usage for this endpoint if per-end hardware is set - (src_end, dst_end, per_end) = resolve_link_end_components( - lk.attrs, self.components_library - ) - if per_end: - end = src_end if lk.source == nd.name else dst_end - end_comp, end_cnt, _end_excl = end - if end_comp is not None: - # Ports used equals count * ports per optic (fractional allowed) - ports_per_optic = float(getattr(end_comp, "ports", 0) or 0) - if ports_per_optic > 0: - used_ports += end_cnt * ports_per_optic + attached_capacity += float(lk.capacity) + + # Compute optics usage for this endpoint if per-end hardware is set + (src_end, dst_end, per_end) = resolve_link_end_components( + lk.attrs, self.components_library + ) + if per_end: + end = src_end if lk.source == nd.name else dst_end + end_comp, end_cnt, _end_excl = end + if end_comp is not None: + # Ports used equals count * ports per optic (fractional allowed) + ports_per_optic = float(getattr(end_comp, "ports", 0) or 0) + if ports_per_optic > 0: + used_ports += end_cnt * ports_per_optic # Compute ports availability and violations total_ports_available = float(getattr(comp, "ports", 0) or 0) * float(hw_count) @@ -485,7 +518,6 @@ def _validate_node_utilization( ports_utilization=float(ports_utilization), capacity_violation=bool(capacity_violation), ports_violation=bool(ports_violation), - disabled=_node_is_disabled(nd), ) # Enforce strict behavior after recording @@ -605,6 +637,8 @@ def _compute_link_stats(self, node_has_hw: Dict[str, bool]) -> None: src_node = self._node_map[src] dst_node = self._node_map[dst] + src_path = self._node_path_map[src_node] + dst_path = self._node_path_map[dst_node] A_src = self._get_ancestors(src_node) A_dst = self._get_ancestors(dst_node) @@ -618,10 +652,7 @@ def _compute_link_stats(self, node_has_hw: Dict[str, bool]) -> None: for an in xor_anc: an.stats.external_link_count += 1 an.stats.external_link_capacity += cap - if an in A_src: - other_path = self._compute_full_path(dst_node) - else: - other_path = self._compute_full_path(src_node) + other_path = dst_path if an in A_src else src_path bd = an.stats.external_link_details.setdefault( other_path, ExternalLinkBreakdown() ) @@ -686,10 +717,7 @@ def _compute_link_stats(self, node_has_hw: Dict[str, bool]) -> None: for an in xor_anc: an.active_stats.external_link_count += 1 an.active_stats.external_link_capacity += cap - if an in A_src: - other_path = self._compute_full_path(dst_node) - else: - other_path = self._compute_full_path(src_node) + other_path = dst_path if an in A_src else src_path bd = an.active_stats.external_link_details.setdefault( other_path, ExternalLinkBreakdown() ) @@ -821,9 +849,9 @@ def _roll_up_if_leaf(self, path: str) -> str: node = self._path_map.get(path) if not node: return path - while node.parent and node.parent.name != "root" and node.is_leaf(): + while node.parent and node.parent.parent is not None and node.is_leaf(): node = node.parent - return self._compute_full_path(node) + return self._node_path_map[node] # ----------------------------- BOM accessors ----------------------------- def get_bom(self, include_disabled: bool = True) -> Dict[str, float]: @@ -883,26 +911,25 @@ def get_bom_map( if include_root: result[root_label] = self.get_bom(include_disabled=include_disabled) for path, node in self._path_map.items(): + # Root is registered under "" in the path map; its inclusion is + # governed solely by include_root/root_label above. + if path == "": + continue stats = node.stats if include_disabled else node.active_stats result[path] = dict(stats.bom) return result # ---------------------- Validation/utilization accessors ---------------------- - def get_node_utilization( - self, include_disabled: bool = True - ) -> List[NodeUtilization]: + def get_node_utilization(self) -> List[NodeUtilization]: """Return hardware utilization per node based on active topology. - Args: - include_disabled: Include nodes marked disabled in the result. + Snapshots are recorded only for enabled nodes whose hardware component + resolves with a positive capacity. Returns: - List of NodeUtilization entries for nodes with declared hardware. + List of NodeUtilization entries. """ - items = list(self._node_utilization.values()) - if include_disabled: - return list(items) - return [u for u in items if not u.disabled] + return list(self._node_utilization.values()) def get_link_issues(self) -> List[LinkCapacityIssue]: """Return recorded link capacity issues discovered in non-strict mode.""" diff --git a/ngraph/lib/__init__.py b/ngraph/lib/__init__.py index a2a5945..9e88a90 100644 --- a/ngraph/lib/__init__.py +++ b/ngraph/lib/__init__.py @@ -1,7 +1,4 @@ -"""Library utilities for ngraph. - -This package contains integration modules for external libraries. -""" +"""Integration modules for external libraries (currently NetworkX).""" from ngraph.lib.nx import EdgeMap, NodeMap, from_networkx, to_networkx diff --git a/ngraph/lib/nx.py b/ngraph/lib/nx.py index fa03773..e4c2e2d 100644 --- a/ngraph/lib/nx.py +++ b/ngraph/lib/nx.py @@ -1,7 +1,7 @@ """NetworkX graph conversion utilities. -This module provides functions to convert between NetworkX graphs and the -internal graph representation used by ngraph for high-performance algorithms. +Convert between NetworkX graphs and the internal graph representation that +ngraph's algorithms run on. Example: >>> import networkx as nx @@ -100,11 +100,13 @@ class EdgeMap: Example: >>> graph, node_map, edge_map = from_networkx(G) - >>> # After running algorithms, map flow results back to original edges - >>> for ext_id, flow in enumerate(flow_state.edge_flow_view()): + >>> # edge_flow_view() is indexed by internal Core edge index, so + >>> # translate through ext_edge_ids_view() before using to_ref. + >>> ext_edge_ids = graph.ext_edge_ids_view() + >>> for edge_idx, flow in enumerate(flow_state.edge_flow_view()): ... if flow > 0: - ... u, v, key = edge_map.to_ref[ext_id] - ... G.edges[u, v, key]["flow"] = flow + ... u, v, key = edge_map.to_ref[int(ext_edge_ids[edge_idx])] + ... G.edges[u, v, key]["flow"] = flow # G.edges[u, v] for a DiGraph """ to_ref: Dict[int, NxEdgeTuple] = field(default_factory=dict) @@ -122,7 +124,7 @@ def from_networkx( cost_attr: str = "cost", default_capacity: float = 1.0, default_cost: int = 1, - bidirectional: bool = False, + bidirectional: Optional[bool] = None, ) -> Tuple[netgraph_core.StrictMultiDiGraph, NodeMap, EdgeMap]: """Convert a NetworkX graph to ngraph's internal graph format. @@ -133,11 +135,17 @@ def from_networkx( Args: G: NetworkX graph (DiGraph, MultiDiGraph, Graph, or MultiGraph) capacity_attr: Edge attribute name for capacity (default: "capacity") - cost_attr: Edge attribute name for cost (default: "cost") + cost_attr: Edge attribute name for cost (default: "cost"). Cost values + must be integers (netgraph_core requires int64 costs); fractional + values raise ValueError. default_capacity: Capacity value when attribute is missing (default: 1.0) - default_cost: Cost value when attribute is missing (default: 1) - bidirectional: If True, add reverse edge for each edge. Useful for - undirected connectivity analysis. (default: False) + default_cost: Cost value when attribute is missing (default: 1). + Must be an integer value. + bidirectional: If True, add a reverse edge for each edge. If None + (default), inferred from the graph type: directed inputs get one + arc per edge, undirected inputs get antiparallel arc pairs (the + standard undirected-to-directed reduction for max-flow and + reachability). Pass an explicit True or False to override. Returns: Tuple of (graph, node_map, edge_map) where: @@ -147,7 +155,8 @@ def from_networkx( Raises: TypeError: If G is not a NetworkX graph - ValueError: If graph has no nodes + ValueError: If graph has no nodes, or an edge cost is not an integer + value Example: >>> import networkx as nx @@ -156,10 +165,10 @@ def from_networkx( >>> graph, node_map, edge_map = from_networkx(G) >>> graph.num_nodes() 2 - >>> node_map.to_index["src"] - 0 - >>> edge_map.to_ref[0] # First edge - ('dst', 'src', 0) # sorted node order: dst < src + >>> node_map.to_index # node indices assigned in sorted-name order + {'dst': 0, 'src': 1} + >>> edge_map.to_ref[0] # edge refs preserve original (u, v, key) + ('src', 'dst', 0) """ import networkx as nx @@ -172,6 +181,10 @@ def from_networkx( if G.number_of_nodes() == 0: raise ValueError("Graph has no nodes") + # Undirected inputs need antiparallel arcs to preserve connectivity. + if bidirectional is None: + bidirectional = not G.is_directed() + # Build node mapping (sorted for deterministic ordering) node_names = sorted(G.nodes(), key=str) node_map = NodeMap.from_names(node_names) @@ -200,7 +213,16 @@ def from_networkx( src_idx = node_map.to_index[u] dst_idx = node_map.to_index[v] cap = float(data.get(capacity_attr, default_capacity)) - cst = int(data.get(cost_attr, default_cost)) + raw_cost = data.get(cost_attr, default_cost) + cost_f = float(raw_cost) + if not cost_f.is_integer(): + raise ValueError( + f"Edge ({u!r}, {v!r}, {key!r}): cost {raw_cost!r} is not an " + f"integer; netgraph_core requires int64 costs. Pre-scale " + f"fractional costs (e.g., multiply by 10 or 100) before " + f"conversion." + ) + cst = int(cost_f) edge_ref: NxEdgeTuple = (u, v, key) # Forward edge @@ -227,9 +249,8 @@ def from_networkx( edge_map = EdgeMap(to_ref=edge_to_ref, from_ref=ref_to_edges) - # Handle graphs with nodes but no edges + # Graphs with nodes but no edges still need correctly typed empty arrays. if not src_list: - # Create minimal arrays for empty edge set src_arr = np.array([], dtype=np.int32) dst_arr = np.array([], dtype=np.int32) capacity_arr = np.array([], dtype=np.float64) @@ -303,7 +324,6 @@ def to_networkx( capacity_arr = graph.capacity_view() cost_arr = graph.cost_view() - # Add edges num_edges = graph.num_edges() for i in range(num_edges): src_idx = int(src_arr[i]) diff --git a/ngraph/logging.py b/ngraph/logging.py index cbe7b0b..498de37 100644 --- a/ngraph/logging.py +++ b/ngraph/logging.py @@ -1,4 +1,13 @@ -"""Centralized logging configuration for NetGraph.""" +"""Centralized logging configuration for NetGraph. + +Follows the standard library pattern: importing the package attaches only a +``logging.NullHandler`` to the root ``ngraph`` logger and never installs +stream handlers or sets levels. Applications opt into console output by +calling ``setup_root_logger()`` explicitly, or implicitly via +``set_global_log_level()``. The CLI does both in ``main()``: it calls +``setup_root_logger()`` first, then sets the level from +``--verbose``/``--quiet``. +""" import logging import sys @@ -15,12 +24,12 @@ def setup_root_logger( ) -> None: """Set up the root NetGraph logger with a single handler. - This should only be called once to avoid duplicate handlers. + Subsequent calls are no-ops until ``reset_logging()`` is called. Args: level: Logging level (default: INFO). format_string: Custom format string (optional). - handler: Custom handler (optional, defaults to StreamHandler). + handler: Custom handler (optional, defaults to StreamHandler(sys.stderr)). """ global _ROOT_LOGGER_CONFIGURED @@ -30,47 +39,46 @@ def setup_root_logger( root_logger = logging.getLogger("ngraph") root_logger.setLevel(level) - # Clear any existing handlers to avoid duplicates + # Replace the import-time NullHandler (and any stale handlers) root_logger.handlers.clear() # Default format with timestamps, level, logger name, and message if format_string is None: format_string = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" - # Default to console output, but allow override for testing + # Default to stderr so machine-readable stdout (e.g. `ngraph run --stdout`) + # stays free of log lines. if handler is None: - handler = logging.StreamHandler(sys.stdout) + handler = logging.StreamHandler(sys.stderr) formatter = logging.Formatter(format_string) handler.setFormatter(formatter) root_logger.addHandler(handler) - # Let logs propagate to root logger so pytest can capture them + # Let logs propagate to the root logger so host applications and pytest + # can capture them root_logger.propagate = True _ROOT_LOGGER_CONFIGURED = True def get_logger(name: str) -> logging.Logger: - """Get a logger with NetGraph's standard configuration. + """Get a logger under NetGraph's logging hierarchy. - This is the main function that should be used throughout the package. - All loggers will inherit from the root 'ngraph' logger configuration. + Use this everywhere in the package. It configures nothing: handlers and + levels are inherited from the root 'ngraph' logger, which is configured + only when an application calls setup_root_logger() (directly or via + set_global_log_level()). Args: name: Logger name (typically __name__ from calling module). Returns: - Configured logger instance. + Logger instance inheriting from the root ngraph logger. """ - # Ensure root logger is set up - setup_root_logger() - - # Get the logger - it will inherit from the root ngraph logger logger = logging.getLogger(name) # Don't add handlers to child loggers - they inherit from root - # Just set the level logger.setLevel(logging.NOTSET) # Inherit from parent return logger @@ -79,11 +87,15 @@ def get_logger(name: str) -> logging.Logger: def set_global_log_level(level: int) -> None: """Set the log level for all NetGraph loggers. + Installs the default console handler via setup_root_logger() if logging + has not been configured yet. Intended for applications (e.g. the CLI); + library code never calls this implicitly. + Args: level: Logging level (e.g., logging.DEBUG, logging.INFO). """ - # Ensure root logger is set up - setup_root_logger() + # Ensure a console handler exists for applications that only call this + setup_root_logger(level=level) # Set the root level for all ngraph loggers root_logger = logging.getLogger("ngraph") @@ -109,11 +121,14 @@ def reset_logging() -> None: global _ROOT_LOGGER_CONFIGURED _ROOT_LOGGER_CONFIGURED = False - # Clear any existing handlers from ngraph logger + # Restore the unconfigured import-time state: only a NullHandler attached root_logger = logging.getLogger("ngraph") root_logger.handlers.clear() root_logger.setLevel(logging.NOTSET) + root_logger.addHandler(logging.NullHandler()) -# Initialize the root logger when the module is imported -setup_root_logger() +# Library pattern: attach only a NullHandler at import time so unconfigured +# use emits nothing (and avoids the logging.lastResort fallback) while records +# still propagate to handlers configured by the host application. +logging.getLogger("ngraph").addHandler(logging.NullHandler()) diff --git a/ngraph/model/__init__.py b/ngraph/model/__init__.py index 99a6c29..fa919d4 100644 --- a/ngraph/model/__init__.py +++ b/ngraph/model/__init__.py @@ -1,8 +1,8 @@ """Network model package. -This package defines the core network data model used across NetGraph, including -nodes, links, risk groups, and the scenario-level `Network`. Temporary exclusions -for analysis are handled via node_mask and edge_mask parameters in Core algorithms. +Nodes, links, risk groups, and the scenario-level `Network`. Temporary +exclusions for analysis are handled via node_mask and edge_mask parameters in +Core algorithms, not by mutating this model. """ from ngraph.model.demand import TrafficDemand diff --git a/ngraph/model/components.py b/ngraph/model/components.py index cf2df0f..f4f42d2 100644 --- a/ngraph/model/components.py +++ b/ngraph/model/components.py @@ -8,8 +8,11 @@ import yaml +from ngraph.logging import get_logger from ngraph.utils.yaml_utils import normalize_yaml_dict_keys +LOGGER = get_logger(__name__) + @dataclass class Component: @@ -54,11 +57,11 @@ def total_capex(self) -> float: return single_instance_capex * self.count def total_power(self) -> float: - """Computes the total *typical* (recursive) power usage of this component, - including children, multiplied by this component's count. + """Computes *typical* power for this component and all descendants. Returns: - float: The total typical power in watts. + float: Typical power in watts, summed over children and multiplied + by this component's ``count``. """ single_instance_power = self.power_watts for child in self.children.values(): @@ -66,11 +69,11 @@ def total_power(self) -> float: return single_instance_power * self.count def total_power_max(self) -> float: - """Computes the total *peak* (recursive) power usage of this component, - including children, multiplied by this component's count. + """Computes *peak* power for this component and all descendants. Returns: - float: The total maximum (peak) power in watts. + float: Maximum (peak) power in watts, summed over children and + multiplied by this component's ``count``. """ single_instance_power_max = self.power_watts_max for child in self.children.values(): @@ -78,11 +81,11 @@ def total_power_max(self) -> float: return single_instance_power_max * self.count def total_capacity(self) -> float: - """Computes the total (recursive) capacity of this component, - including children, multiplied by this component's count. + """Computes capacity for this component and all descendants. Returns: - float: The total capacity (dimensionless or user-defined units). + float: Capacity summed over children and multiplied by this + component's ``count``, in dimensionless or user-defined units. """ single_instance_capacity = self.capacity for child in self.children.values(): @@ -96,7 +99,8 @@ def as_dict(self, include_children: bool = True) -> Dict[str, Any]: include_children (bool): If True, recursively includes children. Returns: - Dict[str, Any]: Dictionary representation of this component. + Dict[str, Any]: One key per field, with ``attrs`` shallow-copied and + a ``children`` key present only when ``include_children``. """ data = { "name": self.name, @@ -127,19 +131,19 @@ class ComponentsLibrary: components: BigSwitch: component_type: chassis - cost: 20000 + capex: 20000 power_watts: 1750 capacity: 25600 children: PIM16Q-16x200G: component_type: linecard - cost: 1000 + capex: 1000 power_watts: 10 ports: 16 count: 8 200G-FR4: component_type: optic - cost: 2000 + capex: 2000 power_watts: 6 power_watts_max: 6.5 """ @@ -150,22 +154,26 @@ def get(self, name: str) -> Optional[Component]: """Retrieves a Component by its name from the library. Args: - name (str): Name of the component. + name (str): Top-level component name as registered in the library. Returns: - Optional[Component]: The requested Component or None if not found. + Optional[Component]: The requested Component, or None if the name + is not registered. """ return self.components.get(name) def merge( self, other: ComponentsLibrary, override: bool = True ) -> ComponentsLibrary: - """Merges another ComponentsLibrary into this one. By default (override=True), - duplicate components in `other` overwrite those in the current library. + """Merges another ComponentsLibrary into this one. + + Component objects are shared, not copied: both libraries then reference + the same instances. Args: - other (ComponentsLibrary): Another library to merge into this one. - override (bool): If True, components in `other` override existing ones. + other (ComponentsLibrary): Library whose components are merged in. + override (bool): If True (default), duplicate names in `other` + replace the existing entries; if False, existing entries win. Returns: ComponentsLibrary: This instance, updated in place. @@ -185,10 +193,10 @@ def clone(self) -> ComponentsLibrary: @classmethod def from_dict(cls, data: Dict[str, Any]) -> ComponentsLibrary: - """Constructs a ComponentsLibrary from a dictionary of raw component definitions. + """Constructs a ComponentsLibrary from raw component definitions. Args: - data (Dict[str, Any]): Raw component definitions. + data (Dict[str, Any]): Mapping of component name -> definition dict. Returns: ComponentsLibrary: A newly constructed library. @@ -205,8 +213,9 @@ def _build_component(cls, name: str, definition_data: Dict[str, Any]) -> Compone """Recursively constructs a single Component from a dictionary definition. Args: - name (str): Name of the component. - definition_data (Dict[str, Any]): Dictionary data for the component definition. + name (str): Name to give the constructed component. + definition_data (Dict[str, Any]): Component definition. Recognized + keys map to fields; anything else is folded into ``attrs``. Returns: Component: The constructed Component instance. @@ -246,6 +255,15 @@ def _build_component(cls, name: str, definition_data: Dict[str, Any]) -> Compone } # Normalize leftover keys too leftover_keys = normalize_yaml_dict_keys(leftover_keys) + if "cost" in leftover_keys: + # Likely confusion with link 'cost'; without 'capex' the component + # contributes 0 to capex totals. + LOGGER.warning( + "Component '%s' defines unrecognized key 'cost'; it is stored " + "in attrs and ignored by capex calculations. Use 'capex' for " + "monetary cost.", + name, + ) attrs.update(leftover_keys) return Component( @@ -282,7 +300,12 @@ def from_yaml(cls, yaml_str: str) -> ComponentsLibrary: if not isinstance(data, dict): raise ValueError("Top-level must be a dict in Components YAML.") - components_data = data.get("components") or data + if "components" in data: + # Presence-based dispatch: an explicit empty/null 'components' + # mapping yields an empty library, not phantom components. + components_data = data["components"] or {} + else: + components_data = data if not isinstance(components_data, dict): raise ValueError("'components' must be a dict if present.") @@ -350,10 +373,11 @@ def _coerce_positive_float(value: Any, default: float = 1.0) -> float: Args: value: Arbitrary value to parse as float. - default: Value to return if parsing fails or non-positive. + default: Returned when parsing fails or the result is <= 0. Returns: - Positive float. + The parsed float when it is > 0, otherwise ``default`` (so the result + is strictly positive only when ``default`` is). """ try: out = float(value) @@ -372,21 +396,17 @@ def resolve_link_end_components( ]: """Resolve per-end hardware components for a link. - Input format inside ``link.attrs``: - - Structured mapping under ``hardware`` key only: + Input format inside ``link.attrs`` is a structured mapping under the + ``hardware`` key only: ``{"hardware": {"source": {"component": NAME, "count": N}, "target": {"component": NAME, "count": N}}}`` + An optional ``exclusive: true`` per end indicates unsharable usage; for + exclusive ends, validation and BOM counting round counts up to integers. Args: attrs: Link attributes mapping. library: Components library for lookups. - Exclusive usage: - - Optional ``exclusive: true`` per end indicates unsharable usage. - For exclusive ends, validation and BOM counting should round-up counts - to integers. - Returns: ((src_comp, src_count, src_exclusive), (dst_comp, dst_count, dst_exclusive), per_end_specified) where components may be ``None`` if name is absent/unknown. ``per_end_specified`` diff --git a/ngraph/model/demand/__init__.py b/ngraph/model/demand/__init__.py index 2decb9c..5c9c76f 100644 --- a/ngraph/model/demand/__init__.py +++ b/ngraph/model/demand/__init__.py @@ -1,7 +1,6 @@ """Traffic demand specification and set containers. -This package provides data structures for defining traffic demands -and organizing them into named demand sets. +Defines individual demands and the named sets that group them for analysis. Public API: TrafficDemand: Individual demand specification with source/target selectors diff --git a/ngraph/model/demand/builder.py b/ngraph/model/demand/builder.py index 0d75c0c..2a9594c 100644 --- a/ngraph/model/demand/builder.py +++ b/ngraph/model/demand/builder.py @@ -66,6 +66,12 @@ def _build_demand(d: Dict[str, Any], set_name: str) -> TrafficDemand: raise ValueError( f"Each demand in set '{set_name}' requires 'source' and 'target' fields" ) + for fld in ("source", "target"): + if not isinstance(d[fld], (str, dict)): + raise ValueError( + f"Demand '{fld}' in set '{set_name}' must be a string or " + f"selector dict, got {type(d[fld]).__name__}" + ) # Build normalized dict for TrafficDemand constructor td_kwargs: Dict[str, Any] = { @@ -84,31 +90,34 @@ def _build_demand(d: Dict[str, Any], set_name: str) -> TrafficDemand: # Coerce flow_policy into FlowPolicyPreset enum when provided if "flow_policy" in d: - td_kwargs["flow_policy"] = _coerce_flow_policy(d["flow_policy"]) + td_kwargs["flow_policy"] = coerce_flow_policy(d["flow_policy"]) return TrafficDemand(**td_kwargs) -def _coerce_flow_policy(value: Any) -> Optional[FlowPolicyPreset]: +def coerce_flow_policy(value: Any) -> Optional[FlowPolicyPreset]: """Return a FlowPolicyPreset from various user-friendly forms. Accepts: - None: returns None - FlowPolicyPreset: returned as-is - - int: mapped by value (e.g., 1 -> SHORTEST_PATHS_ECMP) + - int: mapped by value (e.g., 1 -> SHORTEST_PATHS_ECMP); bools are + rejected (True/False are not presets 1/0) - str: name of enum (case-insensitive); numeric strings are allowed - Any other type is returned unchanged for advanced usages - (e.g., dict configs handled elsewhere). + Raises: + ValueError: If the value is not one of the accepted forms (including + bool and dict/object configs, which are not supported). """ if value is None: return None if isinstance(value, FlowPolicyPreset): return value - if isinstance(value, int): + # bool is a subclass of int; True/False must not coerce to presets 1/0. + if isinstance(value, int) and not isinstance(value, bool): try: return FlowPolicyPreset(value) - except Exception as exc: # pragma: no cover - validated by enum + except Exception as exc: raise ValueError(f"Unknown flow policy value: {value}") from exc if isinstance(value, str): s = value.strip() @@ -126,5 +135,8 @@ def _coerce_flow_policy(value: Any) -> Optional[FlowPolicyPreset]: except KeyError as exc: raise ValueError(f"Unknown flow policy: {value}") from exc - # Preserve other structural forms (e.g., dict) for callers that support them - return value # type: ignore[return-value] + valid = ", ".join(p.name for p in FlowPolicyPreset) + raise ValueError( + f"Invalid flow_policy: {value!r}; expected a FlowPolicyPreset name " + f"or integer (one of: {valid})" + ) diff --git a/ngraph/model/demand/matrix.py b/ngraph/model/demand/matrix.py index b6c7558..e959ac2 100644 --- a/ngraph/model/demand/matrix.py +++ b/ngraph/model/demand/matrix.py @@ -1,14 +1,12 @@ """Demand set containers. -Provides `DemandSet`, a named collection of `TrafficDemand` lists -used as input to demand expansion and placement. This module contains input -containers, not analysis results. +`DemandSet` holds named `TrafficDemand` lists as input to demand expansion and +placement. These are input containers, not analysis results. """ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any from ngraph.model.demand.spec import TrafficDemand @@ -17,9 +15,6 @@ class DemandSet: """Named collection of TrafficDemand lists. - This mutable container maps set names to lists of TrafficDemand objects, - allowing management of multiple demand sets for analysis. - Attributes: sets: Dictionary mapping set names to TrafficDemand lists. """ @@ -27,11 +22,11 @@ class DemandSet: sets: dict[str, list[TrafficDemand]] = field(default_factory=dict) def add(self, name: str, demands: list[TrafficDemand]) -> None: - """Add a demand list to the collection. + """Add a demand list, replacing any set already stored under `name`. Args: name: Set name identifier. - demands: List of TrafficDemand objects for this set. + demands: TrafficDemand objects for this set; stored by reference. """ self.sets[name] = demands @@ -42,7 +37,8 @@ def get_set(self, name: str) -> list[TrafficDemand]: name: Name of the demand set to retrieve. Returns: - List of TrafficDemand objects for the named set. + The stored list for that set, not a copy: mutating it mutates the + DemandSet. Raises: KeyError: If the set name doesn't exist. @@ -52,10 +48,8 @@ def get_set(self, name: str) -> list[TrafficDemand]: def get_default_set(self) -> list[TrafficDemand]: """Get default demand set. - Returns the set named 'default' if it exists. If there is exactly - one set, returns that single set. If there are no sets, - returns an empty list. If there are multiple sets and none is - named 'default', raises an error. + Prefers the set named 'default'. Falls back to the sole set when + exactly one exists, and to an empty list when there are none. Returns: List of TrafficDemand objects for the default set. @@ -81,20 +75,10 @@ def get_all_demands(self) -> list[TrafficDemand]: """Get all traffic demands from all sets combined. Returns: - Flattened list of all TrafficDemand objects across all sets. + A new list of every TrafficDemand, concatenated in set insertion + order. """ all_demands: list[TrafficDemand] = [] for demands in self.sets.values(): all_demands.extend(demands) return all_demands - - def to_dict(self) -> dict[str, Any]: - """Convert to dictionary for JSON serialization. - - Returns: - Dictionary mapping set names to lists of TrafficDemand dictionaries. - """ - return { - name: [demand.__dict__ for demand in demands] - for name, demands in self.sets.items() - } diff --git a/ngraph/model/demand/spec.py b/ngraph/model/demand/spec.py index 0534910..246ccaa 100644 --- a/ngraph/model/demand/spec.py +++ b/ngraph/model/demand/spec.py @@ -1,22 +1,19 @@ """Traffic demand specification. Defines `TrafficDemand`, a user-facing specification used by demand expansion -and placement. It can carry either a concrete `FlowPolicy` instance or a -`FlowPolicyPreset` enum to construct one. +and placement. Routing behavior is selected via an optional `FlowPolicyPreset`. """ from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import Any, Dict, Optional, Union -from ngraph.model.flow.policy_config import FlowPolicyPreset +from ngraph.model.flow.policy_config import FlowPolicyPreset, serialize_policy_preset +from ngraph.types.base import Mode from ngraph.utils.ids import new_base64_uuid -if TYPE_CHECKING: - import netgraph_core - - FlowPolicy = netgraph_core.FlowPolicy -else: - FlowPolicy = None # type: ignore +# Derived from the Mode enum so the vocabulary is defined once. +_VALID_MODES = tuple(m.name.lower() for m in Mode) +_VALID_GROUP_MODES = ("flatten", "per_group", "group_pairwise") @dataclass @@ -27,13 +24,11 @@ class TrafficDemand: source: Source node selector (string path or selector dict). target: Target node selector (string path or selector dict). volume: Total demand volume. - volume_placed: Portion of this demand placed so far. priority: Priority class (lower = higher priority). mode: Node pairing mode ("combine" or "pairwise"). group_mode: How grouped nodes produce demands ("flatten", "per_group", "group_pairwise"). flow_policy: Policy preset for routing. - flow_policy_obj: Concrete policy instance (overrides flow_policy). attrs: Arbitrary user metadata. id: Unique identifier. Auto-generated if empty. """ @@ -41,19 +36,46 @@ class TrafficDemand: source: Union[str, Dict[str, Any]] = "" target: Union[str, Dict[str, Any]] = "" volume: float = 0.0 - volume_placed: float = 0.0 priority: int = 0 mode: str = "combine" group_mode: str = "flatten" flow_policy: Optional[FlowPolicyPreset] = None - flow_policy_obj: Optional["FlowPolicy"] = None # type: ignore[valid-type] attrs: Dict[str, Any] = field(default_factory=dict) id: str = "" def __post_init__(self) -> None: - """Generate id if not provided.""" + """Validate mode fields and generate id if not provided.""" + if self.mode not in _VALID_MODES: + raise ValueError( + f"Unknown demand mode '{self.mode}'. " + f"Expected one of: {', '.join(_VALID_MODES)}" + ) + if self.group_mode not in _VALID_GROUP_MODES: + raise ValueError( + f"Unknown demand group_mode '{self.group_mode}'. " + f"Expected one of: {', '.join(_VALID_GROUP_MODES)}" + ) if not self.id: # Build a stable identifier from source/target src_key = self.source if isinstance(self.source, str) else str(self.source) tgt_key = self.target if isinstance(self.target, str) else str(self.target) self.id = f"{src_key}|{tgt_key}|{new_base64_uuid()}" + + def to_dict(self) -> Dict[str, Any]: + """Return the canonical serialized form (results output, snapshots). + + The flow policy is serialized to its preset name; use the raw + `flow_policy` attribute for analysis wire formats that expect the + preset object. + """ + return { + "id": self.id, + "source": self.source, + "target": self.target, + "volume": float(self.volume), + "priority": int(self.priority), + "mode": self.mode, + "group_mode": self.group_mode, + "flow_policy": serialize_policy_preset(self.flow_policy), + "attrs": dict(self.attrs), + } diff --git a/ngraph/model/failure/generate.py b/ngraph/model/failure/generate.py index 386ac31..a053303 100644 --- a/ngraph/model/failure/generate.py +++ b/ngraph/model/failure/generate.py @@ -1,7 +1,6 @@ """Dynamic risk group generation from entity attributes. -Provides functionality to auto-generate risk groups based on unique -attribute values from nodes or links. +Creates one risk group per unique value of a chosen node or link attribute. """ from __future__ import annotations @@ -11,13 +10,14 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional -from ngraph.dsl.selectors import ( +from ngraph.logging import get_logger +from ngraph.model.network import RiskGroup +from ngraph.model.selectors import ( flatten_link_attrs, flatten_node_attrs, + link_path_key, resolve_attr_path, ) -from ngraph.logging import get_logger -from ngraph.model.network import RiskGroup if TYPE_CHECKING: from ngraph.model.network import Network @@ -58,12 +58,16 @@ def generate_risk_groups(network: "Network", spec: GenerateSpec) -> List[RiskGro Returns: List of newly created RiskGroup objects. + Raises: + ValueError: If `group_by` resolves to an unhashable value, or if the + name template renders the same group name for two distinct values. + Note: - This function modifies entity risk_groups sets in place. + Modifies entity risk_groups sets in place. """ path_pattern = re.compile(spec.path) if spec.path else None - # Collect entities and flatten function + # (entity id, entity, flattened attrs) triples for the target scope if spec.scope == "node": entities = [ (node.name, node, flatten_node_attrs(node)) @@ -87,7 +91,7 @@ def generate_risk_groups(network: "Network", spec: GenerateSpec) -> List[RiskGro entities = [ (eid, entity, attrs) for eid, entity, attrs in entities - if path_pattern.match(f"{attrs['source']}|{attrs['target']}") + if path_pattern.match(link_path_key(attrs)) ] # Group by attribute value @@ -95,18 +99,33 @@ def generate_risk_groups(network: "Network", spec: GenerateSpec) -> List[RiskGro for entity_id, entity, attrs in entities: found, value = resolve_attr_path(attrs, spec.group_by) if found and value is not None: - groups[value].append((entity_id, entity)) + try: + groups[value].append((entity_id, entity)) + except TypeError as exc: + raise ValueError( + f"generate block group_by '{spec.group_by}' resolved to an " + f"unhashable {type(value).__name__} on entity " + f"'{entity_id}'; group_by requires scalar attribute values" + ) from exc # Create risk groups result: List[RiskGroup] = [] + seen_names: Dict[str, Any] = {} for value, members in groups.items(): - # Generate group name from template name = spec.name.replace("${value}", str(value)) + if name in seen_names: + # Distinct group_by values (e.g. int 1 and str "1") can render to + # the same string; a silent merge or a generic downstream conflict + # error would hide the real cause. + raise ValueError( + f"generate block name template '{spec.name}' renders the same " + f"group name '{name}' for distinct group_by values " + f"{seen_names[name]!r} and {value!r}" + ) + seen_names[name] = value - # Create risk group with specified attrs rg = RiskGroup(name=name, attrs=dict(spec.attrs)) - # Add membership to entities for _entity_id, entity in members: entity.risk_groups.add(name) @@ -132,7 +151,9 @@ def parse_generate_spec(raw: Dict[str, Any]) -> GenerateSpec: Parsed GenerateSpec. Raises: - ValueError: If required fields are missing or invalid. + ValueError: If 'scope' is missing or is neither 'node' nor 'link', if + 'group_by' or 'name' is missing, or if 'name' omits the '${value}' + placeholder. """ scope = raw.get("scope") if not scope: diff --git a/ngraph/model/failure/membership.py b/ngraph/model/failure/membership.py index e6d7c3f..d7e9997 100644 --- a/ngraph/model/failure/membership.py +++ b/ngraph/model/failure/membership.py @@ -1,26 +1,26 @@ """Risk group membership rule resolution. -Provides functionality to resolve policy-based membership rules that -auto-assign entities (nodes, links, risk groups) to risk groups based -on attribute conditions. +Resolves policy-based membership rules that auto-assign entities (nodes, +links, risk groups) to risk groups based on attribute conditions. """ from __future__ import annotations import re from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Union -from ngraph.dsl.selectors import ( +from ngraph.logging import get_logger +from ngraph.model.selectors import ( EntityScope, MatchSpec, flatten_link_attrs, flatten_node_attrs, flatten_risk_group_attrs, + link_path_key, match_entity_ids, parse_match_spec, ) -from ngraph.logging import get_logger if TYPE_CHECKING: from ngraph.model.network import Link, Network, Node, RiskGroup @@ -34,7 +34,9 @@ class MembershipSpec: Attributes: scope: Type of entities to match ("node", "link", or "risk_group"). - path: Optional regex pattern to filter entities by name. + path: Optional regex pattern. For node and risk_group scope it is + matched against the entity ID; for link scope it is matched + against the "source|target" key, not the link ID. match: Match specification with conditions. """ @@ -56,9 +58,34 @@ def resolve_membership_rules(network: "Network") -> None: network: Network with risk_groups, nodes, and links populated. Note: - This function modifies entities in place. It should be called after - all risk groups are registered but before validation. + Modifies entities in place. Call after all risk groups are registered + but before validation. """ + # Flattened attribute maps are shared by all membership rules; build each + # lazily once instead of re-flattening every entity per rule. + flat_maps: Dict[str, Dict[str, Dict[str, Any]]] = {} + + def _flat(scope: str) -> Dict[str, Dict[str, Any]]: + cached = flat_maps.get(scope) + if cached is not None: + return cached + if scope == "node": + built = { + node.name: flatten_node_attrs(node) for node in network.nodes.values() + } + elif scope == "link": + built = { + link_id: flatten_link_attrs(link, link_id) + for link_id, link in network.links.items() + } + else: + built = { + rg.name: flatten_risk_group_attrs(rg) + for rg in network.risk_groups.values() + } + flat_maps[scope] = built + return built + for rg_name, rg in network.risk_groups.items(): if rg._membership_raw is None: continue @@ -73,7 +100,7 @@ def resolve_membership_rules(network: "Network") -> None: matched_count = 0 if spec.scope == "risk_group": # Hierarchical: add matched groups as children - matched_rgs = _select_risk_groups(network, spec) + matched_rgs = _select_risk_groups(network, spec, _flat("risk_group")) for matched_rg in matched_rgs: # Don't add self-reference if matched_rg.name != rg_name: @@ -83,7 +110,7 @@ def resolve_membership_rules(network: "Network") -> None: matched_count += 1 else: # Add rg_name to each matched entity's risk_groups - matched_entities = _select_entities(network, spec) + matched_entities = _select_entities(network, spec, _flat(spec.scope)) matched_count = len(matched_entities) for entity in matched_entities: entity.risk_groups.add(rg_name) @@ -106,7 +133,8 @@ def _parse_membership_spec(raw: Dict[str, Any]) -> MembershipSpec: Parsed MembershipSpec. Raises: - ValueError: If required fields are missing or invalid. + ValueError: If 'scope' is missing or is not one of node/link/ + risk_group, or if neither 'path' nor 'match' is given. """ scope = raw.get("scope") if not scope: @@ -121,7 +149,6 @@ def _parse_membership_spec(raw: Dict[str, Any]) -> MembershipSpec: path = raw.get("path") match_raw = raw.get("match") - # At least one of path or match must be specified if path is None and match_raw is None: raise ValueError("membership requires at least 'path' or 'match'") @@ -138,110 +165,79 @@ def _parse_membership_spec(raw: Dict[str, Any]) -> MembershipSpec: return MembershipSpec(scope=scope, path=path, match=match_spec) +def _match_spec_ids( + entity_attrs: Dict[str, Dict[str, Any]], + spec: MembershipSpec, + path_key: Callable[[str, Dict[str, Any]], str], +) -> Set[str]: + """Return entity IDs passing the spec's path filter and match conditions. + + Args: + entity_attrs: Mapping of entity_id -> flattened attribute dict. + spec: Membership specification with path and match. + path_key: Maps (entity_id, attrs) to the string the path regex + matches against (the ID for nodes/risk groups, the + "source|target" form for links). + + Returns: + Set of matching entity IDs. + """ + if spec.path: + path_pattern = re.compile(spec.path) + candidate_ids = { + eid + for eid, attrs in entity_attrs.items() + if path_pattern.match(path_key(eid, attrs)) + } + else: + candidate_ids = set(entity_attrs.keys()) + + if spec.match: + filtered_attrs = {k: v for k, v in entity_attrs.items() if k in candidate_ids} + return match_entity_ids(filtered_attrs, spec.match.conditions, spec.match.logic) + return candidate_ids + + def _select_entities( - network: "Network", spec: MembershipSpec + network: "Network", + spec: MembershipSpec, + entity_attrs: Dict[str, Dict[str, Any]], ) -> List[Union["Node", "Link"]]: """Select nodes or links based on path and/or match conditions. - Uses the shared match_entity_ids() function from selectors. - Args: network: Network to search. - spec: Membership specification with scope, path, and match. + spec: Membership specification with scope ("node" or "link"), + path, and match. + entity_attrs: Pre-flattened attribute map for the spec's scope. Returns: List of matched Node or Link objects. """ - path_pattern = re.compile(spec.path) if spec.path else None - if spec.scope == "node": - # Build flattened attrs dict for all nodes - entity_attrs = { - node.name: flatten_node_attrs(node) for node in network.nodes.values() - } - # Start with all or path-filtered IDs - if path_pattern: - candidate_ids = {eid for eid in entity_attrs if path_pattern.match(eid)} - else: - candidate_ids = set(entity_attrs.keys()) - - # Apply match conditions if specified - if spec.match: - filtered_attrs = { - k: v for k, v in entity_attrs.items() if k in candidate_ids - } - matched_ids = match_entity_ids( - filtered_attrs, spec.match.conditions, spec.match.logic - ) - else: - matched_ids = candidate_ids - + matched_ids = _match_spec_ids(entity_attrs, spec, lambda eid, attrs: eid) return [network.nodes[node_id] for node_id in matched_ids] - elif spec.scope == "link": - # Build flattened attrs dict for all links - entity_attrs = { - link_id: flatten_link_attrs(link, link_id) - for link_id, link in network.links.items() - } - # Start with all or path-filtered IDs - if path_pattern: - candidate_ids = { - eid - for eid, attrs in entity_attrs.items() - if path_pattern.match(f"{attrs['source']}|{attrs['target']}") - } - else: - candidate_ids = set(entity_attrs.keys()) - - # Apply match conditions if specified - if spec.match: - filtered_attrs = { - k: v for k, v in entity_attrs.items() if k in candidate_ids - } - matched_ids = match_entity_ids( - filtered_attrs, spec.match.conditions, spec.match.logic - ) - else: - matched_ids = candidate_ids - - return [network.links[link_id] for link_id in matched_ids] + matched_ids = _match_spec_ids( + entity_attrs, spec, lambda eid, attrs: link_path_key(attrs) + ) + return [network.links[link_id] for link_id in matched_ids] - return [] - -def _select_risk_groups(network: "Network", spec: MembershipSpec) -> List["RiskGroup"]: +def _select_risk_groups( + network: "Network", + spec: MembershipSpec, + entity_attrs: Dict[str, Dict[str, Any]], +) -> List["RiskGroup"]: """Select risk groups based on path and/or match conditions. - Uses the shared match_entity_ids() function from selectors. - Args: network: Network with risk_groups. spec: Membership specification with path and match. + entity_attrs: Pre-flattened risk-group attribute map. Returns: List of matched RiskGroup objects. """ - path_pattern = re.compile(spec.path) if spec.path else None - - # Build flattened attrs dict for all risk groups - entity_attrs = { - rg.name: flatten_risk_group_attrs(rg) for rg in network.risk_groups.values() - } - - # Start with all or path-filtered IDs - if path_pattern: - candidate_ids = {eid for eid in entity_attrs if path_pattern.match(eid)} - else: - candidate_ids = set(entity_attrs.keys()) - - # Apply match conditions if specified - if spec.match: - filtered_attrs = {k: v for k, v in entity_attrs.items() if k in candidate_ids} - matched_ids = match_entity_ids( - filtered_attrs, spec.match.conditions, spec.match.logic - ) - else: - matched_ids = candidate_ids - + matched_ids = _match_spec_ids(entity_attrs, spec, lambda eid, attrs: eid) return [network.risk_groups[rg_name] for rg_name in matched_ids] diff --git a/ngraph/model/failure/parser.py b/ngraph/model/failure/parser.py index 0498ff2..62cd31b 100644 --- a/ngraph/model/failure/parser.py +++ b/ngraph/model/failure/parser.py @@ -4,7 +4,6 @@ from typing import Any, Callable, Dict, List, Optional -from ngraph.dsl.selectors import Condition from ngraph.logging import get_logger from ngraph.model.failure.policy import ( FailureMode, @@ -13,6 +12,7 @@ ) from ngraph.model.failure.policy_set import FailurePolicySet from ngraph.model.network import RiskGroup +from ngraph.model.selectors import parse_match_spec from ngraph.utils.yaml_utils import normalize_yaml_dict_keys _logger = get_logger(__name__) @@ -29,6 +29,11 @@ def build_risk_groups( - Children are also expanded recursively - Generate blocks: {generate: {...}} for dynamic group creation + 'membership', 'disabled', and 'generate' are only honored on top-level + entries: only top-level groups are registered in network.risk_groups, so + these keys would be silently inert on nested children. Child entries + carrying them are rejected with ValueError. + Args: rg_data: List of risk group definitions (strings or dicts). @@ -57,7 +62,7 @@ def build_one(d: Dict[str, Any]) -> RiskGroup: disabled = d.get("disabled", False) # Recursively expand and build children children_list = d.get("children", []) - child_objs = expand_and_build(children_list) + child_objs = expand_and_build(children_list, in_children=True) attrs = normalize_yaml_dict_keys(d.get("attrs", {})) # Extract membership rule for deferred resolution membership_raw = d.get("membership") @@ -69,14 +74,30 @@ def build_one(d: Dict[str, Any]) -> RiskGroup: _membership_raw=membership_raw, ) - def expand_and_build(entries: List[Any]) -> List[RiskGroup]: + def expand_and_build( + entries: List[Any], *, in_children: bool = False + ) -> List[RiskGroup]: """Expand names and build RiskGroups for a list of entries.""" result: List[RiskGroup] = [] for entry in entries: normalized = normalize_entry(entry) - # Skip generate blocks in children (not supported) + # Reject generate blocks in children (not supported) if "generate" in normalized: raise ValueError("'generate' blocks not allowed in children") + if in_children: + # Only top-level groups are registered in network.risk_groups, + # so membership rules and disabled flags on nested children + # would be silently ignored. Reject them instead. + if "membership" in normalized: + raise ValueError( + "'membership' rules not allowed in children; define " + "the group at top level and reference it as a child" + ) + if "disabled" in normalized: + raise ValueError( + "'disabled' not allowed in children; define the group " + "at top level and reference it as a child" + ) name = normalized.get("name", "") if not name: raise ValueError("RiskGroup entry missing 'name' field.") @@ -108,12 +129,9 @@ def build_failure_policy( ) -> FailurePolicy: """Build a FailurePolicy from a raw configuration dictionary. - Parses modes, rules, and conditions from the policy definition and - constructs a fully initialized FailurePolicy object. - Args: fp_data: Policy definition dict with keys: modes (required), attrs, - expand_groups, expand_children. Each mode contains weight and rules. + expand_groups. Each mode contains weight and rules. policy_name: Name identifier for this policy (used for seed derivation). derive_seed: Callable to derive deterministic seeds from component names. @@ -121,7 +139,8 @@ def build_failure_policy( FailurePolicy: Configured policy with parsed modes and rules. Raises: - ValueError: If modes is empty or malformed, or if rules are invalid. + ValueError: If modes is empty or malformed, if rules are invalid, or + if no mode has positive weight. """ def build_rules(rule_dicts: List[Dict[str, Any]]) -> List[FailureRule]: @@ -133,27 +152,15 @@ def build_rules(rule_dicts: List[Dict[str, Any]]) -> List[FailureRule]: "failure rule requires 'scope' field (node, link, or risk_group)" ) - # Get conditions from match block - match_block = rule_dict.get("match", {}) - conditions_data = match_block.get("conditions", []) - logic = match_block.get("logic", "or") - - if not isinstance(conditions_data, list): - raise ValueError("Each rule's 'conditions' must be a list if present.") - conditions: List[Condition] = [] - for cond_dict in conditions_data: - conditions.append( - Condition( - attr=cond_dict["attr"], - op=cond_dict["op"], - value=cond_dict.get("value"), - ) - ) + # Parse the match block with the unified parser + match_spec = parse_match_spec( + rule_dict.get("match", {}), context="failure rule" + ) out.append( FailureRule( scope=scope, - conditions=conditions, - logic=logic, + conditions=match_spec.conditions, + logic=match_spec.logic, mode=rule_dict.get("mode", "all"), probability=rule_dict.get("probability", 1.0), count=rule_dict.get("count", 1), @@ -164,7 +171,6 @@ def build_rules(rule_dicts: List[Dict[str, Any]]) -> List[FailureRule]: return out expand_groups = fp_data.get("expand_groups", False) - expand_children = fp_data.get("expand_children", False) attrs = normalize_yaml_dict_keys(fp_data.get("attrs", {})) modes: List[FailureMode] = [] @@ -182,12 +188,17 @@ def build_rules(rule_dicts: List[Dict[str, Any]]) -> List[FailureRule]: mode_attrs = normalize_yaml_dict_keys(m.get("attrs", {})) modes.append(FailureMode(weight=weight, rules=mode_rules, attrs=mode_attrs)) + if not any(mode.weight > 0.0 for mode in modes): + raise ValueError( + f"failure policy '{policy_name}' has no mode with positive weight; " + "at least one mode must have weight > 0" + ) + policy_seed = derive_seed(policy_name) return FailurePolicy( attrs=attrs, expand_groups=expand_groups, - expand_children=expand_children, seed=policy_seed, modes=modes, ) @@ -218,10 +229,6 @@ def build_failure_policy_set( normalized_fps = normalize_yaml_dict_keys(raw) fps = FailurePolicySet() - # Capture derive_seed in a closure with a different name to avoid confusion - # when passing to build_failure_policy (which also has a derive_seed parameter) - outer_derive_seed = derive_seed - for name, fp_data in normalized_fps.items(): if not isinstance(fp_data, dict): raise ValueError( @@ -230,7 +237,7 @@ def build_failure_policy_set( policy = build_failure_policy( fp_data, policy_name=name, - derive_seed=lambda n, _fn=outer_derive_seed: _fn(f"failure_policy:{n}"), + derive_seed=lambda n, _fn=derive_seed: _fn(f"failure_policy:{n}"), ) fps.add(name, policy) return fps diff --git a/ngraph/model/failure/policy.py b/ngraph/model/failure/policy.py index 0ef2823..f1fad47 100644 --- a/ngraph/model/failure/policy.py +++ b/ngraph/model/failure/policy.py @@ -4,18 +4,27 @@ and risk groups fail in analyses. Conditions match on top-level attributes with simple operators; rules select matches using "all", probabilistic "random" (with `probability`), or fixed-size "choice" (with `count`). -Policies can optionally expand failures by shared risk groups or by -risk-group children. +Policies can optionally expand failures by shared risk groups. Failed risk +groups always cascade to their children downstream (the hierarchy is +inherent), so no policy flag controls that behavior. """ from __future__ import annotations +import heapq +import math import random as _random +import re from collections import defaultdict, deque from dataclasses import dataclass, field -from typing import Any, Dict, List, Literal, Optional, Sequence, Set, Tuple +from typing import Any, Dict, List, Literal, Optional, Sequence, Set, Tuple, get_args -from ngraph.dsl.selectors import Condition, EntityScope, match_entity_ids +from ngraph.model.selectors import ( + Condition, + EntityScope, + link_path_key, + match_entity_ids, +) @dataclass @@ -28,13 +37,17 @@ class FailureRule: conditions: A list of conditions to filter matching entities. logic: "and" (all must be true) or "or" (any must be true, default). mode: The selection strategy among the matched set: - - "random": each matched entity is chosen with probability. + - "random": each matched entity fails independently with + `probability`. - "choice": pick exactly `count` items (random sample). - "all": select every matched entity. probability: Probability in [0,1], used if mode="random". count: Number of entities to pick if mode="choice". weight_by: Optional attribute for weighted sampling in choice mode. - path: Optional regex pattern to filter entities by name. + path: Optional regex pattern applied after condition matching. For + node and risk_group scope it is matched against the entity ID; + for link scope it is matched against the "source|target" key, + not the link ID. """ scope: EntityScope @@ -47,6 +60,19 @@ class FailureRule: path: Optional[str] = None def __post_init__(self) -> None: + if self.scope not in get_args(EntityScope): + raise ValueError( + f"Invalid rule scope '{self.scope}'. " + f"Valid scopes: {', '.join(get_args(EntityScope))}" + ) + if self.mode not in ("random", "choice", "all"): + raise ValueError( + f"Invalid rule mode '{self.mode}'. Valid modes: random, choice, all" + ) + if self.logic not in ("and", "or"): + raise ValueError( + f"Invalid rule logic '{self.logic}'. Must be 'and' or 'or'." + ) if self.mode == "random": if not (0.0 <= self.probability <= 1.0): raise ValueError( @@ -79,34 +105,32 @@ class FailureMode: class FailurePolicy: """A container for failure modes plus optional metadata in `attrs`. - The main entry point is `apply_failures`, which: - 1) Build a single RNG for the entire call (from `seed` or `self.seed`). - 2) Select a mode based on weights (one RNG draw). - 3) For each rule in the mode, gather relevant entities. - 4) Match based on rule conditions using 'and' or 'or' logic. - 5) Apply the selection strategy (all, random, or choice) drawing - from the same RNG, ensuring statistical independence across rules. - 6) Collect the union of all failed entities across all rules. - 7) Optionally expand failures by shared-risk groups or sub-risks. + The main entry point is `apply_failures_typed`, which: + 1) Builds a single RNG for the entire call (from `seed` or `self.seed`). + 2) Selects a mode based on weights (one RNG draw). + 3) Gathers the relevant entities for each rule in that mode. + 4) Matches them against the rule conditions using 'and' or 'or' logic. + 5) Applies the selection strategy (all, random, or choice), drawing + from the same RNG, which keeps rules statistically independent. + 6) Collects the union of all failed entities across all rules. + 7) Optionally expands failures by shared-risk groups. Attributes: attrs: Arbitrary metadata about this policy. expand_groups: If True, expand failures among entities sharing risk groups with failed entities. - expand_children: If True, expand failed risk groups to include - their children recursively. seed: Default seed for reproducible random operations. Overridden - by the ``seed`` parameter on ``apply_failures`` when provided. + by the ``seed`` parameter on ``apply_failures_typed`` when + provided. modes: List of weighted failure modes. """ attrs: Dict[str, Any] = field(default_factory=dict) expand_groups: bool = False - expand_children: bool = False seed: Optional[int] = None modes: List[FailureMode] = field(default_factory=list) - def apply_failures( + def apply_failures_typed( self, network_nodes: Dict[str, Any], network_links: Dict[str, Any], @@ -115,8 +139,13 @@ def apply_failures( seed: Optional[int] = None, failure_trace: Optional[Dict[str, Any]] = None, prepared_matches: Optional[Dict[int, tuple[str, ...]]] = None, - ) -> List[str]: - """Identify which entities fail for this iteration. + prepared_weights: Optional[ + Dict[int, Tuple[Dict[str, float], Tuple[str, ...]]] + ] = None, + prepared_rg_index: Optional[Dict[str, Set[str]]] = None, + prepared_rg_members: Optional[Dict[str, Tuple[frozenset, frozenset]]] = None, + ) -> Tuple[Set[str], Set[str], Set[str]]: + """Identify which entities fail for this iteration, typed by scope. A single ``random.Random`` instance is created from the effective seed (``seed`` if given, else ``self.seed``). All random draws -- mode @@ -135,9 +164,24 @@ def apply_failures( rule selections, expansion). If provided, will be mutated in-place. prepared_matches: Optional mapping from ``id(rule)`` to already-sorted candidate IDs. Used by FailureManager to avoid repeated matching. + prepared_weights: Optional per-rule weight splits from + ``prepare_weights``. Only consulted for rules that also appear + in ``prepared_matches``. + prepared_rg_index: Optional precomputed risk-group -> entity-ID index + (see ``build_risk_group_index``). Used by callers that invoke + ``apply_failures_typed`` repeatedly on a static network to avoid + rebuilding the index on every call. Only consulted when + ``expand_groups`` is True; built on demand when omitted. + prepared_rg_members: Optional transitive risk-group -> (nodes, + links) member index. Seeds expansion for rule-failed risk + groups, including nested children that are not registered + top-level; without it, seeding falls back to the flattened + risk-group map and reaches only registered groups. Returns: - Sorted list of failed entity IDs (nodes, links, and/or risk group names). + Tuple of (failed_nodes, failed_links, failed_risk_groups). Typed + sets let callers classify entities without name probing, which + matters when a risk group shares its name with a node or link. """ if network_risk_groups is None: network_risk_groups = {} @@ -167,14 +211,16 @@ def apply_failures( else _random.Random() ) - # Determine rules from a selected mode (or none if no modes) + # Determine rules from a selected mode (or none if no modes / no mode + # has positive weight) rules_to_apply: Sequence[FailureRule] = [] if self.modes: mode_index = self._select_mode_index(self.modes, rng) - rules_to_apply = self.modes[mode_index].rules - if failure_trace is not None: - failure_trace["mode_index"] = mode_index - failure_trace["mode_attrs"] = dict(self.modes[mode_index].attrs) + if mode_index is not None: + rules_to_apply = self.modes[mode_index].rules + if failure_trace is not None: + failure_trace["mode_index"] = mode_index + failure_trace["mode_attrs"] = dict(self.modes[mode_index].attrs) # Collect matched from each rule, then select for idx, rule in enumerate(rules_to_apply): @@ -183,12 +229,18 @@ def apply_failures( matched_ids = prepared_matches[id(rule)] else: matched_ids = self._match_scope( - idx, rule, network_nodes, network_links, network_risk_groups, ) + weight_split = None + if ( + prepared_weights is not None + and prepared_matches is not None + and id(rule) in prepared_matches + ): + weight_split = prepared_weights.get(id(rule)) selected = self._select_entities( matched_ids, rule, @@ -196,6 +248,7 @@ def apply_failures( network_nodes if rule.scope == "node" else (network_links if rule.scope == "link" else network_risk_groups), + weight_split=weight_split, ) # Record selection in trace if non-empty @@ -210,7 +263,6 @@ def apply_failures( } ) - # Add them to the respective fail sets if rule.scope == "node": failed_nodes |= set(selected) elif rule.scope == "link": @@ -221,22 +273,23 @@ def apply_failures( # Snapshot before expansion for trace pre_nodes: Set[str] = set() pre_links: Set[str] = set() - pre_rgs: Set[str] = set() if failure_trace is not None: pre_nodes = set(failed_nodes) pre_links = set(failed_links) - pre_rgs = set(failed_risk_groups) - # Optionally expand by risk groups + # Optionally expand by risk groups. Members of rule-failed risk + # groups seed the expansion so the same physical failure state + # expands identically regardless of which rule scope produced it. if self.expand_groups: self._expand_risk_groups( - failed_nodes, failed_links, network_nodes, network_links - ) - - # Optionally expand failed risk-group children - if self.expand_children and failed_risk_groups: - self._expand_failed_risk_group_children( - failed_risk_groups, network_risk_groups + failed_nodes, + failed_links, + network_nodes, + network_links, + rg_to_entities=prepared_rg_index, + failed_risk_groups=failed_risk_groups, + network_risk_groups=network_risk_groups, + rg_members=prepared_rg_members, ) # Capture expansion in trace @@ -244,11 +297,48 @@ def apply_failures( failure_trace["expansion"] = { "nodes": sorted(failed_nodes - pre_nodes), "links": sorted(failed_links - pre_links), - "risk_groups": sorted(failed_risk_groups - pre_rgs), + # Expansion adds member nodes/links; the group set never grows. + "risk_groups": [], } - all_failed = set(failed_nodes) | set(failed_links) | set(failed_risk_groups) - return sorted(all_failed) + return failed_nodes, failed_links, failed_risk_groups + + def apply_failures( + self, + network_nodes: Dict[str, Any], + network_links: Dict[str, Any], + network_risk_groups: Dict[str, Any] | None = None, + *, + seed: Optional[int] = None, + failure_trace: Optional[Dict[str, Any]] = None, + prepared_matches: Optional[Dict[int, tuple[str, ...]]] = None, + prepared_weights: Optional[ + Dict[int, Tuple[Dict[str, float], Tuple[str, ...]]] + ] = None, + prepared_rg_index: Optional[Dict[str, Set[str]]] = None, + prepared_rg_members: Optional[Dict[str, Tuple[frozenset, frozenset]]] = None, + ) -> List[str]: + """Identify which entities fail for this iteration. + + Convenience wrapper over ``apply_failures_typed`` returning a single + merged, sorted ID list. Use the typed variant when entity kinds must + be distinguished (IDs are not guaranteed unique across kinds). + + Returns: + Sorted list of failed entity IDs (nodes, links, and/or risk group names). + """ + failed_nodes, failed_links, failed_risk_groups = self.apply_failures_typed( + network_nodes, + network_links, + network_risk_groups, + seed=seed, + failure_trace=failure_trace, + prepared_matches=prepared_matches, + prepared_weights=prepared_weights, + prepared_rg_index=prepared_rg_index, + prepared_rg_members=prepared_rg_members, + ) + return sorted(failed_nodes | failed_links | failed_risk_groups) def prepare_matches( self, @@ -274,14 +364,13 @@ def prepare_matches( prepared: Dict[int, tuple[str, ...]] = {} for mode in self.modes: - for idx, rule in enumerate(mode.rules): + for rule in mode.rules: rule_key = id(rule) if rule_key in prepared: continue prepared[rule_key] = tuple( sorted( self._match_scope( - idx, rule, network_nodes, network_links, @@ -291,9 +380,67 @@ def prepare_matches( ) return prepared + def prepare_weights( + self, + prepared_matches: Dict[int, tuple[str, ...]], + network_nodes: Dict[str, Any], + network_links: Dict[str, Any], + network_risk_groups: Dict[str, Any] | None = None, + ) -> Dict[int, Tuple[Dict[str, float], Tuple[str, ...]]]: + """Precompute per-rule weight splits for weighted-choice rules. + + Weights depend only on the static entity attributes, so callers that + invoke ``apply_failures_typed`` repeatedly (Monte Carlo) should compute + them once. For each ``mode="choice"`` rule with ``weight_by``, maps + ``id(rule)`` to ``(positives, zeros)``: entities with positive weight + (insertion-ordered by sorted entity id) and zero/missing-weight + entities (same order). Only valid together with ``prepared_matches`` + from the same policy and entity maps. + + Args: + prepared_matches: Result of ``prepare_matches`` for this policy. + network_nodes: Mapping of node_id -> flattened attribute dict. + network_links: Mapping of link_id -> flattened attribute dict. + network_risk_groups: Mapping of risk_group_name -> flattened + attribute dict. + + Returns: + Mapping from ``id(rule)`` to its precomputed weight split. + """ + if network_risk_groups is None: + network_risk_groups = {} + + prepared: Dict[int, Tuple[Dict[str, float], Tuple[str, ...]]] = {} + for mode in self.modes: + for rule in mode.rules: + rule_key = id(rule) + if rule_key in prepared or rule_key not in prepared_matches: + continue + if rule.mode != "choice" or not rule.weight_by: + continue + entity_map = ( + network_nodes + if rule.scope == "node" + else ( + network_links if rule.scope == "link" else network_risk_groups + ) + ) + positives: Dict[str, float] = {} + zeros: list[str] = [] + for eid in prepared_matches[rule_key]: + w = FailurePolicy._extract_weight( + entity_map.get(eid), rule.weight_by + ) + w = float(w) if isinstance(w, (int, float)) else 0.0 + if w <= 0.0: + zeros.append(eid) + else: + positives[eid] = w + prepared[rule_key] = (positives, tuple(zeros)) + return prepared + def _match_scope( self, - _rule_idx: int, rule: FailureRule, network_nodes: Dict[str, Any], network_links: Dict[str, Any], @@ -301,12 +448,10 @@ def _match_scope( ) -> Set[str]: """Get the set of IDs matched by the given rule. - Uses the shared match_entity_ids() function from selectors. - Applies optional path filter if specified. + Evaluates the rule's conditions via ``match_entity_ids``, then applies + the optional ``path`` regex. For link scope the regex is matched + against the "source|target" key, otherwise against the entity ID. """ - import re - - # Decide which mapping to iterate if rule.scope == "node": candidates = match_entity_ids(network_nodes, rule.conditions, rule.logic) elif rule.scope == "link": @@ -316,16 +461,13 @@ def _match_scope( network_risk_groups, rule.conditions, rule.logic ) - # Apply path filter if specified if rule.path: pattern = re.compile(rule.path) if rule.scope == "link": candidates = { eid for eid in candidates - if pattern.match( - f"{network_links[eid]['source']}|{network_links[eid]['target']}" - ) + if pattern.match(link_path_key(network_links[eid])) } else: candidates = {eid for eid in candidates if pattern.match(eid)} @@ -338,6 +480,7 @@ def _select_entities( rule: FailureRule, rng: _random.Random, entity_map: Dict[str, Any], + weight_split: Optional[Tuple[Dict[str, float], Tuple[str, ...]]] = None, ) -> Set[str]: """Select entities for failure per rule. @@ -350,42 +493,55 @@ def _select_entities( (from ``prepare_matches``) or a set (sorted internally). rule: The failure rule specifying selection strategy. rng: Random instance shared across the entire apply_failures call. - entity_map: Mapping of entity_id -> attribute dict. + entity_map: Mapping of entity_id -> attribute dict, consulted only + to read ``weight_by`` values. + weight_split: Precomputed ``(positives, zeros)`` split from + ``prepare_weights``; computed here when omitted. """ if not entity_ids: return set() # Ensure deterministic mapping from RNG draws to entity IDs. Prepared - # matches are already ordered; sets must still be sorted here. - ordered_ids = ( - list(entity_ids) - if isinstance(entity_ids, (tuple, list)) - else sorted(entity_ids) + # matches are already ordered (used as-is, no copy); sets must still + # be sorted here. + ordered_ids: Sequence[str] = ( + entity_ids if isinstance(entity_ids, (tuple, list)) else sorted(entity_ids) ) if rule.mode == "random": + # Draw the failure count from Binomial(n, p) and sample uniformly: + # distributionally identical to per-entity Bernoulli trials but + # O(failures) instead of O(matched). binomialvariate requires + # Python 3.12+; fall back to the per-entity loop on 3.11. + binomialvariate = getattr(rng, "binomialvariate", None) + if binomialvariate is not None: + k = binomialvariate(len(ordered_ids), rule.probability) + return set(rng.sample(ordered_ids, k=k)) return {eid for eid in ordered_ids if rng.random() < rule.probability} elif rule.mode == "choice": count = min(rule.count, len(ordered_ids)) if count <= 0: return set() - # Weighted without replacement if weight_by provided + # Weighted without replacement if weight_by provided. The + # positive/zero split is static per rule; use the precomputed one + # (see prepare_weights) when the caller supplies it. if rule.weight_by: - weights: Dict[str, float] = {} - positives: Dict[str, float] = {} - zeros: list[str] = [] - for eid in ordered_ids: - w = FailurePolicy._extract_weight( - entity_map.get(eid), rule.weight_by - ) - w = float(w) if isinstance(w, (int, float)) else 0.0 - if w <= 0.0: - zeros.append(eid) - weights[eid] = 0.0 - else: - positives[eid] = w - weights[eid] = w + if weight_split is not None: + positives, zeros = weight_split + else: + positives = {} + zeros_list: list[str] = [] + for eid in ordered_ids: + w = FailurePolicy._extract_weight( + entity_map.get(eid), rule.weight_by + ) + w = float(w) if isinstance(w, (int, float)) else 0.0 + if w <= 0.0: + zeros_list.append(eid) + else: + positives[eid] = w + zeros = tuple(zeros_list) selected: set[str] = set() if positives: @@ -400,31 +556,23 @@ def _select_entities( pool = [z for z in zeros if z not in selected] if pool: selected |= set(rng.sample(pool, k=min(remaining, len(pool)))) - if selected: - return selected + return selected + + # Uniform sampling when no weighting is requested + return set(rng.sample(ordered_ids, k=count)) - # Fallback to uniform sampling - entity_list = ordered_ids - return set(rng.sample(entity_list, k=count)) - elif rule.mode == "all": - return set(ordered_ids) - else: - raise ValueError(f"Unsupported mode: {rule.mode}") + # mode == "all" (validated in FailureRule.__post_init__) + return set(ordered_ids) @staticmethod - def _extract_weight(entity: Any, attr_name: str) -> float: - """Extract weight attribute from entity which can be dict-like or object. + def _extract_weight(entity: Optional[Dict[str, Any]], attr_name: str) -> float: + """Extract a numeric weight from a flattened attribute dict. - Returns 0.0 on missing attributes or non-numeric values. + Returns 0.0 on missing entities, missing attributes, or non-numeric values. """ if entity is None: return 0.0 - # Dict mapping from merged attributes - if isinstance(entity, dict): - value = entity.get(attr_name) - else: - # RiskGroup object or similar with .attrs - value = getattr(entity, "attrs", {}).get(attr_name) + value = entity.get(attr_name) try: return float(value) if value is not None else 0.0 except (TypeError, ValueError): @@ -454,39 +602,44 @@ def _weighted_sample_without_replacement( if not positive_items: return set() - # Compute keys and select top `count` + # Efraimidis-Spirakis keys computed in the log domain: ln(u) / w is a + # monotone transform of u ** (1/w), so the ranking is identical where + # the linear form is exact, but the log form neither underflows to 0.0 + # for tiny weights (~1e-5, e.g. per-hour failure rates) nor saturates + # to 1.0 for huge ones -- both of which silently degenerated selection + # into descending-id order regardless of weights. scored: List[Tuple[float, str]] = [] for item_id, w in positive_items: u = rng.random() # Guard against u=0.0 -> use minimal positive number if u <= 0.0: u = 1e-12 - key = u ** (1.0 / w) - scored.append((key, item_id)) - # Largest keys first - scored.sort(reverse=True) - selected_ids = {item_id for _, item_id in scored[:count]} - return selected_ids + scored.append((math.log(u) / w, item_id)) + # Largest keys win; nlargest avoids sorting the full candidate list. + return {item_id for _, item_id in heapq.nlargest(count, scored)} @staticmethod - def _select_mode_index(modes: Sequence["FailureMode"], rng: _random.Random) -> int: + def _select_mode_index( + modes: Sequence["FailureMode"], rng: _random.Random + ) -> Optional[int]: """Select a mode index based on normalized weights. - Modes with non-positive weights are ignored. + Modes with non-positive weights are ignored. Returns None when no mode + has positive weight, in which case no rules should be applied. Args: modes: Sequence of FailureMode objects to select from. rng: Random instance shared across the entire apply_failures call. """ - # Build cumulative weights + # Weights need not sum to 1; normalize against the positive ones only. effective: List[Tuple[int, float]] = [ (idx, float(m.weight)) for idx, m in enumerate(modes) if float(m.weight) > 0.0 ] if not effective: - # Degenerate: no positive weights -> fall back to first mode if exists - return 0 + # Degenerate: no positive weights -> no mode is selected + return None total = sum(w for _, w in effective) r = rng.random() * total cumulative = 0.0 @@ -497,52 +650,126 @@ def _select_mode_index(modes: Sequence["FailureMode"], rng: _random.Random) -> i # Fallback due to FP rounding return effective[-1][0] - def _expand_risk_groups( - self, - failed_nodes: Set[str], - failed_links: Set[str], + @staticmethod + def build_risk_group_index( network_nodes: Dict[str, Any], network_links: Dict[str, Any], - ) -> None: - """Expand failures among any node/link that shares a risk group - with a failed entity. BFS until no new failures. + ) -> Dict[str, Set[str]]: + """Build a risk-group -> entity-ID index for risk-group expansion. + + The index depends only on the network, so callers that invoke + ``apply_failures_typed`` repeatedly (e.g., Monte Carlo iterations) should + build it once and pass it via ``prepared_rg_index``. + + Args: + network_nodes: Mapping of node_id -> flattened attribute dict. + network_links: Mapping of link_id -> flattened attribute dict. + + Returns: + Mapping of risk-group name -> set of node and link IDs in the group. """ - # We'll handle node + link expansions only. (Risk group expansions are separate.) - # Build a map risk_group -> set of node or link IDs rg_to_entities: Dict[str, Set[str]] = defaultdict(set) - - # Gather risk_groups from nodes for n_id, nd in network_nodes.items(): if "risk_groups" in nd and nd["risk_groups"]: for rg in nd["risk_groups"]: rg_to_entities[rg].add(n_id) - - # Gather risk_groups from links for l_id, lk in network_links.items(): if "risk_groups" in lk and lk["risk_groups"]: for rg in lk["risk_groups"]: rg_to_entities[rg].add(l_id) + return rg_to_entities + + def _expand_risk_groups( + self, + failed_nodes: Set[str], + failed_links: Set[str], + network_nodes: Dict[str, Any], + network_links: Dict[str, Any], + rg_to_entities: Optional[Dict[str, Set[str]]] = None, + failed_risk_groups: Optional[Set[str]] = None, + network_risk_groups: Optional[Dict[str, Any]] = None, + rg_members: Optional[Dict[str, Tuple[frozenset, frozenset]]] = None, + ) -> None: + """Expand failures among any node/link that shares a risk group + with a failed entity. BFS until no new failures. + + When ``failed_risk_groups`` is given, the members of those groups + (transitively through child groups) seed the expansion as well, so a + rule-failed risk group expands exactly like the equivalent set of + rule-failed member entities. + + Args: + failed_nodes: Set of failed node IDs; mutated in place. + failed_links: Set of failed link IDs; mutated in place. + network_nodes: Mapping of node_id -> flattened attribute dict. + network_links: Mapping of link_id -> flattened attribute dict. + rg_to_entities: Optional precomputed risk-group -> entity-ID index + (see ``build_risk_group_index``); built here when omitted. + failed_risk_groups: Risk groups failed by risk_group-scoped rules. + network_risk_groups: Mapping of risk_group_name -> flattened + attribute dict (provides ``children`` for transitive members). + rg_members: Optional transitive risk-group -> (node IDs, link IDs) + index. Preferred over ``network_risk_groups`` for seeding, + since it also covers nested groups that are not registered + top-level. + """ + # Expansion only ever adds nodes and links; the failed risk-group set + # is never grown here. + if rg_to_entities is None: + rg_to_entities = self.build_risk_group_index(network_nodes, network_links) + + # Seed with members of failed risk groups so a rule-failed group + # expands like the equivalent set of rule-failed member entities. + if failed_risk_groups: + if rg_members is not None: + # Transitive member index (covers nested child groups even + # when they are not registered top-level; FailureManager + # passes its recursive index here). + for rg_name in failed_risk_groups: + member_nodes, member_links = rg_members.get( + rg_name, (frozenset(), frozenset()) + ) + failed_nodes.update(member_nodes) + failed_links.update(member_links) + else: + # Best-effort fallback: walk children via the flattened + # risk-group map. Nested groups that are absent from the map + # (only top-level groups are registered in + # network.risk_groups) cannot be traversed here, so direct + # callers wanting full-depth seeding should pass rg_members. + network_risk_groups = network_risk_groups or {} + rg_queue = deque(failed_risk_groups) + seen_rgs = set(failed_risk_groups) + while rg_queue: + rg_name = rg_queue.popleft() + for member_id in rg_to_entities.get(rg_name, ()): + if member_id in network_nodes: + failed_nodes.add(member_id) + elif member_id in network_links: + failed_links.add(member_id) + for child in network_risk_groups.get(rg_name, {}).get( + "children", [] + ): + if child not in seen_rgs: + seen_rgs.add(child) + rg_queue.append(child) - # Combined set of failed node/link IDs queue = deque(failed_nodes | failed_links) - visited = set(queue) # track which entity IDs we've processed + visited = set(queue) while queue: current_id = queue.popleft() - # figure out if current_id is a node or a link by seeing where it appears + # An ID can name either a node or a link; look it up in both maps. current_rgs = [] if current_id in network_nodes: - # node nd = network_nodes[current_id] current_rgs = nd.get("risk_groups", []) elif current_id in network_links: - # link lk = network_links[current_id] current_rgs = lk.get("risk_groups", []) for rg in current_rgs: - # all entity IDs in rg_to_entities[rg] should be failed - for other_id in rg_to_entities[rg]: + for other_id in rg_to_entities.get(rg, ()): if other_id not in visited: visited.add(other_id) queue.append(other_id) @@ -550,55 +777,22 @@ def _expand_risk_groups( failed_nodes.add(other_id) elif other_id in network_links: failed_links.add(other_id) - # if other_id in risk_groups => not handled here - - def _expand_failed_risk_group_children( - self, - failed_rgs: Set[str], - all_risk_groups: Dict[str, Any], - ) -> None: - """If we fail a risk_group, also fail its descendants recursively. - - We assume each entry in all_risk_groups is something like: - rg_name -> RiskGroup object or { 'name': .., 'children': [...] } - - BFS or DFS any children to mark them as failed as well. - """ - queue = deque(failed_rgs) - while queue: - rg_name = queue.popleft() - rg_data = all_risk_groups.get(rg_name) - if not rg_data: - continue - # Suppose the children are in rg_data["children"] - # or if it's an actual RiskGroup object => rg_data.children - child_list = [] - if isinstance(rg_data, dict): - child_list = rg_data.get("children", []) - else: - # assume it's a RiskGroup object with a .children - child_list = rg_data.children - - for child_obj in child_list: - # child_obj might be a dict or RiskGroup with name - child_name = ( - child_obj["name"] if isinstance(child_obj, dict) else child_obj.name - ) - if child_name not in failed_rgs: - failed_rgs.add(child_name) - queue.append(child_name) def to_dict(self) -> Dict[str, Any]: """Convert to dictionary for JSON serialization. + The output matches the scenario YAML failure-policy format: rule + conditions and logic are nested under a ``match`` key, so the result + round-trips through ``build_failure_policy``. The policy ``seed`` is + derived at scenario load time and is not part of the format, so it is + not serialized. + Returns: Dictionary representation with all fields as JSON-serializable primitives. """ data: Dict[str, Any] = { - "attrs": self.attrs, + "attrs": dict(self.attrs), "expand_groups": self.expand_groups, - "expand_children": self.expand_children, - "seed": self.seed, } if self.modes: data["modes"] = [ @@ -607,15 +801,17 @@ def to_dict(self) -> Dict[str, Any]: "rules": [ { "scope": rule.scope, - "conditions": [ - { - "attr": cond.attr, - "op": cond.op, - "value": cond.value, - } - for cond in rule.conditions - ], - "logic": rule.logic, + "match": { + "logic": rule.logic, + "conditions": [ + { + "attr": cond.attr, + "op": cond.op, + "value": cond.value, + } + for cond in rule.conditions + ], + }, "mode": rule.mode, "probability": rule.probability, "count": rule.count, @@ -624,7 +820,7 @@ def to_dict(self) -> Dict[str, Any]: } for rule in mode.rules ], - "attrs": mode.attrs, + "attrs": dict(mode.attrs), } for mode in self.modes ] diff --git a/ngraph/model/failure/policy_set.py b/ngraph/model/failure/policy_set.py index efe65e9..fdea06e 100644 --- a/ngraph/model/failure/policy_set.py +++ b/ngraph/model/failure/policy_set.py @@ -17,9 +17,6 @@ class FailurePolicySet: """Named collection of FailurePolicy objects. - This mutable container maps failure policy names to FailurePolicy objects, - allowing management of multiple failure policies for analysis. - Attributes: policies: Dictionary mapping failure policy names to FailurePolicy objects. """ @@ -27,11 +24,11 @@ class FailurePolicySet: policies: dict[str, FailurePolicy] = field(default_factory=dict) def add(self, name: str, policy: FailurePolicy) -> None: - """Add a failure policy to the collection. + """Add a policy, replacing any policy already stored under `name`. Args: name: Failure policy name identifier. - policy: FailurePolicy object for this failure policy. + policy: FailurePolicy to store; kept by reference, not copied. """ self.policies[name] = policy @@ -53,7 +50,7 @@ def get_all_policies(self) -> list[FailurePolicy]: """Get all failure policies from the collection. Returns: - List of all FailurePolicy objects. + A new list of the stored policies, in insertion order. """ return list(self.policies.values()) diff --git a/ngraph/model/failure/validation.py b/ngraph/model/failure/validation.py index edfc1c7..9dbe384 100644 --- a/ngraph/model/failure/validation.py +++ b/ngraph/model/failure/validation.py @@ -15,11 +15,10 @@ def validate_risk_group_references(network: "Network") -> None: - """Ensure all risk group references resolve to defined groups. + """Ensure every risk group named by a node or link is defined. - Checks that every risk group name referenced by nodes and links - exists in network.risk_groups. This catches typos and missing - definitions that would otherwise cause silent failures in simulations. + Names are checked against network.risk_groups; typos and missing + definitions would otherwise cause silent failures in simulations. Args: network: Network with nodes, links, and risk_groups populated. @@ -32,13 +31,11 @@ def validate_risk_group_references(network: "Network") -> None: defined: Set[str] = set(network.risk_groups.keys()) errors: List[str] = [] - # Check nodes for node in network.nodes.values(): undefined = node.risk_groups - defined if undefined: errors.append(f"Node '{node.name}': {sorted(undefined)}") - # Check links for link in network.links.values(): undefined = link.risk_groups - defined if undefined: @@ -58,9 +55,8 @@ def validate_risk_group_references(network: "Network") -> None: def validate_risk_group_hierarchy(network: "Network") -> None: """Detect circular references in risk group parent-child relationships. - Uses DFS-based cycle detection to find any risk group that is part of - a cycle in the children hierarchy. This can happen when membership rules - with scope='risk_group' create mutual parent-child relationships. + Cycles arise when membership rules with scope='risk_group' create mutual + parent-child relationships. Detection is a DFS over the children hierarchy. Args: network: Network with risk_groups populated (after membership resolution). diff --git a/ngraph/model/flow/__init__.py b/ngraph/model/flow/__init__.py index af1bf31..d910de9 100644 --- a/ngraph/model/flow/__init__.py +++ b/ngraph/model/flow/__init__.py @@ -1,7 +1,7 @@ """Flow policy configuration for NetGraph. -This package provides preset configurations for traffic routing policies -used in demand placement and flow analysis. +Preset traffic routing configurations used by demand placement and flow +analysis. Public API: FlowPolicyPreset: Enum of common flow policy configurations diff --git a/ngraph/model/flow/policy_config.py b/ngraph/model/flow/policy_config.py index 036b021..d1d98e3 100644 --- a/ngraph/model/flow/policy_config.py +++ b/ngraph/model/flow/policy_config.py @@ -1,7 +1,7 @@ """Flow policy preset configurations for NetGraph. -Provides convenient factory functions to create common FlowPolicy configurations -using NetGraph-Core's FlowPolicy and FlowPolicyConfig. +Named routing presets and the factory that materializes them as NetGraph-Core +FlowPolicy objects built from a FlowPolicyConfig. """ from __future__ import annotations @@ -88,7 +88,8 @@ def create_flow_policy( Args: algorithms: NetGraph-Core Algorithms instance. graph: NetGraph-Core Graph handle. - preset: FlowPolicyPreset enum value specifying the desired policy. + preset: Preset whose path algorithm, placement, edge selection, and + flow-count bounds to apply. node_mask: Optional numpy bool array for node exclusions (True = include). edge_mask: Optional numpy bool array for edge exclusions (True = include). @@ -197,14 +198,14 @@ def create_flow_policy( def serialize_policy_preset(cfg: Any) -> Optional[str]: """Serialize a FlowPolicyPreset to its string name for JSON storage. - Handles FlowPolicyPreset enum values, integer enum values, and string inputs. - Returns None for None input. - Args: - cfg: FlowPolicyPreset enum, integer, or other value to serialize. + cfg: FlowPolicyPreset enum, an integer coercible to one, or any other + value. Returns: - String name of the preset (e.g., "SHORTEST_PATHS_ECMP"), or None if input is None. + Preset name (e.g. "SHORTEST_PATHS_ECMP"); None when ``cfg`` is None. + Values that do not map to a preset are logged at debug level and + returned as ``str(cfg)``. """ if cfg is None: return None diff --git a/ngraph/model/network.py b/ngraph/model/network.py index df18951..02a9943 100644 --- a/ngraph/model/network.py +++ b/ngraph/model/network.py @@ -1,7 +1,6 @@ """Network topology modeling with Node, Link, RiskGroup, and Network classes. -This module provides the core network model classes (Node, Link, RiskGroup, Network) -that can be used independently. +These classes carry no analysis machinery and can be used on their own. """ from __future__ import annotations @@ -10,11 +9,8 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Set -from ngraph.logging import get_logger from ngraph.utils.ids import new_base64_uuid -LOGGER = get_logger(__name__) - @dataclass class Node: @@ -40,10 +36,10 @@ class Node: class Link: """Represents one directed link between two nodes. - The model stores a single direction (``source`` -> ``target``). When building - the working graph for analysis, a reverse edge is added by default to provide - bidirectional connectivity. Disable with ``add_reverse=False`` in - ``Network.to_strict_multidigraph``. + The model stores a single direction (``source`` -> ``target``). When the + analysis graph is built (via ``AnalysisContext`` / netgraph-core), a reverse + edge is added automatically for each link to provide bidirectional + connectivity. Attributes: source (str): Name of the source node. @@ -53,7 +49,10 @@ class Link: disabled (bool): Whether the link is disabled. risk_groups (Set[str]): Set of risk group names this link belongs to. attrs (Dict[str, Any]): Additional metadata (e.g., distance). - id (str): Auto-generated unique identifier: "{source}|{target}|". + id (str): Unique identifier. ``Network.add_link`` assigns the + deterministic form "{source}|{target}|", where is a + per-(source, target) insertion sequence number; links never added + to a Network keep a provisional uuid-suffixed id. """ source: str @@ -66,7 +65,12 @@ class Link: id: str = field(init=False) def __post_init__(self) -> None: - """Generate the link's unique ID upon initialization.""" + """Assign a provisional unique ID. + + Network.add_link replaces it with the deterministic + "source|target|" form; the uuid suffix only guarantees + uniqueness for links never added to a Network. + """ self.id = f"{self.source}|{self.target}|{new_base64_uuid()}" @@ -122,6 +126,7 @@ class Network: _selection_cache: Dict[str, Dict[str, List[Node]]] = field( default_factory=dict, init=False, repr=False ) + _link_seq: Dict[tuple, int] = field(default_factory=dict, init=False, repr=False) def add_node(self, node: Node) -> None: """Add a node to the network (keyed by node.name). @@ -135,22 +140,51 @@ def add_node(self, node: Node) -> None: if node.name in self.nodes: raise ValueError(f"Node '{node.name}' already exists in the network.") self.nodes[node.name] = node - self._selection_cache.clear() # Invalidate cache on modification + self._selection_cache.clear() def add_link(self, link: Link) -> None: - """Add a link to the network (keyed by the link's auto-generated ID). + """Add a link to the network, assigning its deterministic ID. + + The link's ID is (re)assigned here as "source|target|", where + is a per-(source, target) insertion sequence number, so ids and + their sort order are stable across identical scenario builds. Args: link (Link): Link to add. Raises: ValueError: If the link's source or target node does not exist. + ValueError: If this Link object was already added to this network. + ValueError: If the generated "source|target|" id collides with + an existing one, which distinct endpoint pairs can do when node + names contain '|' (e.g. 'a|b'->'c' vs 'a'->'b|c'). """ if link.source not in self.nodes: raise ValueError(f"Source node '{link.source}' not found in network.") if link.target not in self.nodes: raise ValueError(f"Target node '{link.target}' not found in network.") + # Reassign a deterministic per-pair sequence id. The uuid suffix from + # construction makes parallel links sort in a rebuild-dependent order, + # which breaks seeded reproducibility of failure sampling and makes + # link ids unstable across identical scenario builds. + if self.links.get(link.id) is link: + raise ValueError( + f"Link '{link.id}' has already been added to this network." + ) + pair = (link.source, link.target) + seq = self._link_seq.get(pair, 0) + self._link_seq[pair] = seq + 1 + new_id = f"{link.source}|{link.target}|{seq}" + if new_id in self.links: + # Distinct endpoint pairs can render to the same string when node + # names contain '|' (e.g. 'a|b'->'c' vs 'a'->'b|c'); refuse + # rather than silently overwriting the earlier link. + raise ValueError( + f"Link id '{new_id}' already exists; node names containing " + "'|' can make distinct endpoint pairs ambiguous." + ) + link.id = new_id self.links[link.id] = link def select_node_groups_by_path(self, path: str) -> Dict[str, List[Node]]: @@ -167,11 +201,15 @@ def select_node_groups_by_path(self, path: str) -> Dict[str, List[Node]]: path: Regex pattern for node name. Returns: - Mapping from group label to list of nodes. + A fresh mapping from group label to a fresh list of nodes; the Node + objects themselves are shared, so mutating the returned mapping or + lists does not affect the internal selection cache. """ - # Check cache first - if path in self._selection_cache: - return self._selection_cache[path] + # Check cache first. A shallow copy protects the cache from caller + # mutation (groups map and lists are fresh; Node objects are shared). + cached = self._selection_cache.get(path) + if cached is not None: + return {label: list(nodes) for label, nodes in cached.items()} pattern = re.compile(path) groups_map: Dict[str, List[Node]] = {} @@ -187,7 +225,7 @@ def select_node_groups_by_path(self, path: str) -> Dict[str, List[Node]]: groups_map.setdefault(label, []).append(node) self._selection_cache[path] = groups_map - return groups_map + return {label: list(nodes) for label, nodes in groups_map.items()} def disable_node(self, node_name: str) -> None: """Mark a node as disabled. @@ -256,15 +294,15 @@ def disable_all(self) -> None: link.disabled = True def get_links_between(self, source: str, target: str) -> List[str]: - """Retrieve all link IDs that connect the specified source node - to the target node. + """Retrieve the IDs of all direct links from source to target. Args: source (str): Source node name. target (str): Target node name. Returns: - List[str]: A list of link IDs for all direct links from source to target. + List[str]: Link IDs for every direct source -> target link. Empty + if the nodes are unconnected or unknown. """ matches = [] for link_id, link in self.links.items(): @@ -278,15 +316,21 @@ def find_links( target_regex: Optional[str] = None, any_direction: bool = False, ) -> List[Link]: - """Search for links using optional regex patterns for source or target node names. + """Search for links by regex on source and/or target node names. + + Unlike selector paths (which anchor at the start via ``re.match``), + these patterns use unanchored ``re.search`` and match anywhere in the + node name; anchor explicitly (``^...$``) for exact-name matching. Args: - source_regex (Optional[str]): Regex to match link.source. If None, matches all sources. - target_regex (Optional[str]): Regex to match link.target. If None, matches all targets. + source_regex (Optional[str]): Regex matched against link.source; + None matches every source. + target_regex (Optional[str]): Regex matched against link.target; + None matches every target. any_direction (bool): If True, also match reversed source/target. Returns: - List[Link]: A list of unique Link objects that match the criteria. + List[Link]: Matching Link objects, deduplicated by link ID. """ src_pat = re.compile(source_regex) if source_regex else None tgt_pat = re.compile(target_regex) if target_regex else None @@ -312,12 +356,14 @@ def find_links( return results def disable_risk_group(self, name: str, recursive: bool = True) -> None: - """Disable all nodes/links that have 'name' in their risk_groups. - If recursive=True, also disable items belonging to child risk groups. + """Disable every node/link that has 'name' in its risk_groups. + + Unknown group names are ignored. Args: - name (str): The name of the risk group to disable. - recursive (bool): If True, also disable subgroups recursively. + name (str): Name of the risk group to disable. + recursive (bool): If True, also disable members of child groups, + transitively. """ if name not in self.risk_groups: return @@ -326,23 +372,24 @@ def disable_risk_group(self, name: str, recursive: bool = True) -> None: queue = [self.risk_groups[name]] while queue: grp = queue.pop() + if grp.name in to_disable: + continue to_disable.add(grp.name) if recursive: queue.extend(grp.children) - # Disable nodes for node_name, node_obj in self.nodes.items(): if node_obj.risk_groups & to_disable: self.disable_node(node_name) - # Disable links for link_id, link_obj in self.links.items(): if link_obj.risk_groups & to_disable: self.disable_link(link_id) def enable_risk_group(self, name: str, recursive: bool = True) -> None: - """Enable all nodes/links that have 'name' in their risk_groups. - If recursive=True, also enable items belonging to child risk groups. + """Enable every node/link that has 'name' in its risk_groups. + + Unknown group names are ignored. Note: If a node or link is in multiple risk groups, enabling this group @@ -350,8 +397,9 @@ def enable_risk_group(self, name: str, recursive: bool = True) -> None: remain disabled. Args: - name (str): The name of the risk group to enable. - recursive (bool): If True, also enable subgroups recursively. + name (str): Name of the risk group to enable. + recursive (bool): If True, also enable members of child groups, + transitively. """ if name not in self.risk_groups: return @@ -360,16 +408,16 @@ def enable_risk_group(self, name: str, recursive: bool = True) -> None: queue = [self.risk_groups[name]] while queue: grp = queue.pop() + if grp.name in to_enable: + continue to_enable.add(grp.name) if recursive: queue.extend(grp.children) - # Enable nodes for node_name, node_obj in self.nodes.items(): if node_obj.risk_groups & to_enable: self.enable_node(node_name) - # Enable links for link_id, link_obj in self.links.items(): if link_obj.risk_groups & to_enable: self.enable_link(link_id) diff --git a/ngraph/model/path.py b/ngraph/model/path.py index b5c69ed..48f6320 100644 --- a/ngraph/model/path.py +++ b/ngraph/model/path.py @@ -1,9 +1,8 @@ """Lightweight representation of a single routing path. -The ``Path`` dataclass stores a node-and-parallel-edges sequence and a numeric -cost. Cached properties expose derived sequences for nodes and edges, and -helpers provide equality, ordering by cost, and sub-path extraction with cost -recalculation. +``Path`` stores a sequence of (node, parallel edges) elements plus a numeric +cost. Paths sort by cost, compare by structure and cost, and support sub-path +extraction, which leaves the cost for the caller to recompute. """ from __future__ import annotations @@ -67,7 +66,7 @@ def __len__(self) -> int: """Return the number of elements in the path. Returns: - The length of `path`. + Count of (node, parallel_edges) elements, i.e. hop count plus one. """ return len(self.path) @@ -127,10 +126,11 @@ def __repr__(self) -> str: @cached_property def edges_seq(self) -> Tuple[Tuple[EdgeRef, ...], ...]: - """Return a tuple containing the sequence of parallel-edge tuples for each path element except the last. + """Return the parallel-edge tuples of every path element except the last. Returns: - A tuple of parallel-edge tuples; returns an empty tuple if the path has 1 or fewer elements. + A tuple of parallel-edge tuples; empty if the path has 1 or fewer + elements. """ if len(self.path) <= 1: return () @@ -141,7 +141,7 @@ def nodes_seq(self) -> Tuple[str, ...]: """Return a tuple of node names in order along the path. Returns: - A tuple containing the ordered sequence of nodes from source to destination. + Node names from source to destination, repeats included. """ return tuple(node for node, _ in self.path) diff --git a/ngraph/model/selectors/__init__.py b/ngraph/model/selectors/__init__.py new file mode 100644 index 0000000..311fd83 --- /dev/null +++ b/ngraph/model/selectors/__init__.py @@ -0,0 +1,53 @@ +"""Runtime selector engine for the network model. + +Schema types and the evaluation engine for node, link, and risk-group +selection over `Network` objects. It lives in the model layer so that failure +policies and analysis code can evaluate selectors without depending on the DSL +package. + +YAML-facing selector parsing (`normalize_selector`) lives in +`ngraph.dsl.selectors`, which builds the schema types defined here +(dsl -> model direction only). `parse_match_spec` lives here because it +builds model types from plain dicts and is used by model-layer parsers. + +Usage: + from ngraph.model.selectors import NodeSelector, select_nodes + + selector = NodeSelector(path="^dc1/.*") + groups = select_nodes(network, selector, default_active_only=True) +""" + +from .conditions import evaluate_condition, evaluate_conditions, resolve_attr_path +from .parse import parse_match_spec +from .schema import VALID_OPERATORS, Condition, EntityScope, MatchSpec, NodeSelector +from .select import ( + flatten_link_attrs, + flatten_node_attrs, + flatten_risk_group_attrs, + link_path_key, + match_entity_ids, + select_nodes, +) + +__all__ = [ + # Schema + "Condition", + "EntityScope", + "MatchSpec", + "NodeSelector", + "VALID_OPERATORS", + # Parsing + "parse_match_spec", + # Evaluation + "select_nodes", + "evaluate_condition", + "evaluate_conditions", + "resolve_attr_path", + # Attribute flattening + "flatten_node_attrs", + "flatten_link_attrs", + "flatten_risk_group_attrs", + "link_path_key", + # Entity matching + "match_entity_ids", +] diff --git a/ngraph/dsl/selectors/conditions.py b/ngraph/model/selectors/conditions.py similarity index 93% rename from ngraph/dsl/selectors/conditions.py rename to ngraph/model/selectors/conditions.py index e13746a..346a043 100644 --- a/ngraph/dsl/selectors/conditions.py +++ b/ngraph/model/selectors/conditions.py @@ -1,10 +1,10 @@ """Condition evaluation for node/entity filtering. -Provides evaluation logic for attribute conditions used in selectors -and failure policies. Supports operators: ==, !=, <, <=, >, >=, -contains, not_contains, in, not_in, exists, not_exists. +Evaluates the attribute conditions used by selectors and failure policies. +Operators: ==, !=, <, <=, >, >=, contains, not_contains, in, not_in, exists, +not_exists. -Supports dot-notation for nested attribute access (e.g., "hardware.vendor"). +Attribute names support dot-notation for nested access (e.g. "hardware.vendor"). """ from __future__ import annotations @@ -36,9 +36,9 @@ def resolve_attr_path(attrs: Dict[str, Any], path: str) -> Tuple[bool, Any]: Examples: >>> resolve_attr_path({"role": "spine"}, "role") - (True, "spine") + (True, 'spine') >>> resolve_attr_path({"hardware": {"vendor": "Acme"}}, "hardware.vendor") - (True, "Acme") + (True, 'Acme') >>> resolve_attr_path({"role": "spine"}, "missing") (False, None) """ diff --git a/ngraph/model/selectors/parse.py b/ngraph/model/selectors/parse.py new file mode 100644 index 0000000..58e0859 --- /dev/null +++ b/ngraph/model/selectors/parse.py @@ -0,0 +1,77 @@ +"""Parsing of match specifications from plain dicts. + +Builds model schema types (`Condition`, `MatchSpec`) from raw dict input. +Lives in the model layer so failure-policy and membership parsing can use +it without a runtime dependency on the DSL package. +""" + +from __future__ import annotations + +from typing import Any, Dict, Literal + +from .schema import Condition, MatchSpec + + +def parse_match_spec( + raw: Dict[str, Any], + *, + default_logic: Literal["and", "or"] = "or", + require_conditions: bool = False, + context: str = "match", +) -> MatchSpec: + """Parse a match specification from raw dict. + + Shared by adjacency, demands, membership rules, and failure policies. + + Args: + raw: Dict with 'conditions' list and optional 'logic'. Both keys are + optional; a missing 'conditions' yields an empty condition list. + default_logic: Used when 'logic' is absent. + require_conditions: If True, raise when conditions list is empty. + context: Name of the enclosing construct, quoted in error messages. + + Returns: + Parsed MatchSpec. + + Raises: + ValueError: If 'logic' is not 'and'/'or', 'conditions' is not a list, + a condition is not a dict or lacks 'attr'/'op', 'in'/'not_in' is + given a non-list value, or conditions are required but empty. + """ + logic = raw.get("logic", default_logic) + if logic not in ("and", "or"): + raise ValueError( + f"Invalid logic '{logic}' in {context}. Must be 'and' or 'or'." + ) + + conditions_raw = raw.get("conditions", []) + if not isinstance(conditions_raw, list): + raise ValueError(f"'conditions' in {context} must be a list") + if require_conditions and not conditions_raw: + raise ValueError(f"{context} requires at least one condition") + + conditions = [] + for cond_dict in conditions_raw: + if not isinstance(cond_dict, dict): + raise ValueError( + f"Condition in {context} must be a dict, got {type(cond_dict).__name__}" + ) + if "attr" not in cond_dict or "op" not in cond_dict: + raise ValueError(f"Condition in {context} must have 'attr' and 'op'") + if cond_dict["op"] in ("in", "not_in") and not isinstance( + cond_dict.get("value"), list + ): + raise ValueError( + f"Condition in {context}: operator '{cond_dict['op']}' " + "requires a list value" + ) + + conditions.append( + Condition( + attr=cond_dict["attr"], + op=cond_dict["op"], + value=cond_dict.get("value"), + ) + ) + + return MatchSpec(conditions=conditions, logic=logic) diff --git a/ngraph/dsl/selectors/schema.py b/ngraph/model/selectors/schema.py similarity index 77% rename from ngraph/dsl/selectors/schema.py rename to ngraph/model/selectors/schema.py index 69924a0..3b596cb 100644 --- a/ngraph/dsl/selectors/schema.py +++ b/ngraph/model/selectors/schema.py @@ -1,36 +1,36 @@ """Schema definitions for unified node selection. -Provides dataclasses for node selection configuration used across -network rules, demands, and workflow steps. +Dataclasses shared by network rules, demands, and workflow steps. """ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, List, Literal, Optional +from typing import Any, List, Literal, Optional, get_args # Type alias for entity scope used in condition-based selection EntityScope = Literal["node", "link", "risk_group"] """Type of network entity for condition-based selection.""" -# Valid operators for conditions -VALID_OPERATORS: frozenset[str] = frozenset( - { - "==", - "!=", - "<", - "<=", - ">", - ">=", - "contains", - "not_contains", - "in", - "not_in", - "exists", - "not_exists", - } -) +ConditionOp = Literal[ + "==", + "!=", + "<", + "<=", + ">", + ">=", + "contains", + "not_contains", + "in", + "not_in", + "exists", + "not_exists", +] +"""Comparison operators supported by conditions.""" + +# Valid operators for conditions (derived from ConditionOp) +VALID_OPERATORS: frozenset[str] = frozenset(get_args(ConditionOp)) @dataclass @@ -47,20 +47,7 @@ class Condition: """ attr: str - op: Literal[ - "==", - "!=", - "<", - "<=", - ">", - ">=", - "contains", - "not_contains", - "in", - "not_in", - "exists", - "not_exists", - ] + op: ConditionOp value: Any = None def __post_init__(self) -> None: @@ -113,3 +100,7 @@ def __post_init__(self) -> None: raise ValueError( "NodeSelector requires at least one of: path, group_by, or match" ) + if self.path is not None and not isinstance(self.path, str): + raise ValueError( + f"Selector 'path' must be a string, got {type(self.path).__name__}" + ) diff --git a/ngraph/dsl/selectors/select.py b/ngraph/model/selectors/select.py similarity index 63% rename from ngraph/dsl/selectors/select.py rename to ngraph/model/selectors/select.py index f25535a..f8f9208 100644 --- a/ngraph/dsl/selectors/select.py +++ b/ngraph/model/selectors/select.py @@ -1,12 +1,13 @@ """Node selection and evaluation. -Provides the unified select_nodes() function that handles regex matching, -attribute filtering, active-only filtering, and grouping. +`select_nodes()` combines regex matching, attribute filtering, active-only +filtering, and grouping; the flatten helpers build the attribute dicts that +condition evaluation runs against. """ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union +from typing import TYPE_CHECKING, Any, Dict, List, Set from .conditions import evaluate_conditions from .schema import Condition, MatchSpec, NodeSelector @@ -19,6 +20,7 @@ "flatten_node_attrs", "flatten_link_attrs", "flatten_risk_group_attrs", + "link_path_key", "match_entity_ids", ] @@ -27,28 +29,26 @@ def select_nodes( network: "Network", selector: NodeSelector, default_active_only: bool, - excluded_nodes: Optional[Set[str]] = None, ) -> Dict[str, List["Node"]]: """Unified entry point for node selection. Evaluation order: 1. Select nodes matching `path` regex (or all nodes if path is None) 2. Filter by `match` conditions - 3. Filter by `active_only` flag and excluded_nodes + 3. Filter by `active_only` flag 4. Group by `group_by` attribute (overrides regex capture grouping) Args: - network: The network graph. + network: Network whose nodes are searched. selector: Node selection specification. - default_active_only: Context-aware default for active_only flag. - Required parameter to prevent silent bugs. - excluded_nodes: Additional node names to exclude. + default_active_only: Used when the selector leaves `active_only` + unset. Required rather than defaulted so callers cannot silently + inherit the wrong policy. Returns: - Dict mapping group labels to lists of nodes. + Dict mapping group labels to lists of nodes. Groups that filter down + to nothing are dropped. """ - excluded = excluded_nodes or set() - # Resolve effective active_only flag active_only = ( selector.active_only @@ -56,9 +56,10 @@ def select_nodes( else default_active_only ) - # Step 1: Select by path regex (or all nodes) + # Step 1: Select by path regex (or all nodes). Regex selection delegates + # to Network.select_node_groups_by_path() which provides caching. if selector.path is not None: - candidates = _select_by_regex(network, selector.path) + candidates = network.select_node_groups_by_path(selector.path) else: candidates = {"_all_": list(network.nodes.values())} @@ -66,9 +67,9 @@ def select_nodes( if selector.match is not None: candidates = _filter_by_match(candidates, selector.match) - # Step 3: Filter active only + excluded - if active_only or excluded: - candidates = _filter_active_and_excluded(candidates, active_only, excluded) + # Step 3: Filter active only + if active_only: + candidates = _filter_active(candidates) # Step 4: Apply grouping (overrides regex capture grouping) if selector.group_by is not None: @@ -77,14 +78,6 @@ def select_nodes( return candidates -def _select_by_regex(network: "Network", pattern: str) -> Dict[str, List["Node"]]: - """Select nodes by regex pattern with capture group handling. - - Delegates to Network.select_node_groups_by_path() which provides caching. - """ - return network.select_node_groups_by_path(pattern) - - def _filter_by_match( groups: Dict[str, List["Node"]], match: MatchSpec, @@ -119,7 +112,8 @@ def flatten_node_attrs(node: "Node") -> Dict[str, Any]: attrs: Dict[str, Any] = { "name": node.name, "disabled": node.disabled, - "risk_groups": list(node.risk_groups), + # Sorted for deterministic group_by labels and ==/in comparisons. + "risk_groups": sorted(node.risk_groups), } # Add user attrs, but don't overwrite top-level fields attrs.update({k: v for k, v in node.attrs.items() if k not in attrs}) @@ -146,50 +140,40 @@ def flatten_link_attrs(link: "Link", link_id: str) -> Dict[str, Any]: "capacity": link.capacity, "cost": link.cost, "disabled": link.disabled, - "risk_groups": list(link.risk_groups), + # Sorted for deterministic group_by labels and ==/in comparisons. + "risk_groups": sorted(link.risk_groups), } attrs.update({k: v for k, v in link.attrs.items() if k not in attrs}) return attrs -def flatten_risk_group_attrs( - rg: Union["RiskGroup", Dict[str, Any]], -) -> Dict[str, Any]: +def link_path_key(attrs: Dict[str, Any]) -> str: + """Return the "source|target" key used when path-matching links. + + Links have no name of their own, so path regexes match against this + canonical endpoint-pair form of the flattened link attributes. + """ + return f"{attrs['source']}|{attrs['target']}" + + +def flatten_risk_group_attrs(rg: "RiskGroup") -> Dict[str, Any]: """Build flat attribute dict for condition evaluation on risk groups. Merges risk group's top-level fields (name, disabled, children) with rg.attrs. Top-level fields take precedence on key conflicts. - Supports both RiskGroup objects and dict representations (for flexibility - in failure policy matching). - Args: - rg: RiskGroup object or dict representation. + rg: RiskGroup object. Returns: Flat dict suitable for condition evaluation. """ - if isinstance(rg, dict): - # Dict representation - children_raw = rg.get("children", []) - child_names = [ - c.get("name") if isinstance(c, dict) else c.name for c in children_raw - ] - attrs: Dict[str, Any] = { - "name": rg.get("name", ""), - "disabled": rg.get("disabled", False), - "children": child_names, - } - attrs.update({k: v for k, v in rg.get("attrs", {}).items() if k not in attrs}) - else: - # RiskGroup object - attrs = { - "name": rg.name, - "disabled": rg.disabled, - "children": [c.name for c in rg.children], - } - attrs.update({k: v for k, v in rg.attrs.items() if k not in attrs}) - + attrs: Dict[str, Any] = { + "name": rg.name, + "disabled": rg.disabled, + "children": [c.name for c in rg.children], + } + attrs.update({k: v for k, v in rg.attrs.items() if k not in attrs}) return attrs @@ -221,42 +205,54 @@ def match_entity_ids( } -def _filter_active_and_excluded( +def _filter_active( groups: Dict[str, List["Node"]], - active_only: bool, - excluded: Set[str], ) -> Dict[str, List["Node"]]: - """Remove disabled and/or explicitly excluded nodes.""" + """Remove disabled nodes, dropping groups that become empty.""" result: Dict[str, List["Node"]] = {} for label, nodes in groups.items(): - filtered = [] - for n in nodes: - if n.name in excluded: - continue - if active_only and n.disabled: - continue - filtered.append(n) + filtered = [n for n in nodes if not n.disabled] if filtered: result[label] = filtered return result +_MISSING = object() + + +def _node_attr_value(node: "Node", attr_name: str) -> Any: + """Resolve a single node attribute without building the full flat dict. + + Mirrors flatten_node_attrs semantics: top-level fields (name, disabled, + risk_groups) take precedence over node.attrs. Returns _MISSING when the + attribute is absent. + """ + if attr_name == "name": + return node.name + if attr_name == "disabled": + return node.disabled + if attr_name == "risk_groups": + # Sorted for deterministic group_by labels. + return sorted(node.risk_groups) + return node.attrs.get(attr_name, _MISSING) + + def _group_by_attribute( groups: Dict[str, List["Node"]], attr_name: str, ) -> Dict[str, List["Node"]]: """Re-group nodes by attribute value. - Uses flatten_node_attrs to support both top-level fields (name, disabled, - risk_groups) and custom attrs, consistent with match condition evaluation. + Supports both top-level fields (name, disabled, risk_groups) and custom + attrs, consistent with match condition evaluation. Nodes lacking the + attribute are dropped. Note: This discards any existing grouping (including regex captures). """ result: Dict[str, List["Node"]] = {} for nodes in groups.values(): for node in nodes: - flat_attrs = flatten_node_attrs(node) - if attr_name in flat_attrs: - key = str(flat_attrs[attr_name]) - result.setdefault(key, []).append(node) + value = _node_attr_value(node, attr_name) + if value is not _MISSING: + result.setdefault(str(value), []).append(node) return result diff --git a/ngraph/profiling/__init__.py b/ngraph/profiling/__init__.py index 4097b43..726f956 100644 --- a/ngraph/profiling/__init__.py +++ b/ngraph/profiling/__init__.py @@ -1,6 +1,6 @@ """Profiling instrumentation and reporting for NetGraph. -This package exposes public profiling APIs for workflow execution: +Public API: - ``PerformanceProfiler``: CPU and wall-time profiling per workflow step. - ``PerformanceReporter``: Text report generation from profiling results. diff --git a/ngraph/profiling/profiler.py b/ngraph/profiling/profiler.py index 3896617..28dca3e 100644 --- a/ngraph/profiling/profiler.py +++ b/ngraph/profiling/profiler.py @@ -71,7 +71,8 @@ class ProfileResults: class PerformanceProfiler: """CPU profiler for NetGraph workflow execution. - Profiles workflow steps using cProfile and identifies bottlenecks. + Profiles each workflow step with cProfile and flags steps that take more + than 10% of total wall time as bottlenecks. """ def __init__(self, track_memory: bool = False): @@ -130,12 +131,11 @@ def profile_step( """ logger.debug(f"Starting profiling for step: {step_name} ({step_type})") - # Initialize profiling data start_time = time.perf_counter() profiler = cProfile.Profile() profiler.enable() - # Optional: per-step tracemalloc to capture peak memory + # Per-step tracemalloc, when requested, to capture peak memory mem_tracing_started = False if self._track_memory: try: @@ -148,14 +148,11 @@ def profile_step( try: yield finally: - # Capture end time end_time = time.perf_counter() wall_time = end_time - start_time - # Capture CPU profiling data profiler.disable() - # Create stats object for analysis stats_stream = io.StringIO() stats = pstats.Stats(profiler, stream=stats_stream) @@ -171,7 +168,6 @@ def profile_step( stat_tuple[0] for stat_tuple in stats_data.values() ) # cc = call count - # Optional: capture peak memory usage memory_peak_bytes: Optional[int] = None if mem_tracing_started: try: @@ -186,7 +182,6 @@ def profile_step( except Exception as exc: logger.debug("Failed to stop tracemalloc: %s", exc) - # Create step profile step_profile = StepProfile( step_name=step_name, step_type=step_type, @@ -213,7 +208,6 @@ def merge_child_profiles(self, profile_dir: Path, step_name: str) -> None: profile_dir: Directory containing worker profile files. step_name: Name of the workflow step these workers belong to. """ - # Find the step profile to merge into step_profile = None for profile in self.results.step_profiles: if profile.step_name == step_name: @@ -224,15 +218,15 @@ def merge_child_profiles(self, profile_dir: Path, step_name: str) -> None: logger.warning(f"No parent profile found for step: {step_name}") return - # Find all worker profile files for this step - worker_files = list(profile_dir.glob("*_worker_*.pstats")) + # Find all worker profile files for this step. Workers in + # analysis/failure_manager.py write {analysis_name}_thread_{tid}_{uuid}.pstats. + worker_files = list(profile_dir.glob("*_thread_*.pstats")) if not worker_files: logger.debug(f"No worker profiles found in {profile_dir}") return logger.debug(f"Found {len(worker_files)} worker profiles to merge") - # Merge all worker stats into the parent stats try: merged_count = 0 for worker_file in worker_files: @@ -274,12 +268,10 @@ def analyze_performance(self) -> None: logger.debug("Starting performance analysis") - # Identify time-consuming steps sorted_steps = sorted( self.results.step_profiles, key=lambda p: p.wall_time, reverse=True ) - # Calculate percentage of total time for each step total_time = self.results.total_wall_time step_percentages = [] @@ -307,7 +299,6 @@ def analyze_performance(self) -> None: self.results.bottlenecks = bottlenecks - # Generate analysis summary self.results.analysis_summary = { "total_steps": len(self.results.step_profiles), "slowest_step": sorted_steps[0].step_name if sorted_steps else None, @@ -401,10 +392,9 @@ def save_detailed_profile( class PerformanceReporter: - """Format and render performance profiling results. + """Render profiling results as a plain-text report. - Generates plain-text reports with timing analysis, bottleneck identification, - and practical performance tuning suggestions. + Covers per-step timing, bottleneck identification, and tuning suggestions. """ def __init__(self, results: ProfileResults): @@ -589,7 +579,6 @@ def _generate_detailed_analysis(self) -> List[str]: step_name = bottleneck["step_name"] lines.append(f"Top CPU-consuming functions in '{step_name}':") - # Get profiler reference to access top functions profiler = None for profile in self.results.step_profiles: if profile.step_name == step_name: diff --git a/ngraph/results/artifacts.py b/ngraph/results/artifacts.py index 47cda78..cb13c16 100644 --- a/ngraph/results/artifacts.py +++ b/ngraph/results/artifacts.py @@ -1,26 +1,22 @@ """Serializable result artifacts for analysis workflows. -This module defines dataclasses that capture outputs from analyses and -simulations in a JSON-serializable form: - -- `CapacityEnvelope`: frequency-based capacity distributions and optional - aggregated flow statistics -- `FailurePatternResult`: capacity results for specific failure patterns +`CapacityEnvelope` captures a frequency-based capacity distribution, plus +optional aggregated flow statistics, in JSON-serializable form. """ from __future__ import annotations -import hashlib from dataclasses import dataclass, field from typing import Any, Dict, List @dataclass class CapacityEnvelope: - """Frequency-based capacity envelope that stores capacity values as frequencies. + """Capacity distribution stored as a value -> occurrence-count map. - This approach is memory-efficient for Monte Carlo analysis where we care - about statistical distributions rather than individual sample order. + Monte Carlo runs repeat the same capacity values many times, so counting + them keeps memory proportional to the number of distinct values. Individual + sample order is not preserved. Attributes: source_pattern: Regex pattern used to select source nodes. @@ -81,15 +77,11 @@ def from_values( max_capacity = float("-inf") for value in values: - # Update frequency map frequencies[value] = frequencies.get(value, 0) + 1 - - # Update statistics total_sum += value min_capacity = min(min_capacity, value) max_capacity = max(max_capacity, value) - # Calculate derived statistics n = len(values) mean_capacity = total_sum / n @@ -103,7 +95,6 @@ def from_values( variance_sum += count * diff * diff stdev_capacity = (variance_sum / n) ** 0.5 - # Process flow summaries if provided flow_summary_stats = {} if flow_summaries: flow_summary_stats = cls._aggregate_flow_summaries(flow_summaries) @@ -131,7 +122,7 @@ def _aggregate_flow_summaries(cls, flow_summaries: List[Any]) -> Dict[str, Any]: Returns: Dictionary with aggregated flow analytics including cost distribution statistics. """ - from collections import defaultdict + from collections import Counter, defaultdict # Aggregate cost distributions cost_data = defaultdict(list) # cost -> list of flow volumes @@ -176,7 +167,7 @@ def _aggregate_flow_summaries(cls, flow_summaries: List[Any]) -> Dict[str, Any]: "min": min(volumes), "max": max(volumes), "total_samples": len(volumes), - "frequencies": {vol: volumes.count(vol) for vol in set(volumes)}, + "frequencies": dict(Counter(volumes)), } return { @@ -199,7 +190,6 @@ def to_dict(self) -> Dict[str, Any]: "total_samples": self.total_samples, } - # Include flow summary stats if available if self.flow_summary_stats: result["flow_summary_stats"] = self.flow_summary_stats @@ -217,13 +207,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "CapacityEnvelope": """ # Frequencies keys may arrive as strings via JSON; normalize to float freqs_raw = data.get("frequencies", {}) or {} - freqs: Dict[float, int] = {} - for k, v in freqs_raw.items(): - try: - key_f = float(k) - except (TypeError, ValueError): - key_f = float(k) # Will raise again if irrecoverable - freqs[key_f] = int(v) + freqs: Dict[float, int] = {float(k): int(v) for k, v in freqs_raw.items()} return cls( source_pattern=str(data.get("source", "")), @@ -255,7 +239,6 @@ def get_percentile(self, percentile: float) -> float: target_count = (percentile / 100.0) * self.total_samples - # Sort capacities and accumulate counts sorted_capacities = sorted(self.frequencies.keys()) cumulative_count = 0 @@ -276,69 +259,3 @@ def expand_to_values(self) -> List[float]: for capacity, count in self.frequencies.items(): values.extend([capacity] * count) return values - - -@dataclass -class FailurePatternResult: - """Result for a unique failure pattern with associated capacity matrix. - - Attributes: - excluded_nodes: List of failed node IDs. - excluded_links: List of failed link IDs. - capacity_matrix: Dictionary mapping flow keys to capacity values. - count: Number of times this pattern occurred. - is_baseline: Whether this represents the baseline (no failures) case. - """ - - excluded_nodes: List[str] - excluded_links: List[str] - capacity_matrix: Dict[str, float] - count: int - is_baseline: bool = False - _pattern_key_cache: str = field(default="", init=False, repr=False) - - def to_dict(self) -> Dict[str, Any]: - """Convert to dictionary for JSON serialization.""" - return { - "excluded_nodes": self.excluded_nodes, - "excluded_links": self.excluded_links, - "capacity_matrix": self.capacity_matrix, - "count": self.count, - "is_baseline": self.is_baseline, - } - - @property - def pattern_key(self) -> str: - """Generate a deterministic key for this failure pattern. - - Uses a stable BLAKE2s hash of the sorted excluded entity list to avoid - Python's randomized hash() variability across processes. - - Returns empty string for patterns with no exclusions (including baseline). - """ - # Cache to avoid recomputation when accessed repeatedly - if self._pattern_key_cache: - return self._pattern_key_cache - - # Empty exclusions (no failures) return empty string - if not self.excluded_nodes and not self.excluded_links: - return "" - - # Create deterministic key from excluded entities using fast BLAKE2s - excluded_str = ",".join(sorted(self.excluded_nodes + self.excluded_links)) - digest = hashlib.blake2s( - excluded_str.encode("utf-8"), digest_size=8 - ).hexdigest() - self._pattern_key_cache = f"pattern_{digest}" - return self._pattern_key_cache - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "FailurePatternResult": - """Construct FailurePatternResult from a dictionary.""" - return cls( - excluded_nodes=list(data.get("excluded_nodes", [])), - excluded_links=list(data.get("excluded_links", [])), - capacity_matrix=dict(data.get("capacity_matrix", {})), - count=int(data.get("count", 0)), - is_baseline=bool(data.get("is_baseline", False)), - ) diff --git a/ngraph/results/flow.py b/ngraph/results/flow.py index 55578b1..d96aaaa 100644 --- a/ngraph/results/flow.py +++ b/ngraph/results/flow.py @@ -9,9 +9,8 @@ `data.flow_results` by steps. Utilities: - _fmt_float_key: Formats floats as stable string keys for JSON serialization. - Uses fixed-point notation with trailing zeros stripped for human-readable, - canonical representations of numeric keys like cost distributions. + _fmt_float_key: Formats floats as stable string keys for JSON serialization, + in fixed-point notation with trailing zeros stripped. """ from __future__ import annotations @@ -49,7 +48,7 @@ def _fmt_float_key(x: float, places: int = 9) -> str: @dataclass(slots=True) class FlowEntry: - """Represents a single source→destination flow outcome within an iteration. + """One source→destination flow outcome within an iteration. Fields are unit-agnostic. Callers can interpret numbers as needed for presentation (e.g., Gbit/s). @@ -75,7 +74,7 @@ class FlowEntry: data: Dict[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: - """Validate invariants and types for early error detection. + """Validate field types and invariants. Raises: ValueError: If any numeric fields are NaN/inf or logically inconsistent. @@ -179,7 +178,6 @@ def to_dict(self) -> Dict[str, Any]: except Exception: # pragma: no cover - defensive normalized_costs[str(k)] = float(v) - # Build dict directly from known fields (avoids asdict() overhead) return { "source": self.source, "destination": self.destination, @@ -211,7 +209,7 @@ class FlowSummary: num_flows: int def __post_init__(self) -> None: - """Validate summary invariants for correctness. + """Validate summary invariants. Raises: ValueError: If totals/ratio are inconsistent or invalid. diff --git a/ngraph/results/snapshot.py b/ngraph/results/snapshot.py index 3716c59..53b7d81 100644 --- a/ngraph/results/snapshot.py +++ b/ngraph/results/snapshot.py @@ -17,10 +17,6 @@ def build_scenario_snapshot( ) -> Dict[str, Any]: """Build a concise dictionary snapshot of the scenario state. - Creates a serializable representation of the scenario's failure policies - and demand sets, suitable for export into results without keeping heavy - domain objects. - Args: seed: Scenario-level seed for reproducibility, or None if unseeded. failure_policy_set: FailurePolicySet containing named failure policies. @@ -29,60 +25,16 @@ def build_scenario_snapshot( Returns: Dict containing: seed, failures (policy snapshots), demands (demand snapshots). """ - snapshot_failure_policies: Dict[str, Any] = {} - for name, policy in getattr(failure_policy_set, "policies", {}).items(): - modes_list: list[dict[str, Any]] = [] - for mode in getattr(policy, "modes", []) or []: - mode_dict = { - "weight": float(getattr(mode, "weight", 0.0)), - "rules": [], - "attrs": dict(getattr(mode, "attrs", {}) or {}), - } - for rule in getattr(mode, "rules", []) or []: - mode_dict["rules"].append( - { - "scope": getattr(rule, "scope", "node"), - "logic": getattr(rule, "logic", "or"), - "mode": getattr(rule, "mode", "all"), - "probability": float(getattr(rule, "probability", 1.0)), - "count": int(getattr(rule, "count", 1)), - "path": getattr(rule, "path", None), - "conditions": [ - { - "attr": c.attr, - "op": c.op, - "value": c.value, - } - for c in getattr(rule, "conditions", []) or [] - ], - } - ) - modes_list.append(mode_dict) - snapshot_failure_policies[name] = { - "attrs": dict(getattr(policy, "attrs", {}) or {}), - "expand_groups": getattr(policy, "expand_groups", False), - "expand_children": getattr(policy, "expand_children", False), - "modes": modes_list, - } + # Delegate policy serialization to FailurePolicy.to_dict so the snapshot + # matches the scenario YAML format (rule conditions nested under "match"). + snapshot_failure_policies: Dict[str, Any] = { + name: policy.to_dict() for name, policy in failure_policy_set.policies.items() + } - snapshot_demands: Dict[str, list[dict[str, Any]]] = {} - for sname, demands in getattr(demand_set, "sets", {}).items(): - entries: list[dict[str, Any]] = [] - for d in demands: - entries.append( - { - "id": getattr(d, "id", None), - "source": getattr(d, "source", ""), - "target": getattr(d, "target", ""), - "volume": float(getattr(d, "volume", 0.0)), - "priority": int(getattr(d, "priority", 0)), - "mode": getattr(d, "mode", "pairwise"), - "group_mode": getattr(d, "group_mode", "flatten"), - "flow_policy": getattr(d, "flow_policy", None), - "attrs": dict(getattr(d, "attrs", {}) or {}), - } - ) - snapshot_demands[sname] = entries + snapshot_demands: Dict[str, list[dict[str, Any]]] = { + sname: [d.to_dict() for d in demands] + for sname, demands in demand_set.sets.items() + } return { "seed": seed, diff --git a/ngraph/results/store.py b/ngraph/results/store.py index dfa266d..547c9a8 100644 --- a/ngraph/results/store.py +++ b/ngraph/results/store.py @@ -17,12 +17,40 @@ from typing import Any, Dict, Optional +def _deep_convert(v: Any) -> Any: + """Recursively convert a value into a JSON-safe structure. + + Objects exposing a callable ``to_dict()`` are converted and their output is + converted recursively, dictionary keys are coerced to strings, and tuples + are emitted as lists. + + Args: + v: Value to convert. + + Returns: + JSON-safe representation of ``v``. + """ + to_dict = getattr(v, "to_dict", None) + if callable(to_dict): + converted = to_dict() + # Recurse only into plain containers to avoid unbounded recursion on + # objects whose to_dict() returns another convertible object. + if isinstance(converted, (dict, list, tuple)): + return _deep_convert(converted) + return converted + if isinstance(v, dict): + return {str(k): _deep_convert(val) for k, val in v.items()} + if isinstance(v, (list, tuple)): + return [_deep_convert(x) for x in v] + return v + + @dataclass class WorkflowStepMetadata: """Metadata for a workflow step execution. Attributes: - step_type: The workflow step class name (e.g., 'CapacityEnvelopeAnalysis'). + step_type: The workflow step class name (e.g., 'NetworkStats'). step_name: The instance name of the step. execution_order: Order in which this step was executed (0-based). scenario_seed: Scenario-level seed provided in the YAML (if any). @@ -204,26 +232,15 @@ def to_dict(self) -> Dict[str, Any]: f"Step '{step_name}' must store dicts for 'metadata' and 'data'" ) - def deep_convert(v: Any) -> Any: - # Convert nested structures; apply to_dict to any object that supports it - if hasattr(v, "to_dict") and callable(v.to_dict): - return v.to_dict() - if isinstance(v, dict): - return {str(k): deep_convert(val) for k, val in v.items()} - if isinstance(v, (list, tuple)): - return [deep_convert(x) for x in v] - return v - steps[step_name] = { - "metadata": deep_convert(metadata_part), - "data": deep_convert(data_part), + "metadata": _deep_convert(metadata_part), + "data": _deep_convert(data_part), } - # Compose final out: Dict[str, Any] = { "workflow": workflow, "steps": steps, } if self._scenario: - out["scenario"] = self._scenario + out["scenario"] = _deep_convert(self._scenario) return out diff --git a/ngraph/scenario.py b/ngraph/scenario.py index 65e7aa9..6392c49 100644 --- a/ngraph/scenario.py +++ b/ngraph/scenario.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import List, Optional +from typing import Callable, ContextManager, List, Optional from ngraph.dsl.blueprints.expand import expand_network_dsl from ngraph.dsl.loader import load_scenario_yaml @@ -23,18 +23,18 @@ from ngraph.results import Results from ngraph.results.snapshot import build_scenario_snapshot from ngraph.utils.seed_manager import SeedManager -from ngraph.workflow.base import WorkflowStep +from ngraph.workflow.base import WorkflowStep, validate_unique_step_names from ngraph.workflow.parse import build_workflow_steps @dataclass class Scenario: - """Represents a complete scenario for building and executing network workflows. + """A complete scenario for building and executing network workflows. - This scenario includes: + Holds: - A network (nodes/links), constructed via blueprint expansion. - A failure policy set (one or more named failure policies). - - A traffic matrix set containing one or more named traffic matrices. + - A demand set containing one or more named demand collections. - A list of workflow steps to execute. - A results container for storing outputs. - A components_library for hardware/optics definitions. @@ -60,25 +60,31 @@ class Scenario: # Module-level logger _logger = get_logger(__name__) - @property - def seed_manager(self) -> SeedManager: - """Get the seed manager for this scenario. + def run( + self, + step_hook: Optional[Callable[[WorkflowStep], ContextManager[None]]] = None, + ) -> None: + """Execute the scenario's workflow steps in order. - Returns: - SeedManager instance configured with this scenario's seed. - """ - return SeedManager(self.seed) - - def run(self) -> None: - """Executes the scenario's workflow steps in order. + A step may modify scenario data or store outputs in scenario.results. - Each step may modify scenario data or store outputs - in scenario.results. + Args: + step_hook: Optional callable invoked once per step with the step + about to run. It must return a context manager, which is + entered before ``step.execute`` and exited after it returns + (e.g., for per-step profiling). Exceptions raised by a step + propagate through the context manager. """ + # Reject duplicate effective step names before executing anything. + validate_unique_step_names(self.workflow) # Reset instance execution counter for this run self._execution_counter = 0 for step in self.workflow: - step.execute(self) + if step_hook is None: + step.execute(self) + else: + with step_hook(step): + step.execute(self) @classmethod def from_yaml( @@ -86,8 +92,8 @@ def from_yaml( yaml_str: str, default_components: Optional[ComponentsLibrary] = None, ) -> Scenario: - """Constructs a Scenario from a YAML string, optionally merging - with a default ComponentsLibrary if provided. + """Construct a Scenario from a YAML string, merging in a default + ComponentsLibrary when one is given. Top-level YAML keys can include: - vars: YAML anchors for value reuse @@ -101,10 +107,17 @@ def from_yaml( - seed: Master seed for reproducible randomness Risk group processing: - 1. Direct definitions and membership rules are registered - 2. Generate blocks create groups from unique attribute values - 3. Membership rules auto-assign entities to groups - 4. References are validated (undefined groups and circular hierarchies detected) + 1. Direct risk-group definitions are registered + 2. Membership rules auto-assign entities to groups + 3. The risk-group hierarchy is validated (cycle detection) + 4. Generate blocks create groups from unique attribute values + (membership rules cannot match generated groups, since generation + runs after membership resolution) + 5. Members of groups declared disabled are disabled (recursive + cascade; runs after membership and generate processing so rule- + and generate-assigned entities are covered) + 6. Risk-group references on nodes/links are validated (undefined + groups detected) If no 'workflow' key is provided, the scenario has no steps to run. If 'failures' is omitted, scenario.failure_policy_set is empty. @@ -138,15 +151,11 @@ def from_yaml( if network_obj is None: network_obj = Network() else: - try: - Scenario._logger.debug( - "Expanded network: nodes=%d, links=%d", - len(getattr(network_obj, "nodes", {})), - len(getattr(network_obj, "links", {})), - ) - except Exception as exc: - # Defensive: network object may not be fully initialised in some error paths - Scenario._logger.debug("Failed to log network stats: %s", exc) + Scenario._logger.debug( + "Expanded network: nodes=%d, links=%d", + len(network_obj.nodes), + len(network_obj.links), + ) # 2) Build the failure policy set seed_manager = SeedManager(seed) @@ -156,37 +165,27 @@ def from_yaml( ) if failure_policy_set.policies: - try: - policy_names = sorted(list(failure_policy_set.policies.keys())) - Scenario._logger.debug( - "Built FailurePolicySet: %d policies (%s)", - len(policy_names), - ", ".join(policy_names[:5]) - + ("..." if len(policy_names) > 5 else ""), - ) - except Exception as exc: - Scenario._logger.debug("Failed to log policy set stats: %s", exc) + policy_names = sorted(failure_policy_set.policies.keys()) + Scenario._logger.debug( + "Built FailurePolicySet: %d policies (%s)", + len(policy_names), + ", ".join(policy_names[:5]) + ("..." if len(policy_names) > 5 else ""), + ) # 3) Build demand sets raw = data.get("demands", {}) ds = build_demand_set(raw) - try: - set_names = sorted(list(getattr(ds, "sets", {}).keys())) - total_demands = 0 - for _sname, demands in getattr(ds, "sets", {}).items(): - total_demands += len(demands) - Scenario._logger.debug( - "Constructed DemandSet: sets=%d, total_demands=%d%s", - len(set_names), - total_demands, - ( - f" ({', '.join(set_names[:5])}{'...' if len(set_names) > 5 else ''})" - if set_names - else "" - ), - ) - except Exception as exc: - Scenario._logger.debug("Failed to log demand set stats: %s", exc) + set_names = sorted(ds.sets.keys()) + Scenario._logger.debug( + "Constructed DemandSet: sets=%d, total_demands=%d%s", + len(set_names), + sum(len(demands) for demands in ds.sets.values()), + ( + f" ({', '.join(set_names[:5])}{'...' if len(set_names) > 5 else ''})" + if set_names + else "" + ), + ) # 4) Build workflow steps workflow_data = data.get("workflow", []) @@ -194,22 +193,16 @@ def from_yaml( workflow_data, derive_seed=lambda name: seed_manager.derive_seed("workflow_step", name), ) - try: - labels: list[str] = [] - for idx, step in enumerate(workflow_steps): - label = (step.name or step.__class__.__name__) or f"step_{idx}" - labels.append(label) - Scenario._logger.debug( - "Built workflow: steps=%d%s", - len(workflow_steps), - ( - f" ({', '.join(labels[:8])}{'...' if len(labels) > 8 else ''})" - if labels - else "" - ), - ) - except Exception as exc: - Scenario._logger.debug("Failed to log workflow stats: %s", exc) + labels = [step.name or step.__class__.__name__ for step in workflow_steps] + Scenario._logger.debug( + "Built workflow: steps=%d%s", + len(workflow_steps), + ( + f" ({', '.join(labels[:8])}{'...' if len(labels) > 8 else ''})" + if labels + else "" + ), + ) # 5) Build/merge components library scenario_comps_data = data.get("components", {}) @@ -231,15 +224,9 @@ def from_yaml( risk_groups, generate_specs_raw = build_risk_groups(rg_data) for rg in risk_groups: network_obj.risk_groups[rg.name] = rg - if rg.disabled: - network_obj.disable_risk_group(rg.name, recursive=True) - try: - Scenario._logger.debug( - "Attached risk groups: %d", - len(getattr(network_obj, "risk_groups", {})), - ) - except Exception as exc: - Scenario._logger.debug("Failed to log risk group stats: %s", exc) + Scenario._logger.debug( + "Attached risk groups: %d", len(network_obj.risk_groups) + ) # 7) Resolve membership rules (adds entities to risk groups based on conditions) resolve_membership_rules(network_obj) @@ -265,14 +252,17 @@ def from_yaml( except ValueError as e: raise ValueError(f"Invalid generate block: {e}") from e - try: - if generate_specs_raw: - Scenario._logger.debug( - "Generated risk groups: total now %d", - len(getattr(network_obj, "risk_groups", {})), - ) - except Exception as exc: - Scenario._logger.debug("Failed to log generate stats: %s", exc) + if generate_specs_raw: + Scenario._logger.debug( + "Generated risk groups: total now %d", len(network_obj.risk_groups) + ) + + # Disable members of risk groups declared disabled. This runs after + # membership rules and generate blocks so entities assigned to groups + # by those mechanisms are covered by the cascade. + for rg in network_obj.risk_groups.values(): + if rg.disabled: + network_obj.disable_risk_group(rg.name, recursive=True) # 10) Validate risk group references # Ensures all risk group names referenced by nodes/links are defined @@ -300,16 +290,13 @@ def from_yaml( # Snapshot should never block scenario construction Scenario._logger.debug("Failed to attach scenario snapshot: %s", exc) - try: - Scenario._logger.debug( - "Scenario constructed: nodes=%d, links=%d, policies=%d, matrices=%d, steps=%d", - len(getattr(network_obj, "nodes", {})), - len(getattr(network_obj, "links", {})), - len(getattr(failure_policy_set, "policies", {})), - len(getattr(ds, "sets", {})), - len(workflow_steps), - ) - except Exception as exc: - Scenario._logger.debug("Failed to log scenario construction stats: %s", exc) + Scenario._logger.debug( + "Scenario constructed: nodes=%d, links=%d, policies=%d, demand_sets=%d, steps=%d", + len(network_obj.nodes), + len(network_obj.links), + len(failure_policy_set.policies), + len(ds.sets), + len(workflow_steps), + ) return scenario_obj diff --git a/ngraph/schemas/scenario.json b/ngraph/schemas/scenario.json index 31a1349..5ff8a62 100644 --- a/ngraph/schemas/scenario.json +++ b/ngraph/schemas/scenario.json @@ -5,6 +5,37 @@ "description": "JSON Schema for NetGraph network scenario YAML files", "type": "object", "$defs": { + "riskGroupChild": { + "oneOf": [ + { + "type": "string", + "description": "String shorthand for a simple child risk group" + }, + { + "type": "object", + "description": "Child risk group definition. 'membership', 'disabled', and 'generate' are top-level-only: define such groups at top level and reference them by name as children", + "properties": { + "name": { + "type": "string", + "description": "Unique risk group name" + }, + "attrs": { + "type": "object", + "description": "Additional metadata for the risk group" + }, + "children": { + "type": "array", + "description": "Nested child risk groups", + "items": { + "$ref": "#/$defs/riskGroupChild" + } + } + }, + "required": ["name"], + "additionalProperties": false + } + ] + }, "condition": { "type": "object", "description": "A single attribute condition for entity filtering", @@ -121,37 +152,6 @@ }, "additionalProperties": false }, - "linkProperties": { - "type": "object", - "description": "Common link properties (flattened, no wrapper)", - "properties": { - "capacity": { - "type": "number", - "description": "Link capacity" - }, - "cost": { - "type": "number", - "description": "Link cost" - }, - "disabled": { - "type": "boolean", - "description": "Whether the link is disabled" - }, - "risk_groups": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Risk groups this link belongs to" - }, - "attrs": { - "type": "object", - "description": "Additional link attributes", - "additionalProperties": true - } - }, - "additionalProperties": false - }, "nodeDefinition": { "type": "object", "description": "Node definition - either single node, counted group, blueprint reference, or nested nodes", @@ -374,6 +374,10 @@ "$ref": "#/$defs/expandBlock" } }, + "required": [ + "source", + "target" + ], "additionalProperties": false } } @@ -433,7 +437,7 @@ "type": "array", "description": "Nested child risk groups", "items": { - "$ref": "#/properties/risk_groups/items" + "$ref": "#/$defs/riskGroupChild" } }, "membership": { @@ -531,10 +535,6 @@ "type": "boolean", "description": "Whether to fail risk groups" }, - "expand_children": { - "type": "boolean", - "description": "Whether to recursively fail risk group children" - }, "modes": { "type": "array", "description": "Weighted mode list; exactly one mode is chosen per iteration.", @@ -658,7 +658,7 @@ "$ref": "#/$defs/expandBlock" }, "flow_policy": { - "description": "Routing policy configuration (preset name or inline object)", + "description": "Routing policy configuration (preset name or integer)", "oneOf": [ { "type": "string" @@ -666,9 +666,6 @@ { "type": "integer" }, - { - "type": "object" - }, { "type": "null" } diff --git a/ngraph/types/__init__.py b/ngraph/types/__init__.py index 34f9684..62a6603 100644 --- a/ngraph/types/__init__.py +++ b/ngraph/types/__init__.py @@ -1,11 +1,11 @@ """Shared typing constructs for NetGraph. -This package defines public type aliases and protocols used across the codebase -to describe nodes, edges, demands, and workflow interfaces. It centralizes -typing to improve readability and static analysis and contains no runtime logic. +This package defines the public `Cost` and `EdgeDir` aliases, the `EdgeSelect`, +`FlowPlacement`, and `Mode` enums, and the edge-reference DTOs `EdgeRef` and +`MaxFlowResult`. Apart from enum parsing helpers it holds no runtime logic. """ -from ngraph.types.base import MIN_CAP, MIN_FLOW, Cost, EdgeSelect, FlowPlacement, Mode +from ngraph.types.base import Cost, EdgeSelect, FlowPlacement, Mode from ngraph.types.dto import EdgeDir, EdgeRef, MaxFlowResult __all__ = [ @@ -15,8 +15,6 @@ "EdgeSelect", # Type aliases and constants "Cost", - "MIN_CAP", - "MIN_FLOW", "EdgeDir", # DTOs "EdgeRef", diff --git a/ngraph/types/base.py b/ngraph/types/base.py index 0b80575..80be926 100644 --- a/ngraph/types/base.py +++ b/ngraph/types/base.py @@ -8,12 +8,6 @@ #: Represents numeric cost in the network (e.g. distance, latency, etc.). Cost = Union[int, float] -#: Capacity threshold below which capacity values are treated as effectively zero. -MIN_CAP = 2**-12 - -#: Flow threshold below which flow values are treated as effectively zero. -MIN_FLOW = 2**-12 - class EdgeSelect(IntEnum): """Edge selection criteria for shortest-path algorithms. @@ -69,3 +63,24 @@ class Mode(IntEnum): #: Analyze each (source_group, sink_group) pair independently. #: Returns flow values for each pair separately. PAIRWISE = 2 + + @classmethod + def from_string(cls, value: str) -> "Mode": + """Parse a string into a Mode enum value. + + Args: + value: Case-insensitive string name (e.g., "combine", "PAIRWISE"). + + Returns: + The corresponding Mode enum member. + + Raises: + ValueError: If the string doesn't match any enum member. + """ + try: + return cls[value.upper()] + except KeyError: + valid = ", ".join(e.name.lower() for e in cls) + raise ValueError( + f"Invalid mode '{value}'. Valid values are: {valid}" + ) from None diff --git a/ngraph/types/dto.py b/ngraph/types/dto.py index fbf6e16..25f9f49 100644 --- a/ngraph/types/dto.py +++ b/ngraph/types/dto.py @@ -18,8 +18,8 @@ class EdgeRef: """Reference to a directed edge via scenario link_id and direction. - Provides stable, scenario-native edge identification across Core reorderings - using the link's unique ID rather than node name tuples. + Identifying an edge by the link's unique ID rather than by a node-name + tuple keeps the reference valid across Core edge reorderings. Attributes: link_id: Scenario link identifier (matches Network.links keys) @@ -39,7 +39,7 @@ class MaxFlowResult: Attributes: total_flow: Maximum flow value achieved. cost_distribution: Mapping of path cost to flow volume placed at that cost. - min_cut: Saturated edges forming the min-cut (None if not computed). + min_cut: Edges forming a minimum cut (None if not computed). """ total_flow: float diff --git a/ngraph/utils/ids.py b/ngraph/utils/ids.py index efc2ffb..e0e0783 100644 --- a/ngraph/utils/ids.py +++ b/ngraph/utils/ids.py @@ -7,12 +7,11 @@ def new_base64_uuid() -> str: """Return a 22-character URL-safe Base64-encoded UUID without padding. - The function generates a random version 4 UUID, encodes the 16 raw bytes - using URL-safe Base64, removes the two trailing padding characters, and - decodes to ASCII. The resulting string length is 22 characters. + The 16 raw bytes of a random version 4 UUID are encoded with URL-safe + Base64; the two trailing padding characters are dropped, leaving 22 ASCII + characters. Returns: - A 22-character URL-safe Base64 representation of a UUID4 without - padding. + A 22-character URL-safe Base64 representation of a UUID4, unpadded. """ return base64.urlsafe_b64encode(uuid.uuid4().bytes)[:-2].decode("ascii") diff --git a/ngraph/utils/output_paths.py b/ngraph/utils/output_paths.py index d3018d4..77856fc 100644 --- a/ngraph/utils/output_paths.py +++ b/ngraph/utils/output_paths.py @@ -1,9 +1,8 @@ """Utilities for building CLI artifact output paths. -This module centralizes logic for composing file and directory paths for -artifacts produced by the NetGraph CLI. Paths are built from an optional -output directory, a prefix (usually derived from the scenario file or -results file), and a per-artifact suffix. +Every artifact path the NetGraph CLI writes is composed here, from an optional +output directory, a prefix (usually derived from the scenario file or results +file), and a per-artifact suffix. """ from __future__ import annotations diff --git a/ngraph/utils/seed_manager.py b/ngraph/utils/seed_manager.py index 25d1bbc..2285310 100644 --- a/ngraph/utils/seed_manager.py +++ b/ngraph/utils/seed_manager.py @@ -30,8 +30,8 @@ def __init__(self, master_seed: Optional[int] = None) -> None: def derive_seed(self, *components: Any) -> Optional[int]: """Derive a deterministic seed from master seed and component identifiers. - Uses a hash-based approach to generate consistent seeds for different - components while ensuring good distribution of seed values. + The master seed and the component identifiers are joined and hashed + with SHA-256, so distinct components get unrelated seeds. Args: *components: Component identifiers (strings, integers, etc.) that diff --git a/ngraph/utils/yaml_utils.py b/ngraph/utils/yaml_utils.py index 4afea81..08c7429 100644 --- a/ngraph/utils/yaml_utils.py +++ b/ngraph/utils/yaml_utils.py @@ -2,16 +2,14 @@ from typing import Any, Dict, TypeVar -K = TypeVar("K") V = TypeVar("V") def normalize_yaml_dict_keys(data: Dict[Any, V]) -> Dict[str, V]: """Normalize dictionary keys from YAML parsing to ensure consistent string keys. - YAML 1.1 boolean keys (e.g., true, false, yes, no, on, off) get converted to - Python True/False boolean values. This function converts them to predictable - string representations ("True"/"False") and ensures all keys are strings. + YAML 1.1 parses true/false/yes/no/on/off keys as Python booleans. Those + become "True"/"False"; every other key is coerced with str(). Args: data: Dictionary that may contain boolean or other non-string keys from YAML parsing @@ -28,11 +26,10 @@ def normalize_yaml_dict_keys(data: Dict[Any, V]) -> Dict[str, V]: """ normalized = {} for key, value in data.items(): - # Handle YAML parsing quirks: YAML 1.1 boolean keys (e.g., true, false, - # yes, no, on, off) get converted to Python True/False. Convert them to - # predictable string representations. + # YAML 1.1 turns true/false/yes/no/on/off keys into Python bools; + # normalize those to "True"/"False". if isinstance(key, bool): - key = str(key) # Convert True/False to "True"/"False" - key = str(key) # Ensure all keys are strings + key = str(key) + key = str(key) normalized[key] = value return normalized diff --git a/ngraph/workflow/base.py b/ngraph/workflow/base.py index f687ab4..7ccc162 100644 --- a/ngraph/workflow/base.py +++ b/ngraph/workflow/base.py @@ -12,7 +12,7 @@ import time from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Type, Union from ngraph.logging import get_logger @@ -43,27 +43,88 @@ def decorator(cls: Type["WorkflowStep"]) -> Type["WorkflowStep"]: return decorator +def validate_unique_step_names(workflow: "list[WorkflowStep]") -> None: + """Validate that effective step names in a workflow list are unique. + + Effective names follow the same rule used for results storage: + ``step.name`` or the step class name when no name is set. Duplicate + effective names would silently overwrite each other's namespace in the + results store. + + Args: + workflow: Workflow step list. + + Raises: + ValueError: If two or more steps share the same effective name. + """ + counts: Dict[str, int] = {} + for step in workflow: + effective = step.name or type(step).__name__ + counts[effective] = counts.get(effective, 0) + 1 + duplicates = sorted(name for name, count in counts.items() if count > 1) + if duplicates: + dup_list = ", ".join(f"'{name}'" for name in duplicates) + raise ValueError( + f"Duplicate workflow step name(s): {dup_list}. Results are stored " + "per step name, so duplicates would overwrite each other; set a " + "unique 'name' on each step." + ) + + def resolve_parallelism(parallelism: Union[int, str]) -> int: - """Resolve parallelism setting to a concrete worker count. + """Validate and resolve a parallelism setting to a concrete worker count. Args: - parallelism: Either an integer worker count or "auto" for CPU count. + parallelism: Either a positive integer worker count or "auto" for + the CPU count. Returns: Positive integer worker count (minimum 1). + + Raises: + ValueError: If parallelism is a string other than "auto", or an + integer < 1. """ if isinstance(parallelism, str): + if parallelism != "auto": + raise ValueError("parallelism must be an integer or 'auto'") return max(1, int(os.cpu_count() or 1)) - return max(1, int(parallelism)) + if int(parallelism) < 1: + raise ValueError("parallelism must be >= 1") + return int(parallelism) + + +def serialize_monte_carlo_results(raw: Dict[str, Any]) -> tuple[Any, list[dict]]: + """Convert FailureManager Monte Carlo output into JSON-safe dicts. + + Args: + raw: Dict with optional "baseline" entry and "results" list, whose + items expose to_dict() (e.g. FlowIterationResult) or are already + plain dicts. + + Returns: + Tuple of (baseline_dict, flow_results): the baseline iteration (or + None) and the failure iterations, converted via to_dict() when + available. + """ + + def _to_dict(item: Any) -> Any: + to_dict = getattr(item, "to_dict", None) + return to_dict() if callable(to_dict) else item + + baseline = raw.get("baseline") + baseline_dict = _to_dict(baseline) if baseline is not None else None + flow_results = [_to_dict(item) for item in raw.get("results", [])] + return baseline_dict, flow_results @dataclass class WorkflowStep(ABC): """Base class for all workflow steps. - All workflow steps are automatically logged with execution timing information. - All workflow steps support seeding for reproducible random operations. - Workflow metadata is automatically stored in scenario.results for analysis. + Every step is logged with execution timing, supports seeding for + reproducible random operations, and has its metadata stored in + scenario.results for analysis. YAML Configuration: ```yaml @@ -76,21 +137,22 @@ class WorkflowStep(ABC): Attributes: name: Optional custom identifier for this workflow step instance, - used for logging and result storage purposes. + used for logging and result storage. When empty, the class name + is used instead. seed: Optional seed for reproducible random operations. If None, random operations will be non-deterministic. """ name: str = "" seed: Optional[int] = None - # Internal: provenance of the step seed ("explicit-step" or "scenario-derived" or "none"). + # Internal: seed provenance, one of "explicit-step", "scenario-derived", "none". _seed_source: str = "" def execute(self, scenario: "Scenario") -> None: """Execute the workflow step with logging and metadata storage. - This method wraps the abstract run() method with timing, logging, and - automatic metadata storage for the analysis registry system. + Wraps `run()` with timing, logging, and metadata storage for the + analysis registry system. Args: scenario: The scenario to execute the step on. @@ -105,25 +167,20 @@ def execute(self, scenario: "Scenario") -> None: step_type = self.__class__.__name__ # Guarantee a stable results namespace even when name is not provided step_name = self.name or step_type - display_name = step_name - # Determine seed provenance and effective seed for this step - scenario_seed = getattr(scenario, "seed", None) + # Determine seed provenance from the seed the step actually uses. + # run() only ever consults self.seed; a scenario-level seed without a + # concrete step seed means the step runs unseeded. + scenario_seed = scenario.seed step_seed = self.seed - explicit_source = getattr(self, "_seed_source", None) - if step_seed is not None and explicit_source == "explicit-step": - seed_source = "explicit-step" - active_seed = step_seed - elif step_seed is not None and explicit_source == "scenario-derived": - # Step received a derived seed at construction time - seed_source = "scenario-derived" + if step_seed is not None: + explicit_source = getattr(self, "_seed_source", None) + seed_source = ( + explicit_source + if explicit_source in ("explicit-step", "scenario-derived") + else "explicit-step" + ) active_seed = step_seed - elif scenario_seed is not None: - seed_source = "scenario-derived" - # Scenario.from_yaml derives per-step seeds when seed is provided; if a - # concrete seed was not set on the step (self.seed is None), treat the - # scenario seed as the active base (workers may derive offsets internally). - active_seed = scenario_seed else: seed_source = "none" active_seed = None @@ -151,7 +208,7 @@ def execute(self, scenario: "Scenario") -> None: step_type, str(self.seed), ) - logger.info(f"Starting workflow step: {display_name} ({step_type})") + logger.info(f"Starting workflow step: {step_name} ({step_type})") start_time = time.time() try: @@ -166,20 +223,19 @@ def execute(self, scenario: "Scenario") -> None: updated_md["duration_sec"] = float(duration) scenario.results.put("metadata", updated_md) logger.info( - f"Completed workflow step: {display_name} ({step_type}) " + f"Completed workflow step: {step_name} ({step_type}) " f"in {duration:.3f} seconds" ) try: - store = getattr(scenario.results, "_store", {}) - keys = ", ".join(sorted(list(store.get(step_name, {}).keys()))) + keys = ", ".join(sorted(scenario.results.get_step(step_name).keys())) except Exception as exc: logger.debug( - "Failed to read results keys for step %s: %s", display_name, exc + "Failed to read results keys for step %s: %s", step_name, exc ) keys = "-" logger.debug( "Step %s finished: duration=%.3fs, results_keys=%s", - display_name, + step_name, duration, keys or "-", ) @@ -187,7 +243,7 @@ def execute(self, scenario: "Scenario") -> None: end_time = time.time() duration = end_time - start_time logger.error( - f"Failed workflow step: {display_name} ({step_type}) " + f"Failed workflow step: {step_name} ({step_type}) " f"after {duration:.3f} seconds - {type(e).__name__}: {e}" ) raise @@ -197,15 +253,15 @@ def execute(self, scenario: "Scenario") -> None: scenario.results.exit_step() except Exception as exc: logger.warning( - "Failed to exit step scope cleanly for %s: %s", display_name, exc + "Failed to exit step scope cleanly for %s: %s", step_name, exc ) @abstractmethod def run(self, scenario: "Scenario") -> None: """Execute the workflow step logic. - This method should be implemented by concrete workflow step classes. - It is called by execute() which handles logging, timing, and metadata storage. + Called by `execute()`, which handles logging, timing, and metadata + storage. Args: scenario: The scenario to execute the step on. diff --git a/ngraph/workflow/build_graph.py b/ngraph/workflow/build_graph.py index 02fe66d..72471e7 100644 --- a/ngraph/workflow/build_graph.py +++ b/ngraph/workflow/build_graph.py @@ -1,9 +1,8 @@ """Graph building workflow component. -Validates and exports network topology as a node-link representation using NetworkX. -Actual graph building for analysis happens in analysis functions; this step -primarily validates the network and stores a serializable representation for -inspection. +Validates the network topology and exports it as a NetworkX node-link +representation for inspection. Graph building for analysis happens in the +analysis functions, not here. YAML Configuration Example: ```yaml @@ -13,9 +12,9 @@ add_reverse: true # Optional: Add reverse edges (default: true) ``` -The `add_reverse` parameter controls whether reverse edges are added for each link. -When `True` (default), each Link(A→B) gets both forward(A→B) and reverse(B→A) edges -for bidirectional connectivity. Set to `False` for directed-only graphs. +With `add_reverse: true` (the default), each Link(A→B) gets both a forward +(A→B) and a reverse (B→A) edge for bidirectional connectivity. Set it to +`false` for directed-only graphs. Results stored in `scenario.results` under the step name as two keys: - metadata: Step-level execution metadata (node/link counts) @@ -42,9 +41,8 @@ class BuildGraph(WorkflowStep): """Validates network topology and stores node-link representation. - This step validates the network structure and stores a JSON-serializable - node-link representation using NetworkX. Core graph building happens in - analysis functions as needed. + The stored representation is JSON-serializable NetworkX node-link data. + Core graph building for analysis happens in analysis functions as needed. Attributes: add_reverse: If True, adds reverse edges for bidirectional connectivity. @@ -68,27 +66,27 @@ def run(self, scenario: Scenario) -> None: # Build NetworkX MultiDiGraph from Network graph = nx.MultiDiGraph() - # Add nodes with attributes + # Add nodes with attributes. Reserved keys win over user attrs to + # avoid kwarg collisions when attrs contain e.g. "disabled". for node_name in sorted(network.nodes.keys()): node = network.nodes[node_name] - graph.add_node( - node_name, - disabled=node.disabled, - **node.attrs, - ) + graph.add_node(node_name, **{**node.attrs, "disabled": node.disabled}) - # Add edges (links) with attributes + # Add edges (links) with attributes. Reserved keys (id, capacity, + # cost, disabled) win over user attrs with the same names. for link_id in sorted(network.links.keys()): link = network.links[link_id] # Add forward edge graph.add_edge( link.source, link.target, - id=link_id, - capacity=float(link.capacity), - cost=float(link.cost), - disabled=link.disabled, - **link.attrs, + **{ + **link.attrs, + "id": link_id, + "capacity": float(link.capacity), + "cost": float(link.cost), + "disabled": link.disabled, + }, ) # Add reverse edge if configured (for bidirectional connectivity) if self.add_reverse: @@ -96,11 +94,13 @@ def run(self, scenario: Scenario) -> None: graph.add_edge( link.target, link.source, - id=reverse_id, - capacity=float(link.capacity), - cost=float(link.cost), - disabled=link.disabled, - **link.attrs, + **{ + **link.attrs, + "id": reverse_id, + "capacity": float(link.capacity), + "cost": float(link.cost), + "disabled": link.disabled, + }, ) # Convert to node-link format for serialization diff --git a/ngraph/workflow/cost_power.py b/ngraph/workflow/cost_power.py index 6b7e85e..ce66597 100644 --- a/ngraph/workflow/cost_power.py +++ b/ngraph/workflow/cost_power.py @@ -1,8 +1,7 @@ """CostPower workflow step: collect capex and power by hierarchy level. -This step aggregates capex and power from the network hardware inventory without -performing any normalization or reporting. It separates contributions into two -categories: +Aggregates capex and power from the network hardware inventory, with no +normalization or reporting. Contributions are split into two categories: - platform_*: node hardware (e.g., chassis, linecards) resolved from node attrs - optics_*: per-end link hardware (e.g., optics) resolved from link attrs @@ -51,7 +50,6 @@ from dataclasses import dataclass from typing import Any, Dict, List -from ngraph.explorer import NetworkExplorer from ngraph.logging import get_logger from ngraph.model.components import ( ComponentsLibrary, @@ -70,7 +68,8 @@ class CostPower(WorkflowStep): Attributes: include_disabled: If True, include disabled nodes and links. - aggregation_level: Inclusive depth for aggregation. 0=root only. + aggregation_level: Inclusive depth for aggregation; 0 = root only. + Must be >= 0. """ include_disabled: bool = False @@ -101,9 +100,6 @@ def run(self, scenario: Any) -> None: network = scenario.network library: ComponentsLibrary = scenario.components_library - explorer = NetworkExplorer.explore_network(network, components_library=library) - - # Helper: enabled checks def node_enabled(nd: Any) -> bool: return not bool(nd.disabled) @@ -150,11 +146,7 @@ def add_values( if comp is None: continue capex, power, _ = totals_with_multiplier(comp, count) - tree_node = explorer._node_map.get(nd.name) - if tree_node is None: - continue - full_path = explorer._compute_full_path(tree_node) - add_values(full_path, float(capex), float(power), 0.0, 0.0) + add_values(nd.name, float(capex), float(power), 0.0, 0.0) # --- Optics aggregation (per-end link hardware) --- for lk in network.links.values(): @@ -175,19 +167,13 @@ def add_values( src_comp, src_cnt, _src_excl = src_end if src_comp is not None and node_has_hw.get(lk.source, False): capex, power, _ = totals_with_multiplier(src_comp, src_cnt) - src_tree = explorer._node_map.get(lk.source) - if src_tree is not None: - src_path = explorer._compute_full_path(src_tree) - add_values(src_path, 0.0, 0.0, float(capex), float(power)) + add_values(lk.source, 0.0, 0.0, float(capex), float(power)) # Destination endpoint dst_comp, dst_cnt, _dst_excl = dst_end if dst_comp is not None and node_has_hw.get(lk.target, False): capex, power, _ = totals_with_multiplier(dst_comp, dst_cnt) - dst_tree = explorer._node_map.get(lk.target) - if dst_tree is not None: - dst_path = explorer._compute_full_path(dst_tree) - add_values(dst_path, 0.0, 0.0, float(capex), float(power)) + add_values(lk.target, 0.0, 0.0, float(capex), float(power)) # Build payload levels_payload: Dict[int, List[Dict[str, Any]]] = {} diff --git a/ngraph/workflow/max_flow_step.py b/ngraph/workflow/max_flow_step.py index 2d4e8b6..db86866 100644 --- a/ngraph/workflow/max_flow_step.py +++ b/ngraph/workflow/max_flow_step.py @@ -3,8 +3,8 @@ Monte Carlo analysis of maximum flow capacity between node groups using FailureManager. Produces unified `flow_results` per iteration under `data.flow_results`. -Baseline (no failures) is always run first as a separate reference. The `iterations` -parameter specifies how many failure scenarios to run. +Baseline (no failures) always runs first as a separate reference; `iterations` +counts failure scenarios only. YAML Configuration Example: @@ -34,12 +34,12 @@ from ngraph.analysis.failure_manager import FailureManager from ngraph.logging import get_logger -from ngraph.results.flow import FlowIterationResult -from ngraph.types.base import FlowPlacement +from ngraph.types.base import FlowPlacement, Mode from ngraph.workflow.base import ( WorkflowStep, register_workflow_step, resolve_parallelism, + serialize_monte_carlo_results, ) if TYPE_CHECKING: @@ -52,24 +52,26 @@ class MaxFlow(WorkflowStep): """Maximum flow Monte Carlo workflow step. - Baseline (no failures) is always run first as a separate reference. Results are - returned with baseline in a separate field. The flow_results list contains unique - failure patterns (deduplicated); each result has occurrence_count indicating how - many iterations matched that pattern. + Baseline (no failures) always runs first and is returned in a separate field. + The flow_results list holds unique failure patterns (deduplicated); each result + carries an occurrence_count of how many iterations matched that pattern. Attributes: source: Source node selector (string path or selector dict). target: Target node selector (string path or selector dict). mode: Flow analysis mode ("combine" or "pairwise"). failure_policy: Name of failure policy in scenario.failure_policy_set. - iterations: Number of failure iterations to run. - parallelism: Number of parallel worker processes. - shortest_path: Whether to use shortest paths only. + If None, no failure policy is applied. + iterations: Number of failure iterations to run; must be >= 0. + parallelism: Worker thread count, or "auto" for the CPU count. + shortest_path: Restrict flow to lowest-cost paths (IP/IGP mode). require_capacity: If True (default), path selection considers capacity. If False, path selection is cost-only (true IP/IGP semantics). flow_placement: Flow placement strategy. seed: Optional seed for reproducible results. - store_failure_patterns: Whether to store failure patterns in results. + store_failure_patterns: Record the failure trace on each result. + Iterations are deduplicated, so a trace describes the first + iteration of its pattern, not every matching iteration. include_flow_details: Whether to collect cost distribution per flow. include_min_cut: Whether to include min-cut edges per flow. """ @@ -91,14 +93,8 @@ class MaxFlow(WorkflowStep): def __post_init__(self) -> None: if self.iterations < 0: raise ValueError("iterations must be >= 0") - if isinstance(self.parallelism, str): - if self.parallelism != "auto": - raise ValueError("parallelism must be an integer or 'auto'") - else: - if self.parallelism < 1: - raise ValueError("parallelism must be >= 1") - if self.mode not in {"combine", "pairwise"}: - raise ValueError("mode must be 'combine' or 'pairwise'") + resolve_parallelism(self.parallelism) # validate at construction + Mode.from_string(self.mode) # validate; raises ValueError on bad values if isinstance(self.flow_placement, str): self.flow_placement = FlowPlacement.from_string(self.flow_placement) @@ -118,6 +114,9 @@ def run(self, scenario: "Scenario") -> None: self.include_min_cut, ) + # __post_init__ converts string flow_placement values to the enum + assert isinstance(self.flow_placement, FlowPlacement) + fm = FailureManager( network=scenario.network, failure_policy_set=scenario.failure_policy_set, @@ -141,24 +140,7 @@ def run(self, scenario: "Scenario") -> None: scenario.results.put("metadata", raw.get("metadata", {})) - # Handle baseline (separate from failure results) - baseline_result = raw.get("baseline") - baseline_dict = None - if baseline_result is not None: - if hasattr(baseline_result, "to_dict"): - baseline_dict = baseline_result.to_dict() - else: - baseline_dict = baseline_result - - # Handle failure results - flow_results: list[dict] = [] - for item in raw.get("results", []): - if isinstance(item, FlowIterationResult): - flow_results.append(item.to_dict()) - elif hasattr(item, "to_dict") and callable(item.to_dict): - flow_results.append(item.to_dict()) # type: ignore[union-attr] - else: - flow_results.append(item) + baseline_dict, flow_results = serialize_monte_carlo_results(raw) context = { "source": self.source, @@ -166,9 +148,7 @@ def run(self, scenario: "Scenario") -> None: "mode": self.mode, "shortest_path": bool(self.shortest_path), "require_capacity": bool(self.require_capacity), - "flow_placement": getattr( - self.flow_placement, "name", str(self.flow_placement) - ), + "flow_placement": self.flow_placement.name, "include_flow_details": bool(self.include_flow_details), "include_min_cut": bool(self.include_min_cut), } diff --git a/ngraph/workflow/maximum_supported_demand_step.py b/ngraph/workflow/maximum_supported_demand_step.py index b731f2f..0753654 100644 --- a/ngraph/workflow/maximum_supported_demand_step.py +++ b/ngraph/workflow/maximum_supported_demand_step.py @@ -28,16 +28,17 @@ import time from dataclasses import dataclass +from dataclasses import field as dataclasses_field from typing import TYPE_CHECKING, Any import netgraph_core import numpy as np -from ngraph.analysis.demand import ExpandedDemand, expand_demands +from ngraph.analysis.demand import ExpandedDemand +from ngraph.analysis.functions import build_demand_placement_inputs from ngraph.analysis.placement import place_demands from ngraph.logging import get_logger from ngraph.model.demand.spec import TrafficDemand -from ngraph.model.flow.policy_config import FlowPolicyPreset from ngraph.workflow.base import WorkflowStep, register_workflow_step if TYPE_CHECKING: @@ -63,26 +64,31 @@ class _MSDCache: edge_mask: np.ndarray base_expanded: list[ExpandedDemand] resolved_ids: list[tuple[int, int]] + # Persistent base-SPF DAG cache shared by all probes (masks never change + # during MSD, so cached DAGs stay valid across alpha evaluations). + dag_cache: dict = dataclasses_field(default_factory=dict) @dataclass class MaximumSupportedDemand(WorkflowStep): """Finds the maximum uniform traffic multiplier that is fully placeable. - Uses binary search to find alpha_star, the maximum multiplier for all - demands in the set that can still be fully placed on the network. + Binary search yields alpha_star: the largest multiplier at which every + demand in the set still places fully on the network. Attributes: demand_set: Name of the demand set to analyze. - acceptance_rule: Currently only "hard" is implemented. + acceptance_rule: Currently only "hard" is implemented; anything else + raises ValueError at run time. alpha_start: Starting multiplier for binary search. - growth_factor: Factor for bracket expansion. + growth_factor: Factor for bracket expansion; must be > 1.0. alpha_min: Minimum allowed alpha value. alpha_max: Maximum allowed alpha value. - resolution: Convergence threshold for binary search. + resolution: Convergence threshold for binary search; must be positive. max_bracket_iters: Maximum iterations for bracketing phase. max_bisect_iters: Maximum iterations for bisection phase. - placement_rounds: Placement optimization rounds. + placement_rounds: Deprecated; accepted for backward compatibility but + has no effect (placement optimization is handled by the core engine). """ demand_set: str = "default" @@ -97,6 +103,11 @@ class MaximumSupportedDemand(WorkflowStep): placement_rounds: int | str = "auto" def __post_init__(self) -> None: + if self.placement_rounds != "auto": + logger.warning( + "MaximumSupportedDemand 'placement_rounds' is deprecated and has " + "no effect; placement optimization is handled by the core engine." + ) try: self.alpha_start = float(self.alpha_start) self.growth_factor = float(self.growth_factor) @@ -128,23 +139,8 @@ def run(self, scenario: "Any") -> None: ) # Serialize base demands for result output - from ngraph.model.flow.policy_config import serialize_policy_preset - base_tds = scenario.demand_set.get_set(self.demand_set) - base_demands: list[dict[str, Any]] = [ - { - "id": getattr(td, "id", None), - "source": getattr(td, "source", ""), - "target": getattr(td, "target", ""), - "volume": float(getattr(td, "volume", 0.0)), - "mode": getattr(td, "mode", "pairwise"), - "priority": int(getattr(td, "priority", 0)), - "flow_policy": serialize_policy_preset( - getattr(td, "flow_policy", None) - ), - } - for td in base_tds - ] + base_demands: list[dict[str, Any]] = [td.to_dict() for td in base_tds] if not base_demands: raise ValueError( @@ -153,7 +149,7 @@ def run(self, scenario: "Any") -> None: ) # Build cache once for all probes - cache = self._build_cache(scenario, self.demand_set) + cache = self._build_cache(scenario, base_tds) logger.debug( "MSD cache built: %d expanded demands", len(cache.base_expanded), @@ -180,7 +176,6 @@ def probe(alpha: float) -> tuple[bool, dict[str, Any]]: "max_bracket_iters": self.max_bracket_iters, "max_bisect_iters": self.max_bisect_iters, "demand_set": self.demand_set, - "placement_rounds": self.placement_rounds, } scenario.results.put("metadata", {}) scenario.results.put( @@ -246,7 +241,15 @@ def _binary_search(self, probe: "Any") -> float: break upper = alpha if lower is None: - raise ValueError("No feasible alpha found above alpha_min") + # Mirror the upward branch: bracket iterations can run out + # before the halving sequence reaches alpha_min (e.g. a large + # alpha_start), so probe alpha_min directly before giving up. + if upper <= self.alpha_min: + raise ValueError("No feasible alpha found above alpha_min") + feas, _ = probe(self.alpha_min) + if not feas: + raise ValueError("No feasible alpha found above alpha_min") + lower = self.alpha_min assert lower is not None and upper is not None and lower < upper @@ -264,53 +267,23 @@ def _binary_search(self, probe: "Any") -> float: return left @staticmethod - def _build_cache(scenario: Any, demand_set_name: str) -> _MSDCache: + def _build_cache(scenario: Any, base_tds: list[TrafficDemand]) -> _MSDCache: """Build cache for MSD binary search. - Creates stable TrafficDemand objects, expands them once, and builds - AnalysisContext. Called once at search start. + Reuses build_demand_placement_inputs for the expand-once context and + resolved node IDs, then adds the no-exclusion masks shared by all + probes. TrafficDemand ids are stable, so pseudo-node names (which + embed them) stay consistent across probes. Called once at search + start. """ - from ngraph.analysis import AnalysisContext - - base_tds = scenario.demand_set.get_set(demand_set_name) - - # Create stable TrafficDemand objects (same IDs for all probes) - stable_demands: list[TrafficDemand] = [ - TrafficDemand( - id=getattr(td, "id", "") or "", - source=getattr(td, "source", ""), - target=getattr(td, "target", ""), - priority=int(getattr(td, "priority", 0)), - volume=float(getattr(td, "volume", 0.0)), - flow_policy=getattr(td, "flow_policy", None), - mode=str(getattr(td, "mode", "pairwise")), - group_mode=str(getattr(td, "group_mode", "flatten")), - ) - for td in base_tds - ] - - # Expand once (augmentations depend on td.id, now stable) - expansion = expand_demands( + ctx, expansion, resolved_ids = build_demand_placement_inputs( scenario.network, - stable_demands, - default_policy_preset=FlowPolicyPreset.SHORTEST_PATHS_ECMP, - ) - - # Build AnalysisContext once - ctx = AnalysisContext.from_network( - scenario.network, - augmentations=expansion.augmentations, + [{**td.to_dict(), "flow_policy": td.flow_policy} for td in base_tds], ) # Build masks once (no exclusions during MSD) - node_mask = ctx._build_node_mask(excluded_nodes=None) - edge_mask = ctx._build_edge_mask(excluded_links=None) - - # Pre-resolve node IDs once - resolved_ids = [ - (ctx.node_mapper.to_id(d.src_name), ctx.node_mapper.to_id(d.dst_name)) - for d in expansion.demands - ] + node_mask = ctx.build_node_mask(excluded_nodes=None) + edge_mask = ctx.build_edge_mask(excluded_links=None) return _MSDCache( ctx=ctx, @@ -343,6 +316,7 @@ def _evaluate_alpha( cache.edge_mask, resolved_ids=cache.resolved_ids, collect_entries=False, + dag_cache=cache.dag_cache, ) if result.summary.total_demand == 0.0: @@ -355,28 +329,5 @@ def _evaluate_alpha( "placement_ratio": result.summary.ratio, } - @staticmethod - def _build_scaled_demands( - base_demands: list[dict[str, Any]], alpha: float - ) -> list[TrafficDemand]: - """Build scaled TrafficDemand objects from serialized demands. - - Utility for tests to verify results at specific alpha values. - Preserves ID if present for stable context caching. - """ - return [ - TrafficDemand( - id=d.get("id") or "", - source=d["source"], - target=d["target"], - priority=int(d["priority"]), - volume=float(d["volume"]) * alpha, - flow_policy=d.get("flow_policy"), - mode=str(d.get("mode", "pairwise")), - group_mode=str(d.get("group_mode", "flatten")), - ) - for d in base_demands - ] - register_workflow_step("MaximumSupportedDemand")(MaximumSupportedDemand) diff --git a/ngraph/workflow/network_stats.py b/ngraph/workflow/network_stats.py index 058bea9..f289cdf 100644 --- a/ngraph/workflow/network_stats.py +++ b/ngraph/workflow/network_stats.py @@ -1,8 +1,9 @@ """Workflow step for basic node and link statistics. Computes and stores network statistics including node/link counts, -capacity distributions, cost distributions, and degree distributions. Supports -optional exclusion simulation and disabled entity handling. +capacity distributions, cost distributions, and degree distributions. Excluded +entities are filtered out without modifying the base network; disabled nodes +and links are excluded too unless `include_disabled` is set. YAML Configuration Example: ```yaml @@ -25,7 +26,7 @@ from dataclasses import dataclass from statistics import mean, median -from typing import TYPE_CHECKING, Dict, Iterable, List +from typing import TYPE_CHECKING, Dict, Iterable from ngraph.logging import get_logger from ngraph.workflow.base import WorkflowStep, register_workflow_step @@ -67,7 +68,7 @@ def run(self, scenario: Scenario) -> None: """ logger.info("Starting NetworkStats: name=%s", self.name) - # Convert exclusion iterables to sets for efficient lookup + # Sets, so the per-node/per-link membership tests below stay O(1) excluded_nodes_set = set(self.excluded_nodes) if self.excluded_nodes else set() excluded_links_set = set(self.excluded_links) if self.excluded_links else set() @@ -104,10 +105,7 @@ def run(self, scenario: Scenario) -> None: and link.target in nodes } - # Compute node statistics node_count = len(nodes) - - # Compute link statistics link_count = len(links) total_capacity_val = mean_capacity_val = median_capacity_val = 0.0 @@ -128,8 +126,7 @@ def run(self, scenario: Scenario) -> None: min_cost_val = min(costs) max_cost_val = max(costs) - # Compute degree statistics (only for enabled nodes) - degree_values: List[int] = [] + # Compute degree statistics over the selected node set mean_degree_val = median_degree_val = min_degree_val = max_degree_val = 0.0 if nodes: degrees: Dict[str, int] = {name: 0 for name in nodes} @@ -148,32 +145,24 @@ def run(self, scenario: Scenario) -> None: # Store results scenario.results.put("metadata", {}) - # Ensure locals exist even when sets are empty - if not links: - total_capacity_val = mean_capacity_val = median_capacity_val = 0.0 - min_capacity_val = max_capacity_val = 0.0 - mean_cost_val = median_cost_val = min_cost_val = max_cost_val = 0.0 - if not nodes: - mean_degree_val = median_degree_val = min_degree_val = max_degree_val = 0.0 - scenario.results.put( "data", { "node_count": int(node_count), "link_count": int(link_count), - "total_capacity": float(total_capacity_val) if links else 0.0, - "mean_capacity": float(mean_capacity_val) if links else 0.0, - "median_capacity": float(median_capacity_val) if links else 0.0, - "min_capacity": float(min_capacity_val) if links else 0.0, - "max_capacity": float(max_capacity_val) if links else 0.0, - "mean_cost": float(mean_cost_val) if links else 0.0, - "median_cost": float(median_cost_val) if links else 0.0, - "min_cost": float(min_cost_val) if links else 0.0, - "max_cost": float(max_cost_val) if links else 0.0, - "mean_degree": float(mean_degree_val) if nodes else 0.0, - "median_degree": float(median_degree_val) if nodes else 0.0, - "min_degree": float(min_degree_val) if nodes else 0.0, - "max_degree": float(max_degree_val) if nodes else 0.0, + "total_capacity": float(total_capacity_val), + "mean_capacity": float(mean_capacity_val), + "median_capacity": float(median_capacity_val), + "min_capacity": float(min_capacity_val), + "max_capacity": float(max_capacity_val), + "mean_cost": float(mean_cost_val), + "median_cost": float(median_cost_val), + "min_cost": float(min_cost_val), + "max_cost": float(max_cost_val), + "mean_degree": float(mean_degree_val), + "median_degree": float(median_degree_val), + "min_degree": float(min_degree_val), + "max_degree": float(max_degree_val), }, ) @@ -182,7 +171,7 @@ def run(self, scenario: Scenario) -> None: self.name, node_count, link_count, - float(total_capacity_val) if links else 0.0, + float(total_capacity_val), ) diff --git a/ngraph/workflow/traffic_matrix_placement_step.py b/ngraph/workflow/traffic_matrix_placement_step.py index 9d7f4e1..3d3871a 100644 --- a/ngraph/workflow/traffic_matrix_placement_step.py +++ b/ngraph/workflow/traffic_matrix_placement_step.py @@ -3,8 +3,8 @@ Runs Monte Carlo demand placement using a named demand set and produces unified `flow_results` per iteration under `data.flow_results`. -Baseline (no failures) is always run first as a separate reference. The `iterations` -parameter specifies how many failure scenarios to run. +Baseline (no failures) always runs first as a separate reference; `iterations` +counts failure scenarios only. YAML Configuration Example: ```yaml @@ -14,7 +14,7 @@ demand_set: "default" failure_policy: "single_link" # Optional: failure policy name iterations: 100 # Number of failure scenarios - parallelism: 4 # Worker processes (or "auto") + parallelism: 4 # Worker threads (or "auto") alpha: 1.0 # Demand volume multiplier include_flow_details: true # Include cost distribution per flow ``` @@ -28,11 +28,11 @@ from ngraph.analysis.failure_manager import FailureManager from ngraph.logging import get_logger -from ngraph.results.flow import FlowIterationResult from ngraph.workflow.base import ( WorkflowStep, register_workflow_step, resolve_parallelism, + serialize_monte_carlo_results, ) if TYPE_CHECKING: @@ -45,23 +45,29 @@ class TrafficMatrixPlacement(WorkflowStep): """Monte Carlo demand placement using a named demand set. - Baseline (no failures) is always run first as a separate reference. Results are - returned with baseline in a separate field. The flow_results list contains unique - failure patterns (deduplicated); each result has occurrence_count indicating how - many iterations matched that pattern. + Baseline (no failures) always runs first and is returned in a separate field. + The flow_results list holds unique failure patterns (deduplicated); each result + carries an occurrence_count of how many iterations matched that pattern. Attributes: - demand_set: Name of the demand set to analyze. - failure_policy: Optional failure policy name in scenario.failure_policy_set. - iterations: Number of failure iterations to run. - parallelism: Number of parallel worker processes. - placement_rounds: Placement optimization rounds (int or "auto"). + demand_set: Name of the demand set to analyze. Required; an empty + value raises ValueError. + failure_policy: Failure policy name in scenario.failure_policy_set. + If None, no failure policy is applied. + iterations: Number of failure iterations to run; must be >= 0. + parallelism: Worker thread count, or "auto" for the CPU count. + placement_rounds: Deprecated; accepted for backward compatibility but + has no effect (placement optimization is handled by the core engine). seed: Optional seed for reproducibility. - store_failure_patterns: Whether to store failure pattern results. + store_failure_patterns: Record the failure trace on each result. + Iterations are deduplicated, so a trace describes the first + iteration of its pattern, not every matching iteration. include_flow_details: When True, include cost_distribution per flow. include_used_edges: When True, include set of used edges per demand in entry data. - alpha: Numeric scale for demands in the set. - alpha_from_step: Optional producer step name to read alpha from. + alpha: Numeric scale for demands in the set; must be > 0.0. Ignored + when alpha_from_step is set. + alpha_from_step: Optional producer step name to read alpha from; it + must run before this step. alpha_from_field: Dotted field path in producer step (default: "data.alpha_star"). """ @@ -79,14 +85,14 @@ class TrafficMatrixPlacement(WorkflowStep): alpha_from_field: str = "data.alpha_star" def __post_init__(self) -> None: + if self.placement_rounds != "auto": + logger.warning( + "TrafficMatrixPlacement 'placement_rounds' is deprecated and has " + "no effect; placement optimization is handled by the core engine." + ) if self.iterations < 0: raise ValueError("iterations must be >= 0") - if isinstance(self.parallelism, str): - if self.parallelism != "auto": - raise ValueError("parallelism must be an integer or 'auto'") - else: - if self.parallelism < 1: - raise ValueError("parallelism must be >= 1") + resolve_parallelism(self.parallelism) # validate at construction if not (float(self.alpha) > 0.0): raise ValueError("alpha must be > 0.0") @@ -98,11 +104,10 @@ def run(self, scenario: "Scenario") -> None: logger.info("Starting TrafficMatrixPlacement: name=%s", self.name) logger.debug( "TrafficMatrixPlacement params: demand_set=%s failure_iters=%d " - "parallelism=%s placement_rounds=%s failure_policy=%s alpha=%s", + "parallelism=%s failure_policy=%s alpha=%s", self.demand_set, self.iterations, self.parallelism, - self.placement_rounds, self.failure_policy, self.alpha, ) @@ -115,8 +120,6 @@ def run(self, scenario: "Scenario") -> None: f"Demand set '{self.demand_set}' not found in scenario." ) from exc - from ngraph.model.flow.policy_config import serialize_policy_preset - # Resolve alpha effective_alpha = self._resolve_alpha(scenario) alpha_src = getattr(self, "_alpha_source", None) or "explicit" @@ -126,37 +129,17 @@ def run(self, scenario: "Scenario") -> None: str(alpha_src), ) - # Build demands_config with scaled demands (used for analysis) - # Also build base_demands for output (with serialized policy, unscaled) - demands_config: list[dict[str, Any]] = [] - base_demands: list[dict[str, Any]] = [] - for td in td_list: - demands_config.append( - { - "id": td.id, - "source": td.source, - "target": td.target, - "volume": float(td.volume) * float(effective_alpha), - "mode": getattr(td, "mode", "pairwise"), - "flow_policy": getattr(td, "flow_policy", None), - "priority": getattr(td, "priority", 0), - "group_mode": getattr(td, "group_mode", "flatten"), - } - ) - base_demands.append( - { - "id": td.id, - "source": getattr(td, "source", ""), - "target": getattr(td, "target", ""), - "volume": float(getattr(td, "volume", 0.0)), - "mode": getattr(td, "mode", "pairwise"), - "priority": int(getattr(td, "priority", 0)), - "flow_policy": serialize_policy_preset( - getattr(td, "flow_policy", None) - ), - "group_mode": getattr(td, "group_mode", "flatten"), - } - ) + # base_demands: canonical serialized form for output (unscaled). + # demands_config: analysis wire format (scaled volume, raw preset). + base_demands: list[dict[str, Any]] = [td.to_dict() for td in td_list] + demands_config: list[dict[str, Any]] = [ + { + **td.to_dict(), + "volume": float(td.volume) * float(effective_alpha), + "flow_policy": td.flow_policy, + } + for td in td_list + ] # Run via FailureManager fm = FailureManager( @@ -170,7 +153,6 @@ def run(self, scenario: "Scenario") -> None: demands_config=demands_config, iterations=self.iterations, parallelism=effective_parallelism, - placement_rounds=self.placement_rounds, seed=self.seed, store_failure_patterns=self.store_failure_patterns, include_flow_details=self.include_flow_details, @@ -186,24 +168,7 @@ def run(self, scenario: "Scenario") -> None: # Store outputs scenario.results.put("metadata", raw.get("metadata", {})) - # Handle baseline (separate from failure results) - baseline_result = raw.get("baseline") - baseline_dict = None - if baseline_result is not None: - if hasattr(baseline_result, "to_dict"): - baseline_dict = baseline_result.to_dict() - else: - baseline_dict = baseline_result - - # Handle failure results - flow_results: list[dict] = [] - for item in raw.get("results", []): - if isinstance(item, FlowIterationResult): - flow_results.append(item.to_dict()) - elif hasattr(item, "to_dict") and callable(item.to_dict): - flow_results.append(item.to_dict()) # type: ignore[union-attr] - else: - flow_results.append(item) + baseline_dict, flow_results = serialize_monte_carlo_results(raw) alpha_value = float(effective_alpha) alpha_source_value = getattr(self, "_alpha_source", "explicit") @@ -215,7 +180,6 @@ def run(self, scenario: "Scenario") -> None: "flow_results": flow_results, "context": { "demand_set": self.demand_set, - "placement_rounds": self.placement_rounds, "include_flow_details": self.include_flow_details, "include_used_edges": self.include_used_edges, "base_demands": base_demands, @@ -240,9 +204,11 @@ def run(self, scenario: "Scenario") -> None: def _resolve_alpha(self, scenario: "Scenario") -> float: if self.alpha_from_step: step = scenario.results.get_step(self.alpha_from_step) - if not isinstance(step, dict): + # Results.get_step returns {} for unknown or not-yet-run steps. + if not step: raise ValueError( - f"alpha_from_step='{self.alpha_from_step}' not found or invalid" + f"alpha_from_step '{self.alpha_from_step}' has no results - " + "check the step name and that it runs before this step" ) parts = [p for p in str(self.alpha_from_field).split(".") if p] cursor: Any = step diff --git a/pyproject.toml b/pyproject.toml index 0a7b03a..71faf55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "pyyaml>=6.0", "pandas>=2.0", "jsonschema>=4.0", - "netgraph-core>=0.3.0", + "netgraph-core>=0.7.0", ] [project.urls] diff --git a/scenarios/backbone_clos.yml b/scenarios/backbone_clos.yml index 33280eb..5884c6c 100644 --- a/scenarios/backbone_clos.yml +++ b/scenarios/backbone_clos.yml @@ -1005,42 +1005,45 @@ failures: - scope: link mode: choice count: 3 - conditions: - - attr: link_type - op: == - value: dc_to_pop - logic: and + match: + conditions: + - attr: link_type + op: == + value: dc_to_pop + logic: and weight_by: target_capacity - weight: 0.25 rules: - scope: node mode: choice count: 1 - conditions: - - attr: node_type - op: "!=" - value: dc_region - logic: and + match: + conditions: + - attr: node_type + op: "!=" + value: dc_region + logic: and weight_by: attached_capacity_gbps - weight: 0.1 rules: - scope: link mode: choice count: 4 - conditions: - - attr: link_type - op: == - value: leaf_spine - - attr: link_type - op: == - value: intra_group - - attr: link_type - op: == - value: inter_group - - attr: link_type - op: == - value: internal_mesh - logic: or + match: + conditions: + - attr: link_type + op: == + value: leaf_spine + - attr: link_type + op: == + value: intra_group + - attr: link_type + op: == + value: inter_group + - attr: link_type + op: == + value: internal_mesh + logic: or demands: baseline_traffic_matrix: - source: ^metro1/dc1/.* @@ -1105,7 +1108,6 @@ workflow: resolution: 0.05 max_bracket_iters: 16 max_bisect_iters: 32 - placement_rounds: 2 - type: TrafficMatrixPlacement name: tm_placement seed: 42 @@ -1113,7 +1115,6 @@ workflow: failure_policy: weighted_modes iterations: 1000 parallelism: 7 - placement_rounds: auto store_failure_patterns: false include_flow_details: true include_used_edges: false diff --git a/scenarios/nsfnet.yaml b/scenarios/nsfnet.yaml index 9ccea0d..e30ff5a 100644 --- a/scenarios/nsfnet.yaml +++ b/scenarios/nsfnet.yaml @@ -554,7 +554,6 @@ failures: description: | Approximates 1992 backbone reliability: each physical DS-3 has ~99.9 % monthly availability (p=0.001 failure), and each CNSS or ENSS router has ~99.95 % availability (p=0.0005 failure). expand_groups: false - expand_children: false modes: - weight: 1.0 rules: diff --git a/scenarios/square_mesh.yaml b/scenarios/square_mesh.yaml index 8ea0b40..cace280 100644 --- a/scenarios/square_mesh.yaml +++ b/scenarios/square_mesh.yaml @@ -58,14 +58,12 @@ workflow: resolution: 0.05 max_bracket_iters: 16 max_bisect_iters: 32 - placement_rounds: 2 - type: TrafficMatrixPlacement name: tm_placement demand_set: baseline_traffic_matrix failure_policy: single_link_failure iterations: 1000 parallelism: 8 - placement_rounds: auto seed: 42 store_failure_patterns: true include_flow_details: true diff --git a/tests/analysis/test_context_review_fixes.py b/tests/analysis/test_context_review_fixes.py new file mode 100644 index 0000000..573121b --- /dev/null +++ b/tests/analysis/test_context_review_fixes.py @@ -0,0 +1,364 @@ +"""Regression tests for AnalysisContext review fixes. + +Covers: +- Fractional link costs raise ValueError instead of silent int64 truncation. +- Unbound contexts preserve custom augmentations in flow methods. +- Bound flow calls do not re-run node selection to fill missing pairs. +- Missing pairs are filled with fresh default objects, never one shared alias. +- PAIRWISE pseudo-node attachment edges are emitted once per group member. +- Unbound contexts build the Core graph lazily. +- sensitivity_with_flow computes max flow and sensitivity in one pass. +""" + +from __future__ import annotations + +import pytest + +from ngraph import Link, Mode, Network, Node, analyze +from ngraph.analysis import AugmentationEdge + + +def _two_node_network() -> Network: + """Build a minimal A->B network with unit capacity.""" + net = Network() + net.add_node(Node("A")) + net.add_node(Node("B")) + net.add_link(Link("A", "B", capacity=1.0, cost=1.0)) + return net + + +class TestFractionalCostValidation: + """Fractional link costs must raise instead of truncating to int64.""" + + @staticmethod + def _fractional_cost_network() -> Network: + net = Network() + for name in ["A", "B"]: + net.add_node(Node(name)) + net.add_link(Link("A", "B", capacity=1.0, cost=2.5)) + return net + + def test_bound_context_raises_naming_link(self) -> None: + net = self._fractional_cost_network() + link_id = next(iter(net.links)) + + with pytest.raises(ValueError, match="Non-integer link costs") as excinfo: + analyze(net, source="^A$", sink="^B$") + + assert link_id in str(excinfo.value) + assert "2.5" in str(excinfo.value) + + def test_unbound_context_raises_on_first_graph_use(self) -> None: + net = self._fractional_cost_network() + ctx = analyze(net) # Lazy build: no error yet + + with pytest.raises(ValueError, match="Non-integer link costs"): + ctx.shortest_path_cost("^A$", "^B$") + + def test_unbound_flow_call_raises(self) -> None: + net = self._fractional_cost_network() + + with pytest.raises(ValueError, match="Non-integer link costs"): + analyze(net).max_flow("^A$", "^B$") + + def test_fractional_augmentation_cost_raises(self) -> None: + net = _two_node_network() + + with pytest.raises(ValueError, match="Non-integer link costs") as excinfo: + analyze( + net, + source="^A$", + sink="^B$", + augmentations=[AugmentationEdge("A", "B", 5.0, 0.5)], + ) + + assert "augmentation 'A'->'B'" in str(excinfo.value) + + def test_integral_float_costs_accepted(self) -> None: + net = Network() + for name in ["A", "B"]: + net.add_node(Node(name)) + net.add_link(Link("A", "B", capacity=1.0, cost=2.0)) + + result = analyze(net).shortest_path_cost("^A$", "^B$") + assert result[("^A$", "^B$")] == 2.0 + + +class TestUnboundAugmentationsPreserved: + """Unbound flow methods must use custom augmentations.""" + + def test_unbound_max_flow_uses_augmentations(self) -> None: + net = _two_node_network() + augs = [AugmentationEdge("A", "B", 5.0, 1)] + + ctx = analyze(net, augmentations=augs) + result = ctx.max_flow("^A$", "^B$") + + # 1.0 from the real link + 5.0 from the augmentation + assert result[("^A$", "^B$")] == pytest.approx(6.0) + + def test_unbound_matches_bound_with_same_augmentations(self) -> None: + net = _two_node_network() + augs = [AugmentationEdge("A", "B", 5.0, 1)] + + bound = analyze(net, source="^A$", sink="^B$", augmentations=augs) + unbound = analyze(net, augmentations=augs) + + assert unbound.max_flow("^A$", "^B$") == bound.max_flow() + + def test_unbound_max_flow_detailed_uses_augmentations(self) -> None: + net = _two_node_network() + augs = [AugmentationEdge("A", "B", 5.0, 1)] + + ctx = analyze(net, augmentations=augs) + result = ctx.max_flow_detailed("^A$", "^B$") + + assert result[("^A$", "^B$")].total_flow == pytest.approx(6.0) + + def test_unbound_sensitivity_uses_augmentations(self) -> None: + net = _two_node_network() + augs = [AugmentationEdge("A", "B", 5.0, 1)] + + bound = analyze(net, source="^A$", sink="^B$", augmentations=augs) + unbound = analyze(net, augmentations=augs) + + assert unbound.sensitivity("^A$", "^B$") == bound.sensitivity() + + +class TestFillMissingPairsPrecomputed: + """Bound flow calls fill missing pairs without re-running selection.""" + + @staticmethod + def _pairwise_overlap_network() -> Network: + net = Network() + for name in ["S1", "S2", "T1"]: + net.add_node(Node(name)) + net.add_link(Link("S1", "T1", capacity=5.0, cost=1.0)) + net.add_link(Link("S2", "T1", capacity=3.0, cost=1.0)) + return net + + def test_overlapping_pair_filled_with_default(self) -> None: + net = self._pairwise_overlap_network() + ctx = analyze(net, source=r"^(S\d)$", sink=r"^(S1|T1)$", mode=Mode.PAIRWISE) + + results = ctx.max_flow() + + # All 4 pairs present; (S1, S1) overlaps and is filled with 0.0 + assert set(results) == {("S1", "S1"), ("S1", "T1"), ("S2", "S1"), ("S2", "T1")} + assert results[("S1", "S1")] == 0.0 + assert results[("S1", "T1")] == pytest.approx(5.0) + + def test_bound_flow_calls_do_not_rerun_node_selection( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + net = self._pairwise_overlap_network() + ctx = analyze(net, source=r"^(S\d)$", sink=r"^(S1|T1)$", mode=Mode.PAIRWISE) + + import ngraph.model.selectors as selectors_mod + + def _fail(*args: object, **kwargs: object) -> None: + raise AssertionError("select_nodes must not run on bound flow calls") + + monkeypatch.setattr(selectors_mod, "select_nodes", _fail) + + results = ctx.max_flow() + assert results[("S1", "S1")] == 0.0 + + detailed = ctx.max_flow_detailed() + assert detailed[("S1", "S1")].total_flow == 0.0 + + sens = ctx.sensitivity() + assert sens[("S1", "S1")] == {} + + combined = ctx.sensitivity_with_flow() + assert combined[("S1", "S1")] == (0.0, {}) + + def test_missing_pair_defaults_are_not_aliased(self) -> None: + """Each missing pair gets a fresh default object. + + Regression: one mutable default (dict, MaxFlowResult) was stored + under every missing pair key, so mutating one entry of a public + API result would silently mutate the others. + """ + net = Network() + for name in ["A", "B"]: + net.add_node(Node(name)) + net.add_link(Link("A", "B", capacity=1.0, cost=1.0)) + # Pairs (A, A) and (B, B) overlap and are filled with defaults + ctx = analyze(net, source=r"^(A|B)$", sink=r"^(A|B)$", mode=Mode.PAIRWISE) + + sens = ctx.sensitivity() + assert sens[("A", "A")] == {} and sens[("B", "B")] == {} + assert sens[("A", "A")] is not sens[("B", "B")] + + detailed = ctx.max_flow_detailed() + assert detailed[("A", "A")].total_flow == 0.0 + assert ( + detailed[("A", "A")].cost_distribution + is not detailed[("B", "B")].cost_distribution + ) + + combined = ctx.sensitivity_with_flow() + assert combined[("A", "A")] == (0.0, {}) + assert combined[("A", "A")][1] is not combined[("B", "B")][1] + + +class TestSensitivityWithFlow: + """Combined max-flow + sensitivity matches the separate calls.""" + + @staticmethod + def _pairwise_network() -> Network: + net = Network() + for name in ["S1", "S2", "T1"]: + net.add_node(Node(name)) + net.add_link(Link("S1", "T1", capacity=5.0, cost=1.0)) + net.add_link(Link("S2", "T1", capacity=3.0, cost=1.0)) + return net + + def test_bound_combined_matches_separate_calls(self) -> None: + net = self._pairwise_network() + ctx = analyze(net, source=r"^(S\d)$", sink=r"^(S1|T1)$", mode=Mode.PAIRWISE) + + combined = ctx.sensitivity_with_flow() + flows = ctx.max_flow() + sens = ctx.sensitivity() + + assert set(combined) == set(flows) == set(sens) + for pair_key, (flow_value, sensitivity_map) in combined.items(): + assert flow_value == flows[pair_key] + assert sensitivity_map == sens[pair_key] + + def test_overlapping_pair_filled_with_default(self) -> None: + net = self._pairwise_network() + ctx = analyze(net, source=r"^(S\d)$", sink=r"^(S1|T1)$", mode=Mode.PAIRWISE) + + combined = ctx.sensitivity_with_flow() + + assert combined[("S1", "S1")] == (0.0, {}) + assert combined[("S1", "T1")][0] == pytest.approx(5.0) + + def test_unbound_dispatch_preserves_augmentations(self) -> None: + net = _two_node_network() + augs = [AugmentationEdge("A", "B", 5.0, 1)] + + bound = analyze(net, source="^A$", sink="^B$", augmentations=augs) + unbound = analyze(net, augmentations=augs) + + combined = unbound.sensitivity_with_flow("^A$", "^B$") + assert combined == bound.sensitivity_with_flow() + # 1.0 from the real link + 5.0 from the augmentation + assert combined[("^A$", "^B$")][0] == pytest.approx(6.0) + + def test_selector_argument_validation(self) -> None: + net = _two_node_network() + bound = analyze(net, source="^A$", sink="^B$") + unbound = analyze(net) + + with pytest.raises(ValueError, match="source/sink already configured"): + bound.sensitivity_with_flow("^A$", "^B$") + with pytest.raises(ValueError, match="source and sink are required"): + unbound.sensitivity_with_flow() + + def test_combined_builds_masks_once(self, monkeypatch: pytest.MonkeyPatch) -> None: + from ngraph.analysis.context import AnalysisContext + + net = self._pairwise_network() + ctx = analyze(net, source="^S1$", sink="^T1$") + + calls = {"node_mask": 0, "edge_mask": 0} + orig_node_mask = AnalysisContext.build_node_mask + orig_edge_mask = AnalysisContext.build_edge_mask + + def counting_node_mask(self: AnalysisContext, excluded_nodes=None): + calls["node_mask"] += 1 + return orig_node_mask(self, excluded_nodes) + + def counting_edge_mask(self: AnalysisContext, excluded_links=None): + calls["edge_mask"] += 1 + return orig_edge_mask(self, excluded_links) + + monkeypatch.setattr(AnalysisContext, "build_node_mask", counting_node_mask) + monkeypatch.setattr(AnalysisContext, "build_edge_mask", counting_edge_mask) + + ctx.sensitivity_with_flow() + + assert calls == {"node_mask": 1, "edge_mask": 1} + + +class TestPairwisePseudoEdgeDeduplication: + """PAIRWISE attachment edges are emitted once per group member.""" + + @staticmethod + def _grouped_network() -> Network: + net = Network() + # 3 source groups and 3 sink groups with 2 members each + for prefix in ["s1", "s2", "s3", "t1", "t2", "t3"]: + for suffix in ["a", "b"]: + net.add_node(Node(f"{prefix}{suffix}")) + net.add_link(Link("s1a", "t1a", capacity=1.0, cost=1.0)) + net.add_link(Link("s2a", "t2a", capacity=2.0, cost=1.0)) + return net + + def test_pseudo_edge_count_linear_in_group_members(self) -> None: + net = self._grouped_network() + ctx = analyze(net, source=r"^(s\d)", sink=r"^(t\d)", mode=Mode.PAIRWISE) + + # 2 links x 2 directions = 4 real edges; pseudo edges: one per + # member per participating group = (3 + 3) groups * 2 members = 12 + # (previously duplicated once per opposing group: 24). + assert ctx.edge_count == 4 + 12 + + def test_pairwise_flows_unchanged(self) -> None: + net = self._grouped_network() + ctx = analyze(net, source=r"^(s\d)", sink=r"^(t\d)", mode=Mode.PAIRWISE) + + results = ctx.max_flow() + + assert len(results) == 9 + assert results[("s1", "t1")] == pytest.approx(1.0) + assert results[("s2", "t2")] == pytest.approx(2.0) + assert results[("s3", "t3")] == 0.0 + + +class TestLazyCoreGraphBuild: + """Unbound contexts defer the Core graph build until needed.""" + + def test_unbound_context_is_lazy(self) -> None: + net = _two_node_network() + ctx = analyze(net) + + assert ctx._core is None + + # Unbound flow analysis builds a temporary bound context and does + # not need this context's own Core graph. + flow = ctx.max_flow("^A$", "^B$") + assert flow[("^A$", "^B$")] == pytest.approx(1.0) + assert ctx._core is None + + # Path analysis triggers the lazy build + cost = ctx.shortest_path_cost("^A$", "^B$") + assert cost[("^A$", "^B$")] == 1.0 + assert ctx._core is not None + + def test_bound_context_builds_eagerly(self) -> None: + net = _two_node_network() + ctx = analyze(net, source="^A$", sink="^B$") + + assert ctx._core is not None + assert ctx.max_flow()[("^A$", "^B$")] == pytest.approx(1.0) + + def test_unbound_with_augmentations_builds_eagerly(self) -> None: + net = _two_node_network() + ctx = analyze(net, augmentations=[AugmentationEdge("A", "B", 5.0, 1)]) + + assert ctx._core is not None + # Augmented edge visible in the graph: 2 real + 1 augmentation + assert ctx.edge_count == 3 + + def test_core_accessors_trigger_lazy_build(self) -> None: + net = _two_node_network() + ctx = analyze(net) + + assert ctx._core is None + assert ctx.node_count == 2 + assert ctx._core is not None diff --git a/tests/analysis/test_demand_expansion_semantics.py b/tests/analysis/test_demand_expansion_semantics.py new file mode 100644 index 0000000..75e51da --- /dev/null +++ b/tests/analysis/test_demand_expansion_semantics.py @@ -0,0 +1,319 @@ +"""Expansion semantics tests for group_mode='per_group'. + +Pins the documented behavior: +- combine + per_group: one demand per source group with all targets combined. +- pairwise + per_group: pairwise within each group label present on both sides. +- td.volume is split evenly across groups; a skipped group's share (targets + empty after overlap exclusion, or no non-self pairs) is not redistributed, + so total expanded volume can be less than td.volume in those edge cases. +- combine mode excludes nodes selected on both sides (no zero-cost pseudo + bypass when source/target selections overlap). +""" + +import pytest + +from ngraph.analysis.demand import expand_demands +from ngraph.analysis.functions import demand_placement_analysis +from ngraph.model.demand.spec import TrafficDemand +from ngraph.model.network import Link, Network, Node + + +def _dc_network() -> Network: + """Two DCs with two nodes each, plus two hub nodes, fully meshed to hubs.""" + network = Network() + node_names = ["dc1/a", "dc1/b", "dc2/a", "dc2/b", "hub1", "hub2"] + for name in node_names: + network.add_node(Node(name)) + for dc_node in ("dc1/a", "dc1/b", "dc2/a", "dc2/b"): + for hub in ("hub1", "hub2"): + network.add_link(Link(dc_node, hub, capacity=10.0)) + return network + + +class TestPerGroupCombine: + """combine + per_group: one demand per source group, targets combined.""" + + def test_one_demand_per_source_group(self) -> None: + network = _dc_network() + td = TrafficDemand( + id="d1", + source="^(dc[0-9]+)/.*", + target="^hub.*", + volume=8.0, + mode="combine", + group_mode="per_group", + ) + + expansion = expand_demands(network, [td]) + + assert len(expansion.demands) == 2 + # Volume split evenly across source groups; total preserved + assert all(d.volume == pytest.approx(4.0) for d in expansion.demands) + assert sum(d.volume for d in expansion.demands) == pytest.approx(8.0) + # One pseudo source/sink pair per source group + assert {d.src_name for d in expansion.demands} == { + "_src_d1|dc1", + "_src_d1|dc2", + } + assert {d.dst_name for d in expansion.demands} == { + "_snk_d1|dc1", + "_snk_d1|dc2", + } + + def test_targets_combined_per_source_group(self) -> None: + network = _dc_network() + td = TrafficDemand( + id="d1", + source="^(dc[0-9]+)/.*", + target="^hub.*", + volume=8.0, + mode="combine", + group_mode="per_group", + ) + + expansion = expand_demands(network, [td]) + + # Per source group: 2 source attachments + 2 target attachments + # (BOTH hubs combined), so 8 augmentation edges in total. + assert len(expansion.augmentations) == 8 + dc1_sources = { + aug.target for aug in expansion.augmentations if aug.source == "_src_d1|dc1" + } + dc1_sinks = { + aug.source for aug in expansion.augmentations if aug.target == "_snk_d1|dc1" + } + assert dc1_sources == {"dc1/a", "dc1/b"} + assert dc1_sinks == {"hub1", "hub2"} + + +class TestPerGroupPairwise: + """pairwise + per_group: pairwise within each same-label group only.""" + + def test_pairwise_within_each_group(self) -> None: + network = _dc_network() + td = TrafficDemand( + id="d1", + source="^(dc[0-9]+)/.*", + target="^(dc[0-9]+)/.*", + volume=8.0, + mode="pairwise", + group_mode="per_group", + ) + + expansion = expand_demands(network, [td]) + + # Within dc1: (a,b), (b,a); within dc2: (a,b), (b,a). No cross-DC pairs. + assert len(expansion.demands) == 4 + pairs = {(d.src_name, d.dst_name) for d in expansion.demands} + assert pairs == { + ("dc1/a", "dc1/b"), + ("dc1/b", "dc1/a"), + ("dc2/a", "dc2/b"), + ("dc2/b", "dc2/a"), + } + # 8.0 split across 2 groups, then across 2 pairs per group + assert all(d.volume == pytest.approx(2.0) for d in expansion.demands) + assert sum(d.volume for d in expansion.demands) == pytest.approx(8.0) + # Pairwise mode creates no pseudo nodes + assert expansion.augmentations == [] + + def test_no_shared_labels_yields_no_demands(self) -> None: + network = _dc_network() + td = TrafficDemand( + id="d1", + source="^(dc[0-9]+)/.*", + target="^(hub[0-9]+)$", + volume=8.0, + mode="pairwise", + group_mode="per_group", + ) + + # Source labels (dc1, dc2) and target labels (hub1, hub2) are + # disjoint, so per_group pairwise produces nothing. + with pytest.raises(ValueError, match="No demands could be expanded"): + expand_demands(network, [td]) + + +class TestCombineOverlapExclusion: + """Combine mode excludes nodes selected on both sides. + + Regression: overlapping source/target selections previously attached + shared nodes to both pseudo endpoints, creating a zero-cost + pseudo_src -> node -> pseudo_snk bypass over two LARGE_CAPACITY + augmentation edges that absorbed the entire demand without touching + the real network. + """ + + @staticmethod + def _two_group_single_link_network() -> Network: + """Four nodes in two groups joined by a single capacity-1.0 link.""" + network = Network() + for name in ("A/1", "A/2", "B/1", "B/2"): + network.add_node(Node(name)) + network.add_link(Link("A/1", "B/1", capacity=1.0)) + return network + + def test_per_group_combine_excludes_own_group_nodes(self) -> None: + network = self._two_group_single_link_network() + td = TrafficDemand( + id="d1", + source="^(A|B)/", + target="^(A|B)/", + volume=100.0, + mode="combine", + group_mode="per_group", + ) + + expansion = expand_demands(network, [td]) + + assert len(expansion.demands) == 2 + # Each source group's pseudo sink attaches only the OTHER group's nodes + snk_a_sources = { + aug.source for aug in expansion.augmentations if aug.target == "_snk_d1|A" + } + snk_b_sources = { + aug.source for aug in expansion.augmentations if aug.target == "_snk_d1|B" + } + assert snk_a_sources == {"B/1", "B/2"} + assert snk_b_sources == {"A/1", "A/2"} + # No node is attached to both pseudo endpoints of the same demand + # (such a node would form a zero-cost bypass) + for demand in expansion.demands: + attached_to_src = { + aug.target + for aug in expansion.augmentations + if aug.source == demand.src_name + } + attached_to_snk = { + aug.source + for aug in expansion.augmentations + if aug.target == demand.dst_name + } + assert not attached_to_src & attached_to_snk + + def test_per_group_combine_placement_bounded_by_real_capacity(self) -> None: + """Placement is bounded by real capacity, not the pseudo bypass.""" + network = self._two_group_single_link_network() + demands_config = [ + { + "source": "^(A|B)/", + "target": "^(A|B)/", + "volume": 100.0, + "mode": "combine", + "group_mode": "per_group", + } + ] + + result = demand_placement_analysis( + network=network, + excluded_nodes=set(), + excluded_links=set(), + demands_config=demands_config, + ) + + # The single capacity-1.0 link (plus its reverse edge) bounds + # placement at 2.0; the bypass previously placed all 100. + assert result.summary.total_demand == pytest.approx(100.0) + assert result.summary.total_placed == pytest.approx(2.0) + assert result.summary.overall_ratio == pytest.approx(0.02) + + def test_flatten_combine_full_overlap_raises(self) -> None: + network = self._two_group_single_link_network() + td = TrafficDemand( + id="d1", + source="^(A|B)/", + target="^(A|B)/", + volume=100.0, + mode="combine", + group_mode="flatten", + ) + + # All targets are excluded as overlapping, so nothing expands + with pytest.raises(ValueError, match="No demands could be expanded"): + expand_demands(network, [td]) + + def test_flatten_combine_partial_overlap_excludes_shared_node(self) -> None: + network = self._two_group_single_link_network() + td = TrafficDemand( + id="d1", + source="^A/", + target="^(A/1|B)", # A/1 is also selected as a source + volume=10.0, + mode="combine", + group_mode="flatten", + ) + + expansion = expand_demands(network, [td]) + + assert len(expansion.demands) == 1 + snk_sources = { + aug.source for aug in expansion.augmentations if aug.target == "_snk_d1" + } + assert snk_sources == {"B/1", "B/2"} + + +class TestPerGroupVolumeConservation: + """Total expanded volume equals td.volume regardless of group count.""" + + def test_combine_volume_conserved_with_three_groups(self) -> None: + network = Network() + for name in ("g1/x", "g2/x", "g3/x", "sink"): + network.add_node(Node(name)) + for name in ("g1/x", "g2/x", "g3/x"): + network.add_link(Link(name, "sink", capacity=10.0)) + + td = TrafficDemand( + id="d1", + source="^(g[0-9]+)/.*", + target="^sink$", + volume=9.0, + mode="combine", + group_mode="per_group", + ) + + expansion = expand_demands(network, [td]) + + assert len(expansion.demands) == 3 + assert all(d.volume == pytest.approx(3.0) for d in expansion.demands) + assert sum(d.volume for d in expansion.demands) == pytest.approx(9.0) + + +def test_cross_demand_composed_pseudo_collision_raises() -> None: + """Composed per_group ids can render identically across demands when ids + or labels contain '|'; the shared pseudo endpoint must be rejected, not + silently merged (which would recreate the zero-cost bypass).""" + import pytest + + from ngraph.analysis.functions import demand_placement_analysis + from ngraph.model.network import Link, Network, Node + + net = Network() + for name, attrs in (("A1", {"grp": "Y|Z"}), ("T", {}), ("B1", {"grp2": "Z"})): + net.add_node(Node(name, attrs=attrs)) + net.add_link(Link("A1", "T", capacity=20.0)) + + cfg = [ + { + "id": "X", + "source": {"path": "^A1$", "group_by": "grp"}, + "target": "^T$", + "volume": 10.0, + "mode": "combine", + "group_mode": "per_group", + }, + { + "id": "X|Y", + "source": {"path": "^B1$", "group_by": "grp2"}, + "target": "^T$", + "volume": 10.0, + "mode": "combine", + "group_mode": "per_group", + }, + ] + with pytest.raises(ValueError, match="pseudo endpoint"): + demand_placement_analysis( + network=net, + excluded_nodes=set(), + excluded_links=set(), + demands_config=cfg, + ) diff --git a/tests/analysis/test_failure_manager.py b/tests/analysis/test_failure_manager.py index 494d3f4..347b3c3 100644 --- a/tests/analysis/test_failure_manager.py +++ b/tests/analysis/test_failure_manager.py @@ -10,7 +10,6 @@ import pytest from ngraph.analysis.failure_manager import FailureManager -from ngraph.dsl.selectors.schema import Condition from ngraph.model.failure.policy import ( FailureMode, FailurePolicy, @@ -18,6 +17,7 @@ ) from ngraph.model.failure.policy_set import FailurePolicySet from ngraph.model.network import Link, Network, Node, RiskGroup +from ngraph.model.selectors import Condition @pytest.fixture @@ -206,8 +206,10 @@ def prepare_matches( ) -> dict[int, tuple[str, ...]]: return {} - def apply_failures(self, *args: Any, **kwargs: Any) -> list[str]: - return [self.failed_group] + def apply_failures_typed( + self, *args: Any, **kwargs: Any + ) -> tuple[set[str], set[str], set[str]]: + return set(), set(), {self.failed_group} fm = FailureManager( network=network, @@ -258,15 +260,15 @@ def prepare_matches( ) -> dict[int, tuple[str, ...]]: return {} - def apply_failures( + def apply_failures_typed( self, *args: Any, failure_trace: dict[str, Any] | None = None, **kwargs: Any, - ) -> list[str]: + ) -> tuple[set[str], set[str], set[str]]: if failure_trace is not None: failure_trace.update(expected_trace) - return [parent.name] + return set(), set(), {parent.name} fm = FailureManager( network=network, @@ -308,8 +310,10 @@ def prepare_matches( ) -> dict[int, tuple[str, ...]]: return {} - def apply_failures(self, *args: Any, **kwargs: Any) -> list[str]: - return [group_a.name] + def apply_failures_typed( + self, *args: Any, **kwargs: Any + ) -> tuple[set[str], set[str], set[str]]: + return set(), set(), {group_a.name} fm = FailureManager( network=network, diff --git a/tests/analysis/test_failure_manager_fixes.py b/tests/analysis/test_failure_manager_fixes.py new file mode 100644 index 0000000..e68b7d1 --- /dev/null +++ b/tests/analysis/test_failure_manager_fixes.py @@ -0,0 +1,549 @@ +"""Regression tests for FailureManager review fixes. + +Covers: +- Demand-placement Monte Carlo with id-less demand configs (stable demand ids). +- Seed fallback to policy.seed when FailureManager seed is None. +- Context injection gated on the analysis function declaring 'context'. +- Prepared-matches cache identity check (id() address-reuse hazard). +- No forced serial execution for __main__-defined analysis functions. +- Transitive risk-group exclusions across a 3-level hierarchy. +- Risk-group expansion index built once per manager, not per iteration. +""" + +from typing import Any + +import pytest + +from ngraph.analysis.failure_manager import FailureManager +from ngraph.analysis.functions import ( + _reconstruct_traffic_demands, + build_demand_placement_inputs, + demand_placement_analysis, +) +from ngraph.model.failure.policy import FailureMode, FailurePolicy, FailureRule +from ngraph.model.failure.policy_set import FailurePolicySet +from ngraph.model.network import Link, Network, Node, RiskGroup + + +def _chain_network() -> Network: + """A -> B -> C chain with capacity 10 links.""" + network = Network() + for name in ("A", "B", "C"): + network.add_node(Node(name)) + network.add_link(Link("A", "B", capacity=10.0)) + network.add_link(Link("B", "C", capacity=10.0)) + return network + + +def _manager_with_link_choice_policy(network: Network) -> FailureManager: + policy = FailurePolicy( + seed=7, + modes=[ + FailureMode( + weight=1.0, + rules=[FailureRule(scope="link", mode="choice", count=1)], + ) + ], + ) + policy_set = FailurePolicySet() + policy_set.add("p", policy) + return FailureManager(network, policy_set, "p") + + +def _manager_without_policy(network: Network) -> FailureManager: + return FailureManager(network, FailurePolicySet(), None) + + +class TestIdlessDemandConfigs: + """Demand configs without 'id' must not crash Monte Carlo analysis.""" + + def test_run_demand_placement_mc_idless_combine(self) -> None: + fm = _manager_with_link_choice_policy(_chain_network()) + + result = fm.run_demand_placement_monte_carlo( + [{"source": "^A$", "target": "^C$", "volume": 5.0, "mode": "combine"}], + iterations=3, + ) + + assert result["baseline"] is not None + assert result["baseline"].summary.total_placed == pytest.approx(5.0) + assert len(result["results"]) >= 1 + + def test_run_demand_placement_mc_idless_per_group(self) -> None: + network = Network() + for name in ("dc1/a", "dc1/b", "dc2/a", "dc2/b", "hub"): + network.add_node(Node(name)) + for name in ("dc1/a", "dc1/b", "dc2/a", "dc2/b"): + network.add_link(Link(name, "hub", capacity=10.0)) + fm = _manager_with_link_choice_policy(network) + + result = fm.run_demand_placement_monte_carlo( + [ + { + "source": "^(dc[0-9]+)/.*", + "target": "^hub$", + "volume": 4.0, + "mode": "combine", + "group_mode": "per_group", + } + ], + iterations=2, + ) + + baseline = result["baseline"] + # One demand per source group (dc1, dc2), each volume 2.0 + assert baseline.summary.num_flows == 2 + assert baseline.summary.total_demand == pytest.approx(4.0) + assert baseline.summary.total_placed == pytest.approx(4.0) + + def test_reconstructed_ids_are_deterministic(self) -> None: + config = [ + {"source": "^A$", "target": "^C$", "volume": 5.0, "mode": "combine"}, + {"source": "^B$", "target": "^C$", "volume": 1.0}, + ] + ids_first = [td.id for td in _reconstruct_traffic_demands(config)] + ids_second = [td.id for td in _reconstruct_traffic_demands(config)] + + assert ids_first == ids_second + assert all(ids_first) + assert len(set(ids_first)) == len(ids_first) + + def test_prebuilt_context_without_expansion_idless_config(self) -> None: + """Fallback path: context provided but expansion absent must still work.""" + network = _chain_network() + config = [{"source": "^A$", "target": "^C$", "volume": 5.0, "mode": "combine"}] + ctx, _, _ = build_demand_placement_inputs(network, config) + + result = demand_placement_analysis( + network=network, + excluded_nodes=set(), + excluded_links=set(), + demands_config=config, + context=ctx, + ) + + assert result.summary.total_placed == pytest.approx(5.0) + + +class TestSeedFallback: + """FailureManager seed=None must fall back to the policy's own seed.""" + + def test_policy_seed_produces_varied_iterations(self) -> None: + network = Network() + names = [f"n{i}" for i in range(7)] + for name in names: + network.add_node(Node(name)) + for left, right in zip(names, names[1:], strict=False): + network.add_link(Link(left, right, capacity=1.0)) + + policy = FailurePolicy( + seed=42, + modes=[ + FailureMode( + weight=1.0, + rules=[FailureRule(scope="link", mode="choice", count=1)], + ) + ], + ) + policy_set = FailurePolicySet() + policy_set.add("p", policy) + fm = FailureManager(network, policy_set, "p") + + def fake_analysis( + network: Network, excluded_nodes: set, excluded_links: set + ) -> dict[str, Any]: + return {"excluded": len(excluded_links)} + + result = fm.run_monte_carlo_analysis(analysis_func=fake_analysis, iterations=10) + + # Without the fallback, every iteration reuses the same policy RNG + # and collapses to a single failure pattern. + assert result["metadata"]["unique_patterns"] > 1 + + def test_policy_seed_fallback_is_reproducible(self) -> None: + def run_once() -> list[tuple]: + network = Network() + for name in ("a", "b", "c", "d"): + network.add_node(Node(name)) + policy = FailurePolicy( + seed=13, + modes=[ + FailureMode( + weight=1.0, + rules=[FailureRule(scope="node", mode="choice", count=1)], + ) + ], + ) + policy_set = FailurePolicySet() + policy_set.add("p", policy) + fm = FailureManager(network, policy_set, "p") + patterns = [] + for i in range(5): + excluded_nodes, excluded_links = fm.compute_exclusions( + seed_offset=13 + i + ) + patterns.append((tuple(sorted(excluded_nodes)))) + return patterns + + assert run_once() == run_once() + + +class TestContextInjectionGating: + """Context pre-building requires an explicit prepare_inputs hook.""" + + def test_custom_function_with_source_target_params(self) -> None: + fm = _manager_with_link_choice_policy(_chain_network()) + + def custom( + network: Network, + excluded_nodes: set, + excluded_links: set, + source: str, + target: str, + ) -> dict[str, Any]: + return {"src": source, "dst": target} + + result = fm.run_monte_carlo_analysis( + analysis_func=custom, iterations=2, source="^A$", target="^C$" + ) + + assert result["baseline"] == {"src": "^A$", "dst": "^C$"} + + def test_var_keyword_function_does_not_opt_in(self) -> None: + fm = _manager_without_policy(_chain_network()) + captured: list[dict[str, Any]] = [] + + def custom( + network: Network, excluded_nodes: set, excluded_links: set, **kwargs: Any + ) -> dict[str, Any]: + captured.append(dict(kwargs)) + return {} + + # 'source' is not a valid selector regex; injection would crash here. + fm.run_monte_carlo_analysis( + analysis_func=custom, iterations=1, source="dc(", target="x" + ) + + assert captured + assert all("context" not in kwargs for kwargs in captured) + + def test_declared_context_without_hook_gets_no_injection(self) -> None: + # Declaring a `context` parameter is no longer enough on its own; + # only a prepare_inputs hook opts a function into pre-building. + fm = _manager_without_policy(_chain_network()) + + def custom( + network: Network, + excluded_nodes: set, + excluded_links: set, + demands_config: list, + context: Any = None, + ) -> dict[str, bool]: + return {"has_context": context is not None} + + result = fm.run_monte_carlo_analysis( + analysis_func=custom, + iterations=1, + demands_config=[ + {"source": "^A$", "target": "^C$", "volume": 1.0, "mode": "combine"} + ], + ) + + assert result["baseline"] == {"has_context": False} + + def test_prepare_inputs_hook_receives_prebuilt_inputs(self) -> None: + fm = _manager_without_policy(_chain_network()) + + def custom( + network: Network, + excluded_nodes: set, + excluded_links: set, + demands_config: list, + context: Any = None, + expansion: Any = None, + resolved_ids: Any = None, + ) -> dict[str, bool]: + return { + "has_context": context is not None, + "has_expansion": expansion is not None, + "has_resolved_ids": resolved_ids is not None, + } + + def prepare(network: Network, kwargs: dict) -> dict: + ctx, expansion, resolved_ids = build_demand_placement_inputs( + network, kwargs["demands_config"] + ) + return { + "context": ctx, + "expansion": expansion, + "resolved_ids": resolved_ids, + } + + custom.prepare_inputs = prepare + + result = fm.run_monte_carlo_analysis( + analysis_func=custom, + iterations=1, + demands_config=[ + {"source": "^A$", "target": "^C$", "volume": 1.0, "mode": "combine"} + ], + ) + + assert result["baseline"] == { + "has_context": True, + "has_expansion": True, + "has_resolved_ids": True, + } + + def test_builtin_analysis_functions_carry_hook(self) -> None: + from ngraph.analysis.functions import ( + demand_placement_analysis, + max_flow_analysis, + sensitivity_analysis, + ) + + for func in ( + demand_placement_analysis, + max_flow_analysis, + sensitivity_analysis, + ): + assert callable(getattr(func, "prepare_inputs", None)) + + +class TestParallelismNotForcedSerial: + """__main__-defined functions run with the requested thread parallelism.""" + + def test_main_module_function_keeps_parallelism(self) -> None: + fm = _manager_with_link_choice_policy(_chain_network()) + + def main_func( + network: Network, excluded_nodes: set, excluded_links: set + ) -> dict[str, Any]: + return {"ok": True} + + main_func.__module__ = "__main__" + + result = fm.run_monte_carlo_analysis( + analysis_func=main_func, iterations=4, parallelism=2 + ) + + assert result["metadata"]["parallelism"] == 2 + assert result["baseline"] == {"ok": True} + + +class TestPreparedMatchesCacheIdentity: + """Cache entries must be ignored when the stored policy is a different object.""" + + def test_stale_entry_with_reused_id_is_not_used(self) -> None: + network = Network() + network.add_node(Node("node1")) + network.add_node(Node("node2")) + fm = _manager_without_policy(network) + + policy_a = FailurePolicy( + modes=[ + FailureMode( + weight=1.0, + rules=[FailureRule(scope="node", mode="all", path="^node1$")], + ) + ] + ) + excluded_nodes, _ = fm.compute_exclusions(policy=policy_a) + assert excluded_nodes == {"node1"} + + policy_b = FailurePolicy( + modes=[ + FailureMode( + weight=1.0, + rules=[FailureRule(scope="node", mode="all", path="^node2$")], + ) + ] + ) + rule_b = policy_b.modes[0].rules[0] + # Simulate CPython address reuse: a stale entry stored under + # id(policy_b) that belongs to policy_a and maps policy_b's rule + # to the wrong candidate pool. + fm._prepared_policy_matches[id(policy_b)] = ( + policy_a, + {id(rule_b): ("node1",)}, + ) + + excluded_nodes_b, _ = fm.compute_exclusions(policy=policy_b) + assert excluded_nodes_b == {"node2"} + + def test_same_policy_object_uses_cache(self) -> None: + network = Network() + network.add_node(Node("node1")) + fm = _manager_without_policy(network) + policy = FailurePolicy( + modes=[ + FailureMode( + weight=1.0, + rules=[FailureRule(scope="node", mode="all")], + ) + ] + ) + + first, _ = fm.compute_exclusions(policy=policy) + cached_policy, cached_prepared = fm._prepared_policy_matches[id(policy)] + second, _ = fm.compute_exclusions(policy=policy) + + assert first == second == {"node1"} + assert cached_policy is policy + assert fm._prepared_policy_matches[id(policy)][1] is cached_prepared + + +class TestRiskGroupIndexCache: + """Risk-group expansion index is built once per manager, not per call.""" + + @staticmethod + def _network_with_shared_risk_group() -> Network: + network = Network() + network.add_node(Node("A", risk_groups={"rg1"})) + network.add_node(Node("B", risk_groups={"rg1"})) + network.add_node(Node("C")) + network.add_link(Link("A", "C", capacity=1.0)) + return network + + @staticmethod + def _expanding_policy_manager(network: Network) -> FailureManager: + policy = FailurePolicy( + expand_groups=True, + modes=[ + FailureMode( + weight=1.0, + rules=[FailureRule(scope="node", mode="all", path="^A$")], + ) + ], + ) + policy_set = FailurePolicySet() + policy_set.add("p", policy) + return FailureManager(network, policy_set, "p") + + def test_index_built_once_across_many_exclusion_calls( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + fm = self._expanding_policy_manager(self._network_with_shared_risk_group()) + + calls = {"count": 0} + original = FailurePolicy.build_risk_group_index + + def counting( + network_nodes: dict[str, Any], network_links: dict[str, Any] + ) -> dict[str, set[str]]: + calls["count"] += 1 + return original(network_nodes, network_links) + + monkeypatch.setattr( + FailurePolicy, "build_risk_group_index", staticmethod(counting) + ) + + for i in range(10): + excluded_nodes, _excluded_links = fm.compute_exclusions(seed_offset=i) + # Expansion via the shared risk group must still take effect: + # B shares rg1 with the failed node A. + assert {"A", "B"} <= excluded_nodes + + assert calls["count"] == 1 + + def test_index_not_built_when_policy_does_not_expand( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + network = self._network_with_shared_risk_group() + policy = FailurePolicy( + modes=[ + FailureMode( + weight=1.0, + rules=[FailureRule(scope="node", mode="all", path="^A$")], + ) + ], + ) + policy_set = FailurePolicySet() + policy_set.add("p", policy) + fm = FailureManager(network, policy_set, "p") + + def boom(*args: Any, **kwargs: Any) -> None: + raise AssertionError( + "build_risk_group_index must not run when expand_groups is False" + ) + + monkeypatch.setattr(FailurePolicy, "build_risk_group_index", staticmethod(boom)) + + excluded_nodes, _excluded_links = fm.compute_exclusions() + assert excluded_nodes == {"A"} + + +class TestRiskGroupHierarchyExclusions: + """Nested (3-level) risk-group members must all be excluded.""" + + def test_grandchild_members_excluded_when_top_fails(self) -> None: + network = Network() + network.add_node(Node("n_top", risk_groups={"top"})) + network.add_node(Node("n_mid", risk_groups={"mid"})) + network.add_node(Node("n_leaf", risk_groups={"leaf"})) + network.add_node(Node("other")) + leaf_link = Link("n_leaf", "other", risk_groups={"leaf"}) + network.add_link(leaf_link) + + # Only the top-level group is registered in network.risk_groups; + # 'mid' and 'leaf' exist solely as nested children. + network.risk_groups["top"] = RiskGroup( + name="top", + children=[RiskGroup(name="mid", children=[RiskGroup(name="leaf")])], + ) + + policy = FailurePolicy( + modes=[ + FailureMode( + weight=1.0, + rules=[FailureRule(scope="risk_group", mode="all")], + ) + ] + ) + policy_set = FailurePolicySet() + policy_set.add("p", policy) + fm = FailureManager(network, policy_set, "p") + + excluded_nodes, excluded_links = fm.compute_exclusions() + + assert {"n_top", "n_mid", "n_leaf"} <= excluded_nodes + assert "other" not in excluded_nodes + assert leaf_link.id in excluded_links + + +class TestExpandGroupsScopeSymmetry: + """expand_groups must produce identical exclusions whether a failure came + from an entity rule or a risk_group rule, including depth>=3 hierarchies + whose nested groups are not registered top-level.""" + + def test_deep_hierarchy_rg_rule_matches_node_rule(self) -> None: + from ngraph.model.network import Network, Node, RiskGroup + + def build() -> Network: + net = Network() + for n in ("n_leaf", "n_mid", "n_lat"): + net.add_node(Node(n)) + leaf = RiskGroup(name="leaf") + mid = RiskGroup(name="mid", children=[leaf]) + net.risk_groups["top"] = RiskGroup(name="top", children=[mid]) + net.nodes["n_leaf"].risk_groups |= {"leaf", "SHARED"} + net.nodes["n_mid"].risk_groups |= {"mid"} + net.nodes["n_lat"].risk_groups |= {"SHARED"} + return net + + def exclusions(rule: FailureRule) -> list[str]: + net = build() + pol = FailurePolicy( + expand_groups=True, modes=[FailureMode(weight=1.0, rules=[rule])] + ) + fps = FailurePolicySet() + fps.add("p", pol) + fm = FailureManager(network=net, failure_policy_set=fps, policy_name="p") + nodes, _ = fm.compute_exclusions(pol, 1) + return sorted(nodes) + + via_rg = exclusions(FailureRule(scope="risk_group", mode="all", path="^top$")) + via_nodes = exclusions( + FailureRule(scope="node", mode="all", path="^(n_leaf|n_mid)$") + ) + assert via_rg == via_nodes == ["n_lat", "n_leaf", "n_mid"] diff --git a/tests/analysis/test_functions.py b/tests/analysis/test_functions.py index 0f2ce33..d9db9d2 100644 --- a/tests/analysis/test_functions.py +++ b/tests/analysis/test_functions.py @@ -149,7 +149,6 @@ def test_demand_placement_analysis_basic(self, diamond_network: Network) -> None excluded_nodes=set(), excluded_links=set(), demands_config=demands_config, - placement_rounds=1, ) # Verify results structure @@ -187,7 +186,6 @@ def test_demand_placement_analysis_zero_total_demand( excluded_nodes=set(), excluded_links=set(), demands_config=demands_config, - placement_rounds=1, ) assert isinstance(result, FlowIterationResult) @@ -206,7 +204,7 @@ class TestDemandPlacementWithContextCaching: def test_context_caching_pairwise_mode(self, diamond_network: Network) -> None: """Context caching works with pairwise mode.""" - from ngraph.analysis.functions import build_demand_context + from ngraph.analysis.functions import build_demand_placement_inputs demands_config = [ { @@ -219,7 +217,7 @@ def test_context_caching_pairwise_mode(self, diamond_network: Network) -> None: ] # Build context once - ctx = build_demand_context(diamond_network, demands_config) + ctx, _, _ = build_demand_placement_inputs(diamond_network, demands_config) # Use context for analysis result = demand_placement_analysis( @@ -235,7 +233,7 @@ def test_context_caching_pairwise_mode(self, diamond_network: Network) -> None: def test_context_caching_combine_mode(self, diamond_network: Network) -> None: """Context caching works with combine mode (uses pseudo nodes).""" - from ngraph.analysis.functions import build_demand_context + from ngraph.analysis.functions import build_demand_placement_inputs demands_config = [ { @@ -248,7 +246,7 @@ def test_context_caching_combine_mode(self, diamond_network: Network) -> None: ] # Build context once - ctx = build_demand_context(diamond_network, demands_config) + ctx, _, _ = build_demand_placement_inputs(diamond_network, demands_config) # Use context for analysis - this is where the bug manifested result = demand_placement_analysis( @@ -266,7 +264,7 @@ def test_context_caching_combine_multiple_iterations( self, diamond_network: Network ) -> None: """Context can be reused for multiple analysis iterations.""" - from ngraph.analysis.functions import build_demand_context + from ngraph.analysis.functions import build_demand_placement_inputs demands_config = [ { @@ -278,7 +276,7 @@ def test_context_caching_combine_multiple_iterations( }, ] - ctx = build_demand_context(diamond_network, demands_config) + ctx, _, _ = build_demand_placement_inputs(diamond_network, demands_config) # Run multiple iterations with different exclusions for excluded in [set(), {"B"}, {"C"}]: @@ -291,11 +289,17 @@ def test_context_caching_combine_multiple_iterations( ) assert isinstance(result, FlowIterationResult) - def test_context_caching_without_id_raises(self, diamond_network: Network) -> None: - """Context caching without stable ID raises KeyError for combine mode.""" - from ngraph.analysis.functions import build_demand_context + def test_context_caching_without_id_works(self, diamond_network: Network) -> None: + """Context caching works without explicit IDs (deterministic ids). - # Config without explicit ID - each reconstruction generates new ID + Regression: configs without "id" previously got a fresh uuid on + every reconstruction, so pseudo node names diverged from the + pre-built context and analysis crashed with KeyError. IDs derived + from source/target/position keep them stable. + """ + from ngraph.analysis.functions import build_demand_placement_inputs + + # Config without explicit ID - deterministic ID is derived demands_config = [ { "source": "[AB]", @@ -305,19 +309,18 @@ def test_context_caching_without_id_raises(self, diamond_network: Network) -> No }, ] - # Build context - creates pseudo nodes with auto-generated ID (uuid1) - ctx = build_demand_context(diamond_network, demands_config) + ctx, _, _ = build_demand_placement_inputs(diamond_network, demands_config) - # Analysis reconstructs TrafficDemand without ID -> generates new ID (uuid2) - # Tries to find pseudo nodes _src_...|uuid2 which don't exist -> KeyError - with pytest.raises(KeyError): - demand_placement_analysis( - network=diamond_network, - excluded_nodes=set(), - excluded_links=set(), - demands_config=demands_config, - context=ctx, - ) + result = demand_placement_analysis( + network=diamond_network, + excluded_nodes=set(), + excluded_links=set(), + demands_config=demands_config, + context=ctx, + ) + + assert result.summary.total_placed == 50.0 + assert result.summary.overall_ratio == 1.0 class TestSensitivityAnalysis: @@ -372,3 +375,36 @@ def test_sensitivity_analysis_empty_result(self, simple_network: Network) -> Non source="nonexistent.*", target="also_nonexistent.*", ) + + def test_sensitivity_analysis_single_pass( + self, simple_network: Network, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Computes flow and sensitivity in one pass (one mask pair per call).""" + from ngraph.analysis.context import AnalysisContext + from ngraph.analysis.functions import build_maxflow_context + + ctx = build_maxflow_context(simple_network, "A", "C", mode="combine") + + calls = {"node_mask": 0} + original = AnalysisContext.build_node_mask + + def counting(self: AnalysisContext, excluded_nodes=None): + calls["node_mask"] += 1 + return original(self, excluded_nodes) + + monkeypatch.setattr(AnalysisContext, "build_node_mask", counting) + + result = sensitivity_analysis( + network=simple_network, + excluded_nodes=set(), + excluded_links=set(), + source="A", + target="C", + context=ctx, + ) + + # Previously two full passes (max_flow + sensitivity) built two masks. + assert calls["node_mask"] == 1 + entry = result.flows[0] + assert entry.demand == entry.placed == 10.0 + assert len(entry.data["sensitivity"]) == 2 diff --git a/tests/analysis/test_functions_details.py b/tests/analysis/test_functions_details.py index 38f76c6..9fee1c5 100644 --- a/tests/analysis/test_functions_details.py +++ b/tests/analysis/test_functions_details.py @@ -35,7 +35,6 @@ def test_demand_placement_analysis_includes_flow_details_costs_and_edges() -> No excluded_nodes=set(), excluded_links=set(), demands_config=demands_config, - placement_rounds=1, include_flow_details=True, include_used_edges=True, ) diff --git a/tests/analysis/test_functions_mode_validation.py b/tests/analysis/test_functions_mode_validation.py new file mode 100644 index 0000000..1894e26 --- /dev/null +++ b/tests/analysis/test_functions_mode_validation.py @@ -0,0 +1,65 @@ +"""Mode string validation in analysis functions. + +Invalid mode strings must raise ValueError instead of silently falling back +to PAIRWISE. +""" + +import pytest + +from ngraph.analysis.functions import ( + build_maxflow_context, + max_flow_analysis, + sensitivity_analysis, +) +from ngraph.model.network import Link, Network, Node + + +def _simple_network() -> Network: + network = Network() + for name in ("A", "B"): + network.add_node(Node(name)) + network.add_link(Link("A", "B", capacity=1.0)) + return network + + +@pytest.mark.parametrize("bad_mode", ["aggregate", "Combined", "pair", "all", ""]) +def test_max_flow_analysis_invalid_mode_raises(bad_mode: str) -> None: + with pytest.raises(ValueError, match="Invalid mode"): + max_flow_analysis( + _simple_network(), + set(), + set(), + source="^A$", + target="^B$", + mode=bad_mode, + ) + + +def test_sensitivity_analysis_invalid_mode_raises() -> None: + with pytest.raises(ValueError, match="Invalid mode"): + sensitivity_analysis( + _simple_network(), + set(), + set(), + source="^A$", + target="^B$", + mode="aggregate", + ) + + +def test_build_maxflow_context_invalid_mode_raises() -> None: + with pytest.raises(ValueError, match="Invalid mode"): + build_maxflow_context(_simple_network(), "^A$", "^B$", mode="bogus") + + +@pytest.mark.parametrize("mode", ["combine", "pairwise", "COMBINE", "Pairwise"]) +def test_valid_modes_accepted(mode: str) -> None: + result = max_flow_analysis( + _simple_network(), + set(), + set(), + source="^A$", + target="^B$", + mode=mode, + ) + assert result.summary.num_flows >= 1 diff --git a/tests/analysis/test_maxflow_api.py b/tests/analysis/test_maxflow_api.py index a9d6578..326cdbe 100644 --- a/tests/analysis/test_maxflow_api.py +++ b/tests/analysis/test_maxflow_api.py @@ -135,10 +135,11 @@ def test_max_flow_with_details_total_matches() -> None: def test_max_flow_with_details_include_min_cut() -> None: - """Test that include_min_cut correctly returns saturated edges. + """Test that include_min_cut returns a true minimum cut. Uses the simple network with two parallel paths S->A->T and S->B->T. - All 4 edges should be saturated (form the min-cut). + A minimum cut is {S->A, S->B} (2 edges, total capacity == max flow), + not the full set of 4 saturated edges. """ net = _simple_network() @@ -146,19 +147,23 @@ def test_max_flow_with_details_include_min_cut() -> None: res_no_cut = analyze(net).max_flow_detailed("^S$", "^T$", mode=Mode.COMBINE) assert res_no_cut[("^S$", "^T$")].min_cut is None - # With include_min_cut=True, min_cut should contain saturated edges + # With include_min_cut=True, min_cut should be a minimum cut res_with_cut = analyze(net).max_flow_detailed( "^S$", "^T$", mode=Mode.COMBINE, include_min_cut=True ) summary = res_with_cut[("^S$", "^T$")] assert summary.min_cut is not None - assert len(summary.min_cut) == 4 # All 4 edges are saturated + assert len(summary.min_cut) == 2 # True min cut, not all saturated edges - # Verify edge refs have expected structure + # Min-cut edges are distinct links in the forward direction link_ids = {e.link_id for e in summary.min_cut} - # Each link appears once (forward direction) - assert len(link_ids) == 4 + assert len(link_ids) == 2 + assert all(e.direction == "fwd" for e in summary.min_cut) + + # Max-flow/min-cut duality: cut capacity equals max flow + cut_capacity = sum(net.links[e.link_id].capacity for e in summary.min_cut) + assert pytest.approx(cut_capacity, rel=0, abs=1e-9) == 2.0 # Verify total flow is still correct assert pytest.approx(summary.total_flow, rel=0, abs=1e-9) == 2.0 @@ -280,3 +285,26 @@ def test_require_capacity_parameter() -> None: "^S$", "^T$", mode=Mode.COMBINE, shortest_path=True, require_capacity=False ) assert result_ip[("^S$", "^T$")] == pytest.approx(0.0, abs=1e-6) + + +def test_cost_and_capacity_bounds_rejected() -> None: + """Graph build rejects costs >= 2**62 (core overflow) and capacities at or + above the internal pseudo-edge capacity, instead of silently corrupting.""" + import pytest + + from ngraph.analysis import analyze + from ngraph.model.network import Link, Network, Node + + net = Network() + net.add_node(Node("A")) + net.add_node(Node("B")) + net.add_link(Link("A", "B", capacity=10.0, cost=float(2**62))) + with pytest.raises(ValueError, match="2\\*\\*62"): + analyze(net, source="^A$", sink="^B$") + + net2 = Network() + net2.add_node(Node("A")) + net2.add_node(Node("B")) + net2.add_link(Link("A", "B", capacity=5e15, cost=1)) + with pytest.raises(ValueError, match="pseudo-edge capacity"): + analyze(net2, source="^A$", sink="^B$") diff --git a/tests/analysis/test_paths.py b/tests/analysis/test_paths.py index c9191d4..60bace8 100644 --- a/tests/analysis/test_paths.py +++ b/tests/analysis/test_paths.py @@ -8,6 +8,12 @@ from __future__ import annotations +import os +import subprocess +import sys +import textwrap +from typing import Any + import pytest from ngraph import Link, Mode, Network, Node, analyze @@ -257,6 +263,182 @@ def test_max_path_cost_factor(self) -> None: # Both paths (cost 2 and 3) should be included since 3 <= 3.0 assert len(paths) == 2 + @staticmethod + def _multi_member_group_network() -> Network: + """Build a network where groups contain multiple nodes. + + Topology: + x1 -> y1 (cost 10) + x2 -> y2 (cost 11) + x1 -> m -> y1 (cost 5 + 10 = 15) + + Group-to-group paths sorted by cost: 10 (x1->y1), 11 (x2->y2), + 15 (x1->m->y1). + """ + net = Network() + for name in ["x1", "x2", "y1", "y2", "m"]: + net.add_node(Node(name)) + + net.add_link(Link("x1", "y1", capacity=10.0, cost=10.0)) + net.add_link(Link("x2", "y2", capacity=10.0, cost=11.0)) + net.add_link(Link("x1", "m", capacity=10.0, cost=5.0)) + net.add_link(Link("m", "y1", capacity=10.0, cost=10.0)) + return net + + def test_multi_node_groups_merge_paths_across_pairs(self) -> None: + """KSP between multi-node groups merges paths from all node pairs. + + Regression: KSP previously ran only between the single best (src, + snk) node pair, silently omitting cheaper paths from other pairs + (here the cost-11 x2->y2 path lost to the cost-15 x1->m->y1 path). + """ + net = self._multi_member_group_network() + + results = analyze(net).k_shortest_paths("^x", "^y", max_k=2, mode=Mode.COMBINE) + + assert len(results) == 1 + paths = list(results.values())[0] + assert [p.cost for p in paths] == [10.0, 11.0] + + def test_multi_node_groups_all_paths_in_cost_order(self) -> None: + """KSP with a larger max_k returns merged paths in cost order.""" + net = self._multi_member_group_network() + + results = analyze(net).k_shortest_paths("^x", "^y", max_k=5, mode=Mode.COMBINE) + + paths = list(results.values())[0] + assert [p.cost for p in paths] == [10.0, 11.0, 15.0] + + def test_multi_node_groups_cost_factor_relative_to_group_best(self) -> None: + """max_path_cost_factor applies to the best cost across all pairs.""" + net = self._multi_member_group_network() + + # Cap = 10 * 1.2 = 12: includes costs 10 and 11, excludes 15 + results = analyze(net).k_shortest_paths( + "^x", "^y", max_k=5, mode=Mode.COMBINE, max_path_cost_factor=1.2 + ) + + paths = list(results.values())[0] + assert [p.cost for p in paths] == [10.0, 11.0] + + @staticmethod + def _many_pair_group_network() -> Network: + """Build 10 sources and 10 sinks through one hub, all pair costs distinct. + + Pair (Ai, Bj) costs (i + 1) + (j + 1) * 100, so the cheapest + pairs are (A0, B0) = 101, (A1, B0) = 102, (A2, B0) = 103, ... + """ + net = Network() + net.add_node(Node("X")) + for i in range(10): + net.add_node(Node(f"A{i}")) + net.add_node(Node(f"B{i}")) + net.add_link(Link(f"A{i}", "X", capacity=10.0, cost=float(i + 1))) + net.add_link(Link("X", f"B{i}", capacity=10.0, cost=float((i + 1) * 100))) + return net + + def test_multi_node_groups_prune_pairs_beyond_kth_best_cost(self) -> None: + """Per-pair KSP stops once later pairs cannot reach the top-k. + + Regression: every reachable node pair previously ran a full KSP + (100 runs here) even though only the cheapest few pairs can + contribute paths that survive the max_k truncation. + """ + net = self._many_pair_group_network() + ctx = analyze(net) + core = ctx._ensure_core() + inner = core._algorithms + calls = {"ksp": 0} + + class CountingAlgorithms: + def __getattr__(self, name: str) -> Any: + return getattr(inner, name) + + def ksp(self, *args: Any, **kwargs: Any) -> Any: + calls["ksp"] += 1 + return inner.ksp(*args, **kwargs) + + core._algorithms = CountingAlgorithms() # type: ignore[assignment] + + results = ctx.k_shortest_paths("^A", "^B", max_k=3, mode=Mode.COMBINE) + + paths = list(results.values())[0] + assert [p.cost for p in paths] == [101.0, 102.0, 103.0] + # 3 KSP runs collect the top-3; the 4th-cheapest pair (cost 104) + # exceeds the k-th best cost (103) and terminates the pair loop. + assert calls["ksp"] <= 4 + + def test_equal_cost_truncation_is_deterministic(self) -> None: + """Equal-cost ties beyond max_k truncate by structural path order. + + Regression: with more equal-cost paths than max_k, truncation + previously kept a set-iteration-order (hash-dependent) subset. + """ + net = Network() + for name in ["S1", "S2", "M1", "M2", "M3", "T1", "T2"]: + net.add_node(Node(name)) + for src in ["S1", "S2"]: + for mid in ["M1", "M2", "M3"]: + net.add_link(Link(src, mid, capacity=10.0, cost=1.0)) + for mid in ["M1", "M2", "M3"]: + for dst in ["T1", "T2"]: + net.add_link(Link(mid, dst, capacity=10.0, cost=1.0)) + + # 12 equal-cost (cost 2) paths exist; max_k=3 forces truncation. + # The smallest 3 by (cost, node sequence) must always win. + for _ in range(2): # Identical across repeated context builds + results = analyze(net).k_shortest_paths( + "^S", "^T", max_k=3, mode=Mode.COMBINE + ) + paths = list(results.values())[0] + assert [p.nodes_seq for p in paths] == [ + ("S1", "M1", "T1"), + ("S1", "M1", "T2"), + ("S1", "M2", "T1"), + ] + + def test_equal_cost_truncation_stable_across_hash_seeds(self) -> None: + """Truncated path selection is identical across PYTHONHASHSEED values. + + Regression: the selected subset of equal-cost paths previously + varied across processes with different string-hash seeds. + """ + script = textwrap.dedent( + """ + from ngraph import Link, Mode, Network, Node, analyze + + net = Network() + for name in ["S1", "S2", "M1", "M2", "M3", "T1", "T2"]: + net.add_node(Node(name)) + for src in ["S1", "S2"]: + for mid in ["M1", "M2", "M3"]: + net.add_link(Link(src, mid, capacity=10.0, cost=1.0)) + for mid in ["M1", "M2", "M3"]: + for dst in ["T1", "T2"]: + net.add_link(Link(mid, dst, capacity=10.0, cost=1.0)) + + results = analyze(net).k_shortest_paths( + "^S", "^T", max_k=3, mode=Mode.COMBINE + ) + paths = list(results.values())[0] + print(sorted(p.nodes_seq for p in paths)) + """ + ) + + outputs = set() + for seed in ("1", "42"): + env = dict(os.environ, PYTHONHASHSEED=seed) + proc = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + env=env, + check=True, + ) + outputs.add(proc.stdout.strip()) + + assert len(outputs) == 1 + class TestDictSelectorsWithShortestPaths: """Tests for dict-based selectors with shortest path methods. diff --git a/tests/analysis/test_placement.py b/tests/analysis/test_placement.py index afb7956..0ecd825 100644 --- a/tests/analysis/test_placement.py +++ b/tests/analysis/test_placement.py @@ -74,8 +74,8 @@ def _run_demand_placement_without_cache( node_mapper = ctx.node_mapper edge_mapper = ctx.edge_mapper algorithms = ctx.algorithms - node_mask = ctx._build_node_mask(set()) - edge_mask = ctx._build_edge_mask(set()) + node_mask = ctx.build_node_mask(set()) + edge_mask = ctx.build_edge_mask(set()) flow_graph = netgraph_core.FlowGraph(multidigraph) @@ -1096,7 +1096,15 @@ def test_cached_equals_noncached( multi_dest_constrained_network: Network, multi_source_multi_dest_network: Network, ) -> None: - """Cached placement must produce identical results to FlowPolicy placement.""" + """Cached placement matches FlowPolicy placement in uncontended cases. + + The equivalence holds when demands do not compete for capacity. Under + contention the cached path is canonical for SHORTEST_PATHS presets: + it admits flow onto the cost-only shortest paths of the base topology + and drops the overflow (IGP semantics), whereas Core's FlowPolicy + reroutes onto costlier residual paths (TE semantics, available via + the TE_* presets). + """ # Select network based on source count if len(sources) > 1: network = multi_source_multi_dest_network @@ -1216,3 +1224,95 @@ def test_te_overlapping_paths(self, overlapping_paths_network: Network) -> None: f"Flow {i} ({cached_flow.source}->{cached_flow.destination}): " f"placed mismatch - cached={cached_flow.placed}, ref={ref_flow.placed}" ) + + +def test_mixed_preset_same_endpoints_no_flow_index_collision() -> None: + """Regression: cached and policy-based demands sharing (src, dst, priority) + must not merge flows via colliding FlowIndex values. Pre-fix this scenario + reported 15.0 placed across a 10-unit min cut.""" + from ngraph.analysis.functions import demand_placement_analysis + from ngraph.model.network import Link, Network, Node + + net = Network() + for n in ("A", "B", "M1", "M2"): + net.add_node(Node(n)) + net.add_link(Link("A", "M1", capacity=5.0, cost=1)) + net.add_link(Link("M1", "B", capacity=5.0, cost=1)) + net.add_link(Link("A", "M2", capacity=5.0, cost=1)) + net.add_link(Link("M2", "B", capacity=5.0, cost=1)) + + cfg = [ + { + "source": "^A$", + "target": "^B$", + "volume": 5.0, + "mode": "pairwise", + "priority": 0, + "flow_policy": "SHORTEST_PATHS_ECMP", + "id": "d1", + }, + { + "source": "^A$", + "target": "^B$", + "volume": 5.0, + "mode": "pairwise", + "priority": 0, + "flow_policy": "TE_ECMP_16_LSP", + "id": "d2", + }, + { + "source": "^A$", + "target": "^B$", + "volume": 5.0, + "mode": "pairwise", + "priority": 1, + "flow_policy": "TE_WCMP_UNLIM", + "id": "d3", + }, + ] + result = demand_placement_analysis( + network=net, excluded_nodes=set(), excluded_links=set(), demands_config=cfg + ) + assert result.summary.total_placed <= 10.0 + 1e-9 + + +def test_duplicate_policy_demand_triple_rejected() -> None: + """Two policy-based demands sharing (src, dst, priority) raise instead of + silently merging/stealing each other's flows.""" + import pytest + + from ngraph.analysis.functions import demand_placement_analysis + from ngraph.model.network import Link, Network, Node + + net = Network() + for n in ("A", "B"): + net.add_node(Node(n)) + net.add_link(Link("A", "B", capacity=10.0, cost=1)) + + cfg = [ + { + "source": "^A$", + "target": "^B$", + "volume": 5.0, + "mode": "pairwise", + "priority": 0, + "flow_policy": "TE_ECMP_16_LSP", + "id": "p1", + }, + { + "source": "^A$", + "target": "^B$", + "volume": 5.0, + "mode": "pairwise", + "priority": 0, + "flow_policy": "TE_ECMP_16_LSP", + "id": "p2", + }, + ] + with pytest.raises(ValueError, match="Duplicate policy-based demand"): + demand_placement_analysis( + network=net, + excluded_nodes=set(), + excluded_links=set(), + demands_config=cfg, + ) diff --git a/tests/analysis/test_profile_env_restore.py b/tests/analysis/test_profile_env_restore.py new file mode 100644 index 0000000..bdd331d --- /dev/null +++ b/tests/analysis/test_profile_env_restore.py @@ -0,0 +1,31 @@ +"""NGRAPH_PROFILE_DIR must be restored even when serial analysis raises.""" + +import os + +import pytest + +from ngraph.analysis.failure_manager import FailureManager +from ngraph.model.failure.policy_set import FailurePolicySet +from ngraph.model.network import Link, Network, Node + + +def _failing_analysis(network, excluded_nodes, excluded_links, **kwargs): + raise RuntimeError("analysis blew up") + + +def test_run_serial_restores_profile_dir_on_exception(tmp_path, monkeypatch): + profile_dir = str(tmp_path / "profiles") + monkeypatch.setenv("NGRAPH_PROFILE_DIR", profile_dir) + + network = Network() + network.add_node(Node("A")) + network.add_node(Node("B")) + network.add_link(Link("A", "B", capacity=1.0, cost=1.0)) + fm = FailureManager(network, FailurePolicySet(), policy_name=None) + + with pytest.raises(RuntimeError, match="analysis blew up"): + fm.run_monte_carlo_analysis( + analysis_func=_failing_analysis, iterations=1, parallelism=1 + ) + + assert os.environ.get("NGRAPH_PROFILE_DIR") == profile_dir diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 2ab00c5..78bd924 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -8,31 +8,6 @@ from ngraph import cli -# Utilities - - -def extract_json_from_stdout(output: str) -> str: - """Return the JSON payload from stdout that may include status lines. - - This helper isolates the first balanced JSON object for reliable parsing. - """ - json_start = output.find("{") - if json_start == -1: - return output - - brace_count = 0 - json_end = -1 - for i in range(json_start, len(output)): - if output[i] == "{": - brace_count += 1 - elif output[i] == "}": - brace_count -= 1 - if brace_count == 0: - json_end = i + 1 - break - return output[json_start:json_end] if json_end != -1 else output - - # High-value CLI run command tests @@ -56,9 +31,11 @@ def test_run_stdout_and_default_results(tmp_path: Path, capsys, monkeypatch) -> cli.main(["run", str(scenario), "--stdout"]) captured = capsys.readouterr() - payload = json.loads(extract_json_from_stdout(captured.out)) + # stdout must be pure JSON (status banners go to stderr) + payload = json.loads(captured.out) assert "steps" in payload and "build_graph" in payload["steps"] + assert "Results written to" in captured.err # default .results.json is created when --results not passed assert (tmp_path / "scenario_1.results.json").exists() @@ -69,11 +46,26 @@ def test_run_no_results_flag_produces_no_file( scenario = Path("tests/integration/scenario_1.yaml").resolve() monkeypatch.chdir(tmp_path) - cli.main(["run", str(scenario), "--no-results"]) # still prints a status line + cli.main(["run", str(scenario), "--no-results"]) # status line goes to stderr captured = capsys.readouterr() assert not (tmp_path / "scenario_1.results.json").exists() - assert "Scenario execution completed" in captured.out + assert "Scenario execution completed" in captured.err + assert captured.out == "" + + +def test_run_stdout_no_results_emits_pure_json( + tmp_path: Path, capsys, monkeypatch +) -> None: + scenario = Path("tests/integration/scenario_1.yaml").resolve() + monkeypatch.chdir(tmp_path) + + cli.main(["run", str(scenario), "--stdout", "--no-results"]) + captured = capsys.readouterr() + + # The full stdout stream must parse as JSON (safe to pipe to jq) + payload = json.loads(captured.out) + assert "steps" in payload and "build_graph" in payload["steps"] def test_run_custom_results_path_disables_default(tmp_path: Path, monkeypatch) -> None: diff --git a/tests/cli/test_cli_helpers.py b/tests/cli/test_cli_helpers.py index 4edbe2d..26c92f8 100644 --- a/tests/cli/test_cli_helpers.py +++ b/tests/cli/test_cli_helpers.py @@ -30,7 +30,7 @@ def __init__(self, disabled: bool) -> None: return {"G1": [N(False), N(True)], "G2": [N(False)]} -def test_format_table_and_plural() -> None: +def test_format_table() -> None: table = cli_mod._format_table( ["H1", "H2"], [["abc", "1"], ["defghi", "2"]], max_col_width=5 ) @@ -38,10 +38,6 @@ def test_format_table_and_plural() -> None: # Ensure clipping with ASCII ellipsis (max_col_width=5 -> keep 2 chars + '...') assert "de..." in table - assert cli_mod._plural(1, "node") == "node" - assert cli_mod._plural(2, "node") == "nodes" - assert cli_mod._plural(2, "node", "vertices") == "vertices" - def test_collect_and_summarize_node_matches() -> None: step = DummyStep() diff --git a/tests/cli/test_cli_inspect_fixes.py b/tests/cli/test_cli_inspect_fixes.py new file mode 100644 index 0000000..e8f991e --- /dev/null +++ b/tests/cli/test_cli_inspect_fixes.py @@ -0,0 +1,158 @@ +"""Regression tests for `ngraph inspect` output fixes. + +Covers: +- Single-pass per-node capacity/link-count aggregation in + ``_print_network_structure`` (previously O(V*E) nested scans), including + self-loop semantics. +- "Top demands (by offered volume)" sorting by the ``volume`` attribute + (previously keyed on a nonexistent ``demand`` attribute). +""" + +from __future__ import annotations + +import io +from pathlib import Path +from unittest.mock import patch + +from ngraph import cli +from ngraph.model.components import ComponentsLibrary +from ngraph.model.network import Link, Network, Node + + +def _capture_print(func, *args, **kwargs) -> str: + """Run func with print patched and return the joined printed output.""" + with patch("sys.stdout", new=io.StringIO()), patch("builtins.print") as mprint: + func(*args, **kwargs) + return "\n".join(str(c.args[0]) for c in mprint.call_args_list if c.args) + + +def _table_rows(output: str, heading: str) -> list[list[str]]: + """Extract table rows (split on '|') printed after a heading line.""" + lines = output.split("\n") + start = next(i for i, ln in enumerate(lines) if ln.strip() == heading) + rows: list[list[str]] = [] + in_table = False + for ln in lines[start + 1 :]: + stripped = ln.strip() + if not stripped or set(stripped) <= {"-", "+"}: + if in_table and not stripped: + break + continue # blank before table or separator row + if "|" in ln: + in_table = True + rows.append([cell.strip() for cell in ln.split("|")]) + elif in_table: + break + return rows + + +def _build_network_with_self_loop() -> Network: + net = Network() + for name in ("A", "B", "C"): + net.add_node(Node(name=name)) + net.add_link(Link(source="A", target="B", capacity=10.0)) + net.add_link(Link(source="B", target="C", capacity=20.0)) + # Self-loop must contribute its capacity and link count exactly once + net.add_link(Link(source="A", target="A", capacity=5.0)) + # Disabled links are excluded from capacity/link-count aggregation + net.add_link(Link(source="A", target="C", capacity=100.0, disabled=True)) + return net + + +def test_node_table_capacity_and_link_counts_single_pass() -> None: + net = _build_network_with_self_loop() + out = _capture_print( + cli._print_network_structure, net, ComponentsLibrary(), detail=True + ) + + rows = _table_rows(out, "Nodes:") + # Drop header row + data = {r[0]: r for r in rows if r[0] in ("A", "B", "C")} + # A: 10 (A-B) + 5 (self-loop counted once); 2 enabled links + assert data["A"][2] == "15" + assert data["A"][3] == "2" + # B: 10 (A-B) + 20 (B-C); 2 enabled links + assert data["B"][2] == "30" + assert data["B"][3] == "2" + # C: 20 (B-C); the disabled A-C link must not count + assert data["C"][2] == "20" + assert data["C"][3] == "1" + + +def test_node_capacity_statistics_match_naive_aggregation() -> None: + net = _build_network_with_self_loop() + out = _capture_print( + cli._print_network_structure, net, ComponentsLibrary(), detail=False + ) + + rows = _table_rows(out, "Node Capacity Statistics:") + stats = {r[0]: r[1] for r in rows if len(r) >= 2} + # Per-node capacities: A=15, B=30, C=20 + assert stats["Min"] == "15.0" + assert stats["Max"] == "30.0" + assert stats["Mean"] == "21.7" + assert stats["Median"] == "20.0" + assert stats["Total"] == "65.0" + + +def test_node_capacity_statistics_exclude_isolated_nodes() -> None: + net = Network() + for name in ("A", "B", "ISOLATED"): + net.add_node(Node(name=name)) + net.add_link(Link(source="A", target="B", capacity=8.0)) + + out = _capture_print( + cli._print_network_structure, net, ComponentsLibrary(), detail=False + ) + + rows = _table_rows(out, "Node Capacity Statistics:") + stats = {r[0]: r[1] for r in rows if len(r) >= 2} + # Only A and B have enabled links; ISOLATED must not drag Min to 0 + assert stats["Min"] == "8.0" + assert stats["Total"] == "16.0" + + +def test_inspect_top_demands_sorted_by_volume(tmp_path: Path) -> None: + scenario_file = tmp_path / "top_demands.yaml" + scenario_file.write_text( + """ +seed: 1 +network: + nodes: + A: {} + B: {} + C: {} + links: + - source: A + target: B + capacity: 1000 + - source: B + target: C + capacity: 1000 +demands: + default: + - source: "^A$" + target: "^B$" + volume: 10 + - source: "^B$" + target: "^C$" + volume: 300 + - source: "^A$" + target: "^C$" + volume: 50 +workflow: + - type: BuildGraph +""" + ) + + with patch("sys.stdout", new=io.StringIO()), patch("builtins.print") as mprint: + cli.main(["inspect", str(scenario_file), "--detail"]) + + out = "\n".join(str(c.args[0]) for c in mprint.call_args_list if c.args) + assert "Top demands (by offered volume):" in out + + rows = _table_rows(out, "Top demands (by offered volume):") + offered = [ + float(r[2].replace(",", "")) for r in rows if r[2] not in ("Offered", "") + ] + assert offered == [300.0, 50.0, 10.0] diff --git a/tests/cli/test_cli_profile_hook.py b/tests/cli/test_cli_profile_hook.py new file mode 100644 index 0000000..81c28c1 --- /dev/null +++ b/tests/cli/test_cli_profile_hook.py @@ -0,0 +1,69 @@ +"""Tests for the CLI --profile path driven through Scenario.run step hooks.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from ngraph import cli + +_SCENARIO_YAML = """ +seed: 1 +network: + nodes: + A: {} + B: {} + links: + - source: A + target: B + capacity: 1 +workflow: + - type: NetworkStats + name: stats +""" + + +def test_run_profile_prints_performance_report( + tmp_path: Path, monkeypatch, capsys +) -> None: + """``ngraph run --profile`` still emits the per-step performance report.""" + scenario_file = tmp_path / "p.yaml" + scenario_file.write_text(_SCENARIO_YAML) + monkeypatch.chdir(tmp_path) + + cli.main(["run", str(scenario_file), "--profile", "--no-results"]) + + captured = capsys.readouterr() + # The report is human-facing output and goes to stderr; stdout stays + # reserved for machine-readable results. + assert "NETGRAPH PERFORMANCE PROFILING REPORT" in captured.err + # Per-step profiling was driven through Scenario.run's step hook + assert "stats" in captured.err + + +def test_run_profile_restores_profile_dir_env(tmp_path: Path, monkeypatch) -> None: + """NGRAPH_PROFILE_DIR is restored after a --profile run completes.""" + scenario_file = tmp_path / "p.yaml" + scenario_file.write_text(_SCENARIO_YAML) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("NGRAPH_PROFILE_DIR", raising=False) + + cli.main(["run", str(scenario_file), "--profile", "--no-results"]) + + # Previously unset, so it must be removed (not left pointing at a stale + # directory that would silently re-enable worker profiling later). + assert "NGRAPH_PROFILE_DIR" not in os.environ + + +def test_run_profile_restores_preexisting_profile_dir_env( + tmp_path: Path, monkeypatch +) -> None: + """A pre-existing NGRAPH_PROFILE_DIR value is restored after --profile.""" + scenario_file = tmp_path / "p.yaml" + scenario_file.write_text(_SCENARIO_YAML) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("NGRAPH_PROFILE_DIR", "/tmp/preexisting-profile-dir") + + cli.main(["run", str(scenario_file), "--profile", "--no-results"]) + + assert os.environ["NGRAPH_PROFILE_DIR"] == "/tmp/preexisting-profile-dir" diff --git a/tests/cli/test_package_layering.py b/tests/cli/test_package_layering.py new file mode 100644 index 0000000..1bf733c --- /dev/null +++ b/tests/cli/test_package_layering.py @@ -0,0 +1,22 @@ +"""Tests for package-level import layering. + +The package root must not eagerly import the CLI module: the console entry +point (``ngraph.cli:main``) and ``python -m ngraph`` import it explicitly. +""" + +from __future__ import annotations + +import subprocess +import sys + + +def test_import_ngraph_does_not_import_cli() -> None: + """Importing the package root must not pull ngraph.cli into sys.modules.""" + code = "import sys, ngraph; assert 'ngraph.cli' not in sys.modules" + subprocess.run([sys.executable, "-c", code], check=True) + + +def test_from_ngraph_import_cli_still_works() -> None: + """``from ngraph import cli`` resolves via submodule import fallback.""" + code = "from ngraph import cli; assert callable(cli.main)" + subprocess.run([sys.executable, "-c", code], check=True) diff --git a/tests/dsl/test_dsl_features_validation.py b/tests/dsl/test_dsl_features_validation.py index 036c98d..de8fba1 100644 --- a/tests/dsl/test_dsl_features_validation.py +++ b/tests/dsl/test_dsl_features_validation.py @@ -3,6 +3,9 @@ These tests verify the behavior of DSL features to ensure documentation accuracy. """ +import jsonschema +import pytest + from ngraph.scenario import Scenario @@ -326,8 +329,8 @@ def test_flow_policy_preset_string(self): assert demands[0].flow_policy == FlowPolicyPreset.SHORTEST_PATHS_ECMP - def test_flow_policy_inline_object_preserved(self): - """Inline object flow_policy should be preserved (not converted to preset).""" + def test_flow_policy_inline_object_rejected(self): + """Inline object flow_policy is rejected; only preset names/ints work.""" yaml_str = """ network: nodes: @@ -345,14 +348,8 @@ def test_flow_policy_inline_object_preserved(self): path_alg: SPF flow_placement: PROPORTIONAL """ - scenario = Scenario.from_yaml(yaml_str) - demands = scenario.demand_set.sets.get("test", []) - assert len(demands) == 1 - # Inline object should be preserved as dict - fp = demands[0].flow_policy - assert isinstance(fp, dict), f"Expected dict, got {type(fp)}" - assert fp.get("path_alg") == "SPF" - assert fp.get("flow_placement") == "PROPORTIONAL" + with pytest.raises((jsonschema.ValidationError, ValueError)): + Scenario.from_yaml(yaml_str) # Run with: pytest tests/dsl/test_dsl_features_validation.py -v diff --git a/tests/dsl/test_examples.py b/tests/dsl/test_examples.py index 5d6e97c..bcc5671 100644 --- a/tests/dsl/test_examples.py +++ b/tests/dsl/test_examples.py @@ -306,7 +306,6 @@ def test_failure_policy_example(): failures: default: expand_groups: true - expand_children: false attrs: custom_key: "value" modes: @@ -328,7 +327,6 @@ def test_failure_policy_example(): assert len(policies) > 0 default_policy = scenario.failure_policy_set.get_policy("default") assert default_policy.expand_groups - assert not default_policy.expand_children assert len(default_policy.modes) == 1 mode = default_policy.modes[0] assert len(mode.rules) == 1 diff --git a/tests/dsl/test_expand_review_fixes.py b/tests/dsl/test_expand_review_fixes.py new file mode 100644 index 0000000..f763733 --- /dev/null +++ b/tests/dsl/test_expand_review_fixes.py @@ -0,0 +1,481 @@ +"""Regression tests for DSL blueprint expansion review fixes. + +Covers: +- Full variable substitution in link expand blocks (attrs, risk_groups, + selector match values), not just source/target path strings. +- Regex-escaping of literal parent paths when joining blueprint link + selectors (metacharacters in group names must not leak into the regex). +- Validation of blueprint params override keys (typos and unsupported + deep dotted 'params.*' paths raise instead of silently no-opping). +- link_rules requiring source/target at both schema and expansion level. +- node_rules/link_rules of the wrong type raising instead of being ignored. +- Deterministic (sorted) risk_groups in flattened node/link attrs. +- Per-expansion mesh reversed-pair dedup semantics (pinned behavior). +""" + +import jsonschema +import pytest + +from ngraph.dsl.blueprints.expand import expand_network_dsl +from ngraph.dsl.loader import load_scenario_yaml +from ngraph.dsl.selectors import flatten_link_attrs, flatten_node_attrs +from ngraph.model.network import Link, Node + +# ────────────────────────────────────────────────────────────────────────────── +# Link expand block: full variable substitution +# ────────────────────────────────────────────────────────────────────────────── + + +class TestLinkExpandFullSubstitution: + """Variables in a link expand block substitute into the whole definition.""" + + def test_attrs_are_substituted_per_combination(self) -> None: + """Link attrs receive per-combination values, not literal templates.""" + data = { + "network": { + "nodes": { + "dc1": {"nodes": {"gw": {}}}, + "dc2": {"nodes": {"gw": {}}}, + "hub": {}, + }, + "links": [ + { + "source": "${dc}/gw", + "target": "hub", + "expand": {"vars": {"dc": ["dc1", "dc2"]}}, + "attrs": { + "datacenter": "${dc}", + "ring": "${dc}_internal", + }, + } + ], + } + } + net = expand_network_dsl(data) + + assert len(net.links) == 2 + by_source = {link.source: link for link in net.links.values()} + assert by_source["dc1/gw"].attrs["datacenter"] == "dc1" + assert by_source["dc1/gw"].attrs["ring"] == "dc1_internal" + assert by_source["dc2/gw"].attrs["datacenter"] == "dc2" + assert by_source["dc2/gw"].attrs["ring"] == "dc2_internal" + + def test_risk_groups_are_substituted(self) -> None: + """Link risk_groups receive per-combination values.""" + data = { + "network": { + "nodes": { + "dc1": {"nodes": {"gw": {}}}, + "dc2": {"nodes": {"gw": {}}}, + "hub": {}, + }, + "links": [ + { + "source": "${dc}/gw", + "target": "hub", + "expand": {"vars": {"dc": ["dc1", "dc2"]}}, + "risk_groups": ["RG_${dc}"], + } + ], + } + } + net = expand_network_dsl(data) + + by_source = {link.source: link for link in net.links.values()} + assert by_source["dc1/gw"].risk_groups == {"RG_dc1"} + assert by_source["dc2/gw"].risk_groups == {"RG_dc2"} + + def test_match_only_selector_is_substituted(self) -> None: + """A dict selector with only 'match' honors the expand block.""" + data = { + "network": { + "nodes": { + "gw1": {"attrs": {"dc": "dc1"}}, + "gw2": {"attrs": {"dc": "dc2"}}, + "hub": {}, + }, + "links": [ + { + "source": { + "match": { + "conditions": [ + {"attr": "dc", "op": "==", "value": "${d}"} + ] + } + }, + "target": "hub", + "expand": {"vars": {"d": ["dc1", "dc2"]}}, + } + ], + } + } + net = expand_network_dsl(data) + + sources = sorted(link.source for link in net.links.values()) + assert sources == ["gw1", "gw2"] + + def test_match_value_keeps_native_type_for_int_attr(self) -> None: + """A whole-placeholder match value compares as int against int attrs.""" + data = { + "network": { + "nodes": { + "gw1": {"attrs": {"tier": 2}}, + "gw2": {"attrs": {"tier": 1}}, + "hub": {}, + }, + "links": [ + { + "source": { + "match": { + "conditions": [ + {"attr": "tier", "op": "==", "value": "${t}"} + ] + } + }, + "target": "hub", + "expand": {"vars": {"t": [2]}}, + } + ], + } + } + net = expand_network_dsl(data) + + sources = sorted(link.source for link in net.links.values()) + assert sources == ["gw1"] + + def test_node_rule_match_value_keeps_native_type(self) -> None: + """node_rules with '${t}' match values apply attrs to int-attr nodes.""" + data = { + "network": { + "nodes": { + "n1": {"attrs": {"tier": 2}}, + "n2": {"attrs": {"tier": 1}}, + }, + "node_rules": [ + { + "path": ".*", + "match": { + "conditions": [ + {"attr": "tier", "op": "==", "value": "${t}"} + ] + }, + "attrs": {"selected_tier": "${t}"}, + "expand": {"vars": {"t": [2]}}, + } + ], + } + } + net = expand_network_dsl(data) + + assert net.nodes["n1"].attrs.get("selected_tier") == 2 + assert "selected_tier" not in net.nodes["n2"].attrs + + def test_mesh_dedup_is_per_expansion_combination(self) -> None: + """Each variable combination is an independent link definition. + + Cartesian expansion over symmetric variable lists produces both + orientations as separate parallel links (documented semantics). + """ + data = { + "network": { + "nodes": { + "dc1": {"nodes": {"gw": {}}}, + "dc2": {"nodes": {"gw": {}}}, + }, + "links": [ + { + "source": "dc${a}/gw", + "target": "dc${b}/gw", + "expand": {"vars": {"a": [1, 2], "b": [1, 2]}}, + } + ], + } + } + net = expand_network_dsl(data) + + pairs = sorted((link.source, link.target) for link in net.links.values()) + assert pairs == [("dc1/gw", "dc2/gw"), ("dc2/gw", "dc1/gw")] + + +# ────────────────────────────────────────────────────────────────────────────── +# Parent path regex escaping in blueprint links +# ────────────────────────────────────────────────────────────────────────────── + + +class TestBlueprintParentPathEscaping: + """Literal parent paths are escaped before regex compilation.""" + + BLUEPRINT = { + "nodes": { + "leaf": {"count": 2, "template": "leaf-{n}"}, + "spine": {"count": 2, "template": "spine-{n}"}, + }, + "links": [{"source": "/leaf", "target": "/spine", "pattern": "mesh"}], + } + + def test_dot_in_group_name_does_not_match_siblings(self) -> None: + """A '.' in a group name must not act as a regex wildcard.""" + data = { + "blueprints": {"bp": self.BLUEPRINT}, + "network": { + "nodes": { + "dc.1": {"blueprint": "bp"}, + "dcX1": {"blueprint": "bp"}, + } + }, + } + net = expand_network_dsl(data) + + # 2x2 mesh inside each of the two instances; no cross-subtree links. + assert len(net.links) == 8 + for link in net.links.values(): + src_root = link.source.split("/")[0] + tgt_root = link.target.split("/")[0] + assert src_root == tgt_root + + def test_plus_in_group_name_still_creates_links(self) -> None: + """A '+' in a group name must not break blueprint link selection.""" + data = { + "blueprints": {"bp": self.BLUEPRINT}, + "network": {"nodes": {"agg+core": {"blueprint": "bp"}}}, + } + net = expand_network_dsl(data) + + assert len(net.links) == 4 + for link in net.links.values(): + assert link.source.startswith("agg+core/") + assert link.target.startswith("agg+core/") + + +# ────────────────────────────────────────────────────────────────────────────── +# Blueprint params override validation +# ────────────────────────────────────────────────────────────────────────────── + + +class TestBlueprintParamsValidation: + """params override keys must address an existing blueprint subgroup.""" + + def test_typo_in_subgroup_prefix_raises(self) -> None: + data = { + "blueprints": { + "bp": {"nodes": {"spine": {"count": 2, "template": "s-{n}"}}} + }, + "network": { + "nodes": {"pod1": {"blueprint": "bp", "params": {"spnie.count": 8}}} + }, + } + with pytest.raises(ValueError, match="matches no node group"): + expand_network_dsl(data) + + def test_key_without_field_part_raises(self) -> None: + data = { + "blueprints": { + "bp": {"nodes": {"spine": {"count": 2, "template": "s-{n}"}}} + }, + "network": {"nodes": {"pod1": {"blueprint": "bp", "params": {"spine": 8}}}}, + } + with pytest.raises(ValueError, match="must be of the form"): + expand_network_dsl(data) + + def test_deep_dotted_nested_params_raises(self) -> None: + """'group.params.sub.field' no longer silently no-ops.""" + data = { + "blueprints": { + "inner": {"nodes": {"spine": {"count": 2, "template": "s-{n}"}}}, + "outer": {"nodes": {"pod1": {"blueprint": "inner"}}}, + }, + "network": { + "nodes": { + "site": { + "blueprint": "outer", + "params": {"pod1.params.spine.count": 8}, + } + } + }, + } + with pytest.raises(ValueError, match="must be of the form"): + expand_network_dsl(data) + + def test_dict_valued_nested_params_pass_through_works(self) -> None: + """The supported dict-valued form overrides nested blueprint params.""" + data = { + "blueprints": { + "inner": {"nodes": {"spine": {"count": 2, "template": "s-{n}"}}}, + "outer": {"nodes": {"pod1": {"blueprint": "inner"}}}, + }, + "network": { + "nodes": { + "site": { + "blueprint": "outer", + "params": {"pod1.params": {"spine.count": 4}}, + } + } + }, + } + net = expand_network_dsl(data) + + spines = [n for n in net.nodes if n.startswith("site/pod1/spine/")] + assert len(spines) == 4 + + def test_dotted_subgroup_name_override_applies(self) -> None: + """Overrides address subgroup names that themselves contain dots.""" + data = { + "blueprints": { + "bp": {"nodes": {"rack.a": {"count": 1, "template": "n-{n}"}}} + }, + "network": { + "nodes": {"pod1": {"blueprint": "bp", "params": {"rack.a.count": 3}}} + }, + } + net = expand_network_dsl(data) + + rack_nodes = [n for n in net.nodes if n.startswith("pod1/rack.a/")] + assert len(rack_nodes) == 3 + + def test_dotted_subgroup_longest_prefix_wins(self) -> None: + """When one group name extends another, the longest prefix applies.""" + data = { + "blueprints": { + "bp": { + "nodes": { + "rack": {"count": 1, "template": "r-{n}"}, + "rack.a": {"count": 1, "template": "a-{n}"}, + } + } + }, + "network": { + "nodes": { + "pod1": { + "blueprint": "bp", + "params": {"rack.a.count": 3, "rack.count": 2}, + } + } + }, + } + net = expand_network_dsl(data) + + rack_a_nodes = [n for n in net.nodes if n.startswith("pod1/rack.a/")] + rack_nodes = [n for n in net.nodes if n.startswith("pod1/rack/")] + assert len(rack_a_nodes) == 3 + assert len(rack_nodes) == 2 + + def test_key_equal_to_dotted_group_name_raises_form_error(self) -> None: + """A key naming a dotted group without a field raises the form error.""" + data = { + "blueprints": { + "bp": {"nodes": {"rack.a": {"count": 1, "template": "n-{n}"}}} + }, + "network": { + "nodes": {"pod1": {"blueprint": "bp", "params": {"rack.a": 3}}} + }, + } + with pytest.raises(ValueError, match="must be of the form"): + expand_network_dsl(data) + + def test_valid_override_with_bracket_pattern_group(self) -> None: + """Overrides keyed on literal (unexpanded) bracket names stay valid.""" + data = { + "blueprints": { + "bp": {"nodes": {"plane[1-2]": {"count": 1, "template": "n-{n}"}}} + }, + "network": { + "nodes": { + "pod1": {"blueprint": "bp", "params": {"plane[1-2].count": 2}} + } + }, + } + net = expand_network_dsl(data) + + plane_nodes = [n for n in net.nodes if "/plane" in n] + assert len(plane_nodes) == 4 + + +# ────────────────────────────────────────────────────────────────────────────── +# link_rules source/target requirement +# ────────────────────────────────────────────────────────────────────────────── + + +class TestLinkRulesSourceTargetRequired: + """link_rules entries must declare both source and target.""" + + def test_schema_rejects_link_rule_without_source_target(self) -> None: + yaml_content = """ +network: + nodes: + A: {} + B: {} + links: + - source: A + target: B + link_rules: + - capacity: 99 +""" + with pytest.raises(jsonschema.ValidationError): + load_scenario_yaml(yaml_content) + + def test_expansion_raises_value_error_not_key_error(self) -> None: + data = { + "network": { + "nodes": {"A": {}, "B": {}}, + "links": [{"source": "A", "target": "B"}], + "link_rules": [{"capacity": 99}], + } + } + with pytest.raises(ValueError, match="'source' and 'target'"): + expand_network_dsl(data) + + +# ────────────────────────────────────────────────────────────────────────────── +# Malformed rules sections raise +# ────────────────────────────────────────────────────────────────────────────── + + +class TestMalformedRulesSectionsRaise: + """Non-list node_rules/link_rules raise instead of being ignored.""" + + def test_node_rules_mapping_raises(self) -> None: + data = { + "network": { + "nodes": {"A": {}}, + "node_rules": {"path": "A"}, + } + } + with pytest.raises(ValueError, match="'node_rules' must be a list"): + expand_network_dsl(data) + + def test_link_rules_mapping_raises(self) -> None: + data = { + "network": { + "nodes": {"A": {}, "B": {}}, + "links": [{"source": "A", "target": "B"}], + "link_rules": {"source": "A", "target": "B"}, + } + } + with pytest.raises(ValueError, match="'link_rules' must be a list"): + expand_network_dsl(data) + + def test_absent_rules_sections_are_noop(self) -> None: + data = {"network": {"nodes": {"A": {}}}} + net = expand_network_dsl(data) + assert "A" in net.nodes + + +# ────────────────────────────────────────────────────────────────────────────── +# Deterministic flattened risk_groups +# ────────────────────────────────────────────────────────────────────────────── + + +class TestFlattenedRiskGroupsSorted: + """flatten_node_attrs/flatten_link_attrs expose sorted risk_groups.""" + + def test_node_risk_groups_sorted(self) -> None: + node = Node("n1") + node.risk_groups = {"rg1", "rg3", "beta", "alpha", "rg2"} + attrs = flatten_node_attrs(node) + assert attrs["risk_groups"] == ["alpha", "beta", "rg1", "rg2", "rg3"] + + def test_link_risk_groups_sorted(self) -> None: + link = Link(source="a", target="b") + link.risk_groups = {"zeta", "alpha", "mid"} + attrs = flatten_link_attrs(link, "link-id-1") + assert attrs["risk_groups"] == ["alpha", "mid", "zeta"] diff --git a/tests/dsl/test_expansion.py b/tests/dsl/test_expansion.py index 209794d..83e8a73 100644 --- a/tests/dsl/test_expansion.py +++ b/tests/dsl/test_expansion.py @@ -2,7 +2,6 @@ Tests for ngraph.dsl.expansion modules: - ExpansionSpec: schema for expansion configuration -- expand_templates: variable substitution in templates - substitute_vars: single template substitution - expand_name_patterns: bracket expansion for names """ @@ -13,7 +12,6 @@ ExpansionSpec, expand_name_patterns, expand_risk_group_refs, - expand_templates, substitute_vars, ) @@ -91,96 +89,28 @@ def test_underscore_in_var_name(self) -> None: result = substitute_vars("${my_var}", {"my_var": "value"}) assert result == "value" - -# ────────────────────────────────────────────────────────────────────────────── -# expand_templates Tests -# ────────────────────────────────────────────────────────────────────────────── - - -class TestExpandTemplatesCartesian: - """Tests for expand_templates with cartesian mode.""" - - def test_single_var_expands(self) -> None: - """Single variable expands to multiple results.""" - spec = ExpansionSpec(vars={"dc": [1, 2, 3]}) - results = list(expand_templates({"path": "dc${dc}"}, spec)) - - assert len(results) == 3 - assert results[0] == {"path": "dc1"} - assert results[1] == {"path": "dc2"} - assert results[2] == {"path": "dc3"} - - def test_multiple_vars_cartesian(self) -> None: - """Multiple variables create cartesian product.""" - spec = ExpansionSpec(vars={"dc": [1, 2], "rack": ["a", "b"]}) - results = list(expand_templates({"path": "dc${dc}_rack${rack}"}, spec)) - - assert len(results) == 4 # 2 * 2 - paths = [r["path"] for r in results] - assert "dc1_racka" in paths - assert "dc1_rackb" in paths - assert "dc2_racka" in paths - assert "dc2_rackb" in paths - - def test_multiple_templates(self) -> None: - """Multiple template fields are all expanded.""" - spec = ExpansionSpec(vars={"dc": [1, 2]}) - results = list( - expand_templates( - {"source": "dc${dc}/leaf", "target": "dc${dc}/spine"}, spec - ) + def test_whole_string_placeholder_preserves_type(self) -> None: + """A string that is exactly one placeholder keeps the native type.""" + result = substitute_vars("${num}", {"num": 2}) + assert result == 2 + assert isinstance(result, int) + assert substitute_vars("$rate", {"rate": 1.5}) == 1.5 + + def test_embedded_placeholder_stringifies(self) -> None: + """A placeholder embedded in a longer string is interpolated as text.""" + assert substitute_vars("dc${num}", {"num": 2}) == "dc2" + + def test_whole_string_placeholder_in_nested_structure(self) -> None: + """Type preservation applies recursively in dicts and lists.""" + result = substitute_vars( + {"value": "${t}", "items": ["${t}", "tier${t}"]}, {"t": 2} ) + assert result == {"value": 2, "items": [2, "tier2"]} - assert len(results) == 2 - assert results[0] == {"source": "dc1/leaf", "target": "dc1/spine"} - assert results[1] == {"source": "dc2/leaf", "target": "dc2/spine"} - - def test_empty_vars_yields_original(self) -> None: - """Empty expand_vars yields original template.""" - spec = ExpansionSpec() - results = list(expand_templates({"path": "static"}, spec)) - - assert len(results) == 1 - assert results[0] == {"path": "static"} - - -class TestExpandTemplatesZip: - """Tests for expand_templates with zip mode.""" - - def test_zip_pairs_by_index(self) -> None: - """Zip mode pairs variables by index.""" - spec = ExpansionSpec(vars={"src": ["a", "b"], "dst": ["x", "y"]}, mode="zip") - results = list(expand_templates({"path": "${src}->${dst}"}, spec)) - - assert len(results) == 2 - assert results[0] == {"path": "a->x"} - assert results[1] == {"path": "b->y"} - - def test_zip_mismatched_lengths_raises(self) -> None: - """Zip mode with mismatched list lengths raises.""" - spec = ExpansionSpec( - vars={"src": ["a", "b"], "dst": ["x", "y", "z"]}, - mode="zip", - ) - with pytest.raises(ValueError, match="equal-length"): - list(expand_templates({"path": "${src}->${dst}"}, spec)) - - -class TestExpandTemplatesLimits: - """Tests for expansion limits.""" - - def test_large_expansion_raises(self) -> None: - """Expansion exceeding limit raises.""" - # Create vars that would produce > 10,000 combinations - spec = ExpansionSpec( - vars={ - "a": list(range(50)), - "b": list(range(50)), - "c": list(range(50)), - } - ) - with pytest.raises(ValueError, match="limit"): - list(expand_templates({"path": "${a}${b}${c}"}, spec)) + def test_whole_string_placeholder_missing_raises(self) -> None: + """Missing variable raises even for whole-string placeholders.""" + with pytest.raises(KeyError, match="not found"): + substitute_vars("${missing}", {"x": 1}) # ────────────────────────────────────────────────────────────────────────────── diff --git a/tests/dsl/test_native_substitution_guards.py b/tests/dsl/test_native_substitution_guards.py new file mode 100644 index 0000000..e6fa430 --- /dev/null +++ b/tests/dsl/test_native_substitution_guards.py @@ -0,0 +1,93 @@ +"""Guards for non-string values leaking out of native-type variable substitution. + +Whole-string ``${var}`` placeholders substitute the variable's native type +(so match conditions compare correctly against numeric attrs). String-only +positions must reject non-string values with a clear ValueError instead of +crashing later with a context-free TypeError. +""" + +import pytest + +from ngraph.dsl.blueprints.expand import expand_network_dsl +from ngraph.dsl.expansion.brackets import expand_risk_group_refs +from ngraph.dsl.selectors import normalize_selector +from ngraph.model.demand.builder import build_demand_set +from ngraph.model.selectors import NodeSelector + + +class TestSelectorPathGuard: + """Selector 'path' must be a string.""" + + def test_node_selector_rejects_non_string_path(self): + with pytest.raises(ValueError, match="Selector 'path' must be a string"): + NodeSelector(path=1) # type: ignore[arg-type] + + def test_normalize_selector_dict_rejects_non_string_path(self): + with pytest.raises(ValueError, match="Selector 'path' must be a string"): + normalize_selector({"path": 1}, "demand") + + def test_node_rule_native_var_path_raises_value_error(self): + """A bare placeholder bound to an int var fails loudly, not TypeError.""" + data = { + "nodes": {"A": {}}, + "node_rules": [ + { + "path": "${p}", + "attrs": {"tier": 1}, + "expand": {"vars": {"p": [1, 2]}}, + } + ], + } + with pytest.raises(ValueError, match="Selector 'path' must be a string"): + expand_network_dsl({"network": data}) + + +class TestRiskGroupRefGuard: + """Risk group references must be strings.""" + + def test_non_string_ref_rejected(self): + with pytest.raises(ValueError, match="Risk group reference must be a string"): + expand_risk_group_refs(["RG1", 1]) # type: ignore[list-item] + + def test_native_var_risk_group_raises_value_error(self): + """An int-typed var in a link risk_groups list fails loudly.""" + data = { + "nodes": {"A": {}, "B": {}}, + "links": [ + { + "source": "A", + "target": "B", + "risk_groups": ["${rg}"], + "expand": {"vars": {"rg": [1]}}, + } + ], + } + with pytest.raises(ValueError, match="Risk group reference must be a string"): + expand_network_dsl({"network": data}) + + +class TestDemandSourceTargetGuard: + """Demand source/target must be a string or selector dict at build time.""" + + def test_int_source_rejected_with_set_context(self): + with pytest.raises( + ValueError, match="Demand 'source' in set 'tm' must be a string" + ): + build_demand_set({"tm": [{"source": 1, "target": "^B$"}]}) + + def test_native_var_source_rejected_at_build(self): + """Expansion substituting an int source fails at build, not analysis.""" + demands = { + "tm": [ + { + "source": "${s}", + "target": "^B$", + "volume": 1.0, + "expand": {"vars": {"s": [1]}}, + } + ] + } + with pytest.raises( + ValueError, match="Demand 'source' in set 'tm' must be a string" + ): + build_demand_set(demands) diff --git a/tests/dsl/test_selectors.py b/tests/dsl/test_selectors.py index f002ed0..891eb7d 100644 --- a/tests/dsl/test_selectors.py +++ b/tests/dsl/test_selectors.py @@ -329,21 +329,6 @@ def test_active_only_false_includes_disabled( all_nodes = [n for nodes in groups.values() for n in nodes] assert len(all_nodes) == 3 # Includes disabled dc2_leaf_2 - def test_excluded_nodes_always_excluded(self, attributed_network: Network) -> None: - """excluded_nodes parameter always excludes specified nodes.""" - sel = NodeSelector(path="^dc1_.*") - groups = select_nodes( - attributed_network, - sel, - default_active_only=False, - excluded_nodes={"dc1_leaf_1"}, - ) - - all_nodes = [n for nodes in groups.values() for n in nodes] - node_names = [n.name for n in all_nodes] - assert "dc1_leaf_1" not in node_names - assert len(all_nodes) == 2 - class TestSelectNodesByGroupBy: """Tests for group_by attribute grouping.""" diff --git a/tests/dsl/test_skill_examples_validation.py b/tests/dsl/test_skill_examples_validation.py index d07ded7..36a2e35 100644 --- a/tests/dsl/test_skill_examples_validation.py +++ b/tests/dsl/test_skill_examples_validation.py @@ -703,7 +703,6 @@ def test_example_11_advanced_failures(): failures: mixed_failures: expand_groups: true - expand_children: false modes: # 40% chance: fail 1 edge node weighted by capacity - weight: 0.4 @@ -769,7 +768,6 @@ def test_example_11_advanced_failures(): policy = scenario.failure_policy_set.get_policy("mixed_failures") assert len(policy.modes) == 4, f"Expected 4 modes, got {len(policy.modes)}" assert policy.expand_groups is True - assert policy.expand_children is False # ============================================================================= @@ -1047,11 +1045,13 @@ def test_example_15_demand_variables(): def test_example_16_hierarchical_risk_groups(): """Example 16: Hierarchical Risk Groups - nested risk group structure. - Expected: Hierarchical risk groups with children, recursive failure expansion. + Expected: Hierarchical risk groups with children. A failed parent group + always cascades to its children downstream (cascading is inherent to the + hierarchy; no policy flag controls it). - Note: Nodes must reference risk groups defined at the top level. Child groups - are used for hierarchical failure expansion (expand_children: true) but nodes - reference the leaf-level groups which must be defined at the top level. + Note: Nodes must reference risk groups defined at the top level. Child + groups cascade on parent failure but nodes reference the leaf-level groups + which must be defined at the top level. """ yaml_content = """ network: @@ -1074,7 +1074,7 @@ def test_example_16_hierarchical_risk_groups(): - name: Rack2 disabled: false attrs: {location: "DC1-Row2"} - # Parent risk group with children for hierarchical failure expansion + # Parent risk group with children; failing it cascades to the children - name: Rack1 attrs: {location: "DC1-Row1"} children: @@ -1084,7 +1084,6 @@ def test_example_16_hierarchical_risk_groups(): failures: hierarchical: expand_groups: true - expand_children: true modes: - weight: 1.0 rules: @@ -1117,7 +1116,6 @@ def test_example_16_hierarchical_risk_groups(): # Validate failure policy policy = scenario.failure_policy_set.get_policy("hierarchical") assert policy.expand_groups is True - assert policy.expand_children is True # ============================================================================= diff --git a/tests/explorer/test_explorer_review_fixes.py b/tests/explorer/test_explorer_review_fixes.py new file mode 100644 index 0000000..743e051 --- /dev/null +++ b/tests/explorer/test_explorer_review_fixes.py @@ -0,0 +1,184 @@ +"""Regression tests for NetworkExplorer review fixes. + +Covers: +- get_bom_map include_root/root_label contract. +- get_node_utilization signature cleanup (no include_disabled, no disabled field). +- Node-utilization validation equivalence after the O(E) adjacency pre-pass. +- External link path attribution after hoisting path computation. +""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from ngraph.explorer import NetworkExplorer, NodeUtilization +from ngraph.model.components import Component, ComponentsLibrary +from ngraph.model.network import Link, Network, Node + + +def _library() -> ComponentsLibrary: + lib = ComponentsLibrary() + lib.components["box"] = Component( + name="box", + capex=100.0, + power_watts=10.0, + capacity=1000.0, + ports=8, + ) + lib.components["optic"] = Component( + name="optic", + capex=5.0, + power_watts=1.0, + capacity=100.0, + ports=1, + ) + return lib + + +def _network_with_hw() -> Network: + net = Network() + net.nodes["dc1/a"] = Node(name="dc1/a", attrs={"hardware": {"component": "box"}}) + net.nodes["dc2/b"] = Node(name="dc2/b", attrs={"hardware": {"component": "box"}}) + net.links["L1"] = Link(source="dc1/a", target="dc2/b", capacity=100.0) + return net + + +class TestGetBomMapIncludeRoot: + """get_bom_map must honor include_root and not duplicate the root entry.""" + + def test_include_root_false_omits_root(self) -> None: + explorer = NetworkExplorer.explore_network( + _network_with_hw(), components_library=_library() + ) + bom_map = explorer.get_bom_map(include_root=False) + assert "" not in bom_map + # Subtree entries remain present. + assert "dc1" in bom_map + assert "dc2/b" in bom_map + assert bom_map["dc1"] == {"box": 1.0} + + def test_include_root_default_label_yields_single_root_entry(self) -> None: + explorer = NetworkExplorer.explore_network( + _network_with_hw(), components_library=_library() + ) + bom_map = explorer.get_bom_map(include_root=True) + assert bom_map[""] == explorer.get_bom() + assert bom_map[""] == {"box": 2.0} + + def test_include_root_custom_label_not_duplicated(self) -> None: + explorer = NetworkExplorer.explore_network( + _network_with_hw(), components_library=_library() + ) + bom_map = explorer.get_bom_map(include_root=True, root_label="ROOT") + assert bom_map["ROOT"] == explorer.get_bom() + # Root must not also appear under its path-map key "". + assert "" not in bom_map + + +class TestGetNodeUtilizationSignature: + """get_node_utilization takes no filter; snapshots cover enabled nodes only.""" + + def test_no_include_disabled_parameter(self) -> None: + explorer = NetworkExplorer.explore_network( + _network_with_hw(), components_library=_library() + ) + with pytest.raises(TypeError): + explorer.get_node_utilization(include_disabled=False) # type: ignore[call-arg] + + def test_disabled_field_removed(self) -> None: + field_names = {f.name for f in dataclasses.fields(NodeUtilization)} + assert "disabled" not in field_names + + def test_disabled_nodes_have_no_snapshot(self) -> None: + net = _network_with_hw() + net.nodes["dc2/b"].disabled = True + explorer = NetworkExplorer.explore_network(net, components_library=_library()) + utils = explorer.get_node_utilization() + assert [u.node_name for u in utils] == ["dc1/a"] + + +class TestUtilizationAdjacencyPrePass: + """Utilization results must be identical after the O(E) adjacency index.""" + + def test_disabled_links_and_endpoints_excluded(self) -> None: + net = Network() + net.nodes["A"] = Node(name="A", attrs={"hardware": {"component": "box"}}) + net.nodes["B"] = Node(name="B") + net.nodes["C"] = Node(name="C", disabled=True) + net.links["L1"] = Link(source="A", target="B", capacity=100.0) + net.links["L2"] = Link(source="B", target="A", capacity=50.0) + # Disabled link: ignored. + net.links["L3"] = Link(source="A", target="B", capacity=70.0, disabled=True) + # Opposite endpoint disabled: ignored in active view. + net.links["L4"] = Link(source="A", target="C", capacity=30.0) + + explorer = NetworkExplorer.explore_network(net, components_library=_library()) + utils = {u.node_name: u for u in explorer.get_node_utilization()} + assert utils["A"].attached_capacity_active == pytest.approx(150.0) + assert utils["A"].capacity_utilization == pytest.approx(0.15) + assert not utils["A"].capacity_violation + + def test_self_loop_counted_once(self) -> None: + net = Network() + net.nodes["A"] = Node(name="A", attrs={"hardware": {"component": "box"}}) + net.links["L1"] = Link(source="A", target="A", capacity=40.0) + + explorer = NetworkExplorer.explore_network(net, components_library=_library()) + utils = {u.node_name: u for u in explorer.get_node_utilization()} + assert utils["A"].attached_capacity_active == pytest.approx(40.0) + + def test_ports_usage_from_per_end_optics(self) -> None: + net = Network() + net.nodes["A"] = Node(name="A", attrs={"hardware": {"component": "box"}}) + net.nodes["B"] = Node(name="B", attrs={"hardware": {"component": "box"}}) + net.links["L1"] = Link( + source="A", + target="B", + capacity=100.0, + attrs={ + "hardware": { + "source": {"component": "optic", "count": 2}, + "target": {"component": "optic", "count": 3}, + } + }, + ) + + explorer = NetworkExplorer.explore_network(net, components_library=_library()) + utils = {u.node_name: u for u in explorer.get_node_utilization()} + assert utils["A"].ports_used == pytest.approx(2.0) + assert utils["B"].ports_used == pytest.approx(3.0) + assert utils["A"].ports_available == pytest.approx(8.0) + assert utils["A"].ports_utilization == pytest.approx(0.25) + + def test_capacity_violation_still_raises_in_strict_mode(self) -> None: + net = Network() + net.nodes["A"] = Node(name="A", attrs={"hardware": {"component": "box"}}) + net.nodes["B"] = Node(name="B") + net.links["L1"] = Link(source="A", target="B", capacity=1500.0) + + with pytest.raises(ValueError, match="total attached capacity"): + NetworkExplorer.explore_network(net, components_library=_library()) + + +class TestExternalLinkPathAttribution: + """External link details must name the opposite endpoint's full path.""" + + def test_external_details_after_path_hoist(self) -> None: + explorer = NetworkExplorer.explore_network( + _network_with_hw(), components_library=_library() + ) + root = explorer.root_node + assert root is not None + dc1 = root.children["dc1"] + dc2 = root.children["dc2"] + + for stats in (dc1.stats, dc1.active_stats): + assert stats.external_link_count == 1 + assert set(stats.external_link_details) == {"dc2/b"} + assert stats.external_link_details["dc2/b"].link_capacity == pytest.approx( + 100.0 + ) + for stats in (dc2.stats, dc2.active_stats): + assert set(stats.external_link_details) == {"dc1/a"} diff --git a/tests/lib/test_nx.py b/tests/lib/test_nx.py index 747e337..77e5bac 100644 --- a/tests/lib/test_nx.py +++ b/tests/lib/test_nx.py @@ -154,18 +154,21 @@ def test_multidigraph(self): assert len(edge_map) == 2 def test_undirected_graph(self): - """Convert undirected Graph.""" + """Undirected Graph defaults to antiparallel arc pairs per edge.""" G = nx.Graph() G.add_edge("X", "Y", capacity=75.0, cost=3) graph, node_map, edge_map = from_networkx(G) assert graph.num_nodes() == 2 - assert graph.num_edges() == 1 - assert len(edge_map) == 1 + assert graph.num_edges() == 2 + assert len(edge_map) == 2 + # Both arcs map back to the same original undirected edge + assert edge_map.to_ref[0] == ("X", "Y", 0) + assert edge_map.to_ref[1] == ("X", "Y", 0) def test_multigraph(self): - """Convert undirected MultiGraph.""" + """Undirected MultiGraph defaults to antiparallel arc pairs per edge.""" G = nx.MultiGraph() G.add_edge(1, 2, capacity=10.0) G.add_edge(1, 2, capacity=20.0) @@ -173,8 +176,8 @@ def test_multigraph(self): graph, node_map, edge_map = from_networkx(G) assert graph.num_nodes() == 2 - assert graph.num_edges() == 2 - assert len(edge_map) == 2 + assert graph.num_edges() == 4 + assert len(edge_map) == 4 def test_bidirectional_adds_reverse_edges(self): """bidirectional=True adds reverse edge for each edge.""" diff --git a/tests/lib/test_nx_regressions.py b/tests/lib/test_nx_regressions.py new file mode 100644 index 0000000..51febdc --- /dev/null +++ b/tests/lib/test_nx_regressions.py @@ -0,0 +1,160 @@ +"""Regression tests for ngraph.lib.nx conversion fixes. + +Covers: +- Fractional edge costs raise ValueError instead of silent int() truncation. +- Undirected Graph/MultiGraph inputs default to antiparallel arc pairs so + connectivity is preserved (bidirectional=None inference). +- Docstring-example semantics: node indices follow sorted-name order while + edge refs preserve the original (u, v, key) orientation. +""" + +import networkx as nx +import pytest + +from ngraph.lib.nx import from_networkx + + +class TestFractionalCostRejection: + """from_networkx must reject fractional costs, not truncate them.""" + + def test_fractional_cost_raises_value_error(self): + """Fractional edge cost raises ValueError naming the edge.""" + G = nx.DiGraph() + G.add_edge("A", "B", capacity=10.0, cost=1.9) + + with pytest.raises(ValueError, match=r"'A', 'B'.*1\.9.*not an integer"): + from_networkx(G) + + def test_fractional_cost_error_suggests_prescaling(self): + """Error message includes the pre-scaling hint.""" + G = nx.DiGraph() + G.add_edge("A", "B", cost=0.5) + + with pytest.raises(ValueError, match="Pre-scale fractional costs"): + from_networkx(G) + + def test_integral_float_cost_accepted(self): + """Float costs with integral values are accepted and converted.""" + G = nx.DiGraph() + G.add_edge("A", "B", capacity=10.0, cost=10.0) + + graph, _, _ = from_networkx(G) + + assert int(graph.cost_view()[0]) == 10 + + def test_fractional_default_cost_raises(self): + """Fractional default_cost is rejected when applied to an edge.""" + G = nx.DiGraph() + G.add_edge("A", "B") # no cost attribute + + with pytest.raises(ValueError, match="not an integer"): + from_networkx(G, default_cost=1.5) # type: ignore[arg-type] + + def test_fractional_cost_in_multigraph_raises(self): + """Fractional cost on a parallel edge is also rejected.""" + G = nx.MultiDiGraph() + G.add_edge("A", "B", cost=1) + G.add_edge("A", "B", cost=2.5) + + with pytest.raises(ValueError, match="not an integer"): + from_networkx(G) + + +class TestUndirectedDefaultBidirectional: + """Undirected inputs must default to antiparallel arc pairs.""" + + def test_undirected_graph_default_creates_both_arcs(self): + """Undirected edge yields arcs in both directions by default.""" + G = nx.Graph() + G.add_edge("X", "Y", capacity=75.0, cost=3) + + graph, node_map, edge_map = from_networkx(G) + + assert graph.num_edges() == 2 + src_arr = graph.edge_src_view() + dst_arr = graph.edge_dst_view() + arcs = set(zip(src_arr.tolist(), dst_arr.tolist(), strict=True)) + x_idx = node_map.to_index["X"] + y_idx = node_map.to_index["Y"] + assert (x_idx, y_idx) in arcs + assert (y_idx, x_idx) in arcs + # Both internal IDs map back to the same undirected edge + assert edge_map.from_ref[("X", "Y", 0)] == [0, 1] + + def test_undirected_graph_flow_works_in_both_directions(self): + """Max flow over an undirected edge is nonzero in both directions.""" + import netgraph_core + + G = nx.Graph() + G.add_edge("A", "B", capacity=100.0, cost=1) + + graph, node_map, _ = from_networkx(G) + backend = netgraph_core.Backend.cpu() + algorithms = netgraph_core.Algorithms(backend) + handle = algorithms.build_graph(graph) + + a_idx = node_map.to_index["A"] + b_idx = node_map.to_index["B"] + + flow_fwd, _ = algorithms.max_flow(handle, a_idx, b_idx) + flow_rev, _ = algorithms.max_flow(handle, b_idx, a_idx) + assert flow_fwd == 100.0 + assert flow_rev == 100.0 + + def test_undirected_explicit_false_yields_single_arc(self): + """Explicit bidirectional=False overrides the undirected default.""" + G = nx.Graph() + G.add_edge("X", "Y", capacity=75.0, cost=3) + + graph, _, edge_map = from_networkx(G, bidirectional=False) + + assert graph.num_edges() == 1 + assert len(edge_map) == 1 + + def test_directed_graph_default_single_arc(self): + """Directed inputs keep one arc per edge by default.""" + G = nx.DiGraph() + G.add_edge("A", "B", capacity=10.0, cost=1) + + graph, _, edge_map = from_networkx(G) + + assert graph.num_edges() == 1 + assert len(edge_map) == 1 + + def test_directed_explicit_true_adds_reverse(self): + """Explicit bidirectional=True still works for directed inputs.""" + G = nx.DiGraph() + G.add_edge("A", "B", capacity=10.0, cost=1) + + graph, _, edge_map = from_networkx(G, bidirectional=True) + + assert graph.num_edges() == 2 + assert len(edge_map) == 2 + + def test_multigraph_default_creates_arc_pairs_per_parallel_edge(self): + """Each parallel undirected edge yields its own antiparallel pair.""" + G = nx.MultiGraph() + G.add_edge(1, 2, capacity=10.0) + G.add_edge(1, 2, capacity=20.0) + + graph, _, edge_map = from_networkx(G) + + assert graph.num_edges() == 4 + assert len(edge_map) == 4 + assert len(edge_map.from_ref[(1, 2, 0)]) == 2 + assert len(edge_map.from_ref[(1, 2, 1)]) == 2 + + +class TestDocstringExampleSemantics: + """Pin the corrected from_networkx docstring example outputs.""" + + def test_node_indices_sorted_edge_refs_original_orientation(self): + """Node indices follow sorted names; edge refs keep (u, v, key).""" + G = nx.DiGraph() + G.add_edge("src", "dst", capacity=100.0, cost=10) + + graph, node_map, edge_map = from_networkx(G) + + assert graph.num_nodes() == 2 + assert node_map.to_index == {"dst": 0, "src": 1} + assert edge_map.to_ref[0] == ("src", "dst", 0) diff --git a/tests/logging/test_library_logging_pattern.py b/tests/logging/test_library_logging_pattern.py new file mode 100644 index 0000000..14edf53 --- /dev/null +++ b/tests/logging/test_library_logging_pattern.py @@ -0,0 +1,107 @@ +"""Regression tests for the library logging pattern (NullHandler at import). + +Importing ngraph must not install stream handlers or set levels: a bare +``import ngraph`` previously attached a StreamHandler(sys.stdout) to the +'ngraph' logger, duplicating records in host applications and corrupting +machine-readable stdout (``ngraph run --stdout``). +""" + +import io +import logging +import subprocess +import sys + +import pytest + +from ngraph.logging import ( + get_logger, + reset_logging, + set_global_log_level, + setup_root_logger, +) + + +@pytest.fixture(autouse=True) +def _reset_logging_each_test(): + """Reset logging state before and after each test to avoid cross-test bleed.""" + reset_logging() + yield + reset_logging() + + +def test_import_attaches_only_null_handler(): + """A fresh `import ngraph` leaves only a NullHandler on the 'ngraph' logger.""" + code = ( + "import logging, ngraph; " + "h = logging.getLogger('ngraph').handlers; " + "assert len(h) == 1, h; " + "assert type(h[0]) is logging.NullHandler, h" + ) + subprocess.run([sys.executable, "-c", code], check=True) + + +def test_unconfigured_import_emits_nothing(): + """Without explicit setup, library log records produce no console output.""" + code = ( + "from ngraph.logging import get_logger; " + "get_logger('ngraph.test').warning('sentinel-warning')" + ) + proc = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert "sentinel-warning" not in proc.stdout + assert "sentinel-warning" not in proc.stderr + + +def test_no_duplicate_records_when_host_configures_root(): + """Host applications configuring the root logger see each record once.""" + stream = io.StringIO() + root_handler = logging.StreamHandler(stream) + logging.getLogger().addHandler(root_handler) + try: + get_logger("ngraph.dup_check").warning("dup-check-message") + finally: + logging.getLogger().removeHandler(root_handler) + + assert stream.getvalue().count("dup-check-message") == 1 + + +def test_get_logger_does_not_install_stream_handlers(): + """get_logger() must not configure the root 'ngraph' logger.""" + get_logger("ngraph.some.module") + + root_logger = logging.getLogger("ngraph") + assert all(isinstance(h, logging.NullHandler) for h in root_logger.handlers) + # No level configured implicitly either + assert root_logger.level == logging.NOTSET + + +def test_setup_root_logger_default_handler_uses_stderr(): + """The default console handler writes to stderr, keeping stdout clean.""" + setup_root_logger() + + handlers = logging.getLogger("ngraph").handlers + assert len(handlers) == 1 + handler = handlers[0] + assert isinstance(handler, logging.StreamHandler) + assert handler.stream is sys.stderr + + +def test_set_global_log_level_installs_console_handler(): + """set_global_log_level() configures the console handler (CLI entry path).""" + set_global_log_level(logging.DEBUG) + + root_logger = logging.getLogger("ngraph") + assert root_logger.level == logging.DEBUG + non_null = [ + h for h in root_logger.handlers if not isinstance(h, logging.NullHandler) + ] + assert len(non_null) == 1 + + +def test_records_propagate_for_caplog(caplog): + """Records propagate to ancestor handlers (pytest caplog, host apps).""" + with caplog.at_level(logging.INFO, logger="ngraph"): + get_logger("ngraph.caplog_check").info("caplog-message") + + assert "caplog-message" in caplog.text diff --git a/tests/logging/test_logging.py b/tests/logging/test_logging.py index 5d45b39..15aec94 100644 --- a/tests/logging/test_logging.py +++ b/tests/logging/test_logging.py @@ -24,7 +24,8 @@ def _reset_logging_each_test(): def test_effective_levels_enable_disable(): - """Verify effective levels: INFO by default, DEBUG after enable, back to INFO after disable.""" + """Verify effective levels: INFO after setup, DEBUG after enable, back to INFO after disable.""" + setup_root_logger(level=logging.INFO, handler=logging.NullHandler()) logger = get_logger("ngraph.test") capture = StringIO() @@ -33,7 +34,7 @@ def test_effective_levels_enable_disable(): logger.handlers.clear() logger.addHandler(handler) - # INFO should be emitted by default + # INFO should be emitted after explicit setup logger.info("info-1") assert "info-1" in capture.getvalue() @@ -58,10 +59,11 @@ def test_effective_levels_enable_disable(): def test_global_level_propagates_to_children_and_new_loggers(): """Changing global level updates effective level of existing and new child loggers.""" + setup_root_logger(level=logging.INFO, handler=logging.NullHandler()) logger1 = get_logger("ngraph.module1") logger2 = get_logger("ngraph.module2") - # Default effective level is INFO + # Effective level is INFO after explicit setup assert logger1.getEffectiveLevel() == logging.INFO assert logger2.getEffectiveLevel() == logging.INFO diff --git a/tests/model/components/test_components_yaml_edge_cases.py b/tests/model/components/test_components_yaml_edge_cases.py new file mode 100644 index 0000000..62aba32 --- /dev/null +++ b/tests/model/components/test_components_yaml_edge_cases.py @@ -0,0 +1,73 @@ +"""Edge-case tests for ComponentsLibrary YAML parsing. + +Covers presence-based dispatch of the top-level 'components' key and the +warning emitted when a component definition uses 'cost' instead of 'capex'. +""" + +import logging + +from ngraph.model.components import ComponentsLibrary + + +def test_from_yaml_empty_components_mapping_yields_empty_library() -> None: + """An explicit `components: {}` yields an empty library, not a phantom one.""" + lib = ComponentsLibrary.from_yaml("components: {}") + assert lib.components == {} + + +def test_from_yaml_null_components_yields_empty_library() -> None: + """An explicit `components:` (null) yields an empty library.""" + lib = ComponentsLibrary.from_yaml("components:\n") + assert lib.components == {} + + +def test_from_yaml_empty_components_ignores_sibling_keys() -> None: + """Sibling top-level keys are not parsed as components when key is present.""" + yaml_str = """ +components: {} +other_section: + capex: 5 +""" + lib = ComponentsLibrary.from_yaml(yaml_str) + assert lib.components == {} + + +def test_build_component_warns_on_cost_key(caplog) -> None: + """A leftover 'cost' key logs a warning and contributes 0 to capex.""" + yaml_str = """ +components: + Switch: + component_type: chassis + cost: 20000 +""" + with caplog.at_level(logging.WARNING, logger="ngraph.model.components"): + lib = ComponentsLibrary.from_yaml(yaml_str) + + comp = lib.get("Switch") + assert comp is not None + assert comp.capex == 0.0 + assert comp.attrs["cost"] == 20000 + assert any( + "'cost'" in record.message + and "Switch" in str(record.args or ()) + or "Switch" in record.getMessage() + for record in caplog.records + ) + assert any("capex" in record.getMessage() for record in caplog.records) + + +def test_build_component_no_warning_with_capex(caplog) -> None: + """No warning is emitted when 'capex' is used as documented.""" + yaml_str = """ +components: + Switch: + component_type: chassis + capex: 20000 +""" + with caplog.at_level(logging.WARNING, logger="ngraph.model.components"): + lib = ComponentsLibrary.from_yaml(yaml_str) + + comp = lib.get("Switch") + assert comp is not None + assert comp.capex == 20000.0 + assert not caplog.records diff --git a/tests/model/demand/test_builder.py b/tests/model/demand/test_builder.py index bce4dc2..69c90b4 100644 --- a/tests/model/demand/test_builder.py +++ b/tests/model/demand/test_builder.py @@ -3,8 +3,8 @@ import pytest from ngraph.model.demand.builder import ( - _coerce_flow_policy, build_demand_set, + coerce_flow_policy, ) from ngraph.model.flow.policy_config import FlowPolicyPreset @@ -141,83 +141,124 @@ def test_build_demand_set_invalid_demand_type(): def test_coerce_flow_policy_none(): """Test coercing None.""" - assert _coerce_flow_policy(None) is None + assert coerce_flow_policy(None) is None def test_coerce_flow_policy_enum(): """Test coercing FlowPolicyPreset enum.""" preset = FlowPolicyPreset.SHORTEST_PATHS_ECMP - assert _coerce_flow_policy(preset) == preset + assert coerce_flow_policy(preset) == preset def test_coerce_flow_policy_int(): """Test coercing integer to enum.""" - assert _coerce_flow_policy(1) == FlowPolicyPreset.SHORTEST_PATHS_ECMP - assert _coerce_flow_policy(2) == FlowPolicyPreset.SHORTEST_PATHS_WCMP - assert _coerce_flow_policy(3) == FlowPolicyPreset.TE_WCMP_UNLIM - assert _coerce_flow_policy(4) == FlowPolicyPreset.TE_ECMP_UP_TO_256_LSP - assert _coerce_flow_policy(5) == FlowPolicyPreset.TE_ECMP_16_LSP + assert coerce_flow_policy(1) == FlowPolicyPreset.SHORTEST_PATHS_ECMP + assert coerce_flow_policy(2) == FlowPolicyPreset.SHORTEST_PATHS_WCMP + assert coerce_flow_policy(3) == FlowPolicyPreset.TE_WCMP_UNLIM + assert coerce_flow_policy(4) == FlowPolicyPreset.TE_ECMP_UP_TO_256_LSP + assert coerce_flow_policy(5) == FlowPolicyPreset.TE_ECMP_16_LSP def test_coerce_flow_policy_string(): """Test coercing string to enum.""" assert ( - _coerce_flow_policy("SHORTEST_PATHS_ECMP") + coerce_flow_policy("SHORTEST_PATHS_ECMP") == FlowPolicyPreset.SHORTEST_PATHS_ECMP ) assert ( - _coerce_flow_policy("shortest_paths_ecmp") + coerce_flow_policy("shortest_paths_ecmp") == FlowPolicyPreset.SHORTEST_PATHS_ECMP ) assert ( - _coerce_flow_policy("SHORTEST_PATHS_WCMP") + coerce_flow_policy("SHORTEST_PATHS_WCMP") == FlowPolicyPreset.SHORTEST_PATHS_WCMP ) - assert _coerce_flow_policy("TE_WCMP_UNLIM") == FlowPolicyPreset.TE_WCMP_UNLIM + assert coerce_flow_policy("TE_WCMP_UNLIM") == FlowPolicyPreset.TE_WCMP_UNLIM assert ( - _coerce_flow_policy("TE_ECMP_UP_TO_256_LSP") + coerce_flow_policy("TE_ECMP_UP_TO_256_LSP") == FlowPolicyPreset.TE_ECMP_UP_TO_256_LSP ) - assert _coerce_flow_policy("TE_ECMP_16_LSP") == FlowPolicyPreset.TE_ECMP_16_LSP + assert coerce_flow_policy("TE_ECMP_16_LSP") == FlowPolicyPreset.TE_ECMP_16_LSP def test_coerce_flow_policy_string_numeric(): """Test coercing numeric string to enum.""" - assert _coerce_flow_policy("1") == FlowPolicyPreset.SHORTEST_PATHS_ECMP - assert _coerce_flow_policy("2") == FlowPolicyPreset.SHORTEST_PATHS_WCMP - assert _coerce_flow_policy("3") == FlowPolicyPreset.TE_WCMP_UNLIM + assert coerce_flow_policy("1") == FlowPolicyPreset.SHORTEST_PATHS_ECMP + assert coerce_flow_policy("2") == FlowPolicyPreset.SHORTEST_PATHS_WCMP + assert coerce_flow_policy("3") == FlowPolicyPreset.TE_WCMP_UNLIM def test_coerce_flow_policy_empty_string(): """Test coercing empty string.""" - assert _coerce_flow_policy("") is None - assert _coerce_flow_policy(" ") is None + assert coerce_flow_policy("") is None + assert coerce_flow_policy(" ") is None def test_coerce_flow_policy_invalid_string(): """Test error handling for invalid string.""" with pytest.raises(ValueError, match="Unknown flow policy"): - _coerce_flow_policy("INVALID_POLICY") + coerce_flow_policy("INVALID_POLICY") def test_coerce_flow_policy_invalid_numeric_string(): """Test error handling for invalid numeric string.""" with pytest.raises(ValueError, match="Unknown flow policy value"): - _coerce_flow_policy("999") + coerce_flow_policy("999") def test_coerce_flow_policy_invalid_int(): """Test error handling for invalid integer.""" with pytest.raises(ValueError, match="Unknown flow policy value"): - _coerce_flow_policy(999) + coerce_flow_policy(999) -def test_coerce_flow_policy_other_types(): - """Test that other types are passed through unchanged.""" - # Dict config for advanced usage - dict_config = {"custom": "config"} - assert _coerce_flow_policy(dict_config) == dict_config +def test_coerce_flow_policy_rejects_bool(): + """Booleans are not presets 1/0 and fail fast with a clear error.""" + with pytest.raises(ValueError, match="Invalid flow_policy"): + coerce_flow_policy(True) - # List (unusual but should pass through) - list_config = ["a", "b"] - assert _coerce_flow_policy(list_config) == list_config + with pytest.raises(ValueError, match="Invalid flow_policy"): + coerce_flow_policy(False) + + +def test_coerce_flow_policy_rejects_dict_and_list(): + """Unsupported structural forms fail fast with a clear error.""" + with pytest.raises(ValueError, match="Invalid flow_policy"): + coerce_flow_policy({"custom": "config"}) + + with pytest.raises(ValueError, match="Invalid flow_policy"): + coerce_flow_policy(["a", "b"]) + + +def test_build_demand_set_rejects_inline_dict_flow_policy(): + """Inline dict flow_policy raises at build time, not deep in analysis.""" + raw = { + "tm1": [ + { + "source": "A", + "target": "B", + "volume": 100.0, + "flow_policy": {"path_alg": "SPF", "flow_placement": "PROPORTIONAL"}, + } + ] + } + + with pytest.raises(ValueError, match="Invalid flow_policy"): + build_demand_set(raw) + + +def test_build_demand_set_rejects_bool_flow_policy(): + """Bool flow_policy raises instead of coercing to SHORTEST_PATHS_ECMP.""" + raw = { + "tm1": [ + { + "source": "A", + "target": "B", + "volume": 100.0, + "flow_policy": True, + } + ] + } + + with pytest.raises(ValueError, match="Invalid flow_policy"): + build_demand_set(raw) diff --git a/tests/model/demand/test_spec.py b/tests/model/demand/test_spec.py index 5324264..16a0bf8 100644 --- a/tests/model/demand/test_spec.py +++ b/tests/model/demand/test_spec.py @@ -1,7 +1,16 @@ +import dataclasses + from ngraph.model.demand.spec import TrafficDemand from ngraph.model.flow.policy_config import FlowPolicyPreset as FlowPolicyConfig +def test_removed_legacy_fields_absent() -> None: + """Dead pre-netgraph-core fields are not part of the dataclass.""" + field_names = {f.name for f in dataclasses.fields(TrafficDemand)} + assert "volume_placed" not in field_names + assert "flow_policy_obj" not in field_names + + def test_defaults_and_id_generation() -> None: """TrafficDemand sets sane defaults and generates a unique, structured id.""" demand = TrafficDemand(source="Src", target="Dst") @@ -9,7 +18,6 @@ def test_defaults_and_id_generation() -> None: # Defaults assert demand.priority == 0 assert demand.volume == 0.0 - assert demand.volume_placed == 0.0 assert demand.mode == "combine" assert demand.attrs == {} @@ -80,7 +88,6 @@ def test_custom_assignment_including_policy_config() -> None: target="TargetNode", priority=5, volume=42.5, - volume_placed=10.0, attrs={"description": "test"}, mode="pairwise", flow_policy=FlowPolicyConfig.SHORTEST_PATHS_ECMP, @@ -90,7 +97,6 @@ def test_custom_assignment_including_policy_config() -> None: assert demand.target == "TargetNode" assert demand.priority == 5 assert demand.volume == 42.5 - assert demand.volume_placed == 10.0 assert demand.attrs == {"description": "test"} assert demand.mode == "pairwise" assert demand.flow_policy == FlowPolicyConfig.SHORTEST_PATHS_ECMP diff --git a/tests/model/failure/test_conditions_unit.py b/tests/model/failure/test_conditions_unit.py index 1c4061f..db3e63f 100644 --- a/tests/model/failure/test_conditions_unit.py +++ b/tests/model/failure/test_conditions_unit.py @@ -2,7 +2,7 @@ import pytest -from ngraph.dsl.selectors import Condition, evaluate_condition, evaluate_conditions +from ngraph.model.selectors import Condition, evaluate_condition, evaluate_conditions class TestEvaluateCondition: diff --git a/tests/model/failure/test_failure_trace.py b/tests/model/failure/test_failure_trace.py index d3fafdb..f5dfece 100644 --- a/tests/model/failure/test_failure_trace.py +++ b/tests/model/failure/test_failure_trace.py @@ -3,7 +3,6 @@ import pytest from ngraph.analysis.failure_manager import FailureManager -from ngraph.dsl.selectors.schema import Condition from ngraph.model.failure.policy import ( FailureMode, FailurePolicy, @@ -11,6 +10,7 @@ ) from ngraph.model.failure.policy_set import FailurePolicySet from ngraph.model.network import Link, Network, Node +from ngraph.model.selectors import Condition # ----------------------------------------------------------------------------- # FailurePolicy.apply_failures trace tests @@ -122,30 +122,6 @@ def test_trace_captures_expansion_nodes_links(self) -> None: assert "nodes" in trace["expansion"] assert "links" in trace["expansion"] - def test_trace_captures_expansion_risk_groups(self) -> None: - """Test expansion tracking for risk group children.""" - # Select only the parent, then expansion should add child - rule = FailureRule( - scope="risk_group", - conditions=[Condition(attr="name", op="==", value="parent_rg")], - mode="all", - ) - policy = FailurePolicy( - modes=[FailureMode(weight=1.0, rules=[rule])], - expand_children=True, - ) - - risk_groups = { - "parent_rg": {"name": "parent_rg", "children": [{"name": "child_rg"}]}, - "child_rg": {"name": "child_rg", "children": []}, - } - - trace: dict = {} - policy.apply_failures({}, {}, risk_groups, failure_trace=trace) - - # child_rg should appear in expansion.risk_groups (added by expansion, not selection) - assert "child_rg" in trace["expansion"]["risk_groups"] - def test_trace_no_modes_returns_null_mode_index(self) -> None: """Test that mode_index is None when no modes configured.""" policy = FailurePolicy(modes=[]) diff --git a/tests/model/failure/test_policy.py b/tests/model/failure/test_policy.py index 525c6aa..67f98c9 100644 --- a/tests/model/failure/test_policy.py +++ b/tests/model/failure/test_policy.py @@ -1,11 +1,11 @@ import pytest -from ngraph.dsl.selectors.schema import Condition from ngraph.model.failure.policy import ( FailureMode, FailurePolicy, FailureRule, ) +from ngraph.model.selectors import Condition def _single_mode_policy(rule: FailureRule, **kwargs) -> FailurePolicy: @@ -37,7 +37,7 @@ def test_failure_rule_invalid_probability(): def test_failure_policy_evaluate_conditions_or_logic(): """Test condition evaluation with 'or' logic via shared evaluate_conditions.""" - from ngraph.dsl.selectors import evaluate_conditions + from ngraph.model.selectors import evaluate_conditions conditions = [ Condition(attr="vendor", op="==", value="cisco"), @@ -60,7 +60,7 @@ def test_failure_policy_evaluate_conditions_or_logic(): def test_failure_policy_evaluate_conditions_invalid_logic(): """Test condition evaluation with invalid logic via shared evaluate_conditions.""" - from ngraph.dsl.selectors import evaluate_conditions + from ngraph.model.selectors import evaluate_conditions conditions = [Condition(attr="vendor", op="==", value="cisco")] attrs = {"vendor": "cisco"} @@ -417,19 +417,24 @@ def test_serialization(): policy = FailurePolicy(modes=[FailureMode(weight=1.0, rules=[rule])]) policy_dict = policy.to_dict() + assert "seed" not in policy_dict + assert "expand_children" not in policy_dict assert "modes" in policy_dict and len(policy_dict["modes"]) == 1 mode_dict = policy_dict["modes"][0] assert len(mode_dict["rules"]) == 1 rule_dict = mode_dict["rules"][0] assert rule_dict["scope"] == "node" - assert rule_dict["logic"] == "and" assert rule_dict["mode"] == "random" assert rule_dict["probability"] == 0.2 assert rule_dict["count"] == 3 - assert len(rule_dict["conditions"]) == 1 - condition_dict = rule_dict["conditions"][0] + # Conditions and logic are nested under "match", matching the YAML format + match_dict = rule_dict["match"] + assert match_dict["logic"] == "and" + assert len(match_dict["conditions"]) == 1 + + condition_dict = match_dict["conditions"][0] assert condition_dict["attr"] == "equipment_vendor" assert condition_dict["op"] == "==" assert condition_dict["value"] == "cisco" diff --git a/tests/model/failure/test_policy_expansion.py b/tests/model/failure/test_policy_expansion.py index 35a15b8..b4cf059 100644 --- a/tests/model/failure/test_policy_expansion.py +++ b/tests/model/failure/test_policy_expansion.py @@ -1,9 +1,9 @@ -"""Tests for FailurePolicy expansion by shared risk groups and children.""" +"""Tests for FailurePolicy expansion by shared risk groups.""" from __future__ import annotations -from ngraph.dsl.selectors.schema import Condition from ngraph.model.failure.policy import FailurePolicy, FailureRule +from ngraph.model.selectors import Condition def test_expand_by_shared_risk_groups() -> None: @@ -38,13 +38,13 @@ def test_expand_by_shared_risk_groups() -> None: assert "N2" not in failed and "L2" not in failed -def test_expand_failed_risk_group_children() -> None: - """Failing a parent risk group should also fail its children when enabled.""" - # No nodes/links needed here; we validate risk_group expansion output itself - nodes: dict[str, dict] = {} - links: dict[str, dict] = {} +def test_failed_risk_group_returns_group_name_only() -> None: + """A risk_group-scoped rule returns the failed group names. - # Rule selects top-level risk group name directly via risk_group scope + Cascading a failed parent group to its children is inherent to the + risk-group hierarchy and happens downstream (in FailureManager), not in + the policy itself. + """ rule = FailureRule( scope="risk_group", conditions=[Condition(attr="name", op="==", value="parent")], @@ -53,17 +53,43 @@ def test_expand_failed_risk_group_children() -> None: ) from ngraph.model.failure.policy import FailureMode - policy = FailurePolicy( - modes=[FailureMode(weight=1.0, rules=[rule])], expand_children=True - ) + policy = FailurePolicy(modes=[FailureMode(weight=1.0, rules=[rule])]) - # Risk group hierarchy as dicts (the policy supports dict objects for groups) risk_groups = { - "parent": {"name": "parent", "children": [{"name": "child1", "children": []}]}, - "child1": {"name": "child1", "children": [{"name": "grand", "children": []}]}, - "grand": {"name": "grand", "children": []}, + "parent": {"name": "parent", "children": ["child1"]}, + "child1": {"name": "child1", "children": []}, + } + + failed = policy.apply_failures({}, {}, network_risk_groups=risk_groups) + assert failed == ["parent"] + + +def test_expand_risk_groups_with_prepared_index() -> None: + """A precomputed risk-group index must yield the same expansion result.""" + nodes = { + "N1": {"risk_groups": {"rg1"}}, + "N2": {"risk_groups": {"rg2"}}, + } + links = { + "L1": {"risk_groups": {"rg1"}}, + "L2": {"risk_groups": set()}, } + rule = FailureRule( + scope="node", + conditions=[Condition(attr="risk_groups", op="contains", value="rg1")], + logic="and", + mode="all", + ) + from ngraph.model.failure.policy import FailureMode + + policy = FailurePolicy( + modes=[FailureMode(weight=1.0, rules=[rule])], expand_groups=True + ) + + index = FailurePolicy.build_risk_group_index(nodes, links) + assert index == {"rg1": {"N1", "L1"}, "rg2": {"N2"}} - failed = policy.apply_failures(nodes, links, network_risk_groups=risk_groups) - # Should include parent, child1, and grand due to recursive expansion - assert set(failed) == {"parent", "child1", "grand"} + baseline = policy.apply_failures(nodes, links, seed=7) + with_index = policy.apply_failures(nodes, links, seed=7, prepared_rg_index=index) + assert with_index == baseline + assert set(with_index) == {"N1", "L1"} diff --git a/tests/model/failure/test_policy_random_selection.py b/tests/model/failure/test_policy_random_selection.py new file mode 100644 index 0000000..1bc7ba7 --- /dev/null +++ b/tests/model/failure/test_policy_random_selection.py @@ -0,0 +1,99 @@ +"""Tests for mode='random' selection behavior in FailurePolicy. + +Pin the semantics of the binomial-count + uniform-sample implementation: +exact behavior at probability 0 and 1, seeded determinism, candidate-subset +containment (including the prepared-matches tuple path), and the marginal +failure rate. +""" + +from __future__ import annotations + +import pytest + +from ngraph.model.failure.policy import FailureMode, FailurePolicy, FailureRule + + +def _random_policy(probability: float) -> FailurePolicy: + rule = FailureRule(scope="node", mode="random", probability=probability) + return FailurePolicy(modes=[FailureMode(weight=1.0, rules=[rule])]) + + +def test_random_probability_zero_selects_none() -> None: + policy = _random_policy(0.0) + nodes = {f"N{i}": {} for i in range(20)} + for seed in range(10): + assert policy.apply_failures(nodes, {}, seed=seed) == [] + + +def test_random_probability_one_selects_all() -> None: + policy = _random_policy(1.0) + nodes = {f"N{i}": {} for i in range(20)} + for seed in range(10): + assert set(policy.apply_failures(nodes, {}, seed=seed)) == set(nodes) + + +def test_random_seeded_determinism_and_subset() -> None: + policy = _random_policy(0.4) + nodes = {f"N{i}": {} for i in range(30)} + + failed1 = policy.apply_failures(nodes, {}, seed=123) + failed2 = policy.apply_failures(nodes, {}, seed=123) + assert failed1 == failed2 + assert set(failed1).issubset(set(nodes)) + + # Different seeds should eventually produce different selections + results = {tuple(policy.apply_failures(nodes, {}, seed=s)) for s in range(30)} + assert len(results) > 1 + + +def test_random_with_prepared_matches_tuple() -> None: + """Prepared candidate tuples are used as-is and yield identical results.""" + policy = _random_policy(0.5) + nodes = {f"N{i}": {} for i in range(25)} + prepared = policy.prepare_matches(nodes, {}, {}) + + for seed in range(10): + baseline = policy.apply_failures(nodes, {}, seed=seed) + with_prepared = policy.apply_failures( + nodes, {}, seed=seed, prepared_matches=prepared + ) + assert with_prepared == baseline + + +def test_random_marginal_rate_matches_probability() -> None: + """Mean failure fraction over many seeds must be close to the probability.""" + probability = 0.5 + policy = _random_policy(probability) + n_entities = 100 + nodes = {f"N{i:03d}": {} for i in range(n_entities)} + + trials = 200 + total_failed = sum( + len(policy.apply_failures(nodes, {}, seed=seed)) for seed in range(trials) + ) + mean_fraction = total_failed / (trials * n_entities) + # Std of the mean is ~0.0035; 0.05 tolerance is far beyond noise. + assert abs(mean_fraction - probability) < 0.05 + + +class TestWeightedSamplingExtremeScales: + """Regression: E-S keys are computed in the log domain, so weighted + selection stays proportional for tiny (e.g. per-hour failure rates) and + huge weight scales instead of degenerating into descending-id order.""" + + @pytest.mark.parametrize("scale", [1e-6, 1e-3, 1.0, 1e6, 1e17]) + def test_selection_proportional_across_scales(self, scale: float) -> None: + import random + from collections import Counter + + weights = {"a": 2.0 * scale, "b": 1.0 * scale, "c": 1.0 * scale} + rng = random.Random(7) + counts: Counter[str] = Counter() + trials = 6000 + for _ in range(trials): + counts.update( + FailurePolicy._weighted_sample_without_replacement(weights, 1, rng) + ) + # "a" carries half the total weight; allow generous sampling noise. + assert abs(counts["a"] / trials - 0.5) < 0.05 + assert counts["b"] > 0 and counts["c"] > 0 diff --git a/tests/model/failure/test_policy_serialization_roundtrip.py b/tests/model/failure/test_policy_serialization_roundtrip.py new file mode 100644 index 0000000..43f7009 --- /dev/null +++ b/tests/model/failure/test_policy_serialization_roundtrip.py @@ -0,0 +1,104 @@ +"""Round-trip tests for FailurePolicy.to_dict. + +The serialized form must match the scenario YAML failure-policy format +(conditions/logic nested under "match") so it can be read back by +build_failure_policy and validates against the scenario JSON schema. +""" + +from __future__ import annotations + +import json +from importlib import resources + +import jsonschema + +from ngraph.model.failure.parser import build_failure_policy +from ngraph.model.failure.policy import FailureMode, FailurePolicy, FailureRule +from ngraph.model.failure.policy_set import FailurePolicySet +from ngraph.model.selectors import Condition + + +def _sample_policy() -> FailurePolicy: + rule_a = FailureRule( + scope="node", + conditions=[Condition(attr="role", op="==", value="spine")], + logic="and", + mode="choice", + count=2, + weight_by="capacity", + ) + rule_b = FailureRule( + scope="link", + conditions=[Condition(attr="link_type", op="==", value="fiber")], + logic="or", + mode="random", + probability=0.25, + path="^dc1/", + ) + return FailurePolicy( + attrs={"name": "sample"}, + expand_groups=True, + modes=[ + FailureMode(weight=0.7, rules=[rule_a], attrs={"label": "spines"}), + FailureMode(weight=0.3, rules=[rule_b]), + ], + ) + + +def test_to_dict_round_trips_through_parser() -> None: + """build_failure_policy(policy.to_dict()) must preserve all rule fields.""" + policy = _sample_policy() + rebuilt = build_failure_policy( + policy.to_dict(), policy_name="sample", derive_seed=lambda _name: None + ) + + assert rebuilt.attrs == policy.attrs + assert rebuilt.expand_groups == policy.expand_groups + assert len(rebuilt.modes) == len(policy.modes) + for orig_mode, new_mode in zip(policy.modes, rebuilt.modes, strict=True): + assert new_mode.weight == orig_mode.weight + assert new_mode.attrs == orig_mode.attrs + for orig_rule, new_rule in zip(orig_mode.rules, new_mode.rules, strict=True): + assert new_rule.scope == orig_rule.scope + assert new_rule.conditions == orig_rule.conditions + assert new_rule.logic == orig_rule.logic + assert new_rule.mode == orig_rule.mode + assert new_rule.probability == orig_rule.probability + assert new_rule.count == orig_rule.count + assert new_rule.weight_by == orig_rule.weight_by + assert new_rule.path == orig_rule.path + + +def test_to_dict_matches_scenario_schema() -> None: + """to_dict output must validate against the scenario JSON schema.""" + policy = _sample_policy() + schema = json.loads( + resources.files("ngraph.schemas").joinpath("scenario.json").read_text() + ) + jsonschema.validate({"failures": {"sample": policy.to_dict()}}, schema) + + +def test_to_dict_excludes_seed_and_nests_match() -> None: + """Serialized policies omit seed and nest conditions under match.""" + policy = _sample_policy() + policy.seed = 42 + data = policy.to_dict() + + assert "seed" not in data + assert "expand_children" not in data + rule_dict = data["modes"][0]["rules"][0] + assert "conditions" not in rule_dict + assert "logic" not in rule_dict + assert rule_dict["match"] == { + "logic": "and", + "conditions": [{"attr": "role", "op": "==", "value": "spine"}], + } + + +def test_policy_set_to_dict_delegates_to_policy_to_dict() -> None: + """FailurePolicySet.to_dict must emit the same shape per policy.""" + fps = FailurePolicySet() + policy = _sample_policy() + fps.add("sample", policy) + + assert fps.to_dict() == {"sample": policy.to_dict()} diff --git a/tests/model/failure/test_policy_zero_weight_modes.py b/tests/model/failure/test_policy_zero_weight_modes.py new file mode 100644 index 0000000..ffe7a69 --- /dev/null +++ b/tests/model/failure/test_policy_zero_weight_modes.py @@ -0,0 +1,91 @@ +"""Tests for zero-weight failure mode semantics. + +Modes with zero weight must never be selected, including the degenerate case +where no mode has positive weight (the policy then fails nothing). The parser +rejects policies whose modes all have zero weight. +""" + +from __future__ import annotations + +import pytest + +from ngraph.model.failure.parser import build_failure_policy +from ngraph.model.failure.policy import FailureMode, FailurePolicy, FailureRule + + +def test_all_zero_weight_modes_apply_no_failures() -> None: + """A policy whose only mode has weight 0 must not fail anything.""" + rule = FailureRule(scope="node", mode="all") + policy = FailurePolicy(modes=[FailureMode(weight=0.0, rules=[rule])]) + + nodes = {"N1": {}, "N2": {}} + trace: dict = {} + failed = policy.apply_failures(nodes, {}, failure_trace=trace, seed=1) + + assert failed == [] + assert trace["mode_index"] is None + assert trace["mode_attrs"] == {} + assert trace["selections"] == [] + + +def test_all_zero_weight_modes_no_failures_across_seeds() -> None: + """No seed may ever select a zero-weight mode.""" + rule = FailureRule(scope="node", mode="all") + policy = FailurePolicy( + modes=[ + FailureMode(weight=0.0, rules=[rule]), + FailureMode(weight=0.0, rules=[rule]), + ] + ) + nodes = {"N1": {}, "N2": {}} + + for seed in range(20): + assert policy.apply_failures(nodes, {}, seed=seed) == [] + + +def test_zero_weight_mode_never_selected_among_positive() -> None: + """A zero-weight mode must never win against a positive-weight mode.""" + node_rule = FailureRule(scope="node", mode="all") + link_rule = FailureRule(scope="link", mode="all") + policy = FailurePolicy( + modes=[ + FailureMode(weight=0.0, rules=[node_rule]), + FailureMode(weight=1.0, rules=[link_rule]), + ] + ) + nodes = {"N1": {}} + links = {"L1": {}} + + for seed in range(50): + trace: dict = {} + failed = policy.apply_failures(nodes, links, failure_trace=trace, seed=seed) + assert trace["mode_index"] == 1 + assert failed == ["L1"] + + +def test_parser_rejects_all_zero_weight_policy() -> None: + """An all-zero-weight policy must fail loudly at parse time.""" + fp_data = { + "modes": [ + {"weight": 0.0, "rules": [{"scope": "node", "mode": "all"}]}, + {"weight": 0, "rules": []}, + ] + } + with pytest.raises(ValueError, match="no mode with positive weight"): + build_failure_policy( + fp_data, policy_name="all_zero", derive_seed=lambda _name: None + ) + + +def test_parser_accepts_mixed_zero_and_positive_weights() -> None: + """Zero-weight modes are allowed as long as one mode has positive weight.""" + fp_data = { + "modes": [ + {"weight": 0.0, "rules": [{"scope": "node", "mode": "all"}]}, + {"weight": 2.5, "rules": [{"scope": "link", "mode": "all"}]}, + ] + } + policy = build_failure_policy( + fp_data, policy_name="mixed", derive_seed=lambda _name: None + ) + assert [mode.weight for mode in policy.modes] == [0.0, 2.5] diff --git a/tests/model/failure/test_risk_group_parser.py b/tests/model/failure/test_risk_group_parser.py new file mode 100644 index 0000000..1bbf0c1 --- /dev/null +++ b/tests/model/failure/test_risk_group_parser.py @@ -0,0 +1,210 @@ +"""Tests for risk group parsing in ngraph.model.failure.parser. + +Focuses on rejection of keys that are silently inert on nested children: +only top-level groups are registered in network.risk_groups, so 'membership', +'disabled', and 'generate' blocks on children must fail fast. +""" + +import jsonschema +import pytest + +from ngraph.model.failure.parser import build_risk_groups +from ngraph.scenario import Scenario + + +class TestBuildRiskGroupsChildRejection: + """Inert keys on child entries are rejected with ValueError.""" + + def test_child_membership_rejected(self): + """A 'membership' block on a child raises instead of being ignored.""" + rg_data = [ + { + "name": "Parent", + "children": [ + { + "name": "Child", + "membership": { + "scope": "node", + "match": { + "conditions": [ + {"attr": "role", "op": "==", "value": "leaf"} + ] + }, + }, + } + ], + } + ] + + with pytest.raises( + ValueError, match="'membership' rules not allowed in children" + ): + build_risk_groups(rg_data) + + def test_child_disabled_rejected(self): + """A 'disabled' flag on a child raises instead of being ignored.""" + rg_data = [ + { + "name": "Parent", + "children": [{"name": "Child", "disabled": True}], + } + ] + + with pytest.raises(ValueError, match="'disabled' not allowed in children"): + build_risk_groups(rg_data) + + def test_child_disabled_false_rejected(self): + """Even 'disabled: false' on a child is rejected (the key is inert).""" + rg_data = [ + { + "name": "Parent", + "children": [{"name": "Child", "disabled": False}], + } + ] + + with pytest.raises(ValueError, match="'disabled' not allowed in children"): + build_risk_groups(rg_data) + + def test_grandchild_membership_rejected(self): + """Rejection applies recursively to nested grandchildren.""" + rg_data = [ + { + "name": "Parent", + "children": [ + { + "name": "Child", + "children": [ + { + "name": "Grandchild", + "membership": { + "scope": "link", + "match": { + "conditions": [ + { + "attr": "fiber.conduit_id", + "op": "==", + "value": "C1", + } + ] + }, + }, + } + ], + } + ], + } + ] + + with pytest.raises( + ValueError, match="'membership' rules not allowed in children" + ): + build_risk_groups(rg_data) + + def test_child_generate_rejected(self): + """A 'generate' block on a child raises (existing behavior).""" + rg_data = [ + { + "name": "Parent", + "children": [{"generate": {"scope": "node", "group_by": "site"}}], + } + ] + + with pytest.raises(ValueError, match="'generate' blocks not allowed"): + build_risk_groups(rg_data) + + def test_top_level_membership_and_disabled_still_accepted(self): + """Top-level entries keep full support for membership and disabled.""" + rg_data = [ + { + "name": "Parent", + "disabled": True, + "membership": { + "scope": "node", + "match": { + "conditions": [{"attr": "role", "op": "==", "value": "leaf"}] + }, + }, + "children": [{"name": "Child"}], + } + ] + + groups, generate_specs = build_risk_groups(rg_data) + + assert generate_specs == [] + assert len(groups) == 1 + parent = groups[0] + assert parent.name == "Parent" + assert parent.disabled is True + assert parent._membership_raw is not None + assert [c.name for c in parent.children] == ["Child"] + + def test_plain_children_still_accepted(self): + """Children without inert keys parse unchanged.""" + rg_data = [ + { + "name": "Parent", + "children": [ + {"name": "Child[1-2]", "attrs": {"tier": "conduit"}}, + "ChildShorthand", + ], + } + ] + + groups, _ = build_risk_groups(rg_data) + child_names = [c.name for c in groups[0].children] + assert child_names == ["Child1", "Child2", "ChildShorthand"] + + +class TestScenarioChildMembershipRejection: + """Scenario.from_yaml surfaces the child-key rejection to users. + + The JSON schema rejects these keys before the parser runs, so the + scenario-level error is a jsonschema.ValidationError; the parser's + ValueError remains the guard for direct build_risk_groups callers. + """ + + def test_from_yaml_child_membership_raises(self): + """A scenario with a membership block on a child fails to load.""" + yaml_content = """ +network: + nodes: + A: + attrs: + role: leaf + +risk_groups: + - name: Parent + children: + - name: Child + membership: + scope: node + match: + conditions: + - attr: role + op: "==" + value: leaf +""" + with pytest.raises( + jsonschema.ValidationError, + match="Additional properties are not allowed", + ): + Scenario.from_yaml(yaml_content) + + def test_from_yaml_child_disabled_raises(self): + """A scenario with a disabled flag on a child fails to load.""" + yaml_content = """ +network: + nodes: + A: {} + +risk_groups: + - name: Parent + children: + - name: Child + disabled: true +""" + with pytest.raises( + jsonschema.ValidationError, + match="Additional properties are not allowed", + ): + Scenario.from_yaml(yaml_content) diff --git a/tests/model/test_layering.py b/tests/model/test_layering.py new file mode 100644 index 0000000..b4f9eb9 --- /dev/null +++ b/tests/model/test_layering.py @@ -0,0 +1,106 @@ +"""Tests for model/DSL package layering. + +The runtime selector engine lives in ``ngraph.model.selectors`` so that the +model layer (failure policies in particular) evaluates selectors without +importing the DSL package. ``ngraph.dsl.selectors`` keeps YAML-facing parsing +and re-exports the moved names for backward compatibility. + +The subprocess tests stub parent packages with path-only modules so that +importing a model module does not execute ``ngraph/__init__.py`` (which pulls +in the analysis layer, a legitimate DSL consumer). +""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap +from pathlib import Path + +import ngraph + +_NGRAPH_DIR = Path(ngraph.__file__).resolve().parent + + +def _run_python(code: str) -> None: + """Run a Python snippet in a fresh interpreter and require success.""" + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + +def test_failure_policy_import_does_not_load_dsl() -> None: + """Importing the failure policy module must not load any ngraph.dsl module.""" + code = textwrap.dedent( + f""" + import sys + import types + + # Path-only stubs skip package __init__ side effects of parents. + for name, path in [ + ("ngraph", {str(_NGRAPH_DIR)!r}), + ("ngraph.model", {str(_NGRAPH_DIR / "model")!r}), + ("ngraph.model.failure", {str(_NGRAPH_DIR / "model" / "failure")!r}), + ]: + mod = types.ModuleType(name) + mod.__path__ = [path] + sys.modules[name] = mod + + import ngraph.model.failure.policy # noqa: F401 + + dsl_modules = sorted(m for m in sys.modules if m.startswith("ngraph.dsl")) + assert not dsl_modules, f"policy import pulled in DSL modules: {{dsl_modules}}" + """ + ) + _run_python(code) + + +def test_model_packages_do_not_load_dsl_selectors() -> None: + """Importing the model packages must not load ngraph.dsl.selectors modules. + + The only accepted residual model -> dsl dependency is the dependency-free + string expansion helpers in ``ngraph.dsl.expansion``. + """ + code = textwrap.dedent( + f""" + import sys + import types + + stub = types.ModuleType("ngraph") + stub.__path__ = [{str(_NGRAPH_DIR)!r}] + sys.modules["ngraph"] = stub + + import ngraph.model # noqa: F401 + import ngraph.model.failure # noqa: F401 + import ngraph.model.failure.parser # noqa: F401 + import ngraph.model.selectors # noqa: F401 + + unexpected = sorted( + m + for m in sys.modules + if m.startswith("ngraph.dsl") + and m != "ngraph.dsl" + and not m.startswith("ngraph.dsl.expansion") + ) + assert not unexpected, f"model packages imported DSL modules: {{unexpected}}" + """ + ) + _run_python(code) + + +def test_dsl_selectors_reexports_model_selector_names() -> None: + """ngraph.dsl.selectors re-exports the moved names for backward compatibility.""" + import ngraph.dsl.selectors as dsl_selectors + import ngraph.model.selectors as model_selectors + + for name in model_selectors.__all__: + assert getattr(dsl_selectors, name) is getattr(model_selectors, name), ( + f"ngraph.dsl.selectors.{name} is not the ngraph.model.selectors object" + ) + + # Parsing entry points remain in the DSL layer. + assert callable(dsl_selectors.normalize_selector) + assert callable(dsl_selectors.parse_match_spec) diff --git a/tests/model/test_network_basics.py b/tests/model/test_network_basics.py index 71cfd0d..8a4cbcc 100644 --- a/tests/model/test_network_basics.py +++ b/tests/model/test_network_basics.py @@ -252,3 +252,36 @@ def test_get_links_between(self): ab_links = net.get_links_between("A", "B") assert set(ab_links) == {link_ab1.id, link_ab2.id} + + +class TestDeterministicLinkIds: + """Link ids are per-pair sequences assigned at add time; collisions and + re-adds raise instead of silently overwriting.""" + + def test_ids_are_deterministic_sequences(self): + net = Network() + net.add_node(Node("A")) + net.add_node(Node("B")) + l1 = Link("A", "B") + l2 = Link("A", "B") + net.add_link(l1) + net.add_link(l2) + assert l1.id == "A|B|0" + assert l2.id == "A|B|1" + + def test_pipe_in_node_names_collision_raises(self): + net = Network() + for n in ("a|b", "c", "a", "b|c"): + net.add_node(Node(n)) + net.add_link(Link("a|b", "c")) + with pytest.raises(ValueError, match="ambiguous"): + net.add_link(Link("a", "b|c")) + + def test_re_adding_same_link_raises(self): + net = Network() + net.add_node(Node("A")) + net.add_node(Node("B")) + link = Link("A", "B") + net.add_link(link) + with pytest.raises(ValueError, match="already been added"): + net.add_link(link) diff --git a/tests/model/test_types_base.py b/tests/model/test_types_base.py new file mode 100644 index 0000000..f28e336 --- /dev/null +++ b/tests/model/test_types_base.py @@ -0,0 +1,34 @@ +"""Tests for ngraph.types public surface (enums and exports).""" + +import pytest + +import ngraph.types +from ngraph.types import FlowPlacement, Mode + + +def test_min_cap_min_flow_removed() -> None: + """Dead MIN_CAP/MIN_FLOW constants are no longer exported.""" + assert not hasattr(ngraph.types, "MIN_CAP") + assert not hasattr(ngraph.types, "MIN_FLOW") + assert "MIN_CAP" not in ngraph.types.__all__ + assert "MIN_FLOW" not in ngraph.types.__all__ + + +def test_mode_from_string_valid() -> None: + """Mode.from_string parses names case-insensitively.""" + assert Mode.from_string("combine") is Mode.COMBINE + assert Mode.from_string("PAIRWISE") is Mode.PAIRWISE + assert Mode.from_string("Combine") is Mode.COMBINE + + +def test_mode_from_string_invalid() -> None: + """Mode.from_string raises ValueError for unknown values.""" + with pytest.raises(ValueError, match="Invalid mode 'aggregate'"): + Mode.from_string("aggregate") + + +def test_flow_placement_from_string_still_works() -> None: + """FlowPlacement.from_string remains the parsing counterpart.""" + assert FlowPlacement.from_string("proportional") is FlowPlacement.PROPORTIONAL + with pytest.raises(ValueError, match="Invalid flow_placement"): + FlowPlacement.from_string("bogus") diff --git a/tests/profiling/test_worker_profile_merge.py b/tests/profiling/test_worker_profile_merge.py new file mode 100644 index 0000000..fa17971 --- /dev/null +++ b/tests/profiling/test_worker_profile_merge.py @@ -0,0 +1,87 @@ +"""Regression tests for worker-profile merging. + +merge_child_profiles() previously globbed ``*_worker_*.pstats`` while workers +in analysis/failure_manager.py write ``{analysis_name}_thread_{tid}_{uuid}.pstats``, +so worker profiles were never merged into step profiles. +""" + +import cProfile +import pstats +import threading +import uuid +from pathlib import Path +from typing import Any + +import pytest + +from ngraph.analysis.failure_manager import _generic_worker +from ngraph.profiling.profiler import PerformanceProfiler + + +def _dump_worker_style_profile(profile_dir: Path, analysis_name: str) -> Path: + """Dump a real cProfile run using the exact worker filename convention. + + Mirrors the writer side in ngraph/analysis/failure_manager.py + (_generic_worker), which this test suite must keep in sync with. + """ + profiler = cProfile.Profile() + try: + profiler.enable() + except ValueError: + pytest.skip("another profiler is active; cannot exercise cProfile") + sum(range(1000)) + profiler.disable() + + unique_id = uuid.uuid4().hex[:8] + thread_id = threading.current_thread().ident + profile_path = ( + profile_dir / f"{analysis_name}_thread_{thread_id}_{unique_id}.pstats" + ) + pstats.Stats(profiler).dump_stats(profile_path) + return profile_path + + +def test_merge_child_profiles_matches_worker_file_naming(tmp_path: Path) -> None: + """Files named per the worker convention are merged and then removed.""" + perf = PerformanceProfiler() + with perf.profile_step("step1", "MaxFlowStep"): + sum(range(100)) + + worker_file = _dump_worker_style_profile(tmp_path, "max_flow_analysis") + baseline_calls = perf.results.step_profiles[0].function_calls + + perf.merge_child_profiles(tmp_path, "step1") + + step_profile = perf.results.step_profiles[0] + assert step_profile.worker_profiles_merged == 1 + assert step_profile.function_calls > baseline_calls + # Merged worker files are cleaned up + assert not worker_file.exists() + + +def test_generic_worker_profiles_round_trip(tmp_path: Path, monkeypatch) -> None: + """Profiles written by the real worker are merged into the step profile.""" + monkeypatch.setenv("NGRAPH_PROFILE_DIR", str(tmp_path)) + + def dummy_analysis( + network: Any, excluded_nodes: set[str], excluded_links: set[str] + ) -> int: + return sum(range(500)) + + args = (None, set(), set(), dummy_analysis, {}, 0, False, "dummy_analysis") + result = _generic_worker(args) + assert result == sum(range(500)) + + written = list(tmp_path.glob("*.pstats")) + if not written: + pytest.skip("worker profiling disabled (another profiler is active)") + + perf = PerformanceProfiler() + with perf.profile_step("dummy_step", "DummyStep"): + pass + + perf.merge_child_profiles(tmp_path, "dummy_step") + + step_profile = perf.results.step_profiles[0] + assert step_profile.worker_profiles_merged == len(written) + assert list(tmp_path.glob("*.pstats")) == [] diff --git a/tests/results/test_capacity_envelope_unit.py b/tests/results/test_capacity_envelope_unit.py new file mode 100644 index 0000000..d3ce059 --- /dev/null +++ b/tests/results/test_capacity_envelope_unit.py @@ -0,0 +1,59 @@ +"""Unit tests for CapacityEnvelope aggregation and deserialization edge cases.""" + +from __future__ import annotations + +from collections import namedtuple + +import pytest + +from ngraph.results.artifacts import CapacityEnvelope + + +def test_aggregate_frequencies_count_duplicates() -> None: + """Frequency counting (Counter-based) matches duplicate volumes exactly.""" + Summary = namedtuple("Summary", ["cost_distribution", "min_cut"]) + summaries = [ + Summary(cost_distribution={1.0: 5.0}, min_cut=[]), + Summary(cost_distribution={1.0: 5.0}, min_cut=[]), + Summary(cost_distribution={1.0: 7.0}, min_cut=[]), + ] + + env = CapacityEnvelope.from_values( + source_pattern="S", + sink_pattern="T", + mode="combine", + values=[1.0, 2.0, 3.0], + flow_summaries=summaries, + ) + + freqs = env.flow_summary_stats["cost_distribution_stats"][1.0]["frequencies"] + assert freqs == {5.0: 2, 7.0: 1} + + +def test_from_dict_rejects_non_numeric_frequency_key() -> None: + with pytest.raises(ValueError): + CapacityEnvelope.from_dict( + { + "source": "S", + "sink": "T", + "mode": "combine", + "frequencies": {"not-a-number": 1}, + } + ) + + +def test_from_dict_normalizes_string_keys() -> None: + env = CapacityEnvelope.from_dict( + { + "source": "S", + "sink": "T", + "mode": "combine", + "frequencies": {"10.0": 2, "20": 1}, + "min": 10.0, + "max": 20.0, + "mean": 13.33, + "stdev": 4.71, + "total_samples": 3, + } + ) + assert env.frequencies == {10.0: 2, 20.0: 1} diff --git a/tests/results/test_result.py b/tests/results/test_result.py index 2e0fb58..d646b76 100644 --- a/tests/results/test_result.py +++ b/tests/results/test_result.py @@ -1,5 +1,5 @@ from ngraph.results import Results -from ngraph.results.artifacts import FailurePatternResult +from ngraph.results.artifacts import CapacityEnvelope def test_put_and_get(): @@ -103,17 +103,17 @@ def test_results_to_dict_includes_workflow_and_step_data(): results.enter_step("stepA") results.put("metadata", {}) # Include an artifact object to confirm to_dict conversion - fpr = FailurePatternResult( - excluded_nodes=["n1"], - excluded_links=["l1"], - capacity_matrix={"A->B": 10.0}, - count=2, + env = CapacityEnvelope.from_values( + source_pattern="^A$", + sink_pattern="^B$", + mode="combine", + values=[10.0, 10.0, 20.0], ) - results.put("data", {"pattern": fpr, "value": 1}) + results.put("data", {"envelope": env, "value": 1}) results.exit_step() d = results.to_dict() assert "workflow" in d assert "stepA" in d["workflow"] assert d["steps"]["stepA"]["data"]["value"] == 1 - assert isinstance(d["steps"]["stepA"]["data"]["pattern"], dict) + assert isinstance(d["steps"]["stepA"]["data"]["envelope"], dict) diff --git a/tests/results/test_store_deep_convert.py b/tests/results/test_store_deep_convert.py new file mode 100644 index 0000000..1498613 --- /dev/null +++ b/tests/results/test_store_deep_convert.py @@ -0,0 +1,65 @@ +"""Regression tests for Results.to_dict deep conversion. + +Covers recursion into ``to_dict()`` output and conversion of the scenario +snapshot section, which previously escaped JSON-safe normalization. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict + +from ngraph.results import Results +from ngraph.results.artifacts import CapacityEnvelope + + +class _Inner: + def to_dict(self) -> Dict[Any, Any]: + return {2.5: ("a", "b")} + + +class _Outer: + def to_dict(self) -> Dict[Any, Any]: + # Float keys, a tuple, and a nested convertible object + return {1.5: 2, "nested": _Inner(), "items": (1, 2)} + + +def test_to_dict_recurses_into_to_dict_output() -> None: + results = Results() + results.put_step_metadata("s1", "Dummy", 0) + results.enter_step("s1") + results.put("metadata", {}) + results.put("data", {"obj": _Outer()}) + results.exit_step() + + exported = results.to_dict() + obj = exported["steps"]["s1"]["data"]["obj"] + assert obj == {"1.5": 2, "nested": {"2.5": ["a", "b"]}, "items": [1, 2]} + # The whole document must be JSON-serializable without fallbacks + json.dumps(exported) + + +def test_to_dict_converts_capacity_envelope_frequencies_keys() -> None: + results = Results() + results.put_step_metadata("s1", "Dummy", 0) + results.enter_step("s1") + results.put("metadata", {}) + env = CapacityEnvelope.from_values("^A$", "^B$", "combine", [10.0, 10.0, 20.0]) + results.put("data", {"envelope": env}) + results.exit_step() + + exported = results.to_dict() + freqs = exported["steps"]["s1"]["data"]["envelope"]["frequencies"] + assert all(isinstance(k, str) for k in freqs) + assert freqs["10.0"] == 2 + assert freqs["20.0"] == 1 + json.dumps(exported) + + +def test_to_dict_converts_scenario_snapshot() -> None: + results = Results() + results.set_scenario_snapshot({"meta": {1: (1, 2)}, "name": "demo"}) + + exported = results.to_dict() + assert exported["scenario"] == {"meta": {"1": [1, 2]}, "name": "demo"} + json.dumps(exported) diff --git a/tests/scenario/test_scenario.py b/tests/scenario/test_scenario.py index 69c890a..fbdc46c 100644 --- a/tests/scenario/test_scenario.py +++ b/tests/scenario/test_scenario.py @@ -97,7 +97,6 @@ def valid_scenario_yaml() -> str: name: "multi_rule_example" description: "Testing modal policy." expand_groups: false - expand_children: false modes: - weight: 1.0 rules: @@ -286,7 +285,6 @@ def test_scenario_from_yaml_valid(valid_scenario_yaml: str) -> None: simple_policy = scenario.failure_policy_set.get_policy("default") assert isinstance(simple_policy, FailurePolicy) assert not simple_policy.expand_groups - assert not simple_policy.expand_children assert len(simple_policy.modes) == 1 assert simple_policy.attrs.get("name") == "multi_rule_example" @@ -524,3 +522,62 @@ def test_yaml_anchors_and_aliases(): ## Removed redundant anchor test without assertions on attribute merging. The ## remaining anchor test validates both anchors and attribute overrides. + + +def test_scenario_snapshot_serialization_format(): + """Scenario snapshot must serialize policies in YAML format and presets by name. + + Regression tests: the snapshot delegates failure-policy serialization to + FailurePolicy.to_dict (conditions nested under "match", no expand_children) + and stores flow_policy as the preset name string instead of a raw IntEnum. + """ + import json + + yaml_content = """ +network: + nodes: + A: {} + B: {} + links: + - source: A + target: B + capacity: 10 +failures: + default: + modes: + - weight: 1.0 + rules: + - scope: link + mode: choice + count: 1 + match: + logic: and + conditions: + - attr: capacity + op: ">=" + value: 1 +demands: + default: + - source: A + target: B + volume: 5 + flow_policy: SHORTEST_PATHS_ECMP +""" + scenario = Scenario.from_yaml(yaml_content) + exported = scenario.results.to_dict() + snapshot = exported["scenario"] + + # Failure policies use the parser-compatible to_dict shape + policy_dict = snapshot["failures"]["default"] + assert "expand_children" not in policy_dict + assert "seed" not in policy_dict + rule_dict = policy_dict["modes"][0]["rules"][0] + assert rule_dict["match"]["logic"] == "and" + assert rule_dict["match"]["conditions"][0]["attr"] == "capacity" + + # flow_policy is the preset name string, consistent with step exports + demand_entry = snapshot["demands"]["default"][0] + assert demand_entry["flow_policy"] == "SHORTEST_PATHS_ECMP" + + # The whole export must be JSON-serializable + json.dumps(exported) diff --git a/tests/scenario/test_scenario_disabled_risk_groups.py b/tests/scenario/test_scenario_disabled_risk_groups.py new file mode 100644 index 0000000..f72dcc2 --- /dev/null +++ b/tests/scenario/test_scenario_disabled_risk_groups.py @@ -0,0 +1,144 @@ +"""Regression tests for disabled risk groups and late-assigned members. + +The disable cascade for risk groups declared ``disabled: true`` must run after +membership rules and generate blocks, so entities assigned to groups by those +mechanisms are disabled as well. +""" + +from ngraph.scenario import Scenario + + +def test_disabled_group_disables_membership_rule_nodes() -> None: + """Nodes matched by a membership rule of a disabled group end up disabled.""" + yaml_content = """ +network: + nodes: + RouterA: + attrs: + power_zone: "PZ-A" + RouterB: + attrs: + power_zone: "PZ-A" + RouterC: + attrs: + power_zone: "PZ-B" + +risk_groups: + - name: PowerZoneA + disabled: true + membership: + scope: node + match: + conditions: + - attr: power_zone + op: "==" + value: "PZ-A" +""" + scenario = Scenario.from_yaml(yaml_content) + network = scenario.network + + assert "PowerZoneA" in network.nodes["RouterA"].risk_groups + assert "PowerZoneA" in network.nodes["RouterB"].risk_groups + assert network.nodes["RouterA"].disabled is True + assert network.nodes["RouterB"].disabled is True + # Unmatched node remains enabled + assert network.nodes["RouterC"].disabled is False + + +def test_disabled_group_disables_membership_rule_links() -> None: + """Links matched by a membership rule of a disabled group end up disabled.""" + yaml_content = """ +network: + nodes: + NYC: {} + CHI: {} + LA: {} + links: + - source: NYC + target: CHI + attrs: + conduit_id: "C1" + - source: CHI + target: LA + attrs: + conduit_id: "C2" + +risk_groups: + - name: Conduit_C1 + disabled: true + membership: + scope: link + match: + conditions: + - attr: conduit_id + op: "==" + value: "C1" +""" + scenario = Scenario.from_yaml(yaml_content) + network = scenario.network + + by_conduit = {link.attrs.get("conduit_id"): link for link in network.links.values()} + assert "Conduit_C1" in by_conduit["C1"].risk_groups + assert by_conduit["C1"].disabled is True + assert by_conduit["C2"].disabled is False + + +def test_disabled_parent_cascades_to_membership_rule_children() -> None: + """Recursive disable covers children added to a disabled parent via rules.""" + yaml_content = """ +network: + nodes: + NodeC1: + risk_groups: ["Conduit1"] + NodeC2: + risk_groups: ["Conduit2"] + NodeOther: {} + +risk_groups: + - name: Conduit1 + attrs: + route: "NYC-CHI" + - name: Conduit2 + attrs: + route: "NYC-CHI" + - name: Route_NYC_CHI + disabled: true + membership: + scope: risk_group + match: + conditions: + - attr: route + op: "==" + value: "NYC-CHI" +""" + scenario = Scenario.from_yaml(yaml_content) + network = scenario.network + + parent = network.risk_groups["Route_NYC_CHI"] + child_names = {child.name for child in parent.children} + assert child_names == {"Conduit1", "Conduit2"} + + # Members of children added by the membership rule are disabled + assert network.nodes["NodeC1"].disabled is True + assert network.nodes["NodeC2"].disabled is True + assert network.nodes["NodeOther"].disabled is False + + +def test_disabled_group_still_disables_direct_members() -> None: + """Moving the cascade later keeps direct-member disabling intact.""" + yaml_content = """ +network: + nodes: + A: + risk_groups: ["RG1"] + B: {} + +risk_groups: + - name: RG1 + disabled: true +""" + scenario = Scenario.from_yaml(yaml_content) + network = scenario.network + + assert network.nodes["A"].disabled is True + assert network.nodes["B"].disabled is False diff --git a/tests/scenario/test_scenario_run_hook.py b/tests/scenario/test_scenario_run_hook.py new file mode 100644 index 0000000..27911af --- /dev/null +++ b/tests/scenario/test_scenario_run_hook.py @@ -0,0 +1,141 @@ +"""Tests for Scenario.run step_hook and pre-run validation semantics.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import Callable, ContextManager, Iterator, List + +import pytest + +from ngraph.model.network import Network +from ngraph.scenario import Scenario +from ngraph.workflow.base import WorkflowStep + + +@dataclass +class _EventStep(WorkflowStep): + """Workflow step that records its execution into a shared event list.""" + + events: List[str] = field(default_factory=list) + + def run(self, scenario: Scenario) -> None: + """Record that this step executed.""" + self.events.append(f"execute:{self.name}") + + +@dataclass +class _FailingStep(WorkflowStep): + """Workflow step that raises to exercise exception propagation.""" + + events: List[str] = field(default_factory=list) + + def run(self, scenario: Scenario) -> None: + """Record execution, then fail.""" + self.events.append(f"execute:{self.name}") + raise RuntimeError("boom") + + +def _make_hook( + events: List[str], +) -> Callable[[WorkflowStep], ContextManager[None]]: + """Return a step hook that records enter/exit events around each step.""" + + @contextmanager + def hook(step: WorkflowStep) -> Iterator[None]: + events.append(f"enter:{step.name}") + yield + events.append(f"exit:{step.name}") + + return hook + + +def test_run_without_hook_executes_steps() -> None: + events: List[str] = [] + scenario = Scenario( + network=Network(), + workflow=[_EventStep(name="s1", events=events)], + ) + + scenario.run() + + assert events == ["execute:s1"] + + +def test_step_hook_wraps_each_step_in_order() -> None: + events: List[str] = [] + scenario = Scenario( + network=Network(), + workflow=[ + _EventStep(name="s1", events=events), + _EventStep(name="s2", events=events), + ], + ) + + scenario.run(step_hook=_make_hook(events)) + + assert events == [ + "enter:s1", + "execute:s1", + "exit:s1", + "enter:s2", + "execute:s2", + "exit:s2", + ] + + +def test_step_hook_exception_propagates_and_skips_post_yield() -> None: + """A step failure propagates through the hook and skips its exit code. + + Code after the hook's ``yield`` (e.g., profile merging) must not run for + a failed step, and remaining steps must not execute. + """ + events: List[str] = [] + scenario = Scenario( + network=Network(), + workflow=[ + _FailingStep(name="bad", events=events), + _EventStep(name="after", events=events), + ], + ) + + with pytest.raises(RuntimeError, match="boom"): + scenario.run(step_hook=_make_hook(events)) + + assert events == ["enter:bad", "execute:bad"] + + +def test_run_resets_execution_counter_between_runs() -> None: + """Repeated runs restart execution_order from zero, also with a hook.""" + events: List[str] = [] + scenario = Scenario( + network=Network(), + workflow=[ + _EventStep(name="s1", events=events), + _EventStep(name="s2", events=events), + ], + ) + + scenario.run(step_hook=_make_hook(events)) + scenario.run(step_hook=_make_hook(events)) + + workflow_metadata = scenario.results.to_dict()["workflow"] + assert workflow_metadata["s1"]["execution_order"] == 0 + assert workflow_metadata["s2"]["execution_order"] == 1 + + +def test_run_validates_unique_step_names_before_any_execution() -> None: + """Duplicate effective step names fail before any step or hook runs.""" + events: List[str] = [] + scenario = Scenario( + network=Network(), + workflow=[ + _EventStep(name="dup", events=events), + _EventStep(name="dup", events=events), + ], + ) + + with pytest.raises(ValueError, match="Duplicate workflow step name"): + scenario.run(step_hook=_make_hook(events)) + + assert events == [] diff --git a/tests/scenario/test_schema_validation.py b/tests/scenario/test_schema_validation.py index dd5d27e..2bf4782 100644 --- a/tests/scenario/test_schema_validation.py +++ b/tests/scenario/test_schema_validation.py @@ -337,7 +337,6 @@ def test_schema_validates_complex_failure_policies(self, schema): value: 5 risk_group_failure: expand_groups: true - expand_children: true modes: - weight: 1.0 rules: @@ -381,9 +380,7 @@ def test_schema_validates_traffic_matrices(self, schema): target: "storage.*" volume: 5000.0 mode: "pairwise" - flow_policy: - shortest_path: false - flow_placement: "EQUAL_BALANCED" + flow_policy: "SHORTEST_PATHS_ECMP" network: name: Traffic Test Network diff --git a/tests/workflow/test_alpha_resolution_errors.py b/tests/workflow/test_alpha_resolution_errors.py new file mode 100644 index 0000000..1ce621b --- /dev/null +++ b/tests/workflow/test_alpha_resolution_errors.py @@ -0,0 +1,61 @@ +"""Regression tests for TrafficMatrixPlacement._resolve_alpha error reporting. + +A missing/misordered producer step must be reported as such, instead of the +misleading alpha_from_field error (Results.get_step returns {} for unknown +steps, so the old isinstance guard was dead code). +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from ngraph.results.store import Results +from ngraph.workflow.traffic_matrix_placement_step import TrafficMatrixPlacement + + +def _mock_scenario() -> MagicMock: + mock_scenario = MagicMock() + td = MagicMock() + td.source = "A" + td.target = "B" + td.volume = 10.0 + td.mode = "pairwise" + td.priority = 0 + td.flow_policy = None + mock_scenario.demand_set.get_set.return_value = [td] + mock_scenario.results = Results() + return mock_scenario + + +def test_missing_producer_step_reports_step_error() -> None: + scenario = _mock_scenario() + step = TrafficMatrixPlacement( + name="tm", + demand_set="default", + iterations=1, + alpha_from_step="no_such_step", + ) + with pytest.raises( + ValueError, match="alpha_from_step 'no_such_step' has no results" + ): + step.execute(scenario) + + +def test_missing_field_in_existing_step_reports_field_error() -> None: + scenario = _mock_scenario() + scenario.results.enter_step("msd") + scenario.results.put("metadata", {}) + scenario.results.put("data", {"other": 1.0}) + scenario.results.exit_step() + + step = TrafficMatrixPlacement( + name="tm", + demand_set="default", + iterations=1, + alpha_from_step="msd", + alpha_from_field="data.alpha_star", + ) + with pytest.raises(ValueError, match="alpha_from_field 'data.alpha_star' missing"): + step.execute(scenario) diff --git a/tests/workflow/test_build_graph_attrs.py b/tests/workflow/test_build_graph_attrs.py new file mode 100644 index 0000000..0e3f338 --- /dev/null +++ b/tests/workflow/test_build_graph_attrs.py @@ -0,0 +1,123 @@ +"""Regression tests for BuildGraph with reserved-key collisions in attrs. + +BuildGraph previously crashed with TypeError ("got multiple values for +keyword argument") when node attrs contained "disabled" or link attrs +contained "id", "capacity", "cost", or "disabled". Reserved keys must win +over user attrs, matching the precedence documented for flatten_node_attrs +and flatten_link_attrs. +""" + +from unittest.mock import MagicMock + +import pytest + +from ngraph.model.network import Link, Network, Node +from ngraph.results.store import Results +from ngraph.workflow.build_graph import BuildGraph + + +@pytest.fixture +def scenario_with_reserved_attrs(): + """Scenario whose node/link attrs collide with reserved graph keys.""" + scenario = MagicMock() + scenario.seed = None + scenario._execution_counter = 0 + scenario.network = Network() + scenario.results = Results() + + scenario.network.add_node(Node("A", attrs={"disabled": "user-value", "site": "X"})) + scenario.network.add_node(Node("B")) + scenario.network.add_link( + Link( + "A", + "B", + capacity=10.0, + cost=2.0, + attrs={ + "id": "circuit-123", + "capacity": "10G", + "cost": "external", + "disabled": "maybe", + "vendor": "acme", + }, + ) + ) + return scenario + + +def _step_data(scenario, step_name: str) -> dict: + return scenario.results.to_dict()["steps"][step_name]["data"] + + +def test_build_graph_runs_with_reserved_attr_keys(scenario_with_reserved_attrs): + """Step completes instead of raising TypeError on reserved attr keys.""" + step = BuildGraph(name="build_graph") + + step.execute(scenario_with_reserved_attrs) + + data = _step_data(scenario_with_reserved_attrs, "build_graph") + assert data["graph"] is not None + + +def test_reserved_node_keys_win_over_user_attrs(scenario_with_reserved_attrs): + """Node 'disabled' reflects model state, not the user attr value.""" + step = BuildGraph(name="build_graph") + step.execute(scenario_with_reserved_attrs) + + graph_dict = _step_data(scenario_with_reserved_attrs, "build_graph")["graph"] + nodes = {n["id"]: n for n in graph_dict["nodes"]} + + assert nodes["A"]["disabled"] is False + # Non-reserved user attrs are preserved + assert nodes["A"]["site"] == "X" + + +def test_reserved_link_keys_win_over_user_attrs(scenario_with_reserved_attrs): + """Edge id/capacity/cost/disabled reflect model state, not user attrs.""" + network = scenario_with_reserved_attrs.network + link_id = next(iter(network.links)) + + step = BuildGraph(name="build_graph") + step.execute(scenario_with_reserved_attrs) + + graph_dict = _step_data(scenario_with_reserved_attrs, "build_graph")["graph"] + edges = {e["id"]: e for e in graph_dict["edges"]} + + forward = edges[link_id] + assert forward["source"] == "A" + assert forward["target"] == "B" + assert forward["capacity"] == 10.0 + assert forward["cost"] == 2.0 + assert forward["disabled"] is False + # Non-reserved user attrs are preserved + assert forward["vendor"] == "acme" + + reverse = edges[f"{link_id}_reverse"] + assert reverse["source"] == "B" + assert reverse["target"] == "A" + assert reverse["capacity"] == 10.0 + assert reverse["cost"] == 2.0 + assert reverse["disabled"] is False + assert reverse["vendor"] == "acme" + + +def test_build_graph_without_reserved_keys_unchanged(): + """Plain attrs still pass through unchanged.""" + scenario = MagicMock() + scenario.seed = None + scenario._execution_counter = 0 + scenario.network = Network() + scenario.results = Results() + scenario.network.add_node(Node("A", attrs={"role": "leaf"})) + scenario.network.add_node(Node("B", disabled=True)) + scenario.network.add_link(Link("A", "B", capacity=5.0, cost=1.0)) + + step = BuildGraph(name="build_graph") + step.execute(scenario) + + graph_dict = _step_data(scenario, "build_graph")["graph"] + nodes = {n["id"]: n for n in graph_dict["nodes"]} + assert nodes["A"]["role"] == "leaf" + assert nodes["A"]["disabled"] is False + assert nodes["B"]["disabled"] is True + assert len(graph_dict["edges"]) == 2 # forward + reverse diff --git a/tests/workflow/test_capacity_envelope_analysis.py b/tests/workflow/test_capacity_envelope_analysis.py index 34ad923..95586d8 100644 --- a/tests/workflow/test_capacity_envelope_analysis.py +++ b/tests/workflow/test_capacity_envelope_analysis.py @@ -112,7 +112,7 @@ def test_validation_errors(self): with pytest.raises(ValueError, match="parallelism must be >= 1"): MaxFlow(source="^A", target="^C", parallelism=0) - with pytest.raises(ValueError, match="mode must be 'combine' or 'pairwise'"): + with pytest.raises(ValueError, match="Invalid mode"): MaxFlow(source="^A", target="^C", mode="invalid") def test_flow_placement_enum_usage(self): diff --git a/tests/workflow/test_cost_power.py b/tests/workflow/test_cost_power.py index 7abb677..49cd843 100644 --- a/tests/workflow/test_cost_power.py +++ b/tests/workflow/test_cost_power.py @@ -4,6 +4,7 @@ import pytest +from ngraph.explorer import NetworkExplorer from ngraph.model.components import Component, ComponentsLibrary from ngraph.model.network import Link, Network, Node from ngraph.results.store import Results @@ -67,6 +68,7 @@ class _Scenario: components_library = comps results = Results() _execution_counter = 0 + seed = None scenario = _Scenario() @@ -126,6 +128,7 @@ class _Scenario: components_library = comps results = Results() _execution_counter = 0 + seed = None scenario = _Scenario() @@ -163,6 +166,7 @@ class _Scenario: components_library = comps results = Results() _execution_counter = 0 + seed = None scenario = _Scenario() step = CostPower(name="cp3", include_disabled=False, aggregation_level=0) @@ -175,3 +179,58 @@ class _Scenario: assert root["platform_power_watts"] == pytest.approx(10.0) assert root["optics_capex"] == pytest.approx(1.5) assert root["optics_power_watts"] == pytest.approx(0.5) + + +def test_cost_power_runs_despite_hardware_capacity_violation() -> None: + """CostPower aggregates costs even when hardware validation would fail. + + Previously the step built a NetworkExplorer with strict validation as a + side effect, so a node whose attached link capacity exceeded its hardware + capacity crashed the cost aggregation. The step must not depend on the + explorer and must complete regardless of hardware violations. + """ + net = Network() + net.add_node( + Node("dc1/leaf/A", attrs={"hardware": {"component": "NodeHW", "count": 1}}) + ) + net.add_node( + Node("dc1/leaf/B", attrs={"hardware": {"component": "NodeHW", "count": 1}}) + ) + + # NodeHW supports 100.0 of capacity; a 500.0 link exceeds it on both ends. + link = Link("dc1/leaf/A", "dc1/leaf/B", capacity=500.0) + link.attrs["hardware"] = { + "source": {"component": "LinkHW", "count": 1}, + "target": {"component": "LinkHW", "count": 1}, + } + net.add_link(link) + + comps = _build_simple_components() + + # Sanity check: explorer strict validation rejects this network, which is + # exactly what made the old CostPower implementation crash. + with pytest.raises(ValueError, match="exceeds hardware"): + NetworkExplorer.explore_network(net, components_library=comps) + + class _Scenario: + network = net + components_library = comps + results = Results() + _execution_counter = 0 + seed = None + + scenario = _Scenario() + step = CostPower(name="cp4", include_disabled=False, aggregation_level=2) + step.execute(scenario) # type: ignore[arg-type] + + exported = scenario.results.to_dict() + root = _extract_root(exported, "cp4") + assert root["platform_capex"] == pytest.approx(200.0) + assert root["platform_power_watts"] == pytest.approx(20.0) + assert root["optics_capex"] == pytest.approx(3.0) + assert root["optics_power_watts"] == pytest.approx(1.0) + + # Hierarchy paths derive from node names directly. + data = exported["steps"]["cp4"]["data"] + lvl2 = {row["path"] for row in data["levels"]["2"]} + assert lvl2 == {"dc1/leaf"} diff --git a/tests/workflow/test_maximum_supported_demand.py b/tests/workflow/test_maximum_supported_demand.py index 0d1e71e..6eed8c0 100644 --- a/tests/workflow/test_maximum_supported_demand.py +++ b/tests/workflow/test_maximum_supported_demand.py @@ -4,19 +4,14 @@ import pytest +from ngraph.model.demand.spec import TrafficDemand from ngraph.results.store import Results from ngraph.workflow.maximum_supported_demand_step import MaximumSupportedDemand def _mock_scenario_with_matrix() -> MagicMock: mock_scenario = MagicMock() - td = MagicMock() - td.source = "A" - td.target = "B" - td.volume = 10.0 - td.mode = "pairwise" - td.priority = 0 - td.flow_policy = None + td = TrafficDemand(source="A", target="B", volume=10.0, mode="pairwise") mock_scenario.demand_set.get_set.return_value = [td] return mock_scenario @@ -92,9 +87,6 @@ def _eval(cache, alpha): def test_msd_end_to_end_single_link() -> None: """Test MSD end-to-end with a simple single-link scenario.""" from ngraph.analysis.functions import demand_placement_analysis - from ngraph.workflow.maximum_supported_demand_step import ( - MaximumSupportedDemand as MSD, - ) from tests.integration.helpers import ScenarioDataBuilder # Build a tiny deterministic scenario: A --(cap=10)--> B, demand base=5 @@ -126,54 +118,28 @@ def test_msd_end_to_end_single_link() -> None: base_demands = data.get("base_demands") assert isinstance(base_demands, list) and base_demands - # Verify feasibility at alpha* using demand_placement_analysis - scaled_demands = MSD._build_scaled_demands(base_demands, float(alpha_star)) - demands_config = [ - { - "id": d.id, - "source": d.source, - "target": d.target, - "volume": d.volume, - "mode": d.mode, - "priority": d.priority, - "flow_policy": d.flow_policy, - } - for d in scaled_demands - ] + def _scaled_config(alpha: float) -> list[dict]: + return [{**d, "volume": float(d["volume"]) * alpha} for d in base_demands] + # Verify feasibility at alpha* using demand_placement_analysis result = demand_placement_analysis( network=scenario.network, excluded_nodes=set(), excluded_links=set(), - demands_config=demands_config, - placement_rounds=1, + demands_config=_scaled_config(float(alpha_star)), ) # At alpha*, all demands should be fully placed assert result.summary.overall_ratio >= 1.0 - 1e-9 # Verify infeasibility just above alpha* - alpha_above = float(alpha_star) + 0.05 - scaled_demands_above = MSD._build_scaled_demands(base_demands, alpha_above) - demands_config_above = [ - { - "id": d.id, - "source": d.source, - "target": d.target, - "volume": d.volume, - "mode": d.mode, - "priority": d.priority, - "flow_policy": d.flow_policy, - } - for d in scaled_demands_above - ] + demands_config_above = _scaled_config(float(alpha_star) + 0.05) result_above = demand_placement_analysis( network=scenario.network, excluded_nodes=set(), excluded_links=set(), demands_config=demands_config_above, - placement_rounds=1, ) # Above alpha*, placement should fail (ratio < 1.0) diff --git a/tests/workflow/test_msd_perf_safety.py b/tests/workflow/test_msd_perf_safety.py index 0c87930..b9f25e7 100644 --- a/tests/workflow/test_msd_perf_safety.py +++ b/tests/workflow/test_msd_perf_safety.py @@ -11,6 +11,7 @@ def __init__(self, network: Any, demand_set: Any, results: Any) -> None: self.demand_set = demand_set self.results = results self._execution_counter = 0 + self.seed = None def test_msd_deterministic_evaluation(monkeypatch): diff --git a/tests/workflow/test_placement_rounds_deprecated.py b/tests/workflow/test_placement_rounds_deprecated.py new file mode 100644 index 0000000..3defdf4 --- /dev/null +++ b/tests/workflow/test_placement_rounds_deprecated.py @@ -0,0 +1,101 @@ +"""Regression tests for the deprecated placement_rounds parameter. + +placement_rounds never affected placement (the core engine handles +optimization internally). It must remain accepted for YAML backward +compatibility but emit a deprecation warning, must not be forwarded to +FailureManager, and must not be exported in result contexts as if it +influenced the run. +""" + +from __future__ import annotations + +import logging +from unittest.mock import MagicMock, patch + +from ngraph.results.store import Results +from ngraph.workflow.maximum_supported_demand_step import MaximumSupportedDemand +from ngraph.workflow.traffic_matrix_placement_step import TrafficMatrixPlacement + + +def test_msd_placement_rounds_warns_when_set(caplog) -> None: + with caplog.at_level( + logging.WARNING, logger="ngraph.workflow.maximum_supported_demand_step" + ): + MaximumSupportedDemand(name="msd", demand_set="default", placement_rounds=2) + assert any("placement_rounds" in rec.message for rec in caplog.records) + + +def test_tm_placement_rounds_warns_when_set(caplog) -> None: + with caplog.at_level( + logging.WARNING, logger="ngraph.workflow.traffic_matrix_placement_step" + ): + TrafficMatrixPlacement(name="tm", demand_set="default", placement_rounds=2) + assert any("placement_rounds" in rec.message for rec in caplog.records) + + +def test_default_placement_rounds_does_not_warn(caplog) -> None: + with caplog.at_level(logging.WARNING, logger="ngraph.workflow"): + MaximumSupportedDemand(name="msd", demand_set="default") + TrafficMatrixPlacement(name="tm", demand_set="default") + assert not any("placement_rounds" in rec.message for rec in caplog.records) + + +@patch.object(MaximumSupportedDemand, "_evaluate_alpha") +@patch.object(MaximumSupportedDemand, "_build_cache") +def test_msd_context_omits_placement_rounds( + mock_build_cache: MagicMock, mock_eval: MagicMock +) -> None: + mock_build_cache.return_value = MagicMock() + mock_eval.side_effect = lambda cache, alpha: ( + alpha <= 1.0, + {"placement_ratio": 1.0}, + ) + + mock_scenario = MagicMock() + td = MagicMock() + td.source = "A" + td.target = "B" + td.volume = 10.0 + td.mode = "pairwise" + td.priority = 0 + td.flow_policy = None + mock_scenario.demand_set.get_set.return_value = [td] + mock_scenario.results = Results() + + step = MaximumSupportedDemand(name="msd", demand_set="default", placement_rounds=3) + step.execute(mock_scenario) + + context = mock_scenario.results.to_dict()["steps"]["msd"]["data"]["context"] + assert "placement_rounds" not in context + + +@patch("ngraph.workflow.traffic_matrix_placement_step.FailureManager") +def test_tm_does_not_forward_placement_rounds(mock_fm_class) -> None: + mock_scenario = MagicMock() + td = MagicMock() + td.source = "A" + td.target = "B" + td.volume = 10.0 + td.mode = "pairwise" + td.priority = 0 + td.flow_policy = None + mock_scenario.demand_set.get_set.return_value = [td] + mock_scenario.results = Results() + + mock_fm = MagicMock() + mock_fm_class.return_value = mock_fm + mock_fm.run_demand_placement_monte_carlo.return_value = { + "results": [], + "metadata": {"iterations": 1, "unique_patterns": 0}, + } + + step = TrafficMatrixPlacement( + name="tm", demand_set="default", iterations=1, placement_rounds=5 + ) + step.execute(mock_scenario) + + _, kwargs = mock_fm.run_demand_placement_monte_carlo.call_args + assert "placement_rounds" not in kwargs + + context = mock_scenario.results.to_dict()["steps"]["tm"]["data"]["context"] + assert "placement_rounds" not in context diff --git a/tests/workflow/test_seed_provenance.py b/tests/workflow/test_seed_provenance.py new file mode 100644 index 0000000..46095ec --- /dev/null +++ b/tests/workflow/test_seed_provenance.py @@ -0,0 +1,68 @@ +"""Regression tests for seed provenance metadata in WorkflowStep.execute(). + +The recorded seed_source/active_seed must reflect the seed the step actually +uses (self.seed), not the scenario-level seed it never consumes. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from unittest.mock import MagicMock + +from ngraph.results import Results +from ngraph.scenario import Scenario +from ngraph.workflow.base import WorkflowStep + + +@dataclass +class _Dummy(WorkflowStep): + def run(self, scenario) -> None: + scenario.results.put("metadata", {}) + + +def _scenario(seed=None) -> MagicMock: + scen = MagicMock(spec=Scenario) + scen.results = Results() + scen.seed = seed + scen._execution_counter = 0 + return scen + + +def test_unseeded_step_with_scenario_seed_reports_none() -> None: + # The step runs with self.seed=None, so claiming "scenario-derived" + # would falsely advertise reproducibility. + scen = _scenario(seed=42) + _Dummy(name="d1").execute(scen) + + md = scen.results.get_step_metadata("d1") + assert md is not None + assert md.scenario_seed == 42 + assert md.step_seed is None + assert md.seed_source == "none" + assert md.active_seed is None + + +def test_directly_seeded_step_reports_explicit() -> None: + scen = _scenario(seed=None) + _Dummy(name="d2", seed=99).execute(scen) + + md = scen.results.get_step_metadata("d2") + assert md is not None + assert md.scenario_seed is None + assert md.step_seed == 99 + assert md.seed_source == "explicit-step" + assert md.active_seed == 99 + + +def test_scenario_derived_seed_reports_derived() -> None: + scen = _scenario(seed=7) + step = _Dummy(name="d3", seed=1234) + step._seed_source = "scenario-derived" + step.execute(scen) + + md = scen.results.get_step_metadata("d3") + assert md is not None + assert md.scenario_seed == 7 + assert md.step_seed == 1234 + assert md.seed_source == "scenario-derived" + assert md.active_seed == 1234 diff --git a/tests/workflow/test_step_name_collision.py b/tests/workflow/test_step_name_collision.py new file mode 100644 index 0000000..33708e3 --- /dev/null +++ b/tests/workflow/test_step_name_collision.py @@ -0,0 +1,62 @@ +"""Regression tests for workflow step-name collision detection. + +Programmatic scenarios with two unnamed steps of the same type previously +wrote to the same results namespace, silently dropping the first step's data. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import pytest + +from ngraph.model.network import Network +from ngraph.scenario import Scenario +from ngraph.workflow.base import WorkflowStep, validate_unique_step_names + + +@dataclass +class _DummyStep(WorkflowStep): + def run(self, scenario) -> None: + scenario.results.put("metadata", {}) + scenario.results.put("data", {"ran": True}) + + +def _make_scenario(workflow: list[WorkflowStep]) -> Scenario: + return Scenario(network=Network(), workflow=workflow) + + +def test_duplicate_unnamed_steps_raise() -> None: + scenario = _make_scenario([_DummyStep(), _DummyStep()]) + with pytest.raises(ValueError, match="Duplicate workflow step name"): + scenario.run() + + +def test_duplicate_explicit_names_raise() -> None: + scenario = _make_scenario([_DummyStep(name="x"), _DummyStep(name="x")]) + with pytest.raises(ValueError, match="'x'"): + scenario.run() + + +def test_unique_names_run_and_rerun() -> None: + scenario = _make_scenario([_DummyStep(name="a"), _DummyStep(name="b")]) + scenario.run() + # Re-running the same scenario object must not trigger the collision check + scenario.run() + exported = scenario.results.to_dict() + assert exported["steps"]["a"]["data"]["ran"] is True + assert exported["steps"]["b"]["data"]["ran"] is True + + +def test_direct_step_execute_reuse_not_flagged() -> None: + # Executing the same step twice directly (test/REPL pattern) is allowed; + # only duplicates within scenario.workflow are rejected. + scenario = _make_scenario([]) + step = _DummyStep(name="solo") + step.execute(scenario) + step.execute(scenario) + assert scenario.results.to_dict()["steps"]["solo"]["data"]["ran"] is True + + +def test_validate_unique_step_names_accepts_unique() -> None: + validate_unique_step_names([_DummyStep(name="a"), _DummyStep(name="b")]) diff --git a/tests/workflow/test_traffic_matrix_placement.py b/tests/workflow/test_traffic_matrix_placement.py index 2b756c4..a241d4c 100644 --- a/tests/workflow/test_traffic_matrix_placement.py +++ b/tests/workflow/test_traffic_matrix_placement.py @@ -4,6 +4,7 @@ import pytest +from ngraph.model.demand.spec import TrafficDemand from ngraph.results.store import Results from ngraph.workflow.traffic_matrix_placement_step import ( TrafficMatrixPlacement, @@ -16,12 +17,12 @@ def test_traffic_matrix_placement_stores_core_outputs( ) -> None: # Prepare mock scenario with traffic matrix and results store mock_scenario = MagicMock() - mock_td = MagicMock() - mock_td.source = "A" - mock_td.target = "B" - mock_td.volume = 10.0 - mock_td.mode = "pairwise" - mock_td.priority = 0 + mock_td = TrafficDemand( + source="A", + target="B", + volume=10.0, + mode="pairwise", + ) mock_scenario.demand_set.get_set.return_value = [mock_td] # Mock FailureManager return value: baseline separate, failure iterations in results @@ -109,12 +110,12 @@ def test_traffic_matrix_placement_flow_details_edges( ) -> None: # Prepare mock scenario with traffic matrix and results store mock_scenario = MagicMock() - mock_td = MagicMock() - mock_td.source = "A" - mock_td.target = "B" - mock_td.volume = 10.0 - mock_td.mode = "pairwise" - mock_td.priority = 0 + mock_td = TrafficDemand( + source="A", + target="B", + volume=10.0, + mode="pairwise", + ) mock_scenario.demand_set.get_set.return_value = [mock_td] # Mock FailureManager return value with edges used (baseline separate) @@ -212,12 +213,12 @@ def test_traffic_matrix_placement_alpha_scales_demands( ) -> None: # Prepare mock scenario with a single traffic demand mock_scenario = MagicMock() - mock_td = MagicMock() - mock_td.source = "S" - mock_td.target = "T" - mock_td.volume = 10.0 - mock_td.mode = "pairwise" - mock_td.priority = 0 + mock_td = TrafficDemand( + source="S", + target="T", + volume=10.0, + mode="pairwise", + ) mock_scenario.demand_set.get_set.return_value = [mock_td] # Mock FailureManager return value (minimal valid structure) @@ -263,12 +264,12 @@ def test_traffic_matrix_placement_metadata_includes_alpha( mock_failure_manager_class, ) -> None: mock_scenario = MagicMock() - mock_td = MagicMock() - mock_td.source = "A" - mock_td.target = "B" - mock_td.volume = 1.0 - mock_td.mode = "pairwise" - mock_td.priority = 0 + mock_td = TrafficDemand( + source="A", + target="B", + volume=1.0, + mode="pairwise", + ) mock_scenario.demand_set.get_set.return_value = [mock_td] mock_raw = { @@ -309,13 +310,12 @@ def test_traffic_matrix_placement_alpha_auto_uses_msd( ) -> None: # Scenario with one TD mock_scenario = MagicMock() - td = MagicMock() - td.source = "S" - td.target = "T" - td.volume = 4.0 - td.mode = "pairwise" - td.priority = 0 - td.flow_policy = None + td = TrafficDemand( + source="S", + target="T", + volume=4.0, + mode="pairwise", + ) mock_scenario.demand_set.get_set.return_value = [td] # Populate results metadata: prior MSD step @@ -381,13 +381,12 @@ def test_traffic_matrix_placement_alpha_auto_missing_msd_raises( mock_failure_manager_class, ) -> None: mock_scenario = MagicMock() - td = MagicMock() - td.source = "S" - td.target = "T" - td.volume = 4.0 - td.mode = "pairwise" - td.priority = 0 - td.flow_policy = None + td = TrafficDemand( + source="S", + target="T", + volume=4.0, + mode="pairwise", + ) mock_scenario.demand_set.get_set.return_value = [td] # No MSD metadata @@ -411,12 +410,12 @@ def test_traffic_matrix_placement_failure_trace_on_results( ) -> None: """Test that failure_trace is present on flow_results when store_failure_patterns=True.""" mock_scenario = MagicMock() - mock_td = MagicMock() - mock_td.source = "A" - mock_td.target = "B" - mock_td.volume = 10.0 - mock_td.mode = "pairwise" - mock_td.priority = 0 + mock_td = TrafficDemand( + source="A", + target="B", + volume=10.0, + mode="pairwise", + ) mock_scenario.demand_set.get_set.return_value = [mock_td] # Create mock result with failure_trace and occurrence_count @@ -511,12 +510,12 @@ def test_traffic_matrix_placement_no_trace_when_disabled( ) -> None: """Test that failure_trace is None when store_failure_patterns=False.""" mock_scenario = MagicMock() - mock_td = MagicMock() - mock_td.source = "A" - mock_td.target = "B" - mock_td.volume = 10.0 - mock_td.mode = "pairwise" - mock_td.priority = 0 + mock_td = TrafficDemand( + source="A", + target="B", + volume=10.0, + mode="pairwise", + ) mock_scenario.demand_set.get_set.return_value = [mock_td] mock_result = MagicMock()