Skip to content

Generate

This module provides functions for generating power flow data.

_setup_environment

Setup the environment for data generation.

Parameters:

Name Type Description Default
config Union[str, Dict[str, Any], NestedNamespace]

Configuration can be provided in three ways: 1. Path to a YAML config file (str) 2. Configuration dictionary (Dict) 3. NestedNamespace object (NestedNamespace)

required
allow_checkpointing bool

Whether the calling pipeline supports chunk recovery.

False

Returns:

Type Description
Tuple[NestedNamespace, str, Dict[str, str], int]

Tuple of (args, base_path, file_paths, seed)

Raises:

Type Description
TypeError

If config is not a path, dictionary, or NestedNamespace.

ValueError

If a static configuration is invalid.

Source code in gridfm_datakit/generate.py
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
def _setup_environment(
    config: Union[str, Dict[str, Any], NestedNamespace],
    *,
    allow_checkpointing: bool = False,
) -> Tuple[NestedNamespace, str, Dict[str, str], int]:
    """Setup the environment for data generation.

    Args:
        config: Configuration can be provided in three ways:
            1. Path to a YAML config file (str)
            2. Configuration dictionary (Dict)
            3. NestedNamespace object (NestedNamespace)
        allow_checkpointing: Whether the calling pipeline supports chunk recovery.

    Returns:
        Tuple of (args, base_path, file_paths, seed)

    Raises:
        TypeError: If config is not a path, dictionary, or NestedNamespace.
        ValueError: If a static configuration is invalid.
    """
    if isinstance(config, str):
        with open(config, "r", encoding="utf-8") as f:
            config = yaml.safe_load(f)
    elif isinstance(config, NestedNamespace):
        config = config.to_dict()
    elif not isinstance(config, dict):
        raise TypeError(
            "config must be a YAML path, a dictionary, or a NestedNamespace, "
            f"got {type(config).__name__}",
        )

    if not isinstance(config, dict):
        raise ValueError(
            f"Configuration must contain a YAML mapping, got {type(config).__name__}",
        )

    is_dynamic_config = bool(config.get("dynamic"))
    if not is_dynamic_config:
        config = validate_static_config(config)
    checkpointing = config.get("settings", {}).get("enable_checkpointing", False)
    if checkpointing and (is_dynamic_config or not allow_checkpointing):
        raise ValueError(
            "enable_checkpointing is supported only by static generate_power_flow_data_distributed",
        )
    base_path = os.path.join(
        config["settings"]["data_dir"],
        config["network"]["name"],
        "raw",
    )
    checkpoint = GenerationCheckpoint.load(Path(base_path), config)
    if checkpoint is not None:
        config["settings"]["seed"] = checkpoint.state.seed
        config = validate_static_config(config)
    args = NestedNamespace(**config)

    # Dynamic configs share setup but retain their own configuration validator.
    if is_dynamic_config:
        args.settings.coinhsl = NestedNamespace(
            **CoinHSLConfig.model_validate(
                config.get("settings", {}).get("coinhsl", {}),
            ).model_dump(),
        )

    # Set global seed if provided, otherwise generate a unique seed for this generation
    if (
        hasattr(args.settings, "seed")
        and args.settings.seed is not None
        and args.settings.seed != ""
    ):
        seed = args.settings.seed
        print(f"Global random seed set to: {seed}")

    else:
        # Generate a unique seed for non-reproducible but independent scenarios
        # This ensures scenarios are i.i.d. within a run, but different across runs
        seed = _DEFAULT_SEED_POLICY.random_base_seed()
        print(f"No seed provided. Using seed={seed}")

    if checkpointing:
        args.settings.seed = seed

    # Dynamic generation retains its dedicated validator for now. Static
    # configurations receive these defaults from the Pydantic schema above.
    if is_dynamic_config:
        reader = getattr(args.network, "reader", "native")
        if reader not in ("native", "powsybl"):
            raise ValueError(
                f"network.reader must be 'native' or 'powsybl', got {reader!r}",
            )
        args.network.reader = reader

        # pf_solver controls which engine solves the power flow in PF mode.
        # 'powsybl' requires network.reader 'powsybl'. OPF is always solved
        # by PowerModels, so this value is only stored in OPF mode.
        pf_solver = getattr(args.settings, "pf_solver", "powermodel")
        if pf_solver not in ("powermodel", "powsybl", "lightsim2grid"):
            raise ValueError(
                "settings.pf_solver must be 'powermodel', 'powsybl' or 'lightsim2grid', "
                f"got {pf_solver!r}",
            )
        args.settings.pf_solver = pf_solver

        opf_formulation = getattr(args.settings, "opf_formulation", "polar")
        if opf_formulation not in ("polar", "rectangular"):
            raise ValueError(
                "settings.opf_formulation must be 'polar' or 'rectangular', "
                f"got {opf_formulation!r}",
            )
        args.settings.opf_formulation = opf_formulation

    # Static configs are checked by the schema. This still has to run for both
    # paths: the schema cannot tell whether the package is installed, and the
    # check must happen before an existing output directory is removed.
    if args.settings.pf_solver == "lightsim2grid":
        lightsim2grid.check_lightsim2grid_available()

    # Setup output directory
    base_path = os.path.join(args.settings.data_dir, args.network.name, "raw")
    if os.path.exists(base_path) and args.settings.overwrite:
        shutil.rmtree(base_path)
    os.makedirs(base_path, exist_ok=True)

    # Setup solver logs directory under data_dir/solver_log
    solver_log_dir = (
        os.path.join(base_path, "solver_log")
        if args.settings.enable_solver_logs
        else None
    )
    os.makedirs(solver_log_dir, exist_ok=True) if solver_log_dir is not None else None

    # Setup file paths
    file_paths = {
        "tqdm_log": os.path.join(base_path, "tqdm.log"),
        "error_log": os.path.join(base_path, "error.log"),
        "args_log": os.path.join(base_path, "args.log"),
        "config": os.path.join(base_path, "config.yaml"),
        "solver_log_dir": solver_log_dir,
        "bus_data": os.path.join(base_path, "bus_data.parquet"),
        "branch_data": os.path.join(base_path, "branch_data.parquet"),
        "gen_data": os.path.join(base_path, "gen_data.parquet"),
        "y_bus_data": os.path.join(base_path, "y_bus_data.parquet"),
        "runtime_data": os.path.join(base_path, "runtime_data.parquet"),
        "scenarios": os.path.join(
            base_path,
            f"scenarios_{args.load.generator}.parquet",
        ),
        "scenarios_plot": os.path.join(
            base_path,
            f"scenarios_{args.load.generator}.html",
        ),
        "scenarios_log": os.path.join(
            base_path,
            f"scenarios_{args.load.generator}.log",
        ),
    }
    if checkpointing:
        file_paths["checkpoint"] = os.path.join(base_path, "checkpoint.json")

    # Initialize logs
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    for log_file in [
        file_paths["tqdm_log"],
        file_paths["error_log"],
        file_paths["scenarios_log"],
        file_paths["args_log"],
    ]:
        with open(log_file, "a", encoding="utf-8") as f:
            f.write(f"\nNew generation started at {timestamp}\n")
            if log_file == file_paths["args_log"]:
                yaml.safe_dump(args.to_dict(), f)

    with open(file_paths["config"], "w", encoding="utf-8") as f:
        yaml.safe_dump(args.to_dict(), f)

    return args, base_path, file_paths, seed

_prepare_network_and_scenarios

Prepare the network and generate load scenarios.

Parameters:

Name Type Description Default
args NestedNamespace

Configuration object

required
file_paths Dict[str, str]

Dictionary of file paths

required
seed int

Global random seed for reproducibility.

required

Returns:

Type Description
Tuple[Network, ndarray, Dict[str, Any]]

Tuple of (network, scenarios)

Source code in gridfm_datakit/generate.py
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
def _prepare_network_and_scenarios(
    args: NestedNamespace,
    file_paths: Dict[str, str],
    seed: int,
) -> Tuple[Network, np.ndarray, Dict[str, Any]]:
    """Prepare the network and generate load scenarios.

    Args:
        args: Configuration object
        file_paths: Dictionary of file paths
        seed: Global random seed for reproducibility.

    Returns:
        Tuple of (network, scenarios)
    """
    meta = {}
    reader = args.network.reader  # already validated in _setup_environment

    if args.network.source == "pglib":
        if reader == "powsybl":
            network_path = _powsybl_network_path(args)
            loaded_net = powsybl.load_net(network_path)
            meta["pp_net"] = loaded_net.pp_net
            meta["network_path"] = network_path
            meta["mapping_p2g"] = loaded_net.mapping_p2g
            net = loaded_net.gfm_net
        else:
            net = load_net_from_pglib(args.network.name)
    elif args.network.source == "file":
        if reader == "powsybl":
            network_path = _powsybl_network_path(args)
            loaded_net = powsybl.load_net(network_path)
            meta["pp_net"] = loaded_net.pp_net
            meta["network_path"] = network_path
            meta["mapping_p2g"] = loaded_net.mapping_p2g
            net = loaded_net.gfm_net
        else:
            net = load_net_from_file(
                os.path.join(args.network.network_dir, args.network.name) + ".m",
            )
    else:
        raise ValueError(
            f"network.source must be 'pglib' or 'file', got {args.network.source!r}",
        )

    # Generate load scenarios
    load_scenario_generator = get_load_scenario_generator(args.load)
    scenarios = load_scenario_generator(
        net,
        args.load.scenarios,
        file_paths["scenarios_log"],
        max_iter=args.settings.max_iter,
        seed=seed,
        coinhsl=args.settings.coinhsl.to_dict(),
    )
    scenarios_df = load_scenarios_to_df(scenarios)
    write_parquet(scenarios_df, file_paths["scenarios"])
    if net.buses.shape[0] <= 100:
        plot_load_scenarios_combined(scenarios_df, file_paths["scenarios_plot"])
    else:
        print("Skipping plot of scenarios for large networks (number of buses > 100)")

    return net, scenarios, meta

_save_generated_data

Save the generated data to files.

Parameters:

Name Type Description Default
net Network

Network object

required
processed_data List

List of processed data arrays

required
file_paths Dict[str, str]

Dictionary of file paths

required
base_path str

Base output directory

required
args NestedNamespace

Configuration object

required
Source code in gridfm_datakit/generate.py
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
def _save_generated_data(
    net: Network,
    processed_data: List,
    file_paths: Dict[str, str],
    base_path: str,
    args: NestedNamespace,
) -> None:
    """Save the generated data to files.

    Args:
        net: Network object
        processed_data: List of processed data arrays
        file_paths: Dictionary of file paths
        base_path: Base output directory
        args: Configuration object
    """
    if len(processed_data) > 0:
        save_node_edge_data(
            net,
            file_paths["bus_data"],
            file_paths["branch_data"],
            file_paths["gen_data"],
            file_paths["y_bus_data"],
            file_paths["runtime_data"],
            processed_data,
            include_dc_res=args.settings.include_dc_res,
        )

generate_power_flow_data

Generate power flow data based on the provided configuration using sequential processing.

Parameters:

Name Type Description Default
config Union[str, Dict[str, Any], NestedNamespace]

Configuration can be provided in three ways: 1. Path to a YAML config file (str) 2. Configuration dictionary (Dict) 3. NestedNamespace object (NestedNamespace) The config must include settings, network, load, and perturbation configurations.

required

Returns:

Type Description
Dict[str, str]

Dictionary with paths to generated artifacts:

Dict[str, str]

{ 'tqdm_log': progress log file, 'error_log': error log file, 'args_log': configuration dump file, 'config': run configuration (YAML), 'bus_data': bus-level features CSV (BUS_COLUMNS), 'branch_data': branch-level features CSV (BRANCH_COLUMNS), 'gen_data': generator features CSV (GEN_COLUMNS), 'y_bus_data': Y-bus nonzero entries CSV, 'scenarios': load scenarios Parquet, 'scenarios_plot': load scenarios plot HTML, 'scenarios_log': load scenario generation log

Dict[str, str]

}

Note

The function creates output files under {settings.data_dir}/{network.name}/raw/:

  • tqdm.log: Progress tracking
  • error.log: Error messages
  • args.log: Configuration parameters (YAML dump, appended per run)
  • config.yaml: Run configuration, rewritten each run
  • bus_data.parquet: Bus-level features for each scenario
  • branch_data.parquet: Branch-level features for each scenario
  • gen_data.parquet: Generator features for each scenario
  • y_bus_data.parquet: Nonzero Y-bus entries for each scenario
  • scenarios_{generator}.parquet: Load scenarios (per-element time series)
  • scenarios_{generator}.html: Load scenario plots
  • scenarios_{generator}.log: Load scenario generation notes
Source code in gridfm_datakit/generate.py
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
def generate_power_flow_data(
    config: Union[str, Dict[str, Any], NestedNamespace],
) -> Dict[str, str]:
    """Generate power flow data based on the provided configuration using sequential processing.

    Args:
        config: Configuration can be provided in three ways:
            1. Path to a YAML config file (str)
            2. Configuration dictionary (Dict)
            3. NestedNamespace object (NestedNamespace)
            The config must include settings, network, load, and perturbation configurations.

    Returns:
        Dictionary with paths to generated artifacts:
        {
            'tqdm_log': progress log file,
            'error_log': error log file,
            'args_log': configuration dump file,
            'config': run configuration (YAML),
            'bus_data': bus-level features CSV (BUS_COLUMNS),
            'branch_data': branch-level features CSV (BRANCH_COLUMNS),
            'gen_data': generator features CSV (GEN_COLUMNS),
            'y_bus_data': Y-bus nonzero entries CSV,
            'scenarios': load scenarios Parquet,
            'scenarios_plot': load scenarios plot HTML,
            'scenarios_log': load scenario generation log
        }

    Note:
        The function creates output files under {settings.data_dir}/{network.name}/raw/:

        - tqdm.log: Progress tracking
        - error.log: Error messages
        - args.log: Configuration parameters (YAML dump, appended per run)
        - config.yaml: Run configuration, rewritten each run
        - bus_data.parquet: Bus-level features for each scenario
        - branch_data.parquet: Branch-level features for each scenario
        - gen_data.parquet: Generator features for each scenario
        - y_bus_data.parquet: Nonzero Y-bus entries for each scenario
        - scenarios_{generator}.parquet: Load scenarios (per-element time series)
        - scenarios_{generator}.html: Load scenario plots
        - scenarios_{generator}.log: Load scenario generation notes
    """

    # Setup environment
    args, base_path, file_paths, seed = _setup_environment(config)

    # Prepare network and scenarios
    net, scenarios, meta = _prepare_network_and_scenarios(args, file_paths, seed)

    # Initialize topology generator
    topology_generator = initialize_topology_generator(args.topology_perturbation, net)

    # Initialize generation generator
    generation_generator = initialize_generation_generator(
        args.generation_perturbation,
        net,
    )

    # Initialize admittance generator
    admittance_generator = initialize_admittance_generator(
        args.admittance_perturbation,
        net,
    )

    jl = init_julia(
        args.settings.max_iter,
        file_paths["solver_log_dir"],
        opf_formulation=args.settings.opf_formulation,
        coinhsl=args.settings.coinhsl.to_dict(),
    )

    processed_data = []
    # Flush results to disk periodically so memory stays bounded, mirroring
    # the distributed path. Configs without large_chunk_size keep the old
    # save-once-at-the-end behavior.
    flush_every = getattr(args.settings, "large_chunk_size", None)

    # Process scenarios sequentially with deterministic seed
    # Use custom_seed to control randomness for reproducibility
    with custom_seed(_DEFAULT_SEED_POLICY.sequential_seed(seed)):
        with open(file_paths["tqdm_log"], "a", encoding="utf-8") as f:
            with tqdm(
                total=args.load.scenarios,
                desc="Processing scenarios",
                file=Tee(sys.stdout, f),
                miniters=5,
            ) as pbar:
                for scenario_index in range(args.load.scenarios):
                    # Process the scenario
                    if args.settings.mode == "opf":
                        processed_data = process_scenario_opf_mode(
                            net,
                            scenarios,
                            scenario_index,
                            topology_generator,
                            generation_generator,
                            admittance_generator,
                            processed_data,
                            file_paths["error_log"],
                            args.settings.include_dc_res,
                            jl,
                        )
                    elif args.settings.mode == "pf":
                        processed_data = process_scenario_pf_mode(
                            net,
                            scenarios,
                            scenario_index,
                            topology_generator,
                            generation_generator,
                            admittance_generator,
                            processed_data,
                            file_paths["error_log"],
                            args.settings.include_dc_res,
                            args.settings.pf_fast,
                            args.settings.dcpf_fast,
                            jl,
                            args.settings.pf_solver,
                            meta=meta,
                        )
                    else:
                        raise ValueError("Invalid mode!")

                    pbar.update(1)

                    if flush_every and (scenario_index + 1) % flush_every == 0:
                        _save_generated_data(
                            net,
                            processed_data,
                            file_paths,
                            base_path,
                            args,
                        )
                        processed_data = []

    # Save final data
    _save_generated_data(
        net,
        processed_data,
        file_paths,
        base_path,
        args,
    )

    return file_paths

generate_power_flow_data_distributed

Generate power flow data based on the provided configuration using distributed processing.

settings.process_backend selects multiprocessing_pool for a spawned Pool with live progress, or process_pool_executor (the default). Both backends share worker initialization, chunk boundaries, and seed policy. With settings.enable_checkpointing, completed large chunks and the run seed are persisted; overwrite: false resumes compatible saved progress.

Parameters:

Name Type Description Default
config Union[str, Dict[str, Any], NestedNamespace]

Configuration can be provided in three ways: 1. Path to a YAML config file (str) 2. Configuration dictionary (Dict) 3. NestedNamespace object (NestedNamespace) The config must include settings, network, load, and perturbation configurations.

required

Returns:

Type Description
Dict[str, str]

Dictionary with paths to generated artifacts (same as generate_power_flow_data)

Note

The function creates output files under {settings.data_dir}/{network.name}/raw/:

  • tqdm.log: Progress tracking
  • error.log: Error messages
  • args.log: Configuration parameters (YAML dump, appended per run)
  • config.yaml: Run configuration, rewritten each run
  • bus_data.parquet: Bus-level features for each scenario
  • branch_data.parquet: Branch-level features for each scenario
  • gen_data.parquet: Generator features for each scenario
  • y_bus_data.parquet: Nonzero Y-bus entries for each scenario
  • scenarios_{generator}.parquet: Load scenarios (per-element time series)
  • scenarios_{generator}.html: Load scenario plots
  • scenarios_{generator}.log: Load scenario generation notes
Source code in gridfm_datakit/generate.py
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
def generate_power_flow_data_distributed(
    config: Union[str, Dict[str, Any], NestedNamespace],
) -> Dict[str, str]:
    """Generate power flow data based on the provided configuration using distributed processing.

    ``settings.process_backend`` selects ``multiprocessing_pool`` for a spawned
    Pool with live progress, or ``process_pool_executor`` (the default).
    Both backends share worker initialization, chunk boundaries, and seed policy.
    With ``settings.enable_checkpointing``, completed large chunks and the run
    seed are persisted; ``overwrite: false`` resumes compatible saved progress.

    Args:
        config: Configuration can be provided in three ways:
            1. Path to a YAML config file (str)
            2. Configuration dictionary (Dict)
            3. NestedNamespace object (NestedNamespace)
            The config must include settings, network, load, and perturbation configurations.

    Returns:
        Dictionary with paths to generated artifacts (same as generate_power_flow_data)

    Note:
        The function creates output files under {settings.data_dir}/{network.name}/raw/:

        - tqdm.log: Progress tracking
        - error.log: Error messages
        - args.log: Configuration parameters (YAML dump, appended per run)
        - config.yaml: Run configuration, rewritten each run
        - bus_data.parquet: Bus-level features for each scenario
        - branch_data.parquet: Branch-level features for each scenario
        - gen_data.parquet: Generator features for each scenario
        - y_bus_data.parquet: Nonzero Y-bus entries for each scenario
        - scenarios_{generator}.parquet: Load scenarios (per-element time series)
        - scenarios_{generator}.html: Load scenario plots
        - scenarios_{generator}.log: Load scenario generation notes
    """
    # Setup environment
    args, base_path, file_paths, seed = _setup_environment(
        config,
        allow_checkpointing=True,
    )

    scenario_count = args.load.scenarios
    large_chunk_size = args.settings.large_chunk_size
    configured_processes = args.settings.num_processes

    # Match np.array_split's balanced boundaries and preserve full-run chunking
    # and worker counts even when resuming.
    n_large_chunks = (scenario_count + large_chunk_size - 1) // large_chunk_size
    large_chunks = _split_range(0, scenario_count, n_large_chunks)
    checkpoint = None
    if args.settings.enable_checkpointing:
        checkpoint = GenerationCheckpoint.load(Path(base_path), args.to_dict())
        if checkpoint is None:
            checkpoint = GenerationCheckpoint.create(
                Path(base_path),
                args.to_dict(),
                seed,
            )
        else:
            checkpoint.recover()
        if checkpoint.state.next_chunk == n_large_chunks:
            print("All large chunks are already checkpointed; generation is complete.")
            return file_paths

    # Prepare network and scenarios
    net, scenarios, meta = _prepare_network_and_scenarios(args, file_paths, seed)

    # Initialize topology generator
    topology_generator = initialize_topology_generator(args.topology_perturbation, net)

    # Initialize generation generator
    generation_generator = initialize_generation_generator(
        args.generation_perturbation,
        net,
    )

    # Initialize admittance generator
    admittance_generator = initialize_admittance_generator(
        args.admittance_perturbation,
        net,
    )

    max_large_chunk_size = max(stop - start for start, stop in large_chunks)
    worker_count = min(configured_processes, max_large_chunk_size)
    process_backend = getattr(args.settings, "process_backend", "process_pool_executor")
    use_multiprocessing = process_backend == "multiprocessing_pool"

    # pp_net wraps JVM state and cannot cross process boundaries. Spawned
    # workers reload it once from network_path; the in-process path can reuse it.
    worker_meta = (
        meta
        if worker_count == 1 and not use_multiprocessing
        else {key: value for key, value in meta.items() if key != "pp_net"}
    )
    worker_initargs = (
        args.settings.mode,
        net,
        topology_generator,
        generation_generator,
        admittance_generator,
        file_paths["error_log"],
        args.settings.include_dc_res,
        args.settings.pf_fast,
        args.settings.dcpf_fast,
        file_paths["solver_log_dir"],
        args.settings.max_iter,
        seed,
        args.settings.pf_solver,
        worker_meta,
        args.settings.opf_formulation,
        args.settings.coinhsl.to_dict(),
    )

    with open(file_paths["tqdm_log"], "a", encoding="utf-8") as f:
        with tqdm(
            total=scenario_count,
            desc="Processing scenarios",
            file=Tee(sys.stdout, f),
            miniters=5,
        ) as pbar:

            def process_large_chunks(
                executor: Optional[ProcessPoolExecutor] = None,
                pool: Optional[Pool] = None,
                progress_queue: Optional[Any] = None,
            ) -> None:
                for chunk_index, (large_start, large_stop) in enumerate(large_chunks):
                    if (
                        checkpoint is not None
                        and chunk_index < checkpoint.state.next_chunk
                    ):
                        pbar.update(large_stop - large_start)
                        continue
                    write_ram_usage_distributed(f)
                    chunk_size = large_stop - large_start
                    scenario_chunks = _split_range(
                        large_start,
                        large_stop,
                        min(worker_count, chunk_size),
                    )
                    results_by_start = {}

                    if pool is not None:
                        tasks = [
                            (start, np.ascontiguousarray(scenarios[:, start:stop, :]))
                            for start, stop in scenario_chunks
                        ]
                        results_by_start = _collect_pool_results(
                            pool,
                            tasks,
                            progress_queue,
                            pbar,
                        )
                    elif executor is None:
                        for start, stop in scenario_chunks:
                            result = _process_scenario_worker(
                                (start, scenarios[:, start:stop, :]),
                            )
                            results_by_start[start] = result
                    else:
                        futures = {}
                        for start, stop in scenario_chunks:
                            # A contiguous slice is serialized roughly twice as
                            # fast as a strided view and contains only this task's
                            # scenarios, not the tensor for the entire run.
                            scenario_slice = np.ascontiguousarray(
                                scenarios[:, start:stop, :],
                            )
                            future = executor.submit(
                                _process_scenario_worker,
                                (start, scenario_slice),
                            )
                            futures[future] = (start, stop)

                        for future in as_completed(futures):
                            start, stop = futures[future]
                            try:
                                results_by_start[start] = future.result()
                            except Exception as error:
                                raise RuntimeError(
                                    f"Scenario worker failed for [{start}, {stop})",
                                ) from error

                    processed_data = []
                    for start in sorted(results_by_start):
                        (
                            returned_start,
                            processed_count,
                            error,
                            traceback_text,
                            local_processed_data,
                        ) = results_by_start[start]
                        if returned_start != start:
                            raise RuntimeError(
                                f"Scenario worker returned chunk {returned_start}, expected {start}",
                            )
                        if error is not None:
                            raise RuntimeError(
                                f"Scenario chunk starting at {start} failed: {error}\n"
                                f"{traceback_text}",
                            ) from error
                        if local_processed_data is None:
                            raise RuntimeError(
                                f"Scenario chunk starting at {start} returned no data",
                            )
                        if progress_queue is None:
                            pbar.update(processed_count)
                        processed_data.extend(local_processed_data)

                    # Save processed data
                    _save_generated_data(
                        net,
                        processed_data,
                        file_paths,
                        base_path,
                        args,
                    )
                    if checkpoint is not None:
                        checkpoint.commit(chunk_index + 1)

            if use_multiprocessing:
                mp_context = multiprocessing.get_context("spawn")
                with mp_context.Manager() as manager:
                    progress_queue = manager.Queue()
                    with mp_context.Pool(
                        processes=worker_count,
                        initializer=_initialize_scenario_worker,
                        initargs=worker_initargs + (progress_queue,),
                    ) as pool:
                        process_large_chunks(pool=pool, progress_queue=progress_queue)
            elif worker_count == 1:
                # Avoid spawning a process, serializing state, and starting a
                # second Python interpreter when no parallelism was requested.
                _initialize_scenario_worker(*worker_initargs)
                process_large_chunks()
            else:
                mp_context = multiprocessing.get_context("spawn")
                with ProcessPoolExecutor(
                    max_workers=worker_count,
                    mp_context=mp_context,
                    initializer=_initialize_scenario_worker,
                    initargs=worker_initargs,
                ) as executor:
                    process_large_chunks(executor)

    return file_paths

_collect_pool_results

Collect pool results by chunk offset while displaying scenario progress.

Parameters:

Name Type Description Default
pool Pool

Spawned pool initialized with the run's worker state.

required
tasks List[Tuple[int, ndarray]]

Global offsets paired with contiguous scenario slices.

required
progress_queue Any

Manager queue receiving completed-scenario counts.

required
progress_bar tqdm

Parent-process progress display.

required

Returns:

Type Description
Dict[int, Tuple]

Worker results keyed by their submitted global offsets.

Raises:

Type Description
RuntimeError

If a task raises or a worker reports a chunk failure.

Source code in gridfm_datakit/generate.py
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
def _collect_pool_results(
    pool: Pool,
    tasks: List[Tuple[int, np.ndarray]],
    progress_queue: Any,
    progress_bar: tqdm,
) -> Dict[int, Tuple]:
    """Collect pool results by chunk offset while displaying scenario progress.

    Args:
        pool: Spawned pool initialized with the run's worker state.
        tasks: Global offsets paired with contiguous scenario slices.
        progress_queue: Manager queue receiving completed-scenario counts.
        progress_bar: Parent-process progress display.

    Returns:
        Worker results keyed by their submitted global offsets.

    Raises:
        RuntimeError: If a task raises or a worker reports a chunk failure.
    """
    pending = {
        start: pool.apply_async(_process_scenario_worker, ((start, scenarios),))
        for start, scenarios in tasks
    }
    results = {}
    while pending:
        for start in list(pending):
            if not pending[start].ready():
                continue
            try:
                result = pending.pop(start).get()
            except Exception as error:
                raise RuntimeError(
                    f"Scenario worker failed for chunk starting at {start}",
                ) from error
            if result[2] is not None:
                raise RuntimeError(
                    f"Scenario chunk starting at {start} failed: {result[2]}\n{result[3]}",
                ) from result[2]
            results[start] = result
        try:
            progress_bar.update(progress_queue.get(timeout=0.1))
        except Empty:
            pass

    # Queue puts complete before a worker returns; drain final ticks before
    # the next large chunk so progress cannot be counted twice or leak across chunks.
    while True:
        try:
            progress_bar.update(progress_queue.get_nowait())
        except Empty:
            break
    return results