Skip to content

Utils

This module provides utility functions for the project.

Functions

get_num_scenarios

Get total number of scenarios from data directory.

Reads from n_scenarios.txt metadata file in the data directory.

Parameters:

Name Type Description Default
data_dir str

Directory containing parquet files and n_scenarios.txt

required

Returns:

Type Description
int

Total number of scenarios

Raises:

Type Description
ValueError

If n_scenarios.txt metadata file not found

Source code in gridfm_datakit/utils/utils.py
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
def get_num_scenarios(data_dir: str) -> int:
    """Get total number of scenarios from data directory.

    Reads from n_scenarios.txt metadata file in the data directory.

    Args:
        data_dir: Directory containing parquet files and n_scenarios.txt

    Returns:
        Total number of scenarios

    Raises:
        ValueError: If n_scenarios.txt metadata file not found
    """
    n_scenarios_file = os.path.join(data_dir, "n_scenarios.txt")
    if os.path.exists(n_scenarios_file):
        with open(n_scenarios_file, "r", encoding="utf-8") as f:
            return int(f.read().strip())

    else:
        print(
            f"No n_scenarios metadata file found in {data_dir}, using bus_data.parquet to get total number of scenarios",
        )
        return int(
            pd.read_parquet(
                os.path.join(data_dir, "bus_data.parquet"),
                engine="pyarrow",
            )["scenario"].max()
            + 1,
        )

read_partitions

Read sampled partition folders in parallel and concatenate them.

Source code in gridfm_datakit/utils/utils.py
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
def read_partitions(
    base_path: str,
    sampled: list,
    max_workers: int = None,
) -> pd.DataFrame:
    """Read sampled partition folders in parallel and concatenate them."""
    if max_workers is None:
        from os import cpu_count

        max_workers = min(32, cpu_count())  # sensible default

    dfs = []

    # Submit all partition reads to the ThreadPool
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        futures = {
            executor.submit(
                pd.read_parquet,
                os.path.join(base_path, f"scenario_partition={k}"),
                engine="pyarrow",
            ): k
            for k in sampled
        }

        # Collect results as they complete with tqdm
        for future in tqdm(
            as_completed(futures),
            total=len(futures),
            desc=f"Reading {len(sampled)} partitions from {base_path}",
        ):
            df = future.result()
            dfs.append(df)

    # Concatenate all partitions
    return pd.concat(dfs, ignore_index=True)

write_ram_usage_distributed

Source code in gridfm_datakit/utils/utils.py
112
113
114
115
116
117
118
119
120
121
122
123
def write_ram_usage_distributed(tqdm_log: TextIO) -> None:
    process = psutil.Process(os.getpid())  # Parent process
    mem_usage = process.memory_info().rss / 1024**2  # Parent memory in MB

    # Sum memory usage of all child processes
    for child in process.children(recursive=True):
        mem_usage += child.memory_info().rss / 1024**2

    _blocking_write(
        tqdm_log,
        f"Total RAM usage (Parent + Children): {mem_usage:.2f} MB\n",
    )

Classes

Tee

Source code in gridfm_datakit/utils/utils.py
126
127
128
129
130
131
132
133
134
135
136
137
class Tee:
    def __init__(self, *streams):
        self.streams = streams

    def write(self, data):
        for s in self.streams:
            _blocking_write(s, data)
            _retry_on_eagain(s.flush)

    def flush(self):
        for s in self.streams:
            _retry_on_eagain(s.flush)

Value specs

parse_value_spec

Parse a config entry into a value spec.

Parameters:

Name Type Description Default
spec Any

A number or a mapping with a distribution key.

required
path str

The config path named in every error.

required
floor Optional[Floor]

A lower bound every value must admit.

None

Returns:

Name Type Description
ValueSpec ValueSpec

The parsed spec.

Raises:

Type Description
ValueError

If the entry is malformed, a parameter is invalid, or a value can fail the floor. The message starts with f"{path}: ".

Source code in gridfm_datakit/utils/value_spec.py
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
def parse_value_spec(
    spec: Any,
    path: str,
    floor: Optional[Floor] = None,
) -> ValueSpec:
    """Parse a config entry into a value spec.

    Args:
        spec: A number or a mapping with a ``distribution`` key.
        path: The config path named in every error.
        floor: A lower bound every value must admit.

    Returns:
        ValueSpec: The parsed spec.

    Raises:
        ValueError: If the entry is malformed, a parameter is invalid, or a value
            can fail the floor. The message starts with ``f"{path}: "``.
    """
    if not isinstance(spec, Mapping):
        value = _number(spec, path, "value")
        _check_floor(value, path, floor)
        return Fixed(value)

    distribution = spec.get("distribution")
    if not isinstance(distribution, str) or distribution not in _KEYS:
        raise ValueError(
            f"{path}: distribution must be one of {sorted(_KEYS)}, "
            f"got {distribution!r}",
        )
    required, optional = _KEYS[distribution]
    unknown = sorted(set(spec) - required - optional - {"distribution"})
    if unknown:
        raise ValueError(f"{path}: unknown key(s) {unknown} for {distribution}")
    missing = sorted(required - set(spec))
    if missing:
        raise ValueError(f"{path}: missing key(s) {missing} for {distribution}")

    if distribution == "normal":
        return _parse_normal(spec, path, floor)
    if distribution == "uniform":
        return _parse_uniform(spec, path, floor)
    return _parse_choice(spec, path, floor)

ValueSpec

A value drawn once per use.

Source code in gridfm_datakit/utils/value_spec.py
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
class ValueSpec:
    """A value drawn once per use."""

    def sample(self, rng: np.random.Generator) -> Any:
        """Draw one value.

        Args:
            rng: The generator every draw comes from.

        Returns:
            Any: A float for numeric specs, the list element for a choice.
        """
        raise NotImplementedError

    def support(self) -> Tuple[Any, Any]:
        """Return the smallest and largest value a draw can take.

        Returns:
            Tuple[Any, Any]: The worst case bounds.
        """
        raise NotImplementedError
sample(rng)

Draw one value.

Parameters:

Name Type Description Default
rng Generator

The generator every draw comes from.

required

Returns:

Name Type Description
Any Any

A float for numeric specs, the list element for a choice.

Source code in gridfm_datakit/utils/value_spec.py
73
74
75
76
77
78
79
80
81
82
def sample(self, rng: np.random.Generator) -> Any:
    """Draw one value.

    Args:
        rng: The generator every draw comes from.

    Returns:
        Any: A float for numeric specs, the list element for a choice.
    """
    raise NotImplementedError
support()

Return the smallest and largest value a draw can take.

Returns:

Type Description
Tuple[Any, Any]

Tuple[Any, Any]: The worst case bounds.

Source code in gridfm_datakit/utils/value_spec.py
84
85
86
87
88
89
90
def support(self) -> Tuple[Any, Any]:
    """Return the smallest and largest value a draw can take.

    Returns:
        Tuple[Any, Any]: The worst case bounds.
    """
    raise NotImplementedError

Floor

Lower bound every drawn value must admit.

Parameters:

Name Type Description Default
bound float

The bound.

required
strict bool

Whether the bound itself is excluded.

required
Source code in gridfm_datakit/utils/value_spec.py
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
@dataclass(frozen=True)
class Floor:
    """Lower bound every drawn value must admit.

    Args:
        bound: The bound.
        strict: Whether the bound itself is excluded.
    """

    bound: float
    strict: bool

    def admits(self, x: float) -> bool:
        """Check a value against the floor.

        Args:
            x: The value.

        Returns:
            bool: Whether ``x`` lies above the bound, or on it when not strict.
        """
        return x > self.bound if self.strict else x >= self.bound
admits(x)

Check a value against the floor.

Parameters:

Name Type Description Default
x float

The value.

required

Returns:

Name Type Description
bool bool

Whether x lies above the bound, or on it when not strict.

Source code in gridfm_datakit/utils/value_spec.py
54
55
56
57
58
59
60
61
62
63
def admits(self, x: float) -> bool:
    """Check a value against the floor.

    Args:
        x: The value.

    Returns:
        bool: Whether ``x`` lies above the bound, or on it when not strict.
    """
    return x > self.bound if self.strict else x >= self.bound