Skip to content

Checkpoint

Distributed static generation uses GenerationCheckpoint when settings.enable_checkpointing is enabled. See the checkpoint and resume manual.

Persist completed chunks and recover an interrupted parquet append.

Attributes:

Name Type Description
root

Output directory containing checkpoint.json and the datasets.

state

Last successfully published checkpoint.

Source code in gridfm_datakit/checkpoint.py
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
class GenerationCheckpoint:
    """Persist completed chunks and recover an interrupted parquet append.

    Attributes:
        root: Output directory containing checkpoint.json and the datasets.
        state: Last successfully published checkpoint.
    """

    def __init__(self, root: Path, state: _CheckpointState) -> None:
        """Bind a validated checkpoint to its output directory.

        Args:
            root: Generation output directory.
            state: Validated saved progress and file manifest.
        """
        self.root = root
        self.state = state

    @classmethod
    def load(
        cls,
        root: Path,
        config: dict[str, Any],
    ) -> Optional["GenerationCheckpoint"]:
        """Check resume eligibility before existing logs or output are changed.

        Args:
            root: Generation output directory.
            config: Validated requested configuration.

        Returns:
            Existing checkpoint, or None for a fresh run or ordinary append.

        Raises:
            ValueError: If settings, checkpoint state, or committed files disagree.
        """
        settings = config["settings"]
        if settings["overwrite"]:
            return None
        path = root / "checkpoint.json"
        enabled = settings.get("enable_checkpointing", False)
        if not path.exists():
            if enabled and any(
                (root / name).exists() for name in (*_DATASETS, "n_scenarios.txt")
            ):
                raise ValueError(
                    "Cannot resume existing output without checkpoint.json; use a new output directory or overwrite: true",
                )
            return None
        if not enabled:
            raise ValueError(
                "Existing output has a checkpoint; keep enable_checkpointing: true to resume or use overwrite: true",
            )
        try:
            state = _CheckpointState.model_validate_json(
                path.read_text(encoding="utf-8"),
            )
        except (ValueError, OSError) as error:
            raise ValueError(f"Cannot read checkpoint {path}: {error}") from error
        if _resume_config(config, state.seed) != state.config:
            raise ValueError(
                "Resume configuration differs from checkpoint; keep the original scenario, chunk, worker, seed, and solver settings",
            )
        chunks = (
            config["load"]["scenarios"] + settings["large_chunk_size"] - 1
        ) // settings["large_chunk_size"]
        if (
            state.next_chunk > chunks
            or (state.next_chunk == 0 and (state.files or state.saved_scenarios))
            or (state.files and not state.saved_scenarios)
        ):
            raise ValueError("Checkpoint contains inconsistent chunk progress")
        for name, size in state.files.items():
            relative = PurePosixPath(name)
            if (
                relative.is_absolute()
                or ".." in relative.parts
                or len(relative.parts) != 3
                or relative.parts[0] not in _DATASETS
                or not relative.parts[1].startswith("scenario_partition=")
                or relative.suffix != ".parquet"
            ):
                raise ValueError(f"Invalid committed file path in checkpoint: {name}")
            file = root / name
            if not file.is_file() or file.stat().st_size != size:
                raise ValueError(
                    f"Committed checkpoint file is missing or changed: {name}",
                )
        if state.saved_scenarios and any(
            not any(PurePosixPath(name).parts[0] == dataset for name in state.files)
            for dataset in _DATASETS
        ):
            raise ValueError("Checkpoint is missing a committed output dataset")
        return cls(root, state)

    @classmethod
    def create(
        cls,
        root: Path,
        config: dict[str, Any],
        seed: int,
    ) -> "GenerationCheckpoint":
        """Persist a fresh seed before any scenarios are generated.

        Args:
            root: Empty output directory prepared by generation setup.
            config: Validated generation configuration.
            seed: Effective random seed for this run.

        Returns:
            Newly published checkpoint with no completed chunks.
        """
        checkpoint = cls(
            root,
            _CheckpointState(seed=seed, config=_resume_config(config, seed)),
        )
        checkpoint._publish(checkpoint.state)
        return checkpoint

    def _files(self) -> dict[str, int]:
        """Return sizes of parquet files belonging to the five output datasets."""
        return {
            path.relative_to(self.root).as_posix(): path.stat().st_size
            for dataset in _DATASETS
            for path in (self.root / dataset).rglob("*.parquet")
        }

    def _publish(self, state: _CheckpointState) -> None:
        """Persist state before replacing the last committed in-memory state.

        Args:
            state: New progress and file manifest to publish atomically.
        """
        _atomic_write(self.root / "checkpoint.json", state.model_dump_json(indent=2))
        self.state = state

    def recover(self) -> None:
        """Remove uncommitted output files and restore the committed sample count."""
        for name in self._files().keys() - self.state.files.keys():
            (self.root / name).unlink()
        _atomic_write(self.root / "n_scenarios.txt", str(self.state.saved_scenarios))

    def commit(self, next_chunk: int) -> None:
        """Advance only after all dataset writes for a large chunk succeeded.

        Args:
            next_chunk: Index of the first large chunk not yet saved.
        """
        count = self.root / "n_scenarios.txt"
        saved = int(count.read_text(encoding="utf-8")) if count.exists() else 0
        self._publish(
            _CheckpointState(
                seed=self.state.seed,
                config=self.state.config,
                next_chunk=next_chunk,
                saved_scenarios=saved,
                files=self._files(),
            ),
        )
__init__(root, state)

Bind a validated checkpoint to its output directory.

Parameters:

Name Type Description Default
root Path

Generation output directory.

required
state _CheckpointState

Validated saved progress and file manifest.

required
Source code in gridfm_datakit/checkpoint.py
89
90
91
92
93
94
95
96
97
def __init__(self, root: Path, state: _CheckpointState) -> None:
    """Bind a validated checkpoint to its output directory.

    Args:
        root: Generation output directory.
        state: Validated saved progress and file manifest.
    """
    self.root = root
    self.state = state
commit(next_chunk)

Advance only after all dataset writes for a large chunk succeeded.

Parameters:

Name Type Description Default
next_chunk int

Index of the first large chunk not yet saved.

required
Source code in gridfm_datakit/checkpoint.py
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
def commit(self, next_chunk: int) -> None:
    """Advance only after all dataset writes for a large chunk succeeded.

    Args:
        next_chunk: Index of the first large chunk not yet saved.
    """
    count = self.root / "n_scenarios.txt"
    saved = int(count.read_text(encoding="utf-8")) if count.exists() else 0
    self._publish(
        _CheckpointState(
            seed=self.state.seed,
            config=self.state.config,
            next_chunk=next_chunk,
            saved_scenarios=saved,
            files=self._files(),
        ),
    )
create(root, config, seed) classmethod

Persist a fresh seed before any scenarios are generated.

Parameters:

Name Type Description Default
root Path

Empty output directory prepared by generation setup.

required
config dict[str, Any]

Validated generation configuration.

required
seed int

Effective random seed for this run.

required

Returns:

Type Description
GenerationCheckpoint

Newly published checkpoint with no completed chunks.

Source code in gridfm_datakit/checkpoint.py
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
@classmethod
def create(
    cls,
    root: Path,
    config: dict[str, Any],
    seed: int,
) -> "GenerationCheckpoint":
    """Persist a fresh seed before any scenarios are generated.

    Args:
        root: Empty output directory prepared by generation setup.
        config: Validated generation configuration.
        seed: Effective random seed for this run.

    Returns:
        Newly published checkpoint with no completed chunks.
    """
    checkpoint = cls(
        root,
        _CheckpointState(seed=seed, config=_resume_config(config, seed)),
    )
    checkpoint._publish(checkpoint.state)
    return checkpoint
load(root, config) classmethod

Check resume eligibility before existing logs or output are changed.

Parameters:

Name Type Description Default
root Path

Generation output directory.

required
config dict[str, Any]

Validated requested configuration.

required

Returns:

Type Description
Optional[GenerationCheckpoint]

Existing checkpoint, or None for a fresh run or ordinary append.

Raises:

Type Description
ValueError

If settings, checkpoint state, or committed files disagree.

Source code in gridfm_datakit/checkpoint.py
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
@classmethod
def load(
    cls,
    root: Path,
    config: dict[str, Any],
) -> Optional["GenerationCheckpoint"]:
    """Check resume eligibility before existing logs or output are changed.

    Args:
        root: Generation output directory.
        config: Validated requested configuration.

    Returns:
        Existing checkpoint, or None for a fresh run or ordinary append.

    Raises:
        ValueError: If settings, checkpoint state, or committed files disagree.
    """
    settings = config["settings"]
    if settings["overwrite"]:
        return None
    path = root / "checkpoint.json"
    enabled = settings.get("enable_checkpointing", False)
    if not path.exists():
        if enabled and any(
            (root / name).exists() for name in (*_DATASETS, "n_scenarios.txt")
        ):
            raise ValueError(
                "Cannot resume existing output without checkpoint.json; use a new output directory or overwrite: true",
            )
        return None
    if not enabled:
        raise ValueError(
            "Existing output has a checkpoint; keep enable_checkpointing: true to resume or use overwrite: true",
        )
    try:
        state = _CheckpointState.model_validate_json(
            path.read_text(encoding="utf-8"),
        )
    except (ValueError, OSError) as error:
        raise ValueError(f"Cannot read checkpoint {path}: {error}") from error
    if _resume_config(config, state.seed) != state.config:
        raise ValueError(
            "Resume configuration differs from checkpoint; keep the original scenario, chunk, worker, seed, and solver settings",
        )
    chunks = (
        config["load"]["scenarios"] + settings["large_chunk_size"] - 1
    ) // settings["large_chunk_size"]
    if (
        state.next_chunk > chunks
        or (state.next_chunk == 0 and (state.files or state.saved_scenarios))
        or (state.files and not state.saved_scenarios)
    ):
        raise ValueError("Checkpoint contains inconsistent chunk progress")
    for name, size in state.files.items():
        relative = PurePosixPath(name)
        if (
            relative.is_absolute()
            or ".." in relative.parts
            or len(relative.parts) != 3
            or relative.parts[0] not in _DATASETS
            or not relative.parts[1].startswith("scenario_partition=")
            or relative.suffix != ".parquet"
        ):
            raise ValueError(f"Invalid committed file path in checkpoint: {name}")
        file = root / name
        if not file.is_file() or file.stat().st_size != size:
            raise ValueError(
                f"Committed checkpoint file is missing or changed: {name}",
            )
    if state.saved_scenarios and any(
        not any(PurePosixPath(name).parts[0] == dataset for name in state.files)
        for dataset in _DATASETS
    ):
        raise ValueError("Checkpoint is missing a committed output dataset")
    return cls(root, state)
recover()

Remove uncommitted output files and restore the committed sample count.

Source code in gridfm_datakit/checkpoint.py
217
218
219
220
221
def recover(self) -> None:
    """Remove uncommitted output files and restore the committed sample count."""
    for name in self._files().keys() - self.state.files.keys():
        (self.root / name).unlink()
    _atomic_write(self.root / "n_scenarios.txt", str(self.state.saved_scenarios))