Repository navigation
Expand file tree
/
Copy pathtest_content_store_ensure_binding.py
More file actions
436 lines (359 loc) · 14.3 KB
/
Copy pathtest_content_store_ensure_binding.py
File metadata and controls
436 lines (359 loc) · 14.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
"""Tests for Order-submitted vs ContentMesh bootstrap ContentStore binding."""
import hashlib
import importlib
import importlib.util
import shutil
import sys
import textwrap
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from cats.executor.structure import InfraStructure
from cats.network import (
ContentMesh,
_bootstrap_content_store_utils_path,
)
from cats import CATS_HOME
REPO_ROOT = Path(__file__).resolve().parents[1]
REPO_CONTENT_STORE_UTILS = (
REPO_ROOT
/ 'data'
/ 'input'
/ 'structure'
/ 'infrastructure'
/ 'content_store_utils.py'
)
def _write_fake_content_store_utils(path: Path, marker_name: str, *, ready=True):
"""Write a tiny content_store_utils that records ensure / is_ready calls."""
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
textwrap.dedent(
f"""\
class ContentStore:
ensured = []
probed = []
_ready = {ready!r}
@classmethod
def ensure(cls, cwd=None):
cls.ensured.append({{'cwd': cwd, 'marker': {marker_name!r}}})
@classmethod
def is_ready(cls):
cls.probed.append({{'marker': {marker_name!r}}})
return cls._ready
"""
),
encoding='utf-8',
)
def _infra_with_structure_home(structure_home: Path, cats_home: Path):
runtime = SimpleNamespace(
INPUT_STRUCTURE_HOME=str(structure_home),
CATS_HOME=str(cats_home),
executeCMD=MagicMock(),
)
infra = InfraStructure.__new__(InfraStructure)
infra.runtime = runtime
infra.structure_cid = 'bafy-test'
infra.INPUT_STRUCTURE_HOME = str(structure_home)
return infra
def test_bootstrap_path_never_under_input_structure_home():
"""Bootstrap ContentStore utils live under checkout input/, never jobs/."""
path = _bootstrap_content_store_utils_path(CATS_HOME)
assert path is not None
assert 'INPUT_STRUCTURE_HOME' not in path
# Job-scoped structure homes live under data/jobs — bootstrap must not.
assert '/jobs/' not in path.replace('\\', '/')
assert path.endswith(
'data/input/structure/infrastructure/content_store_utils.py'
) or path.endswith(
'data\\input\\structure\\infrastructure\\content_store_utils.py'
)
def test_order_loader_path_is_under_input_structure_home(tmp_path):
"""InfraStructure loads ContentStore utils from the Order structure tree."""
structure_home = tmp_path / 'order_structure'
utils_path = (
structure_home
/ 'infrastructure'
/ 'content_store_utils.py'
)
_write_fake_content_store_utils(utils_path, 'order-submitted')
infra = _infra_with_structure_home(structure_home, tmp_path / 'cats_home')
mod = infra._load_content_store_module()
assert hasattr(mod, 'ContentStore')
# Module was loaded from Order tree (fake has marker via ensure).
mod.ContentStore.ensure(cwd='x')
assert mod.ContentStore.ensured[-1]['marker'] == 'order-submitted'
def test_repo_content_store_utils_required_public_api():
"""Repo content_store_utils exposes ContentStore ensure/is_ready and CLI."""
spec = importlib.util.spec_from_file_location(
'content_store_utils_api_check', REPO_CONTENT_STORE_UTILS
)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
sys.modules[spec.name] = module
spec.loader.exec_module(module)
assert hasattr(module, 'ContentStore')
assert callable(module.ContentStore.ensure)
assert callable(module.ContentStore.is_ready)
assert callable(module._ipfs_api_id_url)
assert callable(module._main)
assert module._main(['status']) in (0, 1)
def test_content_store_utils_sha256_matches_when_order_tree_copied_from_repo(
tmp_path,
):
"""Demo invariant: Order tree copied from repo checkout has no republish lag."""
src = REPO_CONTENT_STORE_UTILS
assert src.is_file()
order_utils = (
tmp_path
/ 'structure'
/ 'infrastructure'
/ 'content_store_utils.py'
)
order_utils.parent.mkdir(parents=True)
shutil.copy2(src, order_utils)
def _sha(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
assert _sha(src) == _sha(order_utils)
def test_infrastructure_content_store_ensure_loads_order_tree(tmp_path):
"""À la carte API still loads Order-submitted utils (not used by apply)."""
structure_home = tmp_path / 'order_structure'
utils_path = (
structure_home
/ 'infrastructure'
/ 'content_store_utils.py'
)
_write_fake_content_store_utils(utils_path, 'order-submitted')
infra = _infra_with_structure_home(structure_home, tmp_path / 'cats_home')
infra.content_store_ensure(cwd=infra.runtime.CATS_HOME)
mod = importlib.import_module('infrastructure_content_store_utils_order')
assert mod.ContentStore.ensured == [
{'cwd': infra.runtime.CATS_HOME, 'marker': 'order-submitted'}
]
def test_content_store_assert_passes_when_ready(tmp_path):
"""content_store_assert soft-succeeds when Order-tree ContentStore is ready."""
structure_home = tmp_path / 'order_structure'
utils_path = (
structure_home
/ 'infrastructure'
/ 'content_store_utils.py'
)
_write_fake_content_store_utils(utils_path, 'order-submitted', ready=True)
infra = _infra_with_structure_home(structure_home, tmp_path / 'cats_home')
infra.content_store_assert()
def test_content_store_assert_soft_when_not_ready(tmp_path, capsys):
"""content_store_assert warns but does not raise when ContentStore is down (§6s)."""
structure_home = tmp_path / 'order_structure'
utils_path = (
structure_home
/ 'infrastructure'
/ 'content_store_utils.py'
)
_write_fake_content_store_utils(utils_path, 'order-submitted', ready=False)
infra = _infra_with_structure_home(structure_home, tmp_path / 'cats_home')
infra.content_store_assert()
captured = capsys.readouterr().out
assert 'WARNING' in captured
mod = sys.modules['infrastructure_content_store_utils_order']
assert mod.ContentStore.ensured == []
def test_apply_soft_probes_content_store_not_transport(tmp_path, monkeypatch):
"""apply soft-probes ContentStore; no transport_assert / ensure (§6s)."""
structure_home = tmp_path / 'order_structure'
utils_path = (
structure_home
/ 'infrastructure'
/ 'content_store_utils.py'
)
_write_fake_content_store_utils(utils_path, 'order-submitted', ready=True)
infra = _infra_with_structure_home(structure_home, tmp_path / 'cats_home')
ensure_calls = []
monkeypatch.setattr(
infra,
'content_store_ensure',
lambda *a, **k: ensure_calls.append((a, k)),
)
monkeypatch.setattr(
'cats.executor.structure.infrastructure.configure_terraform_data_dir',
lambda *_a, **_k: None,
)
monkeypatch.setattr(
'cats.executor.structure.infrastructure.ensure_integration_cache_env',
lambda *_a, **_k: None,
)
monkeypatch.setattr(
'cats.executor.structure.infrastructure.terraform_bin',
lambda *_a, **_k: 'terraform',
)
monkeypatch.setattr(
'cats.executor.structure.infrastructure.require_docker_daemon',
lambda: None,
)
plant_utils = SimpleNamespace(
cleanup_stale_plant_state=lambda *_a, **_k: None,
)
plant = SimpleNamespace(_load_plant_utils=lambda: plant_utils)
monkeypatch.setattr(infra, 'compose', lambda: plant)
obj_mod = SimpleNamespace(
cleanup_stale_obj_store_state=lambda *_a, **_k: None,
)
monkeypatch.setattr(infra, '_load_obj_store_module', lambda: obj_mod)
content_assert_calls = []
monkeypatch.setattr(
infra,
'content_store_assert',
lambda: content_assert_calls.append('assert'),
)
infra.apply()
assert ensure_calls == []
assert content_assert_calls == ['assert']
assert not hasattr(InfraStructure, 'transport_assert')
infra.runtime.executeCMD.assert_called()
def test_destroy_runs_stale_structure_cleanup(tmp_path, monkeypatch):
"""destroy heals stale Plant/compose state before terraform destroy."""
structure_home = tmp_path / 'order_structure'
utils_path = (
structure_home
/ 'infrastructure'
/ 'content_store_utils.py'
)
_write_fake_content_store_utils(utils_path, 'order-submitted', ready=True)
infra = _infra_with_structure_home(structure_home, tmp_path / 'cats_home')
cleanup_calls = []
monkeypatch.setattr(
infra,
'_cleanup_stale_structure_state',
lambda: cleanup_calls.append('cleanup'),
)
monkeypatch.setattr(
'cats.executor.structure.infrastructure.configure_terraform_data_dir',
lambda *_a, **_k: None,
)
monkeypatch.setattr(
'cats.executor.structure.infrastructure.ensure_integration_cache_env',
lambda *_a, **_k: None,
)
monkeypatch.setattr(
'cats.executor.structure.infrastructure.terraform_bin',
lambda *_a, **_k: 'terraform',
)
monkeypatch.setattr(
'cats.executor.structure.infrastructure.require_docker_daemon',
lambda: None,
)
infra.destroy()
assert cleanup_calls == ['cleanup']
infra.runtime.executeCMD.assert_called()
assert 'destroy' in infra.runtime.executeCMD.call_args.args[0]
def test_apply_raises_when_docker_daemon_down(tmp_path, monkeypatch):
"""apply fails closed with a clear error when Docker is not running."""
structure_home = tmp_path / 'order_structure'
utils_path = (
structure_home
/ 'infrastructure'
/ 'content_store_utils.py'
)
_write_fake_content_store_utils(utils_path, 'order-submitted', ready=True)
infra = _infra_with_structure_home(structure_home, tmp_path / 'cats_home')
monkeypatch.setattr(
'cats.executor.structure.infrastructure.configure_terraform_data_dir',
lambda *_a, **_k: None,
)
monkeypatch.setattr(
'cats.executor.structure.infrastructure.ensure_integration_cache_env',
lambda *_a, **_k: None,
)
monkeypatch.setattr(
'cats.executor.structure.infrastructure.docker_daemon_ready',
lambda: False,
)
with pytest.raises(RuntimeError, match='Docker daemon is not running'):
infra.apply()
def test_meshclient_init_does_not_call_bootstrap_ensure(tmp_path, monkeypatch):
"""ContentMesh construction does not probe or ensure ContentStore."""
calls = []
def _spy(self):
calls.append('ensure')
self._bootstrap_content_store_ensured = True
monkeypatch.setattr(ContentMesh, 'ensure_bootstrap_content_store', _spy)
client = ContentMesh(ipfsClient=MagicMock(), CATS_HOME=str(tmp_path))
assert calls == []
assert client._bootstrap_content_store_ensured is False
def test_meshclient_put_dir_triggers_bootstrap_readiness_once(tmp_path, monkeypatch):
"""put_dir triggers bootstrap readiness checks (soft probe path)."""
calls = []
def _spy(self):
calls.append('ready_check')
self._bootstrap_content_store_ensured = True
monkeypatch.setattr(ContentMesh, 'ensure_bootstrap_content_store', _spy)
fake_ipfs = MagicMock()
client = ContentMesh(ipfsClient=fake_ipfs, CATS_HOME=str(tmp_path))
assert calls == []
payload = tmp_path / 'payload'
payload.mkdir()
(payload / 'f.txt').write_text('x', encoding='utf-8')
dir_id, dir_name = client.put_dir(str(payload))
assert dir_id.startswith('ni:///sha-256;')
assert dir_name == 'payload'
assert calls == ['ready_check']
# New trees mint on CAS — no Kubo add.
fake_ipfs.add.assert_not_called()
client.put_dir(str(payload))
assert calls == ['ready_check', 'ready_check']
def test_meshclient_put_dir_requires_cats_home(tmp_path, monkeypatch):
"""put_dir without CATS_HOME fails closed (no Kubo add fallback; §6r)."""
monkeypatch.setattr(
ContentMesh,
'ensure_bootstrap_content_store',
lambda self: None,
)
fake_ipfs = MagicMock()
client = ContentMesh(ipfsClient=fake_ipfs, CATS_HOME=None)
payload = tmp_path / 'payload'
payload.mkdir()
(payload / 'f.txt').write_text('x', encoding='utf-8')
with pytest.raises(RuntimeError, match='CATS_HOME is required'):
client.put_dir(str(payload))
fake_ipfs.add.assert_not_called()
def test_meshclient_put_file_requires_cats_home(tmp_path, monkeypatch):
"""put_file without CATS_HOME fails closed (no Kubo add fallback; §6r)."""
monkeypatch.setattr(
ContentMesh,
'ensure_bootstrap_content_store',
lambda self: None,
)
fake_ipfs = MagicMock()
client = ContentMesh(ipfsClient=fake_ipfs, CATS_HOME=None)
payload = tmp_path / 'f.txt'
payload.write_text('x', encoding='utf-8')
with pytest.raises(RuntimeError, match='CATS_HOME is required'):
client.put_file(str(payload))
fake_ipfs.add.assert_not_called()
def test_meshclient_bootstrap_probes_is_ready_not_ensure(tmp_path):
"""Bootstrap path probes is_ready and never calls ContentStore.ensure."""
cats_home = tmp_path / 'cats_home'
utils_path = (
cats_home
/ 'data'
/ 'input'
/ 'structure'
/ 'infrastructure'
/ 'content_store_utils.py'
)
_write_fake_content_store_utils(utils_path, 'bootstrap', ready=True)
client = ContentMesh(ipfsClient=MagicMock(), CATS_HOME=str(cats_home))
assert client._bootstrap_content_store_ensured is False
client.ensure_bootstrap_content_store()
mod = importlib.import_module('infrastructure_content_store_utils_bootstrap')
assert mod.ContentStore.ensured == []
assert mod.ContentStore.probed == [{'marker': 'bootstrap'}]
assert client._bootstrap_content_store_ensured is True
client.ensure_bootstrap_content_store()
assert mod.ContentStore.probed == [{'marker': 'bootstrap'}]
assert mod.ContentStore.ensured == []
def test_meshclient_bootstrap_skips_when_utils_missing(tmp_path):
"""Missing bootstrap utils soft-skips and marks bootstrap as checked."""
cats_home = tmp_path / 'empty_home'
cats_home.mkdir()
client = ContentMesh(ipfsClient=MagicMock(), CATS_HOME=str(cats_home))
client.ensure_bootstrap_content_store()
assert client._bootstrap_content_store_ensured is True