|
3 | 3 | import os |
4 | 4 | import time |
5 | 5 | from pathlib import PurePath |
6 | | -from typing import Any, Optional, List, Union |
| 6 | +from typing import Any, Optional, List, Union, Dict |
7 | 7 | from collections import OrderedDict |
8 | 8 |
|
9 | 9 | from .codec import codec_registry, CodecRegistry |
@@ -55,16 +55,101 @@ def codec_registry(self) -> CodecRegistry: |
55 | 55 | """ |
56 | 56 | The registry of codecs associated to this instance of a store. |
57 | 57 |
|
58 | | - It is not necessaily unique |
| 58 | + It is not necessarily unique |
59 | 59 | It may not be called for mutable operations during an evaluation. In that case, the behavior is not defined. |
60 | 60 | """ |
61 | 61 | pass |
62 | 62 |
|
63 | 63 | # TODO: reset paths to start the store from scratch without losing data |
64 | 64 |
|
65 | 65 |
|
66 | | -# TODO: add a notion of FileSystemType (Local, DBFS, S3) |
67 | | -# We need to have a matrix between FS types and object types |
| 66 | +class NoOpStore(Store): |
| 67 | + """ |
| 68 | + The store that never stores an object. |
| 69 | +
|
| 70 | + This store is in practice of very limited value because it cannot store paths to an object either. |
| 71 | + As a result, dds.load() will not work correctly with this store. |
| 72 | +
|
| 73 | + It is recommended to use this store only to debug specific issues for which DDS would be disabled |
| 74 | + altogether. |
| 75 | + """ |
| 76 | + |
| 77 | + def has_blob(self, key: PyHash) -> bool: |
| 78 | + return False |
| 79 | + |
| 80 | + def fetch_blob(self, key: PyHash) -> Optional[Any]: |
| 81 | + raise DDSException(f"Blob {key} not store (NoOpStore)") |
| 82 | + |
| 83 | + def store_blob(self, key: PyHash, blob: Any, codec: Optional[ProtocolRef]) -> None: |
| 84 | + return |
| 85 | + |
| 86 | + def sync_paths(self, paths: "OrderedDict[DDSPath, PyHash]") -> None: |
| 87 | + """ |
| 88 | + Commits all the paths. |
| 89 | + """ |
| 90 | + return |
| 91 | + |
| 92 | + def fetch_paths(self, paths: List[DDSPath]) -> "OrderedDict[DDSPath, PyHash]": |
| 93 | + """ |
| 94 | + Fetches a set of paths from the store. It is expected that all the paths are returned. |
| 95 | + """ |
| 96 | + raise DDSException(f"Cannot fetch paths (the NoOpStore does not store paths)") |
| 97 | + |
| 98 | + def codec_registry(self) -> CodecRegistry: |
| 99 | + return codec_registry() |
| 100 | + |
| 101 | + |
| 102 | +class MemoryStore(Store): |
| 103 | + """ |
| 104 | + The store that stores all objects in memory, without saving them permanently in storage. |
| 105 | + It is an good example of how to implement a store that is fully functional. |
| 106 | +
|
| 107 | + This store is useful when the following conditions are met: |
| 108 | + - there is limited value in storing objects beyond the lifetime of the process |
| 109 | + - some complex objects are not serializable |
| 110 | + - the objects are not too large in memory |
| 111 | +
|
| 112 | + This store is not useful for most users, but is useful in debugging or testing context. |
| 113 | + """ |
| 114 | + |
| 115 | + def __init__(self): |
| 116 | + self._cache: Dict[PyHash, Any] = {} |
| 117 | + self._paths: Dict[DDSPath, PyHash] = {} |
| 118 | + |
| 119 | + def has_blob(self, key: PyHash) -> bool: |
| 120 | + return key in self._cache |
| 121 | + |
| 122 | + def fetch_blob(self, key: PyHash) -> Optional[Any]: |
| 123 | + return self._cache.get(key) |
| 124 | + |
| 125 | + def store_blob(self, key: PyHash, blob: Any, codec: Optional[ProtocolRef]) -> None: |
| 126 | + if key in self._cache: |
| 127 | + _logger.warning(f"Overwriting key {key}") |
| 128 | + self._cache[key] = blob |
| 129 | + |
| 130 | + def sync_paths(self, paths: "OrderedDict[DDSPath, PyHash]") -> None: |
| 131 | + """ |
| 132 | + Commits all the paths. |
| 133 | + """ |
| 134 | + for (p, k) in paths.items(): |
| 135 | + if p in self._paths: |
| 136 | + _logger.debug(f"Overwriting path: {p} -> {k}") |
| 137 | + else: |
| 138 | + _logger.debug(f"Registering path: {p} -> {k}") |
| 139 | + self._paths[p] = k |
| 140 | + |
| 141 | + def fetch_paths(self, paths: List[DDSPath]) -> "OrderedDict[DDSPath, PyHash]": |
| 142 | + """ |
| 143 | + Fetches a set of paths from the store. It is expected that all the paths are returned. |
| 144 | + """ |
| 145 | + missing_paths = [p for p in paths if p not in self._paths] |
| 146 | + if missing_paths: |
| 147 | + raise DDSException(f"Missing paths in store: {missing_paths}") |
| 148 | + return OrderedDict([(p, self._paths[p]) for p in paths]) |
| 149 | + |
| 150 | + def codec_registry(self) -> CodecRegistry: |
| 151 | + # All the default content |
| 152 | + return codec_registry() |
68 | 153 |
|
69 | 154 |
|
70 | 155 | class LocalFileStore(Store): |
|
0 commit comments