Skip to content

Commit c11eedc

Browse files
committed
Removed unnecessary lazy imports
1 parent 73805d3 commit c11eedc

3 files changed

Lines changed: 329 additions & 39 deletions

File tree

sdk/python/feast/infra/online_stores/faiss_online_store.py

Lines changed: 84 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -43,16 +43,30 @@ def teardown(self):
4343
self.entity_keys = {}
4444

4545

46+
def _table_id(project: str, table: FeatureView, enable_versioning: bool = False) -> str:
47+
"""Compute the table key, including version suffix when versioning is enabled."""
48+
name = table.name
49+
if enable_versioning:
50+
# Prefer version_tag from the projection (set by version-qualified refs like @v2)
51+
# over current_version_number (the FV's active version in metadata).
52+
version = getattr(table.projection, "version_tag", None)
53+
if version is None:
54+
version = getattr(table, "current_version_number", None)
55+
if version is not None and version > 0:
56+
name = f"{table.name}_v{version}"
57+
return f"{project}_{name}"
58+
59+
4660
class FaissOnlineStore(OnlineStore):
47-
_index: Optional[faiss.IndexIVFFlat] = None
48-
_in_memory_store: InMemoryStore = InMemoryStore()
61+
_indices: Dict[str, faiss.IndexIVFFlat] = {}
62+
_in_memory_stores: Dict[str, InMemoryStore] = {}
4963
_config: Optional[FaissOnlineStoreConfig] = None
5064
_logger: logging.Logger = logging.getLogger(__name__)
5165

52-
def _get_index(self, config: RepoConfig) -> faiss.IndexIVFFlat:
53-
if self._index is None or self._config is None:
54-
raise ValueError("Index is not initialized")
55-
return self._index
66+
def _get_index(
67+
self, table_key: str
68+
) -> Optional[faiss.IndexIVFFlat]:
69+
return self._indices.get(table_key)
5670

5771
def update(
5872
self,
@@ -63,32 +77,47 @@ def update(
6377
entities_to_keep: Sequence[Entity],
6478
partial: bool,
6579
):
66-
feature_views = tables_to_keep
67-
if not feature_views:
68-
return
69-
70-
feature_names = [f.name for f in feature_views[0].features]
71-
dimension = len(feature_names)
72-
7380
self._config = FaissOnlineStoreConfig(**config.online_store.dict())
74-
if self._index is None or not partial:
75-
quantizer = faiss.IndexFlatL2(dimension)
76-
self._index = faiss.IndexIVFFlat(quantizer, dimension, self._config.nlist)
77-
self._index.train(
78-
np.random.rand(self._config.nlist * 100, dimension).astype(np.float32)
79-
)
80-
self._in_memory_store = InMemoryStore()
81+
versioning = config.registry.enable_online_feature_view_versioning
82+
83+
for table in tables_to_delete:
84+
table_key = _table_id(config.project, table, versioning)
85+
self._indices.pop(table_key, None)
86+
self._in_memory_stores.pop(table_key, None)
87+
88+
for table in tables_to_keep:
89+
table_key = _table_id(config.project, table, versioning)
90+
feature_names = [f.name for f in table.features]
91+
dimension = len(feature_names)
92+
93+
if table_key not in self._indices or not partial:
94+
quantizer = faiss.IndexFlatL2(dimension)
95+
index = faiss.IndexIVFFlat(
96+
quantizer, dimension, self._config.nlist
97+
)
98+
index.train(
99+
np.random.rand(self._config.nlist * 100, dimension).astype(
100+
np.float32
101+
)
102+
)
103+
self._indices[table_key] = index
104+
self._in_memory_stores[table_key] = InMemoryStore()
81105

82-
self._in_memory_store.update(feature_names, {})
106+
self._in_memory_stores[table_key].update(feature_names, {})
83107

84108
def teardown(
85109
self,
86110
config: RepoConfig,
87111
tables: Sequence[FeatureView],
88112
entities: Sequence[Entity],
89113
):
90-
self._index = None
91-
self._in_memory_store.teardown()
114+
versioning = config.registry.enable_online_feature_view_versioning
115+
for table in tables:
116+
table_key = _table_id(config.project, table, versioning)
117+
self._indices.pop(table_key, None)
118+
store = self._in_memory_stores.pop(table_key, None)
119+
if store is not None:
120+
store.teardown()
92121

93122
def online_read(
94123
self,
@@ -97,23 +126,28 @@ def online_read(
97126
entity_keys: List[EntityKeyProto],
98127
requested_features: Optional[List[str]] = None,
99128
) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]:
100-
if self._index is None:
129+
versioning = config.registry.enable_online_feature_view_versioning
130+
table_key = _table_id(config.project, table, versioning)
131+
index = self._get_index(table_key)
132+
in_memory_store = self._in_memory_stores.get(table_key)
133+
134+
if index is None or in_memory_store is None:
101135
return [(None, None)] * len(entity_keys)
102136

103137
results: List[Tuple[Optional[datetime], Optional[Dict[str, Any]]]] = []
104138
for entity_key in entity_keys:
105139
serialized_key = serialize_entity_key(
106140
entity_key, config.entity_key_serialization_version
107141
).hex()
108-
idx = self._in_memory_store.entity_keys.get(serialized_key, -1)
142+
idx = in_memory_store.entity_keys.get(serialized_key, -1)
109143
if idx == -1:
110144
results.append((None, None))
111145
else:
112-
feature_vector = self._index.reconstruct(int(idx))
146+
feature_vector = index.reconstruct(int(idx))
113147
feature_dict = {
114148
name: ValueProto(double_val=value)
115149
for name, value in zip(
116-
self._in_memory_store.feature_names, feature_vector
150+
in_memory_store.feature_names, feature_vector
117151
)
118152
}
119153
results.append((None, feature_dict))
@@ -128,8 +162,16 @@ def online_write_batch(
128162
],
129163
progress: Optional[Callable[[int], Any]],
130164
) -> None:
131-
if self._index is None:
132-
self._logger.warning("Index is not initialized. Skipping write operation.")
165+
versioning = config.registry.enable_online_feature_view_versioning
166+
table_key = _table_id(config.project, table, versioning)
167+
index = self._get_index(table_key)
168+
in_memory_store = self._in_memory_stores.get(table_key)
169+
170+
if index is None or in_memory_store is None:
171+
self._logger.warning(
172+
"Index for table '%s' is not initialized. Skipping write operation.",
173+
table_key,
174+
)
133175
return
134176

135177
feature_vectors = []
@@ -142,7 +184,7 @@ def online_write_batch(
142184
feature_vector = np.array(
143185
[
144186
feature_dict[name].double_val
145-
for name in self._in_memory_store.feature_names
187+
for name in in_memory_store.feature_names
146188
],
147189
dtype=np.float32,
148190
)
@@ -153,21 +195,21 @@ def online_write_batch(
153195
feature_vectors_array = np.array(feature_vectors)
154196

155197
existing_indices = [
156-
self._in_memory_store.entity_keys.get(sk, -1) for sk in serialized_keys
198+
in_memory_store.entity_keys.get(sk, -1) for sk in serialized_keys
157199
]
158200
mask = np.array(existing_indices) != -1
159201
if np.any(mask):
160-
self._index.remove_ids(
202+
index.remove_ids(
161203
np.array([idx for idx in existing_indices if idx != -1])
162204
)
163205

164206
new_indices = np.arange(
165-
self._index.ntotal, self._index.ntotal + len(feature_vectors_array)
207+
index.ntotal, index.ntotal + len(feature_vectors_array)
166208
)
167-
self._index.add(feature_vectors_array)
209+
index.add(feature_vectors_array)
168210

169211
for sk, idx in zip(serialized_keys, new_indices):
170-
self._in_memory_store.entity_keys[sk] = idx
212+
in_memory_store.entity_keys[sk] = idx
171213

172214
if progress:
173215
progress(len(data))
@@ -189,12 +231,16 @@ def retrieve_online_documents(
189231
Optional[ValueProto],
190232
]
191233
]:
192-
if self._index is None:
234+
versioning = config.registry.enable_online_feature_view_versioning
235+
table_key = _table_id(config.project, table, versioning)
236+
index = self._get_index(table_key)
237+
238+
if index is None:
193239
self._logger.warning("Index is not initialized. Returning empty result.")
194240
return []
195241

196242
query_vector = np.array(embedding, dtype=np.float32).reshape(1, -1)
197-
distances, indices = self._index.search(query_vector, top_k)
243+
distances, indices = index.search(query_vector, top_k)
198244

199245
results: List[
200246
Tuple[
@@ -209,7 +255,7 @@ def retrieve_online_documents(
209255
if idx == -1:
210256
continue
211257

212-
feature_vector = self._index.reconstruct(int(idx))
258+
feature_vector = index.reconstruct(int(idx))
213259

214260
timestamp = Timestamp()
215261
timestamp.GetCurrentTime()

sdk/python/feast/infra/online_stores/online_store.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,9 +256,10 @@ def get_online_features(
256256

257257
def _check_versioned_read_support(self, grouped_refs):
258258
"""Raise an error if versioned reads are attempted on unsupported stores."""
259+
from feast.infra.online_stores.faiss_online_store import FaissOnlineStore
259260
from feast.infra.online_stores.sqlite import SqliteOnlineStore
260261

261-
if isinstance(self, SqliteOnlineStore):
262+
if isinstance(self, (SqliteOnlineStore, FaissOnlineStore)):
262263
return
263264
for table, _ in grouped_refs:
264265
version_tag = getattr(table.projection, "version_tag", None)

0 commit comments

Comments
 (0)