@@ -43,16 +43,30 @@ def teardown(self):
4343 self .entity_keys = {}
4444
4545
46+ def _table_id (project : str , table : FeatureView , enable_versioning : bool = False ) -> str :
47+ """Compute the table key, including version suffix when versioning is enabled."""
48+ name = table .name
49+ if enable_versioning :
50+ # Prefer version_tag from the projection (set by version-qualified refs like @v2)
51+ # over current_version_number (the FV's active version in metadata).
52+ version = getattr (table .projection , "version_tag" , None )
53+ if version is None :
54+ version = getattr (table , "current_version_number" , None )
55+ if version is not None and version > 0 :
56+ name = f"{ table .name } _v{ version } "
57+ return f"{ project } _{ name } "
58+
59+
4660class FaissOnlineStore (OnlineStore ):
47- _index : Optional [ faiss .IndexIVFFlat ] = None
48- _in_memory_store : InMemoryStore = InMemoryStore ()
61+ _indices : Dict [ str , faiss .IndexIVFFlat ] = {}
62+ _in_memory_stores : Dict [ str , InMemoryStore ] = {}
4963 _config : Optional [FaissOnlineStoreConfig ] = None
5064 _logger : logging .Logger = logging .getLogger (__name__ )
5165
52- def _get_index (self , config : RepoConfig ) -> faiss . IndexIVFFlat :
53- if self . _index is None or self . _config is None :
54- raise ValueError ( "Index is not initialized" )
55- return self ._index
66+ def _get_index (
67+ self , table_key : str
68+ ) -> Optional [ faiss . IndexIVFFlat ]:
69+ return self ._indices . get ( table_key )
5670
5771 def update (
5872 self ,
@@ -63,32 +77,47 @@ def update(
6377 entities_to_keep : Sequence [Entity ],
6478 partial : bool ,
6579 ):
66- feature_views = tables_to_keep
67- if not feature_views :
68- return
69-
70- feature_names = [f .name for f in feature_views [0 ].features ]
71- dimension = len (feature_names )
72-
7380 self ._config = FaissOnlineStoreConfig (** config .online_store .dict ())
74- if self ._index is None or not partial :
75- quantizer = faiss .IndexFlatL2 (dimension )
76- self ._index = faiss .IndexIVFFlat (quantizer , dimension , self ._config .nlist )
77- self ._index .train (
78- np .random .rand (self ._config .nlist * 100 , dimension ).astype (np .float32 )
79- )
80- self ._in_memory_store = InMemoryStore ()
81+ versioning = config .registry .enable_online_feature_view_versioning
82+
83+ for table in tables_to_delete :
84+ table_key = _table_id (config .project , table , versioning )
85+ self ._indices .pop (table_key , None )
86+ self ._in_memory_stores .pop (table_key , None )
87+
88+ for table in tables_to_keep :
89+ table_key = _table_id (config .project , table , versioning )
90+ feature_names = [f .name for f in table .features ]
91+ dimension = len (feature_names )
92+
93+ if table_key not in self ._indices or not partial :
94+ quantizer = faiss .IndexFlatL2 (dimension )
95+ index = faiss .IndexIVFFlat (
96+ quantizer , dimension , self ._config .nlist
97+ )
98+ index .train (
99+ np .random .rand (self ._config .nlist * 100 , dimension ).astype (
100+ np .float32
101+ )
102+ )
103+ self ._indices [table_key ] = index
104+ self ._in_memory_stores [table_key ] = InMemoryStore ()
81105
82- self ._in_memory_store .update (feature_names , {})
106+ self ._in_memory_stores [ table_key ] .update (feature_names , {})
83107
84108 def teardown (
85109 self ,
86110 config : RepoConfig ,
87111 tables : Sequence [FeatureView ],
88112 entities : Sequence [Entity ],
89113 ):
90- self ._index = None
91- self ._in_memory_store .teardown ()
114+ versioning = config .registry .enable_online_feature_view_versioning
115+ for table in tables :
116+ table_key = _table_id (config .project , table , versioning )
117+ self ._indices .pop (table_key , None )
118+ store = self ._in_memory_stores .pop (table_key , None )
119+ if store is not None :
120+ store .teardown ()
92121
93122 def online_read (
94123 self ,
@@ -97,23 +126,28 @@ def online_read(
97126 entity_keys : List [EntityKeyProto ],
98127 requested_features : Optional [List [str ]] = None ,
99128 ) -> List [Tuple [Optional [datetime ], Optional [Dict [str , ValueProto ]]]]:
100- if self ._index is None :
129+ versioning = config .registry .enable_online_feature_view_versioning
130+ table_key = _table_id (config .project , table , versioning )
131+ index = self ._get_index (table_key )
132+ in_memory_store = self ._in_memory_stores .get (table_key )
133+
134+ if index is None or in_memory_store is None :
101135 return [(None , None )] * len (entity_keys )
102136
103137 results : List [Tuple [Optional [datetime ], Optional [Dict [str , Any ]]]] = []
104138 for entity_key in entity_keys :
105139 serialized_key = serialize_entity_key (
106140 entity_key , config .entity_key_serialization_version
107141 ).hex ()
108- idx = self . _in_memory_store .entity_keys .get (serialized_key , - 1 )
142+ idx = in_memory_store .entity_keys .get (serialized_key , - 1 )
109143 if idx == - 1 :
110144 results .append ((None , None ))
111145 else :
112- feature_vector = self . _index .reconstruct (int (idx ))
146+ feature_vector = index .reconstruct (int (idx ))
113147 feature_dict = {
114148 name : ValueProto (double_val = value )
115149 for name , value in zip (
116- self . _in_memory_store .feature_names , feature_vector
150+ in_memory_store .feature_names , feature_vector
117151 )
118152 }
119153 results .append ((None , feature_dict ))
@@ -128,8 +162,16 @@ def online_write_batch(
128162 ],
129163 progress : Optional [Callable [[int ], Any ]],
130164 ) -> None :
131- if self ._index is None :
132- self ._logger .warning ("Index is not initialized. Skipping write operation." )
165+ versioning = config .registry .enable_online_feature_view_versioning
166+ table_key = _table_id (config .project , table , versioning )
167+ index = self ._get_index (table_key )
168+ in_memory_store = self ._in_memory_stores .get (table_key )
169+
170+ if index is None or in_memory_store is None :
171+ self ._logger .warning (
172+ "Index for table '%s' is not initialized. Skipping write operation." ,
173+ table_key ,
174+ )
133175 return
134176
135177 feature_vectors = []
@@ -142,7 +184,7 @@ def online_write_batch(
142184 feature_vector = np .array (
143185 [
144186 feature_dict [name ].double_val
145- for name in self . _in_memory_store .feature_names
187+ for name in in_memory_store .feature_names
146188 ],
147189 dtype = np .float32 ,
148190 )
@@ -153,21 +195,21 @@ def online_write_batch(
153195 feature_vectors_array = np .array (feature_vectors )
154196
155197 existing_indices = [
156- self . _in_memory_store .entity_keys .get (sk , - 1 ) for sk in serialized_keys
198+ in_memory_store .entity_keys .get (sk , - 1 ) for sk in serialized_keys
157199 ]
158200 mask = np .array (existing_indices ) != - 1
159201 if np .any (mask ):
160- self . _index .remove_ids (
202+ index .remove_ids (
161203 np .array ([idx for idx in existing_indices if idx != - 1 ])
162204 )
163205
164206 new_indices = np .arange (
165- self . _index . ntotal , self . _index .ntotal + len (feature_vectors_array )
207+ index . ntotal , index .ntotal + len (feature_vectors_array )
166208 )
167- self . _index .add (feature_vectors_array )
209+ index .add (feature_vectors_array )
168210
169211 for sk , idx in zip (serialized_keys , new_indices ):
170- self . _in_memory_store .entity_keys [sk ] = idx
212+ in_memory_store .entity_keys [sk ] = idx
171213
172214 if progress :
173215 progress (len (data ))
@@ -189,12 +231,16 @@ def retrieve_online_documents(
189231 Optional [ValueProto ],
190232 ]
191233 ]:
192- if self ._index is None :
234+ versioning = config .registry .enable_online_feature_view_versioning
235+ table_key = _table_id (config .project , table , versioning )
236+ index = self ._get_index (table_key )
237+
238+ if index is None :
193239 self ._logger .warning ("Index is not initialized. Returning empty result." )
194240 return []
195241
196242 query_vector = np .array (embedding , dtype = np .float32 ).reshape (1 , - 1 )
197- distances , indices = self . _index .search (query_vector , top_k )
243+ distances , indices = index .search (query_vector , top_k )
198244
199245 results : List [
200246 Tuple [
@@ -209,7 +255,7 @@ def retrieve_online_documents(
209255 if idx == - 1 :
210256 continue
211257
212- feature_vector = self . _index .reconstruct (int (idx ))
258+ feature_vector = index .reconstruct (int (idx ))
213259
214260 timestamp = Timestamp ()
215261 timestamp .GetCurrentTime ()
0 commit comments