Skip to content

Commit a37bdee

Browse files
mredolattimredolatti
committed
add localhost support for dynamic configs
1 parent 66f5d29 commit a37bdee

8 files changed

Lines changed: 335 additions & 65 deletions

File tree

‎setup.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
tests_require = ['flake8', 'pytest', 'pytest-mock', 'coverage', 'pytest-cov']
99
install_requires = [
1010
'requests>=2.9.1',
11+
'pyyaml>=5.1',
1112
'future>=0.15.2',
1213
'docopt>=0.6.2',
1314
]

‎splitio/api/impressions.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ def _build_bulk(impressions):
4040
"""
4141
return [
4242
{
43-
'testName': group[0],
43+
'testName': test_name,
4444
'keyImpressions': [
4545
{
4646
'keyName': impression.matching_key,
@@ -50,10 +50,10 @@ def _build_bulk(impressions):
5050
'label': impression.label,
5151
'bucketingKey': impression.bucketing_key
5252
}
53-
for impression in group[1]
53+
for impression in imps
5454
]
5555
}
56-
for group in groupby(
56+
for (test_name, imps) in groupby(
5757
sorted(impressions, key=lambda i: i.feature_name),
5858
lambda i: i.feature_name
5959
)

‎splitio/client/client.py‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,21 @@ def _send_impression_to_listener(self, impression, attributes):
8181
self._logger.debug('Error', exc_info=True)
8282

8383
def get_treatment_with_config(self, key, feature, attributes=None):
84+
"""
85+
Get the treatment and config for a feature and key, with optional dictionary of attributes.
86+
87+
This method never raises an exception. If there's a problem, the appropriate log message
88+
will be generated and the method will return the CONTROL treatment.
89+
90+
:param key: The key for which to get the treatment
91+
:type key: str
92+
:param feature: The name of the feature for which to get the treatment
93+
:type feature: str
94+
:param attributes: An optional dictionary of attributes
95+
:type attributes: dict
96+
:return: The treatment for the key and feature
97+
:rtype: tuple(str, str)
98+
"""
8499
try:
85100
if self.destroyed:
86101
self._logger.error("Client has already been destroyed - no calls possible")
@@ -157,7 +172,7 @@ def get_treatment(self, key, feature, attributes=None):
157172

158173
def get_treatments_with_config(self, key, features, attributes=None):
159174
"""
160-
Evaluate multiple features and return a dictionary with all the feature/treatments.
175+
Evaluate multiple features and return a dict with feature -> (treatment, config).
161176
162177
Get the treatments for a list of features considering a key, with an optional dictionary of
163178
attributes. This method never raises an exception. If there's a problem, the appropriate
@@ -235,7 +250,21 @@ def get_treatments_with_config(self, key, features, attributes=None):
235250

236251

237252
def get_treatments(self, key, features, attributes=None):
238-
"""TODO"""
253+
"""
254+
Evaluate multiple features and return a dictionary with all the feature/treatments.
255+
256+
Get the treatments for a list of features considering a key, with an optional dictionary of
257+
attributes. This method never raises an exception. If there's a problem, the appropriate
258+
log message will be generated and the method will return the CONTROL treatment.
259+
:param key: The key for which to get the treatment
260+
:type key: str
261+
:param features: Array of the names of the features for which to get the treatment
262+
:type feature: list
263+
:param attributes: An optional dictionary of attributes
264+
:type attributes: dict
265+
:return: Dictionary with the result of all the features provided
266+
:rtype: dict
267+
"""
239268
with_config = self.get_treatments_with_config(key, features, attributes)
240269
return {feature: result[0] for (feature, result) in six.iteritems(with_config)}
241270

‎splitio/client/factory.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,7 @@ def _build_localhost_factory(config):
334334
tasks = {'splits': LocalhostSplitSynchronizationTask(
335335
cfg['splitFile'],
336336
storages['splits'],
337+
cfg['featuresRefreshRate'],
337338
ready_event
338339
)}
339340
tasks['splits'].start()

‎splitio/client/localhost.py‎

Lines changed: 125 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
"""Localhost client mocked components."""
22

3+
import itertools
34
import logging
45
import re
5-
from splitio.models.splits import from_raw
6+
7+
from six import raise_from
8+
import yaml
9+
10+
from splitio.models import splits
611
from splitio.storage import ImpressionStorage, EventStorage, TelemetryStorage
712
from splitio.tasks import BaseSynchronizationTask
813
from splitio.tasks.util import asynctask
914

10-
_COMMENT_LINE_RE = re.compile('^#.*$')
11-
_DEFINITION_LINE_RE = re.compile('^(?<![^#])(?P<feature>[\w_-]+)\s+(?P<treatment>[\w_-]+)$')
15+
_LEGACY_COMMENT_LINE_RE = re.compile(r'^#.*$')
16+
_LEGACY_DEFINITION_LINE_RE = re.compile(r'^(?<![^#])(?P<feature>[\w_-]+)\s+(?P<treatment>[\w_-]+)$')
1217

1318

1419
_LOGGER = logging.getLogger(__name__)
@@ -69,7 +74,7 @@ def pop_gauges(self, *_, **__): #pylint: disable=arguments-differ
6974
class LocalhostSplitSynchronizationTask(BaseSynchronizationTask):
7075
"""Split synchronization task that periodically checks the file and updated the splits."""
7176

72-
def __init__(self, filename, storage, ready_event):
77+
def __init__(self, filename, storage, period, ready_event):
7378
"""
7479
Class constructor.
7580
@@ -83,22 +88,23 @@ def __init__(self, filename, storage, ready_event):
8388
self._filename = filename
8489
self._ready_event = ready_event
8590
self._storage = storage
86-
self._task = asynctask.AsyncTask(self._update_splits, 5, self._on_start)
91+
self._period = period
92+
self._task = asynctask.AsyncTask(self._update_splits, period, self._on_start)
8793

8894
def _on_start(self):
8995
"""Sync splits and set event if successful."""
9096
self._update_splits()
9197
self._ready_event.set()
9298

9399
@staticmethod
94-
def _make_all_keys_based_split(split_name, treatment):
100+
def _make_split(split_name, conditions, configs=None):
95101
"""
96102
Make a split with a single all_keys matcher.
97103
98104
:param split_name: Name of the split.
99105
:type split_name: str.
100106
"""
101-
return from_raw({
107+
return splits.from_raw({
102108
'changeNumber': 123,
103109
'trafficTypeName': 'user',
104110
'name': split_name,
@@ -107,30 +113,55 @@ def _make_all_keys_based_split(split_name, treatment):
107113
'seed': 321654,
108114
'status': 'ACTIVE',
109115
'killed': False,
110-
'defaultTreatment': treatment,
116+
'defaultTreatment': 'control',
111117
'algo': 2,
112-
'conditions': [
113-
{
114-
'partitions': [
115-
{'treatment': treatment, 'size': 100}
116-
],
117-
'contitionType': 'WHITELIST',
118-
'label': 'some_other_label',
119-
'matcherGroup': {
120-
'matchers': [
121-
{
122-
'matcherType': 'ALL_KEYS',
123-
'negate': False,
124-
}
125-
],
126-
'combiner': 'AND'
127-
}
128-
}
129-
]
118+
'conditions': conditions,
119+
'configurations': configs
130120
})
131121

122+
@staticmethod
123+
def _make_all_keys_condition(treatment):
124+
return {
125+
'partitions': [
126+
{'treatment': treatment, 'size': 100}
127+
],
128+
'conditionType': 'WHITELIST',
129+
'label': 'some_other_label',
130+
'matcherGroup': {
131+
'matchers': [
132+
{
133+
'matcherType': 'ALL_KEYS',
134+
'negate': False,
135+
}
136+
],
137+
'combiner': 'AND'
138+
}
139+
}
140+
141+
@staticmethod
142+
def _make_whitelist_condition(whitelist, treatment):
143+
return {
144+
'partitions': [
145+
{'treatment': treatment, 'size': 100}
146+
],
147+
'conditionType': 'WHITELIST',
148+
'label': 'some_other_label',
149+
'matcherGroup': {
150+
'matchers': [
151+
{
152+
'matcherType': 'WHITELIST',
153+
'negate': False,
154+
'whitelistMatcherData': {
155+
'whitelist': whitelist
156+
}
157+
}
158+
],
159+
'combiner': 'AND'
160+
}
161+
}
162+
132163
@classmethod
133-
def _read_splits_from_file(cls, filename):
164+
def _read_splits_from_legacy_file(cls, filename):
134165
"""
135166
Parse a splits file and return a populated storage.
136167
@@ -140,53 +171,89 @@ def _read_splits_from_file(cls, filename):
140171
:return: Storage populataed with splits ready to be evaluated.
141172
:rtype: InMemorySplitStorage
142173
"""
143-
splits = {}
174+
to_return = {}
144175
try:
145176
with open(filename, 'r') as flo:
146177
for line in flo:
147-
if line.strip() == '':
148-
continue
149-
150-
comment_match = _COMMENT_LINE_RE.match(line)
151-
if comment_match:
178+
if line.strip() == '' or _LEGACY_COMMENT_LINE_RE.match(line):
152179
continue
153180

154-
definition_match = _DEFINITION_LINE_RE.match(line)
155-
if definition_match:
156-
splits[definition_match.group('feature')] = cls._make_all_keys_based_split(
157-
definition_match.group('feature'),
158-
definition_match.group('treatment')
181+
definition_match = _LEGACY_DEFINITION_LINE_RE.match(line)
182+
if not definition_match:
183+
_LOGGER.warning(
184+
'Invalid line on localhost environment split '
185+
'definition. Line = %s',
186+
line
159187
)
160188
continue
161189

162-
_LOGGER.warning(
163-
'Invalid line on localhost environment split '
164-
'definition. Line = %s',
165-
line
166-
)
167-
return splits
168-
except IOError as e:
169-
raise ValueError("Error parsing split file")
170-
# TODO: ver raise from!
171-
# raise_from(ValueError(
172-
# 'There was a problem with '
173-
# 'the splits definition file "{}"'.format(filename)),
174-
# e
175-
# )
190+
cond = cls._make_all_keys_condition(definition_match.group('treatment'))
191+
splt = cls._make_split(definition_match.group('feature'), [cond])
192+
to_return[splt.name] = splt
193+
return to_return
176194

195+
except IOError as exc:
196+
raise_from(
197+
ValueError("Error parsing file %s. Make sure it's readable." % filename),
198+
exc
199+
)
200+
201+
@classmethod
202+
def _read_splits_from_yaml_file(cls, filename):
203+
"""
204+
Parse a splits file and return a populated storage.
205+
206+
:param filename: Path of the file containing mocked splits & treatments.
207+
:type filename: str.
208+
209+
:return: Storage populataed with splits ready to be evaluated.
210+
:rtype: InMemorySplitStorage
211+
"""
212+
try:
213+
with open(filename, 'r') as flo:
214+
parsed = yaml.load(flo.read(), Loader=yaml.FullLoader)
215+
216+
grouped_by_feature_name = itertools.groupby(
217+
sorted(parsed, key=lambda i: next(iter(i.keys()))),
218+
lambda i: next(iter(i.keys())))
219+
220+
to_return = {}
221+
for (split_name, statements) in grouped_by_feature_name:
222+
configs = {}
223+
whitelist = []
224+
all_keys = []
225+
for statement in statements:
226+
data = next(iter(statement.values())) # grab the first (and only) value.
227+
if 'keys' in data:
228+
keys = data['keys'] if isinstance(data['keys'], list) else [data['keys']]
229+
whitelist.append(cls._make_whitelist_condition(keys, data['treatment']))
230+
else:
231+
all_keys.append(cls._make_all_keys_condition(data['treatment']))
232+
if 'config' in data:
233+
configs[data['treatment']] = data['config']
234+
to_return[split_name] = cls._make_split(split_name, whitelist + all_keys, configs)
235+
return to_return
236+
237+
except IOError as exc:
238+
raise_from(
239+
ValueError("Error parsing file %s. Make sure it's readable." % filename),
240+
exc
241+
)
177242

178243
def _update_splits(self):
179244
"""Update splits in storage."""
180245
_LOGGER.info('Synchronizing splits now.')
181-
splits = self._read_splits_from_file(self._filename)
182-
to_delete = [name for name in self._storage.get_split_names() if name not in splits.keys()]
183-
for split in splits.values():
246+
if self._filename.split('.')[-1].lower() in ('yaml', 'yml'):
247+
fetched = self._read_splits_from_yaml_file(self._filename)
248+
else:
249+
fetched = self._read_splits_from_legacy_file(self._filename)
250+
to_delete = [name for name in self._storage.get_split_names() if name not in fetched.keys()]
251+
for split in fetched.values():
184252
self._storage.put(split)
185253

186254
for split in to_delete:
187255
self._storage.remove(split)
188256

189-
190257
def is_running(self):
191258
"""Return whether the task is running."""
192259
return self._task.running
@@ -195,13 +262,11 @@ def start(self):
195262
"""Start split synchronization."""
196263
self._task.start()
197264

198-
def stop(self, stop_event):
265+
def stop(self, event=None):
199266
"""
200267
Stop task.
201268
202269
:param stop_event: Event top set when the task finishes.
203270
:type stop_event: threading.Event.
204271
"""
205-
self._task.stop(stop_event)
206-
207-
272+
self._task.stop(event)

‎tests/client/files/file1.split‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
events_write_es on
2+
events_routing sqs
3+
impressions_routing sqs
4+
workspaces_v1 on
5+
create_org_with_workspace on
6+
sqs_events_processing on
7+
sqs_impressions_processing on
8+
sqs_events_fetch on
9+
sqs_impressions_fetch off
10+
sqs_impressions_fetch_period 700
11+
sqs_impressions_fetch_threads 10
12+
sqs_events_fetch_period 500
13+
sqs_events_fetch_threads 5
14+

‎tests/client/files/file2.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
- my_feature:
2+
treatment: "on"
3+
keys: "key"
4+
config: "{\"desc\" : \"this applies only to ON treatment\"}"
5+
- other_feature_3:
6+
treatment: "off"
7+
- my_feature:
8+
treatment: "off"
9+
keys: "only_key"
10+
config: "{\"desc\" : \"this applies only to OFF and only for only_key. The rest will receive ON\"}"
11+
- other_feature_3:
12+
treatment: "on"
13+
keys: "key_whitelist"
14+
- other_feature:
15+
treatment: "on"
16+
keys: ["key2","key3"]
17+
- other_feature_2:
18+
treatment: "on"

0 commit comments

Comments
 (0)