dataiku · kevanescence · Nov 25, 2024 · Nov 21, 2024 · Nov 21, 2024 · Nov 25, 2024
diff --git a/HISTORY.txt b/HISTORY.txt
@@ -1,6 +1,12 @@
 Changelog
 ==========
 
+
+13.2.3 (2024-11-25)
+---------------------
+
+* Initial release for DSS 13.2.3
+
 13.2.2.1 (2024-11-06)
 ---------------------
 

diff --git a/dataikuapi/dss/admin.py b/dataikuapi/dss/admin.py
@@ -1303,7 +1303,7 @@ def delete(self):
 
         .. note::
 
-            This call requires an API key with admin rights
+            This call requires an API key with `Manage all code envs` permission
         """
         resp = self.client._perform_json(
             "DELETE", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name))
@@ -1328,7 +1328,7 @@ def get_definition(self):
 
         .. note::
 
-            This call requires an API key with admin rights
+            This call requires an API key with `Create code envs` or `Manage all code envs` permission
 
         :return: the code env definition
         :rtype: dict
@@ -1364,7 +1364,7 @@ def set_definition(self, env):
 
         .. note::
 
-            This call requires an API key with admin rights
+            This call requires an API key with `Create code envs` or `Manage all code envs` permission
 
         .. important::
 
@@ -1441,7 +1441,7 @@ def set_jupyter_support(self, active):
 
         .. note::
 
-            This call requires an API key with admin rights
+            This call requires an API key with `Create code envs` or `Manage all code envs` permission
 
         :param boolean active: True to activate jupyter support, False to deactivate
         """
@@ -1460,7 +1460,7 @@ def update_packages(self, force_rebuild_env=False):
 
         .. note::
 
-            This call requires an API key with admin rights
+            This call requires an API key with `Create code envs` or `Manage all code envs` permission
 
         :param boolean force_rebuild_env: whether to rebuild the code env from scratch
 

diff --git a/dataikuapi/dss/analysis.py b/dataikuapi/dss/analysis.py
@@ -2,7 +2,6 @@
 from ..utils import DataikuUTF8CSVReader
 from ..utils import DataikuStreamedHttpUTF8CSVReader
 import json
-import time
 from .metrics import ComputedMetrics
 from .ml import DSSMLTask
 from .utils import DSSDatasetSelectionBuilder

diff --git a/dataikuapi/dss/future.py b/dataikuapi/dss/future.py
@@ -1,5 +1,4 @@
-import time
-
+from ..utils import _ExponentialBackoff
 
 class DSSFuture(object):
     """
@@ -126,8 +125,11 @@ def wait_for_result(self):
             return self.result_wrapper(self.state.get('result', None))
         if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek:
             self.get_state()
+
+        eb = _ExponentialBackoff()
+
         while not self.state.get('hasResult', False):
-            time.sleep(5)
+            eb.sleep_next()
             self.get_state()
         if self.state.get('hasResult', False):
             return self.result_wrapper(self.state.get('result', None))

diff --git a/dataikuapi/dss/job.py b/dataikuapi/dss/job.py
@@ -1,7 +1,6 @@
-import time
 import sys
 from ..utils import DataikuException
-
+from ..utils import _ExponentialBackoff
 
 class DSSJob(object):
     """
@@ -77,10 +76,9 @@ def wait(self, no_fail=False):
         :rtype: dict
         """
         job_state = self.job.get_status().get("baseStatus", {}).get("state", "")
-        sleep_time = 2
+        eb = _ExponentialBackoff()
         while job_state not in ["DONE", "ABORTED", "FAILED"]:
-            sleep_time = 60 if sleep_time >= 60 else sleep_time * 1.2
-            time.sleep(int(sleep_time))
+            eb.sleep_next()
             job_state = self.job.get_status().get("baseStatus", {}).get("state", "")
 
         if no_fail or (job_state == "DONE"):

diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py
@@ -1,7 +1,6 @@
 import json
 import logging
 import re
-import time
 import warnings
 
 from six import string_types
@@ -11,6 +10,7 @@
 from .utils import DSSFilterBuilder
 from ..utils import DataikuException
 from ..utils import _write_response_content_to_file
+from ..utils import _ExponentialBackoff
 
 logger = logging.getLogger("dataikuapi.dss.ml")
 
@@ -4486,11 +4486,12 @@ def wait_guess_complete(self):
         This should be called immediately after the creation of a new ML Task if the ML Task was created with ``wait_guess_complete = False``,
         before calling :meth:`get_settings` or :meth:`train`.
         """
+        eb = _ExponentialBackoff()
         while True:
             status = self.get_status()
             if status.get("guessing", "???") == False:
                 break
-            time.sleep(0.2)
+            eb.sleep_next()
 
     def get_status(self):
         """
@@ -4627,11 +4628,12 @@ def wait_train_complete(self):
 
         To be used following any asynchronous training started with :meth:`start_train` or :meth:`start_ensembling`
         """
+        eb = _ExponentialBackoff()
         while True:
             status = self.get_status()
             if status.get("training", "???") == False:
                 break
-            time.sleep(2)
+            eb.sleep_next()
 
     def get_trained_models_ids(self, session_id=None, algorithm=None):
         """

diff --git a/dataikuapi/dss/scenario.py b/dataikuapi/dss/scenario.py
@@ -1,6 +1,6 @@
 from datetime import datetime
-import time, warnings
-from ..utils import DataikuException, _timestamp_ms_to_zoned_datetime, _local_timezone
+import warnings
+from ..utils import DataikuException, _timestamp_ms_to_zoned_datetime, _local_timezone, _ExponentialBackoff
 from .discussion import DSSObjectDiscussions
 from .utils import DSSTaggableObjectListItem
 from dateutil.tz import tzlocal
@@ -715,9 +715,10 @@ def wait_for_completion(self, no_fail=False):
 
         :param boolean no_fail: if False, raises an exception if scenario fails
         """
+        eb = _ExponentialBackoff()
         while self.running:
             self.refresh()
-            time.sleep(5)
+            eb.sleep_next()
 
         if self.outcome != 'SUCCESS' and no_fail == False:
             raise DataikuException("Scenario run returned status %s" % outcome)
@@ -963,9 +964,10 @@ def wait(self, no_fail=False):
         :return: the final state of the scenario run (see :meth:`DSSScenarioRun.get_info()`)
         :rtype: dict
         """
+        eb = _ExponentialBackoff()
         while not self.scenario_run.run.get('result', False):
             self.scenario_run = self.trigger_fire.get_scenario_run()
-            time.sleep(5)
+            eb.sleep_next()
         outcome = self.scenario_run.run.get('result', None).get('outcome', 'UNKNOWN')
         if outcome == 'SUCCESS' or no_fail:
             return self.scenario_run
@@ -1012,6 +1014,7 @@ def wait_for_scenario_run(self, no_fail=False):
         """
         scenario_run = None
         refresh_trigger_counter = 0
+        eb = _ExponentialBackoff()
         while scenario_run is None:
             refresh_trigger_counter += 1
             if refresh_trigger_counter == 10:
@@ -1022,7 +1025,7 @@ def wait_for_scenario_run(self, no_fail=False):
                 else:
                     raise DataikuException("Scenario run has been cancelled")
             scenario_run = self.get_scenario_run()
-            time.sleep(5)
+            eb.sleep_next()
         return scenario_run
 
     def get_scenario_run(self):

diff --git a/dataikuapi/dssclient.py b/dataikuapi/dssclient.py
@@ -663,8 +663,6 @@ def list_code_envs(self, as_objects=False):
         """
         List all code envs setup on the DSS instance
 
-        Note: this call requires an API key with admin rights
-
         :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.future.DSSCodeEnv`
         :returns: a list of code envs. Each code env is a dict containing at least "name", "type" and "language"
         """
@@ -689,7 +687,7 @@ def create_code_env(self, env_lang, env_name, deployment_mode, params=None):
         """
         Create a code env, and return a handle to interact with it
 
-        Note: this call requires an API key with admin rights
+        Note: this call requires an API key with `Create code envs` or `Manage all code envs` permission
 
         :param env_lang: the language (PYTHON or R) of the new code env
         :param env_name: the name of the new code env

diff --git a/dataikuapi/utils.py b/dataikuapi/utils.py
@@ -5,6 +5,7 @@
 import zipfile
 import itertools
 import sys
+import time
 from datetime import datetime
 
 if sys.version_info > (3,0):
@@ -159,4 +160,34 @@ def _timestamp_ms_to_zoned_datetime(timestamp_ms):
     if timestamp_ms and timestamp_ms > 0:
         return datetime.fromtimestamp(timestamp_ms / 1000, tz=_local_timezone)
     else:
-        return None
+        return None
+
+
+class _ExponentialBackoff(object):
+    # These values lead to:
+    #  - 25 calls in the first 20 seconds
+    #  - Overhead overall remains below 10%
+    #  - After 1 minute, the interval is 5 seconds
+
+    def __init__(self, initial_time_ms=200, max_time_ms=15000, factor=1.1):
+        self.initial_time_ms = initial_time_ms
+        self.last_time = None
+        self.max_time_ms = max_time_ms
+        self.factor = factor
+
+    def next_sleep_time(self):
+        if self.last_time is None:
+            next_time = self.initial_time_ms
+        else:
+            next_time = self.last_time * self.factor
+        if next_time > self.max_time_ms:
+            next_time = self.max_time_ms
+
+        self.last_time = next_time
+
+        return next_time
+
+    def sleep_next(self):
+        sleep_time = float(self.next_sleep_time()) / 1000.0
+        #print("Sleeping %.3f" % sleep_time)
+        time.sleep(sleep_time)
diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 
 from setuptools import setup
 
-VERSION = "13.2.2.1"
+VERSION = "13.2.3"
 
 long_description = (open('README').read() + '\n\n' +
                     open('HISTORY.txt').read())