Neeratyoy · Jun 27, 2019 · Jul 1, 2019 · Jul 3, 2019 · Jul 3, 2019 · Jul 3, 2019
diff --git a/ci_scripts/test.sh b/ci_scripts/test.sh
@@ -22,7 +22,7 @@ run_tests() {
         PYTEST_ARGS=''
     fi
 
-    pytest -n 4 --duration=20 --timeout=600 --timeout-method=thread -sv --ignore='test_OpenMLDemo.py' $PYTEST_ARGS $test_dir
+    pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread -sv --ignore='test_OpenMLDemo.py' $PYTEST_ARGS $test_dir
 }
 
 if [[ "$RUN_FLAKE8" == "true" ]]; then

diff --git a/doc/progress.rst b/doc/progress.rst
@@ -8,11 +8,14 @@ Changelog
 
 0.10.0
 ~~~~~~
+* FIX #261: Test server is cleared of all files uploaded during unit testing.
+* FIX #447: All files created by unit tests no longer persist in local.
 * FIX #589: Fixing a bug that did not successfully upload the columns to ignore when creating and publishing a dataset.
 * DOC #639: More descriptive documention for function to convert array format.
 * ADD #687: Adds a function to retrieve the list of evaluation measures available.
 * ADD #695: A function to retrieve all the data quality measures available.
 
+
 0.9.0
 ~~~~~
 * ADD #560: OpenML-Python can now handle regression tasks as well.

diff --git a/openml/testing.py b/openml/testing.py
@@ -17,6 +17,8 @@
 import openml
 from openml.tasks import TaskTypeEnum
 
+import pytest
+
 
 class TestBase(unittest.TestCase):
     """Base class for tests
@@ -26,6 +28,9 @@ class TestBase(unittest.TestCase):
     Currently hard-codes a read-write key.
     Hopefully soon allows using a test server, not the production server.
     """
+    tracker = {}
+    test_server = None
+    apikey = None
 
     def setUp(self, n_levels: int = 1):
         """Setup variables and temporary directories.
@@ -75,6 +80,10 @@ def setUp(self, n_levels: int = 1):
         self.production_server = "https://openml.org/api/v1/xml"
         self.test_server = "https://test.openml.org/api/v1/xml"
 
+        # For global file deletion on test server
+        TestBase.test_server = self.test_server
+        TestBase.apikey = openml.config.apikey
+
         openml.config.server = self.test_server
         openml.config.avoid_duplicate_runs = False
         openml.config.cache_directory = self.workdir
@@ -103,6 +112,66 @@ def tearDown(self):
         openml.config.server = self.production_server
         openml.config.connection_n_retries = self.connection_n_retries
 
+    @classmethod
+    def _track_test_server_dumps(self, entity_type, entity_id):
+        if entity_type not in TestBase.tracker:
+            TestBase.tracker[entity_type] = [entity_id]
+        else:
+            TestBase.tracker[entity_type].append(entity_id)
+
+    @pytest.fixture(scope="session", autouse=True)
+    def _cleanup_fixture(self):
+        """Cleans up files generated by Unit tests
+
+        This function is called at the beginning of the invocation of
+        TestBase (defined below), by each of class that inherits TestBase.
+        The 'yield' creates a checkpoint and breaks away to continue running
+        the unit tests of the sub class. When all the tests end, execution
+        resumes from the checkpoint.
+        """
+        # TODO: generalize this path better akin to static_cache_dir
+        directory = "{}/tests/files/".format(os.getcwd())
+        files = os.walk(directory)
+        old_file_list = []
+        for root, _, filenames in files:
+            for filename in filenames:
+                old_file_list.append(os.path.join(root, filename))
+        # context switches to other remaining tests
+        # pauses the code execution here till all tests in the 'session' is over
+        yield
+        # resumes from here after all collected tests are completed
+        files = os.walk(directory)
+        new_file_list = []
+        for root, _, filenames in files:
+            for filename in filenames:
+                new_file_list.append(os.path.join(root, filename))
+        # filtering the files generated during this run
+        new_file_list = list(set(new_file_list) - set(old_file_list))
+        print("Deleting local files generated...")
+        for file in new_file_list:
+            os.remove(file)
+
+        # Test server deletion
+        print("Deleting files uploaded to test server...")
+        openml.config.server = TestBase.test_server
+        openml.config.apikey = TestBase.apikey
+
+        entity_types = list(TestBase.tracker.keys())
+        # deleting 'run' first to allow other dependent entities to be deleted
+        if 'run' in entity_types:
+            index = entity_types.index('run')
+            # putting 'run' in the start of the list
+            entity_types[0], entity_types[index] = entity_types[index], entity_types[0]
+
+        for entity_type in entity_types:
+            for i, entity in enumerate(TestBase.tracker[entity_type]):
+                try:
+                    openml.utils._delete_entity(entity_type, entity)
+                    print("Deleted ({}, {})".format(entity_type, entity))
+                    TestBase.tracker[entity_type].pop(i)
+                except Exception as e:
+                    print("Cannot delete ({}, {}): {}".format(entity_type, entity, e))
+
     def _get_sentinel(self, sentinel=None):
         if sentinel is None:
             # Create a unique prefix for the flow. Necessary because the flow

diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py
@@ -1,4 +1,5 @@
 import os
+import sys
 import random
 from itertools import product
 from unittest import mock
@@ -43,14 +44,18 @@ def tearDown(self):
         super(TestOpenMLDataset, self).tearDown()
 
     def _remove_pickle_files(self):
-        cache_dir = self.static_cache_dir
+        self.lock_path = os.path.join(openml.config.get_cache_directory(), 'locks')
         for did in ['-1', '2']:
             with lockutils.external_lock(
                     name='datasets.functions.get_dataset:%s' % did,
-                    lock_path=os.path.join(openml.config.get_cache_directory(), 'locks'),
+                    lock_path=self.lock_path,
             ):
-                pickle_path = os.path.join(cache_dir, 'datasets', did,
-                                           'dataset.pkl')
+                if sys.version[0] == '3':
+                    pickle_path = os.path.join(openml.config.get_cache_directory(), 'datasets',
+                                               did, 'dataset.pkl.py3')
+                else:
+                    pickle_path = os.path.join(openml.config.get_cache_directory(), 'datasets',
+                                               did, 'dataset.pkl')
                 try:
                     os.remove(pickle_path)
                 except (OSError, FileNotFoundError):
@@ -478,6 +483,8 @@ def test_publish_dataset(self):
             data_file=file_path,
         )
         dataset.publish()
+        TestBase._track_test_server_dumps('data', dataset.dataset_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], dataset.dataset_id))
         self.assertIsInstance(dataset.dataset_id, int)
 
     def test__retrieve_class_labels(self):
@@ -498,6 +505,8 @@ def test_upload_dataset_with_url(self):
             url="https://www.openml.org/data/download/61/dataset_61_iris.arff",
         )
         dataset.publish()
+        TestBase._track_test_server_dumps('data', dataset.dataset_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], dataset.dataset_id))
         self.assertIsInstance(dataset.dataset_id, int)
 
     def test_data_status(self):
@@ -507,6 +516,8 @@ def test_data_status(self):
             version=1,
             url="https://www.openml.org/data/download/61/dataset_61_iris.arff")
         dataset.publish()
+        TestBase._track_test_server_dumps('data', dataset.dataset_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], dataset.dataset_id))
         did = dataset.dataset_id
 
         # admin key for test server (only adminds can activate datasets.
@@ -620,6 +631,8 @@ def test_create_dataset_numpy(self):
         )
 
         upload_did = dataset.publish()
+        TestBase._track_test_server_dumps('data', upload_did)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], upload_did))
 
         self.assertEqual(
             _get_online_dataset_arff(upload_did),
@@ -682,6 +695,8 @@ def test_create_dataset_list(self):
         )
 
         upload_did = dataset.publish()
+        TestBase._track_test_server_dumps('data', upload_did)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], upload_did))
         self.assertEqual(
             _get_online_dataset_arff(upload_did),
             dataset._dataset,
@@ -725,6 +740,8 @@ def test_create_dataset_sparse(self):
         )
 
         upload_did = xor_dataset.publish()
+        TestBase._track_test_server_dumps('data', upload_did)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], upload_did))
         self.assertEqual(
             _get_online_dataset_arff(upload_did),
             xor_dataset._dataset,
@@ -762,6 +779,8 @@ def test_create_dataset_sparse(self):
         )
 
         upload_did = xor_dataset.publish()
+        TestBase._track_test_server_dumps('data', upload_did)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], upload_did))
         self.assertEqual(
             _get_online_dataset_arff(upload_did),
             xor_dataset._dataset,
@@ -885,6 +904,8 @@ def test_create_dataset_pandas(self):
             paper_url=paper_url
         )
         upload_did = dataset.publish()
+        TestBase._track_test_server_dumps('data', upload_did)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], upload_did))
         self.assertEqual(
             _get_online_dataset_arff(upload_did),
             dataset._dataset,
@@ -919,6 +940,8 @@ def test_create_dataset_pandas(self):
             paper_url=paper_url
         )
         upload_did = dataset.publish()
+        TestBase._track_test_server_dumps('data', upload_did)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], upload_did))
         self.assertEqual(
             _get_online_dataset_arff(upload_did),
             dataset._dataset,
@@ -955,6 +978,8 @@ def test_create_dataset_pandas(self):
             paper_url=paper_url
         )
         upload_did = dataset.publish()
+        TestBase._track_test_server_dumps('data', upload_did)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], upload_did))
         downloaded_data = _get_online_dataset_arff(upload_did)
         self.assertEqual(
             downloaded_data,
@@ -1123,6 +1148,8 @@ def test___publish_fetch_ignore_attribute(self):
 
         # publish dataset
         upload_did = dataset.publish()
+        TestBase._track_test_server_dumps('data', upload_did)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], upload_did))
         # test if publish was successful
         self.assertIsInstance(upload_did, int)
         # variables to carry forward for test_publish_fetch_ignore_attribute()
@@ -1253,6 +1280,8 @@ def test_create_dataset_row_id_attribute_inference(self):
             )
             self.assertEqual(dataset.row_id_attribute, output_row_id)
             upload_did = dataset.publish()
+            TestBase._track_test_server_dumps('data', upload_did)
+            print("\ncollected from {}: {}".format( __file__.split('/')[-1], upload_did))
             arff_dataset = arff.loads(_get_online_dataset_arff(upload_did))
             arff_data = np.array(arff_dataset['data'], dtype=object)
             # if we set the name of the index then the index will be added to

diff --git a/tests/test_extensions/test_sklearn_extension/test_sklearn_extension.py b/tests/test_extensions/test_sklearn_extension/test_sklearn_extension.py
@@ -1126,6 +1126,8 @@ def test_openml_param_name_to_sklearn(self):
         task = openml.tasks.get_task(115)
         run = openml.runs.run_flow_on_task(flow, task)
         run = run.publish()
+        TestBase._track_test_server_dumps('run', run.run_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], run.run_id))
         run = openml.runs.get_run(run.run_id)
         setup = openml.setups.get_setup(run.setup_id)
 

diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py
@@ -41,6 +41,10 @@ def setUp(self):
         super().setUp()
         self.extension = openml.extensions.sklearn.SklearnExtension()
 
+    def tearDown(self):
+        # TestBase.tracker.append(TestFlow.tracker)
+        super().tearDown()
+
     def test_get_flow(self):
         # We need to use the production server here because 4024 is not the
         # test server
@@ -177,6 +181,8 @@ def test_publish_flow(self):
         flow, _ = self._add_sentinel_to_flow_name(flow, None)
 
         flow.publish()
+        self._track_test_server_dumps('flow', flow.flow_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow.flow_id))
         self.assertIsInstance(flow.flow_id, int)
 
     @mock.patch('openml.flows.functions.flow_exists')
@@ -187,6 +193,9 @@ def test_publish_existing_flow(self, flow_exists_mock):
 
         with self.assertRaises(openml.exceptions.PyOpenMLError) as context_manager:
             flow.publish(raise_error_if_exists=True)
+            self._track_test_server_dumps('flow', flow.flow_id)
+            print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow.flow_id))
+        print(TestFlow.tracker)
 
         self.assertTrue('OpenMLFlow already exists' in context_manager.exception.message)
 
@@ -197,6 +206,8 @@ def test_publish_flow_with_similar_components(self):
         flow = self.extension.model_to_flow(clf)
         flow, _ = self._add_sentinel_to_flow_name(flow, None)
         flow.publish()
+        self._track_test_server_dumps('flow', flow.flow_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow.flow_id))
         # For a flow where both components are published together, the upload
         # date should be equal
         self.assertEqual(
@@ -213,6 +224,9 @@ def test_publish_flow_with_similar_components(self):
         flow1 = self.extension.model_to_flow(clf1)
         flow1, sentinel = self._add_sentinel_to_flow_name(flow1, None)
         flow1.publish()
+        self._track_test_server_dumps('flow', flow1.flow_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow1.flow_id))
+        print(TestFlow.tracker)
 
         # In order to assign different upload times to the flows!
         time.sleep(1)
@@ -222,6 +236,8 @@ def test_publish_flow_with_similar_components(self):
         flow2 = self.extension.model_to_flow(clf2)
         flow2, _ = self._add_sentinel_to_flow_name(flow2, sentinel)
         flow2.publish()
+        self._track_test_server_dumps('flow', flow2.flow_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow2.flow_id))
         # If one component was published before the other, the components in
         # the flow should have different upload dates
         self.assertNotEqual(flow2.upload_date,
@@ -234,6 +250,8 @@ def test_publish_flow_with_similar_components(self):
         # Child flow has different parameter. Check for storing the flow
         # correctly on the server should thus not check the child's parameters!
         flow3.publish()
+        self._track_test_server_dumps('flow', flow3.flow_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow3.flow_id))
 
     def test_semi_legal_flow(self):
         # TODO: Test if parameters are set correctly!
@@ -246,6 +264,8 @@ def test_semi_legal_flow(self):
         flow, _ = self._add_sentinel_to_flow_name(flow, None)
 
         flow.publish()
+        self._track_test_server_dumps('flow', flow.flow_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow.flow_id))
 
     @mock.patch('openml.flows.functions.get_flow')
     @mock.patch('openml.flows.functions.flow_exists')
@@ -260,6 +280,8 @@ def test_publish_error(self, api_call_mock, flow_exists_mock, get_flow_mock):
         get_flow_mock.return_value = flow
 
         flow.publish()
+        self._track_test_server_dumps('flow', flow.flow_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow.flow_id))
         self.assertEqual(api_call_mock.call_count, 1)
         self.assertEqual(get_flow_mock.call_count, 1)
         self.assertEqual(flow_exists_mock.call_count, 1)
@@ -271,6 +293,8 @@ def test_publish_error(self, api_call_mock, flow_exists_mock, get_flow_mock):
 
         with self.assertRaises(ValueError) as context_manager:
             flow.publish()
+            self._track_test_server_dumps('flow', flow.flow_id)
+            print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow.flow_id))
 
         fixture = (
             "Flow was not stored correctly on the server. "
@@ -336,6 +360,8 @@ def test_existing_flow_exists(self):
             flow, _ = self._add_sentinel_to_flow_name(flow, None)
             # publish the flow
             flow = flow.publish()
+            self._track_test_server_dumps('flow', flow.flow_id)
+            print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow.flow_id))
             # redownload the flow
             flow = openml.flows.get_flow(flow.flow_id)
 
@@ -394,6 +420,8 @@ def test_sklearn_to_upload_to_flow(self):
         flow, sentinel = self._add_sentinel_to_flow_name(flow, None)
 
         flow.publish()
+        self._track_test_server_dumps('flow', flow.flow_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow.flow_id))
         self.assertIsInstance(flow.flow_id, int)
 
         # Check whether we can load the flow again

diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py
@@ -251,6 +251,8 @@ def test_sklearn_to_flow_list_of_lists(self):
         # Test flow is accepted by server
         self._add_sentinel_to_flow_name(flow)
         flow.publish()
+        TestBase._track_test_server_dumps('flow', flow.flow_id)
+        print("\ncollected from {}: {}".format( __file__.split('/')[-1], flow.flow_id))
 
         # Test deserialization works
         server_flow = openml.flows.get_flow(flow.flow_id, reinstantiate=True)