mfeurer · Jul 9, 2019 · Jun 27, 2019 · Jul 1, 2019 · Jul 3, 2019 · Jul 3, 2019
diff --git a/ci_scripts/test.sh b/ci_scripts/test.sh
@@ -1,5 +1,9 @@
 set -e
 
+# check status and branch before running the unit tests
+before="`git status --porcelain -b`"
+before="$before"
+
 run_tests() {
     # Get into a temp directory to run test from the installed scikit learn and
     # check if we  do not leave artifacts
@@ -22,7 +26,7 @@ run_tests() {
         PYTEST_ARGS=''
     fi
 
-    pytest -n 4 --duration=20 --timeout=600 --timeout-method=thread -sv --ignore='test_OpenMLDemo.py' $PYTEST_ARGS $test_dir
+    pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread -sv --ignore='test_OpenMLDemo.py' $PYTEST_ARGS $test_dir
 }
 
 if [[ "$RUN_FLAKE8" == "true" ]]; then
@@ -32,3 +36,11 @@ fi
 if [[ "$SKIP_TESTS" != "true" ]]; then
     run_tests
 fi
+
+# check status and branch after running the unit tests
+# compares with $before to check for remaining files
+after="`git status --porcelain -b`"
+if [[ "$before" != "$after" ]]; then
+    echo "All generated files have not been deleted!"
+    exit 1
+fi
diff --git a/doc/progress.rst b/doc/progress.rst
@@ -11,6 +11,7 @@ Changelog
 
 * ADD #687: Adds a function to retrieve the list of evaluation measures available.
 * ADD #695: A function to retrieve all the data quality measures available.
+* FIX #447: All files created by unit tests are deleted after the completion of all unit tests.
 
 0.9.0
 ~~~~~

diff --git a/openml/testing.py b/openml/testing.py
@@ -17,6 +17,8 @@
 import openml
 from openml.tasks import TaskTypeEnum
 
+import pytest
+
 
 class TestBase(unittest.TestCase):
     """Base class for tests
@@ -44,7 +46,6 @@ def setUp(self, n_levels: int = 1):
             Number of nested directories the test is in. Necessary to resolve the path to the
             ``files`` directory, which is located directly under the ``tests`` directory.
         """
-
         # This cache directory is checked in to git to simulate a populated
         # cache
         self.maxDiff = None
@@ -103,6 +104,48 @@ def tearDown(self):
         openml.config.server = self.production_server
         openml.config.connection_n_retries = self.connection_n_retries
 
+    @pytest.fixture(scope="session", autouse=True)
+    def _cleanup_fixture(self):
+        """Cleans up files generated by Unit tests
+
+        This function is called at the beginning of the invocation of
+        TestBase (defined below), by each of class that inherits TestBase.
+        The 'yield' creates a checkpoint and breaks away to continue running
+        the unit tests of the sub class. When all the tests end, execution
+        resumes from the checkpoint.
+        """
+
+        abspath_this_file = os.path.abspath(inspect.getfile(self.__class__))
+        static_cache_dir = os.path.dirname(abspath_this_file)
+        # Could be a risky while condition, however, going up a directory
+        # n-times will eventually end at main directory
+        while True:
+            if 'openml' in os.listdir(static_cache_dir):
+                break
+            else:
+                static_cache_dir = os.path.join(static_cache_dir, '../')
+        directory = os.path.join(static_cache_dir, 'tests/files/')
+        # directory = "{}/tests/files/".format(static_cache_dir)
+        files = os.walk(directory)
+        old_file_list = []
+        for root, _, filenames in files:
+            for filename in filenames:
+                old_file_list.append(os.path.join(root, filename))
+        # context switches to other remaining tests
+        # pauses the code execution here till all tests in the 'session' is over
+        yield
+        # resumes from here after all collected tests are completed
+        files = os.walk(directory)
+        new_file_list = []
+        for root, _, filenames in files:
+            for filename in filenames:
+                new_file_list.append(os.path.join(root, filename))
+        # filtering the files generated during this run
+        new_file_list = list(set(new_file_list) - set(old_file_list))
+        print("Files to delete in local: {}".format(new_file_list))
+        for file in new_file_list:
+            os.remove(file)
+
     def _get_sentinel(self, sentinel=None):
         if sentinel is None:
             # Create a unique prefix for the flow. Necessary because the flow

diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py
@@ -43,14 +43,14 @@ def tearDown(self):
         super(TestOpenMLDataset, self).tearDown()
 
     def _remove_pickle_files(self):
-        cache_dir = self.static_cache_dir
+        self.lock_path = os.path.join(openml.config.get_cache_directory(), 'locks')
         for did in ['-1', '2']:
             with lockutils.external_lock(
                     name='datasets.functions.get_dataset:%s' % did,
-                    lock_path=os.path.join(openml.config.get_cache_directory(), 'locks'),
+                    lock_path=self.lock_path,
             ):
-                pickle_path = os.path.join(cache_dir, 'datasets', did,
-                                           'dataset.pkl')
+                pickle_path = os.path.join(openml.config.get_cache_directory(), 'datasets',
+                                           did, 'dataset.pkl.py3')
                 try:
                     os.remove(pickle_path)
                 except (OSError, FileNotFoundError):

diff --git a/tests/test_tasks/test_split.py b/tests/test_tasks/test_split.py
@@ -19,8 +19,7 @@ def setUp(self):
             self.directory, "..", "files", "org", "openml", "test",
             "tasks", "1882", "datasplits.arff"
         )
-        # TODO Needs to be adapted regarding the python version
-        self.pd_filename = self.arff_filename.replace(".arff", ".pkl")
+        self.pd_filename = self.arff_filename.replace(".arff", ".pkl.py3")
 
     def tearDown(self):
         try:

diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py
@@ -12,6 +12,12 @@
 class TestTask(TestBase):
     _multiprocess_can_split_ = True
 
+    def setUp(self):
+        super(TestTask, self).setUp()
+
+    def tearDown(self):
+        super(TestTask, self).tearDown()
+
     def test__get_cached_tasks(self):
         openml.config.cache_directory = self.static_cache_dir
         tasks = openml.tasks.functions._get_cached_tasks()

diff --git a/tests/test_tasks/test_task_methods.py b/tests/test_tasks/test_task_methods.py
@@ -7,6 +7,12 @@
 # Common methods between tasks
 class OpenMLTaskMethodsTest(TestBase):
 
+    def setUp(self):
+        super(OpenMLTaskMethodsTest, self).setUp()
+
+    def tearDown(self):
+        super(OpenMLTaskMethodsTest, self).tearDown()
+
     def test_tagging(self):
         task = openml.tasks.get_task(1)
         tag = "testing_tag_{}_{}".format(self.id(), time())
-Original file line number
+Diff line change
@@ Expand Up / @@ -11,6 +11,7 @@ Changelog @@
     * ADD #687: Adds a function to retrieve the list of evaluation measures available.
     * ADD #695: A function to retrieve all the data quality measures available.
+    * FIX #447: All files created by unit tests are deleted after the completion of all unit tests.
 .9.0
     ~~~~~
@@ Expand Down @@