Skip to content

Remove dataset pickles #336

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mfeurer merged 6 commits into from
Oct 11, 2017
Merged

Remove dataset pickles #336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c28258a
Remove unnecessary writes
mfeurer Oct 8, 2017
a212d3f
don't ignore pickle files for easier debugging of tests
amueller Oct 9, 2017
0b05fa4
don't remove files we don't create, remove files we actually create.
amueller Oct 9, 2017
2448826
Merge pull request #323 from openml/fix_#79
amueller Oct 9, 2017
b91b0b8
Merge branch 'remove_dataset_pickles' of https://github.com/amueller/…
mfeurer Oct 10, 2017
af1de06
FIX concurrency issue
mfeurer Oct 11, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,3 @@ target/
# IDE
.idea
*.swp

# Other
*.pkl
4 changes: 2 additions & 2 deletions openml/datasets/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ def get_data(self, target=None, target_dtype=int, include_row_id=False,

path = self.data_pickle_file
if not os.path.exists(path):
raise ValueError("Cannot find a ndarray file for dataset %s at "
raise ValueError("Cannot find a pickle file for dataset %s at "
"location %s " % (self.name, path))
else:
with open(path, "rb") as fh:
Expand Down Expand Up @@ -425,4 +425,4 @@ def _data_features_supported(self):
if self.features[idx].data_type not in ['numeric', 'nominal']:
return False
return True
return True
return True
3 changes: 0 additions & 3 deletions openml/datasets/functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,9 +338,6 @@ def _get_dataset_description(did_cache_dir, dataset_id):
description = xmltodict.parse(dataset_xml)[
"oml:data_set_description"]

with io.open(description_file, "w", encoding='utf8') as fh:
fh.write(dataset_xml)

return description


Expand Down
3 changes: 0 additions & 3 deletions openml/runs/functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -583,9 +583,6 @@ def get_run(run_id):

run = _create_run_from_xml(run_xml)

with io.open(run_file, "w", encoding='utf8') as fh:
fh.write(run_xml)

return run


Expand Down
237 changes: 101 additions & 136 deletions tests/test_datasets/test_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,57 +1,21 @@
import inspect
import unittest
import os

import numpy as np
from scipy import sparse
import six

from openml import OpenMLDataset
from openml.testing import TestBase
import openml


class OpenMLDatasetTest(unittest.TestCase):
# Splitting not helpful, these test's don't rely on the server and take less
# than 5 seconds + rebuilding the test would potentially be costly
class OpenMLDatasetTest(TestBase):
_multiprocess_can_split_ = True

def setUp(self):
# Load dataset id 1
__file__ = inspect.getfile(OpenMLDatasetTest)
self.directory = os.path.dirname(__file__)
self.arff_filename = os.path.join(self.directory, "..", "files",
"datasets", "2", "dataset.arff")
self.pickle_filename = os.path.join(self.directory, "..", "files",
"datasets", "2", "dataset.pkl")
self.dataset = OpenMLDataset(
1, "anneal", 2, "Lorem ipsum.", "arff", None, None, None,
"2014-04-06 23:19:24", None, "Public",
"http://openml.liacs.nl/files/download/2/dataset_2_anneal.ORIG.arff",
"class", None, None, None, None, None, None, None, None, None,
"939966a711925e333bf4aaadeaa71135", data_file=self.arff_filename)

self.sparse_arff_filename = os.path.join(
self.directory, "..", "files", "datasets", "-1", "dataset.arff")
self.sparse_pickle_filename = os.path.join(
self.directory, "..", "files", "datasets", "-1", "dataset.pkl")
self.sparse_dataset = OpenMLDataset(
-1, "dexter", -1, "Lorem ipsum.", "arff", None, None, None, None,
None, "Public",
"http://www.cs.ubc.ca/labs/beta/Projects/autoweka/datasets/dexter.zip",
"class", None, None, None, None, None, None, None, None, None,
None, data_file=self.sparse_arff_filename)

def tearDown(self):
for file_ in [self.pickle_filename, self.sparse_pickle_filename]:
os.remove(file_)

##########################################################################
# Pandas

@unittest.skip("Does not work right now")
def test_get_arff(self):
rval = self.dataset.get_arff()
self.assertIsInstance(rval, tuple)
self.assertIsInstance(rval[0], np.ndarray)
self.assertTrue(hasattr(rval[1], '__dict__'))
self.assertEqual(rval[0].shape, (898, ))
super(OpenMLDatasetTest, self).setUp()
openml.config.server = self.production_server

# Load dataset id 2 - dataset 2 is interesting because it contains
# missing values, categorical features etc.
self.dataset = openml.datasets.get_dataset(2)

def test_get_data(self):
# Basic usage
Expand All @@ -69,22 +33,18 @@ def test_get_data(self):
self.assertTrue(all([isinstance(att, six.string_types)
for att in attribute_names]))

def test_get_sparse_dataset(self):
rval = self.sparse_dataset.get_data()
self.assertIsInstance(rval, np.ndarray)
def test_get_data_with_rowid(self):
self.dataset.row_id_attribute = "condition"
rval, categorical = self.dataset.get_data(
include_row_id=True, return_categorical_indicator=True)
self.assertEqual(rval.dtype, np.float32)
self.assertEqual((2, 20001), rval.shape)
rval, categorical = self.sparse_dataset.get_data(
return_categorical_indicator=True)
self.assertIsInstance(rval, np.ndarray)
self.assertEqual(len(categorical), 20001)
self.assertTrue(all([isinstance(cat, bool) for cat in categorical]))
rval, attribute_names = self.sparse_dataset.get_data(
return_attribute_names=True)
self.assertIsInstance(rval, np.ndarray)
self.assertEqual(len(attribute_names), 20001)
self.assertTrue(all([isinstance(att, six.string_types)
for att in attribute_names]))
self.assertEqual(rval.shape, (898, 39))
self.assertEqual(len(categorical), 39)
rval, categorical = self.dataset.get_data(
include_row_id=False, return_categorical_indicator=True)
self.assertEqual(rval.dtype, np.float32)
self.assertEqual(rval.shape, (898, 38))
self.assertEqual(len(categorical), 38)

def test_get_data_with_target(self):
X, y = self.dataset.get_data(target="class")
Expand All @@ -98,122 +58,127 @@ def test_get_data_with_target(self):
self.assertNotIn("class", attribute_names)
self.assertEqual(y.shape, (898, ))

def test_get_data_rowid_and_ignore_and_target(self):
self.dataset.ignore_attributes = ["condition"]
self.dataset.row_id_attribute = ["hardness"]
X, y = self.dataset.get_data(target="class", include_row_id=False,
include_ignore_attributes=False)
self.assertEqual(X.dtype, np.float32)
self.assertIn(y.dtype, [np.int32, np.int64])
self.assertEqual(X.shape, (898, 36))
X, y, categorical = self.dataset.get_data(
target="class", return_categorical_indicator=True)
self.assertEqual(len(categorical), 36)
self.assertListEqual(categorical, [True] * 3 + [False] + [True] * 2 + [
False] + [True] * 23 + [False] * 3 + [True] * 3)
self.assertEqual(y.shape, (898, ))

def test_get_data_with_ignore_attributes(self):
self.dataset.ignore_attributes = ["condition"]
rval = self.dataset.get_data(include_ignore_attributes=True)
self.assertEqual(rval.dtype, np.float32)
self.assertEqual(rval.shape, (898, 39))
rval, categorical = self.dataset.get_data(
include_ignore_attributes=True, return_categorical_indicator=True)
self.assertEqual(len(categorical), 39)
rval = self.dataset.get_data(include_ignore_attributes=False)
self.assertEqual(rval.dtype, np.float32)
self.assertEqual(rval.shape, (898, 38))
rval, categorical = self.dataset.get_data(
include_ignore_attributes=False, return_categorical_indicator=True)
self.assertEqual(len(categorical), 38)
# TODO test multiple ignore attributes!


class OpenMLDatasetTestSparse(TestBase):
_multiprocess_can_split_ = True

def setUp(self):
super(OpenMLDatasetTestSparse, self).setUp()
openml.config.server = self.production_server

self.sparse_dataset = openml.datasets.get_dataset(4136)

def test_get_sparse_dataset_with_target(self):
X, y = self.sparse_dataset.get_data(target="class")
self.assertIsInstance(X, np.ndarray)
self.assertTrue(sparse.issparse(X))
self.assertEqual(X.dtype, np.float32)
self.assertIsInstance(y, np.ndarray)
self.assertIn(y.dtype, [np.int32, np.int64])
self.assertEqual(X.shape, (2, 20000))
self.assertEqual(X.shape, (600, 20000))
X, y, attribute_names = self.sparse_dataset.get_data(
target="class", return_attribute_names=True)
self.assertIsInstance(X, np.ndarray)
self.assertTrue(sparse.issparse(X))
self.assertEqual(len(attribute_names), 20000)
self.assertNotIn("class", attribute_names)
self.assertEqual(y.shape, (2, ))
self.assertEqual(y.shape, (600, ))

def test_get_data_with_rowid(self):
self.dataset.row_id_attribute = "condition"
rval, categorical = self.dataset.get_data(
include_row_id=True, return_categorical_indicator=True)
self.assertEqual(rval.dtype, np.float32)
self.assertEqual(rval.shape, (898, 39))
self.assertEqual(len(categorical), 39)
rval, categorical = self.dataset.get_data(
include_row_id=False, return_categorical_indicator=True)
def test_get_sparse_dataset(self):
rval = self.sparse_dataset.get_data()
self.assertTrue(sparse.issparse(rval))
self.assertEqual(rval.dtype, np.float32)
self.assertEqual(rval.shape, (898, 38))
self.assertEqual(len(categorical), 38)

# TODO this is not yet supported!
#rowid = ["condition", "formability"]
#self.dataset.row_id_attribute = rowid
#rval = self.dataset.get_pandas(include_row_id=False)
self.assertEqual((600, 20001), rval.shape)
rval, categorical = self.sparse_dataset.get_data(
return_categorical_indicator=True)
self.assertTrue(sparse.issparse(rval))
self.assertEqual(len(categorical), 20001)
self.assertTrue(all([isinstance(cat, bool) for cat in categorical]))
rval, attribute_names = self.sparse_dataset.get_data(
return_attribute_names=True)
self.assertTrue(sparse.issparse(rval))
self.assertEqual(len(attribute_names), 20001)
self.assertTrue(all([isinstance(att, six.string_types)
for att in attribute_names]))

def test_get_sparse_dataset_with_rowid(self):
self.sparse_dataset.row_id_attribute = ["a_0"]
self.sparse_dataset.row_id_attribute = ["V256"]
rval, categorical = self.sparse_dataset.get_data(
include_row_id=True, return_categorical_indicator=True)
self.assertIsInstance(rval, np.ndarray)
self.assertTrue(sparse.issparse(rval))
self.assertEqual(rval.dtype, np.float32)
self.assertEqual(rval.shape, (2, 20001))
self.assertEqual(rval.shape, (600, 20001))
self.assertEqual(len(categorical), 20001)
rval, categorical = self.sparse_dataset.get_data(
include_row_id=False, return_categorical_indicator=True)
self.assertIsInstance(rval, np.ndarray)
self.assertTrue(sparse.issparse(rval))
self.assertEqual(rval.dtype, np.float32)
self.assertEqual(rval.shape, (2, 20000))
self.assertEqual(rval.shape, (600, 20000))
self.assertEqual(len(categorical), 20000)

# TODO this is not yet supported!
# rowid = ["condition", "formability"]
#self.dataset.row_id_attribute = rowid
#rval = self.dataset.get_pandas(include_row_id=False)

def test_get_data_with_ignore_attributes(self):
self.dataset.ignore_attributes = ["condition"]
rval = self.dataset.get_data(include_ignore_attributes=True)
self.assertEqual(rval.dtype, np.float32)
self.assertEqual(rval.shape, (898, 39))
rval, categorical = self.dataset.get_data(
include_ignore_attributes=True, return_categorical_indicator=True)
self.assertEqual(len(categorical), 39)
rval = self.dataset.get_data(include_ignore_attributes=False)
self.assertEqual(rval.dtype, np.float32)
self.assertEqual(rval.shape, (898, 38))
rval, categorical = self.dataset.get_data(
include_ignore_attributes=False, return_categorical_indicator=True)
self.assertEqual(len(categorical), 38)
# TODO test multiple ignore attributes!

def test_get_sparse_dataset_with_ignore_attributes(self):
self.sparse_dataset.ignore_attributes = ["a_0"]
self.sparse_dataset.ignore_attributes = ["V256"]
rval = self.sparse_dataset.get_data(include_ignore_attributes=True)
self.assertIsInstance(rval, np.ndarray)
self.assertTrue(sparse.issparse(rval))
self.assertEqual(rval.dtype, np.float32)
self.assertEqual(rval.shape, (2, 20001))
self.assertEqual(rval.shape, (600, 20001))
rval, categorical = self.sparse_dataset.get_data(
include_ignore_attributes=True, return_categorical_indicator=True)
self.assertIsInstance(rval, np.ndarray)
self.assertTrue(sparse.issparse(rval))
self.assertEqual(len(categorical), 20001)
rval = self.sparse_dataset.get_data(include_ignore_attributes=False)
self.assertIsInstance(rval, np.ndarray)
self.assertTrue(sparse.issparse(rval))
self.assertEqual(rval.dtype, np.float32)
self.assertEqual(rval.shape, (2, 20000))
self.assertEqual(rval.shape, (600, 20000))
rval, categorical = self.sparse_dataset.get_data(
include_ignore_attributes=False, return_categorical_indicator=True)
self.assertIsInstance(rval, np.ndarray)
self.assertTrue(sparse.issparse(rval))
self.assertEqual(len(categorical), 20000)
# TODO test multiple ignore attributes!

def test_get_data_rowid_and_ignore_and_target(self):
self.dataset.ignore_attributes = ["condition"]
self.dataset.row_id_attribute = ["hardness"]
X, y = self.dataset.get_data(target="class", include_row_id=False,
include_ignore_attributes=False)
self.assertEqual(X.dtype, np.float32)
self.assertIn(y.dtype, [np.int32, np.int64])
self.assertEqual(X.shape, (898, 36))
X, y, categorical = self.dataset.get_data(
target="class", return_categorical_indicator=True)
self.assertEqual(len(categorical), 36)
self.assertListEqual(categorical, [True] * 3 + [False] + [True] * 2 + [
False] + [True] * 23 + [False] * 3 + [True] * 3)
self.assertEqual(y.shape, (898, ))

def test_get_sparse_dataset_rowid_and_ignore_and_target(self):
self.sparse_dataset.ignore_attributes = ["a_0"]
self.sparse_dataset.row_id_attribute = ["a_1"]
# TODO: re-add row_id and ignore attributes
self.sparse_dataset.ignore_attributes = ["V256"]
self.sparse_dataset.row_id_attribute = ["V512"]
X, y = self.sparse_dataset.get_data(
target="class", include_row_id=False,
include_ignore_attributes=False)
self.assertIsInstance(X, np.ndarray)
self.assertTrue(sparse.issparse(X))
self.assertEqual(X.dtype, np.float32)
self.assertIn(y.dtype, [np.int32, np.int64])
self.assertEqual(X.shape, (2, 19998))
self.assertEqual(X.shape, (600, 19998))
X, y, categorical = self.sparse_dataset.get_data(
target="class", return_categorical_indicator=True)
self.assertIsInstance(X, np.ndarray)
self.assertTrue(sparse.issparse(X))
self.assertEqual(len(categorical), 19998)
self.assertListEqual(categorical, [False] * 19998)
self.assertEqual(y.shape, (2, ))
self.assertEqual(y.shape, (600, ))
Loading