Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 98 additions & 0 deletions dataikuapi/dss/managedfolder.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
from ..utils import DataikuException
from ..utils import DataikuUTF8CSVReader
from ..utils import DataikuStreamedHttpUTF8CSVReader


class DSSManagedFolder(object):
"""
A managed folder on the DSS instance
"""
def __init__(self, client, project_key, odb_id):
self.client = client
self.project_key = project_key
self.odb_id = odb_id

########################################################
# Managed folder deletion
########################################################

def delete(self):
"""
Delete the managed folder
"""
return self.client._perform_empty(
"DELETE", "/projects/%s/managedfolders/%s" % (self.project_key, self.odb_id))



########################################################
# Managed folder definition
########################################################

def get_definition(self):
"""
Get the definition of the managed folder

Returns:
the definition, as a JSON object
"""
return self.client._perform_json(
"GET", "/projects/%s/managedfolders/%s/" % (self.project_key, self.odb_id))

def set_definition(self, definition):
"""
Set the definition of the managed folder

Args:
definition: the definition, as a JSON object. You should only set a definition object
that has been retrieved using the get_definition call.
"""
return self.client._perform_json(
"PUT", "/projects/%s/managedfolders/%s/" % (self.project_key, self.odb_id),
body=definition)


########################################################
# Managed folder contents
########################################################

def list_contents(self):
"""
Get the list of files in the managed folder

Returns:
the list of files, as a JSON object
"""
return self.client._perform_json(
"GET", "/projects/%s/managedfolders/%s/contents" % (self.project_key, self.odb_id))

def get_file(self, path):
"""
Get a file from the managed folder

Returns:
the file's content, as a stream
"""
return self.client._perform_raw(
"GET", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, path))

def delete_file(self, path):
"""
Delete a file from the managed folder
"""
return self.client._perform_empty(
"DELETE", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, path))

def put_file(self, name, f):
"""
Upload the file to the managed folder

Args:
f: the file contents, as a stream
name: the name of the file
"""
return self.client._perform_json_upload(
"POST", "/projects/%s/managedfolders/%s/contents/" % (self.project_key, self.odb_id),
name, f)


47 changes: 47 additions & 0 deletions dataikuapi/dss/project.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from dataset import DSSDataset
from managedfolder import DSSManagedFolder
from job import DSSJob
from apiservice import DSSAPIService

Expand Down Expand Up @@ -154,6 +155,52 @@ def create_dataset(self, dataset_name, type,
return DSSDataset(self.client, self.project_key, dataset_name)


########################################################
# Managed folders
########################################################

def list_managed_folders(self):
"""
List the managed folders in this project

Returns:
the list of the managed folders, each one as a JSON object
"""
return self.client._perform_json(
"GET", "/projects/%s/managedfolders/" % self.project_key)

def get_managed_folder(self, odb_id):
"""
Get a handle to interact with a specific managed folder

Args:
odb_id: the identifier of the desired managed folder

Returns:
A :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` managed folder handle
"""
return DSSManagedFolder(self.client, self.project_key, odb_id)

def create_managed_folder(self, name):
"""
Create a new managed folder in the project, and return a handle to interact with it

Args:
name: the name of the managed folder

Returns:
A :class:`dataikuapi.dss.managedfolder.DSSManagedFolder` managed folder handle
"""
obj = {
"name" : name,
"projectKey" : self.project_key
}
res = self.client._perform_json("POST", "/projects/%s/managedfolders/" % self.project_key,
body = obj)
odb_id = res['id']
return DSSManagedFolder(self.client, self.project_key, odb_id)


########################################################
# Jobs
########################################################
Expand Down
14 changes: 14 additions & 0 deletions dataikuapi/dssclient.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,3 +359,17 @@ def _perform_json(self, method, path, params=None, body=None):
def _perform_raw(self, method, path, params=None, body=None):
return self._perform_http(method, path, params, body, True)

def _perform_json_upload(self, method, path, name, f):
auth = HTTPBasicAuth(self.api_key, "")

try:
http_res = self._session.request(
method, "%s/dip/publicapi%s" % (self.host, path),
auth=auth,
files = {'file': (name, f, {'Expires': '0'})} )
http_res.raise_for_status()
return http_res
except exceptions.HTTPError:
ex = http_res.json()
raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message")))

95 changes: 95 additions & 0 deletions tests/managedfolder_tests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
from dataikuapi.dssclient import DSSClient
import json
from nose.tools import ok_
from nose.tools import eq_
import tempfile
import filecmp
import os.path as osp
import sys, traceback
from contextlib import closing

host="http://localhost:8082"
apiKey="5YZ4lHpXexhlNk29FqRi8AcO2EVGddtA"
testProjectKey="BOX"

def list_managedfolders_test():
client = DSSClient(host, apiKey)
managedfolders = client.get_project(testProjectKey).list_managed_folders()
ok_(len(managedfolders) > 0)

def create_delete_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
count = len(project.list_managed_folders())

managedfolder = project.create_managed_folder("titi")
eq_(count + 1, len(project.list_managed_folders()))

managedfolder.delete()
eq_(count, len(project.list_managed_folders()))

def get_set_test():
client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")

definition = managedfolder.get_definition()
definition['description'] = 'describe me!'
managedfolder.set_definition(definition)
definition2 = managedfolder.get_definition()

eq_('describe me!', definition2['description'])

managedfolder.delete()

def updload_download_test():
temp_folder = tempfile.mkdtemp()
stuff = osp.join(temp_folder, "test.txt")
with open(stuff, "w") as f:
f.write('some contents\n on several\nlines')

client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")

count = len(managedfolder.list_contents()['items'])
with open(stuff, "r") as f:
managedfolder.put_file('stuff', f)
eq_(count + 1, len(managedfolder.list_contents()['items']))

stuff2 = osp.join(temp_folder, "test2.txt")
with open(stuff2, "w") as f:
with closing(managedfolder.get_file('stuff')) as s:
f.write(s.raw.read())

eq_(True, filecmp.cmp(stuff, stuff2))

managedfolder.delete()

def updload_replace_delete_test():
temp_folder = tempfile.mkdtemp()
stuff = osp.join(temp_folder, "test.txt")
stuffb = osp.join(temp_folder, "testb.txt")
with open(stuff, "w") as f:
f.write('some contents\n on several\nlines')

client = DSSClient(host, apiKey)
project = client.get_project(testProjectKey)
managedfolder = project.create_managed_folder("titi")

count = len(managedfolder.list_contents()['items'])
with open(stuff, "r") as f:
managedfolder.put_file('stuff', f)
eq_(count + 1, len(managedfolder.list_contents()['items']))

with open(stuffb, "w") as f:
with closing(managedfolder.get_file('stuff')) as s:
f.write(s.raw.read())

eq_(True, filecmp.cmp(stuff, stuffb))

managedfolder.delete_file('stuff')
eq_(count, len(managedfolder.list_contents()['items']))

managedfolder.delete()