Skip to content

Manual edits jun29 pm #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mlkimmins merged 2 commits into from
Jun 29, 2023
Merged

Manual edits jun29 pm #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1148123
fix ML chapter references
Jun 29, 2023
159d82c
fix newlines on ray example
Jun 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 28 additions & 41 deletions dask/Dask-Ch11_ML.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,6 @@
# !pip install xgboost








# In[4]:


Expand Down Expand Up @@ -140,9 +134,6 @@
#end::ex_plot_distances





# In[19]:


Expand All @@ -151,9 +142,6 @@
df['trip_distance'].values.compute_chunk_sizes()





# In[20]:


Expand Down Expand Up @@ -211,28 +199,19 @@
plt.show()





# In[25]:


df['log_trip_duration'] = np.log(df['trip_duration'])





plt.hist(df['log_trip_duration'], bins=100)
plt.xlabel('log(trip_duration)')
plt.ylabel('number of records')
plt.show()
sns.distplot(df["log_trip_duration"], bins=100)





# In[30]:


Expand Down Expand Up @@ -436,9 +415,6 @@
plt.show()





train_final = train.drop(['VendorID',
'passenger_count',
'store_and_fwd_flag',
Expand Down Expand Up @@ -473,9 +449,6 @@
# In[2]:





# Just like standard xgb Dmatrix, but note that we are explicitly passing
# in columns since we're dealing with Pandas, and that we need to give the
# colnames for xgb to know feature names
Expand Down Expand Up @@ -511,25 +484,45 @@
# In[62]:


#tag::dask_delayed_load_model[]
#tag::dask_delayed_load[]
from skimage.io import imread
from skimage.io.collection import alphanumeric_key
from dask import delayed
import dask.array as da
import os

root, dirs, filenames = os.walk(dataset_dir)
# sample first file
imread(filenames[0])


@dask.delayed
def load_model(path):
with fs.open(path, 'rb') as f:
img = Image.open(f)
return img
#end::dask_delayed_load_model
def lazy_reader(file):
return imread(file)


# In[63]:
# we have a bunch of delayed readers attached to the files
lazy_arrays = [lazy_reader(file) for file in filenames]

# read individual files from reader into a dask array
# particularly useful if each image is a large file like DICOM radiographs
# mammography dicom tends to be extremely large
dask_arrays = [
da.from_delayed(delayed_reader, shape=(4608, 5200,), dtype=np.float32)
for delayed_reader in lazy_arrays
]
#end::dask_delayed_load


# In[63]:


#tag::Dask_DataFrame_map_partition_inference[]
import dask.dataframe as dd
import dask.bag as db


def rowwise_operation(row, arg *):
def rowwise_operation(row, arg * ):
#row-wise compute
return result

Expand All @@ -546,12 +539,6 @@ def partition_operation(df):
#end::Dask_DataFrame_map_partition_inference








#tag::batched_operations[]
def handle_batch(batch, conn, nlp_model):
#run_inference_here.
Expand Down
33 changes: 27 additions & 6 deletions dask/Dask-Ch13_ML.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1056,13 +1056,34 @@
"metadata": {},
"outputs": [],
"source": [
"#tag::dask_delayed_load_model[]\n",
"#tag::dask_delayed_load[]\n",
"from skimage.io import imread\n",
"from skimage.io.collection import alphanumeric_key\n",
"from dask import delayed\n",
"import dask.array as da\n",
"import os\n",
"\n",
"root, dirs, filenames = os.walk(dataset_dir)\n",
"# sample first file\n",
"imread(filenames[0])\n",
"\n",
"\n",
"@dask.delayed\n",
"def load_model(path):\n",
" with fs.open(path, 'rb') as f:\n",
" img = Image.open(f)\n",
" return img\n",
"#end::dask_delayed_load_model"
"def lazy_reader(file):\n",
" return imread(file)\n",
"\n",
"\n",
"# we have a bunch of delayed readers attached to the files\n",
"lazy_arrays = [lazy_reader(file) for file in filenames]\n",
"\n",
"# read individual files from reader into a dask array\n",
"# particularly useful if each image is a large file like DICOM radiographs\n",
"# mammography dicom tends to be extremely large\n",
"dask_arrays = [\n",
" da.from_delayed(delayed_reader, shape=(4608, 5200,), dtype=np.float32)\n",
" for delayed_reader in lazy_arrays\n",
"]\n",
"#end::dask_delayed_load"
]
},
{
Expand Down
Loading