#collapse_show
%lsmagic

Available line magics:
%alias  %alias_magic  %autoawait  %autocall  %automagic  %autosave  %bookmark  %cat  %cd  %clear  %colors  %conda  %config  %connect_info  %cp  %debug  %dhist  %dirs  %doctest_mode  %ed  %edit  %env  %gui  %hist  %history  %killbgscripts  %ldir  %less  %lf  %lk  %ll  %load  %load_ext  %loadpy  %logoff  %logon  %logstart  %logstate  %logstop  %ls  %lsmagic  %lx  %macro  %magic  %man  %matplotlib  %mkdir  %more  %mv  %notebook  %page  %pastebin  %pdb  %pdef  %pdoc  %pfile  %pinfo  %pinfo2  %pip  %popd  %pprint  %precision  %prun  %psearch  %psource  %pushd  %pwd  %pycat  %pylab  %qtconsole  %quickref  %recall  %rehashx  %reload_ext  %rep  %rerun  %reset  %reset_selective  %rm  %rmdir  %run  %save  %sc  %set_env  %store  %sx  %system  %tb  %time  %timeit  %unalias  %unload_ext  %who  %who_ls  %whos  %xdel  %xmode

Available cell magics:
%%!  %%HTML  %%SVG  %%bash  %%capture  %%debug  %%file  %%html  %%javascript  %%js  %%latex  %%markdown  %%perl  %%prun  %%pypy  %%python  %%python2  %%python3  %%ruby  %%script  %%sh  %%svg  %%sx  %%system  %%time  %%timeit  %%writefile

Automagic is ON, % prefix IS NOT needed for line magics.

#collapse
%matplotlib notebook

import matplotlib.pyplot as plt
import numpy as np
import datetime
from matplotlib import animation
import random

fig, ax = plt.subplots()

theta = np.linspace(0.5*np.pi, 2.5*np.pi, 100) # 0.5pi to make it start in the top centre of circle
r = np.sqrt(25.0) # radius of circle
x = r * np.cos(theta)
y = r * np.sin(theta)

circumference, = ax.plot(x, y, marker='o')

# to adjust BOTH the x and y axes such that the width between each divider is the same
ax.set_aspect(1)

plot_lim = r + 0.25
plt.xlim(-plot_lim, plot_lim)
plt.ylim(-plot_lim, plot_lim)

clock_timing_label_array = np.linspace(0.5*np.pi, 2.5*np.pi, 13)
clock_timing_label_x = (r - 0.2) * np.cos(clock_timing_label_array)
clock_timing_label_y = (r - 0.2) * np.sin(clock_timing_label_array)
time_coordinates = np.array((clock_timing_label_x, clock_timing_label_y)).T

ref_lines = []

for i, (x_pos, y_pos) in enumerate(time_coordinates):
    if(i != 0):
        plt.annotate("{}".format(i), (-x_pos, y_pos))
        
        # Drawing reference lines to the clock for fun
        sx = [0, -x_pos]
        sy = [0, y_pos]
        l, = ax.plot(sx, sy)
        ref_lines.append(l)


clock_hand_x_array, clock_hand_y_array = [], []
clock_hand, = plt.plot([], [], 'r')

clock_hand_ticks = np.linspace(0.5*np.pi, 2.5*np.pi, 61)
clock_hand_ticks_x = (r - 0.2) * np.cos(clock_hand_ticks)
clock_hand_ticks_y = (r - 0.2) * np.sin(clock_hand_ticks)
seconds_coordinates = np.array((clock_hand_ticks_x, clock_hand_ticks_y)).T

def init():
    return clock_hand,

def update(frame):
    clock_hand_x_array, ydata = [], []

    clock_hand_x_array = [0, -frame[0]]
    clock_hand_y_array = [0, frame[1]]
    
    clock_hand.set_data(clock_hand_x_array, clock_hand_y_array)
    
    plt.title('Analog Clock {}'.format(datetime.datetime.now().strftime("%H:%M:%S")), fontsize=18)
    return clock_hand,

ani = animation.FuncAnimation(fig, 
                    update, 
                    frames=seconds_coordinates,
                    init_func=init, 
                    blit=True,
                    interval=1000
                   )


plt.grid(linestyle='--')

# This part of the code increase the size of the graph since default is too small
fig_size = plt.gcf().get_size_inches() #Get current size
sizefactor = 1.5 #Set a zoom factor
# Modify the current size by the factor
plt.gcf().set_size_inches(sizefactor * fig_size) 

plt.title('Analog Clock 😃', fontsize=18)

def generate_color():
    r = lambda: random.randint(0,255)
    color = '#{:02x}{:02x}{:02x}'.format(r(), r(), r())
    return color

def onclick(event):
    circumference.set_color(generate_color())
    
def press(event):
    if event.key=='y':
        for l in ref_lines:
            l.set_color(generate_color())

fig.canvas.mpl_connect('button_press_event', onclick)
fig.canvas.mpl_connect('key_release_event', press)
#mpl_connect connects event onclick with string 'button_press_event'


#plt.savefig("plot_circle_matplotlib_01.png", bbox_inches='tight')
plt.show()

import pandas as pd
df = pd.read_html('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
cik = df[0][['CIK']]
cik.to_clipboard()

import graphviz
def gv(s): return graphviz.Source('digraph G{ rankdir="LR"' + s + '; }')

gv('diagrams[shape=box3d width=1 height=0.7] my -> diagrams -> are->very; diagrams->very->complex; my->complex')

import numpy as np
import matplotlib.pyplot as plt

num_pts=12 # number of points on the circle
ps = np.arange(num_pts)
# j = np.sqrt(-1)
pts = (np.exp(2j*np.pi/num_pts)**ps)

fig, ax = plt.subplots(1)
ax.plot(pts.real, pts.imag , 'o')
ax.set_aspect(1)
plt.show()

# %matplotlib inline
import matplotlib.pyplot as plt
from matplotlib import animation, rc
from IPython.display import HTML
import datetime


fig, ax = plt.subplots()

ax.set(xlim = (-1.5, 1.5), ylim = (-1.5, 1.5));
plt.axis('scaled')

ax.add_artist( plt.Circle((0,0), 1.1, fill=False) )

num_pts = 12 # number of points on the circle
pts = (np.exp(2j*np.pi/num_pts)** np.arange(num_pts) )
ax.plot(pts.real, pts.imag , 'o')

hand, = ax.plot([0, 0], [0, 1]) 
analogClock = ax.text(0, 0, '20', ha='center', va='center')

# plt.show()

# animation function. This is called sequentially
def animate(i):
  analogClock.set_text( str(i) )
  return (analogClock,)

anim = animation.FuncAnimation(fig, animate, 
                             interval=1000, blit=True)

rc('animation', html='jshtml')
anim

import numpy as np
import pandas as pd

df1 = pd.read_stata('http://www.stata-press.com/data/r11/auto.dta')
df1[4:8]   # df1.iloc[4:8,:]

#df1 = df1[df1.rep78.notna()]
df1.rep78 = df1.rep78.astype('Int8')

len(df1), df1.shape, df1.size  #df1.count()

(74, (74, 12), 888)

df1.groupby('rep78').size()

rep78
1     2
2     8
3    30
4    18
5    11
dtype: int64

df = df1.groupby('rep78').agg({'mpg': ['size','mean', 'median'], 'price':'max', 'length':'min'})

df = df1.groupby(['rep78', 'foreign']).size().unstack()
#df.index = df.index.astype('int32')
df

'h' + str(3)

'h3'

def my_aggfunc(arg): return(arg.name + str(len(arg)))

df = df1.pivot_table(index='rep78', columns='foreign', aggfunc=len, margins=True)['mpg'].fillna(0)  #.astype('int32')
df

df1.groupby(['rep78', 'foreign']).agg({'mpg': 'mean', 'weight':'max'})

for i in [['x','y'], ['a','b']]:
  vars()[i[0]] = i[1]
[x, a]

['y', 'b']

df = df1.pivot_table(index='rep78', columns='foreign', aggfunc=['mean'], values=['mpg'], margins=True).fillna(0)
df1a = df.T.reset_index(drop=True).T

df = df1.pivot_table(index='rep78', columns='foreign', aggfunc=['max'], values=['weight'], margins=True).fillna(0)
df1b = df.T.reset_index(drop=True).T

df1c = df1a.iloc[0:0]
df1c

for i in df1a.index:
    df1c = df1c.append(df1a.loc[i])
    df1c = df1c.append(df1b.loc[i])
    
df1c

l = df1.pivot_table(index='rep78', columns='foreign', aggfunc=['mean'], values=['mpg'], margins=True).fillna(0)
r = df1.pivot_table(index='rep78', columns='foreign', aggfunc=['max'], values=['weight'], margins=True).fillna(0)

d = l.join(r)
display(d.columns.levels)
d.columns = d.columns.droplevel(0)
d.stack(0)
# replace droplevel(0) with join(level 0 and 1)
# use ordered Categorical

FrozenList([['max', 'mean'], ['mpg', 'weight'], ['All', 'Domestic', 'Foreign']])

df1.plot.scatter('weight', 'mpg')

import seaborn as sns
sns.lmplot(x="weight", y="mpg", data=df1, fit_reg=True)

# X = df1["weight"]
# y = df1["mpg"]
# X = sm.add_constant(X)
# model = sm.OLS(y, X).fit()
# predictions = model.predict(X)
# model.summary()

Exercise #2

['inc80'[-2:],
(lambda x: [x[:-2], x[-2:]])('inc80')]
# df['variable'].apply(lambda x: x[-2:])

['80', ['inc', '80']]

import re
# .match (first only) vs .findall (all) 
[
  (re.sub(r'(\D+)(\d+)', r'\2 \1', 'inc80')),
  re.split(r'8', 'inc80'),
  (re.search(r'(\D+)(\d+)', 'inc80').groups())
]

['80 inc', ['inc', '0'], ('inc', '80')]

import pandas as pd
df2 = pd.read_stata('http://www.stata-press.com/data/r11/reshape1.dta')
df2 = df2.astype('int32')
df2

[i for i in list(df2.columns) if i[-2:-1]!='8' ]

['id', 'sex']

a = df2.set_index(['id', 'sex'])
for i in a.T :
    for j in a :
        display([i, j])

[(1, 0), 'inc80']

[(1, 0), 'inc81']

[(1, 0), 'inc82']

[(1, 0), 'ue80']

[(1, 0), 'ue81']

[(1, 0), 'ue82']

[(2, 1), 'inc80']

[(2, 1), 'inc81']

[(2, 1), 'inc82']

[(2, 1), 'ue80']

# for loop
table = []
for i in range(df2.shape[0]):
  temp = df2.iloc[i]
  for j in range(3):
    r = [temp[0],80+j,temp[1],temp[2+j],temp[5+j]]
    table.append(r)

pd.DataFrame(table, columns = ["id","year","sex","inc","ue"])

df = df2.melt(['id', 'sex'])
df[['variable', 'year']] = df['variable'] .apply (lambda x: pd.Series([x[:-2], x[-2:]]))
#df[['variable', 'year']] = df['variable'].str.extract('(\D+)(\d+)'); display(df)

df.pivot_table(index=['id', 'year', 'sex'], columns='variable', values = 'value') .reset_index()

## df2 = pd.read_stata('http://www.stata-press.com/data/r11/reshape1.dta')
df = df2.melt(['id', 'sex'])

df['year'] = df['variable'] 
df['variable'] = df['variable'] .apply(lambda x: x[:-2])
df['year']     = df['year']     .apply(lambda x: x[-2:])


df.pivot_table(index=['id', 'year', 'sex'], columns='variable', values = 'value') .reset_index()

df = df2.melt(['id', 'sex'])
df[['variable', 'year']] = df['variable'] .apply (lambda x: pd.Series([x[:-2], x[-2:]]))
df.set_index(['id', 'year', 'sex', 'variable']) .unstack()

df = df2.set_index(['id', 'sex']).stack().reset_index()
df[['level_2', 'year']] = df['level_2'].str.extract('(\D+)(\d+)'); 
df = df.set_index(['id', 'year', 'sex', 'level_2']).unstack()

display(df)

[df.shape, 
df.index.names, 
df.columns.names,
df.columns.levels]

[(9, 2),
 FrozenList(['id', 'year', 'sex']),
 FrozenList([None, 'level_2']),
 FrozenList([[0], ['inc', 'ue']])]

Exercise #3

df3 = pd.read_csv('http://www.ntu.edu.sg/home/fscheong/public/citycolors.csv'); df3

a = df3.melt('city')
b = a.groupby('city') .agg({'value':'max'}) .reset_index()
c = a.merge(b, on='city')
c [c['value_x']==c['value_y']] [['city','variable']] .reset_index(drop=True)  # .iloc[:,0:2]

MultiIndex Columns Names

cols = pd.MultiIndex.from_tuples([("a", "b"), ("a", "c")])
df = pd.DataFrame([[1,2], [3,4]], columns=cols)
df.columns.names = ['x', 'y']
display(df) #display(df['a'][['c']])

[df.shape, 
list(df.index),
list(df.columns),
df.index.names,
df.columns.names,
df.columns.levels]

[(2, 2),
 [0, 1],
 [('a', 'b'), ('a', 'c')],
 FrozenList([None]),
 FrozenList(['x', 'y']),
 FrozenList([['a'], ['b', 'c']])]

#df.columns = df.columns.droplevel()
df.columns = [col[1] for col in df.columns]
#df.columns = ['_'.join(col) for col in df.columns]
df

About

This notebook is a demonstration of some of capabilities of fastpages with notebooks.

With fastpages you can save your jupyter notebooks into the _notebooks folder at the root of your repository, and they will be automatically be converted to Jekyll compliant blog posts!

Front Matter

The first cell in your Jupyter Notebook or markdown blog post contains front matter. Front matter is metadata that can turn on/off options in your Notebook. It is formatted like this:

# "My Title"
> "Awesome summary"

- toc:true- branch: master- badges: true- comments: true
- author: Hamel Husain & Jeremy Howard
- categories: [fastpages, jupyter]

Setting toc: true will automatically generate a table of contents
Setting badges: true will automatically include GitHub and Google Colab links to your notebook.
Setting comments: true will enable commenting on your blog post, powered by utterances.

The title and description need to be enclosed in double quotes only if they include special characters such as a colon. More details and options for front matter can be viewed on the front matter section of the README.

Markdown Shortcuts

A #hide comment at the top of any code cell will hide both the input and output of that cell in your blog post.

A #hide_input comment at the top of any code cell will only hide the input of that cell.

The comment #hide_input was used to hide the code that produced this.

put a #collapse-hide flag at the top of any cell if you want to hide that cell by default, but give the reader the option to show it:

#collapse-hide
import pandas as pd
import altair as alt

put a #collapse-show flag at the top of any cell if you want to show that cell by default, but give the reader the option to hide it:

#collapse-show
cars = 'https://vega.github.io/vega-datasets/data/cars.json'
movies = 'https://vega.github.io/vega-datasets/data/movies.json'
sp500 = 'https://vega.github.io/vega-datasets/data/sp500.csv'
stocks = 'https://vega.github.io/vega-datasets/data/stocks.csv'
flights = 'https://vega.github.io/vega-datasets/data/flights-5k.json'

Interactive Charts With Altair

Charts made with Altair remain interactive. Example charts taken from this repo, specifically this notebook.

Example 1: DropDown

# single-value selection over [Major_Genre, MPAA_Rating] pairs
# use specific hard-wired values as the initial selected values
selection = alt.selection_single(
    name='Select',
    fields=['Major_Genre', 'MPAA_Rating'],
    init={'Major_Genre': 'Drama', 'MPAA_Rating': 'R'},
    bind={'Major_Genre': alt.binding_select(options=genres), 'MPAA_Rating': alt.binding_radio(options=mpaa)}
)
  
# scatter plot, modify opacity based on selection
alt.Chart(df).mark_circle().add_selection(
    selection
).encode(
    x='Rotten_Tomatoes_Rating:Q',
    y='IMDB_Rating:Q',
    tooltip='Title:N',
    opacity=alt.condition(selection, alt.value(0.75), alt.value(0.05))
)

Example 2: Tooltips

alt.Chart(df).mark_circle().add_selection(
    alt.selection_interval(bind='scales', encodings=['x'])
).encode(
    alt.X('Rotten_Tomatoes_Rating', type='quantitative'),
    alt.Y('IMDB_Rating', type='quantitative', axis=alt.Axis(minExtent=30)),
#     y=alt.Y('IMDB_Rating:Q', ), # use min extent to stabilize axis title placement
    tooltip=['Title:N', 'Release_Date:N', 'IMDB_Rating:Q', 'Rotten_Tomatoes_Rating:Q']
).properties(
    width=500,
    height=400
)

Example 3: More Tooltips

# select a point for which to provide details-on-demand
label = alt.selection_single(
    encodings=['x'], # limit selection to x-axis value
    on='mouseover',  # select on mouseover events
    nearest=True,    # select data point nearest the cursor
    empty='none'     # empty selection includes no data points
)

# define our base line chart of stock prices
base = alt.Chart().mark_line().encode(
    alt.X('date:T'),
    alt.Y('price:Q', scale=alt.Scale(type='log')),
    alt.Color('symbol:N')
)

alt.layer(
    base, # base line chart
    
    # add a rule mark to serve as a guide line
    alt.Chart().mark_rule(color='#aaa').encode(
        x='date:T'
    ).transform_filter(label),
    
    # add circle marks for selected time points, hide unselected points
    base.mark_circle().encode(
        opacity=alt.condition(label, alt.value(1), alt.value(0))
    ).add_selection(label),

    # add white stroked text to provide a legible background for labels
    base.mark_text(align='left', dx=5, dy=-5, stroke='white', strokeWidth=2).encode(
        text='price:Q'
    ).transform_filter(label),

    # add text labels for stock prices
    base.mark_text(align='left', dx=5, dy=-5).encode(
        text='price:Q'
    ).transform_filter(label),
    
    data=stocks
).properties(
    width=500,
    height=400
)

Data Tables

You can display tables per the usual way in your blog:

# display table with pandas
df[['Title', 'Worldwide_Gross', 
    'Production_Budget', 'Distributor', 'MPAA_Rating', 'IMDB_Rating', 'Rotten_Tomatoes_Rating']].head()

Images

Local Images

You can reference local images and they will be copied and rendered on your blog automatically. You can include these with the following markdown syntax:

![](my_icons/fastai_logo.png)

Remote Images

Remote images can be included with the following markdown syntax:

![](https://image.flaticon.com/icons/svg/36/36686.svg)

Animated Gifs

Animated Gifs work, too!

![](https://upload.wikimedia.org/wikipedia/commons/7/71/ChessPawnSpecialMoves.gif)

Captions

You can include captions with markdown images like this:

![](https://www.fast.ai/images/fastai_paper/show_batch.png "Credit: https://www.fast.ai/2020/02/13/fastai-A-Layered-API-for-Deep-Learning/")

Other Elements

GitHub Flavored Emojis

Typing I give this post two :+1:! will render this:

I give this post two :+1:!

Tweetcards

Typing > twitter: https://twitter.com/jakevdp/status/1204765621767901185?s=20 will render this:

Altair 4.0 is released! https://t.co/PCyrIOTcvv
Try it with:

pip install -U altair

The full list of changes is at https://t.co/roXmzcsT58 ...read on for some highlights. pic.twitter.com/vWJ0ZveKbZ
— Jake VanderPlas (@jakevdp) December 11, 2019

Youtube Videos

Typing > youtube: https://youtu.be/XfoYk_Z5AkI will render this:

Boxes / Callouts

Typing > Warning: There will be no second warning! will render this:

Warning: There will be no second warning!

Typing > Important: Pay attention! It's important. will render this:

Important: Pay attention! It’s important.

Typing > Tip: This is my tip. will render this:

Tip: This is my tip.

Typing > Note: Take note of this. will render this:

Note: Take note of this.

Typing > Note: A doc link to [an example website: fast.ai](https://www.fast.ai/) should also work fine. will render in the docs:

Note: A doc link to an example website: fast.ai should also work fine.

Footnotes

You can have footnotes in notebooks, however the syntax is different compared to markdown documents. This guide provides more detail about this syntax, which looks like this:

For example, here is a footnote {% fn 1 %}.
And another {% fn 2 %}
{{ 'This is the footnote.' | fndetail: 1 }}
{{ 'This is the other footnote. You can even have a [link](www.github.com)!' | fndetail: 2 }}

For example, here is a footnote ¹.

And another ²

1. This is the footnote.↩

2. This is the other footnote. You can even have a link!↩

	make	price	mpg	rep78	headroom	trunk	weight	length	turn	displacement	gear_ratio	foreign
4	Buick Electra	7827	15	4.0	4.0	20	4080	222	43	350	2.41	Domestic
5	Buick LeSabre	5788	18	3.0	4.0	21	3670	218	43	231	2.73	Domestic
6	Buick Opel	4453	26	NaN	3.0	10	2230	170	34	304	2.87	Domestic
7	Buick Regal	5189	20	3.0	2.0	16	3280	200	42	196	2.93	Domestic

		mpg	weight
rep78	foreign
1.0	Domestic	21.000000	3470.0
1.0	Foreign	NaN	NaN
2.0	Domestic	19.125000	3900.0
2.0	Foreign	NaN	NaN
3.0	Domestic	19.000000	4840.0
3.0	Foreign	23.333333	2130.0
4.0	Domestic	18.444444	4130.0
4.0	Foreign	24.888889	2750.0
5.0	Domestic	32.000000	2120.0
5.0	Foreign	26.333333	3170.0

	0	1	2
rep78
1.0	21.000000	0.000000	21.000000
1.0	3470.000000	0.000000	3470.000000
2.0	19.125000	0.000000	19.125000
2.0	3900.000000	0.000000	3900.000000
3.0	19.000000	23.333333	19.433333
3.0	4840.000000	2130.000000	4840.000000
4.0	18.444444	24.888889	21.666667
4.0	4130.000000	2750.000000	4130.000000
5.0	32.000000	26.333333	27.363636
5.0	2120.000000	3170.000000	3170.000000
All	19.541667	25.285714	21.289855
All	4840.000000	3170.000000	4840.000000

	foreign	All	Domestic	Foreign
rep78
1.0	mpg	21.000000	21.000000	0.000000
1.0	weight	3470.000000	3470.000000	0.000000
2.0	mpg	19.125000	19.125000	0.000000
2.0	weight	3900.000000	3900.000000	0.000000
3.0	mpg	19.433333	19.000000	23.333333
3.0	weight	4840.000000	4840.000000	2130.000000
4.0	mpg	21.666667	18.444444	24.888889
4.0	weight	4130.000000	4130.000000	2750.000000
5.0	mpg	27.363636	32.000000	26.333333
5.0	weight	3170.000000	2120.000000	3170.000000
All	mpg	21.289855	19.541667	25.285714
All	weight	4840.000000	4840.000000	3170.000000

	Title	Worldwide_Gross	Production_Budget	Distributor	MPAA_Rating	IMDB_Rating	Rotten_Tomatoes_Rating
0	The Land Girls	146083.0	8000000.0	Gramercy	R	6.1	NaN
1	First Love, Last Rites	10876.0	300000.0	Strand	R	6.9	NaN
2	I Married a Strange Person	203134.0	250000.0	Lionsgate	None	6.8	NaN
3	Let's Talk About Sex	373615.0	300000.0	Fine Line	None	NaN	13.0
4	Slam	1087521.0	1000000.0	Trimark	R	3.4	62.0

ureka

cfs ureca