Skip to content

Commit b2b0cbe

Browse files
ChrisBarker-NOAAChrisBarker-NOAA
committed
updated trigrams exercise with TDD. Could use a few tests of the text processing still.
1 parent 74c8535 commit b2b0cbe

8 files changed

Lines changed: 261 additions & 38 deletions

File tree

‎source/_static/UWPCE_logo_W.png‎

9.49 KB
Loading

‎source/_static/UWPCE_logo_full.png‎

24.9 KB
Loading

‎source/exercises/trigrams/test_trigrams.py‎

Lines changed: 79 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,14 @@
2020
import random
2121
import trigrams
2222

23-
IWISH = words = "I wish I may I wish I might".split()
23+
IWISH = "I wish I may I wish I might".split()
24+
25+
LONGER_TEXT = """I was seized with a keen desire to see Holmes
26+
again and to know how he was employing his extraordinary powers
27+
His rooms were brilliantly lit and even as I looked up I saw
28+
his tall spare figure pass twice in a dark silhouette against
29+
the blind""".split()
30+
2431

2532
def test_trigrams_pairs():
2633
"""
@@ -59,16 +66,85 @@ def test_trigrams_following_words():
5966

6067
def test_pick_random_pair():
6168
test_pairs = {("one", "two"): [],
62-
("one", "three"): [],
69+
("two", "three"): [],
6370
("four", "five"): [],
6471
("six", "seven"): [],
6572
("eight", "nine"): [],
6673
}
6774
# set the seed so we'll always get the same one
6875
random.seed(1234)
6976
pair = trigrams.pick_random_pair(test_pairs)
70-
77+
print("the pair is:", pair)
7178
assert pair == ('six', 'seven')
7279

7380

81+
def test_get_last_pair():
82+
words = ["this", "that", "the", "other"]
83+
84+
assert trigrams.get_last_pair(words) == ("the", "other")
85+
86+
87+
def test_get_random_follower():
88+
"""
89+
test getting a random word from the trigrams dict
90+
"""
91+
# we only need one entry for this test
92+
tri_dict = {("one", "two"): ["four", "five", "six", "seven"]}
93+
94+
# set the seed so the answer will be consistent
95+
random.seed(1234)
96+
word = trigrams.get_random_follower(tri_dict, ("one", "two"))
97+
print("got word:", word)
98+
assert word == "seven"
99+
100+
101+
def test_get_random_follower_not_there():
102+
"""
103+
test what happens when the word pair is not there
104+
"""
105+
# we only need one entry for this test
106+
tri_dict = {("one", "two"): ["four", "five", "six", "seven"]}
107+
108+
# here's a word pair that isn't there
109+
# make sure you get something back!
110+
word = trigrams.get_random_follower(tri_dict, ("one", "one"))
111+
print("got word:", word)
112+
assert word # this asserts that you got a non-empty string
113+
114+
115+
def test_make_sentence():
116+
"""
117+
test making a trigrams sentence
118+
119+
as it is supposed to be random, this tests for things other than
120+
the actual results.
121+
122+
NOTE that this test relies on the build_trigram() function, so it
123+
will fail if that doesn't work.
124+
"""
125+
126+
# reset the seed, sop that we won't always get the same answer
127+
random.seed()
128+
129+
# use the already tested build_trigram function to make the dict
130+
tri_dict = trigrams.build_trigram(LONGER_TEXT)
131+
132+
133+
# make a sentence of 6 words
134+
sentence = trigrams.make_sentence(tri_dict, 6)
135+
136+
print(sentence)
137+
# check that it has 6 words
138+
assert len(sentence.split()) == 6
139+
# check that the first letter is a capital
140+
assert sentence[0] == sentence[0].upper()
141+
# check that it ends with a period
142+
assert sentence[-1] == "."
143+
# check that there is not a space between the period and the last word.
144+
assert not sentence[-2].isspace()
145+
146+
147+
148+
149+
74150

‎source/exercises/trigrams/trigrams.rst‎

Lines changed: 108 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -412,13 +412,16 @@ This is the fun part. Once you have a mapping of word pairs to following words,
412412
# pick a random item from a sequence
413413
random.choice(a_list)
414414
415-
This is all pretty tricky to test -- after all, you are selecting random words -- you can't know what the result should be! There are two tactics you can take here. We show a bit of both in the provided tests.
415+
This is all pretty tricky to test -- after all, you are selecting random words -- you can't know what the result should be! There are two tactics you can take here.
416416
417417
Tactic one is to break you code down into pieces that you *can* test -- everything BUT the random choices.
418418
419-
Tactic two is to take advantage of the random "`seed <https://en.wikipedia.org/wiki/Random_seed>`_". Computers don't really make truly random numbers. What they do is compute a sequence of numbers that are statistically very much like random numbers. But if you start with the same initial value, known as the "seed", then you will get the same sequence of numbers. We can take advantage of this in our tests, as the built in ``random`` module provides a way to `set the seed <https://docs.python.org/3/library/random.html#random.seed>`_
419+
Tactic two is to set the random seed before each test, to assure the same result.
420+
The built in ``random`` module `provides a way to set the seed <https://docs.python.org/3/library/random.html#random.seed>`_: the ``random.seed()`` function.
420421
421-
The provided tests use a both of these tactics.
422+
.. note:: Computers don't really make truly random numbers. What they do is compute a sequence of numbers that are statistically very much like random numbers, known as `"pseudo random numbers" <https://en.wikipedia.org/wiki/Pseudorandom_number_generator>`_. If you start with the same initial value, known as the "seed", then you will get the same sequence of numbers. `Random seed <https://en.wikipedia.org/wiki/Random_seed>`_
423+
424+
The provided tests use both of these tactics.
422425
423426
- You need to start with the first word pair; picking a random key from a dict is actually a bit tricky. But we have a test for it:
424427
@@ -434,35 +437,116 @@ The provided tests use a both of these tactics.
434437
# set the seed so we'll always get the same one
435438
random.seed(1234)
436439
pair = trigrams.pick_random_pair(test_pairs)
437-
440+
print("the pair is:", pair)
438441
assert pair == ('six', 'seven')
439442
440443
So you'll need to define a function: ``pick_random_pair()`` that takes your trigram dict as input, and returns a random key.
441444
442-
Note that that particular result is using a particular algorithm -- if you use a different one, you might get a different pair -- but you should get the same one every time, so you can make the test check for that.
445+
Note that the particular result in the test is using a particular algorithm -- if you use a different one, you might get a different pair -- but you should get the same one every time, so you can make the test check for that.
446+
447+
Once you've got the first starting pair, you'll need to make your text,
448+
so you'll need a data structure to build it up in. You probably want to build it up in a list, appending one word at a time. You can join it together at the end with ``" ".join(the_list_of_words)``, which will make a string, separating the words with a space.
449+
450+
Remember that after adding a word to a pair to make a three-word text, the next pair is the last two words in that three-word text.
451+
452+
Here is a test for that step:
453+
454+
.. code-block:: python
455+
456+
def test_get_last_pair():
457+
words = ["this", "that", "the", "other"]
458+
459+
assert trigrams.get_last_pair(words) == ("the", "other")
460+
461+
write a function: ``get_last_pair()`` that takes a list of words, and returns the last two words as a tuple.
462+
463+
Then you'll need to pick a random word from the "followers" -- the words that followed that pair of words in the original text. There is a test for that, too:
464+
465+
.. code-block:: python
466+
467+
def test_get_random_follower():
468+
"""
469+
test getting a random word from the trigrams dict
470+
"""
471+
# we only need one entry for this test
472+
tri_dict = {("one", "two"): ["four", "five", "six", "seven"]}
473+
474+
# set the seed so the answer will be consistent
475+
random.seed(1234)
476+
word = trigrams.get_random_follower(tri_dict, ("one", "two"))
477+
print("got word:", word)
478+
assert word == "seven"
479+
480+
Again, this sets the random seed so that you will always get the same answer. If your code returns a different word -- change the test to match.
481+
482+
But what if the word pair is not in the dict? It's not that likely in a long text, but it can happen. So make sure that your code handles that situation by making sure it passes this test:
483+
484+
.. code-block:: python
485+
486+
def test_get_random_follower_not_there():
487+
"""
488+
test what happens when the word pair is not there
489+
"""
490+
# we only need one entry for this test
491+
tri_dict = {("one", "two"): ["four", "five", "six", "seven"]}
492+
493+
# here's a word pair that isn't there
494+
# make sure you get something back!
495+
word = trigrams.get_random_follower(tri_dict, ("one", "one"))
496+
print("got word:", word)
497+
assert word # this asserts that you got a non-empty string
443498
499+
Note that there are a number of options here as to what to do -- but make sure it returns *something*.
444500
445-
- As you build up your text, you probably want to build it up in a list, appending one word at a time. You can join it together at the end with ``" ".join(the_list_of_words)``
501+
Putting it Together
502+
...................
446503
447-
- Remember that after adding a word to a pair to make a three-word text, the next pair is the last two words in that three-word text.
504+
You now have the pieces you need to make some new text. Let's write a function that will make a single sentence a specified number of words long. The first word should be capitalized, and it should end with a period. Here is the test for that function:
448505
449-
- What to do if you end up with a word pair that isn't in the original text? It's unlikely on a long text, but possible.
506+
.. code-block:: python
507+
508+
def test_make_sentence():
509+
"""
510+
test making a trigrams sentence
511+
512+
as it is supposed to be random, this tests for things other than
513+
the actual results.
514+
515+
NOTE that this test relies on the build_trigram() function, so it
516+
will fail if that doesn't work.
517+
"""
518+
# use the already tested build_trigram function to make the dict
519+
tri_dict = trigrams.build_trigram(LONGER_TEXT)
450520
451-
- How to terminate? Probably have a pre-defined length of text!
452521
522+
# make a sentence of 6 words
523+
sentence = trigrams.make_sentence(tri_dict, 6)
453524
525+
print(sentence)
526+
# check that it has 6 words
527+
assert len(sentence.split()) == 6
528+
# check that the first letter is a capital
529+
assert sentence[0] == sentence[0].upper()
530+
# check that it ends with a period
531+
assert sentence[-1] == "."
532+
# check that there is not a space between the period and the last word.
533+
assert not sentence[-2].isspace()
454534
455-
Once you have the basics working, try your code on a longer piece of input text. Then think about making it fancy. Can you make sentences with capitalized first words and punctuation? Anything else to make the text more "real"?
535+
You can now use the previous functions to make a ``make_sentence()`` function that passes these tests.
536+
537+
538+
Once you have the basics working, try your code on a longer piece of input text. Then think about making it fancy: put a number of sentences of random length to form a paragraph? Add in some other random punctuation? Anything else to make the text more "real"?
456539
457540
458541
Processing the Input Text
459542
-------------------------
460543
461544
If you get a book from Project Gutenberg (or anywhere else), it will not be "clean." That is, it will have header information, footer information, chapter headings, punctuation, what have you. So you'll need to clean it up somehow to get a simple list of words to use to build your trigrams.
462545
463-
The first part of the process is pretty straightforward; open the file and loop through the lines of text.
546+
The first part of the process is pretty straightforward; open the file and loop through the lines of text and process them.
464547
465548
You may want to skip the header. How would you do that??
549+
466550
Hint: in a Project Gutenberg e-book, there is a line of text that starts with::
467551
468552
*** START OF THIS PROJECT GUTENBERG EBOOK
@@ -474,27 +558,36 @@ In the loop, you can process a single line of text to break it into words:
474558
Optional steps to cleaning up the text:
475559
476560
- Strip out punctuation?
477-
- If you do this, what about contractions, i.e. the appostrophe in "can't" vs. a single quotation mark -- which are the same character.
561+
- If you do this, what about contractions, i.e. the apostrophe in "can't" vs. a single quotation mark -- which are the same character.
478562
479563
- Remove capitalization?
480564
- If you do this, what about "I"? And proper nouns?
481565
482566
Any other ideas you may have.
483567
568+
Be sure to use TDD as you develop the "clean up" code: write a test for one feature, and then make sure your code passes that test.
569+
570+
Lather, rinse and repeat.
571+
484572
**Hints:**
485573
486574
The ``string`` methods are your friend here.
487575
488576
There are also handy constants in the ``string`` module: ``import string``
577+
(https://docs.python.org/3/library/string.html)
489578
490579
Check out the ``str.translate()`` method; it can make multiple replacements very fast.
491580
492581
Do get the full trigrams code working first, then play with some of the fancier options.
493582
583+
494584
Code Structure
495585
--------------
496586
497-
Break your code down into a handful of separate functions. This way you can test each on its own, and it's easier to refactor one part without messing with the others. For instance, your ``__main__`` block might look something like:
587+
You will have found that following TDD forces you to break your code down into a handful of separate functions, each of which does only one thing. This lets you test each function on its own, and it's easier to refactor one part without messing with the others. Then you can put them all together into a simple program.
588+
589+
590+
For instance, your ``__main__`` block might look something like:
498591
499592
.. code-block:: python
500593
@@ -512,3 +605,5 @@ Break your code down into a handful of separate functions. This way you can test
512605
new_text = build_text(word_pairs)
513606
514607
print(new_text)
608+
609+

source/solutions/Lesson04/arbitrary_key.py renamed to source/solutions/trigrams/arbitrary_key.py

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
dict.popitem() gets you an arbitrary item, but it removes it also.
1010
In this case, we didn't want to remove it.
1111
12-
In this case, we only needed and arbitrary key, but the principle is
12+
In this case, we only needed an arbitrary key, but the principle is
1313
the same if you want the whole item.
1414
"""
1515

@@ -135,14 +135,28 @@
135135

136136
r = random.randint(0, len(tiny) - 1)
137137

138-
print("random number")
138+
print("random number:", r)
139139

140140
# now loop through the keys until you get to that random number:
141141
for i, key in enumerate(tiny.keys()):
142142
if i >= r:
143143
break
144144
print("A random key:", key)
145145

146-
# this is a bit better than crating the whole list and calling choice() on it,
146+
# this is a bit better than creating the whole list and calling choice() on it,
147147
# as it will, on average, only use half the keys
148148

149+
# you can do a similar thing with the iteration protocol:
150+
r = random.randint(0, len(tiny) - 1)
151+
print("random number:", r)
152+
it = iter(dict.keys())
153+
for _ in range(r):
154+
key = next(it)
155+
156+
print("A random key:", key)
157+
158+
159+
160+
161+
162+

source/solutions/Lesson04/sherlock.txt renamed to source/solutions/trigrams/sherlock.txt

File renamed without changes.

source/solutions/Lesson04/sherlock_small.txt renamed to source/solutions/trigrams/sherlock_small.txt

File renamed without changes.

0 commit comments

Comments
 (0)