Create app.py

app.py

@@ -0,0 +1,51 @@
+import streamlit as st
+import nltk
+from nltk.tokenize import word_tokenize, NLTKWordTokenizer, regexp_tokenize, TweetTokenizer
+from nltk.tokenize.treebank import TreebankWordDetokenizer
+
+# Regression Tests: NLTKWordTokenizer
+
+st.write('Regression Tests: NLTKWordTokenizer\nTokenizing some test strings.')
+
+s1 = "On a $50,000 mortgage of 30 years at 8 percent, the monthly payment would be $366.88."
+st.write(f"Tokenized: {word_tokenize(s1)}")
+
+s2 = "\"We beat some pretty good teams to get here,\" Slocum said."
+st.write(f"Tokenized: {word_tokenize(s2)}")
+
+s3 = "Well, we couldn't have this predictable, cliche-ridden, \"Touched by an Angel\" (a show creator John Masius worked on) wanna-be if she didn't."
+st.write(f"Tokenized: {word_tokenize(s3)}")
+
+s4 = "I cannot cannot work under these conditions!"
+st.write(f"Tokenized: {word_tokenize(s4)}")
+
+s5 = "The company spent $30,000,000 last year."
+st.write(f"Tokenized: {word_tokenize(s5)}")
+
+s6 = "The company spent 40.75% of its income last year."
+st.write(f"Tokenized: {word_tokenize(s6)}")
+
+s7 = "He arrived at 3:00 pm."
+st.write(f"Tokenized: {word_tokenize(s7)}")
+
+s8 = "I bought these items: books, pencils, and pens."
+st.write(f"Tokenized: {word_tokenize(s8)}")
+
+s9 = "Though there were 150, 100 of them were old."
+st.write(f"Tokenized: {word_tokenize(s9)}")
+
+s10 = "There were 300,000, but that wasn't enough."
+st.write(f"Tokenized: {word_tokenize(s10)}")
+
+s11 = "It's more'n enough."
+st.write(f"Tokenized: {word_tokenize(s11)}")
+
+s = '''Good muffins cost $3.88\nin New (York).  Please (buy) me\ntwo of them.\n(Thanks).'''
+expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23), (24, 26), (27, 30), (31, 32), (32, 36), (36, 37), (37, 38), (40, 46), (47, 48), (48, 51), (51, 52), (53, 55), (56, 59), (60, 62), (63, 68), (69, 70), (70, 76), (76, 77), (77, 78)]
+st.write(f"Gathering the spans of the tokenized strings: {list(NLTKWordTokenizer().span_tokenize(s)) == expected}")
+
+expected = ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', '(', 'York', ')', '.', 'Please', '(', 'buy', ')', 'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
+st.write(f"Gathering the spans of the tokenized strings: {[s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected}")
+
+sx1 = '\xabNow that I can do.\xbb'
+expected = ['\xab', 'Now', 'that