Update app.py

app.py

@@ -1,51 +1,58 @@
-import streamlit as st
+# Import required libraries
 import nltk
-from nltk.tokenize import word_tokenize, NLTKWordTokenizer, regexp_tokenize, TweetTokenizer
+from nltk.tokenize import word_tokenize, sent_tokenize, regexp_tokenize, TweetTokenizer
 from nltk.tokenize.treebank import TreebankWordDetokenizer
+from nltk.tokenize import NLTKWordTokenizer
 
-# Regression Tests: NLTKWordTokenizer
-
-st.write('Regression Tests: NLTKWordTokenizer\nTokenizing some test strings.')
-
+# Test cases for NLTKWordTokenizer
 s1 = "On a $50,000 mortgage of 30 years at 8 percent, the monthly payment would be $366.88."
-st.write(f"Tokenized: {word_tokenize(s1)}")
+print(word_tokenize(s1))
 
 s2 = "\"We beat some pretty good teams to get here,\" Slocum said."
-st.write(f"Tokenized: {word_tokenize(s2)}")
+print(word_tokenize(s2))
 
 s3 = "Well, we couldn't have this predictable, cliche-ridden, \"Touched by an Angel\" (a show creator John Masius worked on) wanna-be if she didn't."
-st.write(f"Tokenized: {word_tokenize(s3)}")
+print(word_tokenize(s3))
 
 s4 = "I cannot cannot work under these conditions!"
-st.write(f"Tokenized: {word_tokenize(s4)}")
+print(word_tokenize(s4))
 
 s5 = "The company spent $30,000,000 last year."
-st.write(f"Tokenized: {word_tokenize(s5)}")
+print(word_tokenize(s5))
 
 s6 = "The company spent 40.75% of its income last year."
-st.write(f"Tokenized: {word_tokenize(s6)}")
+print(word_tokenize(s6))
 
 s7 = "He arrived at 3:00 pm."
-st.write(f"Tokenized: {word_tokenize(s7)}")
+print(word_tokenize(s7))
 
 s8 = "I bought these items: books, pencils, and pens."
-st.write(f"Tokenized: {word_tokenize(s8)}")
+print(word_tokenize(s8))
 
 s9 = "Though there were 150, 100 of them were old."
-st.write(f"Tokenized: {word_tokenize(s9)}")
+print(word_tokenize(s9))
 
 s10 = "There were 300,000, but that wasn't enough."
-st.write(f"Tokenized: {word_tokenize(s10)}")
+print(word_tokenize(s10))
 
 s11 = "It's more'n enough."
-st.write(f"Tokenized: {word_tokenize(s11)}")
+print(word_tokenize(s11))
 
 s = '''Good muffins cost $3.88\nin New (York).  Please (buy) me\ntwo of them.\n(Thanks).'''
-expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23), (24, 26), (27, 30), (31, 32), (32, 36), (36, 37), (37, 38), (40, 46), (47, 48), (48, 51), (51, 52), (53, 55), (56, 59), (60, 62), (63, 68), (69, 70), (70, 76), (76, 77), (77, 78)]
-st.write(f"Gathering the spans of the tokenized strings: {list(NLTKWordTokenizer().span_tokenize(s)) == expected}")
+expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23),
+(24, 26), (27, 30), (31, 32), (32, 36), (36, 37), (37, 38),
+(40, 46), (47, 48), (48, 51), (51, 52), (53, 55), (56, 59),
+(60, 62), (63, 68), (69, 70), (70, 76), (76, 77), (77, 78)]
+print(list(NLTKWordTokenizer().span_tokenize(s)) == expected)
 
-expected = ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', '(', 'York', ')', '.', 'Please', '(', 'buy', ')', 'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
-st.write(f"Gathering the spans of the tokenized strings: {[s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected}")
+expected = ['Good', 'muffins', 'cost', '$', '3.88', 'in',
+'New', '(', 'York', ')', '.', 'Please', '(', 'buy', ')',
+'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
+print([s[start:end] for start, end in NLTKWordTokenizer().span_tokenize(s)] == expected)
 
 sx1 = '\xabNow that I can do.\xbb'
-expected = ['\xab', 'Now', 'that
+expected = ['\xab', 'Now', 'that', 'I', 'can', 'do', '.', '\xbb']
+print(word_tokenize(sx1) == expected)
+
+sx2 = 'The unicode 201C and 201D \u201cLEFT(RIGHT) DOUBLE QUOTATION MARK\u201d is also OPEN_PUNCT and CLOSE_PUNCT.'
+expected = ['The', 'unicode', '201C', 'and', '201D', '\u201c', 'LEFT', '(', 'RIGHT', ')', 'DOUBLE', 'QUOTATION', 'MARK', '\u201d', 'is', 'also', 'OPEN_PUNCT