Day 3 lecture: Text mining¶

import zipfile
from pathlib import Path

import nltk

NLTK_DIR = Path("data/nltk_data").resolve()
if not (NLTK_DIR / "tokenizers" / "punkt_tab").exists():
    with zipfile.ZipFile(NLTK_DIR / "tokenizers" / "punkt_tab.zip") as z:
        z.extractall(NLTK_DIR / "tokenizers")
nltk.data.path.insert(0, str(NLTK_DIR))

from collections import Counter

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from nltk.corpus import stopwords
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.tokenize import word_tokenize
from wordcloud import WordCloud

BLUE, ORANGE, AQUA = "#2a78d6", "#eb6834", "#1baf7a"
GREY = "#c9c7c1"   # context
INK = "#55534e"    # notes and labels
plt.rcParams.update({
    "figure.figsize": (8, 4),
    "axes.spines.top": False,
    "axes.spines.right": False,
    "axes.titleweight": "bold",
    "axes.titlelocation": "left",
})
pd.set_option("display.max_colwidth", 90)

# SYNTHETIC reviews written for teaching: (branch, stars 1-5, review text)
REVIEWS = [
    ("Riverside", 5, "The brown sugar milk tea is delicious and the pearls are perfectly chewy. My new favourite shop!"),
    ("Riverside", 2, "Waited 35 minutes for one drink. The queue was huge and nobody explained the delay."),
    ("Riverside", 4, "Tasty mango tea and friendly staff, but the wait was a bit long at lunchtime."),
    ("Riverside", 1, "The app crashed twice while I was paying, and my order was lost. Very frustrating."),
    ("Riverside", 5, "Great taste, fair price, quick service. Highly recommend the jasmine green tea."),
    ("Riverside", 3, "The drink was okay. Not bad, not special."),
    ("Riverside", 2, "Too sweet even at 30% sugar, and the ice melted before I got back to the office."),
    ("Riverside", 1, "Great, another 40 minute wait for a lukewarm drink."),
    ("Riverside", 4, "Love the new oat milk option. A little expensive, but worth it."),
    ("Riverside", 2, "The app showed my order as ready but the staff could not find it. I waited another 20 minutes."),
    ("Campus", 5, "Cheap, fast and delicious. Perfect between classes!"),
    ("Campus", 4, "Good tea and friendly staff. The queue moves fast."),
    ("Campus", 3, "Price is fine. The taste is average."),
    ("Campus", 5, "Best taro milk tea in the city. The staff are always kind."),
    ("Campus", 2, "The app would not accept my student discount code, so I paid full price."),
    ("Campus", 4, "Nice cheese foam, quick pickup with the app. Seats are limited."),
    ("Campus", 1, "Found a hair in my drink. Disgusting. I complained and nobody cared."),
    ("Campus", 5, "The app is easy to use and my drink was ready before I reached the counter. Excellent."),
    ("Campus", 3, "Not the worst tea I have had, but the pearls were hard."),
    ("Campus", 4, "Delicious peach oolong. The music is too loud, though."),
    ("Station", 2, "Slow service. I missed my train because the order took 25 minutes."),
    ("Station", 1, "Terrible. Cold drink, rude cashier and a long wait."),
    ("Station", 2, "Convenient location, but the wait times are unpredictable."),
    ("Station", 4, "Good drinks and the app lets me order before I arrive. The wait is shorter that way."),
    ("Station", 2, "The app kept logging me out, and the queue was very long."),
    ("Station", 5, "Friendly staff and the best brown sugar tea. Worth the wait!"),
    ("Station", 1, "Overpriced and watery. Never again."),
    ("Station", 3, "The drink was fine but the app charged me twice. Refund took a week."),
    ("Station", 4, "Tasty and fresh. The staff were busy but polite."),
    ("Station", 2, "Waited 30 minutes at the counter while app orders kept jumping the queue."),
]
reviews = pd.DataFrame(REVIEWS, columns=["branch", "stars", "text"])
reviews.index = pd.RangeIndex(1, len(reviews) + 1, name="review_id")

print("Reviews:", len(reviews))
print(reviews["branch"].value_counts().sort_index().to_string())
reviews.head()

Reviews: 30
branch
Campus       10
Riverside    10
Station      10

example = reviews.loc[6, "text"]

by_spaces = example.split()
by_nltk = word_tokenize(example.lower())

print("split() on spaces:", len(by_spaces), "tokens")
print(by_spaces)
print()
print("word_tokenize on lowercase text:", len(by_nltk), "tokens")
print(by_nltk)

split() on spaces: 8 tokens
['The', 'drink', 'was', 'okay.', 'Not', 'bad,', 'not', 'special.']

word_tokenize on lowercase text: 11 tokens
['the', 'drink', 'was', 'okay', '.', 'not', 'bad', ',', 'not', 'special', '.']

reviews["tokens"] = reviews["text"].str.lower().apply(word_tokenize)
reviews["words"] = reviews["tokens"].apply(lambda tokens: [t for t in tokens if t.isalpha()])

all_words = [w for words in reviews["words"] for w in words]
print("Alphabetic tokens in all reviews:", len(all_words))
print("Different words (the vocabulary):", len(set(all_words)))
print()
print("Ten most common words so far:")
print(Counter(all_words).most_common(10))

Alphabetic tokens in all reviews: 353
Different words (the vocabulary): 172

Ten most common words so far:
[('the', 37), ('and', 15), ('app', 9), ('i', 9), ('was', 8), ('tea', 7), ('my', 7), ('drink', 7), ('but', 7), ('is', 6)]

english_stopwords = set(stopwords.words("english"))
print("Stopwords in NLTK's English list:", len(english_stopwords))
print("A few of them:", sorted(english_stopwords)[:25])
print()
for word in ["not", "no", "very", "but", "too"]:
    print(f"Is '{word}' a stopword? {word in english_stopwords}")

Stopwords in NLTK's English list: 198
A few of them: ['a', 'about', 'above', 'after', 'again', 'against', 'ain', 'all', 'am', 'an', 'and', 'any', 'are', 'aren', "aren't", 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both']

Is 'not' a stopword? True
Is 'no' a stopword? True
Is 'very' a stopword? True
Is 'but' a stopword? True
Is 'too' a stopword? True

DOMAIN_STOPWORDS = {"tea", "drink", "drinks", "milk"}
stop = english_stopwords | DOMAIN_STOPWORDS

reviews["content_words"] = reviews["words"].apply(lambda words: [w for w in words if w not in stop])
content_words = [w for words in reviews["content_words"] for w in words]

print("Words before removing stopwords:", len(all_words))
print("Words after:", len(content_words))
print(f"Share of words removed: {1 - len(content_words) / len(all_words):.0%}")

Words before removing stopwords: 353
Words after: 196
Share of words removed: 44%

freq = nltk.FreqDist(content_words)
top = pd.Series(dict(freq.most_common(15)), name="count")

fig, ax = plt.subplots(figsize=(8, 5))
ax.barh(top.index[::-1], top.values[::-1], color=BLUE)
for i, value in enumerate(top.values[::-1]):
    ax.text(value + 0.1, i, str(value), va="center", fontsize=9, color=INK)
ax.set_title("What LeafLine customers write about most")
ax.set_xlabel("Number of times the word appears in 30 reviews")
ax.grid(axis="x", color="#e6e5e1")
ax.set_axisbelow(True)
plt.show()

cloud = WordCloud(
    width=800, height=400, background_color="white", colormap="viridis", random_state=3
).generate_from_frequencies(freq)

plt.figure(figsize=(10, 5))
plt.imshow(cloud, interpolation="bilinear")
plt.axis("off")
plt.title("LeafLine reviews as a word cloud (size = frequency)")
plt.show()

pairs = Counter()
for words in reviews["content_words"]:
    pairs.update(nltk.bigrams(words))

print("Most common word pairs (after removing stopwords):")
for (first, second), count in pairs.most_common(8):
    print(f"  {first} {second}: {count}")

Most common word pairs (after removing stopwords):
  friendly staff: 3
  brown sugar: 2
  waited minutes: 2
  sugar delicious: 1
  delicious pearls: 1
  pearls perfectly: 1
  perfectly chewy: 1
  chewy new: 1

analyzer = SentimentIntensityAnalyzer()

for word in ["delicious", "friendly", "good", "okay", "hate", "terrible", "disgusting", "wait", "queue", "watery"]:
    rating = analyzer.lexicon.get(word)
    print(f"{word:<11} {'not in the lexicon' if rating is None else rating}")

delicious   2.7
friendly    2.2
good        1.9
okay        0.9
hate        -2.7
terrible    -2.1
disgusting  -2.4
wait        not in the lexicon
queue       not in the lexicon
watery      not in the lexicon

sentences = [
    "The tea is good.",
    "The tea is very good.",
    "The tea is GOOD!",
    "The tea is not good.",
    "The tea is good, but the staff are rude.",
    "The staff are rude, but the tea is good.",
    "The tea is watery.",
]
rule_demo = pd.DataFrame(
    [analyzer.polarity_scores(s) for s in sentences], index=sentences
)[["neg", "neu", "pos", "compound"]]
rule_demo

def sentiment_label(score):
    if score >= 0.05:
        return "Positive"
    if score <= -0.05:
        return "Negative"
    return "Neutral"


reviews["compound"] = reviews["text"].apply(lambda text: analyzer.polarity_scores(text)["compound"])
reviews["sentiment"] = reviews["compound"].apply(sentiment_label)

LABELS = ["Positive", "Neutral", "Negative"]
counts = reviews["sentiment"].value_counts().reindex(LABELS, fill_value=0)
print(counts.to_string())
print(f"\nAverage compound score: {reviews['compound'].mean():.3f}")
reviews[["branch", "stars", "compound", "sentiment", "text"]].head(8)

sentiment
Positive    19
Neutral      5
Negative     6

Average compound score: 0.254

fig, ax = plt.subplots(figsize=(6, 3.5))
ax.bar(counts.index, counts.values, color=[BLUE, GREY, ORANGE])
for i, value in enumerate(counts.values):
    ax.text(i, value + 0.2, str(value), ha="center", color=INK)
ax.set_title("VADER labels for 30 LeafLine reviews")
ax.set_ylabel("Number of reviews")
plt.show()

rng = np.random.default_rng(12)   # fixed seed: the small sideways jitter is the same every run
jitter = rng.uniform(-0.15, 0.15, len(reviews))

fig, ax = plt.subplots(figsize=(8, 4.5))
ax.axhspan(-0.05, 0.05, color=GREY, alpha=0.4, label="Neutral band (-0.05 to 0.05)")
ax.scatter(reviews["stars"] + jitter, reviews["compound"], s=45, color=BLUE)
ax.set_xticks([1, 2, 3, 4, 5])
ax.set_xlabel("Star rating given by the customer (1 = worst, 5 = best)")
ax.set_ylabel("VADER compound score (-1 to +1)")
ax.set_title("Higher ratings tend to score higher, with clear exceptions")
ax.legend(frameon=False, loc="upper left", fontsize=9)
plt.show()

print("Average compound score by star rating:")
print(reviews.groupby("stars")["compound"].agg(["count", "mean"]).round(3).to_string())
print(f"\nCorrelation between stars and compound score: r = {reviews['stars'].corr(reviews['compound']):.2f}")

Average compound score by star rating:
       count   mean
stars              
1          5 -0.256
2          8  0.008
3          4  0.207
4          7  0.422
5          6  0.844

Correlation between stars and compound score: r = 0.79

unhappy_not_negative = reviews[(reviews["stars"] <= 2) & (reviews["sentiment"] != "Negative")]
print("Reviews with 1-2 stars that VADER does NOT label Negative:", len(unhappy_not_negative))
unhappy_not_negative[["stars", "compound", "sentiment", "text"]]

Reviews with 1-2 stars that VADER does NOT label Negative: 8

ASPECTS = {
    "waiting": {"wait", "waited", "queue", "slow", "minutes", "minute", "long", "delay"},
    "app": {"app"},
    "taste": {"taste", "tasty", "delicious", "sweet", "watery", "pearls", "fresh", "lukewarm"},
    "staff": {"staff", "cashier", "rude", "friendly", "polite", "kind", "nobody"},
    "price": {"price", "cheap", "expensive", "overpriced", "discount", "paid", "charged"},
}

for aspect, keywords in ASPECTS.items():
    reviews[aspect] = reviews["words"].apply(lambda words: bool(keywords & set(words)))

rows = []
for aspect in ASPECTS:
    group = reviews[reviews[aspect]]
    rows.append({
        "aspect": aspect,
        "reviews": len(group),
        "% of all reviews": 100 * len(group) / len(reviews),
        "average stars": group["stars"].mean(),
        "% with 1-2 stars": 100 * (group["stars"] <= 2).mean(),
        "average compound": group["compound"].mean(),
    })
aspects = pd.DataFrame(rows).set_index("aspect").sort_values("% with 1-2 stars", ascending=False)
aspects.round(2)

low_share = aspects["% with 1-2 stars"].sort_values()
worst = low_share.index[-1]

fig, ax = plt.subplots(figsize=(8, 3.8))
ax.barh(low_share.index, low_share.values, color=[ORANGE if a == worst else GREY for a in low_share.index])
for i, (aspect, value) in enumerate(low_share.items()):
    ax.text(value + 1, i, f"{value:.0f}%  (n = {aspects.loc[aspect, 'reviews']})", va="center", fontsize=9, color=INK)
ax.set_xlim(0, 115)
ax.set_title(f"Reviews that mention {worst} are the most often unhappy")
ax.set_xlabel("% of reviews mentioning the aspect that give 1 or 2 stars")
plt.show()

by_branch = reviews.groupby("branch").agg(
    reviews=("stars", "size"),
    mention_worst=(worst, "sum"),
    average_stars=("stars", "mean"),
)
by_branch[f"% mentioning {worst}"] = 100 * by_branch["mention_worst"] / by_branch["reviews"]
by_branch.round(2)