Session 1 lab: Python, Colab and always plotting your data¶

price = 120
quantity = 3
total = price * quantity
total

360

discount = 0.10
print("Total before discount:", total)
print("Total after discount:", total * (1 - discount))

Total before discount: 360
Total after discount: 324.0

segment = "Consumer"
monthly_sales = [120, 95, 140, 180]
print(segment, "sales in four months:", monthly_sales)
print("Total:", sum(monthly_sales))
print("Number of months:", len(monthly_sales))

Consumer sales in four months: [120, 95, 140, 180]
Total: 535
Number of months: 4

# Change the numbers in the list, then run the cell again.
monthly_sales = [120, 95, 140, 180]
print("Average monthly sales:", sum(monthly_sales) / len(monthly_sales))

Average monthly sales: 133.75

import pandas as pd

orders = pd.DataFrame({
    "order_id": ["A1", "A2", "A3", "A4", "A5"],
    "region": ["North", "South", "North", "East", "South"],
    "sales": [250.0, 120.0, 90.0, 310.0, 180.0],
    "items": [3, 1, 2, 4, 2],
})
orders

print("Rows and columns:", orders.shape)
print(orders.dtypes)

Rows and columns: (5, 4)
order_id     object
region       object
sales       float64
items         int64
dtype: object

orders["sales"]

0    250.0
1    120.0
2     90.0
3    310.0
4    180.0
Name: sales, dtype: float64

print("Total sales:", orders["sales"].sum())
print("Average sales per order:", orders["sales"].mean())

Total sales: 950.0
Average sales per order: 190.0

north = orders[orders["region"] == "North"]
north

print("Number of orders:", len(orders))
print("Number of different regions:", orders["region"].nunique())

Number of orders: 5
Number of different regions: 3

# Change "North" to "South" and run again. How many South orders are there,
# and what are their total sales?
selected = orders[orders["region"] == "North"]
print(len(selected), "orders, total sales", selected["sales"].sum())

2 orders, total sales 340.0

sales_by_region = orders.groupby("region")["sales"].sum()
sales_by_region

region
East     310.0
North    340.0
South    300.0
Name: sales, dtype: float64

orders.groupby("region").agg(
    orders=("order_id", "count"),
    total_sales=("sales", "sum"),
    average_sales=("sales", "mean"),
)

import matplotlib.pyplot as plt

sales_by_region.sort_values().plot(kind="barh", title="Sales by region (made-up data)")
plt.xlabel("Sales")
plt.ylabel("")
plt.show()

# Change "sales" to "items" to chart the number of items per region instead.
# Then change the title to match.
orders.groupby("region")["sales"].sum().sort_values().plot(kind="barh", title="Sales by region")
plt.show()

plt.scatter(orders["items"], orders["sales"])
plt.title("Sales against items, one dot per order (made-up data)")
plt.xlabel("Items")
plt.ylabel("Sales")
plt.show()

import pandas as pd

# Anscombe's quartet (1973): four small datasets, eleven points each.
x = [10, 8, 13, 9, 11, 14, 6, 4, 12, 7, 5]
anscombe = pd.concat([
    pd.DataFrame({"set": "I",   "x": x, "y": [8.04, 6.95, 7.58, 8.81, 8.33, 9.96, 7.24, 4.26, 10.84, 4.82, 5.68]}),
    pd.DataFrame({"set": "II",  "x": x, "y": [9.14, 8.14, 8.74, 8.77, 9.26, 8.10, 6.13, 3.10, 9.13, 7.26, 4.74]}),
    pd.DataFrame({"set": "III", "x": x, "y": [7.46, 6.77, 12.74, 7.11, 7.81, 8.84, 6.08, 5.39, 8.15, 6.42, 5.73]}),
    pd.DataFrame({"set": "IV",  "x": [8, 8, 8, 8, 8, 8, 8, 19, 8, 8, 8],
                  "y": [6.58, 5.76, 7.71, 8.84, 8.47, 7.04, 5.25, 12.50, 5.56, 7.91, 6.89]}),
], ignore_index=True)

print("Rows, columns:", anscombe.shape)
anscombe.head()

Rows, columns: (44, 3)

# The code computes ONE statistic for each set: the mean of x.
# 1. Add a line for the mean of y:            mean_y=("y", "mean"),
# 2. Add the standard deviation of x and y:   sd_x=("x", "std"),  sd_y=("y", "std"),
# 3. Run it. Which statistics are the same in all four sets?
anscombe.groupby("set").agg(
    mean_x=("x", "mean"),
).round(2)

anscombe.groupby("set")[["x", "y"]].apply(lambda d: d["x"].corr(d["y"])).round(3)

set
I      0.816
II     0.816
III    0.816
IV     0.817
dtype: float64

# 1. Change "I" to "II", "III" and "IV" in turn, and look at each picture.
# 2. Change the title so it names the set you are showing.
one_set = anscombe[anscombe["set"] == "I"]
plt.scatter(one_set["x"], one_set["y"])
plt.title("Anscombe set I")
plt.xlabel("x")
plt.ylabel("y")
plt.xlim(2, 20)
plt.ylim(2, 14)
plt.show()

fig, axes = plt.subplots(2, 2, figsize=(8, 6.5), sharex=True, sharey=True)
for ax, (name, points) in zip(axes.flat, anscombe.groupby("set")):
    ax.scatter(points["x"], points["y"])
    ax.set_title(f"Set {name}")
fig.suptitle("Anscombe's quartet", x=0.01, ha="left", fontweight="bold")
fig.tight_layout()
plt.show()

# Replace each text in quotes with your own words, then run the cell.
my_question = {
    "Decision":   "Which branch gets the extra staff member next quarter",
    "Population": "All customer visits recorded at our five branches",
    "Period":     "January to June this year",
    "Measure":    "Average visits per staff member per day, by branch",
}
pd.Series(my_question, name="My analytical question")

Decision      Which branch gets the extra staff member next ...
Population    All customer visits recorded at our five branches
Period                                January to June this year
Measure       Average visits per staff member per day, by br...
Name: My analytical question, dtype: object

import numpy as np

# 1. Run the cell: it prints the best-fit line for every set.
# 2. Then remove the # from the two plotting lines to draw each line on its panel.
fig, axes = plt.subplots(2, 2, figsize=(8, 6.5), sharex=True, sharey=True)
line_x = np.array([2, 20])
for ax, (name, points) in zip(axes.flat, anscombe.groupby("set")):
    slope, intercept = np.polyfit(points["x"], points["y"], 1)
    print(f"Set {name:>3}: y = {intercept:.2f} + {slope:.3f} x")
    ax.scatter(points["x"], points["y"])
    # ax.plot(line_x, intercept + slope * line_x, color="#d95f02")
    # ax.set_title(f"Set {name}: y = {intercept:.2f} + {slope:.2f} x")
fig.tight_layout()
plt.show()

Set   I: y = 3.00 + 0.500 x
Set  II: y = 3.00 + 0.500 x
Set III: y = 3.00 + 0.500 x
Set  IV: y = 3.00 + 0.500 x

# 1. Run the cell: it compares set III with and without its largest y value.
# 2. Change "III" to "IV" and run again. What happens, and why?
chosen = anscombe[anscombe["set"] == "III"]
without_top = chosen[chosen["y"] != chosen["y"].max()]

print("Correlation, all points:      ", round(chosen["x"].corr(chosen["y"]), 3))
print("Correlation, top point removed:", round(without_top["x"].corr(without_top["y"]), 3))

Correlation, all points:       0.816
Correlation, top point removed: 1.0

import hashlib
import io
import urllib.request

import pandas as pd

# The Datasaurus Dozen (Matejka and Fitzmaurice, 2017), from the open-source
# datasauRus package (MIT licence). The link points to one fixed version, and the
# fingerprint check confirms we got exactly that file.
DATASAURUS_URL = ("https://raw.githubusercontent.com/jumpingrivers/datasauRus/"
                  "94f3c3a82660e256b84f70961f50dc0e216757de/inst/extdata/DatasaurusDozen-Long.tsv")
DATASAURUS_SHA256 = "c8c4792c3ed15238ba07dad6ab778adc79d72e3e1324ead4d8c1ae52849780f8"

file_bytes = urllib.request.urlopen(DATASAURUS_URL).read()
assert hashlib.sha256(file_bytes).hexdigest() == DATASAURUS_SHA256, "Unexpected file: ask your instructor."
dozen = pd.read_csv(io.BytesIO(file_bytes), sep="\t")

print("Rows, columns:", dozen.shape)
print("Datasets:", dozen["dataset"].nunique())
dozen.head()

Rows, columns: (1846, 3)
Datasets: 13

# 1. Run the cell: it shows the mean of x and y for every dataset.
# 2. Add the standard deviations:  sd_x=("x", "std"), sd_y=("y", "std"),
# 3. Then try other statistics:    median_x=("x", "median"), min_y=("y", "min"), max_y=("y", "max"),
# 4. Look for a statistic whose values clearly differ between datasets.
dozen.groupby("dataset").agg(
    mean_x=("x", "mean"),
    mean_y=("y", "mean"),
).round(2)

import matplotlib.pyplot as plt

names = dozen["dataset"].unique()
fig, axes = plt.subplots(3, 5, figsize=(12, 7), sharex=True, sharey=True)
for ax in axes.flat:
    ax.set_visible(False)
for ax, name in zip(axes.flat, names):
    points = dozen[dozen["dataset"] == name]
    ax.set_visible(True)
    ax.scatter(points["x"], points["y"], s=8)
    ax.set_title(name.replace("_", " "))
fig.suptitle("The Datasaurus Dozen: one scatter plot per dataset", x=0.01, ha="left", fontweight="bold")
fig.tight_layout()
plt.show()

# Space for your Challenge work. Example to start from:
dozen[dozen["dataset"] == "dino"].describe().round(2)