In [1]:
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import random
from sklearn import datasets
import warnings
warnings.filterwarnings('ignore')
In [2]:
iris, _ = datasets.load_iris(return_X_y=True, as_frame=True)
iris.rename(columns={"sepal length (cm)": "sepal_length", "sepal width (cm)": "sepal_width",
"petal length (cm)": "petal_length", "petal width (cm)": "petal_width"}, inplace=True)
iris.sample(10)
Out[2]:
sepal_length | sepal_width | petal_length | petal_width | |
---|---|---|---|---|
88 | 5.6 | 3.0 | 4.1 | 1.3 |
89 | 5.5 | 2.5 | 4.0 | 1.3 |
4 | 5.0 | 3.6 | 1.4 | 0.2 |
139 | 6.9 | 3.1 | 5.4 | 2.1 |
11 | 4.8 | 3.4 | 1.6 | 0.2 |
57 | 4.9 | 2.4 | 3.3 | 1.0 |
41 | 4.5 | 2.3 | 1.3 | 0.3 |
84 | 5.4 | 3.0 | 4.5 | 1.5 |
17 | 5.1 | 3.5 | 1.4 | 0.3 |
109 | 7.2 | 3.6 | 6.1 | 2.5 |
K-Means Clustering¶
In today's lecture, we will use the data from the iris
dataset from sklearn
to perform clustering using two features, petal_length
and petal_width.
Summary of the algorithm:
- Repeat until convergence:
- Color points according to the closest center.
- Move the center for each color to the center of points with that color.
In [3]:
sns.scatterplot(data=iris, x="petal_length", y="petal_width", color="black")
plt.xlabel('x')
plt.ylabel('y');
In [4]:
class Center():
def __init__(self, data):
"""generates a random center inside the region bounded by the data"""
num_dimensions = data.shape[1]
self.coordinates = np.array([0.0] * num_dimensions)
for i in range(num_dimensions):
min_value = np.min(data[:, i])
max_value = np.max(data[:, i])
random_value = random.uniform(min_value, max_value)
self.coordinates[i] = random_value
def __str__(self):
return str(self.coordinates)
def __repr__(self):
return repr(self.coordinates)
def dist(self, data_point):
return np.sqrt(np.sum((self.coordinates - data_point)**2, axis = 1))
def dist_sq(self, data_point):
return np.sum((self.coordinates - data_point)**2, axis = 1)
In [5]:
c1 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c2 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
In [6]:
# Force coordinates from the lecture demo
c1.coordinates = np.array([2.52364007, 2.31040024])
c2.coordinates = np.array([6.53276402, 1.211463])
In [7]:
def plot_centers_and_black_data(iris, centers):
for center in centers:
plt.plot(center.coordinates[0], center.coordinates[1], '*', markersize=10)
sns.scatterplot(data=iris, x="petal_length", y="petal_width", color="black")
plt.xlabel('petal_length')
plt.ylabel('petal_width')
legend_text = ['c' + str(i) for i in range(1, len(centers) + 1)]
legend_text.append('data')
plt.legend(legend_text)
In [8]:
plot_centers_and_black_data(iris, (c1, c2))
In [9]:
def get_cluster_number(dists):
return np.where(dists == np.min(dists))[0][0]
In [10]:
iris["dist1"] = c1.dist(iris[["petal_length", "petal_width"]])
iris["dist2"] = c2.dist(iris[["petal_length", "petal_width"]])
iris["cluster"] = iris[["dist1", "dist2"]].apply(get_cluster_number, axis = 1)
iris.head(5)
Out[10]:
sepal_length | sepal_width | petal_length | petal_width | dist1 | dist2 | cluster | |
---|---|---|---|---|---|---|---|
0 | 5.1 | 3.5 | 1.4 | 0.2 | 2.390890 | 5.231474 | 0 |
1 | 4.9 | 3.0 | 1.4 | 0.2 | 2.390890 | 5.231474 | 0 |
2 | 4.7 | 3.2 | 1.3 | 0.2 | 2.439484 | 5.329623 | 0 |
3 | 4.6 | 3.1 | 1.5 | 0.2 | 2.345555 | 5.133398 | 0 |
4 | 5.0 | 3.6 | 1.4 | 0.2 | 2.390890 | 5.231474 | 0 |
In [11]:
iris["cluster"].value_counts()
Out[11]:
cluster 0 79 1 71 Name: count, dtype: int64
In [12]:
def plot_centers_and_colorized_data(iris, centers):
plt.figure()
for center in centers:
plt.plot(center.coordinates[0], center.coordinates[1],
marker='*', markersize=10, linestyle="None")
current_palette = sns.color_palette()[0:len(centers)]
sns.scatterplot(data=iris, x="petal_length", y="petal_width", hue="cluster", palette=current_palette)
plt.xlabel('petal_length')
plt.ylabel('petal_width')
legend_text = ['c' + str(i) for i in range(1, len(centers) + 1)]
legend_text.append('data')
plt.legend(legend_text)
In [13]:
plot_centers_and_colorized_data(iris, (c1, c2))
In [14]:
average_c1_length = np.mean(iris[iris["cluster"] == 0]["petal_length"])
average_c1_width = np.mean(iris[iris["cluster"] == 0]["petal_width"])
c1.coordinates = (average_c1_length, average_c1_width)
average_c2_length = np.mean(iris[iris["cluster"] == 1]["petal_length"])
average_c2_width = np.mean(iris[iris["cluster"] == 1]["petal_width"])
c2.coordinates = (average_c2_length, average_c2_width)
In [15]:
plot_centers_and_black_data(iris, (c1, c2))
In [16]:
iris["dist1"] = c1.dist(iris[["petal_length", "petal_width"]])
iris["dist2"] = c2.dist(iris[["petal_length", "petal_width"]])
iris["cluster"] = iris[["dist1", "dist2"]].apply(get_cluster_number, axis = 1)
In [17]:
plot_centers_and_colorized_data(iris, (c1, c2))
In [18]:
average_c1_length = np.mean(iris[iris["cluster"] == 0]["petal_length"])
average_c1_width = np.mean(iris[iris["cluster"] == 0]["petal_width"])
c1.coordinates = (average_c1_length, average_c1_width)
average_c2_length = np.mean(iris[iris["cluster"] == 1]["petal_length"])
average_c2_width = np.mean(iris[iris["cluster"] == 1]["petal_width"])
c2.coordinates = (average_c2_length, average_c2_width)
In [19]:
plot_centers_and_black_data(iris, (c1, c2))
In [20]:
iris["dist1"] = c1.dist(iris[["petal_length", "petal_width"]])
iris["dist2"] = c2.dist(iris[["petal_length", "petal_width"]])
iris["cluster"] = iris[["dist1", "dist2"]].apply(get_cluster_number, axis = 1)
In [21]:
plot_centers_and_colorized_data(iris, (c1, c2))
In [22]:
average_c1_length = np.mean(iris[iris["cluster"] == 0]["petal_length"])
average_c1_width = np.mean(iris[iris["cluster"] == 0]["petal_width"])
c1.coordinates = (average_c1_length, average_c1_width)
average_c2_length = np.mean(iris[iris["cluster"] == 1]["petal_length"])
average_c2_width = np.mean(iris[iris["cluster"] == 1]["petal_width"])
c2.coordinates = (average_c2_length, average_c2_width)
In [23]:
plot_centers_and_black_data(iris, (c1, c2))
In [24]:
iris["dist1"] = c1.dist(iris[["petal_length", "petal_width"]])
iris["dist2"] = c2.dist(iris[["petal_length", "petal_width"]])
iris["cluster"] = iris[["dist1", "dist2"]].apply(get_cluster_number, axis = 1)
In [25]:
plot_centers_and_colorized_data(iris, (c1, c2))
In [26]:
average_c1_length = np.mean(iris[iris["cluster"] == 0]["petal_length"])
average_c1_width = np.mean(iris[iris["cluster"] == 0]["petal_width"])
c1.coordinates = (average_c1_length, average_c1_width)
average_c2_length = np.mean(iris[iris["cluster"] == 1]["petal_length"])
average_c2_width = np.mean(iris[iris["cluster"] == 1]["petal_width"])
c2.coordinates = (average_c2_length, average_c2_width)
In [27]:
plot_centers_and_black_data(iris, (c1, c2))
In [28]:
iris["dist1"] = c1.dist(iris[["petal_length", "petal_width"]])
iris["dist2"] = c2.dist(iris[["petal_length", "petal_width"]])
iris["cluster"] = iris[["dist1", "dist2"]].apply(get_cluster_number, axis = 1)
In [29]:
plot_centers_and_colorized_data(iris, (c1, c2))
In [30]:
average_c1_length = np.mean(iris[iris["cluster"] == 0]["petal_length"])
average_c1_width = np.mean(iris[iris["cluster"] == 0]["petal_width"])
c1.coordinates = (average_c1_length, average_c1_width)
average_c2_length = np.mean(iris[iris["cluster"] == 1]["petal_length"])
average_c2_width = np.mean(iris[iris["cluster"] == 1]["petal_width"])
c2.coordinates = (average_c2_length, average_c2_width)
In [31]:
plot_centers_and_black_data(iris, (c1, c2))
In [32]:
average_c1_length = np.mean(iris[iris["cluster"] == 0]["petal_length"])
average_c1_width = np.mean(iris[iris["cluster"] == 0]["petal_width"])
c1.coordinates = (average_c1_length, average_c1_width)
average_c2_length = np.mean(iris[iris["cluster"] == 1]["petal_length"])
average_c2_width = np.mean(iris[iris["cluster"] == 1]["petal_width"])
c2.coordinates = (average_c2_length, average_c2_width)
In [33]:
plot_centers_and_colorized_data(iris, (c1, c2))
Example for K > 2¶
In [34]:
import copy
def compute_centers_after_N_iterations(data, column_names, centers, N):
centers = copy.deepcopy(centers)
for i in range(N):
# Recompute clusters
dist_names = []
for center_num in range(len(centers)):
data["dist" + str(center_num)] = centers[center_num].dist(data[column_names])
dist_names.append("dist" + str(center_num))
data["cluster"] = data[dist_names].apply(get_cluster_number, axis = 1)
# Update centers
for center_num in range(len(centers)):
for col_num in range(len(column_names)):
col_name = column_names[col_num]
centers[center_num].coordinates[col_num] = np.mean(data[data["cluster"] == center_num][col_name])
return centers
In [35]:
c1 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c2 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c1.coordinates = np.array([2.52364007, 2.31040024])
c2.coordinates = np.array([6.53276402, 1.211463])
In [36]:
def inertia(data, centers):
total_inertia = 0
for center_num in range(len(centers)):
data_in_this_cluster = data[data["cluster"] == center_num]
total_inertia += np.sum(centers[center_num].dist(data_in_this_cluster[["petal_length", "petal_width"]]))
return total_inertia
In [37]:
def distortion(data, centers):
total_distortion = 0
for center_num in range(len(centers)):
data_in_this_cluster = data[data["cluster"] == center_num]
total_distortion += np.sum(centers[center_num].dist(data_in_this_cluster[["petal_length", "petal_width"]]))/len(data_in_this_cluster)
return total_distortion
In [38]:
random.seed(25)
c1 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c2 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c3 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c4 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
new_centers = compute_centers_after_N_iterations(iris, ['petal_length', 'petal_width'], [c1, c2, c3, c4], 12)
print(f"inertia: {inertia(iris, new_centers)}, distortion: {distortion(iris, new_centers)})")
plot_centers_and_colorized_data(iris, new_centers)
inertia: 44.88363871576328, distortion: 1.2530251953116613)
In [39]:
random.seed(29)
c1 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c2 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c3 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c4 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
new_centers = compute_centers_after_N_iterations(iris, ['petal_length', 'petal_width'], [c1, c2, c3, c4], 12)
print(f"inertia: {inertia(iris, new_centers)}, distortion: {distortion(iris, new_centers)})")
plot_centers_and_colorized_data(iris, new_centers)
inertia: 45.87509130916156, distortion: 1.3068391699161572)
In [40]:
random.seed(40)
c1 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c2 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c3 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c4 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
new_centers = compute_centers_after_N_iterations(iris, ['petal_length', 'petal_width'], [c1, c2, c3, c4], 12)
print(f"inertia: {inertia(iris, new_centers)}, distortion: {distortion(iris, new_centers)})")
plot_centers_and_colorized_data(iris, new_centers)
inertia: 54.272527867765156, distortion: 1.4992328098338596)
In [41]:
random.seed(75)
c1 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c2 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c3 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
c4 = Center(iris.loc[:, ['petal_length', 'petal_width']].values)
new_centers = compute_centers_after_N_iterations(iris, ['petal_length', 'petal_width'], [c1, c2, c3, c4], 12)
print(f"inertia: {inertia(iris, new_centers)}, distortion: {distortion(iris, new_centers)})")
plot_centers_and_colorized_data(iris, new_centers)
inertia: 44.88363871576328, distortion: 1.2530251953116613)
Example of Inertia Failing to Match Intuition¶
In [42]:
c1.coordinates = [1.2, 0.15]
c2.coordinates = [4.906000000000001, 1.6760000000000006]
iris["dist1"] = c1.dist(iris[["petal_length", "petal_width"]])
iris["dist2"] = c2.dist(iris[["petal_length", "petal_width"]])
iris["cluster"] = iris[["dist1", "dist2"]].apply(get_cluster_number, axis = 1)
In [43]:
plot_centers_and_colorized_data(iris, (c1, c2))
In [44]:
print(f"inertia: {inertia(iris, [c1, c2])}, distortion: {distortion(iris, [c1, c2])})")
inertia: 94.3164648130483, distortion: 1.0959547804008838)
In [45]:
average_c1_length = np.mean(iris[iris["cluster"] == 0]["petal_length"])
average_c1_width = np.mean(iris[iris["cluster"] == 0]["petal_width"])
c1.coordinates = (average_c1_length, average_c1_width)
average_c2_length = np.mean(iris[iris["cluster"] == 1]["petal_length"])
average_c2_width = np.mean(iris[iris["cluster"] == 1]["petal_width"])
c2.coordinates = (average_c2_length, average_c2_width)
In [46]:
plot_centers_and_black_data(iris, (c1, c2))
In [47]:
iris["dist1"] = c1.dist(iris[["petal_length", "petal_width"]])
iris["dist2"] = c2.dist(iris[["petal_length", "petal_width"]])
iris["cluster"] = iris[["dist1", "dist2"]].apply(get_cluster_number, axis = 1)
In [48]:
plot_centers_and_colorized_data(iris, (c1, c2))
In [49]:
print(f"inertia: {inertia(iris, [c1, c2])}, distortion: {distortion(iris, [c1, c2])})")
inertia: 87.2103463131798, distortion: 0.9775403068856574)
Hierarchical Agglomerative Clustering¶
In [50]:
np.random.seed(42)
iris_small = iris.sample(13).loc[:, 'sepal_length':'petal_width'].reset_index(drop=True)
iris_small = iris_small.drop(8).reset_index(drop=True)
In [51]:
sns.scatterplot(data=iris_small, x="petal_length", y="petal_width", color="black");