Missing Value Treatment using Clustering Technique

# Let us load the data
from sklearn import datasets
import numpy as np
# Set a seed to get same clusters
np.random.seed(2)

# Import IRIS data from sklearn.datasets - https://www.kaggle.com/arshid/iris-flower-dataset
iris = datasets.load_iris()
X = iris.data
y = iris.target

# Load the data into a pandas dataframe, so its easy to work with
import pandas as pd
df = pd.DataFrame(data=np.concatenate([X,np.reshape(y, [-1,1])], axis=1), columns=["sl", "sw","pl", "pw", "class"])
print("Data Sample:")
print(df.head()) 
print("\n Variable Description:")
print(df.describe())

from sklearn.cluster import KMeans
# Seed is set to get the same set of clusters on execution.
np.random.seed(2)
km = KMeans(n_clusters=3)
km.fit(df[["sl", "sw", "pl", "pw"]])
df['cluster'] = km.labels_
gp =df.groupby("cluster")

avg_std = []
for g in gp:
    avg_std.append(g[1]['class'].std())
    print(f"Cluster {g[0]} std: {avg_std[-1]}")
print(f"Average Std: {sum(avg_std)/len(avg_std)}")

import random
np.random.seed(2)
random.seed(2)
MASKED_CANDIDATES = [random.randrange(0, len(df)) for i in range(5)]
ACTUAL_VALUES  = np.array([df.loc[[id]]["pl"].tolist()[0] for id in MASKS_CANDIDATES])

print(f"Masked Candidates: {MASKED_CANDIDATES}")
print(f"Actual Values of the mask: {ACTUAL_VALUES}")

%%time
m_df = df.copy()
# Setting mask candidates to None
for cand in MASKS_CANDIDATES:
    m_df.at[cand, 'pl'] = None
km = KMeans(n_clusters=3)
# Ignoring first column, create clusters
km.fit(m_df[["sl","sw", "pw"]].to_numpy())


# Approximate the mising values
print("Approximation:")
approx_vals = []
m_df['cluster'] = km.labels_
gp = m_df.groupby('cluster')
for ma in MASKS_CANDIDATES:
    mc = m_df.loc[[ma]]['cluster']
    approx_vals.append(df['pl'][gp.get_group(mc.tolist()[0]).index].mean())
for i in zip(MASKS_CANDIDATES, ACTUAL_VALUES, approx_vals):
    print(f"Candidate: {i[0]}, Actual PL: {i[1]}, Predicted PL: {i[2]}")
print(f"Euclidean Distance between actual and predictions: {np.linalg.norm(ACTUAL_VALUES-approx_vals)}\n")

print("Cluster Performance:")
avg_std = []
for g in gp:
    avg_std.append(g[1]['class'].std())
    print(f"Cluster {g[0]} std: {avg_std[-1]}")
print(f"Average Std: {sum(avg_std)/len(avg_std)}\n")

%%time
np.random.seed(2)

m_df = df.drop(MASKS_CANDIDATES)

km = KMeans(n_clusters=3)
km.fit(m_df[["sl","sw", "pl","pw"]].to_numpy())
m_df['cluster'] = km.labels_
gp = m_df.groupby('cluster')
print("Approximation:")
approx_vals = []
for cand in MASKS_CANDIDATES:
    dist = [np.linalg.norm(df.loc[[cand], ['sl','sw','pw']].to_numpy() - centroid[[0,1,3]]) for centroid in km.cluster_centers_]
    cluster = np.argmin(dist)
    approx_vals.append(gp.get_group(cluster)["pl"].mean())

for i in zip(MASKS_CANDIDATES, ACTUAL_VALUES, approx_vals):
    print(f"Candidate: {i[0]}, Actual PL: {i[1]}, Predicted PL: {i[2]}")
print(f"Euclidean Distance between actual and predictions: {np.linalg.norm(ACTUAL_VALUES-approx_vals)}\n")

print("Cluster Performance:")
avg_std = []
for g in gp:
    avg_std.append(g[1]['class'].std())
    print(f"Cluster {g[0]} std: {avg_std[-1]}")
print(f"Average Std: {sum(avg_std)/len(avg_std)\n")

%%time
# Random masking of sl and pw column
np.random.seed(2)
random.seed(2)
PL_MASKS_CANDIDATES = [random.randrange(0, len(df)) for i in range(5)]
PW_MASKS_CANDIDATES = [random.randrange(0, len(df)) for i in range(5)]
PL_ACTUAL_VALUES  = np.array([df.loc[[id]]["pl"].tolist()[0] for id in SL_MASKS_CANDIDATES])
PW_ACTUAL_VALUES  = np.array([df.loc[[id]]["pw"].tolist()[0] for id in PW_MASKS_CANDIDATES])
m_df = df.copy()

# Setting mask candidates
for scand, pcand in zip(SL_MASKS_CANDIDATES, PW_MASKS_CANDIDATES):
    m_df.at[cand, 'pl']=None
    m_df.at[cand, 'pw']=None
 
km = KMeans(n_clusters=3)
# Ignoring first column, create clusters
km.fit(m_df[["sl", "sw"]].to_numpy())

# Approximate the mising values
print("Approximation: ")
approx_vals_pl = []
approx_vals_pw = []
m_df['cluster'] = km.labels_
gp = m_df.groupby('cluster')
for plma, pwma in zip(PL_MASKS_CANDIDATES, PW_MASKS_CANDIDATES):
    plmc = m_df.loc[[plma]]['cluster']
    pwmc = m_df.loc[[pwma]]['cluster']
    approx_vals_pl.append(df['pl'][gp.get_group(smc.tolist()[0]).index].mean())
    approx_vals_pw.append(df['pw'][gp.get_group(pmc.tolist()[0]).index].mean())
for i in zip(MASKS_CANDIDATES, PL_ACTUAL_VALUES, approx_vals_pl, PW_ACTUAL_VALUES, approx_vals_pw):
    print(f"Candidate: {i[0]}, Actual PL: {i[1]}, Predicted PL: {i[2]}, Actual PW: {i[3]}, Predicted PW: {i[4]}")
DIST = [np.linalg.norm(PL_ACTUAL_VALUES-approx_vals_pl), np.linalg.norm(PW_ACTUAL_VALUES-approx_vals_pw)]
print(f"Euclidean Distance between actual and predictions: {DIST}")
print(f"Mean Euclidean Distance: {sum(DIST)/len(DIST)}\n")

print("Cluster Performance:")
avg_std = []
for g in gp:
    avg_std.append(g[1]['class'].std())
    print(f"Cluster {g[0]} std: {avg_std[-1]}")
print(f"Average Std: {sum(avg_std)/len(avg_std)}\n")

%%time
np.random.seed(2)
m_df = df.drop(PL_MASKS_CANDIDATES)
m_df = df.drop(PW_MASKS_CANDIDATES)

km = KMeans(n_clusters=3)
km.fit(m_df[["sl", "sw", "pl", "pw"]].to_numpy())
m_df['cluster'] = km.labels_
gp = m_df.groupby('cluster')

print('Approximation:')
approx_vals_pl = []
approx_vals_pw = []
for plcand, pwcand in zip(PL_MASKS_CANDIDATES, PW_MASKS_CANDIDATES):
    pldist = [np.linalg.norm(df.loc[[cand], ['sl','sw','pw']].to_numpy() - centroid[[0,1,3]]) for centroid in km.cluster_centers_]
    pwdist = [np.linalg.norm(df.loc[[cand], ['sl','sw','pl']].to_numpy() - centroid[[0,1,2]]) for centroid in km.cluster_centers_]
    plcluster = np.argmin(pldist)
    pwcluster = np.argmin(pwdist)
    approx_vals_pl.append(gp.get_group(plcluster)["pl"].mean())
    approx_vals_pw.append(gp.get_group(pwcluster)["pw"].mean())

for i in zip(MASKS_CANDIDATES,PL_ACTUAL_VALUES, approx_vals_pl, PW_ACTUAL_VALUES, approx_vals_pw):
    print(f"Candidate: {i[0]}, Actual PL: {i[1]}, Predicted PL: {i[2]}, Actual PW: {i[3]}, Predicted PW: {i[4]}")
DIST = [np.linalg.norm(SL_ACTUAL_VALUES-approx_vals_sl), np.linalg.norm(PW_ACTUAL_VALUES-approx_vals_pw)]
print(f"Euclidean Distance between actual and predictions: {DIST}")
print(f"Mean Distance: {sum(DIST)/len(DIST)}\n")

print("Cluster Performance:")
avg_std = []
for g in gp:
    avg_std.append(g[1]['class'].std())
    print(f"Cluster {g[0]} std: {avg_std[-1]}")
print(f"Average Std: {sum(avg_std)/len(avg_std)}\n")