import pandas as pd
import numpy as np
from random import randint

Missing value imputation

Learning Goal:

In this class, we are going to learn how to deal with data that contains missing values.

• check if the dataframe contains missing value.
• how to impute the categorical values.
• how to impute the numeric ones.

#read the data
df = pd.read_csv('data_missing_values.csv')
df.head(10)

	gender	ethnicity	parental_education	lunch	test_preparation_course	math_score	reading_score	writing_score
0	female	group B	bachelor's degree	standard	none	72.0	72	74
1	female	group C	some college	standard	completed	69.0	90	88
2	female	group B	master's degree	standard	none	90.0	95	93
3	male	group A	associate's degree	free/reduced	none	47.0	57	44
4	male	group C	some college	standard	none	76.0	78	75
5	female	group B	associate's degree	standard	NaN	71.0	83	78
6	female	group B	some college	standard	NaN	88.0	95	92
7	male	group B	some college	free/reduced	NaN	40.0	43	39
8	male	group D	high school	free/reduced	NaN	64.0	64	67
9	female	group B	high school	free/reduced	NaN	38.0	60	50

#check out whole dataframe and identify which columns have missing values
df.isnull().sum()

	0
gender	0
ethnicity	0
parental_education	0
lunch	0
test_preparation_course	6
math_score	11
reading_score	0
writing_score	0

dtype: int64

Deal with Missing Categorical Values

#using majority rule
df['test_preparation_course'].value_counts()

	count
test_preparation_course
none	638
completed	356

dtype: int64

majo = 'none'
df['test_preparation_course']= df['test_preparation_course'].fillna(majo)
#check the dataframe again
df.isnull().sum()

	0
gender	0
ethnicity	0
parental_education	0
lunch	0
test_preparation_course	0
math_score	11
reading_score	0
writing_score	0

dtype: int64

#checking tha tthe NaN is gone in the categorical values
df.head(10)

	gender	ethnicity	parental_education	lunch	test_preparation_course	math_score	reading_score	writing_score
0	female	group B	bachelor's degree	standard	none	72.0	72	74
1	female	group C	some college	standard	completed	69.0	90	88
2	female	group B	master's degree	standard	none	90.0	95	93
3	male	group A	associate's degree	free/reduced	none	47.0	57	44
4	male	group C	some college	standard	none	76.0	78	75
5	female	group B	associate's degree	standard	none	71.0	83	78
6	female	group B	some college	standard	none	88.0	95	92
7	male	group B	some college	free/reduced	none	40.0	43	39
8	male	group D	high school	free/reduced	none	64.0	64	67
9	female	group B	high school	free/reduced	none	38.0	60	50

#look at GAN network to see how people are able to
#fake everything with generator and discriminator algorithms
#the two compete making each other better

#pytorch for deep learning models
#

# deal with the math_score
df['math_score'] = df['math_score'].apply(lambda x: np.random.randint(0,101) if np.isnan(x) else x)

#check for missing values again
df.isnull().sum()

	0
gender	0
ethnicity	0
parental_education	0
lunch	0
test_preparation_course	0
math_score	0
reading_score	0
writing_score	0

dtype: int64