#split data into two sections (categorical and numeric sets)
#for the numeric variable sets, we conduct normalization
#for the categorical variable sets, we transform them into dummy variables

import pandas as pd
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt

from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler

df= pd.read_csv('student.csv')
df.head()

	gender	ethnicity	parental_education	lunch	test_preparation_course	math_score	reading_score	writing_score
0	female	group B	bachelor's degree	standard	none	72	72	74
1	female	group C	some college	standard	completed	69	90	88
2	female	group B	master's degree	standard	none	90	95	93
3	male	group A	associate's degree	free/reduced	none	47	57	44
4	male	group C	some college	standard	none	76	78	75

#define Scalers
min_max_scaler= MinMaxScaler()
std_scaler= StandardScaler()

#extract all numeric variables and put them into df_num
#df_num=df.select_dtypes(include=['int64','float64']) a way of doing it of the data wasnt already sorted by numeric and categorical
#df_num.head()

df_num= df.iloc[:, 5:8]
#rescale data
df_num[['math_n', 'reading_n', 'writing_n']]= min_max_scaler.fit_transform(df_num[['math_score','reading_score','writing_score']])
df_num.head()

	math_score	reading_score	writing_score	math_n	reading_n	writing_n
0	72	72	74	0.72	0.662651	0.711111
1	69	90	88	0.69	0.879518	0.866667
2	90	95	93	0.90	0.939759	0.922222
3	47	57	44	0.47	0.481928	0.377778
4	76	78	75	0.76	0.734940	0.722222

#define scalers
min_max_scaler = MinMaxScaler()
std_scaler= StandardScaler()

#rescale data
#df_num_min_max= min_max_scaler.fit_transform(df_num)
#df_num_min_max= pd.DataFrame(df_num_min_max, columns=df_num.columns)
#df_num_min_max

#adjust 2 decimal
df_num= df_num.round(2)
df_num.head()

	math_score	reading_score	writing_score	math_n	reading_n	writing_n
0	72	72	74	0.72	0.66	0.71
1	69	90	88	0.69	0.88	0.87
2	90	95	93	0.90	0.94	0.92
3	47	57	44	0.47	0.48	0.38
4	76	78	75	0.76	0.73	0.72

#extract normalized columns
df_score_new= df_num.iloc[:, 3:6]
df_score_new.head()

	math_n	reading_n	writing_n
0	0.72	0.66	0.71
1	0.69	0.88	0.87
2	0.90	0.94	0.92
3	0.47	0.48	0.38
4	0.76	0.73	0.72

#for the categorical variables set, we transform them into dummy variables
#extract all categorical variables and put them into df_cat
#df_dummy: store all the dummy variables

df_cat= df.iloc[:, 0:5]
df_cat.head()

	gender	ethnicity	parental_education	lunch	test_preparation_course
0	female	group B	bachelor's degree	standard	none
1	female	group C	some college	standard	completed
2	female	group B	master's degree	standard	none
3	male	group A	associate's degree	free/reduced	none
4	male	group C	some college	standard	none

df_dummy= pd.get_dummies(data= df_cat, drop_first= True, dtype= float)
df_dummy.head()

	gender_male	ethnicity_group B	ethnicity_group C	ethnicity_group D	ethnicity_group E	parental_education_bachelor's degree	parental_education_high school	parental_education_master's degree	parental_education_some college	parental_education_some high school	lunch_standard	test_preparation_course_none
0	0.0	1.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	1.0	1.0
1	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	1.0	0.0
2	0.0	1.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	1.0	1.0
3	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
4	1.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	1.0	1.0

df_dummy.shape

(1000, 12)

#normalized data: df_score_new
#dummied data: df_dummy
new_df= pd.concat([df_dummy, df_score_new], axis= 1) #axis = 1 means sorted by column to column
new_df.head()

	gender_male	ethnicity_group B	ethnicity_group C	ethnicity_group D	ethnicity_group E	parental_education_bachelor's degree	parental_education_high school	parental_education_master's degree	parental_education_some college	parental_education_some high school	lunch_standard	test_preparation_course_none	math_n	reading_n	writing_n
0	0.0	1.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	1.0	1.0	0.72	0.66	0.71
1	0.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	1.0	0.0	0.69	0.88	0.87
2	0.0	1.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	1.0	1.0	0.90	0.94	0.92
3	1.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.47	0.48	0.38
4	1.0	0.0	1.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	1.0	1.0	0.76	0.73	0.72

new_df.shape

(1000, 15)

#check the distribution between math_n and reading_n
plt.figure(figsize=(5,5))
sns.scatterplot(new_df, x= 'math_n', y= 'reading_n')
plt.show()