In [ ]:
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
#from matplotlib import pyplot as plt -- another way to word it
In [ ]:
df = pd.read_csv('student.csv')
df.head()
Out[ ]:
gender ethnicity parental_education lunch test_preparation_course math_score reading_score writing_score
0 female group B bachelor's degree standard none 72 72 74
1 female group C some college standard completed 69 90 88
2 female group B master's degree standard none 90 95 93
3 male group A associate's degree free/reduced none 47 57 44
4 male group C some college standard none 76 78 75
In [ ]:
#remove certain column
dropped_df= df.drop(['lunch', 'parental_education'], axis=1) # axis=1 drops the culumn, use brackets to drop more than one
dropped_df.head()
Out[ ]:
gender ethnicity test_preparation_course math_score reading_score writing_score
0 female group B none 72 72 74
1 female group C completed 69 90 88
2 female group B none 90 95 93
3 male group A none 47 57 44
4 male group C none 76 78 75
In [ ]:
#use iloc
#select first 5 rows and first 3 columns
df.iloc[0:5, 0:3] #if i use : by itself, it'll include all of either the rows or columns depending on where its at
Out[ ]:
gender ethnicity parental_education
0 female group B bachelor's degree
1 female group C some college
2 female group B master's degree
3 male group A associate's degree
4 male group C some college
In [ ]:
#select 3-5 rows and keep all columns
df.iloc[3:6,:]
Out[ ]:
gender ethnicity parental_education lunch test_preparation_course math_score reading_score writing_score
3 male group A associate's degree free/reduced none 47 57 44
4 male group C some college standard none 76 78 75
5 female group B associate's degree standard none 71 83 78
In [ ]:
# use iloc: we want all scores columns for entire students
df.iloc[:, 5:8]
Out[ ]:
math_score reading_score writing_score
0 72 72 74
1 69 90 88
2 90 95 93
3 47 57 44
4 76 78 75
... ... ... ...
995 88 99 95
996 62 55 55
997 59 71 65
998 68 78 77
999 77 86 86

1000 rows × 3 columns

In [ ]:
# reconstruct dataframe with certain columns
new_df= df[['gender', 'ethnicity', 'reading_score']]
new_df.head()
Out[ ]:
gender ethnicity reading_score
0 female group B 72
1 female group C 90
2 female group B 95
3 male group A 57
4 male group C 78
In [ ]:
#trim data: drop()- drop specific, iloc()- continuosly, [[]]- choose columns
In [ ]:
#construct dummy variables
df.head()
Out[ ]:
gender ethnicity parental_education lunch test_preparation_course math_score reading_score writing_score
0 female group B bachelor's degree standard none 72 72 74
1 female group C some college standard completed 69 90 88
2 female group B master's degree standard none 90 95 93
3 male group A associate's degree free/reduced none 47 57 44
4 male group C some college standard none 76 78 75
In [ ]:
# turn gender to dummy variables
#gender_dummy= pd.get_dummies(df['gender'])
#gender_dummy.head()

df_01= pd.get_dummies(df, columns=['gender'], drop_first=True, dtype=float) #if drop_first= false, we would get 2 columns for gender, for dtype we can use float or int
df_01.head()
Out[ ]:
ethnicity parental_education lunch test_preparation_course math_score reading_score writing_score gender_male
0 group B bachelor's degree standard none 72 72 74 0.0
1 group C some college standard completed 69 90 88 0.0
2 group B master's degree standard none 90 95 93 0.0
3 group A associate's degree free/reduced none 47 57 44 1.0
4 group C some college standard none 76 78 75 1.0
In [ ]:
#turn all categorical variables into numeric ones
df_02= pd.get_dummies(df, columns=['gender', 'ethnicity', 'parental_education',
                                   'lunch', 'test_preparation_course'], drop_first=True,
                      dtype=float)
df_02.head()

df_02= pd.get_dummies(df, drop_first=True, dtype=float)
df_02.head()
Out[ ]:
math_score reading_score writing_score gender_male ethnicity_group B ethnicity_group C ethnicity_group D ethnicity_group E parental_education_bachelor's degree parental_education_high school parental_education_master's degree parental_education_some college parental_education_some high school lunch_standard test_preparation_course_none
0 72 72 74 0.0 1.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 1.0 1.0
1 69 90 88 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.0 0.0
2 90 95 93 0.0 1.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 1.0 1.0
3 47 57 44 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0
4 76 78 75 1.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.0 1.0
In [ ]:
#filter some information
df.head()
Out[ ]:
gender ethnicity parental_education lunch test_preparation_course math_score reading_score writing_score
0 female group B bachelor's degree standard none 72 72 74
1 female group C some college standard completed 69 90 88
2 female group B master's degree standard none 90 95 93
3 male group A associate's degree free/reduced none 47 57 44
4 male group C some college standard none 76 78 75
In [ ]:
#select students whose identity is group B
#df_03= df[df['ethnicity']== 'group B']
#df_03.head()

refined_df= df.loc[df['ethnicity']== 'group B']
refined_df.head()
Out[ ]:
gender ethnicity parental_education lunch test_preparation_course math_score reading_score writing_score
0 female group B bachelor's degree standard none 72 72 74
2 female group B master's degree standard none 90 95 93
5 female group B associate's degree standard none 71 83 78
6 female group B some college standard completed 88 95 92
7 male group B some college free/reduced none 40 43 39
In [ ]:
#data visualization
#identify the distribution of math score
plt.figure(figsize=(5,4))
sns.histplot(data=df, x='math_score', kde=True, color='steelblue')
plt.title('Distribution of Math Scores')
plt.xlabel('Math Score')
plt.show()
No description has been provided for this image
In [ ]:
#boxplot
plt.figure(figsize=(5,4))
sns.boxplot(data=df, x='math_score', color='steelblue')
plt.title('Distribution of Math Scores')
Out[ ]:
Text(0.5, 1.0, 'Distribution of Math Scores')
No description has been provided for this image
In [ ]:
#trim data: drop(), iloc, [[]]
#dummy variables pd.get_dummies()
# data filtering: loc
#visualizations: histplot(), boxplot()