import pandas as pd
import numpy as np
from random import randint
import seaborn as sns
import matplotlib.pyplot as plt
from collections import Counter

df= pd.read_csv('data_missing_values.csv')
df.head(8)

	gender	ethnicity	parental_education	lunch	test_preparation_course	math_score	reading_score	writing_score
0	female	group B	bachelor's degree	standard	none	72.0	72	74
1	female	group C	some college	standard	completed	69.0	90	88
2	female	group B	master's degree	standard	none	90.0	95	93
3	male	group A	associate's degree	free/reduced	none	47.0	57	44
4	male	group C	some college	standard	none	76.0	78	75
5	female	group B	associate's degree	standard	NaN	71.0	83	78
6	female	group B	some college	standard	NaN	88.0	95	92
7	male	group B	some college	free/reduced	NaN	40.0	43	39

#chekc for missing values
df.isnull().sum()

	0
gender	0
ethnicity	0
parental_education	0
lunch	0
test_preparation_course	6
math_score	11
reading_score	0
writing_score	0

dtype: int64

$$ \Large y_{math} = 0.8 \times \frac{y_{reading}+y_{writing}}{2} $$

# define the imputation function based on the equation above
def impute_math(row):
  if pd.isna(row['math_score']):
    return 0.8 * (row['reading_score']+ row['writing_score'])/2
  else:
    return row['math_score']

missing_index = df[df['math_score'].isna()].index
missing_index

Index([10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], dtype='int64')

df.iloc[10,:]

	10
gender	male
ethnicity	group C
parental_education	associate's degree
lunch	standard
test_preparation_course	NaN
math_score	NaN
reading_score	54
writing_score	52

dtype: object

df['math_score'] = df.apply(impute_math, axis=1) #axis = 1 means we'reimputing into the columns, 0 for rows
df.isnull().sum()

	0
gender	0
ethnicity	0
parental_education	0
lunch	0
test_preparation_course	6
math_score	0
reading_score	0
writing_score	0

dtype: int64

df.iloc[10,:]

	10
gender	male
ethnicity	group C
parental_education	associate's degree
lunch	standard
test_preparation_course	NaN
math_score	42.4
reading_score	54
writing_score	52

dtype: object

#generate a list of random numbers, within range [0, 10]
random_numbers = np.random.uniform(0, 10, 1500)
a = random_numbers.tolist()
#print(a)

sns.kdeplot(a, fill=True)
plt.show()

#generate 20 numbers from a normal distribution
mean= 5
std_dev = 2
numbers = np.random.normal(mean, std_dev, 200)
b = numbers.tolist()
print(b)

[4.43622837496305, 8.475335199892163, 4.129901222947876, 4.147417988985258, 4.452621387502604, 6.71796288375433, 4.80051881971775, 7.72811249022326, 5.115798020545741, 7.959528485622702, 8.491644640767582, 8.89631876784668, 2.433328145087965, 6.670969014174091, 2.6496605106545914, 7.904991129294375, 3.106644879989397, 8.162460350017076, 4.332109130654497, 5.830929512683773, 5.214732383555363, 5.74297572085592, 5.209988466787768, 7.344675554819101, 3.1566876858322037, 8.54371782189834, 7.2801359147979605, 5.278026499413711, 4.570387270305463, 1.8224400741802556, 5.640011639052827, 7.278472039232703, 8.651101242606664, 1.9260047038647619, 5.359111815800773, 3.6046060460256326, 8.00931891899471, 3.4552532146159685, 5.714681042260107, 4.600824292960552, 7.102667238316661, 4.661265007201304, 5.68613265857488, 2.24044201859576, 2.3534654169860936, 7.355506149408795, 5.182833938426716, 4.502714777648293, 7.0740597012994435, 6.313785262784165, 6.705787568358534, 5.942248002282147, 4.458261896909275, 5.173103656656133, 8.573607123013115, 2.309139717320246, 2.5273381188691713, 6.522509640972042, 4.2848249062895, 4.884372466149224, 9.567425160317883, 9.879932075854637, 6.178948642704745, 2.716715535859143, 7.25785215323851, 7.5752905719981865, 4.226776378533221, 7.243587818971772, 5.777325520610651, 7.970007296631828, 6.639013975951496, 3.811061050054268, 4.002025127521938, 7.18533634619595, 7.342342339158904, 3.365658856906124, 5.848408455565334, 7.181024600806353, 4.56030934638791, 3.339200023470233, 2.9234015712352095, 7.223210292644486, 4.401125073782879, 1.6675285761866063, 4.49270201138827, 4.210698882983828, 6.689505093190687, 5.147106902868457, 3.1388840824048483, 8.696826861446667, 7.230345575787926, 0.7693170871411681, 3.95776141464831, 2.4332287035550975, 5.251179494576448, 6.322282894867996, 6.170252176169177, 4.845753150593083, 9.894814108782821, 5.980862649968648, 4.3017857606313745, 5.523219259216743, 7.117614015026094, 6.255376639086615, 6.2103709914709375, 8.251039925569334, 5.889615547619207, 5.495345239331753, 2.6675820519665843, 5.752204574149771, 2.4280557671139564, 4.653305785451168, 5.034985435312945, 5.892323526944236, 6.502701155031922, 2.0369847270940653, 6.070385341818487, 5.444994461244045, 1.9144440190906793, 6.084284383500946, 5.055030278050272, 5.670526592214618, 2.7218323405958897, 1.9752122503906118, 3.1316572119972848, 0.8103551075837405, 6.716830840165636, 6.738559634192809, 8.217841467819122, 5.464069737112052, 5.585393084726887, 2.778588006063417, 1.6880099015378685, 6.770016460563302, 5.845983873980969, 5.884287026820052, 5.987417710699519, 5.5564141493251835, 3.6987304775257748, 4.528096074636531, 1.5758657033704986, 3.2526135413125212, 3.9551927443225523, 2.7965486929729746, 5.466860764677061, 7.945504941348556, 4.813987509180475, 7.4739101855148125, 6.65209021423483, 3.4971993032693223, 4.0556350181797765, 4.173694118893397, 3.5404836168821516, 4.020132818422381, 3.6796830941203282, 5.255661460704882, 3.4753880101433534, 6.927972888839118, 2.9455223283745187, 4.848022286083767, 6.138491018137341, 5.573712534706047, 8.295599786555744, 10.156239965092979, 4.6427483564850895, 4.526489399194326, 2.5658778480396998, 2.550510618920107, 2.1030429797766974, 7.783617087215903, 3.709124468164573, 4.92486783244223, 4.444430606757818, 5.969685272045981, 4.221835677988624, 6.409501476826569, 7.309449920311807, 4.85658172435512, 7.503596270270737, 6.742470887320128, 2.545755612832757, 2.4015337468074853, 3.6555265695187353, 4.045416638752203, 8.021620887472002, 2.549560165143484, 4.336577693982027, 3.9699927346915818, 2.9079386779261474, 3.5652620195435043, 6.372979551712586, 5.14737400391241, 7.431810308784584, 7.009178136627877, 6.771347170266158, 5.465957968217631, 5.906277703894694, 6.572696272376422, 2.6510036016637994, 4.671624253288581]

sns.kdeplot(b, fill=True, color ='lime')
plt.show()

#task: generate some male heights in the U.S.
#mean is 5.9, std is 2
#100 males
m_mean= 5
m_std_dev = 2
m_numbers = np.random.normal(m_mean, m_std_dev, 100).tolist()
#m = numbers.tolist()
print(m_numbers)

[6.052718472401106, 5.074067701175439, 5.496690981057281, 5.622479426561831, 4.993184287256055, 4.169281100996503, 4.5747712667312355, 5.966432965205642, 5.748142239887502, 3.3081900359374288, 4.837055970408982, 7.632118596343508, 6.327574775262792, 7.387449616352029, 6.280653105761905, 5.651176686588299, 4.445717732860204, 6.642466435868968, 2.3701448832741754, 6.918165985150002, 5.130595733589099, 7.458606807772329, 4.864896394027695, 6.352940674721231, 8.901883278973349, 6.354518215143119, -0.12209477639188204, 3.905786530739781, 4.966507086531309, 1.850715104524912, 4.531902082623751, 5.409592200774423, 3.9845896891772377, 6.482178495307698, 3.243939160623465, 7.991323277175475, 5.4985317313116004, 2.5132987052414064, 4.311943686381933, 5.248319246202225, 3.305413664032134, 3.7218544177369317, 6.236498032456218, 1.7012368455870144, -1.3818007196181021, 0.21197460842133786, 5.869942165443374, 3.2306586948561753, 7.334961004396636, 3.6018885933177005, 5.463052682127464, 5.365636110393745, 3.1428195582768903, 6.205795086135069, 9.011608791095657, 3.0726704743335054, 6.206564525415555, 6.825265080805201, 7.304981406906678, 5.774677469280807, 4.311160023866293, 1.659125927720285, 7.984282053165664, 7.736650858092275, 6.170730481569702, 5.57623643051997, 7.584609625300761, 4.013106703774721, 3.8707562312040613, 4.994802824550806, 3.5128066352223057, 6.345184888736664, 5.376947926643089, 0.41231580130964307, 4.15980304552456, 7.69862056381804, 6.524898244024303, 7.144563628209452, 5.715017061217093, 5.237908517029851, 6.985639057195344, 2.7954309101779824, 5.5498716223887055, 7.342711062957456, 5.941674491603067, 3.6809336830757253, 3.88723776219929, 4.891994369748481, 2.920023649397529, 3.339266271682777, 4.136520075793006, 5.166523184112133, 7.368842304342374, 3.85576388958145, 9.162299780588086, 5.866863361891272, 5.459855632032387, 5.552317298511978, 8.142819081239224, 1.3207342958177826]

sns.kdeplot(m_numbers, fill=True, color ='sienna')
plt.show()

#I want to generate a list. apple and orange (50 by 50)
fruits = np.random.choice(['apple', 'orange'], size=100, p=[0.5, 0.5]).tolist()
print(fruits)

['orange', 'apple', 'apple', 'apple', 'apple', 'apple', 'apple', 'orange', 'apple', 'orange', 'orange', 'apple', 'apple', 'orange', 'orange', 'orange', 'apple', 'orange', 'orange', 'orange', 'apple', 'orange', 'apple', 'apple', 'apple', 'apple', 'apple', 'apple', 'apple', 'orange', 'orange', 'orange', 'orange', 'orange', 'orange', 'orange', 'orange', 'orange', 'apple', 'orange', 'apple', 'apple', 'apple', 'apple', 'apple', 'apple', 'orange', 'apple', 'orange', 'apple', 'apple', 'apple', 'orange', 'orange', 'orange', 'orange', 'orange', 'orange', 'apple', 'apple', 'orange', 'apple', 'apple', 'apple', 'apple', 'apple', 'orange', 'apple', 'apple', 'orange', 'apple', 'apple', 'orange', 'orange', 'apple', 'orange', 'orange', 'orange', 'apple', 'apple', 'orange', 'apple', 'orange', 'orange', 'apple', 'orange', 'orange', 'apple', 'orange', 'apple', 'apple', 'apple', 'orange', 'apple', 'orange', 'apple', 'orange', 'apple', 'orange', 'orange']

Counter(fruits)

Counter({'orange': 48, 'apple': 52})

#task: generate some boys and girls (20 total); evenly distributed
genders = np.random.choice(['female', 'male'], size=20, p=[0.5, 0.5]).tolist()
print(genders)

['male', 'female', 'male', 'female', 'male', 'female', 'male', 'female', 'female', 'female', 'male', 'male', 'male', 'female', 'female', 'male', 'female', 'female', 'female', 'male']

Counter(genders)

Counter({'male': 9, 'female': 11})

#generate their scores
mean_math= 70
std_dev_math= 10
math_score = np.random.normal(mean_math, std_dev_math, 20)
math_score

array([55.00104253, 72.142535  , 60.36158727, 54.44410192, 68.90865687,
       55.32627879, 68.93048251, 60.20088598, 69.40216728, 60.82135698,
       79.41135588, 70.39823466, 66.70567242, 61.1036976 , 87.10456707,
       48.73349445, 62.77616389, 79.84709816, 67.07230403, 58.41642897])

#generate their degrees [graduated, undergraduated, dropped ] 0.2, 0.7, 0.1
grad = np.random.choice(['graduated', 'undergraduate', 'dropped'], size= 20, p=[0.2, 0.7, 0.1])
grad

array(['undergraduate', 'undergraduate', 'graduated', 'undergraduate',
       'graduated', 'undergraduate', 'undergraduate', 'undergraduate',
       'undergraduate', 'graduated', 'undergraduate', 'undergraduate',
       'undergraduate', 'graduated', 'undergraduate', 'undergraduate',
       'dropped', 'undergraduate', 'dropped', 'graduated'], dtype='<U13')

new_df = pd.DataFrame(zip(genders, math_score, grad), columns=['gender', 'math', 'graduation_status'])
new_df.head()

	gender	math	graduation_status
0	male	55.001043	undergraduate
1	female	72.142535	undergraduate
2	male	60.361587	graduated
3	female	54.444102	undergraduate
4	male	68.908657	graduated

sns.kdeplot(data= new_df, x='math', color= 'crimson')
plt.show()

new_df.head()

	gender	math	graduation_status
0	male	55.001043	undergraduate
1	female	72.142535	undergraduate
2	male	60.361587	graduated
3	female	54.444102	undergraduate
4	male	68.908657	graduated

# if they have 80 or lower, give them 90
new_df['new_math'] = new_df['math'].apply(lambda x:90 if x <=80 else x)
new_df.head()

	gender	math	graduation_status	new_math
0	male	55.001043	undergraduate	90.0
1	female	72.142535	undergraduate	90.0
2	male	60.361587	graduated	90.0
3	female	54.444102	undergraduate	90.0
4	male	68.908657	graduated	90.0

sns.kdeplot(data= new_df, x='math', color ='grey')
sns.kdeplot(data= new_df, x='new_math', color= 'cyan')
plt.show()