In [1]:
import pandas as pd
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
In [4]:
 #Load the tips dataset
tips_data = pd.read_csv('tips.csv')
tips_data.head()
Out[4]:
total_bill tip sex smoker day time size
0 16.99 1.01 Female No Sun Dinner 2
1 10.34 1.66 Male No Sun Dinner 3
2 21.01 3.50 Male No Sun Dinner 3
3 23.68 3.31 Male No Sun Dinner 2
4 24.59 3.61 Female No Sun Dinner 4
In [5]:
tips_data['total_bill'].iloc[10:15] = None

/tmp/ipykernel_1447/1340756543.py:1: FutureWarning: ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

  tips_data['total_bill'].iloc[10:15] = None
/tmp/ipykernel_1447/1340756543.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  tips_data['total_bill'].iloc[10:15] = None
In [6]:
# Check for missing values
sum(tips_data['total_bill'].isna())
Out[6]:
5
In [8]:
print(f'The number of observation before removing missing rows: {tips_data.shape[0]}')
# Remove rows with missing values
tips_data_clean = tips_data.dropna()
print(f'The number of observation after removing missing rows: {tips_data_clean.shape[0]}')

The number of observation before removing missing rows: 244
The number of observation after removing missing rows: 239
In [10]:
 #Impute missing values with mean
mean_total_bill = tips_data['total_bill'].mean()
tips_data['total_bill'].fillna(mean_total_bill, inplace=True)

/tmp/ipykernel_1447/1619248906.py:3: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.
The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.

For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.


  tips_data['total_bill'].fillna(mean_total_bill, inplace=True)
In [11]:
# Calculate z-scores for the total_bill column
z_scores = (tips_data['total_bill'] - tips_data['total_bill'].mean()) / tips_data['total_bill'].std()
# Set a threshold for outliers (e.g., z-score > 3)
threshold = 3
outliers_zscore = tips_data[np.abs(z_scores) > threshold]
print(outliers_zscore)

     total_bill    tip   sex smoker  day    time  size
59        48.27   6.73  Male     No  Sat  Dinner     4
156       48.17   5.00  Male     No  Sun  Dinner     6
170       50.81  10.00  Male    Yes  Sat  Dinner     3
212       48.33   9.00  Male     No  Sat  Dinner     4
In [12]:
# Create a box plot for the total_bill column
sns.boxplot(x=tips_data['total_bill'])
# Display the plot
plt.show()
No description has been provided for this image
In [13]:
# Create a scatter plot for the total_bill column against the tip column
sns.scatterplot(x='total_bill', y='tip', data=tips_data)
Out[13]:
<Axes: xlabel='total_bill', ylabel='tip'>
No description has been provided for this image
In [14]:
# Calculate the interquartile range (IQR)
q1 = tips_data['total_bill'].quantile(0.25)
q3 = tips_data['total_bill'].quantile(0.75)
iqr = q3 - q1
# Set a threshold for outliers (e.g., 1.5 times the IQR)
threshold = 1.5
lower_fence = q1 - threshold * iqr
upper_fence = q3 + threshold * iqr
outliers_tukey = tips_data[(tips_data['total_bill'] < lower_fence) |(tips_data['total_bill'] > upper_fence)]
In [15]:
# Extract the total bill amount
total_bill = tips_data['total_bill']
# Plotting histograms of original and transformed data
plt.figure(figsize=(10, 4))
plt.subplot(1, 2, 1)
sns.histplot(total_bill, kde=True)
plt.xlabel('Total Bill')
plt.ylabel('Frequency')
plt.title('Original Data')
plt.subplot(1, 2, 2)
sns.histplot(np.log(total_bill), kde=True)
plt.xlabel('Log Total Bill')
plt.ylabel('Frequency')
plt.title('Log Transformed Data')
# Display the plots
plt.tight_layout()
plt.show()
No description has been provided for this image
In [16]:
#-------- case study
In [17]:
# Import necessary libraries
import pandas as pd
In [19]:
# Load the data
KCHouseData = pd.read_csv("kc_house_data.csv")
In [20]:
# Display the first few rows of the DataFrame
KCHouseData.head()
Out[20]:
id date price bedrooms bathrooms sqft_living sqft_lot floors waterfront view ... grade sqft_above sqft_basement yr_built yr_renovated zipcode lat long sqft_living15 sqft_lot15
0 7129300520 20141013T000000 221900.0 3 1.00 1180 5650 1.0 0 0 ... 7 1180 0 1955 0 98178 47.5112 -122.257 1340 5650
1 6414100192 20141209T000000 538000.0 3 2.25 2570 7242 2.0 0 0 ... 7 2170 400 1951 1991 98125 47.7210 -122.319 1690 7639
2 5631500400 20150225T000000 180000.0 2 1.00 770 10000 1.0 0 0 ... 6 770 0 1933 0 98028 47.7379 -122.233 2720 8062
3 2487200875 20141209T000000 604000.0 4 3.00 1960 5000 1.0 0 0 ... 7 1050 910 1965 0 98136 47.5208 -122.393 1360 5000
4 1954400510 20150218T000000 510000.0 3 2.00 1680 8080 1.0 0 0 ... 8 1680 0 1987 0 98074 47.6168 -122.045 1800 7503

5 rows × 21 columns

In [21]:
# Check for missing values
KCHouseData.isna().sum()
Out[21]:
0
id 0
date 0
price 0
bedrooms 0
bathrooms 0
sqft_living 0
sqft_lot 0
floors 0
waterfront 0
view 0
condition 0
grade 0
sqft_above 0
sqft_basement 0
yr_built 0
yr_renovated 0
zipcode 0
lat 0
long 0
sqft_living15 0
sqft_lot15 0

In [22]:
# Import visualization libraries
import matplotlib.pyplot as plt
import seaborn as sns
# Create a simple histogram to visualize the distribution of prices
sns.histplot(KCHouseData['price'])
plt.show()
No description has been provided for this image
In [23]:
# Plot a boxplot to visualize outliers in the 'price' column
sns.boxplot(x=KCHouseData['price'])
# This will show a box plot, and any data points that appear outside the 'whiskers' can be considered outliers.
Out[23]:
<Axes: xlabel='price'>
No description has been provided for this image
In [24]:
 # Create a new feature 'living_space_per_room'
KCHouseData['living_space_per_room'] = KCHouseData['sqft_living'] /KCHouseData['bedrooms']
# Let's take a look at the first few rows of our DataFrame now
KCHouseData.head()
Out[24]:
id date price bedrooms bathrooms sqft_living sqft_lot floors waterfront view ... sqft_above sqft_basement yr_built yr_renovated zipcode lat long sqft_living15 sqft_lot15 living_space_per_room
0 7129300520 20141013T000000 221900.0 3 1.00 1180 5650 1.0 0 0 ... 1180 0 1955 0 98178 47.5112 -122.257 1340 5650 393.333333
1 6414100192 20141209T000000 538000.0 3 2.25 2570 7242 2.0 0 0 ... 2170 400 1951 1991 98125 47.7210 -122.319 1690 7639 856.666667
2 5631500400 20150225T000000 180000.0 2 1.00 770 10000 1.0 0 0 ... 770 0 1933 0 98028 47.7379 -122.233 2720 8062 385.000000
3 2487200875 20141209T000000 604000.0 4 3.00 1960 5000 1.0 0 0 ... 1050 910 1965 0 98136 47.5208 -122.393 1360 5000 490.000000
4 1954400510 20150218T000000 510000.0 3 2.00 1680 8080 1.0 0 0 ... 1680 0 1987 0 98074 47.6168 -122.045 1800 7503 560.000000

5 rows × 22 columns

In [25]:
 #Import the preprocessing module
from sklearn import preprocessing
# Create the StandardScaler object
scaler = preprocessing.StandardScaler()
# Fit and transform the 'price' column
KCHouseData['price_scaled'] = scaler.fit_transform(KCHouseData[['price']])
KCHouseData[['price','price_scaled']].head()
Out[25]:
price price_scaled
0 221900.0 -0.866717
1 538000.0 -0.005688
2 180000.0 -0.980849
3 604000.0 0.174090
4 510000.0 -0.081958
In [ ]: