In [1]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
In [2]:
df_emailinfo=pd.read_csv("email_table.csv")
#df_emailinfo = df_emailinfo.reset_index(drop=True)
In [3]:
df_emailinfo.head(5)
Out[3]:
In [4]:
df_emailinfo.info()
In [5]:
df_emailinfo.describe()
Out[5]:
In [6]:
categorical_feature_mask = df_emailinfo.dtypes==object
categorical_feature_mask
# filter categorical columns using mask and turn it into a list
categorical_cols = df_emailinfo.columns[categorical_feature_mask].tolist()
categorical_cols.remove('weekday')
categorical_cols.remove('user_country')
from sklearn.preprocessing import LabelEncoder
# # # instantiate labelencoder object
le = LabelEncoder()
df_emailinfo[categorical_cols] = df_emailinfo[categorical_cols].apply(lambda col: le.fit_transform(col))
df_emailinfo[categorical_cols].head(10)
Out[6]:
In [7]:
df_emailinfo.head(5)
Out[7]:
In [ ]:
Data Visualisation¶
In [8]:
import seaborn as sns
In [9]:
sns.boxplot(x='user_country',y='hour',data=df_emailinfo,palette='rainbow')
Out[9]:
In [10]:
sns.countplot(x='user_country',data=df_emailinfo,palette='rainbow')
Out[10]:
In [11]:
sns.countplot(x='converted' ,data=df_emailinfo)
Out[11]:
In [12]:
sns.countplot(x='converted',hue='user_country' ,data=df_emailinfo)
Out[12]:
In [13]:
sns.countplot(x='converted', hue='email_version', data=df_emailinfo)
# from graph we can observe that compared to generic emails, personalized emails is being converted
Out[13]:
In [14]:
sns.countplot(x='converted', hue='email_text', data=df_emailinfo)
# no major difference
Out[14]:
In [15]:
df_emailinfo['user_past_purchases'].hist(color='green',bins=50,figsize=(8,4))
Out[15]:
In [16]:
sns.countplot(x='hour', hue='user_country', data=df_emailinfo)
# most of mails were sent during day-time from 5:00 am to 3:00 pm across various countries
Out[16]:
In [17]:
sns.countplot(x='user_past_purchases', hue='user_country', data=df_emailinfo)
# we can observe a constant decline of purchases
Out[17]:
Merging other two tables wil email_info table¶
In [18]:
df_emailopen=pd.read_csv("email_opened_table.csv")
df_emailopen = df_emailopen.reset_index(drop=True)
df_emailopen.shape
Out[18]:
In [19]:
df_emaillink=pd.read_csv("link_clicked_table.csv")
df_emaillink.info()
In [20]:
df_emaillink.shape
Out[20]:
In [21]:
df_emailopen['email_opened']=1
df_emailopen.head(5)
Out[21]:
In [22]:
df_emailinfo = pd.merge(left=df_emailinfo, right=df_emailopen, on='email_id',how='outer')
df_emailinfo.info()
In [23]:
df_emailinfo['email_opened']=df_emailinfo['email_opened'].fillna(0)
df_emailinfo['email_opened']=df_emailinfo['email_opened'].astype(int)
In [24]:
df_emaillink['link_opened']=1
df_emaillink.head(5)
Out[24]:
In [25]:
df_emailinfo = pd.merge(left=df_emailinfo, right=df_emaillink, on='email_id',how='outer')
df_emailinfo.info()
In [26]:
df_emailinfo['link_opened']=df_emailinfo['link_opened'].fillna(0)
df_emailinfo['link_opened']=df_emailinfo['link_opened'].astype(int)
df_emailinfo['user_past_purchases']=df_emailinfo['user_past_purchases'].astype(int)
Data Visualization after merging¶
In [27]:
sns.countplot(x='email_opened',data=df_emailinfo)
Out[27]:
In [28]:
sns.countplot(x='email_opened', hue='email_text', data=df_emailinfo)
# short emails are preferrable
Out[28]:
In [29]:
sns.countplot(x='email_opened', hue='email_version', data=df_emailinfo)
Out[29]:
In [30]:
sns.countplot(x='email_opened', hue='weekday', data=df_emailinfo)
# mostly opened during weekdays
Out[30]:
In [31]:
sns.countplot(x='email_opened', hue='user_country', data=df_emailinfo)
Out[31]:
In [32]:
sns.countplot(x='email_opened', hue='user_country', data=df_emailinfo)
Out[32]:
In [33]:
sns.countplot(x='link_opened', hue='email_text', data=df_emailinfo)
#does not matter whether long or short emails are sent- equal chance
Out[33]:
In [34]:
sns.countplot(x='email_opened', hue='email_version', data=df_emailinfo)
# to prefer more personalised emails to convert into more buyers
Out[34]:
In [35]:
sns.countplot(x='link_opened', hue='weekday', data=df_emailinfo)
# maximum mails should be triggered on weekdays
Out[35]:
In [36]:
sns.countplot(x='link_opened', hue='user_country', data=df_emailinfo)
# should focus on increasing the count for US and UK
# should come up with new strategies to attract France and Spain customers
Out[36]:
Feature engineering¶
Changing categorical to numerical
In [37]:
df_text_ohe=pd.get_dummies(df_emailinfo, columns=['weekday','user_country'])
In [38]:
df_text_ohe.head(5)
Out[38]:
In [39]:
df_emailinfo.drop(['email_text','weekday','email_version','user_country'],axis=1,inplace=True)
In [40]:
df_emailinfo.head(5)
Out[40]:
In [41]:
df_emailinfo=df_emailinfo.merge(df_text_ohe)
df_emailinfo.head()
Out[41]:
In [42]:
df_emailinfo.shape
Out[42]:
In [43]:
#join_df=df_emailinfo.join(df_emailopen,how='outer',rsuffix='_open',on='email_id')
#join_df.info()
In [44]:
#join_df.sample(50)
In [45]:
#df_emailinfo[df_emailinfo['email_id'] == 950040]
In [46]:
#merge_df=pd.concat([df_emailinfo,df_emailopen],ignore_index=True)
#merge_df.info()
#merged_inner = pd.merge(left=df_emailinfo, right=df_emailopen, on='email_id',how='outer',suffixes=('','_open'))
#merged_inner.info()
Transformation - Rescaling and Standardization¶
In [47]:
df_emailinfo.describe()
Out[47]:
In [48]:
from sklearn import preprocessing
x1_array = pd.Series(df_emailinfo['user_past_purchases'])
normalized_X = preprocessing.normalize([x1_array])
df_emailinfo['user_past_purchases'].shape
Out[48]:
In [49]:
user_purchase=normalized_X.reshape(100000,)
user_purchase
Out[49]:
In [50]:
df_emailinfo['user_past_purchases']=user_purchase
In [51]:
df_emailinfo['user_past_purchases'].head(10)
Out[51]:
In [52]:
df_emailinfo.describe()
Out[52]:
In [53]:
#df_emailinfo.user_past_purchases.hist(bins=50)
In [54]:
#df_emailinfo.hour.hist(bins=10)
In [55]:
# x2_array = pd.Series(df_emailinfo['hour'])
# normalized_X2 = preprocessing.normalize([x2_array])
# df_emailinfo['hour'].shape
# hour=normalized_X2.reshape(100000,)
# hour
# df_emailinfo['hour']=hour
# df_emailinfo['hour'].head(10)
In [56]:
df_emailinfo.describe()
Out[56]:
In [57]:
df_emailinfo.hour.hist(bins=10)
Out[57]:
In [58]:
scaler = preprocessing.StandardScaler()
df_emailinfo['hour']=scaler.fit_transform(df_emailinfo[['hour']])
df_emailinfo['hour'].head()
Out[58]:
In [59]:
df_emailinfo.describe()
Out[59]:
In [60]:
df_emailinfo.hour.hist(bins=10)
Out[60]:
In [61]:
import seaborn as sns
sns.kdeplot(df_emailinfo.hour)
Out[61]:
Data Visualisation¶
In [62]:
# correlation=df_emailinfo.corr()
# fig=plt.figure
# plt.show()
In [63]:
sns.heatmap(df_emailinfo.corr(),cmap='coolwarm',)
plt.title('email_info.corr()')
Out[63]:
In [64]:
#df_emailinfo.drop('email_text_short_email',axis=1,inplace=True)
#df_emailinfo.drop('email_version_generic',axis=1,inplace=True)
df_emailinfo.drop('link_opened',axis=1,inplace=True)
df_emailinfo.drop('user_country_ES',axis=1,inplace=True)
df_emailinfo.drop('email_opened',axis=1,inplace=True)
#df_emailinfo.drop('user_past_purchases',axis=1,inplace=True)
In [65]:
sns.heatmap(df_emailinfo.corr(),cmap='coolwarm',)
plt.title('email_info.corr()')
Out[65]:
In [66]:
df_emailinfo.shape
Out[66]:
In [67]:
df_emailinfo.info()
In [68]:
#input_columns=[columns for columns in df_emailinfo.columns if columns!='converted']
In [69]:
#output_column=df_emailinfo['converted']
#not able to execute
#X = df.loc[:,input_columns].values
#y = df.loc[:,output_column]
#print (X.shape, y.shape)
In [70]:
X = df_emailinfo[['email_id','hour','user_past_purchases','email_text','weekday_Friday','weekday_Monday','weekday_Saturday','weekday_Sunday','weekday_Thursday','weekday_Tuesday','weekday_Wednesday','email_version','user_country_FR','user_country_UK','user_country_US']]
y = df_emailinfo['converted']
In [71]:
from sklearn.model_selection import train_test_split
In [72]:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=100, stratify=y)
In [73]:
print (X_train.shape)
print (y_train.shape)
print (X_test.shape)
print (y_test.shape)
In [74]:
non_converted=df_emailinfo[df_emailinfo['converted']==0]
converted=df_emailinfo[df_emailinfo['converted']==1]
In [75]:
print(non_converted.shape, converted.shape)
In [76]:
from sklearn.utils import resample
converted_upsampled = resample(converted,
replace=True, # sample with replacement
n_samples=len(non_converted), # match number in majority class
random_state=27)
In [77]:
converted_upsampled.shape
Out[77]:
In [78]:
non_converted.shape
Out[78]:
In [79]:
upsampled=pd.concat([converted_upsampled,non_converted])
upsampled.head(5)
Out[79]:
In [80]:
upsampled.converted.value_counts()
Out[80]:
In [81]:
y_train = upsampled.converted
X_train = upsampled.drop('converted', axis=1)
In [82]:
from sklearn.linear_model import LogisticRegression
In [83]:
logmodel = LogisticRegression()
logmodel.fit(X_train,y_train)
Out[83]:
In [84]:
logmodel.score(X_train, y_train)
Out[84]:
In [85]:
logmodel.score(X_test, y_test)
Out[85]:
In [86]:
predictions = logmodel.predict(X_test)
predictions
Out[86]:
In [87]:
from sklearn.metrics import classification_report,accuracy_score
from sklearn.metrics import confusion_matrix
In [88]:
print(accuracy_score(y_test,predictions))
In [89]:
print(classification_report(y_test,predictions))
In [90]:
print(confusion_matrix(y_test,predictions))
In [91]:
from sklearn.metrics import roc_auc_score,roc_curve,auc
false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, predictions)
roc_auc = auc(false_positive_rate, true_positive_rate)
roc_auc
Out[91]:
In [92]:
plt.plot(false_positive_rate,true_positive_rate)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.show()
In [93]:
predicted_probs=logmodel.predict_proba(X_test)
In [94]:
predicted_probs[100:110]
Out[94]:
In [95]:
y_pred_lower_threshold = logmodel.predict_proba(X_test)[:,1] <0.5
In [96]:
print(classification_report(y_test, y_pred_lower_threshold))
In [97]:
from sklearn.model_selection import GridSearchCV
In [98]:
log_model_1=LogisticRegression()
In [114]:
param_grid = {'C': [0.001, 0.04, 0.07,0.08, 1,10],'max_iter':[50,100,200,500]}
In [115]:
grid_search = GridSearchCV(log_model_1, param_grid, cv=5)
In [116]:
grid_search.fit(X_train, y_train)
Out[116]:
In [117]:
print("Test set score: {:.2f}".format(grid_search.score(X_test, y_test)))
In [118]:
print("Best parameters: {}".format(grid_search.best_params_))
print("Best cross-validation score: {:.2f}".format(grid_search.best_score_))
In [ ]:
In [119]:
print("Best estimator:\n{}".format(grid_search.best_estimator_))
In [120]:
logreg = LogisticRegression(class_weight='balanced',C=0.002,max_iter=50).fit(X_train,y_train)
In [121]:
logreg.score(X_test, y_test)
Out[121]:
In [122]:
predictions=logreg.predict(X_test)
In [123]:
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
print(classification_report(y_test, predictions,target_names=["Non Converted", "Converted"]))