In [1]:

    import pandas as pd
    import numpy as np
    from sklearn import metrics
    import matplotlib.pyplot as plt
    %matplotlib inline
    import seaborn as sns

    from sklearn.model_selection import train_test_split
    from sklearn.linear_model import LogisticRegression

    from sklearn.tree import DecisionTreeClassifier
    from sklearn import tree

    #Obejective: To predict which customer is more likely to purchase the newly introduced travel package.

In [2]:

    df = pd.read_excel(r"C:\Users\dganapathi\Desktop\aiml\Tourism.xlsx", sheet_name="Tourism")
    df_copy = df.copy()

In [3]:

    df.info()

    <class 'pandas.core.frame.DataFrame'>
    RangeIndex: 4888 entries, 0 to 4887
    Data columns (total 20 columns):
     #   Column                    Non-Null Count  Dtype
    ---  ------                    --------------  -----
     0   CustomerID                4888 non-null   int64
     1   ProdTaken                 4888 non-null   int64
     2   Age                       4662 non-null   float64
     3   TypeofContact             4863 non-null   object
     4   CityTier                  4888 non-null   int64
     5   DurationOfPitch           4637 non-null   float64
     6   Occupation                4888 non-null   object
     7   Gender                    4888 non-null   object
     8   NumberOfPersonVisiting    4888 non-null   int64
     9   NumberOfFollowups         4843 non-null   float64
     10  ProductPitched            4888 non-null   object
     11  PreferredPropertyStar     4862 non-null   float64
     12  MaritalStatus             4888 non-null   object
     13  NumberOfTrips             4748 non-null   float64
     14  Passport                  4888 non-null   int64
     15  PitchSatisfactionScore    4888 non-null   int64
     16  OwnCar                    4888 non-null   int64
     17  NumberOfChildrenVisiting  4822 non-null   float64
     18  Designation               4888 non-null   object
     19  MonthlyIncome             4655 non-null   float64
    dtypes: float64(7), int64(7), object(6)
    memory usage: 763.9+ KB

In [4]:

    df.describe().T

Out[4]:

| count | mean | std | min | 25% | 50% | 75% | max
--- | --- | --- | --- | --- | --- | --- | --- | ---
CustomerID | 4888.0 | 202443.500000 | 1411.188388 | 200000.0 | 201221.75 | 202443.5 | 203665.25 | 204887.0
ProdTaken | 4888.0 | 0.188216 | 0.390925 | 0.0 | 0.00 | 0.0 | 0.00 | 1.0
Age | 4662.0 | 37.622265 | 9.316387 | 18.0 | 31.00 | 36.0 | 44.00 | 61.0
CityTier | 4888.0 | 1.654255 | 0.916583 | 1.0 | 1.00 | 1.0 | 3.00 | 3.0
DurationOfPitch | 4637.0 | 15.490835 | 8.519643 | 5.0 | 9.00 | 13.0 | 20.00 | 127.0
NumberOfPersonVisiting | 4888.0 | 2.905074 | 0.724891 | 1.0 | 2.00 | 3.0 | 3.00 | 5.0
NumberOfFollowups | 4843.0 | 3.708445 | 1.002509 | 1.0 | 3.00 | 4.0 | 4.00 | 6.0
PreferredPropertyStar | 4862.0 | 3.581037 | 0.798009 | 3.0 | 3.00 | 3.0 | 4.00 | 5.0
NumberOfTrips | 4748.0 | 3.236521 | 1.849019 | 1.0 | 2.00 | 3.0 | 4.00 | 22.0
Passport | 4888.0 | 0.290917 | 0.454232 | 0.0 | 0.00 | 0.0 | 1.00 | 1.0
PitchSatisfactionScore | 4888.0 | 3.078151 | 1.365792 | 1.0 | 2.00 | 3.0 | 4.00 | 5.0
OwnCar | 4888.0 | 0.620295 | 0.485363 | 0.0 | 0.00 | 1.0 | 1.00 | 1.0
NumberOfChildrenVisiting | 4822.0 | 1.187267 | 0.857861 | 0.0 | 1.00 | 1.0 | 2.00 | 3.0
MonthlyIncome | 4655.0 | 23619.853491 | 5380.698361 | 1000.0 | 20346.00 | 22347.0 | 25571.00 | 98678.0

In [5]:

    #This is not useful, so removing it
    df.drop(["CustomerID"], axis=1, inplace=True)

In [6]:

    #checking for missing values
    import missingno as mnso

    mnso.bar(df)

Out[6]:

    <AxesSubplot:>

In [7]:

    df.isnull().sum()

Out[7]:

    ProdTaken                     0
    Age                         226
    TypeofContact                25
    CityTier                      0
    DurationOfPitch             251
    Occupation                    0
    Gender                        0
    NumberOfPersonVisiting        0
    NumberOfFollowups            45
    ProductPitched                0
    PreferredPropertyStar        26
    MaritalStatus                 0
    NumberOfTrips               140
    Passport                      0
    PitchSatisfactionScore        0
    OwnCar                        0
    NumberOfChildrenVisiting     66
    Designation                   0
    MonthlyIncome               233
    dtype: int64

In [8]:

    for feature in df.columns:  # Loop through all columns in the dataframe
        if (
            df[feature].dtype == "object"
        ):  # Only apply for columns with categorical strings
            df[feature] = pd.Categorical(df[feature])
    df.head(10)

Out[8]:

| ProdTaken | Age | TypeofContact | CityTier | DurationOfPitch | Occupation | Gender | NumberOfPersonVisiting | NumberOfFollowups | ProductPitched | PreferredPropertyStar | MaritalStatus | NumberOfTrips | Passport | PitchSatisfactionScore | OwnCar | NumberOfChildrenVisiting | Designation | MonthlyIncome
--- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---
0 | 1 | 41.0 | Self Enquiry | 3 | 6.0 | Salaried | Female | 3 | 3.0 | Deluxe | 3.0 | Single | 1.0 | 1 | 2 | 1 | 0.0 | Manager | 20993.0
1 | 0 | 49.0 | Company Invited | 1 | 14.0 | Salaried | Male | 3 | 4.0 | Deluxe | 4.0 | Divorced | 2.0 | 0 | 3 | 1 | 2.0 | Manager | 20130.0
2 | 1 | 37.0 | Self Enquiry | 1 | 8.0 | Free Lancer | Male | 3 | 4.0 | Basic | 3.0 | Single | 7.0 | 1 | 3 | 0 | 0.0 | Executive | 17090.0
3 | 0 | 33.0 | Company Invited | 1 | 9.0 | Salaried | Female | 2 | 3.0 | Basic | 3.0 | Divorced | 2.0 | 1 | 5 | 1 | 1.0 | Executive | 17909.0
4 | 0 | NaN | Self Enquiry | 1 | 8.0 | Small Business | Male | 2 | 3.0 | Basic | 4.0 | Divorced | 1.0 | 0 | 5 | 1 | 0.0 | Executive | 18468.0
5 | 0 | 32.0 | Company Invited | 1 | 8.0 | Salaried | Male | 3 | 3.0 | Basic | 3.0 | Single | 1.0 | 0 | 5 | 1 | 1.0 | Executive | 18068.0
6 | 0 | 59.0 | Self Enquiry | 1 | 9.0 | Small Business | Female | 2 | 2.0 | Basic | 5.0 | Divorced | 5.0 | 1 | 2 | 1 | 1.0 | Executive | 17670.0
7 | 0 | 30.0 | Self Enquiry | 1 | 30.0 | Salaried | Male | 3 | 3.0 | Basic | 3.0 | Married | 2.0 | 0 | 2 | 0 | 1.0 | Executive | 17693.0
8 | 0 | 38.0 | Company Invited | 1 | 29.0 | Salaried | Male | 2 | 4.0 | Standard | 3.0 | Unmarried | 1.0 | 0 | 3 | 0 | 0.0 | Senior Manager | 24526.0
9 | 0 | 36.0 | Self Enquiry | 1 | 33.0 | Small Business | Male | 3 | 3.0 | Deluxe | 3.0 | Divorced | 7.0 | 0 | 3 | 1 | 0.0 | Manager | 20237.0

In [9]:

    df[df.duplicated()].count()

Out[9]:

    ProdTaken                   141
    Age                         131
    TypeofContact               141
    CityTier                    141
    DurationOfPitch             136
    Occupation                  141
    Gender                      141
    NumberOfPersonVisiting      141
    NumberOfFollowups           140
    ProductPitched              141
    PreferredPropertyStar       141
    MaritalStatus               141
    NumberOfTrips               139
    Passport                    141
    PitchSatisfactionScore      141
    OwnCar                      141
    NumberOfChildrenVisiting    135
    Designation                 141
    MonthlyIncome               132
    dtype: int64

In [10]:

    df.info()

    <class 'pandas.core.frame.DataFrame'>
    RangeIndex: 4888 entries, 0 to 4887
    Data columns (total 19 columns):
     #   Column                    Non-Null Count  Dtype
    ---  ------                    --------------  -----
     0   ProdTaken                 4888 non-null   int64
     1   Age                       4662 non-null   float64
     2   TypeofContact             4863 non-null   category
     3   CityTier                  4888 non-null   int64
     4   DurationOfPitch           4637 non-null   float64
     5   Occupation                4888 non-null   category
     6   Gender                    4888 non-null   category
     7   NumberOfPersonVisiting    4888 non-null   int64
     8   NumberOfFollowups         4843 non-null   float64
     9   ProductPitched            4888 non-null   category
     10  PreferredPropertyStar     4862 non-null   float64
     11  MaritalStatus             4888 non-null   category
     12  NumberOfTrips             4748 non-null   float64
     13  Passport                  4888 non-null   int64
     14  PitchSatisfactionScore    4888 non-null   int64
     15  OwnCar                    4888 non-null   int64
     16  NumberOfChildrenVisiting  4822 non-null   float64
     17  Designation               4888 non-null   category
     18  MonthlyIncome             4655 non-null   float64
    dtypes: category(6), float64(7), int64(6)
    memory usage: 526.3 KB

In [11]:

    # ProdTaken would be the dependant variable/variable to be predicted.

In [12]:

    df = df.drop_duplicates(keep = "last")

In [13]:

    df[df.duplicated()].count()

Out[13]:

    ProdTaken                   0
    Age                         0
    TypeofContact               0
    CityTier                    0
    DurationOfPitch             0
    Occupation                  0
    Gender                      0
    NumberOfPersonVisiting      0
    NumberOfFollowups           0
    ProductPitched              0
    PreferredPropertyStar       0
    MaritalStatus               0
    NumberOfTrips               0
    Passport                    0
    PitchSatisfactionScore      0
    OwnCar                      0
    NumberOfChildrenVisiting    0
    Designation                 0
    MonthlyIncome               0
    dtype: int64

In [14]:

    #After removing dups still about 22% of the data is true for ProdTaken that is good.
    df["ProdTaken"].value_counts()

Out[14]:

    0    3853
    1     894
    Name: ProdTaken, dtype: int64

In [15]:

    df.isnull().sum()

Out[15]:

    ProdTaken                     0
    Age                         216
    TypeofContact                25
    CityTier                      0
    DurationOfPitch             246
    Occupation                    0
    Gender                        0
    NumberOfPersonVisiting        0
    NumberOfFollowups            44
    ProductPitched                0
    PreferredPropertyStar        26
    MaritalStatus                 0
    NumberOfTrips               138
    Passport                      0
    PitchSatisfactionScore        0
    OwnCar                        0
    NumberOfChildrenVisiting     60
    Designation                   0
    MonthlyIncome               224
    dtype: int64

In [16]:

    #Lots of missing values lets treat them

In [17]:

    #Reseting index after removing dups
    df.reset_index(inplace=True, drop=True)

In [18]:

    #Treating missing Age. Mean and median very close, lets replace with median
    df["Age"].fillna(df.Age.median(), inplace=True)

In [19]:

    #Lets look at TypeofContact
    df["TypeofContact"].value_counts()

Out[19]:

    Self Enquiry       3350
    Company Invited    1372
    Name: TypeofContact, dtype: int64

In [20]:

    #25 missing values. Good distribution of Self enq and Company invited.Lets see if the 25 missing values contain any ProdTaken = 1
    # checking for missing values in the data.
    df[df.TypeofContact.isnull()]

Out[20]:

| ProdTaken | Age | TypeofContact | CityTier | DurationOfPitch | Occupation | Gender | NumberOfPersonVisiting | NumberOfFollowups | ProductPitched | PreferredPropertyStar | MaritalStatus | NumberOfTrips | Passport | PitchSatisfactionScore | OwnCar | NumberOfChildrenVisiting | Designation | MonthlyIncome
--- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---
209 | 0 | 31.0 | NaN | 1 | NaN | Small Business | Male | 2 | 5.0 | Deluxe | 3.0 | Divorced | 1.0 | 0 | 3 | 1 | 0.0 | Manager | NaN
531 | 0 | 26.0 | NaN | 1 | NaN | Salaried | Female | 3 | 5.0 | Basic | 3.0 | Married | 4.0 | 0 | 4 | 1 | 2.0 | Executive | NaN
532 | 0 | 29.0 | NaN | 1 | NaN | Small Business | Female | 3 | 3.0 | Deluxe | 3.0 | Divorced | 5.0 | 0 | 2 | 1 | 0.0 | Manager | NaN
536 | 0 | 27.0 | NaN | 3 | NaN | Small Business | Male | 2 | 3.0 | Deluxe | 3.0 | Divorced | 1.0 | 0 | 3 | 0 | 1.0 | Manager | NaN
539 | 0 | 34.0 | NaN | 1 | NaN | Small Business | Female | 2 | 4.0 | Basic | 5.0 | Single | 2.0 | 0 | 2 | 1 | 1.0 | Executive | NaN
557 | 1 | 28.0 | NaN | 1 | NaN | Small Business | Male | 2 | 3.0 | Basic | 3.0 | Single | 7.0 | 0 | 3 | 0 | 0.0 | Executive | NaN
581 | 0 | 32.0 | NaN | 3 | NaN | Salaried | Male | 3 | 3.0 | Deluxe | 3.0 | Married | 3.0 | 0 | 2 | 0 | 0.0 | Manager | NaN
672 | 0 | 24.0 | NaN | 1 | NaN | Small Business | Female | 2 | 4.0 | Deluxe | 3.0 | Married | 2.0 | 0 | 3 | 1 | 1.0 | Manager | NaN
778 | 0 | 26.0 | NaN | 1 | NaN | Small Business | Male | 2 | 1.0 | Basic | 3.0 | Divorced | 2.0 | 0 | 5 | 1 | 1.0 | Executive | NaN
951 | 1 | 25.0 | NaN | 3 | NaN | Salaried | Male | 3 | 4.0 | Basic | 5.0 | Divorced | 4.0 | 0 | 1 | 1 | 0.0 | Executive | NaN
977 | 0 | 33.0 | NaN | 3 | NaN | Small Business | Male | 2 | 3.0 | Deluxe | 5.0 | Divorced | 1.0 | 0 | 3 | 0 | 0.0 | Manager | NaN
1073 | 0 | 45.0 | NaN | 3 | NaN | Small Business | Male | 2 | 4.0 | Deluxe | 5.0 | Married | 2.0 | 0 | 3 | 0 | 0.0 | Manager | NaN
1112 | 0 | 36.0 | NaN | 1 | NaN | Small Business | Female | 2 | 4.0 | Deluxe | 3.0 | Married | 1.0 | 0 | 5 | 1 | 1.0 | Manager | NaN
1147 | 0 | 24.0 | NaN | 1 | NaN | Small Business | Male | 3 | 1.0 | Basic | 3.0 | Married | 2.0 | 0 | 1 | 0 | 0.0 | Executive | NaN
1286 | 0 | 41.0 | NaN | 3 | NaN | Small Business | Female | 2 | 3.0 | Deluxe | 4.0 | Married | 6.0 | 0 | 3 | 1 | 1.0 | Manager | NaN
1399 | 0 | 34.0 | NaN | 1 | NaN | Small Business | Male | 2 | 1.0 | Deluxe | 3.0 | Married | 3.0 | 0 | 3 | 0 | 1.0 | Manager | NaN
1624 | 0 | 31.0 | NaN | 1 | NaN | Small Business | Male | 2 | 5.0 | Deluxe | 3.0 | Married | 1.0 | 0 | 3 | 0 | 0.0 | Manager | NaN
1971 | 0 | 26.0 | NaN | 1 | NaN | Salaried | Female | 3 | 5.0 | Basic | 3.0 | Married | 4.0 | 0 | 4 | 1 | 0.0 | Executive | NaN
1972 | 0 | 29.0 | NaN | 1 | NaN | Small Business | Female | 3 | 3.0 | Deluxe | 3.0 | Married | 5.0 | 0 | 1 | 0 | 1.0 | Manager | NaN
1976 | 0 | 27.0 | NaN | 3 | NaN | Small Business | Male | 2 | 3.0 | Deluxe | 3.0 | Married | 1.0 | 0 | 3 | 1 | 1.0 | Manager | NaN
1979 | 0 | 34.0 | NaN | 1 | NaN | Small Business | Female | 2 | 4.0 | Basic | 5.0 | Single | 2.0 | 0 | 1 | 1 | 0.0 | Executive | NaN
1998 | 1 | 28.0 | NaN | 1 | NaN | Small Business | Male | 2 | 3.0 | Basic | 3.0 | Single | 7.0 | 0 | 3 | 1 | 1.0 | Executive | NaN
2022 | 0 | 32.0 | NaN | 3 | NaN | Salaried | Male | 3 | 3.0 | Deluxe | 3.0 | Married | 3.0 | 0 | 1 | 0 | 2.0 | Manager | NaN
2124 | 0 | 24.0 | NaN | 1 | NaN | Small Business | Female | 2 | 4.0 | Deluxe | 3.0 | Married | 2.0 | 0 | 3 | 0 | 0.0 | Manager | NaN
2243 | 0 | 26.0 | NaN | 1 | NaN | Small Business | Male | 2 | 1.0 | Basic | 3.0 | Married | 2.0 | 0 | 5 | 1 | 1.0 | Executive | NaN

In [21]:

    # There are some 1s. Lets create a new category for the missing values.
    df["TypeofContact"] = df["TypeofContact"].astype(str).replace("nan", "unknown")

In [22]:

    #Lets look at DurationOfPitch
    #mean and median are quite close so lets impute with median
    df.reset_index(inplace=True, drop=True)
    df["DurationOfPitch"].fillna(df.DurationOfPitch.median(), inplace=True)

In [23]:

    #Lets look at NumberOfFollowups
    #mean and median are quite close so lets impute with median
    df.reset_index(inplace=True, drop=True)
    df["NumberOfFollowups"].fillna(df.DurationOfPitch.median(), inplace=True)

In [24]:

    #Lets look at PreferredPropertyStar
    #mean and median are quite close so lets impute with median
    df.reset_index(inplace=True, drop=True)
    df["PreferredPropertyStar"].fillna(df.PreferredPropertyStar.median(), inplace=True)

In [25]:

    #Lets look at NumberOfTrips
    #mean and median are quite close so lets impute with median
    df.reset_index(inplace=True, drop=True)
    df["NumberOfTrips"].fillna(df.NumberOfTrips.median(), inplace=True)

In [26]:

    #Lets look at NumberOfChildrenVisiting
    #mean and median are quite close so lets impute with median
    df.reset_index(inplace=True, drop=True)
    df["NumberOfChildrenVisiting"].fillna(df.NumberOfChildrenVisiting.median(), inplace=True)

In [27]:

    #Lets look at MonthlyIncome
    #mean and median are quite close so lets impute with median
    df.reset_index(inplace=True, drop=True)
    df["MonthlyIncome"].fillna(df.MonthlyIncome.median(), inplace=True)

In [28]:

    #Tier of the city is odered 1>2>3. Lets convert this back to categories. Although ordered tiers can be useful adding arbitrary weights is probably not a good idea.
    #Could potentially leave it as is but making it a category makes sense here.
    df['C_Tier'] = 'Tier_' + str(df['CityTier'])

In [29]:

    #coverting object to category
    df['CityTier'] = pd.Categorical(df['CityTier'])

In [30]:

    df.info()

    <class 'pandas.core.frame.DataFrame'>
    RangeIndex: 4747 entries, 0 to 4746
    Data columns (total 20 columns):
     #   Column                    Non-Null Count  Dtype
    ---  ------                    --------------  -----
     0   ProdTaken                 4747 non-null   int64
     1   Age                       4747 non-null   float64
     2   TypeofContact             4747 non-null   object
     3   CityTier                  4747 non-null   category
     4   DurationOfPitch           4747 non-null   float64
     5   Occupation                4747 non-null   category
     6   Gender                    4747 non-null   category
     7   NumberOfPersonVisiting    4747 non-null   int64
     8   NumberOfFollowups         4747 non-null   float64
     9   ProductPitched            4747 non-null   category
     10  PreferredPropertyStar     4747 non-null   float64
     11  MaritalStatus             4747 non-null   category
     12  NumberOfTrips             4747 non-null   float64
     13  Passport                  4747 non-null   int64
     14  PitchSatisfactionScore    4747 non-null   int64
     15  OwnCar                    4747 non-null   int64
     16  NumberOfChildrenVisiting  4747 non-null   float64
     17  Designation               4747 non-null   category
     18  MonthlyIncome             4747 non-null   float64
     19  C_Tier                    4747 non-null   object
    dtypes: category(6), float64(7), int64(5), object(2)
    memory usage: 548.2+ KB

In [31]:

    df.isnull().sum()

Out[31]:

    ProdTaken                   0
    Age                         0
    TypeofContact               0
    CityTier                    0
    DurationOfPitch             0
    Occupation                  0
    Gender                      0
    NumberOfPersonVisiting      0
    NumberOfFollowups           0
    ProductPitched              0
    PreferredPropertyStar       0
    MaritalStatus               0
    NumberOfTrips               0
    Passport                    0
    PitchSatisfactionScore      0
    OwnCar                      0
    NumberOfChildrenVisiting    0
    Designation                 0
    MonthlyIncome               0
    C_Tier                      0
    dtype: int64

In [32]:

    #All missing values treated, dups removed.
    df.describe().T

Out[32]:

| count | mean | std | min | 25% | 50% | 75% | max
--- | --- | --- | --- | --- | --- | --- | --- | ---
ProdTaken | 4747.0 | 0.188329 | 0.391016 | 0.0 | 0.0 | 0.0 | 0.0 | 1.0
Age | 4747.0 | 37.513377 | 9.119956 | 18.0 | 31.0 | 36.0 | 43.0 | 61.0
DurationOfPitch | 4747.0 | 15.380872 | 8.330097 | 5.0 | 9.0 | 13.0 | 19.0 | 127.0
NumberOfPersonVisiting | 4747.0 | 2.911734 | 0.724040 | 1.0 | 2.0 | 3.0 | 3.0 | 5.0
NumberOfFollowups | 4747.0 | 3.791237 | 1.342215 | 1.0 | 3.0 | 4.0 | 4.0 | 13.0
PreferredPropertyStar | 4747.0 | 3.580156 | 0.799316 | 3.0 | 3.0 | 3.0 | 4.0 | 5.0
NumberOfTrips | 4747.0 | 3.226459 | 1.821210 | 1.0 | 2.0 | 3.0 | 4.0 | 22.0
Passport | 4747.0 | 0.289657 | 0.453651 | 0.0 | 0.0 | 0.0 | 1.0 | 1.0
PitchSatisfactionScore | 4747.0 | 3.051612 | 1.369584 | 1.0 | 2.0 | 3.0 | 4.0 | 5.0
OwnCar | 4747.0 | 0.617653 | 0.486012 | 0.0 | 0.0 | 1.0 | 1.0 | 1.0
NumberOfChildrenVisiting | 4747.0 | 1.191068 | 0.855278 | 0.0 | 1.0 | 1.0 | 2.0 | 3.0
MonthlyIncome | 4747.0 | 23541.308827 | 5264.002340 | 1000.0 | 20474.5 | 22311.0 | 25389.0 | 98678.0

In [33]:

    #Age min =18 , max =61 and median is 36
    #Duration of Pitch nin = 9, max = 127 and median is 13mins
    #number of  Person visting ranges from [1,5] with median

In [ ]:

In [34]:

    df["TypeofContact"].value_counts()

Out[34]:

    Self Enquiry       3350
    Company Invited    1372
    unknown              25
    Name: TypeofContact, dtype: int64

In [35]:

    df["Occupation"].value_counts()

Out[35]:

    Salaried          2293
    Small Business    2028
    Large Business     424
    Free Lancer          2
    Name: Occupation, dtype: int64

In [36]:

    df["Gender"].value_counts()

Out[36]:

    Male       2835
    Female     1769
    Fe Male     143
    Name: Gender, dtype: int64

In [37]:

    # Female and Fe Male are the same
    df['Gender'].replace('Fe Male','Female', inplace= True )

In [38]:

    df["ProductPitched"].value_counts()

Out[38]:

    Basic           1800
    Deluxe          1684
    Standard         714
    Super Deluxe     324
    King             225
    Name: ProductPitched, dtype: int64

In [39]:

    df["MaritalStatus"].value_counts()

Out[39]:

    Married      2279
    Divorced      950
    Single        875
    Unmarried     643
    Name: MaritalStatus, dtype: int64

In [40]:

    # Single and unmarried could be the same. this is a little confusing. but replacing umarried with single
    df["MaritalStatus"].replace('Unmarried','Single',inplace = True)
    df["MaritalStatus"].value_counts()

Out[40]:

    Married     2279
    Single      1518
    Divorced     950
    Name: MaritalStatus, dtype: int64

In [41]:

    df["Designation"].value_counts()

Out[41]:

    Executive         1800
    Manager           1684
    Senior Manager     714
    AVP                324
    VP                 225
    Name: Designation, dtype: int64

In [42]:

    !pip install sweetviz

    Requirement already satisfied: sweetviz in c:\users\dganapathi\anaconda3\lib\site-packages (2.1.4)
    Requirement already satisfied: matplotlib>=3.1.3 in c:\users\dganapathi\anaconda3\lib\site-packages (from sweetviz) (3.4.3)
    Requirement already satisfied: scipy>=1.3.2 in c:\users\dganapathi\anaconda3\lib\site-packages (from sweetviz) (1.7.1)
    Requirement already satisfied: pandas!=1.0.0,!=1.0.1,!=1.0.2,>=0.25.3 in c:\users\dganapathi\anaconda3\lib\site-packages (from sweetviz) (1.3.4)
    Requirement already satisfied: jinja2>=2.11.1 in c:\users\dganapathi\anaconda3\lib\site-packages (from sweetviz) (2.11.3)
    Requirement already satisfied: tqdm>=4.43.0 in c:\users\dganapathi\anaconda3\lib\site-packages (from sweetviz) (4.62.3)
    Requirement already satisfied: numpy>=1.16.0 in c:\users\dganapathi\anaconda3\lib\site-packages (from sweetviz) (1.20.3)
    Requirement already satisfied: importlib-resources>=1.2.0 in c:\users\dganapathi\anaconda3\lib\site-packages (from sweetviz) (5.10.1)
    Requirement already satisfied: zipp>=3.1.0 in c:\users\dganapathi\anaconda3\lib\site-packages (from importlib-resources>=1.2.0->sweetviz) (3.6.0)
    Requirement already satisfied: MarkupSafe>=0.23 in c:\users\dganapathi\anaconda3\lib\site-packages (from jinja2>=2.11.1->sweetviz) (1.1.1)
    Requirement already satisfied: pyparsing>=2.2.1 in c:\users\dganapathi\anaconda3\lib\site-packages (from matplotlib>=3.1.3->sweetviz) (3.0.4)
    Requirement already satisfied: pillow>=6.2.0 in c:\users\dganapathi\anaconda3\lib\site-packages (from matplotlib>=3.1.3->sweetviz) (8.4.0)
    Requirement already satisfied: python-dateutil>=2.7 in c:\users\dganapathi\anaconda3\lib\site-packages (from matplotlib>=3.1.3->sweetviz) (2.8.2)
    Requirement already satisfied: cycler>=0.10 in c:\users\dganapathi\anaconda3\lib\site-packages (from matplotlib>=3.1.3->sweetviz) (0.10.0)
    Requirement already satisfied: kiwisolver>=1.0.1 in c:\users\dganapathi\anaconda3\lib\site-packages (from matplotlib>=3.1.3->sweetviz) (1.3.1)
    Requirement already satisfied: six in c:\users\dganapathi\anaconda3\lib\site-packages (from cycler>=0.10->matplotlib>=3.1.3->sweetviz) (1.16.0)
    Requirement already satisfied: pytz>=2017.3 in c:\users\dganapathi\anaconda3\lib\site-packages (from pandas!=1.0.0,!=1.0.1,!=1.0.2,>=0.25.3->sweetviz) (2021.3)
    Requirement already satisfied: colorama in c:\users\dganapathi\anaconda3\lib\site-packages (from tqdm>=4.43.0->sweetviz) (0.4.4)

In [43]:

    #Trying out univariate analysis using sweetviz for all numeric features
    import sweetviz as sv

In [44]:

    analyze_report = sv.analyze(df[['Age','DurationOfPitch','NumberOfPersonVisiting','NumberOfFollowups','PreferredPropertyStar','NumberOfTrips','PitchSatisfactionScore','NumberOfChildrenVisiting','MonthlyIncome']],pairwise_analysis = 'on')

In [45]:

    analyze_report.show_notebook(w=None,
                                 h=None,
                                 scale = 0.8,
                                 layout='vertical',
                                 filepath = None
    )

In [46]:

    #Age as expected is almost normally distributed.
    #over 70% of the Duration of pitch data lies between 5 - 19 mins. 20%  between 20 -40 and 10 % over 40 mins
    #Number of persons visiting =3 for around 49% of the datapoints. 29% - 2 persons
    #Number of followups = 4 for 42% of the data and is 3 for nearly 30%
    #60% of the people prefered property star = 3 with 20% = 5 and 18% =4
    #66% of the customers had 1-4 trips. with 3-% of the people doing 2 trips per year.
    #30% customers rated the Sales Pitch score as 3, 20% ->1 , 19%-5 , 18% - 4
    #monthly income is betwween 1000-98678 . mean and median are close around 22-23k. so this is normalish

In [47]:

    # function to create labeled barplots
    # We'll call this function later.

    def labeled_barplot(data, feature, perc=False, n=None):
        """
        Barplot with percentage at the top

        data: dataframe
        feature: dataframe column
        perc: whether to display percentages instead of count (default is False)
        n: displays the top n category levels (default is None, i.e., display all levels)
        """
        total = len(data[feature])  # length of the column
        count = data[feature].nunique()
        if n is None:
            plt.figure(figsize=(count + 1, 5))
        else:
            plt.figure(figsize=(n + 1, 5))

        plt.xticks(rotation=90, fontsize=15)
        ax = sns.countplot(
            data=data,
            x=feature,
            palette="Paired",
            order=data[feature].value_counts().index[:n].sort_values(),
        )

        for p in ax.patches:
            if perc == True:
                label = "{:.1f}%".format(
                    100 * p.get_height() / total
                )  # percentage of each class of the category
            else:
                label = p.get_height()  # count of each level of the category

            x = p.get_x() + p.get_width() / 2  # width of the plot
            y = p.get_height()  # height of the plot

            ax.annotate(
                label,
                (x, y),
                ha="center",
                va="center",
                size=12,
                xytext=(0, 5),
                textcoords="offset points",
            )  # annotate the percentage

        plt.show()  # show the plot

In [48]:

    #Let's look at Gender
    labeled_barplot(df, "Gender", perc=True)
    #40.3 % are female customers and 59.7 are male customers

In [49]:

    labeled_barplot(df, "ProductPitched", perc=True)

In [50]:

    #Most Pitched Product is Basic followed by Delux. these two alone account for 72% of all pitches. next is Standard at 15%

In [51]:

    #How many customers own cars
    labeled_barplot(df, "OwnCar", perc=True)

In [52]:

    labeled_barplot(df, "Occupation", perc=True)
    #48% of the customers are salaried customers and 43% are small business and 9% larg business owners.

In [53]:

    labeled_barplot(df, "Designation", perc=True)
    #38% customers are executives and 35.5 are managers.

In [54]:

    labeled_barplot(df, "MaritalStatus", perc=True)
    #48% are married, Single and Divorced at 32% and 20% .Technically Single and Divorced are probably single?
    #Already took care of the unmarried and combined that with Single

In [55]:

    labeled_barplot(df, "CityTier", perc=True)

In [56]:

    #65% of the customers are from Tier 1 cities and 31% are from Tier 3. That leaves Tier 2 customers at a low 4%

In [57]:

    #Lets do some Bivariate Analysis
    sns.pairplot(data=df, hue="ProdTaken")
    plt.show()
    # Hard to read this but for completeness sake!

In [58]:

    plt.figure(figsize=(18, 7))
    sns.heatmap(df.corr(), annot=True, vmin=-1, vmax=1, cmap="Spectral")
    plt.show()

In [59]:

    #The NumberOfChildrenVisiting and the NumberOfPersonVisiting are  positively correlated 0.61 ->kinda obvious
    #The ProductTaken and the Passport have low positive correlation. 0.26 -> interesting!!
    #The Age and MonthlyIncome of the customers have moderate correlation 0.46.
    #There are other low correlations but nothing super insightful here

In [60]:

    # Now let's try to create a customer profilse based on product taken based on EDA

In [61]:

    sns.set_theme(style="whitegrid")

    plt.figure(figsize=(10,7))
    plt.title('Barplot: Maritial Status - product taken')
    plt.xlabel('Product Taken')
    sns.countplot(data=df, x='ProdTaken', hue='MaritalStatus');
    plt.show()

In [62]:

    # Single people buy the product the most and Divorced people the least.

In [63]:

    sns.set_theme(style="whitegrid")
    plt.figure(figsize=(10,7))
    plt.title('Barplot: Maritial Status - product pitched')
    plt.xlabel('Product Pitched')
    sns.countplot(data=df, x='ProductPitched', hue='MaritalStatus');
    plt.show()
    # Prod Pitch to Married folks is more than Single folks but Single folks buy more product than marriend folks.
    #

In [ ]:

In [64]:

    sns.set_theme(style="whitegrid")
    plt.figure(figsize=(10,7))
    plt.title('Barplot: Maritial Status - product pitched')
    plt.xlabel('Product Pitched')
    sns.countplot(data=df, x= 'ProdTaken',hue='ProductPitched');
    plt.show()
    #Super Deluxe is the most taken product.Deluxe is the next most taken product. Delux is the one that is most pitched though

In [65]:

    sns.set_theme(style="whitegrid")

    plt.figure(figsize=(10,7))
    plt.title('Barplot: ProductTaken - Type of contact')
    plt.xlabel('Product Pitched')
    sns.countplot(data=df, x='ProdTaken', hue='TypeofContact');
    plt.show()

In [66]:

    #Products taken as a result of Self Enquiry is almost twice as high as that by company invite but the ratio of success
    #( Total Prod taken/Total Prod taken + Total prod not taken) is higher for Company invited. So it is more successful but
    # less used.

In [67]:

    sns.set_theme(style="whitegrid")

    plt.figure(figsize=(10,7))
    plt.title('Barplot: Maritial Status - product taken')
    plt.xlabel('Product Taken')
    sns.countplot(data=df, x='ProdTaken', hue='Gender');
    plt.show()

In [68]:

    #Males buy the product more than females.

In [69]:

    sns.set_theme(style="whitegrid")

    plt.figure(figsize=(10,7))
    plt.title('Barplot: product taken-designation')
    plt.xlabel('Product Taken')
    sns.countplot(data=df, x='ProdTaken', hue='Designation');
    plt.show()

In [70]:

    # Executives are the biggest buyers of the product..

In [71]:

    sns.set_theme(style="whitegrid")

    plt.figure(figsize=(10,7))
    plt.title('Barplot: product taken-Occupation')
    plt.xlabel('Product Taken')
    sns.countplot(data=df, x='ProdTaken', hue='Occupation');
    plt.show()

In [72]:

    #Salaried and small business owners the most buyers of the product.

In [73]:

    sns.set_theme(style="whitegrid")

    plt.figure(figsize=(10,7))
    plt.title('Barplot: product taken-designation')
    plt.xlabel('Product Taken')
    sns.countplot(data=df, x='ProdTaken', hue='Passport');
    plt.show()

In [74]:

    # of the ones that don't buy the product most dont have passport but of the ones that do it is almost a 50-50 split

In [75]:

    sns.set_theme(style="whitegrid")

    plt.figure(figsize=(10,7))
    plt.title('Barplot: product taken-NumberOfFollowup')
    plt.xlabel('Product Taken')
    sns.countplot(data=df, x='ProdTaken', hue='NumberOfFollowups');
    plt.show()

In [76]:

    plt.figure(figsize=(7,7))
    plt.title('Scatter: product -taken - pitched - income')
    plt.xlabel('Product taken')
    plt.ylabel('Monthly Income')
    plt.xticks(rotation=90)
    sns.scatterplot(data=df, x='ProductPitched',y='MonthlyIncome', hue='ProdTaken');
    plt.show()

In [77]:

    #Basic customers income ranges from 15 to 40k. with some outliers
    #King customers are low income around 20k.Also King has few customers
    #Standard customers income range similar to Basic.
    #SuperDelux More skewed towards higher income 25k-40k

In [78]:

    plt.figure(figsize=(7,7))
    plt.title('Scatter: product -taken - pitched - income')
    plt.xlabel('Product Pitched')
    plt.ylabel('age')
    plt.xticks(rotation=90)
    sns.scatterplot(data=df, x='ProductPitched',y='Age', hue='ProdTaken');
    plt.show()

In [79]:

    #super Delux customers are mostly around ages 40-45.
    #King customers are over 55+
    #BAsic and Standard customers are spread ages18-60

In [131]:

    plt.figure(figsize=(7,7))
    plt.title('Scatter: product -taken - pitched - income')
    plt.xlabel('Product Pitched')
    plt.ylabel('age')
    plt.xticks(rotation=90)
    sns.scatterplot(data=df, x='ProductPitched',y='Age', hue='CityTier');
    plt.show()

In [80]:

    #There are a lot of features here We could keep going with this trying more combinations.

In [81]:

    #DATA Processing and FEATURE Eng:
    #We already dealt with missing values, cleaned the data and converted objects to categorical datatypes
    #We converted Tier to categorical variable as well, now will be a good time to encode the categorical variables

In [82]:

    # outlier detection using boxplot
    numeric_columns = df.select_dtypes(include=np.number).columns.tolist()

    plt.figure(figsize=(15, 12))

    for i, variable in enumerate(numeric_columns):
        plt.subplot(4, 4, i + 1)
        plt.boxplot(df[variable], whis=1.5)
        plt.tight_layout()
        plt.title(variable)

    plt.show()

In [83]:

    #There are outliers but they seem normal and possible. so not treating them

In [84]:

    df1 = df.copy()
    X = df.drop('ProdTaken',axis=1)     # Since this is target variable
    y = df['ProdTaken']   #
    #Lets encode County
    X = pd.get_dummies(
        X,
        columns=df.select_dtypes(include=["object", "category"]).columns.tolist(),
        drop_first=True,
    )

In [85]:

    #Let's do this with Stratify
    X_train, X_test, y_train, y_test = train_test_split(X, y, stratify= y,test_size=0.3, random_state=1)

    print("Original ProdTaken True Values    : {0} ({1:0.2f}%)".format(len(df.loc[df['ProdTaken'] == 1]), (len(df.loc[df['ProdTaken'] == 1])/len(df.index)) * 100))
    print("Original ProdTaken False Values   : {0} ({1:0.2f}%)".format(len(df.loc[df['ProdTaken'] == 0]), (len(df.loc[df['ProdTaken'] == 0])/len(df.index)) * 100))
    print("")
    print("Training ProdTaken True Values    : {0} ({1:0.2f}%)".format(len(y_train[y_train[:] == 1]), (len(y_train[y_train[:] == 1])/len(y_train)) * 100))
    print("Training ProdTaken False Values   : {0} ({1:0.2f}%)".format(len(y_train[y_train[:] == 0]), (len(y_train[y_train[:] == 0])/len(y_train)) * 100))
    print("")
    print("Test ProdTaken True Values        : {0} ({1:0.2f}%)".format(len(y_test[y_test[:] == 1]), (len(y_test[y_test[:] == 1])/len(y_test)) * 100))
    print("Test ProdTaken False Values       : {0} ({1:0.2f}%)".format(len(y_test[y_test[:] == 0]), (len(y_test[y_test[:] == 0])/len(y_test)) * 100))
    print("")

    Original ProdTaken True Values    : 894 (18.83%)
    Original ProdTaken False Values   : 3853 (81.17%)

    Training ProdTaken True Values    : 626 (18.84%)
    Training ProdTaken False Values   : 2696 (81.16%)

    Test ProdTaken True Values        : 268 (18.81%)
    Test ProdTaken False Values       : 1157 (81.19%)

In [86]:

    from sklearn import metrics
    from sklearn.tree import DecisionTreeClassifier

    from sklearn import tree

    from sklearn.model_selection import GridSearchCV

    # Libraries to import decision tree classifier and different ensemble classifiers
    from sklearn.ensemble import BaggingClassifier
    from sklearn.ensemble import RandomForestClassifier
    from sklearn.tree import DecisionTreeClassifier
    from sklearn import tree

    # Libtune to tune model, get different metric scores
    from sklearn import metrics
    from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, precision_score, recall_score,f1_score,roc_auc_score
    from sklearn.model_selection import GridSearchCV

    from sklearn.metrics import (
        f1_score,
        accuracy_score,
        recall_score,
        precision_score,
        confusion_matrix,
        plot_confusion_matrix,
        make_scorer,
    )

In [87]:

    #Lets try Bagging, Random forrest and decision trees.

In [88]:

    # defining a function to compute different metrics to check performance of a classification model built using sklearn
    def model_performance_classification_sklearn(model, predictors, target):
        """
        Function to compute different metrics to check classification model performance

        model: classifier
        predictors: independent variables
        target: dependent variable
        """

        # predicting using the independent variables
        pred = model.predict(predictors)

        acc = accuracy_score(target, pred)  # to compute Accuracy
        recall = recall_score(target, pred)  # to compute Recall
        precision = precision_score(target, pred)  # to compute Precision
        f1 = f1_score(target, pred)  # to compute F1-score

        # creating a dataframe of metrics
        df_perf = pd.DataFrame(
            {
                "Accuracy": acc,
                "Recall": recall,
                "Precision": precision,
                "F1": f1,
            },
            index=[0],
        )

        return df_perf
    def confusion_matrix_sklearn(model, predictors, target):
        """
        To plot the confusion_matrix with percentages

        model: classifier
        predictors: independent variables
        target: dependent variable
        """
        y_pred = model.predict(predictors)
        cm = confusion_matrix(target, y_pred)
        labels = np.asarray(
            [
                ["{0:0.0f}".format(item) + "\n{0:.2%}".format(item / cm.flatten().sum())]
                for item in cm.flatten()
            ]
        ).reshape(2, 2)

        plt.figure(figsize=(6, 4))
        sns.heatmap(cm, annot=labels, fmt="")
        plt.ylabel("True label")
        plt.xlabel("Predicted label")

In [89]:

    #Fitting the Decision Tree model
    d_tree = DecisionTreeClassifier(random_state=1)
    d_tree.fit(X_train,y_train)

    #Calculating different metrics
    dtree_model_train_perf=model_performance_classification_sklearn(d_tree,X_train,y_train)
    print("Training performance:\n",dtree_model_train_perf)
    dtree_model_test_perf=model_performance_classification_sklearn(d_tree,X_test,y_test)
    print("Testing performance:\n",dtree_model_test_perf)
    #Creating confusion matrix
    confusion_matrix_sklearn(d_tree, X_test, y_test)

    Training performance:
        Accuracy  Recall  Precision   F1
    0       1.0     1.0        1.0  1.0
    Testing performance:
        Accuracy    Recall  Precision        F1
    0  0.908772  0.787313   0.742958  0.764493

In [90]:

    #Fitting the RandomForrest model
    rf_estimator = RandomForestClassifier(random_state=1)
    rf_estimator.fit(X_train,y_train)

    #Calculating different metrics
    rf_estimator_model_train_perf=model_performance_classification_sklearn(rf_estimator,X_train,y_train)
    print("Training performance:\n",rf_estimator_model_train_perf)
    rf_estimator_model_test_perf=model_performance_classification_sklearn(rf_estimator,X_test,y_test)
    print("Testing performance:\n",rf_estimator_model_test_perf)

    #Creating confusion matrix
    confusion_matrix_sklearn(rf_estimator, X_test, y_test)

    Training performance:
        Accuracy  Recall  Precision   F1
    0       1.0     1.0        1.0  1.0
    Testing performance:
        Accuracy    Recall  Precision        F1
    0  0.907368  0.552239      0.925  0.691589

In [91]:

    #Fitting Bagging Classifier model
    bagging_classifier = BaggingClassifier(random_state=1)
    bagging_classifier.fit(X_train,y_train)

    #Calculating different metrics
    bagging_classifier_model_train_perf=model_performance_classification_sklearn(bagging_classifier,X_train,y_train)
    print("Training performance:\n",bagging_classifier_model_train_perf)
    bagging_classifier_model_test_perf=model_performance_classification_sklearn(bagging_classifier,X_test,y_test)
    print("Testing performance:\n",bagging_classifier_model_test_perf)
    #Creating confusion matrix
    confusion_matrix_sklearn(bagging_classifier, X_test, y_test)

    Training performance:
        Accuracy    Recall  Precision        F1
    0   0.99398  0.971246   0.996721  0.983819
    Testing performance:
        Accuracy   Recall  Precision        F1
    0  0.913684  0.61194   0.896175  0.727273

In [92]:

    #Since the dataset is imbalenced accuracy is not a good measure of how good the model predicts.
    #Recall is the parameter we should look to maximize here, since predicting False negetives is important. It makes sense
    #to increase resources such as follow ups/ pitchminutes etc on customers likely to buy product. the downside would just be
    #wasted resources like the time spent pitching and follow up
    #Decision Tree Recall - .78
    #Random Forrest Recall -.55
    #BAgging Recall - .61
    #Obviously there is overfitting in training data in all 3 models.

In [93]:

    #Lets tune the above three
    #Choose the type of classifier.
    dtree_estimator = DecisionTreeClassifier(class_weight={0:0.35,1:0.65},random_state=1)

    # Grid of parameters to choose from
    parameters = {'max_depth': np.arange(2,10),
                  'min_samples_leaf': [5, 7, 10, 15],
                  'max_leaf_nodes' : [2, 3, 5, 10,15],
                  'min_impurity_decrease': [0.0001,0.001,0.01,0.1]
                 }

    # Type of scoring used to compare parameter combinations
    scorer = metrics.make_scorer(metrics.recall_score)

    # Run the grid search
    grid_obj = GridSearchCV(dtree_estimator, parameters, scoring=scorer,n_jobs=-1)
    grid_obj = grid_obj.fit(X_train, y_train)

    # Set the clf to the best combination of parameters
    dtree_estimator = grid_obj.best_estimator_

    # Fit the best algorithm to the data.
    dtree_estimator.fit(X_train, y_train)

Out[93]:

    DecisionTreeClassifier(class_weight={0: 0.35, 1: 0.65}, max_depth=7,
                           max_leaf_nodes=15, min_impurity_decrease=0.0001,
                           min_samples_leaf=15, random_state=1)

In [94]:

    #Calculating different metrics
    dtree_estimator_model_train_perf=model_performance_classification_sklearn(dtree_estimator,X_train,y_train)
    print("Training performance:\n",dtree_estimator_model_train_perf)
    dtree_estimator_model_test_perf=model_performance_classification_sklearn(dtree_estimator,X_test,y_test)
    print("Testing performance:\n",dtree_estimator_model_test_perf)
    #Creating confusion matrix
    confusion_matrix_sklearn(dtree_estimator, X_test, y_test)

    Training performance:
        Accuracy    Recall  Precision        F1
    0  0.837748  0.583067   0.567652  0.575256
    Testing performance:
        Accuracy    Recall  Precision        F1
    0  0.828772  0.507463   0.548387  0.527132

In [95]:

    #Test recall has gone down with Hyper parameter tuning

In [98]:

    #Lets tune the randon forrest
    #Choose the type of classifier.
    rf_tuned = RandomForestClassifier(class_weight={0:0.35,1:0.65},random_state=1)

    parameters = {
                    'max_depth': list(np.arange(3,10,1)),
                    'max_features': np.arange(0.6,1.1,0.1),
                    'max_samples': np.arange(0.7,1.1,0.1),
                    'min_samples_split': np.arange(2, 20, 5),
                    'n_estimators': np.arange(30,100,20),
                    'min_impurity_decrease': [0.001,0.01,0.1]
    }

    # Type of scoring used to compare parameter combinations
    scorer = metrics.make_scorer(metrics.recall_score)

    # Run the grid search
    grid_obj = GridSearchCV(rf_tuned, parameters, scoring=scorer,cv=3,n_jobs=-1)
    grid_obj = grid_obj.fit(X_train, y_train)

    # Set the clf to the best combination of parameters
    rf_tuned = grid_obj.best_estimator_

    # Fit the best algorithm to the data.
    rf_tuned.fit(X_train, y_train)

    C:\Users\dganapathi\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py:922: UserWarning: One or more of the test scores are non-finite: [0.37378082 0.38174764 0.37532971 ...        nan        nan        nan]
      warnings.warn(

Out[98]:

    RandomForestClassifier(class_weight={0: 0.35, 1: 0.65}, max_depth=9,
                           max_features=0.9999999999999999,
                           max_samples=0.9999999999999999,
                           min_impurity_decrease=0.001, n_estimators=30,
                           random_state=1)

In [99]:

    #IT IS Taking for ever to run this around 45 minutes. I know i can take the error off by supressing the warnings
    # but leaving it here for completeness.

In [100]:

    #Calculating different metrics
    rf_tuned_model_train_perf=model_performance_classification_sklearn(rf_tuned,X_train,y_train)
    print("Training performance:\n",rf_tuned_model_train_perf)
    rf_tuned_model_test_perf=model_performance_classification_sklearn(rf_tuned,X_test,y_test)
    print("Testing performance:\n",rf_tuned_model_test_perf)

    #Creating confusion matrix
    confusion_matrix_sklearn(rf_tuned, X_test, y_test)

    Training performance:
        Accuracy    Recall  Precision        F1
    0  0.934979  0.744409    0.89272  0.811847
    Testing performance:
        Accuracy   Recall  Precision        F1
    0  0.891228  0.58209    0.78392  0.668094

In [101]:

    #Tuning the bagging classifier
    #Choose the type of classifier.
    bagging_estimator_tuned = BaggingClassifier(random_state=1)

    # Grid of parameters to choose from
    parameters = {'max_samples': [0.7,0.8,0.9,1],
                  'max_features': [0.7,0.8,0.9,1],
                  'n_estimators' : [10,20,30,40,50],
                 }

    # Type of scoring used to compare parameter combinations
    acc_scorer = metrics.make_scorer(metrics.recall_score)

    # Run the grid search
    grid_obj = GridSearchCV(bagging_estimator_tuned, parameters, scoring=acc_scorer,cv=3)
    grid_obj = grid_obj.fit(X_train, y_train)

    # Set the clf to the best combination of parameters
    bagging_estimator_tuned = grid_obj.best_estimator_

    # Fit the best algorithm to the data.
    bagging_estimator_tuned.fit(X_train, y_train)

Out[101]:

    BaggingClassifier(max_features=0.9, max_samples=0.9, n_estimators=50,
                      random_state=1)

In [102]:

    #Calculating different metrics
    bagging_estimator_tuned_model_train_perf=model_performance_classification_sklearn(bagging_estimator_tuned,X_train,y_train)
    print("Training performance:\n",bagging_estimator_tuned_model_train_perf)
    bagging_estimator_tuned_model_test_perf=model_performance_classification_sklearn(bagging_estimator_tuned,X_test,y_test)
    print("Testing performance:\n",bagging_estimator_tuned_model_test_perf)

    #Creating confusion matrix
    confusion_matrix_sklearn(bagging_estimator_tuned, X_test, y_test)

    Training performance:
        Accuracy    Recall  Precision       F1
    0  0.998495  0.992013        1.0  0.99599
    Testing performance:
        Accuracy    Recall  Precision       F1
    0  0.922105  0.660448   0.898477  0.76129

In [103]:

    # testing performance comparison

    models_test_comp_df = pd.concat(
        [dtree_model_test_perf.T,dtree_estimator_model_test_perf.T,rf_estimator_model_test_perf.T,rf_tuned_model_test_perf.T,
         bagging_classifier_model_test_perf.T, bagging_estimator_tuned_model_test_perf.T],
        axis=1,
    )
    models_test_comp_df.columns = [
        "Decision Tree",
        "Decision Tree Estimator",
        "Random Forest Estimator",
        "Random Forest Tuned",
        "Bagging Classifier",
        "Bagging Estimator Tuned"]
    print("Testing performance comparison:")
    models_test_comp_df

    Testing performance comparison:

Out[103]:

| Decision Tree | Decision Tree Estimator | Random Forest Estimator | Random Forest Tuned | Bagging Classifier | Bagging Estimator Tuned
--- | --- | --- | --- | --- | --- | ---
Accuracy | 0.908772 | 0.828772 | 0.907368 | 0.891228 | 0.913684 | 0.922105
Recall | 0.787313 | 0.507463 | 0.552239 | 0.582090 | 0.611940 | 0.660448
Precision | 0.742958 | 0.548387 | 0.925000 | 0.783920 | 0.896175 | 0.898477
F1 | 0.764493 | 0.527132 | 0.691589 | 0.668094 | 0.727273 | 0.761290

In [104]:

    feature_names = X_train.columns
    importances = dtree_estimator.feature_importances_
    indices = np.argsort(importances)

    plt.figure(figsize=(12,12))
    plt.title('Feature Importances')
    plt.barh(range(len(indices)), importances[indices], color='violet', align='center')
    plt.yticks(range(len(indices)), [feature_names[i] for i in indices])
    plt.xlabel('Relative Importance')
    plt.show()

In [ ]:

    #Recall is highest for the decision tree classifier at 0.78 but decision tree without pruning is notorious for overfitting to training data
    #The Tunned Bagging estimator has a good recall of 0.66 and precision of .89, giving it a good f1 score of .76. It is well balanced and not overfitted to training data
    #Features that are important for the decision tree are Owning a Passport, Designation-Executive,MaritalStatus - Single living in Tier 3 city and age.
    #Duration of Pitch, PreferredPropertyStar, PitchSatisfactionScore,NumberOfFollowup are important Control variables that can be used to influence the customers

In [ ]:

In [105]:

    from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier
    #To install xgboost library use - !pip install xgboost
    from xgboost import XGBClassifier

In [106]:

    ## Function to create confusion matrix
    def make_confusion_matrix(model,y_actual,labels=[1, 0]):
        '''
        model : classifier to predict values of X
        y_actual : ground truth

        '''
        y_predict = model.predict(X_test)
        cm=metrics.confusion_matrix( y_actual, y_predict, labels=[0, 1])
        df_cm = pd.DataFrame(cm, index = [i for i in ["Actual - No","Actual - Yes"]],
                      columns = [i for i in ['Predicted - No','Predicted - Yes']])
        group_counts = ["{0:0.0f}".format(value) for value in
                    cm.flatten()]
        group_percentages = ["{0:.2%}".format(value) for value in
                             cm.flatten()/np.sum(cm)]
        labels = [f"{v1}\n{v2}" for v1, v2 in
                  zip(group_counts,group_percentages)]
        labels = np.asarray(labels).reshape(2,2)
        plt.figure(figsize = (10,7))
        sns.heatmap(df_cm, annot=labels,fmt='')
        plt.ylabel('True label')
        plt.xlabel('Predicted label')

    ##  Function to calculate different metric scores of the model - Accuracy, Recall and Precision
    def get_metrics_score(model,flag=True):
        '''
        model : classifier to predict values of X

        '''
        # defining an empty list to store train and test results
        score_list=[]

        pred_train = model.predict(X_train)
        pred_test = model.predict(X_test)

        train_acc = model.score(X_train,y_train)
        test_acc = model.score(X_test,y_test)

        train_recall = metrics.recall_score(y_train,pred_train)
        test_recall = metrics.recall_score(y_test,pred_test)

        train_precision = metrics.precision_score(y_train,pred_train)
        test_precision = metrics.precision_score(y_test,pred_test)

        score_list.extend((train_acc,test_acc,train_recall,test_recall,train_precision,test_precision))

        # If the flag is set to True then only the following print statements will be dispayed. The default value is set to True.
        if flag == True:
            print("Accuracy on training set : ",model.score(X_train,y_train))
            print("Accuracy on test set : ",model.score(X_test,y_test))
            print("Recall on training set : ",metrics.recall_score(y_train,pred_train))
            print("Recall on test set : ",metrics.recall_score(y_test,pred_test))
            print("Precision on training set : ",metrics.precision_score(y_train,pred_train))
            print("Precision on test set : ",metrics.precision_score(y_test,pred_test))

        return score_list # returning the list with train and test scores

In [107]:

    # Lets create an Adaboost model with default parameters
    abc = AdaBoostClassifier(random_state=1)
    abc.fit(X_train,y_train)

    #Using above defined function to get accuracy, recall and precision on train and test set
    abc_score=get_metrics_score(abc)

    Accuracy on training set :  0.8509933774834437
    Accuracy on test set :  0.8428070175438597
    Recall on training set :  0.34824281150159747
    Recall on test set :  0.2947761194029851
    Precision on training set :  0.7147540983606557
    Precision on test set :  0.6929824561403509

In [108]:

    make_confusion_matrix(abc,y_test)

In [ ]:

    #Adaboost recall is really low on traingin and test but there is not much overfitting. Tunning could help

In [109]:

    # Lets create an Gradient Boosting Classifier with default parameters
    gbc = GradientBoostingClassifier(random_state=1)
    gbc.fit(X_train,y_train)

Out[109]:

    GradientBoostingClassifier(random_state=1)

In [ ]:

In [110]:

    xgb_score=get_metrics_score(gbc)
    make_confusion_matrix(gbc,y_test)

    Accuracy on training set :  0.8916315472606864
    Accuracy on test set :  0.8659649122807017
    Recall on training set :  0.49201277955271566
    Recall on test set :  0.39925373134328357
    Precision on training set :  0.88
    Precision on test set :  0.781021897810219

In [ ]:

    #Recall using Gradient Boosting is better than Adaboost

In [ ]:

In [111]:

    # XGBoost Classifier
    xgb = XGBClassifier(random_state=1,eval_metric='logloss')
    xgb.fit(X_train, y_train)

Out[111]:

    XGBClassifier(base_score=0.5, booster='gbtree', callbacks=None,
                  colsample_bylevel=1, colsample_bynode=1, colsample_bytree=1,
                  early_stopping_rounds=None, enable_categorical=False,
                  eval_metric='logloss', feature_types=None, gamma=0, gpu_id=-1,
                  grow_policy='depthwise', importance_type=None,
                  interaction_constraints='', learning_rate=0.300000012,
                  max_bin=256, max_cat_threshold=64, max_cat_to_onehot=4,
                  max_delta_step=0, max_depth=6, max_leaves=0, min_child_weight=1,
                  missing=nan, monotone_constraints='()', n_estimators=100,
                  n_jobs=0, num_parallel_tree=1, predictor='auto', random_state=1, ...)

In [112]:

    #Using above defined function to get accuracy, recall and precision on train and test set
    xgb_score=get_metrics_score(xgb)
    make_confusion_matrix(xgb,y_test)

    Accuracy on training set :  0.9996989765201686
    Accuracy on test set :  0.9228070175438596
    Recall on training set :  0.9984025559105432
    Recall on test set :  0.6716417910447762
    Precision on training set :  1.0
    Precision on test set :  0.8910891089108911

In [129]:

    #Recall of XGBoost model is much better->0.67

    importances = xgb.feature_importances_
    indices = np.argsort(importances)
    feature_names = list(X.columns)

    plt.figure(figsize=(12,12))
    plt.title('Feature Importances')
    plt.barh(range(len(indices)), importances[indices], color='violet', align='center')
    plt.yticks(range(len(indices)), [feature_names[i] for i in indices])
    plt.xlabel('Relative Importance')
    plt.show()

In [114]:

    #Hyper parameter tunning for the Ensemble models:
    # Choose the Adaboost classifier.
    abc_tuned = AdaBoostClassifier(random_state=1)

    # Grid of parameters to choose from
    ## add from article
    parameters = {
        #Let's try different max_depth for base_estimator
        "base_estimator":[DecisionTreeClassifier(max_depth=1, random_state=1),DecisionTreeClassifier(max_depth=2, random_state=1),DecisionTreeClassifier(max_depth=3, random_state=1)],
        "n_estimators": np.arange(10,110,10),
        "learning_rate":np.arange(0.1,2,0.1)
    }

    # Type of scoring used to compare parameter combinations
    acc_scorer = metrics.make_scorer(metrics.recall_score)

    # Run the grid search
    grid_obj = GridSearchCV(abc_tuned, parameters, scoring=acc_scorer,cv=3)
    grid_obj = grid_obj.fit(X_train, y_train)

    # Set the clf to the best combination of parameters
    abc_tuned = grid_obj.best_estimator_

    # Fit the best algorithm to the data.
    abc_tuned.fit(X_train, y_train)

Out[114]:

    AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3,
                                                             random_state=1),
                       learning_rate=0.7000000000000001, n_estimators=100,
                       random_state=1)

In [115]:

    #Using above defined function to get accuracy, recall and precision on train and test set
    abc_tuned_score=get_metrics_score(abc_tuned)
    make_confusion_matrix(abc_tuned,y_test)

    Accuracy on training set :  0.9861529199277543
    Accuracy on test set :  0.8863157894736842
    Recall on training set :  0.9361022364217252
    Recall on test set :  0.6044776119402985
    Precision on training set :  0.9898648648648649
    Precision on test set :  0.7431192660550459

In [ ]:

    #Tunned Adaboost still only does .60 recall

In [116]:

    importances = abc_tuned.feature_importances_
    indices = np.argsort(importances)
    feature_names = list(X.columns)

    plt.figure(figsize=(12,12))
    plt.title('Feature Importances')
    plt.barh(range(len(indices)), importances[indices], color='violet', align='center')
    plt.yticks(range(len(indices)), [feature_names[i] for i in indices])
    plt.xlabel('Relative Importance')
    plt.show()

In [ ]:

    ### Important features: Monthly Income>Age >Duration of Pitch > Pitch Satisfaction Score > Number of trips
    ##

In [117]:

    #Let us Try tunning the Gradient Boosted Classifier by using AdaBoost as the initial base classifier
    gbc_init = GradientBoostingClassifier(init=AdaBoostClassifier(random_state=1),random_state=1)
    gbc_init.fit(X_train,y_train)

    #Using above defined function to get accuracy, recall and precision on train and test set
    gbc_init_score=get_metrics_score(gbc_init)

    Accuracy on training set :  0.8922335942203492
    Accuracy on test set :  0.8694736842105263
    Recall on training set :  0.48722044728434505
    Recall on test set :  0.417910447761194
    Precision on training set :  0.8918128654970761
    Precision on test set :  0.7887323943661971

In [118]:

    # Choose the type of classifier.
    gbc_tuned = GradientBoostingClassifier(init=AdaBoostClassifier(random_state=1),random_state=1)

    # Grid of parameters to choose from
    ## add from article
    parameters = {
        "n_estimators": [100,150,200,250],
        "subsample":[0.8,0.9,1],
        "max_features":[0.7,0.8,0.9,1]
    }

    # Type of scoring used to compare parameter combinations
    acc_scorer = metrics.make_scorer(metrics.recall_score)

    # Run the grid search
    grid_obj = GridSearchCV(gbc_tuned, parameters, scoring=acc_scorer,cv=5)
    grid_obj = grid_obj.fit(X_train, y_train)

    # Set the clf to the best combination of parameters
    gbc_tuned = grid_obj.best_estimator_

    # Fit the best algorithm to the data.
    gbc_tuned.fit(X_train, y_train)

Out[118]:

    GradientBoostingClassifier(init=AdaBoostClassifier(random_state=1),
                               max_features=0.7, n_estimators=250, random_state=1,
                               subsample=0.9)

In [119]:

    #This means n_estimators greater than 250 could be bettter!!!! but it is taking forever to run this on my box. Leaving it
    #at this due to time constraint

In [120]:

    #Using above defined function to get accuracy, recall and precision on train and test set
    gbc_tuned_score=get_metrics_score(gbc_tuned)
    make_confusion_matrix(gbc_tuned,y_test)

    Accuracy on training set :  0.9229379891631547
    Accuracy on test set :  0.8849122807017544
    Recall on training set :  0.6437699680511182
    Recall on test set :  0.5111940298507462
    Precision on training set :  0.9243119266055045
    Precision on test set :  0.8058823529411765

In [ ]:

    #Tunned GBoosted classifier gives a .64 recall higher than tunned adaboost

In [121]:

    importances = gbc_tuned.feature_importances_
    indices = np.argsort(importances)
    feature_names = list(X.columns)

    plt.figure(figsize=(12,12))
    plt.title('Feature Importances')
    plt.barh(range(len(indices)), importances[indices], color='violet', align='center')
    plt.yticks(range(len(indices)), [feature_names[i] for i in indices])
    plt.xlabel('Relative Importance')
    plt.show()

In [ ]:

    #Importance : Monthly income > Passport >Age > Designation (Executive) >Duration of Pitch > Single

In [ ]:

    # Choose the type of classifier.
    xgb_tuned = XGBClassifier(random_state=1,eval_metric='logloss')

    # Grid of parameters to choose from
    ## add from
    parameters = {
        "n_estimators": np.arange(10,100,20),
        "scale_pos_weight":[0,1,2,5],
        "subsample":[0.5,0.7,0.9,1],
        "learning_rate":[0.01,0.1,0.2,0.05],
        "gamma":[0,1,3],
        "colsample_bytree":[0.5,0.7,0.9,1],
        "colsample_bylevel":[0.5,0.7,0.9,1]
    }

    # Type of scoring used to compare parameter combinations
    acc_scorer = metrics.make_scorer(metrics.recall_score)

    # Run the grid search
    grid_obj = GridSearchCV(xgb_tuned, parameters,scoring=acc_scorer,cv=3)
    grid_obj = grid_obj.fit(X_train, y_train)

    # Set the clf to the best combination of parameters
    xgb_tuned = grid_obj.best_estimator_

    # Fit the best algorithm to the data.
    xgb_tuned.fit(X_train, y_train)

In [ ]:

    #This is took Significantly longer to run on my box. Nearly 2.5 hours and it was still running

In [ ]:

    #Using above defined function to get accuracy, recall and precision on train and test set
    xgb_tuned_score=get_metrics_score(xgb_tuned)
    make_confusion_matrix(xgb_tuned,y_test)

In [ ]:

    importances = xgb_tuned.feature_importances_
    indices = np.argsort(importances)
    feature_names = list(X.columns)

    plt.figure(figsize=(12,12))
    plt.title('Feature Importances')
    plt.barh(range(len(indices)), importances[indices], color='violet', align='center')
    plt.yticks(range(len(indices)), [feature_names[i] for i in indices])
    plt.xlabel('Relative Importance')
    plt.show()

In [126]:

    # defining list of models
    models = [abc, abc_tuned, gbc, gbc_init, gbc_tuned, xgb]

    # defining empty lists to add train and test results
    acc_train = []
    acc_test = []
    recall_train = []
    recall_test = []
    precision_train = []
    precision_test = []

    # looping through all the models to get the accuracy, precall and precision scores
    for model in models:
        j = get_metrics_score(model,False)
        acc_train.append(np.round(j[0],2))
        acc_test.append(np.round(j[1],2))
        recall_train.append(np.round(j[2],2))
        recall_test.append(np.round(j[3],2))
        precision_train.append(np.round(j[4],2))
        precision_test.append(np.round(j[5],2))

In [127]:

    comparison_frame = pd.DataFrame({'Model':['AdaBoost with default paramters','AdaBoost Tuned',
                                              'Gradient Boosting with default parameters','Gradient Boosting with init=AdaBoost',
                                              'Gradient Boosting Tuned','XGBoost with default parameters'],
                                              'Train_Accuracy': acc_train,'Test_Accuracy': acc_test,
                                              'Train_Recall':recall_train,'Test_Recall':recall_test,
                                              'Train_Precision':precision_train,'Test_Precision':precision_test})
    comparison_frame

Out[127]:

| Model | Train_Accuracy | Test_Accuracy | Train_Recall | Test_Recall | Train_Precision | Test_Precision
--- | --- | --- | --- | --- | --- | --- | ---
0 | AdaBoost with default paramters | 0.85 | 0.84 | 0.35 | 0.29 | 0.71 | 0.69
1 | AdaBoost Tuned | 0.99 | 0.89 | 0.94 | 0.60 | 0.99 | 0.74
2 | Gradient Boosting with default parameters | 0.89 | 0.87 | 0.49 | 0.40 | 0.88 | 0.78
3 | Gradient Boosting with init=AdaBoost | 0.89 | 0.87 | 0.49 | 0.42 | 0.89 | 0.79
4 | Gradient Boosting Tuned | 0.92 | 0.88 | 0.64 | 0.51 | 0.92 | 0.81
5 | XGBoost with default parameters | 1.00 | 0.92 | 1.00 | 0.67 | 1.00 | 0.89

In [128]:

    #XGBOost with default parameters is 0.67 that is higher than all the other tunned models.
    #XGBoost with hyper-parameter tunning will likely increase the recall even more.

In [ ]:

    #Recommendations:

    #King customers are low income around 20k.Also King has few customers and King customers are over 55+
    #Standard customers income range similar to Basic and spread through a wide range of income
    #Basic > Deluxe >Standard > SuperDeluxe is the order of popularity
    #SuperDelux is preferred by higher income folks
    #super Delux customers are mostly around 40-45 age group.
    #BAsic and Standard customers are spread between ages 18-60
    #Overall Single people purchased the product
    #Overall more males purchased the product.
    #Important features: Designation-Executive ,Passport, Marital Status Single, Large business owner, City Tier 3
    # Product Pitched Super Deluxe, City Tier 2, Product Pitched Deluxe /Standard

    #Targeting Single Executives, Large Business owners living in tier 2/3 cities with more follow up calls + increased pitch
    #duration would yeild more customers. Pitching super Deluxe /Deluxe and Standard would be ideal.

    #Product seems to be less popular with Female folks. Investing some thought into how to make this popular for women would
    #help tapping into a new demographic.

    #Focussing on Tier 2 /3 cities would be a good idea.

    #Model for prediction:
    #XGBoost with Default parameters gives a good recall of 0.67 and likely that with hyper paramenter tunning it will significantly inccrease.
    #Decision tree model gives a good recall as well 0.78 -> likely to be improved by prunning.

In [ ]: