train_features_df = pd.read_csv('training_set_features.csv')
train_features_df['feat'] = 'Train'
train_features_df.head()

test_features_df = pd.read_csv('test_set_features.csv')
test_features_df['feat'] = 'Test'
test_features_df.head()

features_df = pd.concat([train_features_df, test_features_df])
features_df#.head()

features_df.columns

features_df.describe()

features_df.info()

labels_df = pd.read_csv('training_set_labels.csv')
labels_df[['h1n1_vaccine','seasonal_vaccine']] = labels_df[
    ['h1n1_vaccine','seasonal_vaccine']].astype('category')
labels_df.head()

labels_df.info()

labels_df['VaccineCategory'] = np.where(
    ((labels_df['h1n1_vaccine']==1)&(labels_df['seasonal_vaccine']==1)), "Both Vaccines",
    np.where(
        ((labels_df['h1n1_vaccine']==1)&(labels_df['seasonal_vaccine']==0)), "Seasonal Only",
        np.where(
            ((labels_df['h1n1_vaccine']==0)&(labels_df['seasonal_vaccine']==1)), "H1N1 Only",
            np.where(
                ((labels_df['h1n1_vaccine']==0)&(labels_df['seasonal_vaccine']==0)), "No Vaccine",
                "Unknown"
            )
        )
    )
)
df_vaccine_categ = labels_df.groupby('VaccineCategory')['respondent_id'].nunique().reset_index(
name='N').sort_values(by="N").reset_index(drop=True)
labels_vaccine_categ = df_vaccine_categ['VaccineCategory'].unique().tolist()
sizes_vaccine_categ = df_vaccine_categ['N'].tolist()
colors_vaccine_categ = ['#ADD8E6','#e57373','#81c784','#FFFF99']
plot_pie_chart(
    sizes_vaccine_categ, 
    labels=labels_vaccine_categ, 
    colors=colors_vaccine_categ,
    title = 'Respondent Distro by VaccineCategory'
)

h1n1_counts = labels_df.groupby('h1n1_vaccine')['respondent_id'].nunique()
seasonal_counts = labels_df.groupby('seasonal_vaccine')['respondent_id'].nunique()
h1n1_percent = (h1n1_counts / h1n1_counts.sum()) * 100
seasonal_percent = (seasonal_counts / seasonal_counts.sum()) * 100
fig, axes = plt.subplots(1, 2, figsize=(12, 5))
h1n1_counts.plot(kind='barh', ax=axes[0], color='skyblue')
for idx, value in enumerate(h1n1_counts):
    axes[0].text(value - 20, idx, f'{value} ({h1n1_percent[idx]:.0f}%)', va='center', ha='right', color='white')
axes[0].set_title('H1N1 Vaccine Counts')
axes[0].set_xlabel('')
axes[0].set_ylabel('')
axes[0].tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=False)
seasonal_counts.plot(kind='barh', ax=axes[1], color='salmon')
for idx, value in enumerate(seasonal_counts):
    axes[1].text(value - 20, idx, f'{value} ({seasonal_percent[idx]:.0f}%)', va='center', ha='right', color='white')
axes[1].set_title('Seasonal Vaccine Counts')
axes[1].set_xlabel('')
axes[1].set_ylabel('')
axes[1].tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=False)
plt.tight_layout()
plt.show()

# Create a contingency table
contingency_table = pd.crosstab(labels_df['h1n1_vaccine'], labels_df['seasonal_vaccine'])

# Perform Chi-Square test of independence
chi2, p, dof, expected = chi2_contingency(contingency_table)

# Print the results
print(f"Chi-Square Statistic: {chi2}")
print(f"P-value: {p}")
print(f"Degrees of Freedom: {dof}")
print("Expected Frequencies Table:")
print(expected)

# Interpretation
alpha = 0.05
if p < alpha:
    print("Reject the null hypothesis. The two variables are dependent.")
else:
    print("Fail to reject the null hypothesis. The two variables are independent.")

features_df[object_cols] = features_df[object_cols].apply(lambda x: x.map({1: 'True', 0: 'False', np.nan: 'No response'})).copy()
features_df[null_object_cols] = features_df[null_object_cols].fillna("No response").copy()
features_df.head()

int_object_cols = []
int_object_cols_medians = []
for each in features_df.columns.tolist():
    df_each = features_df[each]
    type_col = df_each.dtype
    if type_col == float or type_col == int and each not in ['respondent_id']:
        if df_each.isna().any():
            int_object_cols.append(each)
            median_col = df_each.median()
            int_object_cols_medians.append(median_col)
            
features_df_fin = features_df.copy()
for column, median in zip(int_object_cols, int_object_cols_medians):
    features_df_fin[column].fillna(median, inplace=True)
features_df_fin.head()

features_df_fin.info()

labels_df['Y'] = np.where(
    labels_df['VaccineCategory'] == "Both Vaccines", 1, 0
)
labels_df['One_vaccine_only'] = np.where(
    labels_df['VaccineCategory'].isin(["Seasonal Only","H1N1 Only"]), True, False
)
labels_df.head()

df_vaccine_categ2 = labels_df.groupby('Y')[
    'respondent_id'].nunique().reset_index(name='N').sort_values(by="N").reset_index(drop=True)
df_vaccine_categ2

final_df = pd.merge(
    features_df_fin, 
    labels_df[['respondent_id','Y']], 
    on=['respondent_id'], 
    how='left'
)
final_df['Y'] = final_df['Y'].astype(object)
final_df = final_df.drop(columns=['respondent_id'])
final_df.head()

final_df.info()

correlation_matrix_ = final_df.corr()

threshold = 0.70
corr_pairs = correlation_matrix_.abs().unstack().sort_values(kind="quicksort", ascending=False)
filtered_pairs = corr_pairs[(corr_pairs >= threshold) & (corr_pairs < 1)]
filtered_pairs

print("Highly correlated pairs:")
for index, value in filtered_pairs.items():
    print(f"{index[0]} and {index[1]}: {value}")

numeric_scaled = pd.DataFrame(preprocessing.scale(np.asarray(final_df[numerical_cols])))
numeric_scaled.columns = numerical_cols
numeric_scaled#.head()

dummied_df = pd.get_dummies(final_df[categorical_cols], drop_first=True)
dummied_df.head()

print(X.shape, Y.shape)

#LASSO PLOT, FULL
plt.figure(figsize=(14, 8))
ax = plt.gca()
ax.plot(unique_lambdas, save_avg_coef)
ax.set_xscale('log')
ax.set_xlim(ax.get_xlim()[::-1])
plt.xlabel('lambda')
plt.ylabel('weights')
plt.title('Lasso coefficients ALL')
plt.axis('tight')
plt.show()
pred_index = 38
display("PREDICTOR LAMBDA:", unique_lambdas[pred_index])

coeff_df = pd.DataFrame(columns=['ColName','Coeff'])
coeff_df['ColName'] = X_columns
coeff_df['Coeff'] = save_avg_coef[pred_index]
coeff_df['Magnitude'] = np.where(
    coeff_df['Coeff']<0, coeff_df['Coeff']*-1, coeff_df['Coeff']
)
coeff_df = coeff_df.sort_values(by='Magnitude',ascending=False)
coeff_df

chosen_predictors = coeff_df[:40].reset_index(drop=True)
chosen_predictors

positive_color = 'skyblue'
negative_color = 'salmon'
colors = [positive_color if coeff > 0 else negative_color for coeff in chosen_predictors['Coeff'].tolist()]
plt.figure(figsize=(13, 13))
bars = plt.barh(chosen_predictors['ColName'].tolist(), chosen_predictors['Coeff'].tolist(), color=colors)
plt.xlabel('Coefficient Value')
plt.ylabel('Feature')
plt.title('Coefficients of ML Model')
for bar, coeff in zip(bars,  chosen_predictors['Coeff'].tolist()):
    plt.text(bar.get_width() * 1.05, bar.get_y() + bar.get_height() / 2, f'{coeff:.2f}', ha='left', va='center')

plt.gca().invert_yaxis()
plt.show()

models = {
"Logit CV No Penalty": LogisticRegressionCV(random_state=randomstate, cv=cv, max_iter=5000),
"Logit CV L1": LogisticRegressionCV(penalty='l1', random_state=randomstate, cv=cv, solver='liblinear'),
"Logit CV L2": LogisticRegressionCV(penalty='l2', random_state=randomstate, cv=cv, solver='liblinear'),
"Logit CV ElasticNet": LogisticRegressionCV(penalty='elasticnet', random_state=randomstate, cv=cv, 
                                            solver='saga', l1_ratios=np.linspace(0.1,0.9,5), max_iter=5000),
"Logit No Penalty": LogisticRegression(random_state=randomstate, max_iter=5000),
"Logit L1": LogisticRegression(penalty='l1', solver='liblinear', random_state=randomstate),
"Logit L2": LogisticRegression(penalty='l2', solver='liblinear', random_state=randomstate),
# "Logit ElasticNet": LogisticRegression(penalty='elasticnet', solver='saga', random_state=randomstate, 
#                                        l1_ratios=np.linspace(0.1,0.9,5), max_iter=5000),
"SGD Classifier No Penalty": SGDClassifier(loss='log_loss', alpha=0.001, max_iter=1000, random_state=randomstate),
"SGD Classifier L1 Penalty": SGDClassifier(loss='log_loss', alpha=0.001, max_iter=1000, random_state=randomstate, penalty='l1'),
"SGD Classifier L2 Penalty": SGDClassifier(loss='log_loss', alpha=0.001, max_iter=1000, random_state=randomstate, penalty='l2'),
"SGD Classifier ElasticNet Penalty": SGDClassifier(
    loss='log_loss', alpha=0.0001, max_iter=1000, random_state=randomstate, penalty='elasticnet'),
"DecisionTree Classifier Entropy": DecisionTreeClassifier(criterion='entropy',min_samples_leaf=10, random_state=randomstate),
"DecisionTree Classifier Gini": DecisionTreeClassifier(criterion='gini',min_samples_leaf=10, random_state=randomstate),
"SVM RBF": svm.SVC(kernel='rbf', C=20, random_state=randomstate),
"SVM Poly": svm.SVC(kernel='poly', C=20, random_state=randomstate),
}
df_predictions = pd.DataFrame(columns=['Y','Yhat','Model'])
for each, model in models.items():
    print(f"Running {str(each)} model currently....")
    model.fit(X_train, y_train)
    predictions = model.predict(X_test)
    model_predictions = pd.DataFrame()
    model_predictions['Y'] = y_test.tolist()
    model_predictions['Yhat'] = predictions.tolist()
    model_predictions['Model'] = str(each)
    df_predictions = pd.concat([df_predictions,model_predictions])

# print(df_predictions)

df_predictions['correct_pred'] = np.where(
    df_predictions['Y']==df_predictions['Yhat'], 'Correct Prediction', 'Incorrect Prediction'
)
df_predictions

correct_predictions = df_predictions.groupby(['Model','correct_pred']).size().reset_index(name='counts').sort_values(
    by=['counts','Model'])
correct_predictions

grouped_df = correct_predictions.groupby(['correct_pred', 'Model'])['counts'].sum().unstack().reset_index()
for col in grouped_df.columns[1:]:
    grouped_df[col] = grouped_df[col] / grouped_df[col].sum()

# Reshape the DataFrame for sorting
stacked_df = grouped_df.set_index('correct_pred').stack().reset_index(name='percentage')
# Sort by percentage values
stacked_df = stacked_df.sort_values(by='percentage', ascending=False)

# Reconstruct the DataFrame to have 'correct_pred' as columns
grouped_df_sorted = stacked_df.pivot_table(index='correct_pred', columns='Model', values='percentage').reset_index()

fig, ax = plt.subplots(figsize=(10, 8))

category_values = grouped_df_sorted.columns[1:]
colors = {'Incorrect Prediction': '#FF6347', 'Correct Prediction': '#32CD32'}

bar_positions = [i for i in range(len(category_values))]

for i, cat1 in enumerate(grouped_df_sorted['correct_pred'].unique()):
    values = grouped_df_sorted[grouped_df_sorted['correct_pred'] == cat1][category_values].values[0]
    if i == 0:
        left = None
    else:
        left = grouped_df_sorted[grouped_df_sorted['correct_pred'].isin(grouped_df_sorted['correct_pred'].unique()[:i])][category_values].sum().tolist()
    bars = ax.barh(bar_positions, values, left=left, label=cat1, color=colors[cat1])
    for j, (bar, value) in enumerate(zip(bars, values)):
        ax.text(bar.get_width() / 2 + (left[j] if left else 0), bar.get_y() + bar.get_height() / 2,
                f'{value*100:.2f}%', ha='center', va='center', color='white')

ax.set_yticks(bar_positions)
ax.set_yticklabels(category_values)
ax.set_xlabel('% of Predictions')
ax.set_title('Correct VS Incorrect Predictions by model', pad=20)
ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), ncol=2)
plt.tight_layout()
plt.show()

df_metrics = pd.DataFrame()
for each in df_predictions['Model'].unique().tolist():
    model_name = each
    df_model = df_predictions[df_predictions['Model']==each]
    precision, recall, f1_score, _ = precision_recall_fscore_support(df_model['Y'].tolist(), 
                                                                     df_model['Yhat'].tolist())
    metrics_df = pd.DataFrame({
        'Precision': precision,
        'Recall': recall,
        'F1-score': f1_score
    })
    metrics_df.index.name = 'Category'
    metrics_df.reset_index(inplace=True)
    metrics_df['Model'] = each
    df_metrics = pd.concat([df_metrics,metrics_df])
    
df_metrics['Category'] = np.where(
    df_metrics['Category'] == 1, "Vaccinated", "Not Vaccinated"
)
df_metrics