score = 0

for col in rule_1_columns:

if row[col] in [2, 3]:

score += 1

for col in rule_2_columns:

if row[col] in [0, 1]:

score += 1

'asq_11', 'asq_12', 'asq_13', 'asq_14', 'asq_15', 'asq_16', 'asq_17', 'asq_18', 'asq_19',

'asq_20',

'asq_21', 'asq_22', 'asq_23', 'asq_24', 'asq_25', 'asq_26', 'asq_27', 'asq_28', 'asq_29',

'asq_30',

'asq_31', 'asq_32', 'asq_33', 'asq_34', 'asq_35', 'asq_36', 'asq_37', 'asq_38', 'asq_39',

'asq_40',

'asq_41', 'asq_42', 'asq_43', 'asq_44', 'asq_45', 'asq_46', 'asq_47', 'asq_48', 'asq_49',

score = 0

for col in rule_1_columns:

if row[col] in [2, 3, 4]:

score += 1

for col in rule_2_columns:

if row[col] in [3, 4]:

score += 1

df_famd.iloc[:, :num_components].to_numpy(),

columns=[f"FAMD_{i + 1}" for i in range(num_components)],

index=df_postpp.index

'PCA': df_pca_selected,

'FAMD': df_famd_80,

'FAMD_R': df_famd_80R,

'PCAMIX_R': df_pcamix_80R

linkage_methods = ['ward', 'average', 'complete', 'single']

fig, axs = plt.subplots(1, 2, figsize=(14, 6))

for linkage_method in linkage_methods:

silhouette_avgs = []

distortions = []

for k in range(2, 9):

clustering = AgglomerativeClustering(n_clusters=k, linkage=linkage_method)

clustering.fit(X)

labels = clustering.labels_

# Silhouette Score

silhouette_avg = silhouette_score(X, labels)

silhouette_avgs.append(silhouette_avg)

# Elbow Method

cluster_centers = np.array([X[labels == i].mean(axis=0) for i in range(k)])

dist = sum(

np.sum(cdist(X[labels == i], cluster_centers[i].reshape(1, -1), 'euclidean')) for i in range(k))

distortions.append(dist)

axs[0].plot(range(2, 9), silhouette_avgs, marker='o', label=f'{linkage_method}')

for x, y in zip(range(2, 9), silhouette_avgs):

axs[0].text(x, y, f'{y:.3f}', ha='center', va='bottom', fontsize=9)

axs[1].plot(range(2, 9), distortions, marker='o', label=f'{linkage_method}')

for x, y in zip(range(2, 9), distortions):

axs[1].text(x, y, f'{y:.0f}', ha='center', va='bottom', fontsize=9)

axs[0].set_xlabel('Number of clusters')

axs[0].set_ylabel('Silhouette Score')

axs[0].set_title(f'{tag} - Silhouette Score for Linkage Methods')

axs[1].set_xlabel('Number of clusters')

axs[1].set_ylabel('Sum of Intra-cluster Distances')

axs[1].set_title(f'{tag} - Elbow Method for Linkage Methods')

axs[0].legend(title="Linkage", loc='best')

axs[1].legend(title="Linkage", loc='best')

plt.tight_layout()

silhouette_avgs_ward = []

silhouette_avgs_kmedoids = []

distortions_ward = []

distortions_kmedoids = []

ward_labels = []

kmedoids_labels = []

for k in range(2, 9):

# Agglomerative clustering

model_ward = AgglomerativeClustering(n_clusters=k, linkage='ward')

model_ward.fit(X)

labels_ward = model_ward.labels_

silhouette_avg_ward = silhouette_score(X, labels_ward)

silhouette_avgs_ward.append(silhouette_avg_ward)

ward_labels.append(labels_ward)

# Distortion for Agglomerative

cluster_centers = np.array([X[labels_ward == i].mean(axis=0) for i in range(k)])

dist_ward = sum(

np.sum(cdist(X[labels_ward == i], cluster_centers[i].reshape(1, -1), 'euclidean')) for i in range(k))

distortions_ward.append(dist_ward)

# K-Medoids clustering

kmedoids = KMedoids(n_clusters=k, init='k-medoids++', metric='manhattan', max_iter=500, random_state=0)

kmedoids.fit(X)

labels_kmedoids = kmedoids.labels_

silhouette_avg_kmedoids = silhouette_score(X, labels_kmedoids)

silhouette_avgs_kmedoids.append(silhouette_avg_kmedoids)

kmedoids_labels.append(labels_kmedoids)

# Distortion for K-Medoids

medoid_centers = np.array([X[labels_kmedoids == i].mean(axis=0) for i in range(k)])

dist_kmedoids = sum(

np.sum(cdist(X[labels_kmedoids == i], medoid_centers[i].reshape(1, -1), 'euclidean')) for i in range(k))

distortions_kmedoids.append(dist_kmedoids)

if tag == 'PCA':

best_ward_labels = ward_labels[1]

best_kmedoids_labels = kmedoids_labels[1]

elif tag == 'FAMD':

best_ward_labels = ward_labels[3]

best_kmedoids_labels = kmedoids_labels[1]

elif tag == 'FAMD_R':

best_ward_labels = ward_labels[1]

best_kmedoids_labels = kmedoids_labels[2]

elif tag == 'PCAMIX_R':

best_ward_labels = ward_labels[1]

best_kmedoids_labels = kmedoids_labels[1]

# Add results to dataframe

df[f'{tag}_Agglomerative_Cluster'] = best_ward_labels

df[f'{tag}_KMedoids_Cluster'] = best_kmedoids_labels

# Plot results

fig, axs = plt.subplots(1, 2, figsize=(14, 6))

axs[0].plot(range(2, 9), silhouette_avgs_ward, marker='o', label='Agglomerative (Ward)', color='blue')

for x, y in zip(range(2, 9), silhouette_avgs_ward):

axs[0].text(x, y, f'{y:.3f}', ha='center', va='bottom', fontsize=9)

axs[0].plot(range(2, 9), silhouette_avgs_kmedoids, marker='o', label='K-Medoids', color='red')

for x, y in zip(range(2, 9), silhouette_avgs_kmedoids):

axs[0].text(x, y, f'{y:.3f}', ha='center', va='bottom', fontsize=9)

axs[1].plot(range(2, 9), distortions_ward, marker='o', label='Agglomerative (Ward)', color='blue')

for x, y in zip(range(2, 9), distortions_ward):

axs[1].text(x, y, f'{y:.0f}', ha='center', va='bottom', fontsize=9)

axs[1].plot(range(2, 9), distortions_kmedoids, marker='o', label='K-Medoids', color='red')

for x, y in zip(range(2, 9), distortions_kmedoids):

axs[1].text(x, y, f'{y:.0f}', ha='center', va='bottom', fontsize=9)

axs[0].set_xlabel('Number of clusters')

axs[0].set_ylabel('Silhouette Score')

axs[0].set_title(f'{tag} - Silhouette Score Comparison')

axs[0].legend()

axs[1].set_xlabel('Number of clusters')

axs[1].set_ylabel('Sum of Intra-cluster Distances')

axs[1].set_title(f'{tag} - Elbow Method Comparison')

axs[1].legend()

plt.tight_layout()

plt.show()

if tag in ['PCA', 'FAMD_R']:

return df, silhouette_avgs_ward, distortions_ward

elif tag in ['FAMD', 'PCAMIX_R']:

print(f"\nStatistical analysis for clustering '{target_col}':")

# Normality tests

normality_results = []

for var in numeric_vars:

data = df[var]

if data.nunique() > 1:

shapiro_stat, shapiro_p = shapiro(data)

ks_stat, ks_p = kstest(data, 'norm', args=(data.mean(), data.std()))

normality_results.append((var, shapiro_stat, shapiro_p, ks_stat, ks_p))

else:

print(f"Insufficient data or zero variance for Shapiro test on variable '{var}'")

# Store normality test results

normality_results_df = pd.DataFrame(normality_results,

columns=["Variable", "Shapiro_Statistic", "Shapiro_P", "KS_Statistic", "KS_P"])

# Identify variables that do not follow a normal distribution

non_normal_vars_df = normality_results_df[

(normality_results_df['Shapiro_P'] < 0.05) | (normality_results_df['KS_P'] < 0.05)]

print("\nVariables that do not follow a normal distribution:")

print(non_normal_vars_df)

if len(non_normal_vars_df) == len(numeric_vars):

print("\nAll the numerical variables are not normally distributed.")

# Kruskal-Wallis

kruskal_results = []

for var in numeric_vars:

data = df[[var, target_col]]

groups = [data[data[target_col] == group][var].values for group in data[target_col].unique()]

kruskal_stat, kruskal_p = kruskal(*groups)

kruskal_results.append((var, kruskal_stat, kruskal_p))

# Store Kruskal-Wallis results

kruskal_results_df = pd.DataFrame(

kruskal_results,

columns=["Variable", "Kruskal_Statistic", "Kruskal_P"]

)

# Identify significant variables with p-value < 0.05

significant_vars_df = kruskal_results_df[kruskal_results_df['Kruskal_P'] < 0.05]

print("\nSignificant variables after Kruskal-Wallis (p-value < 0.05):")

print(significant_vars_df)

# Mann-Whitney U Test for Pairwise Comparisons

mannwhitney_results = []

for var in significant_vars_df['Variable']:

data = df[[var, target_col]]

for (group1, group2) in combinations(data[target_col].unique(), 2):

group1_data = data[data[target_col] == group1][var].values

group2_data = data[data[target_col] == group2][var].values

mw_stat, mw_p = mannwhitneyu(group1_data, group2_data, alternative='two-sided')

mannwhitney_results.append((var, group1, group2, mw_stat, mw_p))

mannwhitney_results_df = pd.DataFrame(

mannwhitney_results,

columns=["Variable", "Group1", "Group2", "MW_Statistic", "MW_P"]

)

# Apply Bonferroni correction

mannwhitney_results_df['Bonferroni_P'] = multipletests(mannwhitney_results_df['MW_P'], method='bonferroni')[1]

significant_mw_results_df = mannwhitney_results_df[mannwhitney_results_df['Bonferroni_P'] < 0.05]

if len(significant_mw_results_df) > 0:

print("\nSignificant results of Mann-Whitney (Bonferroni corrected p-value < 0.05):")

print(significant_mw_results_df[['Variable', 'Group1', 'Group2', 'MW_Statistic', 'MW_P', 'Bonferroni_P']])

else:

print("\nNo significant Mann-Whitney results after Bonferroni correction.")

# Final Selection of Significant Variables

significant_mw_vars = significant_mw_results_df['Variable'].unique()

final_significant_vars = significant_vars_df[significant_vars_df['Variable'].isin(significant_mw_vars)]

print("\nVariabili finali significative da mantenere:")

print(f"\nAnalisi statistica per il clustering su '{target_col}':")

# Normality test

normality_results = []

for var in numeric_vars:

data = df[var]

if data.nunique() > 1:

shapiro_stat, shapiro_p = shapiro(data)

ks_stat, ks_p = kstest(data, 'norm', args=(data.mean(), data.std()))

normality_results.append((var, shapiro_stat, shapiro_p, ks_stat, ks_p))

else:

print(f"Insufficient data or zero variance for Shapiro test on variable '{var}'")

normality_results_df = pd.DataFrame(normality_results,

columns=["Variable", "Shapiro_Statistic", "Shapiro_P", "KS_Statistic", "KS_P"])

non_normal_vars_df = normality_results_df[

(normality_results_df['Shapiro_P'] < 0.05) | (normality_results_df['KS_P'] < 0.05)]

print("\nVariables that do not follow a normal distribution:")

print(non_normal_vars_df)

if len(non_normal_vars_df) == len(numeric_vars):

print("\nAll numeric variables are not distributed normally.")

# Kruskal-Wallis test

kruskal_results = []

for var in numeric_vars:

data = df[[var, target_col]]

groups = [data[data[target_col] == group][var].values for group in data[target_col].unique()]

kruskal_stat, kruskal_p = kruskal(*groups)

kruskal_results.append((var, kruskal_stat, kruskal_p))

kruskal_results_df = pd.DataFrame(

kruskal_results,

columns=["Variable", "Kruskal_Statistic", "Kruskal_P"]

)

significant_vars_df = kruskal_results_df[kruskal_results_df['Kruskal_P'] < 0.05]

print("\nVSignificant variables after Kruskal-Wallis (p-value < 0.05):")

print(significant_vars_df)

# Test di Mann-Whitney for significant variables

mannwhitney_results = []

for var in significant_vars_df['Variable']:

data = df[[var, target_col]]

for (group1, group2) in combinations(data[target_col].unique(), 2):

group1_data = data[data[target_col] == group1][var].values

group2_data = data[data[target_col] == group2][var].values

mw_stat, mw_p = mannwhitneyu(group1_data, group2_data, alternative='two-sided')

mannwhitney_results.append((var, group1, group2, mw_stat, mw_p))

mannwhitney_results_df = pd.DataFrame(

mannwhitney_results,

columns=["Variable", "Group1", "Group2", "MW_Statistic", "MW_P"]

)

# Bonferroni correction

mannwhitney_results_df['Bonferroni_P'] = multipletests(mannwhitney_results_df['MW_P'], method='bonferroni')[1]

significant_mw_results_df = mannwhitney_results_df[mannwhitney_results_df['Bonferroni_P'] < 0.05]

if len(significant_mw_results_df) > 0:

print("\nSignificant Mann-Whitney achievements (p-value Bonferroni corrected < 0.05):")

print(significant_mw_results_df[['Variable', 'Group1', 'Group2', 'MW_Statistic', 'MW_P', 'Bonferroni_P']])