#🔒 Help with basic biostastics? Why are my categorical variables not being analyzed?
quartz flint
vague tulip
mport pandas as pd
from scipy.stats import chi2_contingency, fisher_exact, ttest_ind, mannwhitneyu
# Load the dataset
file_path = '/Users/dg/Desktop/AMCORT/Full Cohort_AMCort.csv'
data = pd.read_csv(file_path)
# Strip whitespace and convert to lowercase for column names
data.columns = data.columns.str.strip().str.lower()
# Ensure correct encoding of categorical variables
data['gender'] = data['gender'].map({'female': 1, 'male': 0})
data['dos is reoperation'] = data['dos is reoperation'].map({1: 1, 0: 0})
data['gtr (intraoperative)'] = data['gtr (intraoperative)'].map({1: 1, 0: 0})
data['visual loss on presentation'] = data['visual loss on presentation'].map({1: 1, 0: 0})
data['apoplexy on presentation'] = data['apoplexy on presentation'].map({1: 1, 0: 0})
# Define Knosp Group
data['knosp group'] = data['knosp score'].apply(lambda x: 'low invasive knosp' if x in [0, 1, 2] else 'high invasive knosp')
data['knosp group'] = data['knosp group'].map({'low invasive knosp': 0, 'high invasive knosp': 1})
# Define Consistency Group
data['consistency group'] = data['tumor consistency'].apply(lambda x: 'soft tumors' if x in [1, 2, 3] else 'hard tumors')
data['consistency group'] = data['consistency group'].map({'soft tumors': 0, 'hard tumors': 1})
# Ensure 'AI Resolved' is read as a numeric column and fill NaNs with a distinct value (e.g., -1) to exclude them
data['ai resolved'] = pd.to_numeric(data['ai resolved'], errors='coerce').fillna(-1)
# Define groups, ensuring we exclude blank cells
hp = data['developed ai'] == 1
nhp = data['developed ai'] == 0
# Define feature set
categorical_vars = ['gender', 'dos is reoperation', 'gtr (intraoperative)', 'visual loss on presentation', 'apoplexy on presentation', 'knosp group', 'consistency group']
quantitative_vars = ['age at surgery', 'bmi on dos', 'mib-1 (ki-67) labeling index (%)', 'hospital stay (days)', 'pre-op am cortisol', 'pod1 am cortisol level', 'pod2 am cortisol level']
# Define PA Subtype comparison (Nonfunctional (NFA) vs all others)
data['pa subtype_nfa'] = data['pa subtype'].apply(lambda x: 1 if x == 'Nonfunctional (NFA)' else 0)
data['pa subtype_others'] = data['pa subtype_nfa'].apply(lambda x: 1 if x == 0 else 0)
categorical_vars.append('pa subtype_nfa')
# Function to perform chi-square or Fisher exact test for categorical variables
def chi_square_or_fisher_test(data, group1, group2, var):
contingency_table = pd.crosstab(data[group1][var], data[group2][var])
if contingency_table.size == 0:
return None, "No data"
if contingency_table.shape[0] == 2 and contingency_table.shape[1] == 2:
_, p = fisher_exact(contingency_table)
else:
chi2, p, dof, expected = chi2_contingency(contingency_table)
return p, None
# Function to perform t-test or Mann-Whitney U test for quantitative variables
def t_test_or_mannwhitneyu(data, group1, group2, var):
group1_data = data[group1][var].dropna()
group2_data = data[group2][var].dropna()
if len(group1_data) < 2 or len(group2_data) < 2:
return None, "Not enough data"
if len(group1_data) + len(group2_data) < 30:
_, p_val = mannwhitneyu(group1_data, group2_data)
else:
_, p_val = ttest_ind(group1_data, group2_data)
return p_val, None
# Collect all results for HP vs NHP
all_results = []
comparison_name = "HP vs NHP"
for var in categorical_vars:
available_count_group1 = data[hp][var].notnull().sum()
missing_count_group1 = data[hp][var].isnull().sum()
available_count_group2 = data[nhp][var].notnull().sum()
missing_count_group2 = data[nhp][var].isnull().sum()
p_value, reason = chi_square_or_fisher_test(data, hp, nhp, var)
if reason:
significance = None
all_results.append([comparison_name, var, 'Chi-Square/Fisher Test', p_value, significance, available_count_group1, missing_count_group1, available_count_group2, missing_count_group2, reason])
else:
significance = p_value < 0.05
all_results.append([comparison_name, var, 'Chi-Square/Fisher Test', p_value, significance, available_count_group1, missing_count_group1, available_count_group2, missing_count_group2, ""])
for var in quantitative_vars:
available_count_group1 = data[hp][var].notnull().sum()
missing_count_group1 = data[hp][var].isnull().sum()
available_count_group2 = data[nhp][var].notnull().sum()
missing_count_group2 = data[nhp][var].isnull().sum()
p_value, reason = t_test_or_mannwhitneyu(data, hp, nhp, var)
if reason:
significance = None
all_results.append([comparison_name, var, 'T-Test/Mann-Whitney U', p_value, significance, available_count_group1, missing_count_group1, available_count_group2, missing_count_group2, reason])
else:
significance = p_value < 0.05
all_results.append([comparison_name, var, 'T-Test/Mann-Whitney U', p_value, significance, available_count_group1, missing_count_group1, available_count_group2, missing_count_group2, ""])
# Create a DataFrame for results
results_df = pd.DataFrame(all_results, columns=['Comparison', 'Variable', 'Test', 'P Value', 'Significance', 'Available Count Group 1', 'Missing Count Group 1', 'Available Count Group 2', 'Missing Count Group 2', 'Reason'])
# Save results to CSV
output_file_path = '/Users/dg/Desktop/AMCORT/univariate_analysis_results_HP_vs_NHP.csv'
results_df.to_csv(output_file_path, index=False)
my goal here is to see if my variables of choice are associated with patients breing placed in the HP group (diseased) vs NHP (nondiseased)
mossy pecan
You need a good rpc
