🔒 Need help cleaning up code

[mortal edge]

@south nymph

Python help channel opened

Remember to:

• Ask your Python question, not if you can ask or if there's an expert who can help.
• Show a code sample as text (rather than a screenshot) and the error message, if you've got one.
• Explain what you expect to happen and what actually happens.

:warning: Do not pip install anything that isn't related to your question, especially if asked to over DMs.

Closes after a period of inactivity, or when you send !close.

[south nymph]

lemme paste it all

import math
import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import scipy.stats as stats
import statsmodels.api as sm
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler, MaxAbsScaler, QuantileTransformer


# set the Z score. 
def pick_outliers(std, zs=3):
    outliers = (std.abs() > zs)
    return outliers

#Create standardized data
def standardize(df):
    scaler = StandardScaler()
    scaled = scaler.fit_transform(df)
    std = pd.DataFrame(scaled, columns=df.columns)
    return std

def log_transform(x, y):    
    non_zero_mask = y != 0
    x = x[non_zero_mask].reshape(-1, 1)
    y = y[non_zero_mask]
    y = np.log(y)
    return x, y

def std_transform(x, y):
    # Standardize log-transformed values
    scaler = StandardScaler()
    x = scaler.fit_transform(x)
    y = scaler.fit_transform(y.reshape(-1, 1)).flatten()
    return x, y

def get_data(data):
    ##data; original raw data
    ##d2; cleaned data; constraints & no outliers
    ##d3; scaled data of d2

    #SET UP df
    df = pd.DataFrame()
    
    
    for i, name in enumerate(data.columns):
        if not name.startswith("Relative"):
            df[name] = data[name]
    df["Relative.Spread"] = data["Relative.Spread"]
    df["Relative.Mortality"] = data["Relative.Mortality"]
    df["HC_perc"]*=100 ##make sure HC is percentages
    
    #USE ONLY NUMBER COLUMNS
    columns_to_standardize = df.columns[3:] #3 columns afterwards, number values appear
    std = standardize(df[columns_to_standardize]) #standardizes every single value EXCEPT the NaN values

    #MAKE Clean DF to remove outliers and negatives. Nulls will be removed much much later
    clean_df = df.copy(deep=True)

    chosen_outliers = pick_outliers(std, 7)
    clean_df.mask(chosen_outliers, inplace=True)
    
    
    for col in clean_df[columns_to_standardize]: #Constraints
        clean_df = clean_df[(clean_df[col] >= 0) | (clean_df[col].isna())] #remove negatives
    
    mapping_df = clean_df.copy(deep=True)
    
    clean_df.dropna(inplace=True) #removes any null values NOW
    
    #Remove county info from clean df
    clean_df = clean_df[columns_to_standardize]
    clean_std = standardize(clean_df) 

    return df, df.describe(), clean_df, clean_df.describe(), mapping_df, clean_std

#SHOW SCATTER PLOT OF CLEAN DATAFRAME
#Feature: X, Target: Y

#TEST IF LINEAR REGRESSION IS TRUE
#USING CLEAN STD

def show_distribution(df, feature, target, color="green"):
    # Calculate the PDF of the standard normal distribution
    
    x = df[feature].values.reshape(-1, 1)
    y = df[target].values
    
    x, y = log_transform(x, y)
    if color=="green":
        x, y = std_transform(x, y)
        
    x1 = sm.add_constant(x)
    stat_scores = sm.OLS(y, x1).fit()
    with open(f'log_txt/{target}_and_{feature}.txt', 'w') as file:
        file.write(f"{feature} vs {target}\n")
        file.write(str(stat_scores.summary()))
        file.close()


        
    #make linear regression
    model = LinearRegression()
    model.fit(x, y)
    y_fit = model.predict(x)
    #residuals

    correlation = np.corrcoef(x.flatten(), y)[0, 1]
    print(correlation)
    
    # Create the plot
    plt.figure(figsize=(10, 6))
    plt.scatter(x, y, label='Standard Normal Distribution', color=color)
    
    #Create best fit line
    #plt.plot(x, y_fit, color='red', label='Best Fit Line')
    coef = model.coef_[0]
    intercept = model.intercept_
    
    # Add title and labels
    if color=="green":
        plt.title(f'STANDARDIZED {feature} vs {target}\ny = {coef:.2f}x + {intercept:.2f}')
        plt.xlabel(f'{feature} Z-score')
        plt.ylabel(f'{target} Z-score')
        # Show the plot IF standardized
        plt.grid(True)
    else:
        plt.title(f'{feature} vs {target}')#\ny = {coef:.2f}x + {intercept:.2f}
        plt.xlabel(f'{feature}')
        plt.ylabel(f'{target}')
    
    # Add a legend
    plt.legend()
    
    plt.savefig(f"log/{feature}vs{target}_standardized.png")
    plt.show()



def show_heatmap(df):
    plt.figure(figsize=(13, 7))
    plt.title('Correlation Heatmap of the Data')
    sns.heatmap(df.corr(), annot=True, cmap='coolwarm', center=0)
    
    plt.savefig("correlation_heatmap.png")
    plt.show()

def explore_data(spread):
    
    dirty, dirprof, clean_df, cleprof, mapping, std = get_data(spread)
    
    clean_df.to_csv("cleaned_df.csv", index=False)
    
    target = "Relative.Mortality"
    #feature = "Education_perc"
    #for target in std.columns[-2:]:
    for feature in std.columns[:-2]:

        show_distribution(clean_df, feature, target, color="green")

#develop targets and features for regression model

    

# Fit the regression model


# Display the summary which includes t-statistics and p-values

            
        
    
    #show_heatmap(clean_df)
    
    #do RFE

spread = pd.read_csv("spread.csv")
explore_data(spread)

[shrewd kernel]

That's not what you're supposed to do...

!paste

[mortal edge]

Pasting large amounts of code

If your code is too long to fit in a codeblock in Discord, you can paste your code here:
https://paste.pythondiscord.com/

After pasting your code, save it by clicking the Paste! button in the bottom left, or by pressing CTRL + S. After doing that, you will be navigated to the new paste's page. Copy the URL and post it here so others can see it.

[mortal edge]

Python help channel closed

This help channel has been closed and it's no longer possible to send messages here. If your question wasn't answered, feel free to create a new post in #1035199133436354600. To maximize your chances of getting a response, check out this guide on asking good questions.