Add files via upload

Author: SiruiJi

Corporate_Governance_Risk_Measure.py

@@ -0,0 +1,210 @@
+import pandas as pd
+import yfinance as yf
+import datetime as dt
+import matplotlib.pyplot as plt
+import statsmodels.api as sm
+import numpy as np
+from sklearn.covariance import EllipticEnvelope
+
+
+def load_raw_data(ticker, start_date, end_date):
+    crash_data = pd.DataFrame()
+    for i in ticker:
+        raw_data = yf.download(i, start_date, end_date)
+        crash_df = pd.DataFrame()
+        crash_df['RET'] = (raw_data['Adj Close'] / raw_data['Adj Close'].shift(1)) - 1
+        crash_df.index = raw_data.index
+        crash_df['BIDLO'] = raw_data['Low']
+        crash_df['ASKHI'] = raw_data['High']
+        crash_df['PRC'] = raw_data['Close']
+        crash_df['VOL'] = raw_data['Volume']
+        typical_price = (raw_data['High'] + raw_data['Low'] + raw_data['Close']) / 3
+        crash_df['VWAP'] = (typical_price * raw_data['Volume']).cumsum() / raw_data['Volume'].cumsum()
+        crash_df['vwretx'] = (crash_df['VWAP'] / crash_df['VWAP'].shift(1)) - 1
+        crash_df['TICKER'] = i
+        crash_df.dropna(inplace=True)
+        crash_data = pd.concat([crash_data, crash_df])
+
+    return crash_data
+
+
+def weekly_hist_gram(crash_data):
+    crash_dataw = crash_data.groupby('TICKER').resample('W').agg({'RET': 'mean', 'vwretx': 'mean', 'VOL': 'mean',
+                                                                  'BIDLO': 'mean', 'ASKHI': 'mean', 'PRC': 'mean'})
+    crash_dataw = crash_dataw.reset_index()
+    crash_dataw.dropna(inplace=True)
+    stocks = crash_data.TICKER.unique()
+    plt.figure(figsize=(12, 8))
+    k = 1
+    for i in stocks[:4]:
+        plt.subplot(2, 2, k)
+        plt.hist(crash_dataw[crash_dataw.TICKER == i]['RET'])
+        plt.title('Histogram of ' + i)
+        k += 1
+    plt.show()
+    return crash_dataw, stocks
+
+
+def firm_specific_weekly_return(crash_dataw, stocks):
+    residuals_dict = {}  # We will store residuals for each stock in this dictionary
+
+    for i in stocks:
+        Y = crash_dataw.loc[crash_dataw['TICKER'] == i]['RET'].values
+        X = crash_dataw.loc[crash_dataw['TICKER'] == i]['vwretx'].values
+        X = sm.add_constant(X)
+
+        X_transformed = X[2:-2] + X[1:-3] + X[0:-4] + X[3:-1] + X[4:]
+        ols = sm.OLS(Y[2:-2], X_transformed).fit()
+
+        residuals_stock = ols.resid
+        residuals_dict[i] = list(map(lambda x: np.log(1 + x), residuals_stock))
+
+    crash_data_sliced = pd.DataFrame([])
+    for i in stocks:
+        crash_data_sliced = pd.concat([crash_data_sliced, crash_dataw.loc[crash_dataw.TICKER == i][2:-2]],
+                                      ignore_index=True)
+    print(crash_data_sliced.head())
+
+    envelope = EllipticEnvelope(contamination=0.02, support_fraction=1)
+    ee_predictions = {}
+
+    for i in stocks:
+        stock_residuals = np.array(residuals_dict[i]).reshape(-1, 1)
+        if stock_residuals.shape[0] < 2:
+            print(f"Skipping stock {i} due to insufficient residuals.")
+            continue  # Skip the current iteration and move to the next stock
+        envelope.fit(stock_residuals)
+        ee_predictions[i] = envelope.predict(stock_residuals)
+
+    transform = []
+    for i in stocks:
+        if i in ee_predictions:  # Ensure we only process stocks that were not skipped
+            for j in range(len(ee_predictions[i])):
+                transform.append(np.where(ee_predictions[i][j] == 1, 0, -1))
+
+    crash_data_sliced = crash_data_sliced.reset_index()
+    crash_data_sliced['residuals'] = np.concatenate(list(residuals_dict.values()))
+    crash_data_sliced['neg_outliers'] = np.where((np.array(transform)) == -1, 1, 0)
+    crash_data_sliced.loc[(crash_data_sliced.neg_outliers == 1) & (crash_data_sliced.residuals > 0), 'neg_outliers'] = 0
+
+    plt.figure(figsize=(12, 8))
+    k = 1
+
+    for i in stocks[8:12]:
+        plt.subplot(2, 2, k)
+        crash_data_sliced['residuals'][crash_data_sliced.TICKER == i].hist(label='normal', bins=30, color='gray')
+        outliers = crash_data_sliced['residuals'][
+            (crash_data_sliced.TICKER == i) & (crash_data_sliced.neg_outliers > 0)]
+        outliers.hist(color='black', label='anomaly')
+        plt.title(i)
+        plt.legend()
+        k += 1
+    plt.show()
+
+    return crash_data_sliced
+
+
+def weekly_to_annual_data(crash_data_sliced, crash_data, crash_dataw):
+    crash_data_sliced = crash_data_sliced.set_index('Date')
+    crash_data_sliced.index = pd.to_datetime(crash_data_sliced.index)
+
+    std = crash_data.groupby('TICKER')['RET'].resample('W').std().reset_index()
+    crash_dataw['std'] = pd.DataFrame(std['RET'])
+
+    yearly_data = crash_data_sliced.groupby('TICKER').resample('Y')['residuals'].agg(['mean', 'std']).reset_index()
+    print(yearly_data.head())
+
+    merge_crash = pd.merge(crash_data_sliced.reset_index(), yearly_data, how='outer', on=['TICKER', 'Date'])
+    merge_crash[['annual_mean', 'annual_std']] = merge_crash.sort_values(by=['TICKER', 'Date']).iloc[:, -2:].fillna(
+        method='bfill')
+    merge_crash['residuals'] = merge_crash.sort_values(by=['TICKER', 'Date'])['residuals'].fillna(method='ffill')
+    merge_crash = merge_crash.drop(merge_crash.iloc[:, -4:-2], axis=1)
+
+    return merge_crash
+
+
+def crash_risk_measure(merge_crash, stocks):
+    crash_risk_out = []
+    for j in stocks:
+        for k in range(len(merge_crash[merge_crash.TICKER == j])):
+            if merge_crash[merge_crash.TICKER == j]['residuals'].iloc[k] < \
+                    merge_crash[merge_crash.TICKER == j]['annual_mean'].iloc[k] - \
+                    3.09 * merge_crash[merge_crash.TICKER == j]['annual_std'].iloc[k]:
+                crash_risk_out.append(1)
+            else:
+                crash_risk_out.append(0)
+    merge_crash['crash_risk'] = crash_risk_out
+    print(merge_crash['crash_risk'].value_counts())
+
+    merge_crash = merge_crash.set_index('Date')
+    merge_crash_annual = merge_crash.groupby('TICKER').resample('1Y')['crash_risk'].sum().reset_index()
+
+    down = []
+    for j in range(len(merge_crash)):
+        if merge_crash['residuals'].iloc[j] < merge_crash['annual_mean'].iloc[j]:
+            down.append(1)
+        else:
+            down.append(0)
+
+    merge_crash = merge_crash.reset_index()
+    merge_crash['down'] = pd.DataFrame(down)
+    merge_crash['up'] = 1 - merge_crash['down']
+    down_residuals = merge_crash[merge_crash.down == 1][['residuals', 'TICKER', 'Date']]
+    up_residuals = merge_crash[merge_crash.up == 1][['residuals', 'TICKER', 'Date']]
+
+    down_residuals['residuals_down_sq'] = down_residuals['residuals'] ** 2
+    down_residuals['residuals_down_cubic'] = down_residuals['residuals'] ** 3
+    up_residuals['residuals_up_sq'] = up_residuals['residuals'] ** 2
+    up_residuals['residuals_up_cubic'] = up_residuals['residuals'] ** 3
+    down_residuals['down_residuals'] = down_residuals['residuals']
+    up_residuals['up_residuals'] = up_residuals['residuals']
+    del down_residuals['residuals']
+    del up_residuals['residuals']
+    merge_crash['residuals_sq'] = merge_crash['residuals'] ** 2
+    merge_crash['residuals_cubic'] = merge_crash['residuals'] ** 3
+
+    merge_crash_all = merge_crash.merge(down_residuals, on=['TICKER', 'Date'], how='outer')
+    merge_crash_all = merge_crash_all.merge(up_residuals, on=['TICKER', 'Date'], how='outer')
+    cols = ['BIDLO', 'ASKHI', 'residuals', 'annual_std', 'residuals_sq', 'residuals_cubic', 'down', 'up',
+            'residuals_up_sq', 'residuals_down_sq', 'neg_outliers']
+    merge_crash_all = merge_crash_all.set_index('Date')
+    merge_grouped = merge_crash_all.groupby('TICKER')[cols].resample('1Y').sum().reset_index()
+    merge_grouped['neg_outliers'] = np.where(merge_grouped.neg_outliers >= 1, 1, 0)
+
+    merge_grouped = merge_grouped.set_index('Date')
+    merge_all = merge_grouped.groupby('TICKER').resample('1Y').agg({'down': ['sum', 'count'],
+                                                                    'up': ['sum', 'count']}).reset_index()
+    print(merge_all.head())
+
+    merge_grouped['down'] = merge_all['down']['sum'].values
+    merge_grouped['up'] = merge_all['up']['sum'].values
+    merge_grouped['count'] = merge_grouped['down'] + merge_grouped['up']
+
+    merge_grouped = merge_grouped.reset_index()
+    merge_grouped['duvol'] = np.log(((merge_grouped['up'] - 1) * merge_grouped['residuals_down_sq']) /
+                                    ((merge_grouped['down'] - 1) * merge_grouped['residuals_up_sq']))
+    print(merge_grouped.groupby('TICKER')['duvol'].mean())
+
+    merge_grouped['ncskew'] = - (((merge_grouped['count'] * (merge_grouped['count'] - 1) ** (3 / 2)) *
+                                  merge_grouped['residuals_cubic']) / (((merge_grouped['count'] - 1) *
+                                (merge_grouped['count'] - 2)) * merge_grouped['residuals_sq'] ** (3 / 2)))
+    print(merge_grouped.groupby('TICKER')['ncskew'].mean())
+
+    merge_grouped['crash_risk'] = merge_crash_annual['crash_risk']
+    merge_grouped['crash_risk'] = np.where(merge_grouped.crash_risk >= 1, 1, 0)
+    merge_crash_all_grouped2 = merge_crash_all.groupby('TICKER')[['VOL', 'PRC']].resample('1Y').mean().reset_index()
+    merge_grouped[['VOL', 'PRC']] = merge_crash_all_grouped2[['VOL', 'PRC']]
+    print(merge_grouped[['ncskew', 'duvol']].corr())
+    return merge_grouped
+
+
+if __name__ == '__main__':
+    ticker = ['ABBV', 'GOOGL', 'JNJ', 'DLTR', 'HLT', 'JPM', 'DEO', 'PG', 'ALB', 'BA', 'NVDA', 'LUV', 'PEP', 'TSM',
+              'SPY', '^VIX', 'GLD']
+    start_date = dt.datetime(2010, 1, 1)
+    end_date = dt.datetime(2023, 1, 1)
+    crash_data_ = load_raw_data(ticker, start_date, end_date)
+    crash_dataw_, stocks_ = weekly_hist_gram(crash_data_)
+    crash_data_sliced_ = firm_specific_weekly_return(crash_dataw_, stocks_)
+    merge_crash_ = weekly_to_annual_data(crash_data_sliced_, crash_data_, crash_dataw_)
+    merge_grouped_ = crash_risk_measure(merge_crash_, stocks_)

Credit_risk_PD_estimation_with_Deep_Learning.py

@@ -0,0 +1,97 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import StandardScaler
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import roc_auc_score, roc_curve
+from tensorflow import keras
+from keras.wrappers.scikit_learn import KerasClassifier
+from keras.layers import Dense, Dropout
+from sklearn.model_selection import GridSearchCV
+import tensorflow as tf
+import logging
+tf.get_logger().setLevel(logging.ERROR)
+
+def read_original_files(file_path):
+    credit = pd.read_csv(file_path)
+    print(credit.head())
+    del credit['Unnamed: 0']
+    return credit
+
+
+def data_conversion(credit):
+    print(credit.describe())
+    numerical_credit = credit.select_dtypes(include=[np.number])
+    '''obtain all numerical variables'''
+    plt.figure(figsize=(10, 8))
+    k = 0
+    cols = numerical_credit.columns
+    for i, j in zip(range(len(cols)), cols):
+        k += 1
+        plt.subplot(2, 2, k)
+        plt.hist(numerical_credit.iloc[:, i])
+        plt.title(j)
+    plt.show()
+
+    scaler = StandardScaler()
+    scaled_credit = scaler.fit_transform(numerical_credit)
+    scaled_credit = pd.DataFrame(scaled_credit, columns=numerical_credit.columns)
+
+    non_numerical_credit = credit.select_dtypes(include=['object'])
+    dummies_credit = pd.get_dummies(non_numerical_credit, drop_first=True)
+    dummies_credit = dummies_credit.astype(int)
+    print(dummies_credit.head())
+
+    combined_credit = pd.concat([scaled_credit, dummies_credit], axis=1)
+
+    return numerical_credit, scaled_credit, dummies_credit, combined_credit
+
+
+def data_preparation(combined_credit):
+    X = combined_credit.drop("Risk_good", axis=1)
+    y = combined_credit["Risk_good"]
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+    return X_train, X_test, y_train, y_test
+
+
+def DL_risk(dropout_rate, verbose=0):
+    model = keras.Sequential()
+    model.add(Dense(128,kernel_initializer='normal', activation='relu', input_dim=21))
+    model.add(Dense(64, kernel_initializer='normal', activation='relu'))
+    model.add(Dense(8, kernel_initializer='normal', activation='relu'))
+    model.add(Dropout(dropout_rate))
+    model.add(Dense(1, activation='sigmoid'))
+    model.compile(loss='binary_crossentropy', optimizer='rmsprop')
+    return model
+
+
+def training_model(X_train, X_test, y_train, y_test):
+    parameters = {'batch_size': [10, 50, 100],
+                  'epochs': [50, 100, 150],
+                  'dropout_rate': [0.2, 0.4]}
+    model = KerasClassifier(build_fn=DL_risk)
+    gs = GridSearchCV(estimator=model, param_grid=parameters, scoring='roc_auc', error_score='raise')
+
+    gs.fit(X_train, y_train, verbose=0)
+    print('Best hyperparameters for first cluster in DL are {}'.format(gs.best_params_))
+
+    model = KerasClassifier(build_fn=DL_risk,
+                            dropout_rate=gs.best_params_['dropout_rate'],
+                            verbose=0,
+                            batch_size=gs.best_params_['batch_size'],
+                            epochs=gs.best_params_['epochs'])
+    model.fit(X_train, y_train)
+    DL_predict = model.predict(X_test)
+    DL_ROC_AUC = roc_auc_score(y_test, pd.DataFrame(DL_predict.flatten()))
+    print('DL_ROC_AUC is {:.4f}'.format(DL_ROC_AUC))
+    return model, DL_predict
+
+
+if __name__ == '__main__':
+    file_path = 'D:/PyCharm Community Edition 2023.1.2/Python_Project/Finance/py4frm/german_credit_data.csv'
+    credit_ = read_original_files(file_path)
+    numerical_credit_, scaled_credit_, dummies_credit_, combined_credit_ = data_conversion(credit_)
+    X_train_, X_test_, y_train_, y_test_ = data_preparation(combined_credit_)
+    model_, DL_pred = training_model( X_train_, X_test_, y_train_, y_test_)

Credit_risk_PD_estimation_with_Neural_Network.py

@@ -0,0 +1,76 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import StandardScaler
+from sklearn.model_selection import train_test_split
+from sklearn.neural_network import MLPClassifier
+from sklearn.experimental import enable_halving_search_cv
+from sklearn.model_selection import HalvingRandomSearchCV
+from sklearn.metrics import roc_auc_score, roc_curve
+
+
+def read_original_files(file_path):
+    credit = pd.read_csv(file_path)
+    print(credit.head())
+    del credit['Unnamed: 0']
+    return credit
+
+
+def data_conversion(credit):
+    print(credit.describe())
+    numerical_credit = credit.select_dtypes(include=[np.number])
+    '''obtain all numerical variables'''
+    plt.figure(figsize=(10, 8))
+    k = 0
+    cols = numerical_credit.columns
+    for i, j in zip(range(len(cols)), cols):
+        k += 1
+        plt.subplot(2, 2, k)
+        plt.hist(numerical_credit.iloc[:, i])
+        plt.title(j)
+    plt.show()
+
+    scaler = StandardScaler()
+    scaled_credit = scaler.fit_transform(numerical_credit)
+    scaled_credit = pd.DataFrame(scaled_credit, columns=numerical_credit.columns)
+
+    non_numerical_credit = credit.select_dtypes(include=['object'])
+    dummies_credit = pd.get_dummies(non_numerical_credit, drop_first=True)
+    dummies_credit = dummies_credit.astype(int)
+    print(dummies_credit.head())
+
+    combined_credit = pd.concat([scaled_credit, dummies_credit], axis=1)
+
+    return numerical_credit, scaled_credit, dummies_credit, combined_credit
+
+
+def data_preparation(combined_credit):
+    X = combined_credit.drop("Risk_good", axis=1)
+    y = combined_credit["Risk_good"]
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+    return X_train, X_test, y_train, y_test
+
+
+def training_model(X_train, X_test, y_train, y_test):
+    param_NN ={"hidden_layer_sizes": [(100, 50), (50, 50), (10, 100)],
+               "solver": ["lbfgs", "sgd", "adam"],
+               "learning_rate_init": [0.001, 0.05]}
+    MLP = MLPClassifier(random_state=42)
+
+    param_halve_NN = HalvingRandomSearchCV(MLP, param_NN, scoring='roc_auc')
+    param_halve_NN.fit(X_train, y_train)
+
+    y_pred_NN = param_halve_NN.predict(X_test)
+    print('The ROC AUC score of RF is {:4f}'.format(roc_auc_score(y_test, y_pred_NN)))
+
+    return param_halve_NN, y_pred_NN
+
+
+if __name__ == '__main__':
+    file_path = 'D:/PyCharm Community Edition 2023.1.2/Python_Project/Finance/py4frm/german_credit_data.csv'
+    credit_ = read_original_files(file_path)
+    numerical_credit_, scaled_credit_, dummies_credit_, combined_credit_ = data_conversion(credit_)
+    X_train_, X_test_, y_train_, y_test_ = data_preparation(combined_credit_)
+    model, y_pred = training_model(X_train_, X_test_, y_train_, y_test_)