Skip to content

Update some chars and annotate some checked chars #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
TianXie1999 wants to merge 30 commits into
base: master
Choose a base branch
from
Open

Update some chars and annotate some checked chars #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
1529d86
Update accounting.py
TianXie1999 Jul 2, 2020
fc0e1db
Update accounting.py
TianXie1999 Jul 7, 2020
f659bcb
Merge remote-tracking branch 'upstream/master'
TianXie1999 Jul 8, 2020
abb8a27
Merge remote-tracking branch 'upstream/master'
TianXie1999 Jul 14, 2020
85f1783
Update accounting.py
TianXie1999 Jul 14, 2020
c1a615f
Update accounting.py
TianXie1999 Jul 20, 2020
7276644
Update chars
TianXie1999 Jul 21, 2020
477e37a
Update accounting.py
TianXie1999 Jul 22, 2020
8b3eead
Update another 4 investment chars
TianXie1999 Jul 23, 2020
db73ca9
update 2 chars
TianXie1999 Jul 23, 2020
62adbb9
update until ta except pta
TianXie1999 Jul 24, 2020
95601f8
fixed bugs
TianXie1999 Jul 27, 2020
1b435ab
adm and almq
TianXie1999 Jul 28, 2020
28c8a3b
ol,etr
TianXie1999 Jul 30, 2020
2f0d589
olq and seasonality
TianXie1999 Jul 31, 2020
7ced2be
rdm,rdmq,rds
TianXie1999 Aug 4, 2020
6d58f56
Create dtv.py
TianXie1999 Aug 10, 2020
fb510d0
create capm_mom
TianXie1999 Aug 28, 2020
6af4873
Update rvar_capm.py
TianXie1999 Aug 29, 2020
f09353c
Update rvar_capm_mom.py
TianXie1999 Aug 29, 2020
219f79c
Update rvar_capm_mom.py
TianXie1999 Sep 2, 2020
2a0f1fe
Update rmom_capm
TianXie1999 Sep 2, 2020
6db51ab
Update rmom_capm.py
TianXie1999 Sep 2, 2020
8728451
Create rmom_ff3.py
TianXie1999 Sep 8, 2020
2029821
Create dtv.py
TianXie1999 Sep 21, 2020
2473ad0
Update dtv.py
TianXie1999 Sep 26, 2020
8beac9b
Update friction chars
TianXie1999 Sep 27, 2020
be583a4
Update hxz_Ivq.py
TianXie1999 Sep 27, 2020
9ed0389
Create hxz_sv.py
TianXie1999 Sep 28, 2020
780873c
Update hxz_Ivq.py
TianXie1999 Oct 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 225 additions & 17 deletions pychars/accounting.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
from dateutil.relativedelta import *
from pandas.tseries.offsets import *
import pickle as pkl

###################
# Connect to WRDS #
###################
Expand Down Expand Up @@ -59,20 +58,20 @@ def ttm12(series, df):
f.ebit, f.nopi, f.spi, f.pi, f.txp, f.ni, f.txfed, f.txfo, f.txt, f.xint,

/*CF statement and others*/
f.capx, f.oancf, f.dvt, f.ob, f.gdwlia, f.gdwlip, f.gwo, f.mib, f.oiadp, f.ivao,
f.capx, f.oancf, f.dvt, f.ob, f.gdwlia, f.gdwlip, f.gwo, f.mib, f.oiadp, f.ivao, f.ivst,

/*assets*/
f.rect, f.act, f.che, f.ppegt, f.invt, f.at, f.aco, f.intan, f.ao, f.ppent, f.gdwl, f.fatb, f.fatl,

/*liabilities*/
f.lct, f.dlc, f.dltt, f.lt, f.dm, f.dcvt, f.cshrc,
f.dcpstk, f.pstk, f.ap, f.lco, f.lo, f.drc, f.drlt, f.txdi,
f.dcpstk, f.pstk, f.ap, f.lco, f.lo, f.drc, f.drlt, f.txdi, f.dltis, f.dltr, f.dlcch,

/*equity and other*/
f.ceq, f.scstkc, f.emp, f.csho, f.seq, f.txditc, f.pstkrv, f.pstkl, f.np, f.txdc, f.dpc, f.ajex,
f.ceq, f.scstkc, f.emp, f.csho, f.seq, f.txditc, f.pstkrv, f.pstkl, f.np, f.txdc, f.dpc, f.ajex, f.epspx,

/*market*/
abs(f.prcc_f) as prcc_f
abs(f.prcc_f) as prcc_f, abs(f.prcc_c) as prcc_c, f.dvc, f.prstkc, f.sstk, f.fopt, f.wcap, f.oancf

from comp.funda as f
left join comp.company as c
Expand Down Expand Up @@ -191,14 +190,15 @@ def ttm12(series, df):
ccm1 = pd.merge(comp, ccm, how='left', on=['gvkey'])

# we can only get the accounting data after the firm public their report
# for annual data, we ues 6 months lagged data
# for annual data, we use 6 months lagged data
ccm1['yearend'] = ccm1['datadate'] + YearEnd(0)
ccm1['jdate'] = ccm1['yearend'] + MonthEnd(6)

# set link date bounds
ccm2 = ccm1[(ccm1['jdate'] >= ccm1['linkdt']) & (ccm1['jdate'] <= ccm1['linkenddt'])]

# link comp and crsp
# data_rawa only includes annul data because comp is annual. Use inner merge
crsp2 = crsp2.rename(columns={'monthend': 'jdate'})
data_rawa = pd.merge(crsp2, ccm2, how='inner', on=['permno', 'jdate'])

Expand All @@ -210,8 +210,8 @@ def ttm12(series, df):
'''
Note: me is CRSP market equity, mve_f is Compustat market equity. Please choose the me below.
'''
# data_rawa['me'] = data_rawa['me']/1000 # CRSP ME
data_rawa['me'] = data_rawa['mve_f'] # Compustat ME
data_rawa['me'] = data_rawa['me']/1000 # CRSP ME
#data_rawa['me'] = data_rawa['mve_f'] # Compustat ME

# count single stock years
data_rawa['count'] = data_rawa.groupby(['gvkey']).cumcount()
Expand Down Expand Up @@ -270,10 +270,48 @@ def ttm12(series, df):
np.nan]
data_rawa['cfp_n'] = np.select(condlist, choicelist, default=data_rawa['ib']+data_rawa['dp'])

# ep
# ep, checked from Hou and change 'ME' from compustat to crsp，checked
data_rawa['ep'] = data_rawa['ib']/data_rawa['me']
data_rawa['ep_n'] = data_rawa['ib']

#ir
'''
First calculate r(t-5,t). Then rb(t-5,t) and use Bm to perform linear regression and get residue
'''
#r(t-5,t):sum ret from t-5 to t (which is calendar year t-6 to t-1)
lag = pd.DataFrame()
for i in range(1,6):
lag['ret%s' % i] = data_rawa.groupby(['permno'])['ret'].shift(i)

data_rawa['ret5'] = lag['ret1']+lag['ret2']+lag['ret3']+lag['ret4']+lag['ret5']

#bm_t-5 (bm of year t-5)
data_rawa['bm5'] = data_rawa.groupby(['permno'])['bm'].shift(5)

#rB (five year log book return)
#Reference: jf_06 page8 by KENT DANIEL
data_rawa['rB'] = data_rawa['bm'] - data_rawa['bm5'] + data_rawa['ret5']

#Regression and get ir
#First get unique datelist
datelist = data_rawa['jdate'].unique()
for date in datelist:
temp = data_rawa['jdate' == date]
n_row = temp.shape[0]
index = temp.index
X = pd.DataFrame()
X['bm5'] = temp['bm5']
X['rB'] = temp['rB']
X['intercept'] = 1
X = X[['intercept','rB','bm5']]
X = np.mat(X)
Y = np.mat(temp[['ret5']])
#These are residuals on one date
res = (np.identity(n_row) - X.dot(X.T.dot(X).I).dot(X.T)).dot(Y)
#put residuals back into data_rawa
data_rawa.loc[index,'ir'] = res


# ni
data_rawa['csho_l1'] = data_rawa.groupby(['permno'])['csho'].shift(1)
data_rawa['ajex_l1'] = data_rawa.groupby(['permno'])['ajex'].shift(1)
Expand All @@ -282,7 +320,7 @@ def ttm12(series, df):
np.log(data_rawa['csho']*data_rawa['ajex']).replace(-np.inf, 0)-
np.log(data_rawa['csho_l1']*data_rawa['ajex_l1']).replace(-np.inf, 0))

# op
# op: the formula seems different from Hou Page 74?
data_rawa['cogs0'] = np.where(data_rawa['cogs'].isnull(), 0, data_rawa['cogs'])
data_rawa['xint0'] = np.where(data_rawa['xint'].isnull(), 0, data_rawa['xint'])
data_rawa['xsga0'] = np.where(data_rawa['xsga'].isnull(), 0, data_rawa['xsga'])
Expand All @@ -292,6 +330,16 @@ def ttm12(series, df):
data_rawa['op'] = np.select(condlist, choicelist,
default=(data_rawa['revt'] - data_rawa['cogs0'] - data_rawa['xsga0'] - data_rawa['xint0'])/data_rawa['be'])

#nop
data_rawa['net_p'] = data_rawa['dvc'] + data_rawa['prstkc'] + 2*data_rawa['pstkrv'] - data_rawa['sstk']
data_rawa['nop'] = data_rawa['net_p'] / data_rawa['me']
data_rawa['nop'] = np.where(data_rawa['nop']<=0, np.nan, data_rawa['nop'] )

#ocp
data_rawa['ocy'] = np.where(data_rawa['jdate'] < '1988-06-30', data_rawa['fopt'] - data_rawa['wcap'], data_rawa['fopt'] - data_rawa['oancf'])
data_rawa['ocp'] = data_rawa['ocy'] / data_rawa['me']
data_rawa['ocp'] = np.where(data_rawa['ocp']<=0, np.nan, data_rawa['ocp'] )

# rsup
data_rawa['sale_l1'] = data_rawa.groupby(['permno'])['sale'].shift(1)
data_rawa['rsup'] = (data_rawa['sale']-data_rawa['sale_l1'])/data_rawa['me']
Expand All @@ -306,7 +354,7 @@ def ttm12(series, df):
# lev
data_rawa['lev'] = data_rawa['lt']/data_rawa['me']

# sp
# sp, checked
data_rawa['sp'] = data_rawa['sale']/data_rawa['me']
data_rawa['sp_n'] = data_rawa['sale']

Expand All @@ -316,7 +364,7 @@ def ttm12(series, df):
# rdm
data_rawa['rdm'] = data_rawa['xrd']/data_rawa['me']

# adm hxz adm
# adm hxz adm, checked
data_rawa['adm'] = data_rawa['xad']/data_rawa['me']

# gma
Expand Down Expand Up @@ -371,7 +419,7 @@ def ttm12(series, df):
# alm
# data_rawa['alm'] = data_rawa['ala']/(data_rawa['at']+data_rawa['prcc_f']*data_rawa['csho']-data_rawa['ceq'])

# noa
# noa,checked
data_rawa['noa'] = ((data_rawa['at']-data_rawa['che']-data_rawa['ivao'].fillna(0))-
(data_rawa['at']-data_rawa['dlc'].fillna(0)-data_rawa['dltt'].fillna(0)-data_rawa['mib'].fillna(0)
-data_rawa['pstk'].fillna(0)-data_rawa['ceq'])/data_rawa['at_l1'])
Expand Down Expand Up @@ -433,6 +481,75 @@ def ttm12(series, df):
# dy
data_rawa['dy'] = data_rawa['dvt']/data_rawa['me']

#aci
data_rawa['capx_s'] = data_rawa['capx']/data_rawa['sale']
data_rawa['capx3'] = data_rawa.groupby(['permno'])['capx_s'].shift(1) + data_rawa.groupby(['permno'])['capx_s'].shift(2) + data_rawa.groupby(['permno'])['capx_s'].shift(3)
data_rawa['capx3'] = data_rawa['capx3']/3
data_rawa['aci'] = data_rawa['capx_s']/data_rawa['capx3']

#cei
data_rawa['me_5'] = data_rawa.groupby('permno')['me'].shift(5)
data_rawa['cei'] = np.log(data_rawa['me']/data_rawa['me_5']) - data_rawa['ret5']

#dwc
data_rawa['dwc'] = (data_rawa['act'] - data_rawa['che']) - (data_rawa['lct'] - data_rawa['dlc'])
#data_rawa['dwc'] = data_rawa['dwc']/data_rawa['at_l1']

#I/A
data_rawa['ia'] = (data_rawa['at']/data_rawa['at_l1'])-1

#Ig
data_rawa['capx_l1'] = data_rawa.groupby('permno')['capx'].shift(1)
data_rawa['ig'] = data_rawa['capx']/data_rawa['capx_l1']

#2Ig
data_rawa['capx_l2'] = data_rawa.groupby('permno')['capx'].shift(2)
data_rawa['2ig'] = data_rawa['capx']/data_rawa['capx_l2']

#Ivc
data_rawa['atAvg'] = (data_rawa['at']+data_rawa['at_l1'])/2
data_rawa['ivc'] = data_rawa['invt'] / data_rawa['atAvg']

#Ndf
data_rawa['ndf'] = data_rawa['dltis'] - data_rawa['dltr'] + data_rawa['dlcch']

#nsi
data_rawa['sps'] = data_rawa['csho'] * data_rawa['ajex']
data_rawa['sps_l1'] = data_rawa.groupby('permno')['sps'].shift(1)
data_rawa['nsi'] = np.log(data_rawa['sps']/data_rawa['sps_l1'])

#oa
data_rawa['txp'] = np.where(data_rawa['txp'].isnull(), 0, data_rawa['txp'])
data_rawa['oa'] = (data_rawa['act'] - data_rawa['che']) - (data_rawa['lct'] - data_rawa['dlc'] - data_rawa['txp']) - data_rawa['dp']

#Poa
data_rawa['poa'] = data_rawa['oa']/data_rawa['ni']

#dNco
data_rawa['lct'] = np.where(data_rawa['lct'].isnull(), 0, data_rawa['lct'])
data_rawa['dltt'] = np.where(data_rawa['dltt'].isnull(), 0, data_rawa['dltt'])
data_rawa['ivao'] = np.where(data_rawa['ivao'].isnull(), 0, data_rawa['ivao'])
data_rawa['ivst'] = np.where(data_rawa['ivst'].isnull(), 0, data_rawa['ivst'])
data_rawa['pstk'] = np.where(data_rawa['pstk'].isnull(), 0, data_rawa['pstk'])
data_rawa['dnco'] = (data_rawa['at'] - data_rawa['ivao']) - (data_rawa['lt'] - data_rawa['dltt'])

#dFin
data_rawa['dfin'] = (data_rawa['ivst'] + data_rawa['ivao']) - (data_rawa['dltt'] + data_rawa['dlc'] + data_rawa['pstk'])

#Ta
data_rawa['ta'] = data_rawa['dwc'] + data_rawa['dnco'] + data_rawa['dfin']

#Ol
data_rawa['ol'] = (data_rawa['cogs'] + data_rawa['xsga'])/data_rawa['at']

#etr
data_rawa['txtpi'] = data_rawa['txt'] / data_rawa['pi']
data_rawa['txtpi_l1'] = data_rawa.groupby('permno')['txtpi'].shift(1)
data_rawa['txtpi_l2'] = data_rawa.groupby('permno')['txtpi'].shift(2)
data_rawa['txtpi_l3'] = data_rawa.groupby('permno')['txtpi'].shift(3)
data_rawa['deps'] = data_rawa['epspx']/(data_rawa['ajex'] * data_rawa['prcc_f'])
data_rawa['etr'] = (data_rawa['txtpi'] - (data_rawa['txtpi_l1'] + data_rawa['txtpi_l2'] + data_rawa['txtpi_l3'])/3) * data_rawa['deps']

# Annual Accounting Variables
chars_a = data_rawa[['cusip', 'ncusip', 'gvkey', 'permno', 'exchcd', 'shrcd', 'datadate', 'jdate', 'count',
'sic', 'acc', 'agr', 'bm', 'cfp', 'ep', 'ni', 'op', 'rsup', 'cash', 'chcsho',
Expand Down Expand Up @@ -502,8 +619,8 @@ def ttm12(series, df):
'''
Note: me is CRSP market equity, mveq_f is Compustat market equity. Please choose the me below.
'''
# data_rawq['me'] = data_rawq['me']/1000 # CRSP ME
data_rawq['me'] = data_rawq['mveq_f'] # Compustat ME
data_rawq['me'] = data_rawq['me']/1000 # CRSP ME
#data_rawq['me'] = data_rawq['mveq_f'] # Compustat ME

# count single stock years
data_rawq['count'] = data_rawq.groupby(['gvkey']).cumcount()
Expand Down Expand Up @@ -572,7 +689,7 @@ def ttm12(series, df):
(ttm4('ibq', data_rawq)+ttm4('dpq', data_rawq))/data_rawq['me'])
data_rawq['cfp_n'] = data_rawq['cfp']*data_rawq['me']

# ep
# ep, also checked and change 'ME' from compustat to crsp
data_rawq['ep'] = ttm4('ibq', data_rawq)/data_rawq['me']
data_rawq['ep_n'] = data_rawq['ep']*data_rawq['me']

Expand Down Expand Up @@ -630,6 +747,9 @@ def ttm12(series, df):
# rdm
data_rawq['rdm'] = data_rawq['xrdq4']/data_rawq['me']

# rds
data_rawq['rds'] = data_rawq['xrdq4']/data_rawq['saleq']

# sgr
data_rawq['saleq4'] = ttm4('saleq', data_rawq)
data_rawq['saleq4'] = np.where(data_rawq['saleq4'].isnull(), data_rawq['saley'], data_rawq['saleq4'])
Expand Down Expand Up @@ -714,6 +834,21 @@ def ttm12(series, df):
# ato
data_rawq['ato'] = data_rawq['saleq']/data_rawq['noa_l4']

#Iaq
data_rawq['atqlag'] = ttm4('atq',data_rawq)
data_rawq['iaq'] = (data_rawq['atq']/data_rawq['atqlag'])-1

#Almq
data_rawq['intanq'] = np.where(data_rawq['intanq'].isnull(), 0, data_rawq['intanq'])
data_rawq['qal'] = data_rawq['cheq'] + 0.75*(data_rawq['actq']-data_rawq['cheq']) + 0.5*(data_rawq['atq'] - data_rawq['actq'] - data_rawq['intanq'])
data_rawq['mveqa'] = data_rawq['atq'] + data_rawq['mveq_f'] - data_rawq['ceqq']
data_rawq['mveqa_1'] = data_rawq.groupby(['permno'])['mveqa'].shift(1)
data_rawq['almq'] = data_rawq['qal']/data_rawq['mveqa_1']

#Olq
data_rawa['olq'] = (data_rawa['cogsq'] + data_rawa['xsgaq'])/data_rawa['atq']


# Quarterly Accounting Variables
chars_q = data_rawq[['gvkey', 'permno', 'datadate', 'jdate', 'sic', 'exchcd', 'shrcd', 'acc', 'bm', 'cfp',
'ep', 'agr', 'ni', 'op', 'cash', 'chcsho', 'rd', 'cashdebt', 'pctacc', 'gma', 'lev',
Expand All @@ -734,6 +869,77 @@ def ttm12(series, df):
crsp_mom['jdate'] = pd.to_datetime(crsp_mom['date']) + MonthEnd(0)
crsp_mom = crsp_mom.dropna()

#Seasonality

#Rla
crsp_mom['rla'] = crsp_mom.groupby['permno']['ret'].shift(12)

#Rln
lag = pd.DataFrame()
result = 0
for i in range(1, 12):
lag['mom%s' % i] = crsp_mom.groupby(['permno'])['ret'].shift(i)
result = result + lag['mom%s' % i]
crsp_mom['rln'] = result/11

#R[2,5]a
#R[2,5]n
lag = pd.DataFrame()
result = 0
for i in range(13,61):
lag['mom%s' % i] = crsp_mom.groupby(['permno'])['ret'].shift(i)
if i not in [24,36,48,60]:
result = result + lag['mom%s' % i]

crsp_mom['r25a'] = (lag['mom24']+lag['mom36']+lag['mom48']+lag['mom60'])/4
crsp_mom['r25n'] = result/44

#R[6,10]a
#R[6,10]n
lag = pd.DataFrame()
result = 0
for i in range(61,121):
lag['mom%s' % i] = crsp_mom.groupby(['permno'])['ret'].shift(i)
if i not in [72,84,96,108,120]:
result = result + lag['mom%s' % i]

crsp_mom['r610a'] = (lag['mom72']+lag['mom84']+lag['mom96']+lag['mom108']+lag['mom120'])/5
crsp_mom['r610n'] = result/55

#R[11,15]a
lag = pd.DataFrame()
result = 0
for i in [132,144,156,168,180]:
lag['mom%s' % i] = crsp_mom.groupby(['permno'])['ret'].shift(i)
result = result + lag['mom%s' % i]
crsp_mom['r1115a'] = result/5

#R[16,20]a
lag = pd.DataFrame()
result = 0
for i in [192,204,216,228,240]:
lag['mom%s' % i] = crsp_mom.groupby(['permno'])['ret'].shift(i)
result = result + lag['mom%s' % i]
crsp_mom['r1620a'] = result/5



def mom(start, end, df):
"""

:param start: Order of starting lag
:param end: Order of ending lag
:param df: Dataframe
:return: Momentum factor
"""
lag = pd.DataFrame()
result = 1
for i in range(start, end):
lag['mom%s' % i] = df.groupby(['permno'])['ret'].shift(i)
result = result * (1+lag['mom%s' % i])
result = result - 1
return result

# add delisting return
dlret = conn.raw_sql("""
select permno, dlret, dlstdt
Expand Down Expand Up @@ -817,4 +1023,6 @@ def mom(start, end, df):
pkl.dump(chars_a, f)

with open('chars_q.pkl', 'wb') as f:
pkl.dump(chars_q, f)
pkl.dump(chars_q, f)


Loading