Importing the Packages¶
In [347]:
import numpy as np
In [348]:
np.set_printoptions(suppress = True, linewidth = 100, precision = 2)
Importing the Data¶
In [349]:
raw_data_np = np.genfromtxt("loan-data.csv",
delimiter = ';',
skip_header = 1,
autostrip = True)
raw_data_np
Out[349]:
array([[48010226. , nan, 35000. , ..., nan, nan, 9452.96],
[57693261. , nan, 30000. , ..., nan, nan, 4679.7 ],
[59432726. , nan, 15000. , ..., nan, nan, 1969.83],
...,
[50415990. , nan, 10000. , ..., nan, nan, 2185.64],
[46154151. , nan, nan, ..., nan, nan, 3199.4 ],
[66055249. , nan, 10000. , ..., nan, nan, 301.9 ]])
Checking for Incomplete Data¶
In [350]:
np.isnan(raw_data_np).sum()
Out[350]:
88005
In [351]:
temporary_fill = np.nanmax(raw_data_np) + 1
temporary_mean = np.nanmean(raw_data_np, axis = 0)
/var/folders/q7/_hrwfn5d4_v5t3x2qqfx8dwc0000gn/T/ipykernel_25127/3983241459.py:2: RuntimeWarning: Mean of empty slice temporary_mean = np.nanmean(raw_data_np, axis = 0)
In [352]:
temporary_mean
Out[352]:
array([54015809.19, nan, 15273.46, nan, 15311.04, nan, 16.62,
440.92, nan, nan, nan, nan, nan, 3143.85])
In [353]:
temporary_stats = np.array([np.nanmin(raw_data_np, axis =0),
temporary_mean,
np.nanmax(raw_data_np, axis = 0)])
/var/folders/q7/_hrwfn5d4_v5t3x2qqfx8dwc0000gn/T/ipykernel_25127/915496744.py:1: RuntimeWarning: All-NaN slice encountered temporary_stats = np.array([np.nanmin(raw_data_np, axis =0), /var/folders/q7/_hrwfn5d4_v5t3x2qqfx8dwc0000gn/T/ipykernel_25127/915496744.py:3: RuntimeWarning: All-NaN slice encountered np.nanmax(raw_data_np, axis = 0)])
In [355]:
temporary_stats
Out[355]:
array([[ 373332. , nan, 1000. , nan, 1000. , nan, 6. ,
31.42, nan, nan, nan, nan, nan, 0. ],
[54015809.19, nan, 15273.46, nan, 15311.04, nan, 16.62,
440.92, nan, nan, nan, nan, nan, 3143.85],
[68616519. , nan, 35000. , nan, 35000. , nan, 28.99,
1372.97, nan, nan, nan, nan, nan, 41913.62]])
Splitting the Dataset¶
Splitting the Columns¶
In [356]:
column_strings = np.argwhere(np.isnan(temporary_mean)).squeeze()
column_strings
Out[356]:
array([ 1, 3, 5, 8, 9, 10, 11, 12])
In [357]:
column_numeric = np.argwhere(np.isnan(temporary_mean) == False).squeeze()
column_numeric
Out[357]:
array([ 0, 2, 4, 6, 7, 13])
Re-importing the Dataset¶
In [358]:
loan_data_strings = np.genfromtxt('loan-data.csv',
delimiter = ';',
skip_header = 1,
autostrip = True,
usecols = column_strings,
dtype = str)
loan_data_strings
Out[358]:
array([['May-15', 'Current', '36 months', ..., 'Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=48010226', 'CA'],
['', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=57693261', 'NY'],
['Sep-15', 'Current', '36 months', ..., 'Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=59432726', 'PA'],
...,
['Jun-15', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=50415990', 'CA'],
['Apr-15', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=46154151', 'OH'],
['Dec-15', 'Current', '36 months', ..., '',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=66055249', 'IL']],
dtype='<U69')
In [534]:
loan_data_numeric = np.genfromtxt('loan-data.csv',
delimiter =';',
skip_header = 1,
autostrip = True,
usecols = column_numeric,
filling_values = temporary_fill)
loan_data_numeric
Out[534]:
array([[48010226. , 35000. , 35000. , 13.33, 1184.86, 9452.96],
[57693261. , 30000. , 30000. , 68616520. , 938.57, 4679.7 ],
[59432726. , 15000. , 15000. , 68616520. , 494.86, 1969.83],
...,
[50415990. , 10000. , 10000. , 68616520. , 68616520. , 2185.64],
[46154151. , 68616520. , 10000. , 16.55, 354.3 , 3199.4 ],
[66055249. , 10000. , 10000. , 68616520. , 309.97, 301.9 ]])
The Names of the Columns¶
In [361]:
header_strings = np.genfromtxt('loan-data.csv',
delimiter = ';',
autostrip = True,
skip_header = 1,
usecols = column_strings,
dtype = str
)
header_strings
Out[361]:
array([['May-15', 'Current', '36 months', ..., 'Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=48010226', 'CA'],
['', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=57693261', 'NY'],
['Sep-15', 'Current', '36 months', ..., 'Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=59432726', 'PA'],
...,
['Jun-15', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=50415990', 'CA'],
['Apr-15', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=46154151', 'OH'],
['Dec-15', 'Current', '36 months', ..., '',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=66055249', 'IL']],
dtype='<U69')
In [362]:
header_numeric = np.genfromtxt('loan-data.csv',
delimiter = ';',
autostrip = True,
skip_header = 1,
usecols = column_numeric,
dtype = str
)
header_numeric
Out[362]:
array([['48010226', '35000.0', '35000.0', '13.33', '1184.86', '9452.96'],
['57693261', '30000.0', '30000.0', 'þëè.89', '938.57', '4679.7'],
['59432726', '15000.0', '15000.0', 'íîå.53', '494.86', '1969.83'],
...,
['50415990', '10000.0', '10000.0', 'þëè.89', '', '2185.64'],
['46154151', '', '10000.0', '16.55', '354.3', '3199.4'],
['66055249', '10000.0', '10000.0', 'þëè.26', '309.97', '301.9']], dtype='<U11')
In [363]:
header_full = np.genfromtxt('loan-data.csv',
delimiter = ';',
autostrip = True,
skip_footer =raw_data_np.shape[0],
dtype = str )
header_full
Out[363]:
array(['id', 'issue_d', 'loan_amnt', 'loan_status', 'funded_amnt', 'term', 'int_rate',
'installment', 'grade', 'sub_grade', 'verification_status', 'url', 'addr_state',
'total_pymnt'], dtype='<U19')
In [364]:
header_numeric, header_strings = header_full[column_numeric], header_full[column_strings]
Creating Checkpoints:¶
In [365]:
def checkpoint(file_name, checkpoint_header, checkpoint_data):
np.savez(file_name, header = checkpoint_header, data = checkpoint_data)
checkpoint_variable = np.load(file_name + ".npz")
return(checkpoint_variable)
In [370]:
checkpoint_test = checkpoint("checkpoint-test",header_strings, loan_data_strings)
In [371]:
checkpoint_test['header']
Out[371]:
array(['issue_d', 'loan_status', 'term', 'grade', 'sub_grade', 'verification_status', 'url',
'addr_state'], dtype='<U19')
In [372]:
checkpoint_test['data']
Out[372]:
array([['May-15', 'Current', '36 months', ..., 'Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=48010226', 'CA'],
['', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=57693261', 'NY'],
['Sep-15', 'Current', '36 months', ..., 'Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=59432726', 'PA'],
...,
['Jun-15', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=50415990', 'CA'],
['Apr-15', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=46154151', 'OH'],
['Dec-15', 'Current', '36 months', ..., '',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=66055249', 'IL']],
dtype='<U69')
Manipulating String Columns¶
In [373]:
header_strings
Out[373]:
array(['issue_d', 'loan_status', 'term', 'grade', 'sub_grade', 'verification_status', 'url',
'addr_state'], dtype='<U19')
In [59]:
header_strings[0] = 'issue_date'
In [374]:
loan_data_strings
Out[374]:
array([['May-15', 'Current', '36 months', ..., 'Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=48010226', 'CA'],
['', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=57693261', 'NY'],
['Sep-15', 'Current', '36 months', ..., 'Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=59432726', 'PA'],
...,
['Jun-15', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=50415990', 'CA'],
['Apr-15', 'Current', '36 months', ..., 'Source Verified',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=46154151', 'OH'],
['Dec-15', 'Current', '36 months', ..., '',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=66055249', 'IL']],
dtype='<U69')
Issue Date¶
In [375]:
loan_data_strings[:,0]
Out[375]:
array(['May-15', '', 'Sep-15', ..., 'Jun-15', 'Apr-15', 'Dec-15'], dtype='<U69')
In [376]:
loan_data_strings[:,0] = np.chararray.strip(loan_data_strings[:,0],"-15")
In [377]:
np.unique(loan_data_strings[:,0])
Out[377]:
array(['', 'Apr', 'Aug', 'Dec', 'Feb', 'Jan', 'Jul', 'Jun', 'Mar', 'May', 'Nov', 'Oct', 'Sep'],
dtype='<U69')
In [378]:
months = np.array(['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])
In [379]:
for i in range(13):
loan_data_strings[:,0]= np.where(loan_data_strings[:,0] == months[i],
i,
loan_data_strings[:,0])
In [380]:
np.unique(loan_data_strings[:,0])
Out[380]:
array(['0', '1', '10', '11', '12', '2', '3', '4', '5', '6', '7', '8', '9'], dtype='<U69')
Loan Status¶
In [381]:
header_strings
Out[381]:
array(['issue_d', 'loan_status', 'term', 'grade', 'sub_grade', 'verification_status', 'url',
'addr_state'], dtype='<U19')
In [382]:
loan_data_strings[:,1]
Out[382]:
array(['Current', 'Current', 'Current', ..., 'Current', 'Current', 'Current'], dtype='<U69')
In [383]:
np.unique(loan_data_strings[:,1])
Out[383]:
array(['', 'Charged Off', 'Current', 'Default', 'Fully Paid', 'In Grace Period', 'Issued',
'Late (16-30 days)', 'Late (31-120 days)'], dtype='<U69')
In [384]:
status_bad = np.array(['','Charged Off','Default','Late (31-120 days)'])
In [385]:
loan_data_strings[:,1] = np.where(np.isin(loan_data_strings[:,1],status_bad),0,1)
In [386]:
np.unique(loan_data_strings[:,1])
Out[386]:
array(['0', '1'], dtype='<U69')
Term¶
In [387]:
header_strings
Out[387]:
array(['issue_d', 'loan_status', 'term', 'grade', 'sub_grade', 'verification_status', 'url',
'addr_state'], dtype='<U19')
In [388]:
loan_data_strings[:,2]
Out[388]:
array(['36 months', '36 months', '36 months', ..., '36 months', '36 months', '36 months'],
dtype='<U69')
In [389]:
np.unique(loan_data_strings[:,2])
Out[389]:
array(['', '36 months', '60 months'], dtype='<U69')
In [390]:
loan_data_strings[:,2]= np.chararray.strip(loan_data_strings[:,2]," months")
In [391]:
loan_data_strings[:,2] = np.where(loan_data_strings[:,2] == '',
60,
loan_data_strings[:,2]
)
In [392]:
np.unique(loan_data_strings[:,2])
Out[392]:
array(['36', '60'], dtype='<U69')
Grade and Subgrade¶
In [393]:
header_strings
Out[393]:
array(['issue_d', 'loan_status', 'term', 'grade', 'sub_grade', 'verification_status', 'url',
'addr_state'], dtype='<U19')
In [394]:
np.unique(loan_data_strings[:,3])
Out[394]:
array(['', 'A', 'B', 'C', 'D', 'E', 'F', 'G'], dtype='<U69')
In [395]:
np.unique(loan_data_strings[:,4])
Out[395]:
array(['', 'A1', 'A2', 'A3', 'A4', 'A5', 'B1', 'B2', 'B3', 'B4', 'B5', 'C1', 'C2', 'C3', 'C4',
'C5', 'D1', 'D2', 'D3', 'D4', 'D5', 'E1', 'E2', 'E3', 'E4', 'E5', 'F1', 'F2', 'F3', 'F4',
'F5', 'G1', 'G2', 'G3', 'G4', 'G5'], dtype='<U69')
Filling Sub Grade¶
In [396]:
for i in np.unique(loan_data_strings[:,3])[1:]:
loan_data_strings[:,4] = np.where((loan_data_strings[:,4] == '') & (loan_data_strings[:,3] == i),
i + '5',
loan_data_strings[:,4])
In [397]:
np.unique(loan_data_strings[:,4])
Out[397]:
array(['', 'A1', 'A2', 'A3', 'A4', 'A5', 'B1', 'B2', 'B3', 'B4', 'B5', 'C1', 'C2', 'C3', 'C4',
'C5', 'D1', 'D2', 'D3', 'D4', 'D5', 'E1', 'E2', 'E3', 'E4', 'E5', 'F1', 'F2', 'F3', 'F4',
'F5', 'G1', 'G2', 'G3', 'G4', 'G5'], dtype='<U69')
In [398]:
loan_data_strings[:,4]= np.where(loan_data_strings[:,4] == '',
'H1',
loan_data_strings[:,4])
In [400]:
np.unique(loan_data_strings[:,4], return_counts = True)
Out[400]:
(array(['A1', 'A2', 'A3', 'A4', 'A5', 'B1', 'B2', 'B3', 'B4', 'B5', 'C1', 'C2', 'C3', 'C4', 'C5',
'D1', 'D2', 'D3', 'D4', 'D5', 'E1', 'E2', 'E3', 'E4', 'E5', 'F1', 'F2', 'F3', 'F4', 'F5',
'G1', 'G2', 'G3', 'G4', 'G5', 'H1'], dtype='<U69'),
array([285, 278, 239, 323, 592, 509, 517, 530, 553, 633, 629, 567, 586, 564, 577, 391, 267, 250,
255, 288, 235, 162, 171, 139, 160, 94, 52, 34, 43, 24, 19, 10, 3, 7, 5, 9]))
Removing Grade¶
In [401]:
loan_data_strings = np.delete(loan_data_strings,3, axis = 1)
In [402]:
header_strings = np.delete(header_strings,3)
In [403]:
header_strings
Out[403]:
array(['issue_d', 'loan_status', 'term', 'sub_grade', 'verification_status', 'url', 'addr_state'],
dtype='<U19')
In [ ]:
Converting Sub Grade¶
In [404]:
np.unique(loan_data_strings[:,3])
Out[404]:
array(['A1', 'A2', 'A3', 'A4', 'A5', 'B1', 'B2', 'B3', 'B4', 'B5', 'C1', 'C2', 'C3', 'C4', 'C5',
'D1', 'D2', 'D3', 'D4', 'D5', 'E1', 'E2', 'E3', 'E4', 'E5', 'F1', 'F2', 'F3', 'F4', 'F5',
'G1', 'G2', 'G3', 'G4', 'G5', 'H1'], dtype='<U69')
In [ ]:
In [405]:
keys = list(np.unique(loan_data_strings[:,3]))
values = list(range(1, np.unique(loan_data_strings[:,3]).shape[0] + 1))
dict_sub_grade = dict(zip(keys, values))
In [ ]:
In [406]:
for i in np.unique(loan_data_strings[:,3]):
loan_data_strings[:,3] = np.where(loan_data_strings[:,3] == i,
dict_sub_grade[i],
loan_data_strings[:,3])
In [407]:
np.unique(loan_data_strings[:,3])
Out[407]:
array(['1', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '2', '20', '21', '22',
'23', '24', '25', '26', '27', '28', '29', '3', '30', '31', '32', '33', '34', '35', '36',
'4', '5', '6', '7', '8', '9'], dtype='<U69')
Verification Status¶
In [408]:
np.unique(loan_data_strings[:,4])
Out[408]:
array(['', 'Not Verified', 'Source Verified', 'Verified'], dtype='<U69')
In [409]:
loan_data_strings[:,4] = np.where((loan_data_strings[:,4] == "Not Verified") |(loan_data_strings[:,4] == ""),0,1)
In [410]:
loan_data_strings[:,4]
Out[410]:
array(['1', '1', '1', ..., '1', '1', '0'], dtype='<U69')
In [411]:
np.unique(loan_data_strings[:,4])
Out[411]:
array(['0', '1'], dtype='<U69')
URL¶
In [412]:
loan_data_strings[:,5]
Out[412]:
array(['https://www.lendingclub.com/browse/loanDetail.action?loan_id=48010226',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=57693261',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=59432726', ...,
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=50415990',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=46154151',
'https://www.lendingclub.com/browse/loanDetail.action?loan_id=66055249'], dtype='<U69')
In [413]:
np.chararray.strip(loan_data_strings[:,5],"https://www.lendingclub.com/browse/loanDetail.action?loan_id=")
Out[413]:
chararray(['48010226', '57693261', '59432726', ..., '50415990', '46154151', '66055249'],
dtype='<U69')
In [414]:
loan_data_strings[:,5] = np.chararray.strip(loan_data_strings[:,5],"https://www.lendingclub.com/browse/loanDetail.action?loan_id=")
In [415]:
loan_data_strings[:,5]
Out[415]:
array(['48010226', '57693261', '59432726', ..., '50415990', '46154151', '66055249'], dtype='<U69')
In [416]:
header_full
Out[416]:
array(['id', 'issue_d', 'loan_amnt', 'loan_status', 'funded_amnt', 'term', 'int_rate',
'installment', 'grade', 'sub_grade', 'verification_status', 'url', 'addr_state',
'total_pymnt'], dtype='<U19')
In [417]:
loan_data_numeric[:,0].astype(dtype = np.int32)
Out[417]:
array([48010226, 57693261, 59432726, ..., 50415990, 46154151, 66055249], dtype=int32)
In [418]:
loan_data_strings[:,5].astype(dtype = np.int32)
Out[418]:
array([48010226, 57693261, 59432726, ..., 50415990, 46154151, 66055249], dtype=int32)
In [419]:
np.array_equal(loan_data_numeric[:,0].astype(dtype = np.int32),loan_data_strings[:,5].astype(dtype = np.int32))
Out[419]:
True
In [420]:
loan_data_strings = np.delete(loan_data_strings,5,axis = 1)
In [421]:
header_strings = np.delete(header_strings,5)
In [422]:
header_strings
Out[422]:
array(['issue_d', 'loan_status', 'term', 'sub_grade', 'verification_status', 'addr_state'],
dtype='<U19')
In [423]:
header_numeric
Out[423]:
array(['id', 'loan_amnt', 'funded_amnt', 'int_rate', 'installment', 'total_pymnt'], dtype='<U19')
State Address¶
In [424]:
header_strings
Out[424]:
array(['issue_d', 'loan_status', 'term', 'sub_grade', 'verification_status', 'addr_state'],
dtype='<U19')
In [425]:
header_strings[5] = "State_address"
In [426]:
states_names, states_count = np.unique(loan_data_strings[:,5],return_counts = True)
states_count_sorted = np.argsort(-states_count)
states_names[np.argsort(-states_count)],states_count[np.argsort(-states_count)]
Out[426]:
(array(['CA', 'NY', 'TX', 'FL', '', 'IL', 'NJ', 'GA', 'PA', 'OH', 'MI', 'NC', 'VA', 'MD', 'AZ',
'WA', 'MA', 'CO', 'MO', 'MN', 'IN', 'WI', 'CT', 'TN', 'NV', 'AL', 'LA', 'OR', 'SC', 'KY',
'KS', 'OK', 'UT', 'AR', 'MS', 'NH', 'NM', 'WV', 'HI', 'RI', 'MT', 'DE', 'DC', 'WY', 'AK',
'NE', 'SD', 'VT', 'ND', 'ME'], dtype='<U69'),
array([1336, 777, 758, 690, 500, 389, 341, 321, 320, 312, 267, 261, 242, 222, 220,
216, 210, 201, 160, 156, 152, 148, 143, 143, 130, 119, 116, 108, 107, 84,
84, 83, 74, 74, 61, 58, 57, 49, 44, 40, 28, 27, 27, 27, 26,
25, 24, 17, 16, 10]))
In [427]:
loan_data_strings[:,5]= np.where(loan_data_strings[:,5]== '',
0,
loan_data_strings[:,5])
In [428]:
states_west = np.array(['WA', 'OR','CA','NV','ID','MT', 'WY','UT','CO', 'AZ','NM','HI','AK'])
states_south = np.array(['TX','OK','AR','LA','MS','AL','TN','KY','FL','GA','SC','NC','VA','WV','MD','DE','DC'])
states_midwest = np.array(['ND','SD','NE','KS','MN','IA','MO','WI','IL','IN','MI','OH'])
states_east = np.array(['PA','NY','NJ','CT','MA','VT','NH','ME','RI'])
In [429]:
loan_data_strings[:,5] = np.where(np.isin(loan_data_strings[:,5],states_west),1,loan_data_strings[:,5])
loan_data_strings[:,5] = np.where(np.isin(loan_data_strings[:,5],states_south),2,loan_data_strings[:,5])
loan_data_strings[:,5] = np.where(np.isin(loan_data_strings[:,5],states_midwest),3,loan_data_strings[:,5])
loan_data_strings[:,5] = np.where(np.isin(loan_data_strings[:,5],states_east),4,loan_data_strings[:,5])
In [430]:
np.unique(loan_data_strings[:,5])
Out[430]:
array(['0', '1', '2', '3', '4'], dtype='<U69')
Converting to Numbers¶
In [431]:
loan_data_strings
Out[431]:
array([['5', '1', '36', '13', '1', '1'],
['0', '1', '36', '5', '1', '4'],
['9', '1', '36', '10', '1', '4'],
...,
['6', '1', '36', '5', '1', '1'],
['4', '1', '36', '17', '1', '3'],
['12', '1', '36', '4', '0', '3']], dtype='<U69')
In [432]:
loan_data_strings.astype(dtype = int)
Out[432]:
array([[ 5, 1, 36, 13, 1, 1],
[ 0, 1, 36, 5, 1, 4],
[ 9, 1, 36, 10, 1, 4],
...,
[ 6, 1, 36, 5, 1, 1],
[ 4, 1, 36, 17, 1, 3],
[12, 1, 36, 4, 0, 3]])
In [433]:
loan_data_strings = loan_data_strings.astype(dtype = int)
Checkpoint 1: Strings¶
In [434]:
checkpoint_strings = checkpoint('Checkpoint-Strings', header_strings,loan_data_strings)
In [435]:
checkpoint_strings['header']
Out[435]:
array(['issue_d', 'loan_status', 'term', 'sub_grade', 'verification_status', 'State_address'],
dtype='<U19')
In [436]:
checkpoint_strings['data']
Out[436]:
array([[ 5, 1, 36, 13, 1, 1],
[ 0, 1, 36, 5, 1, 4],
[ 9, 1, 36, 10, 1, 4],
...,
[ 6, 1, 36, 5, 1, 1],
[ 4, 1, 36, 17, 1, 3],
[12, 1, 36, 4, 0, 3]])
In [437]:
np.array_equal(checkpoint_strings['data'], loan_data_strings)
Out[437]:
True
Manipulating Numeric Columns¶
In [535]:
loan_data_numeric
Out[535]:
array([[48010226. , 35000. , 35000. , 13.33, 1184.86, 9452.96],
[57693261. , 30000. , 30000. , 68616520. , 938.57, 4679.7 ],
[59432726. , 15000. , 15000. , 68616520. , 494.86, 1969.83],
...,
[50415990. , 10000. , 10000. , 68616520. , 68616520. , 2185.64],
[46154151. , 68616520. , 10000. , 16.55, 354.3 , 3199.4 ],
[66055249. , 10000. , 10000. , 68616520. , 309.97, 301.9 ]])
In [536]:
np.unique(loan_data_numeric, return_counts = True)
Out[536]:
(array([ 0. , 0.74, 6. , ..., 68615915. , 68616519. , 68616520. ]), array([ 384, 1, 1, ..., 1, 1, 8005]))
Substitute "Filler" Values¶
In [506]:
header_numeric
Out[506]:
array(['id', 'loan_amnt_USD', 'loan_amnt_EUR', 'funded_amnt_USD', 'funded_amnt_EUR', 'int_rate',
'installment_USD', 'installment_EUR', 'total_pymnt_USD', 'total_pymnt_EUR', 'exchange rate'],
dtype='<U19')
ID¶
In [507]:
temporary_fill
Out[507]:
68616520.0
In [537]:
np.isin(loan_data_numeric[:,0],temporary_fill)
Out[537]:
array([False, False, False, ..., False, False, False])
In [538]:
np.isin(loan_data_numeric[:,0],temporary_fill).sum()
Out[538]:
0
In [539]:
loan_data_numeric[:,5]
Out[539]:
array([9452.96, 4679.7 , 1969.83, ..., 2185.64, 3199.4 , 301.9 ])
Temporary Stats¶
In [445]:
temporary_stats[:,column_numeric]
Out[445]:
array([[ 373332. , 1000. , 1000. , 6. , 31.42, 0. ],
[54015809.19, 15273.46, 15311.04, 16.62, 440.92, 3143.85],
[68616519. , 35000. , 35000. , 28.99, 1372.97, 41913.62]])
Funded Amount¶
In [540]:
loan_data_numeric
Out[540]:
array([[48010226. , 35000. , 35000. , 13.33, 1184.86, 9452.96],
[57693261. , 30000. , 30000. , 68616520. , 938.57, 4679.7 ],
[59432726. , 15000. , 15000. , 68616520. , 494.86, 1969.83],
...,
[50415990. , 10000. , 10000. , 68616520. , 68616520. , 2185.64],
[46154151. , 68616520. , 10000. , 16.55, 354.3 , 3199.4 ],
[66055249. , 10000. , 10000. , 68616520. , 309.97, 301.9 ]])
In [541]:
loan_data_numeric[:,2] = np.where(loan_data_numeric[:,2] == temporary_fill,
temporary_stats[0, column_numeric[2]],
loan_data_numeric[:,2])
loan_data_numeric[:,2]
Out[541]:
array([35000., 30000., 15000., ..., 10000., 10000., 10000.])
In [542]:
temporary_stats[0,column_numeric[3]]
Out[542]:
6.0
Loaned Amount, Interest Rate, Total Payment, Installment¶
In [448]:
header_numeric
Out[448]:
array(['id', 'loan_amnt', 'funded_amnt', 'int_rate', 'installment', 'total_pymnt'], dtype='<U19')
In [543]:
for i in [1,3,4,5]:
loan_data_numeric[:,i] = np.where(loan_data_numeric[:,i] == temporary_fill,
temporary_stats[2, column_numeric[i]],
loan_data_numeric[:,i])
In [544]:
loan_data_numeric
Out[544]:
array([[48010226. , 35000. , 35000. , 13.33, 1184.86, 9452.96],
[57693261. , 30000. , 30000. , 28.99, 938.57, 4679.7 ],
[59432726. , 15000. , 15000. , 28.99, 494.86, 1969.83],
...,
[50415990. , 10000. , 10000. , 28.99, 1372.97, 2185.64],
[46154151. , 35000. , 10000. , 16.55, 354.3 , 3199.4 ],
[66055249. , 10000. , 10000. , 28.99, 309.97, 301.9 ]])
Currency Change¶
The Exchange Rate¶
In [545]:
EUR_USD = np.genfromtxt('EUR-USD (2).csv',delimiter = ',', autostrip = True, skip_header = 1,usecols = 3 )
EUR_USD
Out[545]:
array([1.13, 1.12, 1.08, 1.11, 1.1 , 1.12, 1.09, 1.13, 1.13, 1.1 , 1.06, 1.09])
In [546]:
np.unique(loan_data_strings[:,0])
Out[546]:
array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])
In [547]:
exchange_rate = loan_data_strings[:,0]
for i in range(1,13):
exchange_rate = np.where(exchange_rate == i,
EUR_USD[i-1],
exchange_rate)
exchange_rate = np.where(exchange_rate ==0,
np.mean(EUR_USD),
exchange_rate)
exchange_rate
Out[547]:
array([1.1 , 1.11, 1.13, ..., 1.12, 1.11, 1.09])
In [548]:
exchange_rate.shape
Out[548]:
(10000,)
In [549]:
exchange_rate = np.reshape(exchange_rate,(10000,1))
In [550]:
loan_data_numeric = np.hstack((loan_data_numeric, exchange_rate))
In [457]:
header_numeric = np.concatenate((header_numeric, np.array(['exchange rate'])))
In [551]:
loan_data_numeric.shape
Out[551]:
(10000, 7)
From USD to EUR¶
In [459]:
header_numeric
Out[459]:
array(['id', 'loan_amnt', 'funded_amnt', 'int_rate', 'installment', 'total_pymnt', 'exchange rate'],
dtype='<U19')
In [553]:
columns_dollar = np.array([1,2,4,5])
In [552]:
loan_data_numeric[:,6]
Out[552]:
array([1.1 , 1.11, 1.13, ..., 1.12, 1.11, 1.09])
In [554]:
for i in columns_dollar:
loan_data_numeric = np.hstack((loan_data_numeric, np.reshape(loan_data_numeric[:,i] / loan_data_numeric[:,6], (10000,1))))
In [555]:
(loan_data_numeric[:,10])
Out[555]:
array([8624.69, 4232.39, 1750.04, ..., 1947.47, 2878.63, 276.11])
In [556]:
loan_data_numeric[:,2]
Out[556]:
array([35000., 30000., 15000., ..., 10000., 10000., 10000.])
Expanding the header¶
In [475]:
header_additional = np.array([column_name + '_EUR' for column_name in header_numeric[columns_dollar]])
In [477]:
header_additional
Out[477]:
array(['loan_amnt_EUR', 'funded_amnt_EUR', 'installment_EUR', 'total_pymnt_EUR'], dtype='<U15')
In [478]:
header_numeric = np.concatenate((header_numeric,header_additional))
In [480]:
header_numeric
Out[480]:
array(['id', 'loan_amnt', 'funded_amnt', 'int_rate', 'installment', 'total_pymnt', 'exchange rate',
'loan_amnt_EUR', 'funded_amnt_EUR', 'installment_EUR', 'total_pymnt_EUR'], dtype='<U19')
In [483]:
header_numeric[columns_dollar] = np.array([column_name + '_USD' for column_name in header_numeric[columns_dollar]])
In [485]:
header_numeric
Out[485]:
array(['id', 'loan_amnt_USD', 'funded_amnt_USD', 'int_rate', 'installment_USD', 'total_pymnt_USD',
'exchange rate', 'loan_amnt_EUR', 'funded_amnt_EUR', 'installment_EUR', 'total_pymnt_EUR'],
dtype='<U19')
In [557]:
columns_index_order = [0,1,7,2,8,3,4,9,5,10,6]
In [488]:
header_numeric = header_numeric[columns_index_order]
In [558]:
loan_data_numeric
Out[558]:
array([[48010226. , 35000. , 35000. , ..., 31933.3 , 1081.04, 8624.69],
[57693261. , 30000. , 30000. , ..., 27132.46, 848.86, 4232.39],
[59432726. , 15000. , 15000. , ..., 13326.3 , 439.64, 1750.04],
...,
[50415990. , 10000. , 10000. , ..., 8910.3 , 1223.36, 1947.47],
[46154151. , 35000. , 10000. , ..., 8997.4 , 318.78, 2878.63],
[66055249. , 10000. , 10000. , ..., 9145.8 , 283.49, 276.11]])
In [559]:
loan_data_numeric = loan_data_numeric[:,columns_index_order]
Interest Rate¶
In [494]:
header_numeric
Out[494]:
array(['id', 'loan_amnt_USD', 'loan_amnt_EUR', 'funded_amnt_USD', 'funded_amnt_EUR', 'int_rate',
'installment_USD', 'installment_EUR', 'total_pymnt_USD', 'total_pymnt_EUR', 'exchange rate'],
dtype='<U19')
In [560]:
loan_data_numeric[:,5]
Out[560]:
array([13.33, 28.99, 28.99, ..., 28.99, 16.55, 28.99])
In [561]:
loan_data_numeric[:,5] = loan_data_numeric[:,5]/100
In [563]:
loan_data_numeric[:,5]
Out[563]:
array([0.13, 0.29, 0.29, ..., 0.29, 0.17, 0.29])
Checkpoint 2: Numeric¶
In [564]:
checkpoint_numeric = checkpoint('Checkpoint-Numeric',header_numeric, loan_data_numeric)
In [565]:
checkpoint_numeric['header'], checkpoint_numeric['data']
Out[565]:
(array(['id', 'loan_amnt_USD', 'loan_amnt_EUR', 'funded_amnt_USD', 'funded_amnt_EUR', 'int_rate',
'installment_USD', 'installment_EUR', 'total_pymnt_USD', 'total_pymnt_EUR', 'exchange rate'],
dtype='<U19'),
array([[48010226. , 35000. , 31933.3 , ..., 9452.96, 8624.69, 1.1 ],
[57693261. , 30000. , 27132.46, ..., 4679.7 , 4232.39, 1.11],
[59432726. , 15000. , 13326.3 , ..., 1969.83, 1750.04, 1.13],
...,
[50415990. , 10000. , 8910.3 , ..., 2185.64, 1947.47, 1.12],
[46154151. , 35000. , 31490.9 , ..., 3199.4 , 2878.63, 1.11],
[66055249. , 10000. , 9145.8 , ..., 301.9 , 276.11, 1.09]]))
Creating the "Complete" Dataset¶
In [568]:
checkpoint_strings['data'].shape
Out[568]:
(10000, 6)
In [569]:
checkpoint_numeric['data'].shape
Out[569]:
(10000, 11)
In [571]:
loan_data = np.hstack((checkpoint_numeric['data'],checkpoint_strings['data']))
In [572]:
loan_data
Out[572]:
array([[48010226. , 35000. , 31933.3 , ..., 13. , 1. , 1. ],
[57693261. , 30000. , 27132.46, ..., 5. , 1. , 4. ],
[59432726. , 15000. , 13326.3 , ..., 10. , 1. , 4. ],
...,
[50415990. , 10000. , 8910.3 , ..., 5. , 1. , 1. ],
[46154151. , 35000. , 31490.9 , ..., 17. , 1. , 3. ],
[66055249. , 10000. , 9145.8 , ..., 4. , 0. , 3. ]])
In [573]:
np.isnan(loan_data).sum()
Out[573]:
0
In [574]:
header_full= np.concatenate((checkpoint_numeric['header'], checkpoint_strings['header']))
Sorting the New Dataset¶
In [577]:
loan_data = loan_data[np.argsort(loan_data[:,0])]
In [579]:
loan_data
Out[579]:
array([[ 373332. , 9950. , 9038.08, ..., 21. , 0. , 1. ],
[ 575239. , 12000. , 10900.2 , ..., 25. , 1. , 2. ],
[ 707689. , 10000. , 8924.3 , ..., 13. , 1. , 0. ],
...,
[68614880. , 5600. , 5121.65, ..., 8. , 1. , 1. ],
[68615915. , 4000. , 3658.32, ..., 10. , 1. , 2. ],
[68616519. , 21600. , 19754.93, ..., 3. , 0. , 2. ]])
In [580]:
np.argsort(loan_data[:,0])
Out[580]:
array([ 0, 1, 2, ..., 9997, 9998, 9999])
Storing the New Dataset¶
In [582]:
loan_data = np.vstack((header_full, loan_data))
In [583]:
np.savetxt('loan_data_preprocessed.csv',
loan_data,
fmt = '%s',
delimiter = ',')
In [ ]: