1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
import numpy as np
from collections import defaultdict
def readDataSet(filename, frequency = 0, training_set_ratio = 0.7, shuffle = True):
'''
read the dataset file, and shuffle, remove all punctuations
Parameters
----------
filename: str, the filename of the data
frequency: int, you will select the words that appeared more than the frequency you specified
for example, if you set frequency equals 1, the program will return all words that they have
appeared more than once.
training_set_ratio: float, the ratio of training data account for in all data
shuffle: bool, whether to shuffle the data
Returns
----------
train_text: list, each element contains a tuple of words that in each sentence
train_labels: list, each element is the label of the corresponding sentence
test_text: list
test_labels: list
'''
with open(filename, 'r', encoding='utf-8') as f:
text = f.read().strip().split('\n')
if shuffle:
np.random.shuffle(text)
import re
dataset = []
for index, i in enumerate(text):
t = i.split('\t')
label = t[0]
t1 = re.sub("[\.\!\/_,$%^*(+\"\']+|[+——！，。？?、~@#￥%……&*（）]+", "", t[1])
dataset.append((label, re.split(re.compile('\s+'), t1)))
print("dataset's size is", len(dataset))
labels, text = zip(*dataset)
split_line = int(len(text) * training_set_ratio)
train_text = text[:split_line]
train_labels = labels[:split_line]
test_text = text[split_line:]
test_labels = labels[split_line:]
return train_text, train_labels, test_text, test_labels
def preprocessing_training_data(text, labels):
'''
use bag of words to build features for training data
Parameters
----------
text: lists, each element contains a list of words in a sentence
labels: lists, each element is the label of the sample corresponding to the element in text
Returns
----------
trainX: ndarray, training data, the shape of it is (number of samples, number of features)
trainY: ndarray, labels of training data, the shape of it is (number of samples, )
words_table: dict, key is words, value is the index in bag of words
labels_table: dict, key is the label, value is the index that represents the corresponding label
'''
bag_of_words = tuple(set(word for words in text for word in words))
words_table = {i: index for index, i in enumerate(bag_of_words)}
trainX = np.empty((len(text), len(bag_of_words)))
for index, words in enumerate(text):
for word in words:
trainX[index, words_table[word]] += 1
labels_table = {i: index for index, i in enumerate(set(labels))}
trainY = np.array([labels_table[i] for i in labels])
return trainX, trainY, words_table, labels_table
def preprocessing_testing_data(text, labels, words_table, labels_table):
'''
use bag of words to build features for testing data
Parameters
----------
text: lists, each element contains a list of words in a sentence
labels: lists, each element is the label of the sample corresponding to the element in text
words_table: dict, key is words, value is the index in bag of words
labels_table: dict, key is the label, value is the index that represents the corresponding label
Returns
----------
testX: ndarray, testing data, the shape of it is (number of samples, number of features)
testY: ndarray, labels of testing data, the shape of it is (number of samples, )
'''
testX = np.empty((len(text), len(words_table)))
for index, words in enumerate(text):
for word in words:
col = words_table.get(word)
if col is not None:
testX[index, words_table[word]] += 1
testY = []
for i in labels:
l = labels_table.get(i)
if l is not None:
testY.append(l)
else:
labels_table[i] = len(labels_table)
testY.append(labels_table[i])
testY = np.array(testY)
return testX, testY
class GaussianNB:
'''
Gaussian naive bayes for continous features
'''
def __init__(self):
self.probability_of_y = {}
self.mean = {}
self.var = {}
def fit(self, trainX, trainY):
'''
use trainX and trainY to compute the prior probability for each class
and then compute the mean and variance for each features for each class
Parameters
----------
trainX: ndarray, training data, the shape of it is (number of samples, number of features)
trainY: ndarray, labels of training data, the shape of it is (number of samples, )
'''
labels = set(trainY.tolist())
for y in labels:
x = trainX[trainY == y, :]
self.probability_of_y[y] = x.shape[0] / trainX.shape[0]
self.mean[y] = x.mean(axis = 0)
var = x.var(axis = 0)
var[var == 0] += 1e-9 * var.max()
self.var[y] = var
def predict(self, testX):
'''
predict the labels of testX
Parameters
----------
testX: ndarray, testing data, the shape of it is (number of samples, number of features)
Returns
----------
ndarray: each element is a str variable, which represent the label of corresponding testing data
'''
results = np.empty((testX.shape[0], len(self.probability_of_y)))
labels = []
for index, (label, py) in enumerate(self.probability_of_y.items()):
t = np.exp(- ((testX - self.mean[label]) ** 2) / (2 * self.var[label])) / np.sqrt(2 * np.pi * self.var[label])
t[t == 0] = np.finfo(np.longdouble).eps
a = np.log(t)
results[:, index] = np.exp(np.sum(a, axis = 1)) * py
labels.append(label)
return np.array(labels)[np.argmax(results, axis = 1)]
def accuracy(prediction, testY):
'''
compute accuracy for prediction
Parameters
----------
prediction: ndarray, the prediction generated by the classifier
testY: ndarray, true labels
Returns
----------
float, accuracy
'''
return np.sum((prediction - testY) == 0) / testY.shape[0]
def main():
datadir = 'SMSSpamCollection'
train_text, train_labels, test_text, test_labels = readDataSet(datadir)
trainX, trainY, words_table, labels_table = preprocessing_training_data(train_text, train_labels)
print('training data shape:', trainX.shape, trainY.shape)
testX, testY = preprocessing_testing_data(test_text, test_labels, words_table, labels_table)
print('testing data shape:', testX.shape, testY.shape)
a = GaussianNB()
a.fit(trainX, trainY)
r = a.predict(testX)
print('accuracy:', accuracy(r, testY))
if __name__ == '__main__':
main()
|
