forked from renjunxiang/Text-Classification
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathTextClassification.py
More file actions
76 lines (63 loc) · 2.43 KB
/
TextClassification.py
File metadata and controls
76 lines (63 loc) · 2.43 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
from .DataPreprocess import DataPreprocess
from .net import CNN
import numpy as np
class TextClassification():
def __init__(self):
self.preprocess = None
self.model = None
def get_preprocess(self, texts, labels, word_len=1, num_words=2000, sentence_len=30):
# 数据预处理
preprocess = DataPreprocess()
# 处理文本
texts_cut = preprocess.cut_texts(texts, word_len)
preprocess.train_tokenizer(texts_cut, num_words)
texts_seq = preprocess.text2seq(texts_cut, sentence_len)
# 得到标签
preprocess.creat_label_set(labels)
labels = preprocess.creat_labels(labels)
self.preprocess = preprocess
return texts_seq, labels
def fit(self, texts_seq, texts_labels, output_type, epochs, batch_size, model=None):
if model is None:
preprocess = self.preprocess
model = CNN(preprocess.num_words,
preprocess.sentence_len,
128,
len(preprocess.label_set),
output_type)
# 训练神经网络
model.fit(texts_seq,
texts_labels,
epochs=epochs,
batch_size=batch_size)
self.model = model
def predict(self, texts):
preprocess = self.preprocess
word_len = preprocess.word_len
sentence_len = preprocess.sentence_len
# 处理文本
texts_cut = preprocess.cut_texts(texts, word_len)
texts_seq = preprocess.text2seq(texts_cut, sentence_len)
return self.model.predict(texts_seq)
def label2toptag(self, predictions, labelset):
labels = []
for prediction in predictions:
label = labelset[prediction == prediction.max()]
labels.append(label.tolist())
return labels
def label2half(self, predictions, labelset):
labels = []
for prediction in predictions:
label = labelset[prediction > 0.5]
labels.append(label.tolist())
return labels
def label2tag(self, predictions, labelset):
labels1 = self.label2toptag(predictions, labelset)
labels2 = self.label2half(predictions, labelset)
labels = []
for i in range(len(predictions)):
if len(labels2[i]) == 0:
labels.append(labels1[i])
else:
labels.append(labels2[i])
return labels