文本分类器构建方法及程序设计

import jieba
import jieba.analyse

#添加自定义词典
jieba.load_userdict('e:\\lexicon\\CBwords.txt')

# 创建停用词列表函数
def creadstoplist(stopwordspath):
    stwlist = [line.strip() for line in open(stopwordspath, 'r', encoding='utf-8').readlines()]
    return stwlist

# 对句子进行分词
def seg_sentence(sentence):
    wordList = jieba.cut(sentence.strip())
    stwlist = creadstoplist('e:\\lexicon\\CBstopwords.txt') #加载停用词路径
    outstr = ''
    for word in wordList:
        if word not in stwlist:
                if word != '\t':
                    outstr += word
                    outstr += " "
    return outstr

#删除第1行和最后3行语句
infile = open('e:\\SVM预处理输入\\in1.txt', 'r', encoding='utf-8')
line_new = infile.readlines() #注意readline 和readlines
infile_new = ''.join(line_new[1:-3]) 
outfile_new = open("e:\\new.txt","w",encoding='utf-8')
outfile_new.write(infile_new)
infile.close()
outfile_new.close()

#结果输出到txt文件夹中
infile_new1 = open("e:\\new.txt","r",encoding='utf-8')
outfile = open('e:\\SVM预处理输出\\outfile1.txt', 'w', encoding='utf-8')
for line in infile_new1:
    line_seg = seg_sentence(line)
    outfile.write(line_seg+'\n')
infile_new1.close()
outfile.close()

import jieba
import jieba.analyse

#添加自定义词典
jieba.load_userdict('e:\\lexicon\\CBwords.txt')

# 创建停用词列表函数
def creadstoplist(stopwordspath):
    stwlist = [line.strip() for line in open(stopwordspath, 'r', encoding='utf-8').readlines()]
    return stwlist

# 对句子进行分词
def seg_sentence(sentence):
    wordList = jieba.cut(sentence.strip())
    stwlist = creadstoplist('e:\\lexicon\\CBstopwords.txt') #加载停用词路径
    outstr = ''
    for word in wordList:
        if word not in stwlist:
                if word != '\t':
                    outstr += word
                    outstr += " "
    return outstr

#删除第1行和最后3行语句
infile = open('e:\\SVM预处理输入\\in1.txt', 'r', encoding='utf-8')
line_new = infile.readlines() #注意readline 和readlines
infile_new = ''.join(line_new[1:-3]) 
outfile_new = open("e:\\new.txt","w",encoding='utf-8')
outfile_new.write(infile_new)
infile.close()
outfile_new.close()

#结果输出到txt文件夹中
infile_new1 = open("e:\\new.txt","r",encoding='utf-8')
outfile = open('e:\\SVM预处理输出\\outfile1.txt', 'w', encoding='utf-8')
for line in infile_new1:
    line_seg = seg_sentence(line)
    outfile.write(line_seg+'\n')
infile_new1.close()
outfile.close()

import csv
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.svm import SVC
from sklearn.pipeline import Pipeline
from sklearn import metrics
from sklearn.grid_search import GridSearchCV

# 1.读取训练集
def readtrain():
    with open('e:\\test1.csv', 'r') as csvfile: 
        reader = csv.reader(csvfile)
        column1 = [row for row in reader]
    content_train = [i[1] for i in column1[1:]] #第一列为文本内容，并去除列名
    opinion_train = [i[2] for i in column1[1:]] #第二列为分类标签，并去除列名
    print ('数据集共有 %s 条句子' % len(content_train)) 
    train = [content_train, opinion_train]
    return train

def changeListCode(b):
    a = []
    for i in b:
        a.append(i.decode('utf8'))
    return a
train = readtrain() #读取readtrain数据集
content = train[0] #第一行文本内容赋给content
opinion = train[1] #第二行分类标签赋给opinion

# 2.划分训练集与测试集
train_content = content[:85]  #训练集文本内容
test_content = content[85:]   #测试集文本内容
train_opinion = opinion[:85]  #训练集分类标签
test_opinion = opinion[85:]   #测试集分类标签（用于计算精确度）

# 3.文本向量化
vectorizer = CountVectorizer() #将向量化函数赋给vectorizer
tfidftransformer = TfidfTransformer()
tfidf = tfidftransformer.fit_transform(vectorizer.fit_transform(train_content))  # 先转换成词频矩阵，再计算TFIDF值
print (tfidf.shape)

# 4.训练文本分类器
text_clf = Pipeline([('vect', CountVectorizer()), ('tfidf', TfidfTransformer()), ('clf', SVC(C=0.99, kernel = 'linear'))])
text_clf = text_clf.fit(train_content, train_opinion)

# 5.预测文本分类结果
predicted = text_clf.predict(test_content)
print ("打印预测结果：")
print (predicted) #打印预测分类结果

# 6.计算精确度
print ("============================================")
print ("计算预测结果精确度")
print ('SVC',np.mean(predicted == test_opinion))