|
从网站抓取训练样本数据，代码： import urllib2
from BeautifulSoup import BeautifulSoup
import sys
import re
import time
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
url=['http://news.baidu.com/n?cmd=4&class=mil&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=finannews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=internet&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=housenews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=autonews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=sportnews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=enternews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=gamenews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=edunews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=healthnews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=technnews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=socianews&pn=1&from=tab']
ff=['E:/baidu/军事.txt' ,'E:/baidu/财经.txt' ,'E:/baidu/互联网.txt' ,'E:/baidu/房产.txt' ,'E:/baidu/汽车.txt' ,'E:/baidu/体育.txt' ,'E:/baidu/娱乐.txt' ,'E:/baidu/游戏.txt' ,'E:/baidu/教育.txt' ,'E:/baidu/女人.txt' ,'E:/baidu/科技.txt' ,'E:/baidu/社会.txt']
for j in range(7,8):
soup=BeautifulSoup(urllib2.urlopen(url[j]).read())
main=soup.find('div',{'class':'p2'})
index=main.findAll('a')
len_0=len(index)
a=[]
for i in range(len_0):
a.append(index[i]['href'])
for i in range(len_0):
try:
soup=BeautifulSoup(urllib2.urlopen(a[i]).read())
txt=soup.findAll(text=re.compile(ur'[\u4e00-\u9fa5] '))
txt_=''.join(txt)
f= open(ff[j],'a')
print >>f,txt_
f.close()
except:
continue
有监督学习的文本分类代码： import jieba
import os
import sys
import codecs
from sklearn import feature_extraction
from sklearn import svm
from sklearn.neighbors import KNeighborsClassifier
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn import tree
from sklearn.naive_bayes import MultinomialNB
#--------------#
def load_data():
corpus_train=[]
target_train=[]
filepath='E:\python_pananteng/程序6：文本挖掘/文本分类/实例2/train'
filelist = os.listdir(filepath)
for num in range(len(filelist)):
filetext=filepath '/' filelist[num]
filename=os.path.basename(filetext)
myfile = codecs.open(filetext, 'r','utf-8')
temp=myfile.readlines()
myfile.close()
for i in range(0,100):
len_0=len(temp)
seg_list=jieba.cut(','.join(temp[int(i*len_0/100):int((i 1)*len_0/100)]), cut_all=False)
words=' '.join(seg_list)
target_train.append(filename)
corpus_train.append(words)
#--------------#
corpus_test=[]
target_test=[]
filepath='E:\python_pananteng/程序6：文本挖掘/文本分类/实例2/test'
filelist = os.listdir(filepath)
for num in range(len(filelist)):
filetext=filepath '/' filelist[num]
myfile = open(filetext, 'r')
temp=myfile.readlines()
myfile.close()
seg_list=jieba.cut(','.join(temp[1:]), cut_all=False)
words=' '.join(seg_list)
target_test.append(temp[0])
corpus_test.append(words)
return [[corpus_train,target_train],[corpus_test,target_test]]
#--------------#
def data_pro():
[[corpus_train,target_train],[corpus_test,target_test]]=load_data()
count_v1=CountVectorizer()
#该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频
counts_train=count_v1.fit_transform(corpus_train)
#fit_transform是将文本转为词频矩阵
transformer=TfidfTransformer()
#该类会统计每个词语的tf-idf权值
tfidf_train=transformer.fit(counts_train).transform(counts_train)
#fit_transform是计算tf-idf
weight_train=tfidf_train.toarray()
#weight[i][j],第i个文本，第j个词的tf-idf值
count_v2=CountVectorizer(vocabulary=count_v1.vocabulary_)
#让两个CountVectorizer共享vocabulary
counts_test=count_v2.fit_transform(corpus_test)
#fit_transform是将文本转为词频矩阵
transformer=TfidfTransformer()
#该类会统计每个词语的tf-idf权值
tfidf_test=transformer.fit(counts_train).transform(counts_test)
#fit_transform是计算tf-idf
weight_test=tfidf_test.toarray()
#weight[i][j],第i个文本，第j个词的tf-idf值
return [[weight_train,target_train],[weight_test,target_test]]
#--------------#
[[weight_train,target_train],[weight_test,target_test]]=data_pro()
#---------------------------------------------#
knnclf = KNeighborsClassifier()
knnclf.fit(weight_train,target_train)
knn_pred = knnclf.predict(weight_test)
#knn模型
#---------------------------------------------#
#---------------------------------------------#
#svm模型
svc = svm.SVC(kernel='linear')
svc.fit(weight_train,target_train)
svc_pred = svc.predict(weight_test)
#---------------------------------------------#
#---------------------------------------------#
#tree模型
tre = tree.DecisionTreeClassifier()
tre.fit(weight_train,target_train)
tre_pred = tre.predict(weight_test)
#---------------------------------------------#
#---------------------------------------------#
#bayes模型
bayes = MultinomialNB(alpha = 0.01)
bayes.fit(weight_train,target_train)
bayes_pred = bayes.predict(weight_test)
#---------------------------------------------#
调用两个开源库，分别是 1、结巴中文分词库，运用该库对网页抓取的中文文章进行分词 2、sklearn机器学习库，调用里面的算法有：tf-idf算法，将文本转换为特征数字矩阵；及knn算法、svm算法、naivebeyes算法、cart算法，这三个算法都是分类的算法，作用是对网页抓取的文章进行有监督的分类学习 效果： 训练：样本 1001 个，其中有3类文章， 第一类，互联网类，样本数量 300个 第二类，军事类，样本数量309个 第三类，财经类，样本数量302个
测试：150个测试样本 KNN算法，命中118个，错误32个 SVM算法，命中125个，错误25个 CART算法，命中122个，错误28个 Bayes算法，命中130个，错误20个 |
|
|
