def_open_file(self,filename):#打开文件,返回所有单词list with open(filename,'r',encoding='utf-8')as f: raw_words = f.read() low_words = raw_words.lower() words = re.findall('[a-z] ',low_words) #正则re找到所有单词 return words
剔除常用单词(is am are do……):
def_filter_words(self,raw_words,count=NUMBERS):#载入未处理的所有单词列表 和 默认count值 new_words = [] for word in raw_words:#找出非exclude 和 长度大于1 的单词 -> new_words if word notin exclude_list and len(word) > 1: new_words.append(word) pass
计数:
from collections import Counter #计数器 pass c = Counter(words) #list new_words return c.most_common(5000) #拿到出现次数最多的5000单词,返回从大到小的排序list[(and,1),....]
数据库初始化Peewee模块:
from peewee import * db = SqliteDatabase('voca.db') class NewWord(Model): # 单词名 name = CharField() # 解释 explanation = TextField(default='') # 词频 frequency = IntegerField(default=0) # 音标 phonogram = CharField(default='') class Meta: database = db
加入单词到数据库:
definsert_data(self,words_times): # 向数据库内插入数据 for word,fre in words_times: word_ins = NewWord.create(name = word , frequency = fre) #直接调用create book.is_analyzed = True book.save()
翻译:
#iciba翻译函数 deftrans(self, word): url = 'http://www.iciba.com/index.php?a=getWordMean&c=search&word=' word try: req = requests.get(url) req.raise_for_status() info = req.json() data = info['baesInfo']['symbols'][0] assert info['baesInfo']['symbols'][0] # 去除没有音标的单词 assert data['ph_am'] and data['ph_en'] # 去除没有词性的单词 assert data['parts'][0]['part'] except: return ('none','none') ph_en = '英 [' data['ph_en'] ']' ph_am = '美 [' data['ph_am'] ']' ex = '' for part in data['parts']: ex = part['part'] ';'.join(part['means']) ';' return ph_en ph_am, ex #调用翻译函数,保存中文到数据库 for i in NewWord.select(): i.explanation = str(t.trans(i.name)[1]) i.save()
提取所有单词到CSV:
import csv #提取所有数据库内容生成迭代对象 yield ~ 好好看看如何使用 defextract() pass for word in NewWord.select(): for i in [word.name, word.explanation, word.frequency]: datas.append(i) yield datas #保存函数 defsave(data): with open('words.csv', 'a ', errors='ignore', newline='')as f: csv_writer = csv.writer(f) csv_writer.writerow(data) #主程序 datas = extract() #yeild 迭代对象 whileTrue: try: data = next(datas) except: break save(data)