垃圾邮件分类2(2)

当前位置:

首页 > temp > 简明python教程 >

垃圾邮件分类2(2)

3.数据划分—训练集和测试集数据划分

from sklearn.model_selection import train_test_split

x_train,x_test, y_train, y_test = train_test_split(data, target, test_size=0.2, random_state=0, stratify=y_train)

				
										# 3、划分数据集

										def split_dataset(data, label):

										     x_train, x_test, y_train, y_test = train_test_split(data, label, test_size=0.2, random_state=0, stratify=label)

										     return x_train, x_test, y_train, y_test

4.文本特征提取

sklearn.feature_extraction.text.CountVectorizer

https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html?highlight=sklearn%20feature_extraction%20text%20tfidfvectorizer

sklearn.feature_extraction.text.TfidfVectorizer

https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html?highlight=sklearn%20feature_extraction%20text%20tfidfvectorizer#sklearn.feature_extraction.text.TfidfVectorizer

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf2 = TfidfVectorizer()

观察邮件与向量的关系

向量还原为邮件

				
										# 4、文本特征提取

										# 把文本转化为tf-idf的特征矩阵

										def tfidf_dataset(x_train,x_test):

										     tfidf = TfidfVectorizer()

										     X_train = tfidf.fit_transform(x_train) 

										     X_test = tfidf.transform(x_test)

										     return X_train, X_test, tfidf

										# 向量还原成邮件

										def revert_mail(x_train, X_train, model):

										    s = X_train.toarray()[0]

										    print("第一封邮件向量表示为：", s)

										    a = np.flatnonzero(X_train.toarray()[0])  # 非零元素的位置（index）

										    print("非零元素的位置:", a)

										    print("向量的非零元素的值：", s[a])

										    b = model.vocabulary_  # 词汇表

										    key_list = []

										    for key, value in b.items():

										        if value in a:

										            key_list.append(key)  # key非0元素对应的单词

										    print("向量非零元素对应的单词：", key_list)

										    print("向量化之前的邮件：", x_train[0])

栏目列表