问性能:提高朴素贝叶斯分类器的准确性

EN

import nltk, random

from nltk.corpus import brown, stopwords
from nltk.stem.porter import PorterStemmer

documents = [(list(brown.words(fileid)), category)
        for category in brown.categories()
        for fileid in brown.fileids(category)]

random.shuffle(documents)

stop = set(stopwords.words('english'))


without_stop = [w for w in brown.words() if w not in stop] 

lowercase = [w.lower() for w in without_stop] # lowercase


porter = PorterStemmer()
stemmed = [porter.stem(w) for w in lowercase] 

wnl = nltk.WordNetLemmatizer()
lemmatized = [wnl.lemmatize(w) for w in stemmed] 
tagged = nltk.pos_tag(lemmatized) 


all_words = nltk.FreqDist(tagged) 

word_features = list(all_words.keys())[:2000]

def document_features(document):
    document_words = set(document)
    features = {}
    for word in word_features:
       features['contains({})'.format(word)] = (word in document_words)
    return features


featuresets = [(document_features(d), c) for (d,c) in documents]
train_set, test_set = featuresets[100:], featuresets[:100]

classifier = nltk.NaiveBayesClassifier.train(train_set)

print(nltk.classify.accuracy(classifier, test_set))