问使用NGram情绪分析-无法获得前5个单词

EN

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.svm import LinearSVC
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression

text_train = ["The quick brown fox jumps over a lazy dog",
        "Pack my box with five dozen liquor jugs",
        "How quickly daft jumping zebras vex",
        "The five boxing wizards jump quickly",
        "the fox of my friend it the most lazy one I have seen in the past five years"]

text_test = ["just for a test"]

target_train = [1, 1, 0, 1, 0]

target_test = [1]

#######################################################################
##       OBSERVING TOKENIZATION OF DATA WITH AND WITOUT NGRAMS       ##
#######################################################################

## WITHOUT NGRAMS

cv = CountVectorizer()
count_vector = cv.fit_transform(text_train)
#Display the dictionary pairing each single word and it's position in the
#"vectorized" version of our text corpus, without any count.
print("")
print(cv.vocabulary_)
print("")
print("")
print(dict(zip(cv.get_feature_names(), count_vector.toarray().sum(axis=0))))

##  WITH NGRAMS

#Now let's also add as meaningfull entities all pair and all trios of words
#using NGrams
cv = CountVectorizer(ngram_range=(1,3))
count_vector = cv.fit_transform(text_train)
#Observe that now, "jump quickly" and "large fawn jumped" for instance are 
#considered as sort of meaningful unique "words" composed of several unique
#words.
print("")
print("")
print(cv.vocabulary_)
print("")
print("")
#List of all words and counts their occurences
print(dict(zip(cv.get_feature_names(), count_vector.toarray().sum(axis=0))))

#######################################################################
##                    YOUR ATTEMPT WITH LINEARSVC                    ##
#######################################################################
cv1 = CountVectorizer(binary=True)
count_vector_train = cv1.fit_transform(text_train)
count_vector_test = cv1.transform(text_test)

final_svm  = LinearSVC(C=1.0)
final_svm.fit(count_vector_train, target_train)
final_accuracy = final_svm.predict(count_vector_test)
final_accuracy_score = accuracy_score(target_test, final_accuracy)
print("")
print("")
print ("Final SVM without NGrams Accuracy: %s" % final_accuracy_score)
feature_names = zip(cv1.get_feature_names(), final_svm.coef_[0])
feature_to_coef = {
    word: coef for word, coef in feature_names
}
itemz = feature_to_coef.items()
list_positive = sorted(
    itemz, 
    key=lambda x: x[1], 
    reverse=True)[:5] #Here you can choose the top 5
print("")
print("SVM without NGrams")
print(list_positive)

#######################################################################
##              YOUR ATTEMPT WITH LOGISTIC REGRESSION                ##
#######################################################################
cv2 = CountVectorizer(binary=True)
count_vector_train = cv2.fit_transform(text_train)
count_vector_test = cv2.transform(text_test)

final_lr  = LogisticRegression(C=1.0)
final_lr.fit(count_vector_train, target_train)
final_accuracy = final_lr.predict(count_vector_test)
final_accuracy_score = accuracy_score(target_test, final_accuracy)
print("")
print("")
print ("Final Logistic regression without NGrams Accuracy: %s" % final_accuracy_score)
feature_names = zip(cv2.get_feature_names(), final_lr.coef_[0])
feature_to_coef = {
    word: coef for word, coef in feature_names
}
itemz = feature_to_coef.items()
list_positive = sorted(
    itemz, 
    key=lambda x: x[1], 
    reverse=True)[:5] #Here you can choose the top 5
print("")
print("Logistic regression without NGrams")
print(list_positive)

#######################################################################
##         YOUR ATTEMPT WITH LOGISTIC REGRESSION AND NGRAMS          ##
#######################################################################
cv3 = CountVectorizer(binary=True, ngram_range=(1,3))
count_vector_train = cv3.fit_transform(text_train)
count_vector_test = cv3.transform(text_test)

final_lr  = LogisticRegression(C=1.0)
final_lr.fit(count_vector_train, target_train)
final_accuracy = final_lr.predict(count_vector_test)
final_accuracy_score = accuracy_score(target_test, final_accuracy)
print("")
print("")
print ("Final Logistic regression with NGrams Accuracy: %s" % final_accuracy_score)
feature_names = zip(cv3.get_feature_names(), final_lr.coef_[0])
feature_to_coef = {
    word: coef for word, coef in feature_names
}
itemz = feature_to_coef.items()
list_positive = sorted(
    itemz, 
    key=lambda x: x[1], 
    reverse=True)[:5] #Here you can choose the top 5
print("")
print("Logistic regression with NGrams")
print(list_positive)

#######################################################################
##         YOUR ATTEMPT WITH LOGISTIC REGRESSION AND NGRAMS          ##
##                BUT EXTRACTS ONLY REAL UNIQUE WORDS                ##
#######################################################################
cv4 = CountVectorizer(binary=True, ngram_range=(1,3))
count_vector_train = cv4.fit_transform(text_train)
count_vector_test = cv4.transform(text_test)

final_lr  = LogisticRegression(C=1.0)
final_lr.fit(count_vector_train, target_train)
final_accuracy = final_lr.predict(count_vector_test)
final_accuracy_score = accuracy_score(target_test, final_accuracy)
print("")
print("")
print ("Final Logistic regression with NGrams Accuracy: %s" % final_accuracy_score)
feature_names = zip(cv4.get_feature_names(), final_lr.coef_[0])
feature_names_unigrams = [(a, b) for a, b in feature_names if len(a.split()) < 2]
feature_to_coef = {
    word: coef for word, coef in feature_names_unigrams
}
itemz = feature_to_coef.items()

list_positive = sorted(
    itemz,
    key=lambda x: x[1], 
    reverse=True)[:5] #Here you can choose the top 5
print("")
print("Logistic regression with NGrams but only getting unigrams")
print(list_positive)