1 sent_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
2 paragraph = "The first time I heard that song was in Hawaii on radio. 」+
3             "I was just a kid, and loved it very much! What a fantastic song!"
4 sentences = sent_tokenizer.tokenize(paragraph)
5 print(sentences)

1 from nltk.tokenize import WordPunctTokenizer
2 sentence = "Are you old enough to remember Michael Jackson attending 」+
3             「the Grammys with Brooke Shields and Webster sat on his lap during the show?"
4 words = WordPunctTokenizer().tokenize(sentence)　
5 print(words)

1 from nltk.tokenize import RegexpTokenizer
2 sentence='Thomas Jefferson began \n building \t Monticello at the age of 26.'
3 tokenizer=RegexpTokenizer(r'\w+|$[0-9.]+|\S+')
4 print(tokenizer.tokenize(sentence))

1 from nltk.tokenize import TreebankWordTokenizer
2 sentence="Sorry! I don't know."
3 tokenizer=TreebankWordTokenizer()
4 print(tokenizer.tokenize(sentence))

1 from nltk.stem import porter as pt
2 
3 words = [ 'wolves', 'playing','boys','this', 'dog', 'the',
4             'beaches', 'grounded','envision','table', 'probably']
5 stemmer=pt.PorterStemmer()
6 for word in words:
7      pt_stem = stemmer.stem(word)
8      print(pt_stem)

1 stemmer=PorterStemmer()
2 wordnet=WordNetLemmatizer()
3 word1=wordnet.lemmatize('boys',pos='n')
4 print(word1)
5 
6 word2=wordnet.lemmatize('goodness',pos='a')
7 word2=stemmer.stem(word2)
8 print(word2)                

1 stopword=stopwords.raw('english').replace('\n',' ')
2 print(stopword)

1 words = [ 'the', 'playing','boys','this', 'dog', 'a',]
2 stopword=stopwords.raw('english').replace('\n',' ')
3 words=[word for word in words if word not in stopword]
4 print(words)

1 sentence='Build the way that works best for you '+\
2                'with support for all your go-to integrations '+\
3                'including Slack Jira and more.'
4 words=sentence.split()
5 print(list(ngrams(words,2)))

1 def cut(self, sentence, cut_all=False, HMM=True,             
2              use_paddle=False):        

1 sentence='嫦娥四號著陸器地形地貌相機對玉兔二號巡視器成像'
2 word1=jieba.cut(sentence,False)
3 print(list(word1))
4 word2=jieba.cut(sentence,True)
5 print(list(word2))

1 def cut_for_search(self,
2       sentence: Any,
3       HMM: bool = True) -> Generator[str, Any, None]

1 word1=jieba.cut_for_search('Nikon Z7II是去年底全新升級的一款全片幅微單相機',False)
2 print(list(word1))
3 word2=jieba.cut_for_search('Nikon Z7II是去年底全新升級的一款全片幅微單相機',True)
4 print(list(word2))

1 word1=jieba.cut('阿里雲是全球領先的雲計算及人工智慧科技公司')
2 print(list(word1))
3 jieba.load_userdict('C://Users/Leslie/Desktop/word.txt')
4 word2=jieba.cut('阿里雲是全球領先的雲計算及人工智慧科技公司')
5 print(list(word2))

1 def add_word(self, word, freq=None, tag=None):

1 def del_word(self, word):

1 word1=jieba.cut('阿里雲是全球領先的雲計算及人工智慧科技公司')
2 print(list(word1))
3 jieba.add_word('阿里雲')
4 jieba.add_word('雲計算')
5 word2=jieba.cut('阿里雲是全球領先的雲計算及人工智慧科技公司')
6 print(list(word2))
7 jieba.del_word('阿里雲')
8 word3=jieba.cut('阿里雲是全球領先的雲計算及人工智慧科技公司')
9 print(list(word3))

1 def suggest_freq(self, segment, tune=False):

word1=jieba.cut('阿里雲是全球領先的雲計算及人工智慧科技公司')
print(list(word1))
jieba.suggest_freq('阿里雲',True)
word2=jieba.cut('阿里雲是全球領先的雲計算及人工智慧科技公司',False,False)
print(list(word2))
jieba.suggest_freq(('阿里','雲'),True)
word3=jieba.cut('阿里雲是全球領先的雲計算及人工智慧科技公司',False,False)
print(list(word3))

1 words=jieba.posseg.cut('阿里雲是全球領先的雲計算及人工智慧科技公司')
2 for word,flag in words:
3      print(word,flag)

 1 def readFile():
 2     # 讀取文件
 3     file=open('C://Users/Leslie/Desktop/word.txt','r',102400,'utf8').read()
 4     # 去標點
 5     text=re.sub('[·。，\』!\"\#$%&\'()＃！（）*+,-./:;<=>?\@，：?￥★、…．＞【】［］《》？\「\」\『\』\[\\]^_`{|}~]+'
 6                 ,'',file)
 7     # 加入常用詞
 8     jieba.add_word('雲計算')
 9     ......
10     # 利用 Jieba 分詞
11     words=jieba.lcut(text)
12     print('總詞數：{0}'.format(len(words)))
13     return words
14 
15 def stopWord():
16     # 讀取停用詞
17     stopword=[line.strip() for line in open('C://Users/Leslie/Desktop/stopword.txt','r',1024,'utf8')
18              .readlines()]
19     return stopword
20 
21 def wordFrequency():
22     # 獲取文章的詞
23     fileWords=readFile()
24     # 獲取停用詞
25     stopWords=stopWord()
26     words={}
27     # 計算詞頻
28     for word in fileWords:
29         if word not in stopWords:
30             if word not in words:
31                 words[word]=1
32             else:
33                 words[word]+=1
34     print('單詞數: {0}'.format(len(words)))
35     return words
36 
37 if __name__=='__main__':
38     words=wordFrequency()
39     for item in words.items():
40         print(item)

 1 def getWords():
 2     # 對句子進行分詞
 3     sentence='珠穆朗瑪峰上的星空是如此的迷人'
 4     words=jieba.lcut(sentence)
 5     print('【原始語句】：{0}'.format(sentence))
 6     print('【分詞數據】：{0}'.format(words))
 7     return words
 8 
 9 def getTestWords():
10     # 把詞集進行重新排序後作為測試數據
11     words=getWords().copy()
12     words.sort()
13     print('【測試數據】：{0}'.format(words))
14     return words
15 
16 def one_hot_test():
17     # 獲取分詞後數據集
18     words=getWords()
19     # 獲取測試數據集
20     testWords=getTestWords()
21     size=len(words)
22     onehot_vectors=np.zeros((size,size),int)
23     # 獲取測試數據 one_hot 向量
24     for i,word in enumerate(testWords):
25         onehot_vectors[i,words.index(word)]=1
26     return onehot_vectors
27 
28 if __name__=='__main__':
29     print(one_hot_test())

  1 # 分詞集合
  2 wordKeys=[]
  3 # 每個分詞的數量集合
  4 wordValues=[]
  5 # 每篇文章的詞量總數
  6 totalCounts=[]
  7 # tdidf 值
  8 tdidf = []
  9 
 10 def readFile(filepath):
 11     # 讀取文件
 12     file=open(filepath,'r',102400,'utf8').read()
 13     # 去標點
 14     text=re.sub('[·。，\』!\"\#$%&\'()＃！（）*+,-./:;<=>?\@，：?￥★、…．＞【】［］《》？\「\」\『\』\[\\]^_`{|}~]+'
 15                 ,'',file)
 16     # 加入常用詞
 17     jieba.add_word('雲計算')
 18     # ...
 19     # 利用 Jieba 分詞
 20     words=jieba.lcut(text)
 21     return words
 22 
 23 def stopWord():
 24     # 讀取停用詞
 25     stopword=[line.strip() for line in open('../stopword.txt','r',1024,'utf8')
 26         .readlines()]
 27     stopword.append('\n')
 28     stopword.append(' ')
 29     stopword.append('\u200b')
 30     return stopword
 31 
 32 def getFilePath():
 33     # 讀取目錄下所有文件路徑
 34     dir=os.walk('../files')
 35     filePath=[]
 36     for path,index,files in dir:
 37         for file in files:
 38             _path=os.path.join(path,file)
 39             filePath.append(_path)
 40     return filePath
 41 
 42 def getKeys():
 43     readData=[]
 44     # 獲取文件路徑
 45     filePath=getFilePath()
 46     # 獲取停用詞
 47     stopWords = stopWord()
 48     # 讀取所有文本的詞
 49     for file in filePath:
 50         readData=readData+readFile(file)
 51     # 過濾停用詞
 52     for word in readData:
 53         if word not in stopWords and word not in wordKeys:
 54             wordKeys.append(word)
 55 
 56 def getValues():
 57     # 獲取文件路徑集合
 58     filePath=getFilePath()
 59     # 行 index
 60     index=0
 61     for file in filePath:
 62         # 行值
 63         values = []
 64         # 記錄每個文檔的分詞總數
 65         totalCount=0
 66         # 獲取分詞
 67         words=readFile(file)
 68         # 獲取每篇文章的分詞數量
 69         counts=Counter(words)
 70         for key in wordKeys:
 71             if key in counts.keys():
 72                 values.append(counts[key])
 73                 totalCount+=counts[key]
 74             else:
 75                 values.append(0)
 76         # 插入行
 77         wordValues.insert(index,values)
 78         totalCounts.append(totalCount)
 79         index+=1
 80 
 81 def getTFIDF():
 82     list = np.array(wordValues)
 83     # 分行計算
 84     for index in range(0,len(wordValues)):
 85         col=0
 86         row=[]
 87         # 分別計算 TF 值與 IDF 值
 88         for value in wordValues[index]:
 89             # 計算 TF 值 
 90             tf=value / totalCounts[index]
 91             # 獲取當前列的集合
 92             cols=list[:,col]
 93             # 計算有多少篇文檔包含當前分詞
 94             nonzerocount=np.count_nonzero(cols)
 95             # 計算 IDF
 96             idf=np.log10(len(list)/(nonzerocount+1))
 97             # 計算 TFIDF 把計算結果加入集合
 98             row.append(tf*idf)
 99             col+=1
100         # 插入行
101         tdidf.insert(index,row)
102         index+=1
103 
104 if __name__=='__main__':
105     getKeys()
106     getValues()
107     # 查看過濾後每個分詞在每篇文章中的數量
108     dataset=pd.DataFrame(wordValues,columns=wordKeys)
109     print(dataset.head(3))
110     # 查看計算後的 TFIDF 值
111     getTFIDF()
112     tfidfSet=pd.DataFrame(tdidf,columns=wordKeys)
113     print(tfidfSet.head(3))

 1 class TfidfVectorizer(CountVectorizer):
 2     @_deprecate_positional_args
 3     def __init__(self, *, input='content', encoding='utf-8',
 4                  decode_error='strict', strip_accents=None, lowercase=True,
 5                  preprocessor=None, tokenizer=None, analyzer='word',
 6                  stop_words=None, token_pattern=r"(?u)\b\w\w+\b",
 7                  ngram_range=(1, 1), max_df=1.0, min_df=1,
 8                  max_features=None, vocabulary=None, binary=False,
 9                  dtype=np.float64, norm='l2', use_idf=True, smooth_idf=True,
10                  sublinear_tf=False):

 1 corpus = [
 2     'In addition to having a row context',
 3     'Usually a smaller text field',
 4     'The TFIDF idea here might be calculating some rareness of words',
 5     'The larger context might be the entire text column',
 6 ]
 7 
 8 def stopWord():
 9     # 讀取停用詞
10     stopword=[line.strip() for line in open('C://Users/Leslie/Desktop/stopword.txt','r',1024,'utf8')
11         .readlines()]
12     return stopword
13 
14 def tfidfVectorizerTest():
15     words=corpus
16     #建立tfidf模型
17     vertorizer=tfidfVectorizer(stop_words=stopWord()，ngram_range=(1,2))
18     #訓練與運算
19     model=vertorizer.fit_transform(words)
20     #顯示分詞
21     print(vertorizer.vocabulary_)
22     #顯示向量
23     print(model)
24 
25 if __name__=='__main__':
26     tfidfVectorizerTest()

 1 corpus = [
 2     '北京冬奧會完美落下帷幕',
 3     '冬奧生態內容方面的表現給用戶留下深刻印象',
 4     '全平台創作者參與冬奧內容創作',
 5     '此次冬奧會對於中國是一次重要的里程碑時刻',
 6 ]
 7 
 8 def stopWord():
 9     # 讀取停用詞
10     stopword=[line.strip() for line in open('C://Users/Leslie/Desktop/stopword.txt','r',1024,'utf8')
11         .readlines()]
12     return stopword
13 
14 def getWord():
15     # 轉換集合格式後再進行分詞
16     list=[jieba.lcut(sentence) for sentence in corpus]
17     # 在每個詞中添加空格符
18     word=[' '.join(word) for word in list]
19     return word
20 
21 def tfidfVectorizerTest():
22     words=getWord()
23     # 列印轉換格式後的分詞
24     print(str.replace('格式轉換:{0}\n'.format(words),',','\n\t\t'))
25     # 建立模型
26     vertorizer=tfidfVectorizer(stop_words=stopWord())
27     # 模型訓練
28     model=vertorizer.fit_transform(words)
29     print('分詞:{0}\n'.format(vertorizer.vocabulary_))
30     print(model)
31 
32 if __name__=='__main__':
33     tfidfVectorizerTest()        

1 def cosine_similarity(X, Y=None, dense_output=True):

 1     x0=np.array([0.895,0.745]).reshape(1,-1)
 2     y0=np.array([0.568,0.445]).reshape(1,-1)
 3     similarity0=cosine_similarity(x0,y0)
 4     print('similarity0 is:\n{0}\n'.format(similarity0))
 5 
 6     x1=np.arange(4).reshape(2,2)
 7     similarity1=cosine_similarity(x1)
 8     print('similarity1 is:\n{0}\n'.format(similarity1))
 9 
10     x2=np.arange(10).reshape(2,5)
11     y2=np.arange(15).reshape(3,5)
12     similarity2=cosine_similarity(x2,y2)
13     print('similarity2 is:\n{0}\n'.format(similarity2))

1 def cosine_distances(X, Y=None):

 1     x0=np.array([0.895,0.745]).reshape(1,-1)
 2     y0=np.array([0.568,0.445]).reshape(1,-1)
 3     distances0=cosine_distances(x0,y0)
 4     print('distances0 is:\n{0}\n'.format(distances0))
 5 
 6     x1=np.arange(4).reshape(2,2)
 7     distances1=cosine_distances(x1)
 8     print('distances1 is:\n{0}\n'.format(distances1))
 9 
10     x2=np.arange(10).reshape(2,5)
11     y2=np.arange(15).reshape(3,5)
12     distances2=cosine_distances(x2,y2)
13     print('distances2 is:\n{0}\n'.format(distances2))

 1 # 車機的命令與回複數組
 2 command=[['請打開車窗','好的，車窗已打開'],
 3          ['我要聽陳奕迅的歌','為你播放富士山下'],
 4          ['我好熱','已為你把溫度調到25度'],
 5          ['幫我打電話給小豬豬','已幫你撥小豬豬的電話'],
 6          ['現在幾點鐘','現在是早上10點'],
 7          ['我要導航到中華廣場','高德地圖已打開'],
 8          ['明天天氣怎麼樣','明天天晴']
 9         ]
10 
11 # 利用 jieba 轉換命令格式
12 def getWords():
13     comm=np.array(command)
14     list=[jieba.lcut(sentence) for sentence in comm[:,0]]
15     words=[' '.join(word) for word in list]
16     return words
17 
18 # 訓練 TfidfVectorizer 模型
19 def getModel():
20     words=getWords()
21     vectorizer=TfidfVectorizer()
22     model=vectorizer.fit(words)
23     print(model.vocabulary_)
24     return model
25 
26 # 計算 consine 餘弦相似度
27 def consine(inputCommand):
28     # 把輸入命令轉化為數組格式
29     sentence=jieba.lcut(inputCommand)
30     words=str.join(' ',sentence)
31     list=[]
32     list.insert(0,words)
33     # 獲取訓練好的 TfidfVectorizer 模型
34     model=getModel()
35     # 獲取車機命令的 TF-IDF 向量
36     data0=model.transform(getWords()).toarray().reshape(len(command),-1)
37     # 獲取輸入命令的 TF-IDF 向量
38     data1=model.transform(list).toarray().reshape(1,-1)
39     # 餘弦相似度對比
40     result=cosine_similarity(data0,data1)
41     print('相似度對比：\n{0}'.format(result))
42     return result
43 
44 if __name__=='__main__':
45     comm='我要聽陳奕迅的歌'
46     # 獲取餘弦相似度
47     result=np.array(consine(comm))
48     # 獲取相似度最高的命令 index
49     argmax=result.argmax()
50     # 讀取命令回復
51     data=command[argmax][1]
52     print('命令：{0}\n回復：{1}'.format(comm,data))

1 class TruncatedSVD(TransformerMixin, BaseEstimator):
2     @_deprecate_positional_args
3     def __init__(self, n_components=2, *, algorithm="randomized", n_iter=5,
4                  random_state=None, tol=0.):
5         self.algorithm = algorithm
6         self.n_components = n_components
7         self.n_iter = n_iter
8         self.random_state = random_state
9         self.tol = tol

 1 # 金融主題的關鍵字
 2 finance=['股份','銀行','股票','投資','股市','黃金','市場','證券','科技','有限公司']
 3 
 4 # 利用 jieba 轉換命令格式
 5 def getWords():
 6     file=open('C://Users/Leslie/Desktop/toutiao/news.txt','r',1024,'utf-8').read()
 7     sentences=np.array(file.split('\n'))
 8     # jieba 分詞
 9     list=[jieba.lcut(sentence) for sentence in sentences]
10     # 轉換中文分詞格式
11     words=[' '.join(word) for word in list]
12     return words
13 
14 # 訓練 TF-IDF 向量
15 def getTfidfVector():
16     tfidf = TfidfVectorizer()
17     # 獲取分詞
18     words=getWords()
19     # 訓練模型，返回 TF-IDF 向量
20     vector=tfidf.fit_transform(words)
21     return tfidf,vector
22 
23 def svdComponent():
24     # 獲取 TF-IDF 向量
25     tfidf,vectors=getTfidfVector()
26     # 建立 SVD 模型
27     svd=TruncatedSVD(n_components=10,n_iter=10)
28     # 獲取 TF-IDF 向量,訓練 SVD 模型
29     svd=svd.fit(vectors)
30     svd_vectors=svd.transform(vectors)
31     # 顯示主題
32     keys=tfidf.vocabulary_.keys()
33     # 獲取相關主題的 components 向量
34     dataframeComponents=pd.DataFrame(svd.components_,columns=keys)
35     # 按照 component 5 進行相關性排序
36     dataframeVectors = pd.DataFrame(svd_vectors).sort_values(5, ascending=False).head(10)
37     svdComponentDisplay(dataframeComponents)
38     svdVectorsDisplay(dataframeVectors)
39 
40 def svdComponentDisplay(dataframe):
41     # 獲取與 finance 金融有關的關鍵詞主題向量
42     topic = dataframe[finance] * 10000
43     # 顯示與金融主題相關的SVD模型component主題
44     pd.options.display.max_columns = 10
45     # 列印主題向量及向量總值
46     print(topic)
47     print(topic.T.sum())
48 
49 def svdVectorsDisplay(dataframe):
50     print(dataframe)
51     words=getWords()
52     # 金融主題 finance 關鍵字相關性語句
53     for row in dataframe.iterrows():
54         index = row[0]
55         print(words[index])
56 
57 if __name__=='__main__':
58     svdComponent()

 1 # 科技、娛樂的兩個文本路徑
 2 paths=['C://Users/Leslie/Desktop/toutiao/news_finance.txt',
 3       'C://Users/Leslie/Desktop/toutiao/news_entertainment.txt']
 4 # 科技類資訊標記為0，娛樂類資訊標記為1
 5 result = []
 6 
 7 # 利用 jieba 轉換命令格式
 8 def getWords():
 9     data = []
10     type = 0
11     # 獲取路徑中的兩類文件
12     for path in paths:
13         file=open(path,'r',1024,'utf-8').read()
14         #分行讀取，由於運行時間較長，所以只拿前2000條數據
15         sentences=np.array(file.split('\n'))[:2000]
16         # jieba 分詞，記錄分類結果
17         for sentence in sentences:
18             data.append(jieba.lcut(sentence))
19             result.append(type)
20         type+=1
21     # 轉換中文分詞格式
22     words=[' '.join(word) for word in data]
23     return words
24 
25 # 訓練 TF-IDF 向量
26 def getTfidfVector():
27     tfidf = TfidfVectorizer()
28     # 獲取分詞
29     words=getWords()
30     # 訓練模型，返回 TF-IDF 向量
31     vector=tfidf.fit_transform(words)
32     return vector.toarray()
33 
34 def ldaTest():
35     # 把 TF-IDF 向量切分為訓練數據與測試數據
36     X_train,X_test,y_train,y_test=train_test_split(getTfidfVector(),result,random_state=22)
37     # 由於只是二分類，n_components 為 1 即可
38     lda=LDA(n_components=1)
39     # 訓練模型
40     lda.fit(X_train,y_train)
41     # 輸出準確率
42     y_model=lda.predict(X_test)
43     print('準確率為：{0}'.format(accuracy_score(y_test,y_model)))
44 
45 if __name__=='__main__':
46     ldaTest()

1 class LatentDirichletAllocation(TransformerMixin, BaseEstimator):
2     @_deprecate_positional_args
3     def __init__(self, n_components=10, *, doc_topic_prior=None,
4                  topic_word_prior=None, learning_method='batch',
5                  learning_decay=.7, learning_offset=10., max_iter=10,
6                  batch_size=128, evaluate_every=-1, total_samples=1e6,
7                  perp_tol=1e-1, mean_change_tol=1e-3, max_doc_update_iter=100,
8                  n_jobs=None, verbose=0, random_state=None):

 1 # 科技、娛樂的兩個文本路徑
 2 paths=['C://Users/Leslie/Desktop/toutiao/news_finance.txt',
 3       'C://Users/Leslie/Desktop/toutiao/news_entertainment.txt']
 4 # 科技類資訊標記為0，娛樂類資訊標記為1
 5 result = []
 6 
 7 # 利用 jieba 轉換命令格式
 8 def getWords():
 9     data = []
10     type = 0
11     # 獲取路徑中的兩類文件
12     for path in paths:
13         file=open(path,'r',1024,'utf-8').read()
14         #分行讀取,為了避免訓練時間過長，只獲取 4000 行數據
15         sentences=np.array(file.split('\n'))[2000:4000]
16         # jieba 分詞，記錄分類結果
17         for sentence in sentences:
18             data.append(jieba.lcut(sentence))
19             result.append(type)
20         type+=1
21     # 轉換中文分詞格式
22     words=[' '.join(word) for word in data]
23     return words
24 
25 # 訓練 TF-IDF 向量
26 def getLdiaVector():
27     tfidf = TfidfVectorizer()
28     # 獲取分詞
29     words=getWords()
30     # 訓練模型，返回 TF-IDF 向量
31     vector=tfidf.fit_transform(words)
32     # 訓練 LDiA 模型，轉換為 200個主題
33     ldia=LDiA(n_components=200,doc_topic_prior=2e-3,topic_word_prior=1e-3,random_state=42)
34     return ldia.fit_transform(vector)
35 
36 def ldaTest():
37     # 把 TF-IDF 向量切分為訓練數據與測試數據
38     X_train,X_test,y_train,y_test=train_test_split(getLdiaVector(),result,random_state=22)
39     # 由於只是二分類，n_components 為 1 即可
40     lda=LDA(n_components=1)
41     # 訓練模型
42     lda.fit(X_train,y_train)
43     # 輸出準確率
44     y_model=lda.predict(X_test)
45     print('準確率為：{0}'.format(accuracy_score(y_test,y_model)))
46 
47 if __name__=='__main__':
48     ldaTest()

 1 @keras_export('keras.layers.Embedding')
 2 class Embedding(Layer):
 3     def __init__(self,
 4                input_dim,
 5                output_dim,
 6                embeddings_initializer='uniform',
 7                embeddings_regularizer=None,
 8                activity_regularizer=None,
 9                embeddings_constraint=None,
10                mask_zero=False,
11                input_length=None,
12                **kwargs):

 1 # 科技、娛樂的兩個文本路徑
 2 paths=['C://Users/Leslie/Desktop/toutiao/news_finance.txt',
 3        'C://Users/Leslie/Desktop/toutiao/news_entertainment.txt']
 4 # 科技類資訊標記為0，娛樂類資訊標記為1
 5 result = []
 6 # 最大單詞數
 7 max_features=10000
 8 # 單個語句最大的詞數量限制
 9 maxlen=10
10 
11 # 利用 jieba 轉換命令格式
12 def getWords():
13     data = []
14     type = 0
15     # 獲取路徑中的兩類文件
16     for path in paths:
17         file=open(path,'r',1024,'utf-8').read()
18         #分行讀取,讀取前 5000 行數據
19         sentences=np.array(file.split('\n'))[:5000]
20         # jieba 分詞，記錄分類結果
21         for sentence in sentences:
22             data.append(jieba.lcut(sentence))
23             result.append(type)
24         type+=1
25     return data
26 
27 # 自編碼
28 def getEncode():
29     # 獲取所有分詞
30     sentences=getWords()
31     encode=list()
32     words= {}
33     index=0
34     # 循環所有句子
35     for sentence in sentences:
36         array = []
37         # 把分詞轉換成編碼
38         for key in sentence:
39             if key not in words:
40                 words[key]=index
41                 index+=1
42             array.append(words[key])
43         # 記錄每個句子的編碼
44         encode.append(array)
45     # 返回自編碼
46     return encode
47 
48 #  Model
49 def getModel():
50     model=Sequential()
51     model.add(Embedding(max_features,20,input_length=maxlen))
52     model.add(Flatten())
53     model.add(Dense(1,activation='sigmoid'))
54     model.compile(optimizer='rmsprop',loss='binary_crossentropy',metrics=['acc'])
55     model.summary()
56     return model
57 
58 def test():
59     model=getModel()
60     # 獲取句子的編碼
61     encodes=getEncode()
62     X_train,X_test,y_train,y_test=train_test_split(encodes,np.array(result),random_state=22)
63     X_train=preprocessing.sequence.pad_sequences(X_train,maxlen=maxlen)
64     X_test=preprocessing.sequence.pad_sequences(X_test,maxlen=maxlen)
65     # 輸出準確率
66     history=model.fit(X_train,y_train,epochs=20,batch_size=500)
67     print(history)
68     model.fit(X_test,y_test)
69 
70 if __name__=='__main__':
71     test()

1 class Word2Vec(utils.SaveLoad):
2     def __init__(
3             self, sentences=None, corpus_file=None, vector_size=100, alpha=0.025, window=5, min_count=5,
4             max_vocab_size=None, sample=1e-3, seed=1, workers=3, min_alpha=0.0001,
5             sg=0, hs=0, negative=5, ns_exponent=0.75, cbow_mean=1, hashfxn=hash, epochs=5, null_word=0,
6             trim_rule=None, sorted_vocab=1, batch_words=MAX_WORDS_IN_BATCH,
7             compute_loss=False, callbacks=(),comment=None, max_final_vocab=None, shrink_windows=True,
8         )

 1 # 定議下載後壓縮文件的路徑，解壓轉換為簡體的新文本路徑
 2 wikipath = 'E://Tools/words/word2vec/zhwiki-latest-pages-articles.xml.bz2'
 3 filepath = 'E://Tools/words/word2vec/wiki.simple.txt'
 4 modelpath = 'E://Python_Projects/ANN/venv/word2ver_wiki_cn.model'
 5 
 6 if __name__=='__main__':
 7     convert()
 8     saveModel()
 9 
10 def saveModel():
11     # 通過 os.path 獲取路徑避免引起LineSentence路徑錯誤
12     sentencesPath = os.path.abspath(filepath)
13     modelPath = os.path.abspath(modelpath)
14     # 生成逐行讀取對象 LineSentence
15     sentences = LineSentence(sentencesPath)
16     # 建議 word2vec 對象進行學習
17     model = Word2Vec(sentences, window=8, min_count=5, workers=10)
18     # 保存模型
19     model.save(modelPath)
20 
21 def convert():
22     # 定義寫入文件對象
23     write=open(filepath,'w',10240,'utf-8')
24     # 讀取 bz2 壓縮文件
25     wiki = WikiCorpus(wikipath)
26     # 分行讀取
27     for sentences in wiki.get_texts():
28         data=''
29         # 分句讀取
30         for sentence in sentences:
31             # 把繁體字轉換為簡體字
32             simpleSentence=zhconv.convert(sentence,'zh-cn')
33             # 通過 jieba 進行分詞
34             for word in jieba.lcut(simpleSentence):
35                 data+=word+' '
36         # 換行
37         data+='\n'
38         # 寫入文件
39         write.write(data)

1 modelPath=os.path.abspath('E://Python_Projects/ANN/venv/word2ver_wiki_cn.model')
2 model=Word2Vec.load(modelPath)
3 print( model.wv[ '朱元璋' ])

 1     modelPath=os.path.abspath('E://Python_Projects/ANN/venv/word2ver_wiki_cn.model')
 2     model=Word2Vec.load(modelPath)
 3 
 4     list0=model.wv.most_similar(positive=['唐朝'],topn=10)
 5     print(list0)
 6     list1=model.wv.most_similar(positive=['唐朝'],negative=['朝代'],topn=10)
 7     print(list1)
 8     list2=model.wv.most_similar(positive=['唐朝','皇帝'],topn=10)
 9     print(list2)

 1 # 科技、娛樂、文學等16類文本路徑
 2 paths=['C://Users/Leslie/Desktop/toutiao/news_finance.txt',
 3        'C://Users/Leslie/Desktop/toutiao/news_entertainment.txt',
 4        'C://Users/Leslie/Desktop/toutiao/news_culture.txt'，
 5         ..........]
 6 # 記錄類型標記 0，1，2，3...
 7 result = []
 8 # 記錄分詞與其對應編碼
 9 words = {}
10 # 定義 maxlen 超過100個截取
11 maxlen=100
12 # 定義向量主題數
13 vector_size=100
14 
15 # 利用 jieba 轉換命令格式
16 def getWords():
17     data = []
18     type = 0
19     # 獲取路徑中的16類文件
20     for path in paths:
21         file=open(path,'r',1024,'utf-8').read()
22         #分行讀取
23         sentences=np.array(file.split('\n'))[:5000]
24         # jieba 分詞，記錄分類結果
25         for sentence in sentences:
26             data.append(jieba.lcut(sentence))
27             result.append(type)
28         type+=1
29     return data
30 
31 # 自編碼
32 def getEncode():
33     # 獲取所有分詞
34     sentences=getWords()
35     encode=list()
36     index=0
37     # 循環所有句子
38     for sentence in sentences:
39         array = []
40         # 把分詞轉換成編碼
41         for key in sentence:
42             if key not in words:
43                 words[key]=index
44                 index+=1
45             array.append(words[key])
46         # 記錄每個句子的編碼
47         encode.append(array)
48     # 返回自編碼
49     return encode
50 
51 # 獲取word2vec中對應的向量生成 Embedding 數組
52 def getEmbedding():
53     # 載入訓練好的 word2vec 模型
54     modelPath=os.path.abspath('E://Python_Projects/ANN/venv/word2ver_wiki_cn.model')
55     model=Word2Vec.load(modelPath)
56     # 向量初始化
57     embedding=np.zeros((len(words), vector_size))
58     # 若word2vec有此分詞則載入此向量，若沒有則設置為0
59     for key,value in words.items():
60         if model.wv.__contains__(key):
61             embedding[value]=model.wv.get_vector(key)
62     return embedding
63 
64 #  生成Model
65 def getModel():
66     model=Sequential()
67     model.add(Embedding(len(words),vector_size,input_length=maxlen))
68     model.add(Flatten())
69     model.add(Dense(500,activation='relu'))
70     model.add(Dense(100,activation='relu'))
71     model.add(Dense(3,activation='sigmoid'))
72     model.compile(optimizer=optimizers.Adam(0.003),
73                   loss=losses.sparse_categorical_crossentropy,
74                   metrics=['acc'])
75     model.summary()
76     return model
77 
78 def test():
79     # 獲取句子的分詞編碼
80     encodes=getEncode()
81     # 獲取模型
82     model=getModel()
83     # 在 Embedding 層加入預訓練好的 word2vec 模型
84     model.layers[0].set_weights([getEmbedding()])
85     # 訓練時不修改 word2vec 模型中的向量
86     model.layers[0].trainable=False
87     # 分拆訓練數據與測試數據
88     X_train,X_test,y_train,y_test=train_test_split(encodes,np.array(result),random_state=60)
89     X_train=preprocessing.sequence.pad_sequences(X_train,maxlen=maxlen)
90     X_test=preprocessing.sequence.pad_sequences(X_test,maxlen=maxlen)
91     # 輸出準確率
92     callback= keras.callbacks.TensorBoard(log_dir='logs')
93     history=model.fit(X_train,y_train,epochs=20,batch_size=500,callbacks=callback)
94     print(history)
95     model.fit(X_test,y_test)
96 
97 if __name__=='__main__':
98     test()         

 1 #準備數據集
 2 sentense = [['摘要','人工智慧','AI','開發'],['我們','是','機器人'],......]
 3 corpus_model = Corpus()
 4 corpus_model.fit(sentense, window=10)
 5 #訓練
 6 glove = Glove(no_components=100, learning_rate=0.05)
 7 glove.fit(corpus_model.matrix, epochs=10,
 8           no_threads=1, verbose=True)
 9 glove.add_dictionary(corpus_model.dictionary)
10 #模型保存
11 glove.save('glove.model')
12 glove = Glove.load('glove.model')
13 #語料保存
14 corpus_model.save('corpus.model')
15 corpus_model = Corpus.load('corpus.model')

 1 # 用於轉換並載入glove預訓練詞向量
 2 from gensim.test.utils import datapath, get_tmpfile
 3 from gensim.models import KeyedVectors
 4 # 將glove轉換為word2vec
 5 from gensim.scripts.glove2word2vec import glove2word2vec
 6 
 7 path='文件夾路徑'
 8 glove_file=datapath(os.path.join(path, "glove.txt"))
 9 word2vec_file=get_tmpfile(os.path.join(path,"word2vec.txt"))
10 glove2word2vec(glove_file, word2vec_file)