FindSimilarTPO/FindSimilarTPO.py at master · SAOHPRWHG/FindSimilarTPO · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
# -*- coding: utf-8 -*-
"""
Created on Mon Mar  2 10:36:44 2020

@author: August Yu
"""
from gensim import corpora,models,similarities
import jieba
from collections import defaultdict
import heapq
import argparse


#寻找最相似的X篇TPO文章
'''
find file name:
    Reading materials : 1-1.txt
    Listening materials : C1-1.txt(Conversation) or L1-1.txt(Lecture)
Each TPO have 3 reading materials, 2 conversations and 4 lectures.
'''
parser = argparse.ArgumentParser(description='Find Similar TPO')
parser.add_argument(
    "-f",
    '--filename',
    metavar="FILE",
    help="filename: \nReading materials : 1-1.txt \n Listening materials : C1-1.txt(Conversation) or L1-1.txt(Lecture)",
    type=str,
)
parser.add_argument(
    "-n",
    '--num',
    default = 10,
    help="Show how many records",
    type=int,
)
parser.add_argument(
    "-t",
    '--tpo',
    default = 54,
    help="Find in how many TPO, default is 54, you may change to 64 if you got them",
    type=int,
)

argv = parser.parse_args()
#print(argv)

find_file = argv.filename
sim_num = argv.num #show the similar text number要显示的最相似的数目
TPO_num = argv.tpo #目前可访问的TPO数目，如增加到64则改为64


lib = []
for i in range(1,TPO_num+1):
    for j in range(1,4):
        file_name = str(i)+'-'+str(j)+'.txt'
        doc = './data/TPO/'
        doc += file_name
        #print(doc)
        d = open(doc, encoding='utf-8').read()
        data = jieba.cut(d)
        data_ = ''
        for item in data:
            data_+=item+' '
        lib.append(data_)
    for j in range(1,5):
        file_name = 'L' + str(i)+'-'+str(j)+'.txt'
        doc = './data/TPO/'
        doc += file_name
        d = open(doc, encoding='utf-8').read()
        data = jieba.cut(d)
        data_ = ''
        for item in data:
            data_+=item+' '
        lib.append(data_)
    for j in range(1,3):
        file_name = 'C' + str(i)+'-'+str(j)+'.txt'
        doc = './data/TPO/'
        doc += file_name
        d = open(doc, encoding='utf-8').read()
        data = jieba.cut(d)
        data_ = ''
        for item in data:
            data_+=item+' '
        lib.append(data_)


#存储文档到列表
#documents=[data11,data21]
texts=[[word for word in document.split()]
       for document in lib]

#计算词语的频率
frequency=defaultdict(int)#构建频率对象
for text in texts:
    for token in text:
        frequency[token]+=1
'''
#如果词汇量过多，去掉低频词
texts=[[word for word in text if frequency[token]>3]
 for text in texts]
'''
#通过语料库建立词典
dictionary=corpora.Dictionary(texts)
dictionary.save('./dictionary.txt')
#加载要对比文档
doc3='./data/TPO/'

#find_file = '1-1.txt'
doc3+=find_file

d3=open(doc3,encoding='utf-8').read()
data3=jieba.cut(d3)
data31=''
for item in data3:
    data31+=item+' '
new_doc=data31
new_vec=dictionary.doc2bow(new_doc.split())#转换为稀疏矩阵
#得到新的语料库
corpus=[dictionary.doc2bow(text) for text in texts]

ifidf=models.TfidfModel(corpus)
featureNUm=len(dictionary.token2id.keys())#得到特征数
index=similarities.SparseMatrixSimilarity(ifidf[corpus],num_features=featureNUm)
sim=index[ifidf[new_vec]]

r = {}
l = {}
for i in range(1,TPO_num+1):
    for j in range(1,4):
        file_name = str(i)+'-'+str(j)+'.txt'
        ind = (i-1)*9 + j
        r[file_name] = sim[ind-1]*100
    for j in range(1,5):
        file_name = 'L' + str(i)+'-'+str(j)+'.txt'
        ind = (i-1)*9 + 3 + j
        l[file_name] = sim[ind-1]*100
    for j in range(1,3):
        file_name = 'C' + str(i)+'-'+str(j)+'.txt'
        ind = (i-1)*9 + 7 + j
        l[file_name] = sim[ind-1]*100

#sim_num = 10
outputR = heapq.nlargest(sim_num, r, key = r.get)
outputL = heapq.nlargest(sim_num, l, key = l.get)
#print(output)
print('Reading:')
print('Rank\tName\t\tSimilarity')
for i in range(len(outputR)):
    print(i,'\t', outputR[i],':\t',r[outputR[i]])

print('\nListening:')
print('Rank\tName\t\tSimilarity')
for i in range(len(outputL)):
    print(i,'\t', outputL[i],':\t',l[outputL[i]])