RanqueamentoPLN/preprocessing.py at master · viksiem/RanqueamentoPLN · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize
from nltk.tokenize import word_tokenize
from nltk.stem.snowball import SnowballStemmer
import string
import os
from collections import Counter
import math
from itertools import repeat

_punctuation = string.punctuation
_stemmer = SnowballStemmer('english')
_stopwords_list = stopwords.words('english')


#os.chdir('C:\Users\meiski\Desktop\RanqueamentoPLN\corpus')
os.chdir('/home/meiski/PycharmProjects/RanqueamentoPLN/corpus')
_LOG_BASE = 10


# ref: https://wiki.python.org.br/TudoSobrePythoneUnicode
def read_file(i_file):
    i_file = open(i_file)
    data_file = i_file.read()
    data_file = unicode(data_file, 'utf-8')
    return data_file


def seg_into_senteces(doc):
    sentences = sent_tokenize(doc)
    return sentences


def seg_into_words(doc):
    words = []
    for d in doc:
        words += word_tokenize(d)
    return words


def add_stopwords():
    _stopwords_list.append('e.g.')
    _stopwords_list.append('without')
    _stopwords_list.append('low')
    _stopwords_list.append('moreover')
    _stopwords_list.append('include')
    _stopwords_list.append('including')
    _stopwords_list.append('de')
    _stopwords_list.append('variants')
    _stopwords_list.append('annotation')


def remove_stopwords(doc):
    add_stopwords()
    finalwords = [w for w in doc if not w.lower() in _stopwords_list]

    return finalwords


def remove_punctuation(doc):
    punctuation_free = [w for w in doc if not w in _punctuation]
    return punctuation_free


def reduce_tostem(doc):
    doc_stems = [_stemmer.stem(w) for w in doc]
    return doc_stems


def count_frequencies(doc):
    terms_plus_freq = Counter(doc).most_common()
    return terms_plus_freq


def flog_tf(_doc_frequency):
    f_docfrequency = []
    for n in range(len(_doc_frequency)):
        f_docfrequency.append(float("%.3f" % (1 + math.log(_doc_frequency[n][1], _LOG_BASE))))
    return f_docfrequency


def tlog_tf(_doc_frequency):
    t_docfrequency = []
    for n in range(len(_doc_frequency)):
        t_docfrequency.append(_doc_frequency[n][0])
    return t_docfrequency


# em quantos documentos o termo aparece
def doc_frequency(_terms_of_all, _docterms):  # docterms = 20, terms of all = 1194
    df = []
    #dfd = []
    for t in _terms_of_all:  # t = algum termo
        tmp = 0
        for d in _docterms:
            if t in d:
                tmp += 1
                #tmp = d.count(t)
            else:
                tmp += 0
                #tmp = 0
            #dfd.append(tmp)
            #tmp = 0
        df.append(tmp)
        #del dfd[:]
    return df


def idf(_df, n_docs):
    _idf = []
    for p in range(len(_df)):
        _idf.append(float("%.3f" % (math.log(n_docs / _df[p], _LOG_BASE))))
    return _idf


#_tf_idf = [[TERMO1[DOC1, DOC2, ...,DOC20], TERMO2[DOC1, DOC2, ...,DOC20]]
def tf_idf(_log_terms, _log_freq, _idf, _docterms, _final_terms):
    bydocTFIDF =[]
    _tf_idf = [] #for j in repeat(None, len(_final_terms))]
    for i, term in enumerate(_final_terms):
        for n, doc in enumerate(_docterms):
            if term in doc:
                #print 'IF'
                termindex = _log_terms[n].index(term)
                bydocTFIDF.append(float("%.3f" % (_log_freq[n][termindex] * _idf[i])))

            else:
                #print 'ELSE'
                bydocTFIDF.append(0)
        #tmp = list(bydocTFIDF)
        _tf_idf.append(list(bydocTFIDF))
        del bydocTFIDF[:]

    return _tf_idf