PRML_Project/text_preprocessing.py at main · bhaveshAswani112/PRML_Project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
import re
import string
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer

def textprocessing(text):
    text = str(text).lower()

    # Remove URLs
    text = re.sub(r'https?://\S+|www\.\S+', '', text)

    # Remove square brackets and their contents
    text = re.sub(r'\[.*?\]', '', text)

    # Replace non-word characters with a space
    text = re.sub(r'\W', ' ', text)

    # Remove XML tags
    text = re.sub(r'<.*?>+', '', text)

    # Remove punctuation
    text = re.sub(r'[%s]' % re.escape(string.punctuation), '', text)

    # Remove newline characters
    text = re.sub(r'\n', '', text)

    # Remove alphanumeric characters
    text = re.sub(r'\w*\d\w*', '', text)


    # Remove '@' and '#' symbols
    text = re.sub(r'\@\w+|\#', " ", text)


    # Tokenization
    text_tokens = word_tokenize(text)

    # Remove stopwords
    stop_words = set(stopwords.words('english')) - {'not','no' , 'never'}
    text_no_stopwords = [word for word in text_tokens if word not in stop_words]

    # Lemmatization
    lem = SnowballStemmer('english')
    text_lemmatized = [lem.stem(word) for word in text_no_stopwords]
    text_processed = " ".join(text_lemmatized)

    return text_processed