Lowercase & Noise removal

In [1]:
import re
In [2]:
def cleanup(txt):
    txt = re.sub('@\w+',' ', txt)             # remove @mentions
    txt = re.sub('\w+:\/\/\S+', ' ', txt)     # remove link://...
    txt = re.sub('[^0-9a-zA-Z \t]', ' ', txt) # remove punctuation
    return txt.strip().lower()
In [3]:
txt = '@Craig_Spur @kev_g1 @davspurs We have put a number now on every single head within the Covid world which makes people in the public feel that every loss is personal to them and I do get it but you think people are gonna go out an exercise when we tell them 17m die globally a year through crap lifestyles?'
print(txt)

@Craig_Spur @kev_g1 @davspurs We have put a number now on every single head within the Covid world which makes people in the public feel that every loss is personal to them and I do get it but you think people are gonna go out an exercise when we tell them 17m die globally a year through crap lifestyles?
In [4]:
print(cleanup(txt))

we have put a number now on every single head within the covid world which makes people in the public feel that every loss is personal to them and i do get it but you think people are gonna go out an exercise when we tell them 17m die globally a year through crap lifestyles

Tokenization

In [5]:
from nltk.tokenize import word_tokenize, sent_tokenize
In [6]:
txt = 'An electrocardiogram is used to record the electrical conduction through a person\'s heart. The readings can be used to diagnose cardiac arrhythmias.'
print(txt)

An electrocardiogram is used to record the electrical conduction through a person's heart. The readings can be used to diagnose cardiac arrhythmias.
In [7]:
# Split on spaces
print(txt.split(' '))

['An', 'electrocardiogram', 'is', 'used', 'to', 'record', 'the', 'electrical', 'conduction', 'through', 'a', "person's", 'heart.', 'The', 'readings', 'can', 'be', 'used', 'to', 'diagnose', 'cardiac', 'arrhythmias.']
In [8]:
# Split on words
print(word_tokenize(txt))

['An', 'electrocardiogram', 'is', 'used', 'to', 'record', 'the', 'electrical', 'conduction', 'through', 'a', 'person', "'s", 'heart', '.', 'The', 'readings', 'can', 'be', 'used', 'to', 'diagnose', 'cardiac', 'arrhythmias', '.']
In [9]:
# Split on sentences
print(sent_tokenize(txt))

["An electrocardiogram is used to record the electrical conduction through a person's heart.", 'The readings can be used to diagnose cardiac arrhythmias.']

Stopwords removal

In [ ]:
import nltk
nltk.download('stopwords')
In [10]:
from nltk.corpus import stopwords
In [11]:
stopwords_eng = set(stopwords.words('english'))
len(stopwords_eng)
Out[11]:
179
In [12]:
stopwords_eng
Out[12]:
{'a',
 'about',
 'above',
 'after',
 'again',
 'against',
 'ain',
 'all',
 'am',
 'an',
 'and',
 'any',
 'are',
 'aren',
 "aren't",
 'as',
 'at',
 'be',
 'because',
 'been',
 'before',
 'being',
 'below',
 'between',
 'both',
 'but',
 'by',
 'can',
 'couldn',
 "couldn't",
 'd',
 'did',
 'didn',
 "didn't",
 'do',
 'does',
 'doesn',
 "doesn't",
 'doing',
 'don',
 "don't",
 'down',
 'during',
 'each',
 'few',
 'for',
 'from',
 'further',
 'had',
 'hadn',
 "hadn't",
 'has',
 'hasn',
 "hasn't",
 'have',
 'haven',
 "haven't",
 'having',
 'he',
 'her',
 'here',
 'hers',
 'herself',
 'him',
 'himself',
 'his',
 'how',
 'i',
 'if',
 'in',
 'into',
 'is',
 'isn',
 "isn't",
 'it',
 "it's",
 'its',
 'itself',
 'just',
 'll',
 'm',
 'ma',
 'me',
 'mightn',
 "mightn't",
 'more',
 'most',
 'mustn',
 "mustn't",
 'my',
 'myself',
 'needn',
 "needn't",
 'no',
 'nor',
 'not',
 'now',
 'o',
 'of',
 'off',
 'on',
 'once',
 'only',
 'or',
 'other',
 'our',
 'ours',
 'ourselves',
 'out',
 'over',
 'own',
 're',
 's',
 'same',
 'shan',
 "shan't",
 'she',
 "she's",
 'should',
 "should've",
 'shouldn',
 "shouldn't",
 'so',
 'some',
 'such',
 't',
 'than',
 'that',
 "that'll",
 'the',
 'their',
 'theirs',
 'them',
 'themselves',
 'then',
 'there',
 'these',
 'they',
 'this',
 'those',
 'through',
 'to',
 'too',
 'under',
 'until',
 'up',
 've',
 'very',
 'was',
 'wasn',
 "wasn't",
 'we',
 'were',
 'weren',
 "weren't",
 'what',
 'when',
 'where',
 'which',
 'while',
 'who',
 'whom',
 'why',
 'will',
 'with',
 'won',
 "won't",
 'wouldn',
 "wouldn't",
 'y',
 'you',
 "you'd",
 "you'll",
 "you're",
 "you've",
 'your',
 'yours',
 'yourself',
 'yourselves'}
In [13]:
txt = "NBC was founded in 1926 making it the oldest major broadcast network in the USA"
txt
Out[13]:
'NBC was founded in 1926 making it the oldest major broadcast network in the USA'
In [14]:
" ".join([w for w in word_tokenize(txt)
            if w not in stopwords_eng])
Out[14]:
'NBC founded 1926 making oldest major broadcast network USA'

Spelling correction

https://pyspellchecker.readthedocs.io/en/latest/code.html

In [ ]:
!pip install spellchecker pyspellchecker
In [15]:
from spellchecker import SpellChecker
In [16]:
txt = 'The qiuck brown fox jmps over the lazy dog'
txt
Out[16]:
'The qiuck brown fox jmps over the lazy dog'
In [17]:
spell = SpellChecker()

" ".join([spell.correction(w)
            for w in word_tokenize(txt)])
Out[17]:
'The quick brown fox mps over the lazy dog'

Stemming

In [18]:
from nltk.stem import PorterStemmer
In [19]:
txt = "NBC was founded in 1926 making it the oldest major broadcast network in the USA"
txt
Out[19]:
'NBC was founded in 1926 making it the oldest major broadcast network in the USA'
In [20]:
stemmer = PorterStemmer()

" ".join([stemmer.stem(w)
            for w in word_tokenize(txt)])
Out[20]:
'nbc wa found in 1926 make it the oldest major broadcast network in the usa'

Lemmatization

In [ ]:
import nltk
nltk.download('wordnet')
In [21]:
from nltk.stem import WordNetLemmatizer
In [22]:
lemmatizer = WordNetLemmatizer()

" ".join([lemmatizer.lemmatize(w)
            for w in word_tokenize(txt)])
Out[22]:
'NBC wa founded in 1926 making it the oldest major broadcast network in the USA'

Lemmatization with Part-of-speech

In [23]:
from nltk import pos_tag
In [24]:
pos_tag(word_tokenize(txt))
Out[24]:
[('NBC', 'NNP'),
 ('was', 'VBD'),
 ('founded', 'VBN'),
 ('in', 'IN'),
 ('1926', 'CD'),
 ('making', 'VBG'),
 ('it', 'PRP'),
 ('the', 'DT'),
 ('oldest', 'JJS'),
 ('major', 'JJ'),
 ('broadcast', 'NN'),
 ('network', 'NN'),
 ('in', 'IN'),
 ('the', 'DT'),
 ('USA', 'NNP')]

Part-of-speed tags of pos_tag() are different than WordNet's, so we need to convert them

In [25]:
treebank_wordnet_pos = {
    'J': 'a', # adjective
    'V': 'v', # verb
    'N': 'n', # noun
    'R': 'r', # adverb
}

def get_wordnet_pos(treebank_pos, default='n'):
    return treebank_wordnet_pos.get(treebank_pos[0], default)
In [26]:
" ".join([lemmatizer.lemmatize(w[0], get_wordnet_pos(w[1]))
            for w in pos_tag(word_tokenize(txt))])
Out[26]:
'NBC be found in 1926 make it the old major broadcast network in the USA'

Synonyms

In [27]:
from nltk.corpus import wordnet as wn
In [28]:
synonyms = wn.synsets('nonetheless')

if len(synonyms) != 0:
    syn = synonyms[0]

    print(syn.pos(), syn._lemma_names[0])

r however