وب اسکرپ سایت ویکی پدیا

import nltk
import urllib
import bs4 as bs
import re # regular expression
from nltk.corpus import stopwords
nltk.download('punkt')

#get the data source
#use url to read the context
source = urllib.request.urlopen('https://simple.wikipedia.org/wiki/Global_warming')

# we need 2 parameter : 1: source 2: way of parsing
# creating beautiful soup object
soup=bs.BeautifulSoup(source,'lxml')

text=""
for paragraph in soup.find_all('p'):
    text+=paragraph.text

# preprocessing the data
# removing all the numbers and replace by ********
text1= re.sub(r'\[[0-9]*\]','******** ',text)

len(text1) #shorter lenght because numbers have been removed

#remove additional spaces
text2=re.sub(r'\s+',' ',text1)
len(text2)

text3_lower=text2.lower()

#converting to sentences
sentences=nltk.sent_tokenize(text3_lower)
len(sentences)

for i in sentences:
    print(i)
    print('----------------')