Web Scraping Part II：Advanced Scrapers (二)：读写自然语言

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
...
txtFile = urlopen("http://pythonscraping.com/files/inaugurationSpeech.txt")
content = str(txtFile.read(), 'utf-8')

ngrams = ngrams(content,2)
sortedNGrams = OrderedDict(sorted(ngrams.items(), key=lambda item: item[1], reverse=True))

for k,v in sortedNGrams.items():
    print(k,v)

#Output
# of the 212
# in the 62
# to the 60
# by the 41
# the Constitution 32
# of our 29
# to be 26
# ...

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
...
def isCommon(ngram):
    commonWords = ["the", "be", "and", "of", "a", "in", "to", "have", "it",
            "i", "that", "for", "you", "he", "with", "on", "do", "say", "this",
            "they", "is", "an", "at", "but","we", "his", "from", "that", "not",
            "by", "she", "or", "as", "what", "go", "their","can", "who", "get",
            "if", "would", "her", "all", "my", "make", "about", "know", "will",
            "as", "up", "one", "time", "has", "been", "there", "year", "so",
            "think", "when", "which", "them", "some", "me", "people", "take", "out", "into",                 "just", "see", "him", "your", "come", "could", "now",
            "than", "like", "other", "how", "then", "its", "our", "two", "more",
            "these", "want", "way", "look", "first", "also", "new", "because",
            "day", "more", "use", "no", "man", "find", "here", "thing", "give",
            "many", "well"]
    for word in ngram:
        if word in commonWords:
            return True
    return False
...

#Output
# united states 10
# general government 4
# executive department 4
# government should 3
# same causes 3
# chief magistrate 3
# mr jefferson 3
# called upon 3

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
from urllib.request import urlopen
from random import randint
import re

#get the total occurence of a word
def subsequentialWordsTotalOccurence(subsequentialWordsDictionary):
    sum = 0
    for word, occurence in subsequentialWordsDictionary.items():
        sum += occurence
    return sum

#get the random next word of the word
def retrieveSubsequentialWordRandomly(subsequentialWordsDictionary):
    randomIndex = randint(1,subsequentialWordsTotalOccurence(subsequentialWordsDictionary))

    for word, occurence in subsequentialWordsDictionary.items():
        randomIndex -=occurence
        if (randomIndex <=0):
            return word

#build a dictionary of dictionary, as the input of Markov Models
def buildWordsDict(text):

    punctuation = [',',':','.','?','!']

    for symbol in punctuation:
        text = text.replace(symbol," "+symbol+" ")

    text = re.sub("\n+"," ",text)
    text = re.sub(" +"," ",text)
    # text = text.lower()
    words = text.split(" ")
    words = [word for word in words if word != ""]

    mainWordDict = dict()

    for i in range(len(words)-1):
        word = words[i]
        subsequentialWord = words[i+1]
        if word not in mainWordDict:
            mainWordDict[word] = dict()
        subsequentialWordsDict = mainWordDict[word]
        if subsequentialWord not in subsequentialWordsDict:
            subsequentialWordsDict[subsequentialWord] = 1
        else:
            subsequentialWordsDict[subsequentialWord] +=1

    return mainWordDict

#get words dictionary
textFile = urlopen("http://pythonscraping.com/files/inaugurationSpeech.txt")
content = str(textFile.read(), 'utf-8')
wordDict = buildWordsDict(content)

wordsLength = 100
makeupSentence = ""
startWord = "I"


for i in range(wordsLength):
    makeupSentence += startWord + " "
    startWord = retrieveSubsequentialWordRandomly(wordDict[startWord])

print(makeupSentence)

# Output
# I sincerely believe to make all that they respectively claim or rather of that of the lapse of their own independence and maintain their country . Some of the great a motive for all that instrument could have acted in war , as is in the way of ultimate decision there was observable as the world may be content with our Constitution to justify me , it is not to control the best historians agree in legislation could then been the Constitution from the tendency of them his assent . Without denying that can unmake , nothing upon the want 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
from urllib.request import urlopen
from bs4 import BeautifulSoup
import pymysql


connection = pymysql.connect(host='127.0.0.1', unix_socket='/tmp/mysql.sock',
                        user='root', passwd='640401', db='mysql', charset='utf8')
cursor = connection.cursor()
cursor.execute("USE wikipedia")

class SolutionFound(RuntimeError):
    def __init__(self,message):
        self.message = message

def getLinks(fromgPageID):
    cursor.execute("SELECT toPageID from links where fromPageID = %s",(fromgPageID))
    if cursor.rowcount == 0:
        return None
    else:
        # the results row only contains toPageID, thus the first item is selected
        return [x[0] for x in cursor.fetchall()]

def constructDict(curentPageID):
    links = getLinks(curentPageID)
    if links:
        # zip key list and value list(item is dictionary) into a dictionary
        return dict(zip(links,[{}]*len(links)))
    return {}

#The link tree may either be empty or contain multiple links

def searchDepth(targetPageID, currentPageID, linkTree, depth):

    if depth == 0:
        #Stop recursing and return, regardless
        return linkTree

    if not linkTree:
        linkTree = constructDict(currentPageID)
        if not linkTree:
            #No links found. Cannot continue at this code
            return {}

    if targetPageID in linkTree.keys():
        print("TARGET " + str(targetPageID) + " Found!")
        raise SolutionFound("Page: " + str(currentPageID))

    for branchKey, branchValue in linkTree.items():
        try:
            #Recurse here to continue building the tree
            linkTree[branchKey] = searchDepth(targetPageID,branchKey,branchValue,depth-1)
        except SolutionFound as e:
            print(e.message)
            raise SolutionFound("PAGE: " + str(currentPageID))
    return linkTree


try:
    searchDepth(18885,1,{},4)
    print("No Solution FOund")
except SolutionFound as e:
    print(e.message)

#Output
# TARGET 18885 Found!   //Terry_Jones
# Page: 18830           //Douglas_Adams
# PAGE: 3               //San_Diego_Comic-Con_International
# PAGE: 1               //Kevin Bacon
# 即Kevin Bacon -> San_Diego_Comic-Con_International -> Douglas Adams -> Terry Jones

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
from nltk import word_tokenize,FreqDist,bigrams,ngrams
from nltk import Text
from nltk.book import text6  #直接可用book下的attribute,如果用improt nltk,book,你得book.attribute


#1. text object
tokens = word_tokenize("Here is some not very interesting text")
text = Text(tokens)
print(len(text6)/len(text))

# Output
# 2423.8

#2. freuqncy distribution
fdist = FreqDist(text6)
print(fdist.most_common(10))
print(fdist["Grail"])

# Output
# [(':', 1197), ('.', 816), ('!', 801), (',', 731), ("'", 421), ('[', 319), (']', 312), ('the', 299), ('I', 255), ('ARTHUR', 225)]
# 34

#3. n-grams
bigrams = bigrams(text6)
bigramsDist =FreqDist(bigrams)
print(bigramsDist[("Sir","Robin")])

# Output
# 18

fourgrams = ngrams(text6,4)
fourgamsDist =FreqDist(fourgrams)
print(fourgamsDist[("father","smelt","of","elderberries")])

# Output
# 1

#4. iteration for text object, frequency distribution, ngrams

fourgrams = ngrams(text6,4)
for fourgram in fourgrams:
    if fourgram[0] == "father":
        print(fourgram)

# Output
# ('father', 'smelt', 'of', 'elderberries')
# ('father', 'owns', 'the', 'biggest')
# ('father', ',', 'who', 'wishes')
# ('father', ',', 'that', "'")
# ('father', '--', 'GUEST', '#')
# ('father', '--', 'GUEST', '#')
# ('father', ',', 'who', ',')

1
2
3
4
5
6
7
8
from nltk import word_tokenize,Text,pos_tag
from nltk.book import *

text = word_tokenize("The dust was thick so he had to dust")
print(pos_tag(text))

# output
# [('The', 'DT'), ('dust', 'NN'), ('was', 'VBD'), ('thick', 'RB'), ('so', 'RB'), ('he', 'PRP'), ('had', 'VBD'), ('to', 'TO'), ('dust', 'VB')]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
from nltk import word_tokenize,sent_tokenize,pos_tag

sentences = sent_tokenize("google is one of the best companies in the world. I constantly google myself to see what I'm up to.")
nouns = ['NN','NNS','NNP','NNPS']

for sentence in sentences:
    if "google" in sentence.lower():
        taggedWords = pos_tag(word_tokenize(sentence))
        for word in taggedWords:
            if word[0].lower() == "google" and word[1] in nouns:
                print(sentence)

# output
# google is one of the best companies in the world.