Web Scraping Part I：Basic Scrapers (四)：BeautifulSoup+API

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
                                                                                 
from twitter import *

ACCESS_TOKEN = "***"
ACCESS_SECRET = "***"
CONSUMER_KEY = "***"
CONSUMER_SECRET = "***"


t = Twitter(auth=OAuth(ACCESS_TOKEN,ACCESS_SECRET,CONSUMER_KEY,CONSUMER_SECRET))
'''
任务1:搜索tweets
pythonTweets = t.search.tweets(q = "#python")
print(pythonTweets)
'''

'''
任务2:发布Twitter
statusUpdate = t.statuses.update(status="Hello, world!!!")
print(statusUpdate)
'''

'''
任务3:搜索特定用户的tweets
pythonStatuses = t.statuses.user_timeline(screen_name="montypython",count=5)
print(pythonStatuses)
'''

1
2
3
4
5
6
7
//任务1：搜索地址
https://maps.googleapis.com/maps/api/geocode/ json?address=1+Science+Park+Boston
    +MA+02114&key=YOUR_API_KEY

//任务2：搜索时区
 https://maps.googleapis.com/maps/api/timezone/json?location=42.3677994,-71.0708
    078&timestamp=1412649030&key=<your API key>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
import json
from urllib.request import urlopen

def getCountry(ipAddress):
    JSONFileHandler = urlopen("http://freegeoip.net/json/"+ipAddress)   #获取JSONFile句柄
    JSONFileString = JSONFileHandler.read().decode('utf-8')             #用UTF-8读取成String
    responseJSON = json.loads(JSONFileString)                           #json模块loads该String
    return responseJSON.get("country_code")                             #获取其"country_code"对应的内容。

print(getCountry("168.70.77.50"))

#Output:
#HK


jsonString = '{"arrayOfNums":[{"number":0},{"number":1},{"number":2}],"arrayOfFruits":[{"fruit": "apple"}, {"fruit": "banana"},{"fruit": "pear"}]}'
jsonObj = json.loads(jsonString)
print(jsonObj.get("arrayOfNums"))
print(jsonObj.get("arrayOfNums")[1])
print(jsonObj.get("arrayOfNums")[1].get("number") + jsonObj.get("arrayOfNums")[2].get("number"))
print(jsonObj.get("arrayOfFruits")[2].get("fruit"))

# #Output:
# [{'number': 0}, {'number': 1}, {'number': 2}]
# {'number': 1}
# 3
# pear

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
from urllib.request import urlopen
from urllib.error import HTTPError
from bs4 import BeautifulSoup
import json
import re
import datetime
import random

def getArticalTags(inputUrl):

    url = "https://en.wikipedia.org" + inputUrl
    htmlHandler = urlopen(url)
    bsObject = BeautifulSoup(htmlHandler.read(), "html.parser")

    return bsObject.find("div",{"id":"bodyContent"}).findAll("a",href=re.compile("^(\/wiki)((.(?!:))*)$"))

def getHistoryIPs(articalUrl):
    # articalUrl="/wiki/Six_Degrees_of_Kevin_Bacon"

    # revision history page pattern
    # https://en.wikipedia.org/w/index.php?title=Kevin_Bacon&action=history
    # https://en.wikipedia.org/w/index.php?title=Six_Degrees_of_Kevin_Bacon&action=history
    title = articalUrl.replace("/wiki/","")
    print("revision history of ",title," -------------------------")
    revisionPageUrl = "https://en.wikipedia.org/w/index.php?title="+title+"&action=history"

    htmlHandler = urlopen(revisionPageUrl)
    bsObject = BeautifulSoup(htmlHandler.read(), "html.parser")
    IPs= set()
    for IPTag in bsObject.find("ul",{"id":"pagehistory"}).findAll("a",{"class":"mw-anonuserlink"}):
        if IPTag.string not in IPs:
            IPs.add(IPTag.string)
    return IPs

def getIPCountry(IP):
    JSONFileHandler = urlopen("http://freegeoip.net/json/" + IP)
    JSONFileString = JSONFileHandler.read().decode('utf-8')
    JSONObject = json.loads(JSONFileString)
    IPCountry = JSONObject.get('country_name')
    return IPCountry



def start(startPage):
    articalTags = getArticalTags(startPage)
    for articalTag in articalTags:
        articalLink = articalTag.attrs["href"]
        for IP in getHistoryIPs(articalLink):
            print(IP, " from: ",getIPCountry(IP))

    while(len(articalTags)>0):
        i = random.randint[0,random.randint(len(articalTags)-1)]
        randomArticalTag = articalTags[i]
        start(randomArticalTag)

start("/wiki/Kevin_Bacon")

#Output:
# revision history of  Kevin_Bacon_(disambiguation)  -------------------------
# revision history of  San_Diego_Comic-Con_International  -------------------------
# 70.67.239.79  from:  Canada
# 220.221.249.249  from:  Japan
# 2602:306:b883:4f0:198d:191a:4f08:5b16  from:  United States
# 2600:8801:9604:3900:7273:cbff:fe5c:336d  from:  United States
# 96.250.2.62  from:  United States
# 134.223.230.151  from:  United States
# 126.229.33.149  from:  Japan
# 1.79.84.210  from:  Japan
# 126.204.160.80  from:  Japan
# 91.122.178.55  from:  Russia
# 176.182.35.113  from:  France
# revision history of  Philadelphia  -------------------------