Web Scraping Part II：Advanced Scrapers (五)：图形处理和文本识别

1
2
3
4
5
6
7
from PIL import Image, ImageFilter

kitten = Image.open("kitten.jpg")
#GaussianBlur is added to the orginal image
blurryKitten = kitten.filter(ImageFilter.GaussianBlur)
blurryKitten.save("kittent_blurred.jpg")
blurryKitten.show()

1
2
3
4
5
6
7
8
9
10
11
from PIL import Image
import pytesseract

image = Image.open("text.png")
string = pytesseract.image_to_string(image)

print(string)

# output:
# This is some text, written in Arial, that will be read by
# Tesseract. Here are some symbols: !@#$%"&*()

1
2
3
4
5
6
7
#2 slightly dirty Well-Formateed Text
image = Image.open("text2.png")
string = pytesseract.image_to_string(image)
print(string)
# output
# This is some text, mitten in Anal, 1!" _,
# Tessetact. Here are some symbols: _

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
from PIL import Image
import pytesseract

def cleanFile(filePath,newFilePath):

    image = Image.open(filePath)

    #Set a threshold value for the image, and save
    image = image.point(lambda x:0 if x<143 else 255)
    image.save(newFilePath)
    return image

image = cleanFile('text2.png','text3_cleaned.png')
string = pytesseract.image_to_string(image)
print(string)


# output
# This IS some text‘ wntten In Anal, that will be readby
# Tesseract Here are some symbols: !@#$%"&'()

可以看到输出提高了很多，尽管某些标点符号有错误。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
import time
from urllib.request import urlretrieve
import subprocess
from selenium import webdriver

#driver = webdriver.PhantomJS(executable_path='/Users/ryan/Documents/pythonscraping/code/headless/phantomjs-1.9.8-macosx/bin/phantomjs')
driver = webdriver.Chrome(executable_path="/Applications/chromedriver")
url = "https://www.amazon.com/War-Peace-Vintage-Classics-Tolstoy/dp/1400079985/ref=sr_1_1?ie=UTF8&qid=1469602093&sr=8-1&keywords=peace+and+war"
driver.get(url)
time.sleep(2)

driver.find_element_by_id("img-canvas").click()
#The easiest way to get exactly one of every page
imageList = set()

#Wait for the page to load
time.sleep(10)
print(driver.find_element_by_id("sitbReaderRightPageTurner").get_attribute("style"))
while "pointer" in driver.find_element_by_id("sitbReaderRightPageTurner").get_attribute("style"):
    #While we can click on the right arrow, move through the pages
    driver.find_element_by_id("sitbReaderRightPageTurner").click()
    time.sleep(2)
    #Get any new pages that have loaded (multiple pages can load at once)
    pages = driver.find_elements_by_xpath("//div[@class='pageImage']/div/img")
    for page in pages:
        image = page.get_attribute("src")
        imageList.add(image)

driver.quit()

#Start processing the images we've collected URLs for with Tesseract
for image in sorted(imageList):
    urlretrieve(image, "page.jpg")
    p = subprocess.Popen(["tesseract", "page.jpg", "page"], stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    p.wait()
    f = open("page.txt", "r")
    print(f.read())

1
2
3
4
5
6
7
8
9
10
from PIL import Image
import pytesseract

image = Image.open("3.1_CAPTCHA.png")
image.show()
string = pytesseract.image_to_string(image)
print(string)

#output:
#

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
from urllib.request import urlretrieve
from urllib.request import urlopen
from bs4 import BeautifulSoup
import subprocess
import requests
from PIL import Image
from PIL import ImageOps

def cleanImage(imagePath):
    image = Image.open(imagePath)
    image = image.point(lambda x: 0 if x<143 else 255)
    borderImage = ImageOps.expand(image,border=20,fill='white')
    borderImage.save(imagePath)

html = urlopen("http://www.pythonscraping.com/humans-only")
bsObj = BeautifulSoup(html.read(),"html.parser")
#Gather prepopulated form values
imageLocation = bsObj.find("img", {"title": "Image CAPTCHA"})["src"]
formBuildId = bsObj.find("input", {"name":"form_build_id"})["value"]
captchaSid = bsObj.find("input", {"name":"captcha_sid"})["value"]
captchaToken = bsObj.find("input", {"name":"captcha_token"})["value"]

captchaUrl = "http://pythonscraping.com"+imageLocation
urlretrieve(captchaUrl, "captcha.jpg")
cleanImage("captcha.jpg")
p = subprocess.Popen(["tesseract", "captcha.jpg", "captcha"], stdout=
    subprocess.PIPE,stderr=subprocess.PIPE)
p.wait()
f = open("captcha.txt", "r")

#Clean any whitespace characters
captchaResponse = f.read().replace(" ", "").replace("\n", "")
print("Captcha solution attempt: "+captchaResponse)

if len(captchaResponse) == 5:
    params = {"captcha_token":captchaToken, "captcha_sid":captchaSid,
              "form_id":"comment_node_page_form", "form_build_id": formBuildId,
                  "captcha_response":captchaResponse, "name":"Ryan Mitchell",
                  "subject": "I come to seek the Grail",
                  "comment_body[und][0][value]":
                                           "...and I am definitely not a bot"}
    r = requests.post("http://www.pythonscraping.com/comment/reply/10",
                          data=params)
    responseObj = BeautifulSoup(r.text)
    if responseObj.find("div", {"class":"messages"}) is not None:
        print(responseObj.find("div", {"class":"messages"}).get_text())
else:
    print("There was a problem reading the CAPTCHA correctly!")


# output:
# Status message
# Your comment has been queued for review by site administrators and will be published after approval.