Web Scraping Part I：Basic Scrapers (六)：阅读文档

1
2
3
4
5
6
7
8
from urllib.request import urlopen

textFileHandler = urlopen("http://www.pythonscraping.com/pages/warandpeace/chapter1.txt")
textContent = textFileHandler.read()
print(textContent)

Output:
# b'CHAPTER I\n\n"Well, Prince, so Genoa and Lucca

1
2
3
4
5
6
7
8
from urllib.request import urlopen

textFileHandler = urlopen("http://www.pythonscraping.com/pages/warandpeace/chapter1.txt")
textContent = textFileHandler.read()
print(textContent)

#Output:
# b"\xd0\xa7\xd0\x90\xd0\xa1\xd0\xa2\xd0\xac

1
2
3
4
5
6
7
8
9
10
from urllib.request import urlopen
from bs4 import BeautifulSoup

html = urlopen("http://en.wikipedia.org/wiki/Python_(programming_language)")
bsObj = BeautifulSoup(html,"html.parser")
content = bsObj.find("div", {"id":"mw-content-text"}).get_text()
content = bytes(content, "UTF-8")  #先转回二进制数
content = content.decode("UTF-8")  #再用UTF-8解码

print(content)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
from urllib.request import urlopen
import csv
from io import StringIO

csvFileHandler = urlopen("http://pythonscraping.com/files/MontyPythonAlbums.csv")

csvString = csvFileHandler.read().decode("ascii",'ignore')

#创建StringIO实例(内存中读写字符串),该实例和fileHandler类似,因此可以在使用fileHandler的地方使用StringIO实例。
memory = StringIO(csvString)
csvReader = csv.reader(memory)   #csv.reader(memory) 返回一个Iterator，每一个item是list
for row in csvReader:
    print(row)

# Output:
# ['Name', 'Year']
# ["Monty Python's Flying Circus", '1970']
# ['Another Monty Python Record', '1971']

1
2
3
4
5
6
7
for row in csvReader:
    print("The album \"" + row[0] + "\" was relased in "+ row[1])

# Output:
# The album "Name" was relased in Year
# The album "Monty Python's Flying Circus" was relased in 1970
# The album "Another Monty Python Record" was relased in 1971

1
2
3
4
5
6
7
8
csvReader = csv.DictReader(memory)
for row in csvReader:
    print(row)

# Output:
# {'Name': "Monty Python's Flying Circus", 'Year': '1970'}
# {'Name': 'Another Monty Python Record', 'Year': '1971'}
# {'Name': "Monty Python's Previous Record", 'Year': '1972'}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
from urllib.request import urlopen
from pdfminer.pdfinterp import PDFResourceManager, process_pdf
from pdfminer.layout import LAParams
from pdfminer.converter import TextConverter
from io import StringIO


def readPDF(pdfFile):
    resourceManager = PDFResourceManager()
    retstr = StringIO()
    laparams = LAParams()
    device = TextConverter(resourceManager,retstr,laparams=laparams)

    process_pdf(resourceManager,device,pdfFile)
    device.close()

    content = retstr.getvalue()
    retstr.close()
    return content

pdfFile = urlopen("http://pythonscraping.com/pages/warandpeace/chapter1.pdf")
# you can change the pdfFile from local path
# pdfFile = open("chapter1.pdf",'rb')
outputString = readPDF(pdfFile)
print(outputString)
pdfFile.close()

# Output:
# CHAPTER I
# 
# "Well, Prince, so Genoa and Lucca are now just family estates of 
# theBuonapartes. But I warn you, if you don't tell me that this

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
from urllib.request import urlopen
from zipfile import ZipFile
from io import BytesIO


wordFileHandler = urlopen("http://pythonscraping.com/pages/AWordDocument.docx")
wordFileString = wordFileHandler.read()             #read String from wordFielHandler
worldFielByte = BytesIO(wordFileString)             #turn String into Bytes
docuement = ZipFile(worldFielByte)                  #unzip it, because all the .docx are zipped to save space，return a fileHandler
xml_content = docuement.read('word/document.xml')   #read the unzipped file into String for name 'word/document.xml'
print(xml_content.decode('utf-8'))

# Output:
# <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
# <w:document xmlns: /w:pPr><w:r><w:t>A Word Document on a Website</w:t></w:r><w:bookmarkStart w:id="0" w:name="_GoBack"/><w:bookmarkEnd w:id="0"/></w:p><w:p w:rsidR="00764658" w:rsidRDefault="00764658" w:rsidP="00764658"/><w:p w:rsidR="00764658" w:rsidRPr="00764658" w:rsidRDefault="00764658" w:rsidP="00764658"><w:r><w:t>This is a Word document, full of content that you want very much. Unfortunately, it’s difficult to access because I’m putting it on my website as a .</w:t></w:r><w:proofErr w:type="spellStart"/><w:r><w:t>docx</w:t></w:r><w:proofErr w:type="spellEnd"/><w:r><w:t xml:space="preserve"> file, rather than just publishing it as HTML</w:t></w:r></w:</w:document>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
from urllib.request import urlopen
from zipfile import ZipFile
from io import BytesIO
from bs4 import BeautifulSoup

wordFileHandler = urlopen("http://pythonscraping.com/pages/AWordDocument.docx")
wordFileString = wordFileHandler.read()             #read String from wordFielHandler
worldFielByte = BytesIO(wordFileString)             #turn String into Bytes
docuement = ZipFile(worldFielByte)                  #unzip it, because all the .docx are zipped to save space
xml_content = docuement.read('word/document.xml')   #read the unzipped file into string,存储为word/document.xml文件。

bsObj = BeautifulSoup(xml_content,'html.parser')    #BeautifulSoup也可以用来解析XML
texts= bsObj.findAll("w:t")
for text in texts:
    print(text.get_text())

# Output:
# A Word Document on a Website
# This is a Word document, full of content that you want very much. Unfortunately, it’s difficult to access because I’m putting it on my website as a .
# docx
#  file, rather than just publishing it as HTML

1
2
3
4
5
6
7
8
9
10
11
12
13
14
...
bsObj = BeautifulSoup(xml_content,'html.parser')    #BeautifulSoup也可以用来解析XML
textTags= bsObj.findAll("w:t")
for textTag in textTags:
    try:
        titleTag = textTag.parent.previousSibling.find("w:pstyle",{"w:val": "Title"})
        if titleTag is not None:
            print("<h1>",textTag.text,"</h1>")
    except AttributeError:
        pass


# Output:
# <h1> A Word Document on a Website </h1>