第 5 章：抓取实时站点

• 我可以拿这个数据吗？
• 我可以重新发布这些数据吗？
• 我是否使网站的服务器超载？
• 我可以用这些数据做什么？

• 机器人排除协议
• 服务条款

User-agent: *
Disallow: /

User-agent: [whom this applies to]
Disallow/Allow: [the directory or folder that should not be crawled or scraped]

--snip--
User-agent: Googlebot
Disallow: /ajax/
Disallow: /album.php
Disallow: /checkpoint/
Disallow: /contact_importer/
Disallow: /feeds/
Disallow: /file_download.php
Disallow: /hashtag/
Disallow: /l.php
Disallow: /live/
Disallow: /moments_app/
Disallow: /p.php
Disallow: /photo.php
Disallow: /photos.php
Disallow: /sharer/
--snip--

# Import our modules or packages that we will need to scrape a website
import csv 
from bs4 import BeautifulSoup
import requests
# make an empty array for your data
rows = []

# open the website
url = "https://en.wikipedia.org/wiki/Category:Women_computer_scientists"①
page = requests.get(url)②
page_content = page.content③
# parse the page with the Beautiful Soup library
soup = BeautifulSoup(page_content, "html.parser")

<div class="mw-category-group"><h3>A</h3>
<ul><li><a href="/wiki/Janet_Abbate" title="Janet Abbate">Janet Abbate</a></li>
<li><a href="/wiki/T%C3%BClay_Adal%C4%B1" title="Tülay Adalı">Tülay Adalı</a></li>
<li><a href="/wiki/Sarita_Adve" title="Sarita Adve">Sarita Adve</a></li>
<li><a href="/wiki/Dorit_Aharonov" title="Dorit Aharonov">Dorit Aharonov</a></li>
<li><a href="/wiki/Anastasia_Ailamaki" title="Anastasia Ailamaki">Anastasia Ailamaki</a></li>
<li><a href="/wiki/Susanne_Albers" title="Susanne Albers">Susanne Albers</a></li>
<li><a href="/wiki/Frances_E._Allen" title="Frances E. Allen">Frances E. Allen</a></li>
<li><a href="/wiki/Sarah_Allen_(software_developer)" title="Sarah Allen (software developer)">Sarah Allen (software developer)</a></li>
<li><a href="/wiki/Nancy_M._Amato" title="Nancy M. Amato">Nancy M. Amato</a></li>
<li><a href="/wiki/Pat_Fothergill" title="Pat Fothergill">Pat Fothergill</a></li>
<li><a href="/wiki/Nina_Amenta" title="Nina Amenta">Nina Amenta</a></li>
<li><a href="/wiki/Dana_Angluin" title="Dana Angluin">Dana Angluin</a></li>
<li><a href="/wiki/Annie_Ant%C3%B3n" title="Annie Antón">Annie Antón</a></li>
<li><a href="/wiki/Cecilia_R._Aragon" title="Cecilia R. Aragon">Cecilia R. Aragon</a></li>
<li><a href="/wiki/Gillian_Arnold_(technologist)" title="Gillian Arnold (technologist)">Gillian Arnold (technologist)</a></li>
<li><a href="/wiki/Chieko_Asakawa" title="Chieko Asakawa">Chieko Asakawa</a></li>
<li><a href="/wiki/Winifred_Asprey" title="Winifred Asprey">Winifred Asprey</a></li>
<li><a href="/wiki/Stella_Atkins" title="Stella Atkins">Stella Atkins</a></li>
<li><a href="/wiki/Hagit_Attiya" title="Hagit Attiya">Hagit Attiya</a></li>
<li><a href="/wiki/Terri_Attwood" title="Terri Attwood">Terri Attwood</a></li>
<li><a href="/wiki/Donna_Auguste" title="Donna Auguste">Donna Auguste</a></li>
<li><a href="/wiki/Chrisanthi_Avgerou" title="Chrisanthi Avgerou">Chrisanthi Avgerou</a></li>
<li><a href="/wiki/Henriette_Avram" title="Henriette Avram">Henriette Avram</a></li></ul></div>

<div class="mw-category-group">①
    <h3>A</h3>②
    <ul>③
      ④ <li><a href="/wiki/Janet_Abbate" title="Janet Abbate"⑤>Janet Abbate</a></li>
        <li><a href="/wiki/T%C3%BClay_Adal%C4%B1" title="Tülay Adalı">Tülay Adalı</a></li>
        <li><a href="/wiki/Sarita_Adve" title="Sarita Adve">Sarita Adve</a></li>
        <li><a href="/wiki/Dorit_Aharonov" title="Dorit Aharonov">Dorit Aharonov</a></li>
        <li><a href="/wiki/Anastasia_Ailamaki" title="Anastasia Ailamaki">Anastasia Ailamaki</a></li>
        <li><a href="/wiki/Susanne_Albers" title="Susanne Albers">Susanne Albers</a></li>
        <li><a href="/wiki/Frances_E._Allen" title="Frances E. Allen">Frances E. Allen</a></li>
        <li><a href="/wiki/Sarah_Allen_(software_developer)" title="Sarah Allen (software developer)">Sarah Allen (software developer)</a></li>
        <li><a href="/wiki/Nancy_M._Amato" title="Nancy M. Amato">Nancy M. Amato</a></li>
        <li><a href="/wiki/Pat_Fothergill" title="Pat Fothergill">Pat Fothergill</a></li>
        <li><a href="/wiki/Nina_Amenta" title="Nina Amenta">Nina Amenta</a></li>
        <li><a href="/wiki/Dana_Angluin" title="Dana Angluin">Dana Angluin</a></li>
        <li><a href="/wiki/Annie_Ant%C3%B3n" title="Annie Antón">Annie Antón</a></li>
        <li><a href="/wiki/Cecilia_R._Aragon" title="Cecilia R. Aragon">Cecilia R. Aragon</a></li>
        <li><a href="/wiki/Gillian_Arnold_(technologist)" title="Gillian Arnold (technologist)">Gillian Arnold (technologist)</a></li>
        <li><a href="/wiki/Chieko_Asakawa" title="Chieko Asakawa">Chieko Asakawa</a></li>
        <li><a href="/wiki/Winifred_Asprey" title="Winifred Asprey">Winifred Asprey</a></li>
        <li><a href="/wiki/Stella_Atkins" title="Stella Atkins">Stella Atkins</a></li>
        <li><a href="/wiki/Hagit_Attiya" title="Hagit Attiya">Hagit Attiya</a></li>
        <li><a href="/wiki/Terri_Attwood" title="Terri Attwood">Terri Attwood</a></li>
        <li><a href="/wiki/Donna_Auguste" title="Donna Auguste">Donna Auguste</a></li>
        <li><a href="/wiki/Chrisanthi_Avgerou" title="Chrisanthi Avgerou">Chrisanthi Avgerou</a></li>
        <li><a href="/wiki/Henriette_Avram" title="Henriette Avram">Henriette Avram</a></li>
    </ul>
</div>

• 从第一页获取所有无序列表的名字。
• 从每个列表项中提取信息，包括女计算机科学家的姓名、指向她的个人资料的链接以及她所属的字母。
• 根据此信息创建一行数据，并将每一行写入一个 .csv 文件。

--snip--
soup = BeautifulSoup(page_content, "html.parser")
content = soup.find("div", class_="mw-category")①
all_groupings = content.find_all("div", class_="mw-category-group")②

--snip--
all_groupings = content.find_all("div", class_="mw-category-group")
for grouping in all_groupings:①
    names_list = grouping.find("ul")②
    category = grouping.find("h3").get_text()③
    alphabetical_names = names_list.find_all("li")④

--snip--
    category = grouping.find("h3").get_text()
    alphabetical_names = names_list.find_all("li")
    for alphabetical_name in alphabetical_names:①
        # get the name
        name = alphabetical_name.text②
        # get the link
        anchortag = alphabetical_name.find("a",href = True)③
        link = anchortag["href"]④
        # get the letter
        letter_name = category

--snip--
        # make a dictionary that will be written into the csv
        row = {"name": name,①
               "link": link,②
               "letter_name": letter_name}③
        rows.append(row)④

# Import our modules or packages that we will need to scrape a website
import csv 
from bs4 import BeautifulSoup
import requests
# make an empty list for your data
rows = []
# open the website
url = "https://en.wikipedia.org/wiki/Category:Women_computer_scientists"
page = requests.get(url)
page_content = page.content
# parse the page through the BeautifulSoup library
soup = BeautifulSoup(page_content, "html.parser")
content = soup.find("div", class_="mw-category")
all_groupings = content.find_all("div", class_="mw-category-group")
for grouping in all_groupings:
    names_list = grouping.find("ul")
    category = grouping.find("h3").get_text()
    alphabetical_names = names_list.find_all("li")
    for alphabetical_name in alphabetical_names:
        # get the name
        name = alphabetical_name.text
        # get the link
        anchortag = alphabetical_name.find("a",href=True)
        link = anchortag["href"]
        # get the letter
        letter_name = category
        # make a data dictionary that will be written into the csv
        row = { "name": name,
                "link": link,
                "letter_name": letter_name}
        rows.append(row)

# Import our modules or packages that we will need to scrape a website
import csv 
from bs4 import BeautifulSoup
import requests
# make an empty list for your data
rows = []
① def scrape_content(url):
    ② page = requests.get(url)
    page_content = page.content
    # parse the page through the BeautifulSoup library
    soup = BeautifulSoup(page_content, "html.parser")
    content = soup.find("div", class_="mw-category")
    all_groupings = content.find_all("div", class_="mw-category-group")
    for grouping in all_groupings:
        names_list = grouping.find("ul")
        category = grouping.find("h3").get_text()
        alphabetical_names = names_list.find_all("li")
        for alphabetical_name in alphabetical_names:
        # get the name
            name = alphabetical_name.text
            # get the link
            anchortag = alphabetical_name.find("a",href=True)
            link = anchortag["href"]
            # get the letter
            letter_name = category
            # make a data dictionary that will be written into the csv
            row = { "name": name,
                    "link": link,
                    "letter_name": letter_name}
            rows.append(row)

url = "https://en.wikipedia.org/wiki/Category:Women_computer_scientists"

scrape_content("<https://en.wikipedia.org/wiki/Category:Women_computer_scientists>")

# open the website
urls = ①["https://en.wikipedia.org/wiki/Category:Women_computer_scientists", "https://en.wikipedia.org/w/index.php?title=Category:Women_computer_scientists&pagefrom=Lin%2C+Ming+C.%0AMing+C.+Lin#mw-pages"]
def scrape_content(url):
--snip--
            rows.append(row)
for url in urls:②
    scrape_content(url)③

--snip--
# make a new csv into which we will write all the rows
with open("all-women-computer-scientists.csv", "w+") as csvfile:①
    # these are the header names:
    fieldnames = ["name", "link", "letter_name"]②
    # this creates your csv
    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)③
    # this writes in the first row, which are the headers
    writer.writeheader()④
    # this loops through your rows (the array you set at the beginning and have updated throughtout)
    for row in rows:⑤
        # this takes each row and writes it into your csv
        writer.writerow(row)⑥

--snip--
headers = {①"user-agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36",
            ② "from": "Your name example@domain.com"}
--snip--
    page = requests.get(url, headers=headers)③

--snip--
# Import our libraries that we will need to scrape a website
import csv 
import time①
from bs4 import BeautifulSoup
import requests
time.sleep(2)②

# Import our modules or packages that we will need to scrape a website
import csv
import time
from bs4 import BeautifulSoup
import requests
# Your identification
headers = {"user-agent" : "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36;",
            "from": "Your name <example@domain.com>"}
# make an empty array for your data
rows = []
# open the website
urls = ["https://en.wikipedia.org/wiki/Category:American_women_computer_scientists", "https://en.wikipedia.org/w/index.php?title=Category:American_women_computer_scientists&pagefrom=Lehnert%2C+Wendy%0AWendy+Lehnert#mw-pages"]
def scrape_content(url):
    time.sleep(2)
    page = requests.get(url, headers= headers)
    page_content = page.content
    # parse the page through the BeautifulSoup library
    soup = BeautifulSoup(page_content, "html.parser")
    content = soup.find("div", class_="mw-category")
    all_groupings = content.find_all("div", class_="mw-category-group")
    for grouping in all_groupings:
        names_list = grouping.find("ul")
        category = grouping.find("h3").get_text()
        alphabetical_names = names_list.find_all("li")
        for alphabetical_name in alphabetical_names:
            # get the name
            name = alphabetical_name.text
            # get the link
            anchortag = alphabetical_name.find("a",href=True)
            link = anchortag["href"]
            # get the letter
            letter_name = category
            # make a data dictionary that will be written into the csv
            row = { "name": name,
                    "link": link,
                    "letter_name": letter_name}
            rows.append(row)
for url in urls:
    scrape_content(url)
# make a new csv into which we will write all the rows
with open("all-women-computer-scientists.csv", "w+") as csvfile:
    # these are the header names:
    fieldnames = ["name", "link", "letter_name"]
    # this creates your csv
    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
    # this writes in the first row, which are the headers
    writer.writeheader()
    # this loops through your rows (the array you set at the beginning and have updated throughtout)
    for row in rows:
        # this takes each row and writes it into your csv
        writer.writerow(row)

python3 wikipediascraper.py

python wikipediascraper.py