知識星球
本次爬取用到的知識點有:
1. selenium
2. pymysql
3 pyquery
'''
文章：Python爬蟲專案--爬取某寶男裝資訊
作者：Star_Zhao
源自：https://www.cnblogs.com/star-zhao/p/9664564.html
'''
正文
1. 分析標的網站
1. 開啟某寶首頁, 輸入”男裝”後點選”搜尋”, 則跳轉到”男裝”的搜尋介面.
2. 空白處”右擊”再點選”檢查”審查網頁元素, 點選”Network”.
1) 找到對應的URL, URL裡的引數正是Query String Parameters的引數, 且請求方式是GET
2) 我們請求該URL得到內容就是”Response”裡的內容, 那麼點選它來確認資訊.
3) 下拉看到”男裝”字樣, 那麼再往下找, 並沒有發現有關”男裝”的商品資訊.
4) 任意複製一個商品資訊, 空白處右擊再點選”檢視網頁原始碼”, 在原始碼查詢該商品, 即可看到該商品的資訊.
5) 對比網頁原始碼和”Response”響應內容, 發現原始碼中的商品資訊被替換, 這便是採用了JS加密
6) 如果去請求上面的URL, 得到的則是加密過的資訊, 這時就可以利用Selenium庫來模擬瀏覽器, 進而得到商品資訊.
2. 獲取單個商品介面
1. 請求網站
# -*- coding: utf-8 -*-
from selenium import webdriver #從selenium匯入瀏覽器驅動
browser = webdriver.Chrome() #宣告驅動物件, 即Chrome瀏覽器
def get_one_page():
'''獲取單個頁面'''
browser.get("https://www.xxxxx.com") #請求網站
2. 輸入”男裝”, 在輸入之前, 需要判斷輸入框是否存在, 如果存在則輸入”男裝”, 不存在則等待顯示成功.
# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.common.by import By #匯入元素定位方法模組
from selenium.webdriver.support.ui import WebDriverWait #匯入等待判斷模組
from selenium.webdriver.support import expected_conditions as EC #匯入判斷條件模組
browser = webdriver.Chrome()
def get_one_page():
'''獲取單個頁面'''
browser.get("https://www.xxxxx.com")
input = WebDriverWait(browser,10).until( #等待判斷
EC.presence_of_element_located((By.CSS_SELECTOR,"#q"))) #若輸入框顯示成功,則獲取,否則等待
input.send_keys("男裝") #輸入商品名稱
3. 下一步就是點選”搜尋”按鈕, 按鈕具有屬性: 可點選, 那麼加入判斷條件.
# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
browser = webdriver.Chrome()
def get_one_page():
'''獲取單個頁面'''
browser.get("https://www.xxxxx.com")
input = WebDriverWait(browser,10).until(
EC.presence_of_element_located((By.CSS_SELECTOR,"#q"))) #
input.send_keys("男裝")
button = WebDriverWait(browser,10).until( #等待判斷
EC.element_to_be_clickable((By.CSS_SELECTOR,"#J_TSearchForm > div.search-button > button"))) #若按鈕可點選, 則獲取, 否則等待
button.click() #點選按鈕
4. 獲取總的頁數, 同樣加入等待判斷.
# -*- coding: utf-8 -*-
import re
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
browser = webdriver.Chrome()
def get_one_page():
'''獲取單個頁面'''
browser.get("https://www.xxxxx.com")
input = WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.CSS_SELECTOR, "#q")))
input.send_keys("男裝")
button = WebDriverWait(browser, 10).until(
EC.element_to_be_clickable(
(By.CSS_SELECTOR, "#J_TSearchForm > div.search-button > button")))
button.click()
pages = WebDriverWait(browser, 10).until( # 等待判斷
EC.presence_of_element_located(
(By.CSS_SELECTOR, "#mainsrp-pager > div > div > div > div.total"))) # 若總頁數載入成功,則獲取總頁數,否則等待
return pages.text
def main():
pages = get_one_page()
print(pages)
if __name__ == '__main__':
main()
5. 打印出來的不是我們想要的結果, 利用正則運算式獲取, 最後再利用try…except捕捉異常
# -*- coding: utf-8 -*-
import re
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
browser = webdriver.Chrome()
def get_one_page():
'''獲取單個頁面'''
try:
browser.get("https://www.xxxxx.com")
input = WebDriverWait(browser,10).until(
EC.presence_of_element_located((By.CSS_SELECTOR,"#q")))
input.send_keys("男裝")
button = WebDriverWait(browser,10).until(
EC.element_to_be_clickable((By.CSS_SELECTOR,"#J_TSearchForm > div.search-button > button")))
button.click()
pages = WebDriverWait(browser,10).until(
EC.presence_of_element_located((By.CSS_SELECTOR,"#mainsrp-pager > div > div > div > div.total")))
return pages.text
except TimeoutException:
return get_one_page() #如果超時,繼續獲取
def main():
pages = get_one_page()
pages = int(re.compile("(\d+)").findall(pages)[0]) #採用正則運算式提取文字中的總頁數
print(pages)
if __name__ == '__main__':
main()
關於Selenium的更多內容,可參看官方檔案https://selenium-python.readthedocs.io/waits.html
3. 獲取多個商品介面
採用獲取”到第 頁”輸入框方式, 切換到下一頁, 同樣是等待判斷
需要註意的是, 最後要加入判斷: 高亮是否是當前頁
def get_next_page(page):
try:
input = WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.CSS_SELECTOR, "#mainsrp-pager > div > div > div > div.form > input"))) # 若輸入框載入成功,則獲取,否則等待
input.send_keys(page) # 輸入頁碼
button = WebDriverWait(browser, 10).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit"))) # 若按鈕可點選,則獲取,否則等待
button.click() # 點選按鈕
WebDriverWait(browser,10).until(
EC.text_to_be_present_in_element((By.CSS_SELECTOR,"#mainsrp-pager > div > div > div > ul > li.item.active > span"),str(page))) # 判斷高亮是否是當前頁
except TimeoutException: # 超時, 繼續請求
return get_next_page(page)
def main():
pages = get_one_page()
pages = int(re.compile("(\d+)").findall(pages)[0])
for page in range(1,pages+1):
get_next_page(page)
if __name__ == '__main__':
main()
4. 獲取商品資訊
首先, 判斷資訊是否載入成功, 緊接著獲取原始碼並初始化, 進而解析.
需要註意的是, 在”get_one_page”和”get_next_page”中呼叫之後, 才可執行
def get_info():
"""獲取詳情"""
WebDriverWait(browser,20).until(EC.presence_of_element_located((
By.CSS_SELECTOR,"#mainsrp-itemlist .items .item"))) #判斷商品資訊是否載入成功
text = browser.page_source #獲取網頁原始碼
html = pq(text) #初始化網頁原始碼
items = html('#mainsrp-itemlist .items .item').items() #採用items方法會得到生成器
for item in items: #遍歷每個節點物件
data = []
image = item.find(".pic .img").attr("data-src") #用find方法查詢子孫節點,用attr方法獲取屬性名稱
price = item.find(".price").text().strip().replace("\n","") #用text方法獲取文字,strip()去掉前後字串,預設是空格
deal = item.find(".deal-cnt").text()[:-2]
title = item.find(".title").text().strip()
shop = item.find(".shop").text().strip()
location = item.find(".location").text()
data.append([shop, location, title, price, deal, image])
print(data)
5. 儲存到MySQL資料庫
def save_to_mysql(data):
"""儲存到資料庫"""
# 建立資料庫連線物件
db= pymysql.connect(host = "localhost",user = "root",password = "password",port = 3306, db = "spiders",charset = "utf8")
# 獲取遊標
cursor = db.cursor()
#建立資料庫
cursor.execute("CREATE TABLE IF NOT EXISTS {0}(shop VARCHAR(20),location VARCHAR(10),title VARCHAR(255),price VARCHAR(20),deal VARCHAR(20), image VARCHAR(255))".format("男裝"))
#SQL陳述句
sql = "INSERT INTO {0} values(%s,%s,%s,%s,%s,%s)".format("男裝")
try:
#傳入引數sql,data
if cursor.execute(sql,data):
#插入資料庫
db.commit()
print("********已入庫**********")
except:
print("#########入庫失敗#########")
#回滾,相當什麼都沒做
db.rollback()
#關閉資料庫
db.close()
完整程式碼
# -*- coding: utf-8 -*-
'''
有需要Python學習資料的小夥伴嗎?小編整理一套Python資料和PDF，感興趣者可以加學習群：548377875，反正閑著也是閑著呢，不如學點東西啦~~
'''
import re
import pymysql
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from pyquery import PyQuery as pq
browser = webdriver.Chrome()
def get_one_page(name):
'''獲取單個頁面'''
print("-----------------------------------------------獲取第一頁-------------------------------------------------------")
try:
browser.get("https://www.xxxxx.com")
input = WebDriverWait(browser,10).until(
EC.presence_of_element_located((By.CSS_SELECTOR,"#q")))
input.send_keys(name)
button = WebDriverWait(browser,10).until(
EC.element_to_be_clickable((By.CSS_SELECTOR,"#J_TSearchForm > div.search-button > button")))
button.click()
pages = WebDriverWait(browser,10).until(
EC.presence_of_element_located((By.CSS_SELECTOR,"#mainsrp-pager > div > div > div > div.total")))
print("----即將解析第一頁資訊----")
get_info(name)
print("----第一頁資訊解析完成----")
return pages.text
except TimeoutException:
return get_one_page(name)
def get_next_page(page,name):
"""獲取下一頁"""
print("---------------------------------------------------正在獲取第{0}頁----------------------------------------".format(page))
try:
input = WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.CSS_SELECTOR, "#mainsrp-pager > div > div > div > div.form > input")))
input.send_keys(page)
button = WebDriverWait(browser, 10).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit")))
button.click()
WebDriverWait(browser,10).until(
EC.text_to_be_present_in_element((By.CSS_SELECTOR,"#mainsrp-pager > div > div > div > ul > li.item.active > span"),str(page)))
print("-----即將解析第{0}頁資訊-----".format(page))
get_info(name)
print("-----第{0}頁資訊解析完成-----".format(page))
except TimeoutException:
return get_next_page(page,name)
def get_info(name):
"""獲取詳情"""
WebDriverWait(browser,20).until(EC.presence_of_element_located((
By.CSS_SELECTOR,"#mainsrp-itemlist .items .item")))
text = browser.page_source
html = pq(text)
items = html('#mainsrp-itemlist .items .item').items()
for item in items:
data = []
image = item.find(".pic .img").attr("data-src")
price = item.find(".price").text().strip().replace("\n","")
deal = item.find(".deal-cnt").text()[:-2]
title = item.find(".title").text().strip()
shop = item.find(".shop").text().strip()
location = item.find(".location").text()
data.append([shop, location, title, price, deal, image])
for dt in data:
save_to_mysql(dt,name)
def save_to_mysql(data,name):
"""儲存到資料庫"""
db= pymysql.connect(host = "localhost",user = "root",password = "password",port = 3306, db = "spiders",charset = "utf8")
cursor = db.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS {0}(shop VARCHAR(20),location VARCHAR(10),title VARCHAR(255),price VARCHAR(20),deal VARCHAR(20), image VARCHAR(255))".format(name))
sql = "INSERT INTO {0} values(%s,%s,%s,%s,%s,%s)".format(name)
try:
if cursor.execute(sql,data):
db.commit()
print("********已入庫**********")
except:
print("#########入庫失敗#########")
db.rollback()
db.close()
def main(name):
pages = get_one_page(name)
pages = int(re.compile("(\d+)").findall(pages)[0])
for page in range(1,pages+1):
get_next_page(page,name)
if __name__ == '__main__':
name = "男裝"
main(name)