详解用python实现爬取CSDN热门评论URL并存入redis

import time
import random
from PIL import Image
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
if __name__ == \'__main__\':
  options = webdriver.ChromeOptions()
  options.binary_location = r\'C:\\Users\\hhh\\AppData\\Local\\Google\\Chrome\\Application\\谷歌浏览器.exe\'
  # driver=webdriver.Chrome(executable_path=r\'D:\\360Chrome\\chromedriver\\chromedriver.exe\')
  driver = webdriver.Chrome(options=options)
  #以java模块为例
  driver.get(\'https://www.csdn.net/nav/java\')
  for i in range(1,20):
    driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight)\")
    time.sleep(2)

from bs4 import BeautifulSoup
from lxml import etree 
html = etree.HTML(driver.page_source)
# soup = BeautifulSoup(html, \'lxml\')
# soup_herf=soup.find_all(\"#feedlist_id > li:nth-child(1) > div > div > h2 > a\")
# soup_herf
title = html.xpath(\'//*[@id=\"feedlist_id\"]/li/div/div/h2/a/@href\')

import redis
r_link = redis.Redis(port=\'6379\', host=\'localhost\', decode_responses=True, db=1)
for u in title:
    print(\"准备写入{}\".format(u))
    r_link.rpush(\"csdn_url\", u)
    print(\"{}写入成功!\".format(u))
print(\'=\' * 30, \'\\n\', \"共计写入url：{}个\".format(len(title)), \'\\n\', \'=\' * 30)

one_url = r_link.rpop(\"csdn_url)\")
while one_url:
  print(\"{}被弹出！\".format(one_url))