工具

pycharm

目的

'''
https://www.77xsw.cc/fenlei/1_1/：第一頁的網(wǎng)址
https://www.77xsw.cc/fenlei/1_2/：第二頁的網(wǎng)址
...
https://www.77xsw.cc/fenlei/1_10/：第十頁的網(wǎng)址
'''

代碼

import requests
import re
import jsonnovel_list = []for i in range(1,11):# 請求網(wǎng)址url = 'https://www.77xsw.cc/fenlei/1_' + str(i) + '/'headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'}# 發(fā)送請求response = requests.get(url, headers=headers)# print(response.text)# 數(shù)據(jù)處理 確定正則表達式規(guī)則時，看抓取到的沒有美化的響應(yīng)結(jié)果data = response.text# 用中文不能匹配中文的標(biāo)點符號# rule = '<span class="sp_2"><a href="(.*?)".*?title="[\u4e00-\u9fa5]*">([\u4e00-\u9fa5]*?)</a></span><span class="sp_3">'  # not# rule = '<span class="sp_2"><a href="(.*?)".*?title=".*?">(.*?)</a></span><span class="sp_3">'  # okrule = '<span class="sp_2"><a href="(.*?)".*?>(.*?)</a></span><span class="sp_3">'result = re.findall(rule,data)  # 返回值為多個匹配結(jié)果組成的列表for novel in result:novel_list.extend(result)# print(novel_list)
novel_tuple = tuple(novel_list)  # 去重
novel_dict = dict([i[1], i[0]] for i in novel_tuple)  # 轉(zhuǎn)為字典
print(novel_dict,len(novel_dict))  # 一頁40個，10頁應(yīng)該有400個，結(jié)果為397個，含有重復(fù)的
# 保存數(shù)據(jù) json格式
with open('novel.json', 'w', encoding='utf8') as f:json.dump(novel_dict,f,indent=2,ensure_ascii=False)

運行結(jié)果

見資源