爬取小说1--高并发

爬取《放开那个女巫》小说

还是采用高并发的协程来进行开启下载。

其实，到现在为止，会了并发技术之后，诸多爬虫比较的，已经不再是用什么库，之类的问题了。
而是，开始研究爬虫的策略问题了。

比如，这里，我的策略就是，要保证每一章必须要爬取到，否则就要接着等下去。（每次爬取200章，然后必须要等所有的都已经爬取完成之后才开始合并，之后再接着爬取接下来的200章。这个策略虽然保证的健壮性，但是在速度上却是不敢恭维。下一步，我们将对这个策略进行改进！）

import requests import os import gevent from gevent import monkey import random import re from lxml import etreemonkey.patch_all(select=False) from urllib import parse import timeIPs = [{'HTTPS': 'HTTPS://182.114.221.180:61202'},{'HTTPS': 'HTTPS://60.162.73.45:61202'},{'HTTPS': 'HTTPS://113.13.36.227:61202'},{'HTTPS': 'HTTPS://1.197.88.101:61202'}] HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8','Accept-Language': 'zh-CN,zh;q=0.9','Cookie': '__cfduid=d820fcba1e8cf74caa407d320e0af6b5d1518500755; UM_distinctid=1618db2bfbb140-060057ff473277-4323461-e1000-1618db2bfbc1e4; CNZZDATA1272873873=2070014299-1518497311-https%253A%252F%252Fwww.baidu.com%252F%7C1520689081; yjs_id=5a4200a91c8aa5629ae0651227ea7fa2; ctrl_time=1; jieqiVisitTime=jieqiArticlesearchTime%3D1520693103' }def setDir():if 'Noval' not in os.listdir('./'):os.mkdir('./Noval')def getNoval(url, id):while True:try:headers = HEADERSIP = random.choice(IPs)res = requests.get(url, headers=headers, proxies=IP)res.encoding = 'GB18030'html = res.text.replace('&nbsp;', ' ') # 替换掉这个字符 换成空格~ 意思是一样的page = etree.HTML(html)content = page.xpath('//div[@id="content"]')ps = page.xpath('//div[@class="bookname"]/h1')if len(ps) != 0:s = ps[0].text + '\n's = s + content[0].xpath("string(.)")with open('./Noval/%d.txt' % id, 'w', encoding='gb18030', errors='ignore') as f:f.write(s)except Exception:continueelse:breakdef getContentFile(url):headers = HEADERSIP = random.choice(IPs)res = requests.get(url, headers=headers, proxies=IP)res.encoding = 'GB18030'page = etree.HTML(res.text)bookname = page.xpath('//div[@id="info"]/h1')[0].xpath('string(.)')dl = page.xpath('//div[@id="list"]/dl/dd/a')splitHTTP = parse.urlsplit(url)url = splitHTTP.scheme + '://' + splitHTTP.netlocreturn list(map(lambda x: url + x.get('href'), dl)), booknamedef BuildGevent(baseurl):content, bookname = getContentFile(baseurl) # version2steps = 200beginIndex, length = steps, len(content)count = 0name = "%s.txt" % booknamewhile (count - 1) * steps < length:WaitigList = [gevent.spawn(getNoval, content[i + count * steps], i + count * steps) for i in range(steps) ifi + count * steps < length]gevent.joinall(WaitigList)NovalFile = list(filter(lambda x: x[:x.index('.')].isdigit(), os.listdir('./Noval')))NovalFile.sort(key=lambda x: int(re.match('\d+', x).group()))String = ''for dirFile in NovalFile:with open('./Noval/' + dirFile, 'r', encoding='gb18030', errors='ignore') as f:String = String + '\n' + f.read()os.remove('./Noval/%s' % dirFile)if count == 0:with open('./Noval/' + name, 'w', encoding='gb18030', errors='ignore') as ff:ff.write(String)else:with open('./Noval/' + name, 'a', encoding='gb18030', errors='ignore') as ff:ff.write(String)count += 1if __name__ == '__main__':starttime = time.time()setDir()url = 'http://www.biquge.com.tw/16_16588/'BuildGevent(url)endtime = time.time()print("Total use time: %.6f" % (endtime - starttime))

总结

以上是生活随笔为你收集整理的爬取小说1--高并发的全部内容，希望文章能够帮你解决所遇到的问题。

如果觉得生活随笔网站内容还不错，欢迎将生活随笔推荐给好友。