爬取四大名著

'''诗词名句网1. 爬取固定书籍2. 爬取书名3. 爬取本部书的章回目录4. 灵活处理，爬取任意书籍的章回目录5. 加入异常处理6. 爬取任意整本书 '''import requests import redef bookSpider(oldurl,bookName):url=oldurl+".html"html=loadPage(url)try:with open("demo.txt",'w',encoding='utf-8') as f:f.write(html)except:print("FILE OPERATION ERROR")findTitle("demo.txt",bookName)cnt=findTileOfPages("demo.txt",bookName)getWholeBook(oldurl,bookName,cnt)def findTitle(filename,bookName):try:f=open(filename,encoding='utf-8')book=open("book.txt",'w',encoding='utf-8')except:print("FILE OPERATION ERROR")while True:line=f.readline()#print("READ:"+line)if not line:breakpattern=re.compile(r'<title>《.{0,10}》')bookName=re.search(pattern,line)flag=Falseif bookName:print("书名：",end="")for ch in str(bookName):if ch == '《':flag=Trueif ch == '》':flag=Falseprint("》")book.write('》'+'\n')if flag:print(ch,end="")book.write(ch)def findTileOfPages(filename,bookName):cnt=0try:f = open(filename,encoding='utf-8')book = open("book.txt",'a', encoding='utf-8')except:print("FILE OPERATION ERROR")book.write("目录：\n")while True:line = f.readline()# print("READ:"+line)if not line:breakpattern = re.compile(r'<li><a href="/book/'+bookName+'/\d+.html">.{10,40}</a></li>')titleOfpages = pattern.findall(line)flag = Falseif titleOfpages:for i in range(0,len(titleOfpages)):cnt+=1for j in range(0,len(titleOfpages[i])):if titleOfpages[i][j] == '第':flag=Trueif titleOfpages[i][j] == '<':flag=Falseif flag:print(titleOfpages[i][j],end="")book.write(titleOfpages[i][j])print()book.write('\n')return cntdef getWholeBook(url,bookName,cnt):print("正在下载全本书，请稍后...")for i in range(1,cnt+1):newUrl=url+'/'+str(i)+".html"print(newUrl)html=loadPage(newUrl)try:with open("bookHtml.txt", 'w', encoding='utf-8') as f:f.write(html)except:print("FILE OPERATION ERROR")f = open('bookHtml.txt', 'r', encoding='utf-8')bookContent = open('book.txt', 'a', encoding='utf-8')while True:line = f.readline()# print("READ:"+line)if not line:breakpattern = re.compile(r'<p>&nbsp;&nbsp;&nbsp;&nbsp;.+</p>')content = re.findall(pattern, line)patternOfTitle=re.compile(r'<h1>.+</h1>')contentOfTitle = re.findall(patternOfTitle, line)flag=Falsefor i in range(0, len(contentOfTitle)):for j in range(0, len(contentOfTitle[i])):if contentOfTitle[i][j] == '>':flag=Truecontinueif contentOfTitle[i][j] == '<':flag=Falsecontinueif flag:bookContent.write(contentOfTitle[i][j])bookContent.write('\n')flag = Falsefor i in range(0, len(content)):for j in range(0, len(content[i])):if content[i][j] == '<':flag=Falsecontinueif content[i][j] == ';' and content[i][j - 1] == 'p' and content[i][j + 1] != '&':flag = Truecontinueif flag:bookContent.write(content[i][j])bookContent.write('\n')f.close()bookContent.close()def loadPage(url):try:header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'}response = requests.get(url, headers=header)return response.content.decode('utf-8')except:print("PAGE LOAD ERROR")if __name__ == "__main__":bookName=input("请输入想看的书名：(全拼)")url = "http://www.shicimingju.com/book/"+bookNamebookSpider(url,bookName)

 

转载于:https://www.cnblogs.com/TheSilverMoon/p/11143203.html

总结

以上是生活随笔为你收集整理的爬取四大名著的全部内容，希望文章能够帮你解决所遇到的问题。

如果觉得生活随笔网站内容还不错，欢迎将生活随笔推荐给好友。