java 批量下载小说天堂的小说

简单的小说下载工具，未进行优化

package com.zrc.demo.execute;import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.net.SocketTimeoutException; import java.util.Set; import java.util.TreeSet;import org.apache.commons.io.IOUtils; import org.jsoup.Connection; import org.jsoup.Connection.Response; import org.jsoup.HttpStatusException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; /*** 用于下载小说天堂的小说* 同时存入本地D：\\novel* @author ZRC**/ class NovelDeal extends Thread{private String cata;public NovelDeal() {}public NovelDeal(String cata) {this.cata = cata;}@Overridepublic void run(){beginWrite(cata); //开启线程执行的方法}public static void main(String[] args) {NovelDeal a1 = new NovelDeal("mingzhu"); //每个线程用于下载一类图书NovelDeal a2 = new NovelDeal("ertong");NovelDeal a3 = new NovelDeal("qingchun");NovelDeal a4 = new NovelDeal("dangdai");NovelDeal a5 = new NovelDeal("gudian");NovelDeal a6 = new NovelDeal("pinglun");NovelDeal a7 = new NovelDeal("waiwen");NovelDeal a8 = new NovelDeal("sanwen");NovelDeal a9 = new NovelDeal("xuanhuan");NovelDeal a10 = new NovelDeal("chuanyue");NovelDeal a11 = new NovelDeal("wangyou");NovelDeal a12 = new NovelDeal("yanqing");NovelDeal a13 = new NovelDeal("xuanyi");NovelDeal a14 = new NovelDeal("wuxia");NovelDeal a15 = new NovelDeal("kehuan");NovelDeal a16 = new NovelDeal("renwen");NovelDeal a17 = new NovelDeal("junshi");NovelDeal a18 = new NovelDeal("zhuanji");NovelDeal a19 = new NovelDeal("lizhi");NovelDeal a20 = new NovelDeal("lishi");NovelDeal a21 = new NovelDeal("shenghuo");a1.start(); a2.start();a3.start();a4.start();a5.start();a6.start();a7.start();a8.start();a9.start();a10.start();a11.start();a12.start();a13.start();a14.start();a15.start();a16.start();a17.start();a18.start();a19.start();a20.start();a21.start();}/*** 传入小说地址* @param address 小说地址*/public static void writeNovel(String address){Connection connect = Jsoup.connect(address).timeout(45000000).header("User-Agent", "chrome");Response execute;Document document=null;try {execute= connect.execute();document= execute.parse();} catch (IOException e) {e.printStackTrace();}String articleName = document.getElementsByTag("h1").text().replaceAll("txt下载", "");String author = document.getElementsByClass("zhaungtai").get(0).getElementsByTag("a").get(0).text();String textName = articleName+"--"+author+".txt";String catalog = document.getElementsByClass("zhaungtai").get(0).getElementsByTag("a").get(1).text();String path = "D:\\novel\\"+catalog+"\\";File pathFile = new File(path);if(!pathFile.exists()){pathFile.mkdirs();}File textFile = new File(path+textName);OutputStream textOs = null;try {textOs=new FileOutputStream(textFile);} catch (FileNotFoundException e) {e.printStackTrace();}Elements tds = document.getElementsByTag("td");Set<String> set = new TreeSet<>();for (Element ele : tds) {Elements as = ele.getElementsByTag("a");for (Element aEle : as) {set.add(aEle.attr("href"));}}StringBuilder content = new StringBuilder();for (String url : set) {Response exe = null;Document doc = null;Connection con = Jsoup.connect("http://www.xiaoshuotxt.net"+url).timeout(45000000).header("User-Agent", "chrome");try {exe = con.execute();doc = exe.parse();content.append(doc.getElementsByTag("h1").get(0).text());} catch (Exception e) {e.printStackTrace();continue;}Elements pEles = doc.getElementsByClass("zw");for (Element pEle : pEles) {content.append(pEle.toString().replaceAll("<div>", "").replaceAll("<a.*[堂]", "").replaceAll("</div>", "").replaceAll("<div class=\"zw\">", "").replaceAll("</p>", "").replaceAll("<p>", "").replaceAll("<br>", "").replaceAll(" ", " ").replaceAll("[w|W|Ｗ].*[堂]", "").replaceAll("<a.*a>", "").replaceAll("</a>", "").replaceAll("小说天.*[t|T]", "").replaceAll("[w|W|Ｗ].*[o|O][m|M]", ""));}content.append("\r\n");}try {IOUtils.write(content.toString(), textOs,"GBK");textOs.close();} catch (IOException e) {e.printStackTrace();}System.out.println(catalog+"--->"+textName+"---->完成");}/*** 通过分类取得每个小说的地址* @param catalog*/public static void beginWrite(String catalog){int index = 1;while(true){String url = "http://www.xiaoshuotxt.net/"+catalog;String relUrl = "";if(index!=1){relUrl=url+"/index_"+index+".html";}else{relUrl=url;}Connection connect = Jsoup.connect(relUrl).timeout(45000000).header("User-Agent", "chrome");Elements elements;Document document;try {Response execute = connect.execute();document = execute.parse();}catch(SocketTimeoutException e){System.err.println("SocketTimeoutException-------------continue"+relUrl);e.printStackTrace();continue ;}catch(HttpStatusException e){System.err.println("HttpStatusException-------------break"+relUrl+"\n\n\n");e.printStackTrace();break;}catch (IOException e) {System.err.println("IOException-------------break"+relUrl+"\n\n\n");e.printStackTrace();break;} elements = document.getElementsByClass("bbox");for (Element element : elements) {String base = element.getElementsByTag("a").get(0).attr("href");writeNovel("http://www.xiaoshuotxt.net"+base);}index++;}} }

总结

以上是生活随笔为你收集整理的java 批量下载小说天堂的小说的全部内容，希望文章能够帮你解决所遇到的问题。

如果觉得生活随笔网站内容还不错，欢迎将生活随笔推荐给好友。