使用HtmlParser抓去网页内容

package parser;

import org.htmlparser.Parser;

import org.htmlparser.beans.StringBean;

importorg.htmlparser.filters.NodeClassFilter;

importorg.htmlparser.parserapplications.StringExtractor;

import org.htmlparser.tags.BodyTag;

import org.htmlparser.util.NodeList;

import org.htmlparser.util.ParserException;

/**

 * 使用HtmlParser抓去网页内容: 要抓去页面的内容最方便的方法就是使用StringBean. 里面有几个控制页面内容的几个参数.

 * 在后面的代码中会有说明. Htmlparser包中还有一个示例StringExtractor 里面有个直接得到内容的方法,

 * 其中也是使用了StringBean . 另外直接解析Parser的每个标签也可以的.

 *

 *@author chenguoyong

 *

 */

public class GetContent {

       publicvoid getContentUsingStringBean(String url) {

              StringBeansb = new StringBean();

              sb.setLinks(true);// 是否显示web页面的连接(Links)

              //为了取得页面的整洁美观一般设置上面两项为true , 如果要保持页面的原有格式, 如代码页面的空格缩进 可以设置为false

              sb.setCollapse(true);// 如果是true的话把一系列空白字符用一个字符替代.

              sb.setReplaceNonBreakingSpaces(true);//If true regular space

              sb

                            .setURL("http://www.blogjava.net/51AOP/archive/2006/07/19/59064.html");

              System.out.println("TheContent is :\n" + sb.getStrings());

 http://c.tieba.baidu.com/p/3389457489
http://c.tieba.baidu.com/p/3389466826
http://c.tieba.baidu.com/p/3389470640
http://c.tieba.baidu.com/p/3389472377
http://c.tieba.baidu.com/p/3389473930
http://c.tieba.baidu.com/p/3389477136
http://c.tieba.baidu.com/p/3389475148
http://c.tieba.baidu.com/p/3389485599
http://c.tieba.baidu.com/p/3389513216
http://c.tieba.baidu.com/p/3389517509
http://c.tieba.baidu.com/p/3389520313
http://c.tieba.baidu.com/p/3389522398
http://c.tieba.baidu.com/p/3390769932
http://c.tieba.baidu.com/p/3392021705
http://c.tieba.baidu.com/p/3392323863
http://c.tieba.baidu.com/p/3392330792
http://c.tieba.baidu.com/p/3392336731
http://c.tieba.baidu.com/p/3392342524
http://c.tieba.baidu.com/p/3392347298
http://c.tieba.baidu.com/p/3375675193
http://c.tieba.baidu.com/p/3375686025
http://c.tieba.baidu.com/p/3375690552
http://c.tieba.baidu.com/p/3375695670
http://c.tieba.baidu.com/p/3375701076
http://c.tieba.baidu.com/p/3375709000
http://c.tieba.baidu.com/p/3375716663
http://c.tieba.baidu.com/p/3375740498
http://c.tieba.baidu.com/p/3375749024
http://c.tieba.baidu.com/p/3375766133
http://c.tieba.baidu.com/p/3392359946
http://c.tieba.baidu.com/p/3387737417
http://c.tieba.baidu.com/p/3387738658
http://c.tieba.baidu.com/p/3387739838
http://c.tieba.baidu.com/p/3387783818
http://c.tieba.baidu.com/p/3387786470
http://c.tieba.baidu.com/p/3387783818
http://c.tieba.baidu.com/p/3382021222
http://c.tieba.baidu.com/p/3382021222
http://c.tieba.baidu.com/p/3382009487
http://c.tieba.baidu.com/p/3382005675
http://c.tieba.baidu.com/p/3382002991
http://c.tieba.baidu.com/p/3381994737
http://c.tieba.baidu.com/p/3381991852
http://c.tieba.baidu.com/p/3381988322
http://c.tieba.baidu.com/p/3381986298
http://c.tieba.baidu.com/p/3381983852
http://c.tieba.baidu.com/p/3381981736
http://c.tieba.baidu.com/p/3381979917
http://c.tieba.baidu.com/p/3381918554
http://c.tieba.baidu.com/p/3381918127
http://c.tieba.baidu.com/p/3381914696
http://c.tieba.baidu.com/p/3381913501
http://c.tieba.baidu.com/p/3381913187
http://c.tieba.baidu.com/p/3381911905
http://c.tieba.baidu.com/p/3381911406
http://c.tieba.baidu.com/p/3381910239
http://c.tieba.baidu.com/p/3381909751
http://c.tieba.baidu.com/p/3381905367
http://c.tieba.baidu.com/p/3381903236
http://c.tieba.baidu.com/p/3381892748
http://c.tieba.baidu.com/p/3387734920
http://c.tieba.baidu.com/p/3387734170
http://c.tieba.baidu.com/p/3387733127
http://c.tieba.baidu.com/p/3387732611
http://c.tieba.baidu.com/p/3387732170
http://c.tieba.baidu.com/p/3387731630
http://c.tieba.baidu.com/p/3387731086
http://c.tieba.baidu.com/p/3387730892
http://c.tieba.baidu.com/p/3387728964
http://c.tieba.baidu.com/p/3387725536
http://c.tieba.baidu.com/p/3374352634
http://c.tieba.baidu.com/p/3374351958
http://c.tieba.baidu.com/p/3374351507
http://c.tieba.baidu.com/p/3374350912
http://c.tieba.baidu.com/p/3374349860
http://c.tieba.baidu.com/p/3374344211
http://c.tieba.baidu.com/p/3374341531
http://c.tieba.baidu.com/p/3374339288
http://c.tieba.baidu.com/p/3374338595
http://c.tieba.baidu.com/p/3374337825
http://c.tieba.baidu.com/p/3374336667
http://c.tieba.baidu.com/p/3374334686
http://c.tieba.baidu.com/p/3374333843
http://c.tieba.baidu.com/p/3374333168
http://c.tieba.baidu.com/p/3392384335
http://c.tieba.baidu.com/p/3392396252