[关闭]
@xuduochoua 2016-05-23T07:08:52.000000Z 字数 27419 阅读 1844

VIC简单爬虫程序

爬虫 demo

项目说明

  1.  maven项目
  2.  利用Jsonp完成对网页内容的抓取,包括文章列表，文章详情，以及
  4.  - 列表项
  6. 详情中的图片，并写入本地文件

一、准备工作

1. 搭建mavne工程

pom文件如下

  1. <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  2.     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  3.     <modelVersion>4.0.0</modelVersion>
  4.     <groupId>craw</groupId>
  5.     <artifactId>craw</artifactId>
  6.     <version>0.0.1-SNAPSHOT</version>
  9.     <dependencies>
  11.         <dependency>
  12.             <groupId>commons-httpclient</groupId>
  13.             <artifactId>commons-httpclient</artifactId>
  14.             <version>3.1</version>
  15.         </dependency>
  18.         <dependency>
  19.             <groupId>org.codehaus.jackson</groupId>
  20.             <artifactId>jackson-mapper-asl</artifactId>
  21.             <version>1.9.12</version>
  22.         </dependency>
  23.         <!-- 提供一些基础的、通用的操作和处理，如自动生成toString()的结果、自动实现hashCode()和equals()方法、数组操作、枚举、日期和时间的处理等等。 -->
  24.         <dependency>
  25.             <groupId>commons-lang</groupId>
  26.             <artifactId>commons-lang</artifactId>
  27.             <version>2.6</version>
  28.         </dependency>
  30.         <!-- Commons项目中用来处理常用的编码方法的工具类包，例如DES、SHA1、MD5、Base64，URL，Soundx等等。[1] 
  31.             不仅是编码，也可用于解码。 -->
  32.         <dependency>
  33.             <groupId>commons-codec</groupId>
  34.             <artifactId>commons-codec</artifactId>
  35.             <version>1.10</version>
  36.         </dependency>
  38.         <!-- Commons-loggin的目的是为“所有的Java日志实现”提供一个统一的接口，它自身的日志功能平常弱 -->
  39.         <dependency>
  40.             <groupId>commons-logging</groupId>
  41.             <artifactId>commons-logging</artifactId>
  42.             <version>1.2</version>
  43.         </dependency>
  45.         <dependency>
  46.             <groupId>commons-io</groupId>
  47.             <artifactId>commons-io</artifactId>
  48.             <version>2.4</version>
  49.         </dependency>
  51.         <dependency>
  52.             <groupId>org.jsoup</groupId>
  53.             <artifactId>jsoup</artifactId>
  54.             <version>1.8.3</version>
  55.         </dependency>
  56.         <!-- slf4j 配合 logback 做日志 start-->
  57.         <dependency>
  58.             <groupId>org.slf4j</groupId>
  59.             <artifactId>slf4j-api</artifactId>
  60.             <version>1.7.8</version>
  61.         </dependency>
  63.         <dependency>
  64.             <groupId>ch.qos.logback</groupId>
  65.             <artifactId>logback-classic</artifactId>
  66.             <version>1.1.3</version>
  67.         </dependency>
  68.         <dependency>
  69.             <groupId>ch.qos.logback</groupId>
  70.             <artifactId>logback-core</artifactId>
  71.             <version>1.1.3</version>
  72.         </dependency>
  73.         <!-- slf4j 配合 logback 做日志 end-->
  74.     </dependencies>
  76.     <build>
  77.         <plugins>
  78.             <plugin>
  79.                 <groupId>org.apache.maven.plugins</groupId>
  80.                 <artifactId>maven-compiler-plugin</artifactId>
  81.                 <configuration>
  82.                     <source>1.7</source>
  83.                     <target>1.7</target>
  84.                 </configuration>
  85.             </plugin>
  86.         </plugins>
  87.     </build>
  88. </project>

2. 引入slf4j日志

  1. 引入jar包
  2. 配置src下logback.xml
  1. <?xml version="1.0" encoding="UTF-8"?>
  2. <!--
  3. scan：当此属性设置为true时，配置文件如果发生改变，将会被重新加载，默认值为true。
  4. scanPeriod：设置监测配置文件是否有修改的时间间隔，如果没有给出时间单位，默认单位是毫秒当scan为true时，此属性生效。默认的时间间隔为1分钟。
  5. debug：当此属性设置为true时，将打印出logback内部日志信息，实时查看logback运行状态。默认值为false。
  6. -->
  7. <configuration scan="false" scanPeriod="60 seconds" debug="false">
  8.     <!-- 定义日志的根目录 -->
  9.     <property name="LOG_HOME" value="/app/log" />
  10.     <!-- 定义日志文件名称 -->
  11.     <property name="appName" value="netty"></property>
  12.     <!-- ch.qos.logback.core.ConsoleAppender 表示控制台输出 -->
  13.     <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
  14.         <Encoding>UTF-8</Encoding>
  15.         <!--
  16.         日志输出格式：%d表示日期时间，%thread表示线程名，%-5level：级别从左显示5个字符宽度
  17.         %logger{50} 表示logger名字最长50个字符，否则按照句点分割。 %msg：日志消息，%n是换行符
  18.         -->
  19.         <layout class="ch.qos.logback.classic.PatternLayout">
  20.             <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
  21.         </layout>
  22.     </appender>
  24.     <!-- 滚动记录文件，先将日志记录到指定文件，当符合某个条件时，将日志记录到其他文件 -->  
  25.     <appender name="appLogAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
  26.         <Encoding>UTF-8</Encoding>
  27.         <!-- 指定日志文件的名称 -->  
  28.         <file>${LOG_HOME}/${appName}.log</file>
  29.         <!--
  30.         当发生滚动时，决定 RollingFileAppender 的行为，涉及文件移动和重命名
  31.         TimeBasedRollingPolicy： 最常用的滚动策略，它根据时间来制定滚动策略，既负责滚动也负责出发滚动。
  32.         -->
  33.         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
  34.             <!--
  35.             滚动时产生的文件的存放位置及文件名称 %d{yyyy-MM-dd}：按天进行日志滚动 
  36.             %i：当文件大小超过maxFileSize时，按照i进行文件滚动
  37.             -->
  38.             <fileNamePattern>${LOG_HOME}/${appName}-%d{yyyy-MM-dd}-%i.log</fileNamePattern>
  39.             <!-- 
  40.             可选节点，控制保留的归档文件的最大数量，超出数量就删除旧文件。假设设置每天滚动，
  41.             且maxHistory是365，则只保存最近365天的文件，删除之前的旧文件。注意，删除旧文件是，
  42.             那些为了归档而创建的目录也会被删除。
  43.             -->
  44.             <MaxHistory>365</MaxHistory>
  45.             <!-- 
  46.             当日志文件超过maxFileSize指定的大小是，根据上面提到的%i进行日志文件滚动 注意此处配置SizeBasedTriggeringPolicy是无法实现按文件大小进行滚动的，必须配置timeBasedFileNamingAndTriggeringPolicy
  47.             -->
  48.             <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
  49.                 <maxFileSize>100MB</maxFileSize>
  50.             </timeBasedFileNamingAndTriggeringPolicy>
  51.         </rollingPolicy>
  52.         <!--
  53.         日志输出格式：%d表示日期时间，%thread表示线程名，%-5level：级别从左显示5个字符宽度 %logger{50} 表示logger名字最长50个字符，否则按照句点分割。 %msg：日志消息，%n是换行符
  54.         -->     
  55.         <layout class="ch.qos.logback.classic.PatternLayout">
  56.             <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [ %thread ] - [ %-5level ] [ %logger{50} : %line ] - %msg%n</pattern>
  57.         </layout>
  58.     </appender>
  60.     <!-- 
  61.     logger主要用于存放日志对象，也可以定义日志类型、级别
  62.     name：表示匹配的logger类型前缀，也就是包的前半部分
  63.     level：要记录的日志级别，包括 TRACE < DEBUG < INFO < WARN < ERROR
  64.     additivity：作用在于children-logger是否使用 rootLogger配置的appender进行输出，false：表示只用当前logger的appender-ref，true：表示当前logger的appender-ref和rootLogger的appender-ref都有效
  65.     -->
  66.     <!-- hibernate logger -->
  67.     <logger name="org.hibernate" level="error" />
  68.     <!-- Spring framework logger -->
  69.     <logger name="org.springframework" level="error" additivity="false"></logger>
  71.     <logger name="com.vic.study" level="info" additivity="true">
  72.         <appender-ref ref="appLogAppender" />
  73.     </logger>
  75.     <!-- 
  76.     root与logger是父子关系，没有特别定义则默认为root，任何一个类只会和一个logger对应，
  77.     要么是定义的logger，要么是root，判断的关键在于找到这个logger，然后判断这个logger的appender和level。 
  78.     -->
  79.     <root level="info">
  80.         <appender-ref ref="stdout" />
  81.         <appender-ref ref="appLogAppender" />
  82.     </root>
  83. </configuration>

3. 引入常用工具类

详见VIC的工具类

最重要的为正则工具类

二、编码

1、 Jsonp连接类CrawlConnect

HTTP请求相关

  1. import com.vic.study.crawl.util.IOUtils;
  2. /**
  3.  * HTTP请求工具类
  4.  * @author VIC
  5.  *
  6.  */
  7. public class CrawlConnect {
  8.     private final Logger logger = LoggerFactory.getLogger(CrawlConnect.class);
  10.     private Connection connection;
  12.     /**
  13.      * 必定先调用这个方法
  14.      * @param url
  15.      * @return
  16.      */
  17.     public CrawlConnect(Connection connection){
  18.         this.connection = connection;
  19.     }
  21.     public CrawlConnect url(String url){
  22.         connection.url(url);
  23.         return this;
  24.     }
  26.     public CrawlConnect url(URL url){
  27.         connection.url(url);
  28.         return this;
  29.     }
  30.     public CrawlConnect cookie(String name, String value){
  31.         connection.cookie(name, value);
  32.         return this;
  33.     }
  35.     public CrawlConnect cookie(Map<String, String> cookies){
  36.         connection.cookies(cookies);
  37.         return this;
  38.     }
  40.     public CrawlConnect data(String... keyvals){
  41.         connection.data(keyvals);
  42.         return this;
  43.     }
  45.     public CrawlConnect data(String key, String value ) {
  46.         connection.data(key, value);
  47.         return this;
  48.     }
  50.     public CrawlConnect data(String key, String filename, InputStream in){
  51.         connection.data(key, filename, in);
  52.         return this;
  53.     }
  55.     public Connection.Response execute() throws IOException{
  56.         return connection.execute();
  57.     }
  59.     public CrawlConnect followRedirects(boolean followRedirects){
  60.         connection.followRedirects(followRedirects);
  61.         return this;
  62.     }
  64.     //get
  65.     public Document getDocument() throws IOException {
  66.         return connection.get();
  67.     }
  69.     public String getHtml() throws IOException{
  70.         return this.getDocument().html();
  71.     }
  73.     public String getBodyText() throws IOException{
  74.         return this.getDocument().body().text();
  75.     }
  77.     public CrawlConnect header(String key, String value) {
  78.         connection.header(key, value);
  79.         return this;
  80.     }
  82.     public CrawlConnect maxBodySize(int bytes) {
  83.         connection.maxBodySize(bytes);
  84.         return this;
  85.     }
  87.     public CrawlConnect method(Connection.Method method){
  88.         connection.method(method);
  89.         return this;
  90.     }
  91.     //Provide an alternate parser to use when parsing the response to a Document. 
  92.     public CrawlConnect parser(Parser parser){
  93.         connection.parser(parser);
  94.         return this;
  95.     }
  97.     //post
  98.     public Document postDocument() throws IOException{
  99.         return connection.post();
  100.     }
  102.     public String postHtml() throws IOException{
  103.         return this.postDocument().html();
  104.     }
  106.     public String postBodyText() throws IOException{
  107.         return this.postDocument().body().text();
  108.     }
  110.     //Sets the default post data character set for x-www-form-urlencoded post data 
  111.     public CrawlConnect postDataCharset(String charset) {
  112.         connection.postDataCharset(charset);
  113.         return this;
  114.     }
  117.     public Connection.Request request(){
  118.         return connection.request();
  119.     }
  121.     public Connection.Response response(){
  122.         return connection.response();
  123.     }
  125.     //Disable/enable TSL certificates validation for HTTPS requests. 
  126.     public  CrawlConnect validateTLSCertificates(boolean value) {
  127.         connection.validateTLSCertificates(value);
  128.         return this;
  129.     }
  130.     /**
  131.      * 下载文件到本地
  132.      * @param path
  133.      * @param fileName
  134.      * @throws IOException 
  135.      */
  136.     public void downFile(String path, String fileName) throws IOException{
  137.         Response response = this.execute();
  138.         File file = IOUtils.getFileByPathAndName(path, fileName);
  139.         FileOutputStream out = new FileOutputStream(file);
  140.         out.write(response.bodyAsBytes());
  141.         out.close();
  142.     }
  145. }

2. 爬虫基类BaseCrawl

爬虫类继承此抽象类，方便连接、操作网页

  1. import org.jsoup.Connection;
  2. import org.jsoup.Jsoup;
  4. import com.vic.study.crawl.util.PropertiesUtil;
  5. /**
  6.  * 爬虫基类
  7.  * @author VIC
  8.  *
  9.  */
  10. public abstract class BaseCrawl {
  12.     /**
  13.      * 连接网页
  14.      * @param url
  15.      * @return
  16.      */
  17.     protected CrawlConnect con(String url){
  18.         Connection conn = Jsoup.connect(url).ignoreContentType(true).timeout(PropertiesUtil.getIntByKey("timeout"));
  19.         return new CrawlConnect(conn);
  20.     }
  22. }

3. 爬虫主程序CrawlCnblogList

根据分类抓取博客园下文章列表及详情及其他 写入本地文件

  1. import java.io.File;
  2. import java.io.IOException;
  3. import java.util.Date;
  4. import java.util.List;
  5. import java.util.UUID;
  7. import org.apache.commons.io.FileUtils;
  8. import org.apache.commons.lang.StringUtils;
  9. import org.slf4j.Logger;
  10. import org.slf4j.LoggerFactory;
  12. import com.vic.study.crawl.util.IOUtils;
  13. import com.vic.study.crawl.util.RegexUtil;
  16. /**
  17.  * 抓取博客园的列表以及详情
  18.  * @author VIC
  19.  *
  20.  */
  21. public class CrawlCnblogList extends BaseCrawl{
  23.     private  Logger logger = LoggerFactory.getLogger(CrawlCnblogList.class);
  25.     private static final String SAVE_LOCATION = "E:/crawl/";//下载的保存位置
  27.     private  static final String baseUrl = "http://www.cnblogs.com/cate/";
  29.     private static final String listRegex = "<div id=\"post_list\">(.*?)</div>\\s*<script>"; //列表页
  30.     private static final String itemRegex = "<div class=\"post_item\">(.*?)<div class=\"clear\"></div>\\s*</div>"; //提取列表页的每个item
  32.     private static final String titleRegex = "<h3><a class=\"titlelnk\" href=\".*?\" target=\"_blank\">(.*?)</a></h3>";//标题
  33.     private static final String linkRegex = "<h3><a class=\"titlelnk\" href=\"(.*?)\" target=\"_blank\">.*?</a></h3>";//链接
  34.     private static final String descRegex = "<p class=\"post_item_summary\"> (.*?)</p>";//简介
  36.     private static final String authorRegex = "<a href=\".*?\" class=\"lightblue\">(.*?)</a>";//作者
  37.     private static final String timeRegex = "发布于 (.*?)<span class=\"article_comment\">";//发布时间
  39.     private static final String detailRegex = "<div id=\"cnblogs_post_body\">(.*?)</div>\\s*<div id=\"MySignature\"></div>";//文章详情内容
  40.     private static final String imgRegex = "<img .*? src=\"(.*?)\" .*?>";//详情页图片地址
  41.     /**
  42.      * 根据分类抓取博客园下文章列表及详情及其他 写入本地文件
  43.      * @param categorys 如java/,php/
  44.      * @throws IOException 
  45.      */
  46.     public  void list(String...categories) {
  47.         if(categories.length < 1) {
  48.             logger.info("请输入至少一个分类");
  49.             return;
  50.         }
  51.         for(String category : categories){
  52.             String url = baseUrl + category +"/";
  53.             try {
  55.                 String basePath = SAVE_LOCATION + category +"/"  + new Date().getTime();
  56.                 //获取文章列表
  57.                 String html = con(url).getHtml();
  58.                 String itemsStr = RegexUtil.getFirstString(html, listRegex, 1);
  59.                 List<String> list = RegexUtil.getList(itemsStr, itemRegex, 1);
  60.                 for(String item : list) {
  62.                     String title = RegexUtil.getFirstString(item, titleRegex,1);
  63.                     String link = RegexUtil.getFirstString(item, linkRegex,1);
  64.                     String desc = RegexUtil.getFirstString(item, descRegex,1);
  65.                     String author = RegexUtil.getFirstString(item, authorRegex,1);
  66.                     String time = RegexUtil.getFirstString(item, timeRegex,1);
  68.                     StringBuffer baseContent = new StringBuffer();
  69.                     baseContent.append("标题:").append(title).append("\r\n");
  70.                     baseContent.append("链接:").append(link).append("\r\n");
  71.                     baseContent.append("作者:").append(author).append("\r\n");
  72.                     baseContent.append("时间:").append(time).append("\r\n");
  73.                     baseContent.append("简介:").append(desc).append("\r\n");
  75.                     String titleDir = StringUtils.isEmpty(title)? ("title" +UUID.randomUUID()) : title; 
  76.                     String baseDir = basePath + File.separator +  titleDir ;//每一篇文章存的目录
  77.                     String fileName = (StringUtils.isEmpty(title)? ("title" +UUID.randomUUID()) : title ) + ".txt";
  78.                     File baseFile = IOUtils.getFileByPathAndName(baseDir, fileName);
  79.                     FileUtils.write(baseFile, baseContent.toString(), "utf-8");
  81.                     //获取详情
  82.                     try{
  83.                         String detailPath = baseDir + "/detail/";
  84.                         String detail = con(link).getHtml();
  85.                         String content = RegexUtil.getFirstString(detail, detailRegex, 1);
  86.                         FileUtils.writeStringToFile(IOUtils.getFileByPathAndName(detailPath, fileName), content, "utf-8");
  87.                         List<String> imgs = RegexUtil.getList(content, imgRegex, 1);
  88.                         if(imgs.size() > 0) {
  89.                             for(String img : imgs){
  90.                                 String imgName = img.substring(img.lastIndexOf("/")+ 1, img.length());
  91.                                 try{
  92.                                     con(img).downFile(detailPath, imgName);
  93.                                     logger.warn("图片{}下载成功{}",imgName, img);
  94.                                 }catch(Exception e) {
  95.                                     logger.warn("图片{}下载失败",imgName);
  96.                                 }
  98.                             }
  100.                         }
  101.                     }catch(Exception e){
  102.                         logger.warn("获得{}详情失败:{}", title, link);
  103.                     }
  106.                 }
  107.             } catch (IOException e) {
  108.                 logger.warn("获得{}列表失败:{}", categories, url);
  109.                 e.printStackTrace();
  110.             }
  111.         }
  112.     }
  114.     public static void main(String[] args) {
  115.         CrawlCnblogList l = new CrawlCnblogList();
  116.         l.list("php", "java");
  117.     }
  118. }

备注

这是个垂直抓取的网络爬虫Demo,

由于未察觉博客园对请求header做相关验证，本Demo中并未对头信息作相应伪装。

HTTPUnit

  1. package com.guoyuan.scene.database.test;
  3. import java.net.URL;
  4. import java.util.List;
  6. import com.gargoylesoftware.htmlunit.Page;
  7. import com.gargoylesoftware.htmlunit.WebClient;
  8. import com.gargoylesoftware.htmlunit.WebRequest;
  9. import com.gargoylesoftware.htmlunit.html.HtmlButton;
  10. import com.gargoylesoftware.htmlunit.html.HtmlDivision;
  11. import com.gargoylesoftware.htmlunit.html.HtmlImageInput;
  12. import com.gargoylesoftware.htmlunit.html.HtmlInput;
  13. import com.gargoylesoftware.htmlunit.html.HtmlPage;
  15. public class HttpTest {
  17.     public static void main(String[] args) throws Exception {
  18.         String url = "http://oa.gyzq.com.cn:8082/welcome.do";// 想采集的网址
  19.         String refer = "http://outofmemory.cn/";
  20.         URL link = new URL(url);
  21.         WebClient wc = new WebClient();
  22.         WebRequest request = new WebRequest(link);
  23.         request.setCharset("UTF-8");
  24.         request.setAdditionalHeader("Referer", refer);// 设置请求报文头里的refer字段
  25.         // //设置请求报文头里的User-Agent字段
  26.         request
  27.                 .setAdditionalHeader("User-Agent",
  28.                         "Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");
  29.         // wc.addRequestHeader("User-Agent",
  30.         // "Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");
  31.         // wc.addRequestHeader和request.setAdditionalHeader功能应该是一样的。选择一个即可。
  32.         // 其他报文头字段可以根据需要添加
  33.         wc.getCookieManager().setCookiesEnabled(true);// 开启cookie管理
  34.         wc.getOptions().setJavaScriptEnabled(true);// 开启js解析。对于变态网页，这个是必须的
  35.         wc.getOptions().setCssEnabled(true);// 开启css解析。对于变态网页，这个是必须的。
  36.         wc.getOptions().setThrowExceptionOnFailingStatusCode(false);
  37.         wc.getOptions().setThrowExceptionOnScriptError(false);
  38.         wc.getOptions().setTimeout(10000);
  39.         // 设置cookie。如果你有cookie，可以在这里设置
  40.         // Set<Cookie> cookies = null;
  41.         // Iterator<Cookie> i = cookies.iterator();
  42.         // while (i.hasNext())
  43.         // {
  44.         // wc.getCookieManager().addCookie(i.next());
  45.         // }
  46.         // 准备工作已经做好了
  47.         HtmlPage page = null;
  48.         page = wc.getPage(request);
  49.         if (page == null) {
  50.             System.out.println("采集 " + url + " 失败!!!");
  51.             return;
  52.         }
  53.         String content = page.asXml();// 网页内容保存在content里
  54.         if (content == null) {
  55.             System.out.println("采集 " + url + " 失败!!!");
  56.             return;
  57.         }
  58.         List<?> inputList = page.getByXPath("//input[@name='image']");
  59.         HtmlImageInput image = (HtmlImageInput)inputList.get(0);
  60.         HtmlInput userid = (HtmlInput)page.getHtmlElementById("userId");
  61.         HtmlInput pass = (HtmlInput)page.getHtmlElementById("pass");
  62.         userid.setValueAttribute("0254");
  63.         pass.setValueAttribute("password123");
  64.         image.click();
  65.         request = new WebRequest(new URL("http://oa.gyzq.com.cn:8082/UIProcessor?Table=vGYMS_JBSQ"));
  66.         request.setCharset("UTF-8");
  67.         request.setAdditionalHeader("Referer", refer);// 设置请求报文头里的refer字段
  68.         // //设置请求报文头里的User-Agent字段
  69.         request
  70.                 .setAdditionalHeader("User-Agent",
  71.                         "Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");
  72.         HtmlPage page2 = wc.getPage(request);
  73.         List<?> buttonList = page2.getElementsByTagName("button");
  74.         List<?> divList = page2.getByXPath("//div[contains(@class,'x-grid3-row')]");
  75.         for (int i = 0; i < buttonList.size(); i++) {
  76.             HtmlButton button = (HtmlButton)buttonList.get(i);
  77.             System.out.println(button.getTextContent());
  78.         }
  79.         HtmlButton button = (HtmlButton)page2.getElementById("ext-gen39");
  80.         for (int i = 0; i < divList.size(); i++) {
  81.             HtmlDivision div = (HtmlDivision)divList.get(i);
  82.             div.click();
  83.             if (button != null) {
  84.                 Page popup = button.click();
  85.                 if (popup.isHtmlPage()) {
  86.                     System.out.println(((HtmlPage) popup).asXml());
  87.                 }
  88.             }
  89.             System.out.println(div.asXml());
  90.         }
  91. //      Page page2 = image.click();
  92. //      if (page2.isHtmlPage()) {
  93. //          HtmlPage htmlPage = (HtmlPage) page2;
  94. //          List<FrameWindow> iframeList = htmlPage.getFrames();
  95. //          for (FrameWindow frame : iframeList) {
  96. //              if (frame.getName().equals("fraRightFrame")) {
  97. //                  final HtmlPage pageTwo = (HtmlPage) frame.getEnclosedPage();
  98. //                  HtmlAnchor anchor = pageTwo.getAnchorByText("陈洋于2015-08-24发起的发文申请流程");
  99. //                  HtmlPage page3 = anchor.click();
  100. //                  System.out.println(page3.asXml());
  101. //                  System.out.println(page3.asXml());
  102. //              }
  103. //          }
  104. //      }
  105. //      // 搞定了
  106. //      CookieManager CM = wc.getCookieManager(); // WC = Your WebClient's name
  107. //      Set<Cookie> cookies_ret = CM.getCookies();// 返回的Cookie在这里，下次请求的时候可能可以用上啦。
  108. //      System.out.println(123);
  109.     }
  110. }

ERPCrawl

  1. package com.vic.study.crawl.erp;
  3. import java.io.File;
  4. import java.io.FileInputStream;
  5. import java.io.IOException;
  6. import java.io.InputStream;
  7. import java.net.URLDecoder;
  8. import java.util.Date;
  9. import java.util.HashMap;
  10. import java.util.List;
  11. import java.util.Map;
  12. import java.util.UUID;
  14. import org.apache.commons.lang.time.DateFormatUtils;
  15. import org.apache.commons.lang.time.DateUtils;
  16. import org.jsoup.Connection.Method;
  17. import org.jsoup.Connection.Response;
  18. import org.jsoup.nodes.Document;
  19. import org.jsoup.nodes.Element;
  21. import com.vic.study.crawl.BaseCrawl;
  22. import com.vic.study.crawl.util.ERPContants;
  23. import com.vic.study.crawl.util.RegexUtil;
  25. /**
  26.  * 抓取ERP   自动提交加班流程
  27.  * @author VIC
  28.  *
  29.  */
  30. public class ErpCrawl extends BaseCrawl{
  33.     public static void printMap(Map<String, String> map){
  34.         System.out.println("打印map start--------------->>");
  35.         for(Map.Entry<String, String> entry : map.entrySet()) {
  36.             System.out.println(entry.getKey() + "—" + entry.getValue());
  37.         }
  38.         System.out.println("打印map end<<---------------------");
  39.     }
  40.     /**
  41.      * 登陆
  42.      * @return
  43.      */
  44.     public  Map<String, String> login(){
  45.         try {
  46.             Map<String, String> cookies = con(ERPContants.LOGIN_URL).data("userId", ERPContants.USERNAME).data("pass", ERPContants.PASSWORD).execute().cookies();
  47.             if(cookies != null) {
  48.                 Document docemnet = con(ERPContants.HOME_PAGE_URL).cookie(cookies).getDocument();
  49.                 Element userNameEle=docemnet.getElementById("userNameCt");
  50.                 if(userNameEle!=null){
  51.                     logger.warn("{} 登陆成功" , ERPContants.USERNAME);
  52.                     return cookies;
  53.                 }else{
  54.                     logger.warn("{}登陆失败" , ERPContants.USERNAME);
  55.                 }
  56.             }
  57.         } catch (IOException e) {
  58.             logger.warn("{} 登陆失败",ERPContants.USERNAME);
  59.         }
  60.         return null;
  61.     }
  64.     /**
  65.      * 2 加班流程
  66.      */
  68.     public void loadOvertimeProcess(){
  69.         Map<String, String> cookies = login();
  70.         if(cookies == null){
  71.             return ;
  72.         }
  74.         String processNo = "";//流程编号
  75.         try {
  76.             String html =  con(ERPContants.OVERTIME_PROCESS_URL).cookie(cookies).getHtml();
  77.             String token = RegexUtil.getFirstString(html, "Token=(.*?)&", 1);
  78.             String workId = RegexUtil.getFirstString(html, "&WorkID=(\\d+?)&", 1);
  79.             String StepID = "initialStep";
  80.             String WorkActionID = RegexUtil.getFirstString(html, "&WorkActionID=(\\d+?)&", 1);
  81.             logger.info("进入加班流程页面 token:{},workID:{},WorkActionID:{}", token,workId, WorkActionID);
  82.             processNo = RegexUtil.getFirstString(html, "提出的加班申请\\[(\\d+?)\\]", 1);
  84.             /**
  85.              * 添加加班详细信息
  86.              */
  87.             goAddDetail(cookies, token, workId);
  88.             /**
  89.              * 添加附加
  90.              */
  91.             goAddAttachment(cookies, token, workId);
  93. //          if(1==1) return;
  94. //          String fromHtml = RegexUtil.getFirstString(html, "<form .*?>(.*?)</form>", 1);//提取form
  95. //          List<String> nameList = RegexUtil.getList(fromHtml, "(<.*? name=.*?>)", 1);//提取所有的带有name的标签
  96.             //提取结果为 需要提交的参数
  97.             Map<String, String> data = new HashMap<>();
  98.             data.put("$C{LCBT}", "0");//流程标题
  99. 			data.put("$C{JBXXXX}", "12");
  100.             data.put("$C{JBNR}", "7");//加班内容
  101. 			data.put("JBNR", "具体的加班内容");// 具体的加班内容 手动填写
  102. 			data.put("$C{FJ}", "12");//附件
  103.             data.put("$C{SPYJ}", "0");//审批意见
  104. 				//data.put("SPYJ", "");//审批意见
  105. 			data.put("$C{CSR}", "25");//抄送信息
  106.                 //data.put("CSR", "");
  107.             data.put("$C{SQR}", "0");//申请人
  108. 			data.put("SQR", "60");//申请人   需/可 抓取
  109. 			data.put("$C{SQRBM}", "0");//申请人部门
  110.             data.put("SQRBM", "142");//申请人部门  需/可 抓取
  111.             data.put("$C{SQSJ}", "0");//申请时间
  112.             data.put("SQSJ", "2015-11-18 15:09:44");//申请时间
  114.             // 保存时通用参数  Token WorkID StepID
  115.             data.put("WorkID", workId);
  116.             data.put("StepID", StepID);
  117.             data.put("Token", token);
  118.             data.put("WorkActionID", WorkActionID);
  120.             //抓取相关参数
  121.             String SQRBM = RegexUtil.getFirstString(html, "name=\"SQRBM\".*?value=\"(\\d*?)\".*?>", 1);//申请人部门
  122.             data.put("SQRBM", SQRBM);
  123.             String SQR = RegexUtil.getFirstString(html, "name=\"SQR\".*?value=\"(\\d*?)\".*?>", 1);//申请人;
  124.             data.put("SQR", SQR);
  126.             //手动添加加班内容
  127.             data.put("JBNR", "这是程序添加的测试加班内容" + UUID.randomUUID() + " 时间：" + DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm"));
  131. //          提交加班流程
  132.             String  resultHtml = con(ERPContants.SUBMIT_OVERTIME_PROCESS_URL).cookie(cookies).data(data).postHtml();
  133.             if(resultHtml.contains("执行提交成功")){
  136.                 logger.info("本次成功提交的流程编号为：{}" , RegexUtil.getFirstString(resultHtml, "提出的加班申请\\[(\\d*?)\\]", 1));
  137.                 System.out.println("processNo:" + processNo);
  138.             }
  140.         } catch (IOException e) {
  141.             logger.warn("OPEN加班流程{}失败" , ERPContants.OVERTIME_PROCESS_URL);
  142.         }
  143.     }
  145.     /**
  146.      * 3 前往新增加班流程明细 并提交
  147.      *   当前token workID 与 “2 加班流程”中的一致  尚未改变
  148.      */
  149.     public void goAddDetail(Map<String, String> cookies, String token, String  workID){
  150.         try {
  151.             String detailHtml = con(ERPContants.ADD_OVERTIME_PROCESS_DETAIL_URL).cookie(cookies).data("Token", token).data("WorkID", workID).getHtml();
  152.             String operateID = RegexUtil.getFirstString(detailHtml, ";OperateID=(.*?)&", 1);
  153.             logger.info("前往新增加班流程明细 token:{},workID:{},operateID:{}", token,workID, operateID);
  155.             Map<String, String> params = new HashMap<>();
  156.             params.put("OperateID", operateID);
  157. //          params.put("Token", token);
  158.             params.put("WorkID", workID);
  159.             params.put("XM", "60");//姓名
  160.             params.put("$C{XM}", "0");
  161.             params.put("BM", "盐城解放南路证券营业部");//部门
  162.             params.put("KSSJ", DateFormatUtils.format(DateUtils.addHours(new Date(), 6)  ,ERPContants.TIME_PATTERN));//加班开始时间
  163.             params.put("JSSJ", DateFormatUtils.format(DateUtils.addHours(new Date(), 8)  ,ERPContants.TIME_PATTERN));//加班结束时间
  164.             params.put("JBSJ", "2.0");//加班时间(小时)
  165.             params.put("TXHBZ", "1");//调休或补助 0 -1 
  166.             try{
  168.                 String html = con(ERPContants.SUBMIT_OVERTIME_PROCESS_DETAIL_URL).cookie(cookies).data(params).postHtml();
  169.                 if(html.contains("执行新增成功")){
  170.                     logger.info("成功新增加班流程明细");
  171.                 }
  172.             }catch(Exception e){
  173.                 logger.warn("提交加班明细{}失败" , ERPContants.SUBMIT_OVERTIME_PROCESS_DETAIL_URL);
  174.             }
  176.         } catch (IOException e) {
  177.             e.printStackTrace();
  178.             logger.warn("前往加班明细{}失败" , ERPContants.ADD_OVERTIME_PROCESS_DETAIL_URL);
  179.         }
  180.     }
  182.     /**
  183.      * 4 前往添加附件
  184.      *  当前token  workID 与传入的token workId尚且一致
  185.      *  operateID  和步骤3中的 operateID  不一致
  186.      */
  187.     public void goAddAttachment(Map<String, String> cookies, String token, String workID){
  188.         InputStream in = null;
  189.         try {
  190. //          String attachmentHtml = con(ERPContants.ADD_ATTACHMENT_URL).cookie(cookies).data("Token", token).data("WorkID", workID).getHtml();
  192.             Response attachmentResponse = con(ERPContants.ADD_ATTACHMENT_URL).cookie(cookies).data("Token", token).data("WorkID", workID).execute();
  194.             String refererUrl = attachmentResponse.url().toString();
  196.             String attachmentHtml = attachmentResponse.parse().html();
  197.             String operateID = RegexUtil.getFirstString(attachmentHtml, ";OperateID=(.*?)&", 1);
  198.             logger.info("前往添加附件 token:{},workID:{},operateID:{}", token,workID, operateID);
  199.             Map<String, String> params = new HashMap<>();//参数  文件参数为  FJ
  200.             params.put("OperateID", operateID);
  201.             params.put("Token", token);
  202.             params.put("WorkID", workID);
  204.             params.put("operate", "Add");
  205.             /*params.put("$C{FJMC}", "1");
  206. 			params.put("$C{FJ}", "9");
  207.             params.put("$C{BZ}", "7");*/
  208.             params.put("BZ", "这是一个附件备注" +UUID.randomUUID());
  209.             File file = new File("E:\\pic\\4.jpg");
  210.             if(!file.exists()){
  211.                 logger.warn("不存在的附件");
  212.                 return ;
  213.             }
  214.             params.put("FJMC", file.getName());//附件名称
  217.             in = new FileInputStream(file);
  218.             Response response = con(ERPContants.SUBMIT_ATTACHMENT_URL).header("Content-Type", "multipart/form-data").
  219.                     cookie(cookies).data(params).data("FJ", file.getName(), in).setMethod(Method.POST).execute();
  220.             String result = response.parse().html();
  221.             System.out.println("ADD RESULT");
  222.             System.out.println(result);
  223. //          if(response.body().contains("执行新增成功")){
  224. //              logger.info("提交附件成功");
  225. //              logger.info("返回提交附件的结果为\n{{}}" ,RegexUtil.getFirstString(result, "var params =\\{(.*?)\\};", 1));
  226. //          }
  227.             /*
  228.              *  var params ={"message":"执行新增成功.","data":{ "results": 2, "records":  [{"id":"-1001","FJMC":"123请问","FJ":"3.jpg","BZ":""},{"id":"-1002","FJMC":"额外发","FJ":"3.jpg","BZ":"阿
  229.                                 萨德"}]},"retVal":true,"success":true};
  230.              * */
  231.         } catch (IOException e) {
  232.             e.printStackTrace();
  233.             logger.info("添加附件失败");
  234.         }finally {
  235.             try {
  236.                 in.close();
  237.             } catch (IOException e) {
  238.                 // TODO Auto-generated catch block
  239.                 e.printStackTrace();
  240.             }
  241.         }
  242.     }
  244.     /********************************************************************/
  247.     public static void main(String[] args) {
  248.         ErpCrawl e = new ErpCrawl();
  249. //      e.testupload();
  250.         e.loadOvertimeProcess(); 
  251. //      testUrl();
  252.     }
  257.     /********************************************************************/
  258.     public  void testupload(){
  259.         File file = new File("E:\\pic\\4.jpg");
  260.         if(!file.exists()){
  261.             logger.warn("不存在的附件");
  262.             return ;
  263.         }
  264.         try{
  266.             Response response = con("http://121.41.76.50/upfile/ajax/upfile").data("upfile", file.getName(), new FileInputStream(file)).setMethod(Method.POST).execute();
  267.             System.out.println(response.body());
  268.             System.out.println(response.statusCode());
  269.         }catch(IOException e){
  270.             e.printStackTrace();
  271.         }
  273.     }
  276.     public static void testUrl(){
  277.         String url="http://oa.gyzq.com.cn:8082/OperateProcessor?operate=WorkSave&Table=wfGYMS_JBSQ&Token=c98007f224b69d598b13f30e27b217ff&WorkID=2577&StepID=4&&isSubmit=1&%24C%7BCSR%7D=25&%24C%7BFJ%7D=12&%24C%7BJBNR%7D=7&%24C%7BJBXXXX%7D=12&%24C%7BLCBT%7D=0&%24C%7BSPYJ%7D=0&%24C%7BSQRBM%7D=0&%24C%7BSQR%7D=0&%24C%7BSQSJ%7D=0&CSR=&JBNR=0002&SPYJ=&SQR=60&SQRBM=142&SQSJ=2015-11-18%2013%3A19%3A29&operate=Update";
  278.         List<String[]> list = RegexUtil.getList(url, "&(.*?)=(.*?)&", new int[]{1,2});
  279.         for(String us[] : list) {
  280.             System.out.println(us[0] + " || " + URLDecoder.decode(us[0]) + " | " +us[1]);
  281.         }
  282.         System.out.println("00000000");
  283.         //解析提交流程的URL   $C{JBXXXX}==加班详细信息  $C{FJ}==附件   $C{SPYJ}==审批意见=0 $C{SQR}==申请人=0 SQRBM==申请人部门=142 WorkID
  284.         String url2 = "http://oa.gyzq.com.cn:8082/OperateProcessor?operate=WorkSave&Table=wfGYMS_JBSQ&Token=1cc072aef0e222b263d9c0e470dd8b1e&WorkID=40&StepID=initialStep&&isSubmit=1&%24C%7BCSR%7D=25&%24C%7BFJ%7D=12&%24C%7BJBNR%7D=7&%24C%7BJBXXXX%7D=12&%24C%7BLCBT%7D=0&%24C%7BSPYJ%7D=0&%24C%7BSQRBM%7D=0&%24C%7BSQR%7D=0&%24C%7BSQSJ%7D=0&CSR=&JBNR=%E6%B5%8B%E8%AF%95%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%82%E3%80%822015%E5%B9%B411%E6%9C%8818%E6%97%A514%3A53%3A48&SPYJ=&SQR=60&SQRBM=142&SQSJ=2015-11-18%2014%3A50%3A51&operate=Add";
  285.         List<String[]> list2 = RegexUtil.getList(url2, "&(.*?)=(.*?)&", new int[]{1,2});
  286.         for(String us[] : list2) {
  287.             System.out.println(us[0] + " || " + URLDecoder.decode(us[0]) + " | " +us[1]);
  288.         }
  291.         String url5 = "&%24C%7BCSR%7D=25&%24C%7BFJ%7D=12&%24C%7BJBNR%7D=7&%24C%7BJBXXXX%7D=12&%24C%7BLCBT%7D=0&%24C%7BSPYJ%7D=0&%24C%7BSQRBM%7D=0&%24C%7BSQR%7D=0&%24C%7BSQSJ%7D=0&CSR=&JBNR=%E9%98%BF%E8%90%A8%E5%BE%B7%E9%98%BF%E8%90%A8%E5%BE%B7%E6%B5%8B%E8%AF%95%20%E6%B5%8B%E8%AF%95%20%E6%B5%8B%E8%AF%95&SPYJ=&SQR=60&SQRBM=142&SQSJ=2015-11-19%2011%3A07%3A59&";
  292.         List<String[]> list5 = RegexUtil.getList(url5, "&(.*?)=(.*?)&", new int[]{1,2});
  293.         for(String us[] : list5) {
  294.             System.out.println(us[0] + " || " + URLDecoder.decode(us[0]) + " | " +us[1]);
  295.         }
  296.     }
  298. }

ErpContants

  1. package com.vic.study.crawl.util;
  3. public class ERPContants {
  5.     public static final String TIME_PATTERN = "yyyy-MM-dd HH:mm";
  7.     public static final String USERNAME = "0254";
  9.     public static final String PASSWORD = "password123";
  11.     public static final String BASE_URL = "http://oa.gyzq.com.cn:8082";// 网址
  13.     public static final String LOGIN_URL = "http://oa.gyzq.com.cn:8082/login.do";// 登陆
  15.     public static final String HOME_PAGE_URL = "http://oa.gyzq.com.cn:8082/welcome.do";// 主页
  16.     //加班流程
  17.     public static final String OVERTIME_PROCESS_URL = "http://oa.gyzq.com.cn:8082/StartWorkflow?Workflow=wfGYMS_JBSQ&extWindow=true&PopupWin=true";
  18.     //保存加班流程 Token WorkID StepID
  19.     public static final String SAVE_OVERTIME_PROCESS_URL = "http://oa.gyzq.com.cn:8082/OperateProcessor?operate=WorkSave&Table=wfGYMS_JBSQ&isSubmit=1&operate=Update";
  20.     //提交加班流程 WorkActionID Token WorkID
  21.     public static final String SUBMIT_OVERTIME_PROCESS_URL = "http://oa.gyzq.com.cn:8082/OperateProcessor?operate=WorkAction.5&Table=wfGYMS_JBSQ&StepID=initialStep&&isSubmit=1&operate=Add";
  24.     // Token WorkID 前往新增详细
  25.     public static final String ADD_OVERTIME_PROCESS_DETAIL_URL = "http://oa.gyzq.com.cn:8082/WorkProcessor?Table=wfGYMS_JBSQ&StepID=initialStep&Column=JBXXXX&SubmitForm=DATA_FORM&EVENT_SOURCE=AddHypotObj&extWindow=true&PopupWin=true";
  26.     // 提交详细 Token WorkID OperateID
  27.     public static final String SUBMIT_OVERTIME_PROCESS_DETAIL_URL = "http://oa.gyzq.com.cn:8082/OperateProcessor?Table=wfGYMS_JBSQ&StepID=initialStep&extWindow=true&extWindow=true&PopupWin=true&NewWindow=false";
  29.     //打开提交附件 Token WorkID
  30.     public static final String ADD_ATTACHMENT_URL = "http://oa.gyzq.com.cn:8082/WorkProcessor?Table=wfGYMS_JBSQ&StepID=initialStep&Column=FJ&SubmitForm=DATA_FORM&EVENT_SOURCE=AddHypotObj&extWindow=true&PopupWin=true";
  31.     //提交附件 OperateID WorkID Token                                                                                   
  32.     public static final String SUBMIT_ATTACHMENT_URL = "http://oa.gyzq.com.cn:8082/OperateProcessor?Table=wfGYMS_JBSQ&StepID=7&extWindow=true&PopupWin=true&NewWindow=false";
  33. //                                                                                      /OperateProcessor?Table=wfGYMS_JBSQ&Token=c04a7c4eca425b738e8686487351af6f&WorkID=202&StepID=initialStep&OperateID=34b53c980468fe3fc10140f46acf28e9&extWindow=true
  34. }
添加新批注
在作者公开此批注前，只有你和作者可见。
回复批注