升级webmagic内核至0.6.0 fixed #7

gsh199449 · Dec 24, 2016 · f8dd7a3 · f8dd7a3
1 parent 164e4ae
commit f8dd7a3
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/pom.xml b/pom.xml
@@ -1,4 +1,4 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.gs</groupId>
@@ -181,7 +181,7 @@
         <dependency>
             <groupId>us.codecraft</groupId>
             <artifactId>webmagic-core</artifactId>
-            <version>0.5.3</version>
+            <version>0.6.0</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>

diff --git a/src/main/java/com/gs/spider/gather/commons/ContentLengthLimitHttpClientDownloader.java b/src/main/java/com/gs/spider/gather/commons/ContentLengthLimitHttpClientDownloader.java
@@ -98,7 +98,7 @@ public Page download(Request request, Task task) {
         int statusCode = 0;
         HttpUriRequest httpUriRequest = null;
         try {
-            httpUriRequest = getHttpUriRequest(request, site, headers);
+            httpUriRequest = getHttpUriRequest(request, site, headers, null);
             httpResponse = getHttpClient(site).execute(httpUriRequest);
             statusCode = httpResponse.getStatusLine().getStatusCode();
             request.putExtra(Request.STATUS_CODE, statusCode);
@@ -140,15 +140,15 @@ public Page download(Request request, Task task) {
 
     private CloseableHttpClient getHttpClient(Site site) {
         if (site == null) {
-            return httpClientGenerator.getClient(null);
+            return httpClientGenerator.getClient(null, null);
         }
         String domain = site.getDomain();
         CloseableHttpClient httpClient = httpClients.get(domain);
         if (httpClient == null) {
             synchronized (this) {
                 httpClient = httpClients.get(domain);
                 if (httpClient == null) {
-                    httpClient = httpClientGenerator.getClient(site);
+                    httpClient = httpClientGenerator.getClient(site, null);
                     httpClients.put(domain, httpClient);
                 }
             }