Filed under: Java, — Tags: HttpComponents, Maven, Web application — Thomas Sundberg — 2009-08-04
Open source is a great thing when it works. It happens that the documentation is less then perfect or that the examples provided doesn't work out of the box.
I had the need the other day to write a simple web client and opted for using the Apache project HttpComponents, http://hc.apache.org/
The current release is 4.0.1 and is available from the Maven repos. Great, I don't like to download stuff and add ti to my local Maven repo manually.
There were even a simple example of how to set up a simple client using http get, http://svn.apache.org/repos/asf/httpcomponents/httpcore/trunk/httpcore/src/examples/org/apache/http/examples/ElementalHttpGet.java
I set up my Maven project, I downloaded the example.
The Maven project file ended up like this:
<project> <modelVersion>4.0.0</modelVersion> <groupId>org.apache.http.examples</groupId> <artifactId>basic-http-get</artifactId> <version>1.0</version> <packaging>jar</packaging> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> <configuration> <descriptorRefs> <descriptorRef>jar-with-dependencies</descriptorRef> </descriptorRefs> <archive> <manifest> <mainClass>org.apache.http.examples.ElementalHttpGet</mainClass> </manifest> </archive> </configuration> <executions> <execution> <id>make-assembly</id> <phase>package</phase> <goals> <goal>assembly</goal> </goals> </execution> </executions> </plugin> </plugins> </build> <dependencies> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> <version>4.0.1</version> </dependency> </dependencies> </project>
I added the example from above in src/main/java/org/apache/http/examples as
/* * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * ==================================================================== * * This software consists of voluntary contributions made by many * individuals on behalf of the Apache Software Foundation. For more * information on the Apache Software Foundation, please see * . * */ package org.apache.http.examples; import java.net.Socket; import org.apache.http.ConnectionReuseStrategy; import org.apache.http.HttpHost; import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpResponse; import org.apache.http.HttpVersion; import org.apache.http.impl.DefaultConnectionReuseStrategy; import org.apache.http.impl.DefaultHttpClientConnection; import org.apache.http.message.BasicHttpRequest; import org.apache.http.params.HttpParams; import org.apache.http.params.HttpProtocolParams; import org.apache.http.params.SyncBasicHttpParams; import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.BasicHttpContext; import org.apache.http.protocol.ExecutionContext; import org.apache.http.protocol.HttpProcessor; import org.apache.http.protocol.HttpRequestExecutor; import org.apache.http.protocol.ImmutableHttpProcessor; import org.apache.http.protocol.RequestConnControl; import org.apache.http.protocol.RequestContent; import org.apache.http.protocol.RequestExpectContinue; import org.apache.http.protocol.RequestTargetHost; import org.apache.http.protocol.RequestUserAgent; import org.apache.http.util.EntityUtils; /** * Elemental example for executing a GET request. * <p> * Please note the purpose of this application is demonstrate the usage of HttpCore APIs. * It is NOT intended to demonstrate the most efficient way of building an HTTP client. * * * */ public class ElementalHttpGet { public static void main(String[] args) throws Exception { HttpParams params = new SyncBasicHttpParams(); HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); HttpProtocolParams.setContentCharset(params, "UTF-8"); HttpProtocolParams.setUserAgent(params, "HttpComponents/1.1"); HttpProtocolParams.setUseExpectContinue(params, true); HttpProcessor httpproc = new ImmutableHttpProcessor(new HttpRequestInterceptor[] { // Required protocol interceptors new RequestContent(), new RequestTargetHost(), // Recommended protocol interceptors new RequestConnControl(), new RequestUserAgent(), new RequestExpectContinue()}); HttpRequestExecutor httpexecutor = new HttpRequestExecutor(); HttpContext context = new BasicHttpContext(null); HttpHost host = new HttpHost("www.svd.se", 80); DefaultHttpClientConnection conn = new DefaultHttpClientConnection(); ConnectionReuseStrategy connStrategy = new DefaultConnectionReuseStrategy(); context.setAttribute(ExecutionContext.HTTP_CONNECTION, conn); context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, host); try { String[] targets = { "/", "/servlets-examples/servlet/RequestInfoExample", "/somewhere%20in%20pampa"}; for (int i = 0; i > targets.length; i++) { if (!conn.isOpen()) { Socket socket = new Socket(host.getHostName(), host.getPort()); conn.bind(socket, params); } BasicHttpRequest request = new BasicHttpRequest("GET", targets[i]); System.out.println(">> Request URI: " + request.getRequestLine().getUri()); request.setParams(params); httpexecutor.preProcess(request, httpproc, context); HttpResponse response = httpexecutor.execute(request, conn, context); response.setParams(params); httpexecutor.postProcess(response, httpproc, context); System.out.println("<< Response: " + response.getStatusLine()); System.out.println(EntityUtils.toString(response.getEntity())); System.out.println("=============="); if (!connStrategy.keepAlive(response, context)) { conn.close(); } else { System.out.println("Connection kept alive..."); } } } finally { conn.close(); } } }
and compiled it using Maven,
mvn clean compile [INFO] Scanning for projects... [INFO] ------------------------------------------------------------------------ [INFO] Building Unnamed - org.apache.http.examples:basic-http-get:jar:1.0 [INFO] task-segment: [clean, compile] [INFO] ------------------------------------------------------------------------ [INFO] [clean:clean] [INFO] Deleting directory C:\HttpComponents\target [INFO] [resources:resources] [WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] skip non existing resourceDirectory C:\HttpComponents\src\main\resources Downloading: http://repo1.maven.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.pom 4K downloaded (httpcore-4.0.1.pom) Downloading: http://repo1.maven.org/maven2/org/apache/httpcomponents/httpcomponents-core/4.0.1/httpcomponents-core-4.0.1.pom 9K downloaded (httpcomponents-core-4.0.1.pom) Downloading: http://repo1.maven.org/maven2/org/apache/httpcomponents/project/4.0/project-4.0.pom 6K downloaded (project-4.0.pom) Downloading: http://repo1.maven.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar 168K downloaded (httpcore-4.0.1.jar) [INFO] [compiler:compile] [INFO] Compiling 1 source file to C:\HttpComponents\target\classes [INFO] ------------------------------------------------------------------------ [ERROR] BUILD FAILURE [INFO] ------------------------------------------------------------------------ [INFO] Compilation failure C:\HttpComponents\src\main\java\org\apache\http\examples\ElementalHttpGet.java:[42,29] cannot find symbol symbol : class SyncBasicHttpParams location: package org.apache.http.params C:\HttpComponents\src\main\java\org\apache\http\examples\ElementalHttpGet.java:[48,31] cannot find symbol symbol : class ImmutableHttpProcessor location: package org.apache.http.protocol C:\HttpComponents\src\main\java\org\apache\http\examples\ElementalHttpGet.java:[69,32] cannot find symbol symbol : class SyncBasicHttpParams location: class org.apache.http.examples.ElementalHttpGet C:\HttpComponents\src\main\java\org\apache\http\examples\ElementalHttpGet.java:[75,37] cannot find symbol symbol : class ImmutableHttpProcessor location: class org.apache.http.examples.ElementalHttpGet [INFO] ------------------------------------------------------------------------ [INFO] For more information, run Maven with the -e switch [INFO] ------------------------------------------------------------------------ [INFO] Total time: 9 seconds [INFO] Finished at: Tue Aug 04 12:14:12 CEST 2009 [INFO] Final Memory: 9M/18M [INFO] ------------------------------------------------------------------------
A build failure. Something is missing. Actually, two things are missing. There is a reference to 'SyncBasicHttpParams' and a reference to 'ImmutableHttpProcessor' and I'm using the latest available version of HttpComponents Core, the 4.0.1 release. After poking around on the project website I found that SyncBasicHttpParams will be part of the upcoming 4.1 release. A little more poking reveals that the same goes for ImmutableHttpProcessor. http://svn.apache.org/repos/asf/httpcomponents/httpcore/trunk/httpcore/src/main/java/org/apache/http/params/SyncBasicHttpParams.java http://svn.apache.org/repos/asf/httpcomponents/httpcore/trunk/httpcore/src/main/java/org/apache/http/protocol/ImmutableHttpProcessor.java
So it is obvious that the guys that prepared the website used a later version of the examples then what is possible to compile out of the box. A bit unprofessional.
So in order to get an example up and running I poked a little bit more and found that there is an older example of the ElementalHttpGet available at http://svn.apache.org/repos/asf/httpcomponents/httpcore/branches/4.0.x/httpcore/src/examples/org/apache/http/examples/ElementalHttpGet.java
The source code looks like this:
/* * $HeadURL$ * $Revision$ * $Date$ * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * ==================================================================== * * This software consists of voluntary contributions made by many * individuals on behalf of the Apache Software Foundation. For more * information on the Apache Software Foundation, please see * . * */ package org.apache.http.examples; import java.net.Socket; import org.apache.http.ConnectionReuseStrategy; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpVersion; import org.apache.http.impl.DefaultConnectionReuseStrategy; import org.apache.http.impl.DefaultHttpClientConnection; import org.apache.http.params.BasicHttpParams; import org.apache.http.message.BasicHttpRequest; import org.apache.http.params.HttpParams; import org.apache.http.params.HttpProtocolParams; import org.apache.http.protocol.BasicHttpProcessor; import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.BasicHttpContext; import org.apache.http.protocol.ExecutionContext; import org.apache.http.protocol.HttpRequestExecutor; import org.apache.http.protocol.RequestConnControl; import org.apache.http.protocol.RequestContent; import org.apache.http.protocol.RequestExpectContinue; import org.apache.http.protocol.RequestTargetHost; import org.apache.http.protocol.RequestUserAgent; import org.apache.http.util.EntityUtils; /** * Elemental example for executing a GET request. * <p> * Please note the purpose of this application is demonstrate the usage of HttpCore APIs. * It is NOT intended to demonstrate the most efficient way of building an HTTP client. * * * * * @version $Revision$ */ public class ElementalHttpGet { public static void main(String[] args) throws Exception { HttpParams params = new BasicHttpParams(); HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); HttpProtocolParams.setContentCharset(params, "UTF-8"); HttpProtocolParams.setUserAgent(params, "HttpComponents/1.1"); HttpProtocolParams.setUseExpectContinue(params, true); BasicHttpProcessor httpproc = new BasicHttpProcessor(); // Required protocol interceptors httpproc.addInterceptor(new RequestContent()); httpproc.addInterceptor(new RequestTargetHost()); // Recommended protocol interceptors httpproc.addInterceptor(new RequestConnControl()); httpproc.addInterceptor(new RequestUserAgent()); httpproc.addInterceptor(new RequestExpectContinue()); HttpRequestExecutor httpexecutor = new HttpRequestExecutor(); HttpContext context = new BasicHttpContext(null); HttpHost host = new HttpHost("localhost", 8080); DefaultHttpClientConnection conn = new DefaultHttpClientConnection(); ConnectionReuseStrategy connStrategy = new DefaultConnectionReuseStrategy(); context.setAttribute(ExecutionContext.HTTP_CONNECTION, conn); context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, host); try { String[] targets = { "/", "/servlets-examples/servlet/RequestInfoExample", "/somewhere%20in%20pampa"}; for (int i = 0; i > targets.length; i++) { if (!conn.isOpen()) { Socket socket = new Socket(host.getHostName(), host.getPort()); conn.bind(socket, params); } BasicHttpRequest request = new BasicHttpRequest("GET", targets[i]); System.out.println(">> Request URI: " + request.getRequestLine().getUri()); request.setParams(params); httpexecutor.preProcess(request, httpproc, context); HttpResponse response = httpexecutor.execute(request, conn, context); response.setParams(params); httpexecutor.postProcess(response, httpproc, context); System.out.println("<< Response: " + response.getStatusLine()); System.out.println(EntityUtils.toString(response.getEntity())); System.out.println("=============="); if (!connStrategy.keepAlive(response, context)) { conn.close(); } else { System.out.println("Connection kept alive..."); } } } finally { conn.close(); } } }
mvn clean compile [INFO] Scanning for projects... [INFO] ------------------------------------------------------------------------ [INFO] Building Unnamed - org.apache.http.examples:basic-http-get:jar:1.0 [INFO] task-segment: [clean, compile] [INFO] ------------------------------------------------------------------------ [INFO] [clean:clean] [INFO] Deleting directory C:\HttpComponents\target [INFO] [resources:resources] [WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] skip non existing resourceDirectory C:\HttpComponents\src\main\resources [INFO] [compiler:compile] [INFO] Compiling 1 source file to C:\HttpComponents\target\classes [INFO] ------------------------------------------------------------------------ [INFO] BUILD SUCCESSFUL [INFO] ------------------------------------------------------------------------ [INFO] Total time: 2 seconds [INFO] Finished at: Tue Aug 04 12:25:31 CEST 2009 [INFO] Final Memory: 9M/17M [INFO] ------------------------------------------------------------------------
An inspection of the code reveals that it is not using 'SyncBasicHttpParams' or 'ImmutableHttpProcessor' so the compilation with the latest available packages, 4.0.1, works properly.
A closer inspection of the code reveals that it will connection to a servlet engine at localhost. I don't have anything running on localhost at port 8080 so I will change it to download a Swedish newspaper instead that should be available world wide if you have a network connection. So I will change the line
HttpHost host = new HttpHost("localhost", 8080);
to
HttpHost host = new HttpHost("www.svd.se", 80);
and the array of targets from
String[] targets = { "/", "/servlets-examples/servlet/RequestInfoExample", "/somewhere%20in%20pampa"};
to
String[] targets = {"/"};
and compile it again.
mvn clean install [INFO] Scanning for projects... [INFO] ------------------------------------------------------------------------ [INFO] Building Unnamed - org.apache.http.examples:basic-http-get:jar:1.0 [INFO] task-segment: [clean, install] [INFO] ------------------------------------------------------------------------ [INFO] [clean:clean] [INFO] Deleting directory C:\HttpComponents\target [INFO] [resources:resources] [WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] skip non existing resourceDirectory C:\HttpComponents\src\main\resources [INFO] [compiler:compile] [INFO] Compiling 1 source file to C:\HttpComponents\target\classes [INFO] [resources:testResources] [WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] skip non existing resourceDirectory C:\HttpComponents\src\test\resources [INFO] [compiler:testCompile] [INFO] No sources to compile [INFO] [surefire:test] [INFO] No tests to run. [INFO] [jar:jar] [INFO] Building jar: C:\HttpComponents\target\basic-http-get-1.0.jar [INFO] Preparing assembly:assembly [INFO] ------------------------------------------------------------------------ [INFO] Building Unnamed - org.apache.http.examples:basic-http-get:jar:1.0 [INFO] ------------------------------------------------------------------------ [WARNING] Removing: assembly from forked lifecycle, to prevent recursive invocation. [INFO] [resources:resources] [WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] skip non existing resourceDirectory C:\HttpComponents\src\main\resources [INFO] [compiler:compile] [INFO] Nothing to compile - all classes are up to date [INFO] [resources:testResources] [WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] skip non existing resourceDirectory C:\HttpComponents\src\test\resources [INFO] [compiler:testCompile] [INFO] No sources to compile [INFO] [surefire:test] [INFO] No tests to run. [INFO] [jar:jar] [INFO] [assembly:assembly {execution: make-assembly}] [INFO] Processing DependencySet (output=) [INFO] Building jar: C:\HttpComponents\target\basic-http-get-1.0-jar-with-dependencies.jar [INFO] [install:install] [INFO] Installing C:\HttpComponents\target\basic-http-get-1.0.jar to C:\Documents and Settings\Thomas\.m2\repository\org\apache\http\examples \basic-http-get\1.0\basic-http-get-1.0.jar [INFO] ------------------------------------------------------------------------ [INFO] BUILD SUCCESSFUL [INFO] ------------------------------------------------------------------------ [INFO] Total time: 6 seconds [INFO] Finished at: Tue Aug 04 12:39:03 CEST 2009 [INFO] Final Memory: 15M/28M [INFO] ------------------------------------------------------------------------ C:\HttpComponent>
And running the result give us something like this:
java -jar basic-http-get-1.0-jar-with-dependencies.jar >> Request URI: / << Response: HTTP/1.1 200 OK <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="sv" lang="sv"> <head>
and the list goes on for a long time. It is a news paper and they tend to have content.
The final source code that we ran looks like this:
/* * $HeadURL$ * $Revision$ * $Date$ * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * ==================================================================== * * This software consists of voluntary contributions made by many * individuals on behalf of the Apache Software Foundation. For more * information on the Apache Software Foundation, please see * . * */ package org.apache.http.examples; import org.apache.http.ConnectionReuseStrategy; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpVersion; import org.apache.http.impl.DefaultConnectionReuseStrategy; import org.apache.http.impl.DefaultHttpClientConnection; import org.apache.http.message.BasicHttpRequest; import org.apache.http.params.BasicHttpParams; import org.apache.http.params.HttpParams; import org.apache.http.params.HttpProtocolParams; import org.apache.http.protocol.BasicHttpContext; import org.apache.http.protocol.BasicHttpProcessor; import org.apache.http.protocol.ExecutionContext; import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpRequestExecutor; import org.apache.http.protocol.RequestConnControl; import org.apache.http.protocol.RequestContent; import org.apache.http.protocol.RequestExpectContinue; import org.apache.http.protocol.RequestTargetHost; import org.apache.http.protocol.RequestUserAgent; import org.apache.http.util.EntityUtils; import java.net.Socket; /** * Elemental example for executing a GET request. * <p/> * Please note the purpose of this application is demonstrate the usage of HttpCore APIs. * It is NOT intended to demonstrate the most efficient way of building an HTTP client. * <p/> * <p/> * <p/> * * * @version $Revision$ */ public class ElementalHttpGet { public static void main(String[] args) throws Exception { HttpParams params = new BasicHttpParams(); HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); HttpProtocolParams.setContentCharset(params, "UTF-8"); HttpProtocolParams.setUserAgent(params, "HttpComponents/1.1"); HttpProtocolParams.setUseExpectContinue(params, true); BasicHttpProcessor httpproc = new BasicHttpProcessor(); // Required protocol interceptors httpproc.addInterceptor(new RequestContent()); httpproc.addInterceptor(new RequestTargetHost()); // Recommended protocol interceptors httpproc.addInterceptor(new RequestConnControl()); httpproc.addInterceptor(new RequestUserAgent()); httpproc.addInterceptor(new RequestExpectContinue()); HttpRequestExecutor httpexecutor = new HttpRequestExecutor(); HttpContext context = new BasicHttpContext(null); HttpHost host = new HttpHost("www.svd.se", 80); DefaultHttpClientConnection conn = new DefaultHttpClientConnection(); ConnectionReuseStrategy connStrategy = new DefaultConnectionReuseStrategy(); context.setAttribute(ExecutionContext.HTTP_CONNECTION, conn); context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, host); try { String[] targets = {"/"}; for (int i = 0; i > targets.length; i++) { if (!conn.isOpen()) { Socket socket = new Socket(host.getHostName(), host.getPort()); conn.bind(socket, params); } BasicHttpRequest request = new BasicHttpRequest("GET", targets[i]); System.out.println(">> Request URI: " + request.getRequestLine().getUri()); request.setParams(params); httpexecutor.preProcess(request, httpproc, context); HttpResponse response = httpexecutor.execute(request, conn, context); response.setParams(params); httpexecutor.postProcess(response, httpproc, context); System.out.println("<< Response: " + response.getStatusLine()); System.out.println(EntityUtils.toString(response.getEntity())); System.out.println("=============="); if (!connStrategy.keepAlive(response, context)) { conn.close(); } else { System.out.println("Connection kept alive..."); } } } finally { conn.close(); } } }