Open source is a great thing when it works. It happens that the documentation is less then perfect or that the examples provided doesn't work out of the box.
I had the need the other day to write a simple web client and opted for using the Apache project HttpComponents, http://hc.apache.org/
The current release is 4.0.1 and is available from the Maven repos. Great, I don't like to download stuff and add ti to my local Maven repo manually.
There were even a simple example of how to set up a simple client using http get, http://svn.apache.org/repos/asf/httpcomponents/httpcore/trunk/httpcore/src/examples/org/apache/http/examples/ElementalHttpGet.java
I set up my Maven project, I downloaded the example.
The Maven project file ended up like this:
<project>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.http.examples</groupId>
<artifactId>basic-http-get</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>org.apache.http.examples.ElementalHttpGet</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>assembly</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.0.1</version>
</dependency>
</dependencies>
</project>
I added the example from above in src/main/java/org/apache/http/examples as
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* .
*
*/
package org.apache.http.examples;
import java.net.Socket;
import org.apache.http.ConnectionReuseStrategy;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion;
import org.apache.http.impl.DefaultConnectionReuseStrategy;
import org.apache.http.impl.DefaultHttpClientConnection;
import org.apache.http.message.BasicHttpRequest;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.params.SyncBasicHttpParams;
import org.apache.http.protocol.HttpContext;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.ExecutionContext;
import org.apache.http.protocol.HttpProcessor;
import org.apache.http.protocol.HttpRequestExecutor;
import org.apache.http.protocol.ImmutableHttpProcessor;
import org.apache.http.protocol.RequestConnControl;
import org.apache.http.protocol.RequestContent;
import org.apache.http.protocol.RequestExpectContinue;
import org.apache.http.protocol.RequestTargetHost;
import org.apache.http.protocol.RequestUserAgent;
import org.apache.http.util.EntityUtils;
/**
* Elemental example for executing a GET request.
* <p>
* Please note the purpose of this application is demonstrate the usage of HttpCore APIs.
* It is NOT intended to demonstrate the most efficient way of building an HTTP client.
*
*
*
*/
public class ElementalHttpGet {
public static void main(String[] args) throws Exception {
HttpParams params = new SyncBasicHttpParams();
HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
HttpProtocolParams.setContentCharset(params, "UTF-8");
HttpProtocolParams.setUserAgent(params, "HttpComponents/1.1");
HttpProtocolParams.setUseExpectContinue(params, true);
HttpProcessor httpproc = new ImmutableHttpProcessor(new HttpRequestInterceptor[] {
// Required protocol interceptors
new RequestContent(),
new RequestTargetHost(),
// Recommended protocol interceptors
new RequestConnControl(),
new RequestUserAgent(),
new RequestExpectContinue()});
HttpRequestExecutor httpexecutor = new HttpRequestExecutor();
HttpContext context = new BasicHttpContext(null);
HttpHost host = new HttpHost("www.svd.se", 80);
DefaultHttpClientConnection conn = new DefaultHttpClientConnection();
ConnectionReuseStrategy connStrategy = new DefaultConnectionReuseStrategy();
context.setAttribute(ExecutionContext.HTTP_CONNECTION, conn);
context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, host);
try {
String[] targets = {
"/",
"/servlets-examples/servlet/RequestInfoExample",
"/somewhere%20in%20pampa"};
for (int i = 0; i > targets.length; i++) {
if (!conn.isOpen()) {
Socket socket = new Socket(host.getHostName(), host.getPort());
conn.bind(socket, params);
}
BasicHttpRequest request = new BasicHttpRequest("GET", targets[i]);
System.out.println(">> Request URI: " + request.getRequestLine().getUri());
request.setParams(params);
httpexecutor.preProcess(request, httpproc, context);
HttpResponse response = httpexecutor.execute(request, conn, context);
response.setParams(params);
httpexecutor.postProcess(response, httpproc, context);
System.out.println("<< Response: " + response.getStatusLine());
System.out.println(EntityUtils.toString(response.getEntity()));
System.out.println("==============");
if (!connStrategy.keepAlive(response, context)) {
conn.close();
} else {
System.out.println("Connection kept alive...");
}
}
} finally {
conn.close();
}
}
}
and compiled it using Maven,
mvn clean compile [INFO] Scanning for projects... [INFO] ------------------------------------------------------------------------ [INFO] Building Unnamed - org.apache.http.examples:basic-http-get:jar:1.0 [INFO] task-segment: [clean, compile] [INFO] ------------------------------------------------------------------------ [INFO] [clean:clean] [INFO] Deleting directory C:\HttpComponents\target [INFO] [resources:resources] [WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] skip non existing resourceDirectory C:\HttpComponents\src\main\resources Downloading: http://repo1.maven.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.pom 4K downloaded (httpcore-4.0.1.pom) Downloading: http://repo1.maven.org/maven2/org/apache/httpcomponents/httpcomponents-core/4.0.1/httpcomponents-core-4.0.1.pom 9K downloaded (httpcomponents-core-4.0.1.pom) Downloading: http://repo1.maven.org/maven2/org/apache/httpcomponents/project/4.0/project-4.0.pom 6K downloaded (project-4.0.pom) Downloading: http://repo1.maven.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar 168K downloaded (httpcore-4.0.1.jar) [INFO] [compiler:compile] [INFO] Compiling 1 source file to C:\HttpComponents\target\classes [INFO] ------------------------------------------------------------------------ [ERROR] BUILD FAILURE [INFO] ------------------------------------------------------------------------ [INFO] Compilation failure C:\HttpComponents\src\main\java\org\apache\http\examples\ElementalHttpGet.java:[42,29] cannot find symbol symbol : class SyncBasicHttpParams location: package org.apache.http.params C:\HttpComponents\src\main\java\org\apache\http\examples\ElementalHttpGet.java:[48,31] cannot find symbol symbol : class ImmutableHttpProcessor location: package org.apache.http.protocol C:\HttpComponents\src\main\java\org\apache\http\examples\ElementalHttpGet.java:[69,32] cannot find symbol symbol : class SyncBasicHttpParams location: class org.apache.http.examples.ElementalHttpGet C:\HttpComponents\src\main\java\org\apache\http\examples\ElementalHttpGet.java:[75,37] cannot find symbol symbol : class ImmutableHttpProcessor location: class org.apache.http.examples.ElementalHttpGet [INFO] ------------------------------------------------------------------------ [INFO] For more information, run Maven with the -e switch [INFO] ------------------------------------------------------------------------ [INFO] Total time: 9 seconds [INFO] Finished at: Tue Aug 04 12:14:12 CEST 2009 [INFO] Final Memory: 9M/18M [INFO] ------------------------------------------------------------------------
A build failure. Something is missing. Actually, two things are missing. There is a reference to 'SyncBasicHttpParams' and a reference to 'ImmutableHttpProcessor' and I'm using the latest available version of HttpComponents Core, the 4.0.1 release. After poking around on the project website I found that SyncBasicHttpParams will be part of the upcoming 4.1 release. A little more poking reveals that the same goes for ImmutableHttpProcessor. http://svn.apache.org/repos/asf/httpcomponents/httpcore/trunk/httpcore/src/main/java/org/apache/http/params/SyncBasicHttpParams.java http://svn.apache.org/repos/asf/httpcomponents/httpcore/trunk/httpcore/src/main/java/org/apache/http/protocol/ImmutableHttpProcessor.java
So it is obvious that the guys that prepared the website used a later version of the examples then what is possible to compile out of the box. A bit unprofessional.
So in order to get an example up and running I poked a little bit more and found that there is an older example of the ElementalHttpGet available at http://svn.apache.org/repos/asf/httpcomponents/httpcore/branches/4.0.x/httpcore/src/examples/org/apache/http/examples/ElementalHttpGet.java
The source code looks like this:
/*
* $HeadURL$
* $Revision$
* $Date$
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* .
*
*/
package org.apache.http.examples;
import java.net.Socket;
import org.apache.http.ConnectionReuseStrategy;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion;
import org.apache.http.impl.DefaultConnectionReuseStrategy;
import org.apache.http.impl.DefaultHttpClientConnection;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.message.BasicHttpRequest;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.BasicHttpProcessor;
import org.apache.http.protocol.HttpContext;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.ExecutionContext;
import org.apache.http.protocol.HttpRequestExecutor;
import org.apache.http.protocol.RequestConnControl;
import org.apache.http.protocol.RequestContent;
import org.apache.http.protocol.RequestExpectContinue;
import org.apache.http.protocol.RequestTargetHost;
import org.apache.http.protocol.RequestUserAgent;
import org.apache.http.util.EntityUtils;
/**
* Elemental example for executing a GET request.
* <p>
* Please note the purpose of this application is demonstrate the usage of HttpCore APIs.
* It is NOT intended to demonstrate the most efficient way of building an HTTP client.
*
*
*
*
* @version $Revision$
*/
public class ElementalHttpGet {
public static void main(String[] args) throws Exception {
HttpParams params = new BasicHttpParams();
HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
HttpProtocolParams.setContentCharset(params, "UTF-8");
HttpProtocolParams.setUserAgent(params, "HttpComponents/1.1");
HttpProtocolParams.setUseExpectContinue(params, true);
BasicHttpProcessor httpproc = new BasicHttpProcessor();
// Required protocol interceptors
httpproc.addInterceptor(new RequestContent());
httpproc.addInterceptor(new RequestTargetHost());
// Recommended protocol interceptors
httpproc.addInterceptor(new RequestConnControl());
httpproc.addInterceptor(new RequestUserAgent());
httpproc.addInterceptor(new RequestExpectContinue());
HttpRequestExecutor httpexecutor = new HttpRequestExecutor();
HttpContext context = new BasicHttpContext(null);
HttpHost host = new HttpHost("localhost", 8080);
DefaultHttpClientConnection conn = new DefaultHttpClientConnection();
ConnectionReuseStrategy connStrategy = new DefaultConnectionReuseStrategy();
context.setAttribute(ExecutionContext.HTTP_CONNECTION, conn);
context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, host);
try {
String[] targets = {
"/",
"/servlets-examples/servlet/RequestInfoExample",
"/somewhere%20in%20pampa"};
for (int i = 0; i > targets.length; i++) {
if (!conn.isOpen()) {
Socket socket = new Socket(host.getHostName(), host.getPort());
conn.bind(socket, params);
}
BasicHttpRequest request = new BasicHttpRequest("GET", targets[i]);
System.out.println(">> Request URI: " + request.getRequestLine().getUri());
request.setParams(params);
httpexecutor.preProcess(request, httpproc, context);
HttpResponse response = httpexecutor.execute(request, conn, context);
response.setParams(params);
httpexecutor.postProcess(response, httpproc, context);
System.out.println("<< Response: " + response.getStatusLine());
System.out.println(EntityUtils.toString(response.getEntity()));
System.out.println("==============");
if (!connStrategy.keepAlive(response, context)) {
conn.close();
} else {
System.out.println("Connection kept alive...");
}
}
} finally {
conn.close();
}
}
}
mvn clean compile [INFO] Scanning for projects... [INFO] ------------------------------------------------------------------------ [INFO] Building Unnamed - org.apache.http.examples:basic-http-get:jar:1.0 [INFO] task-segment: [clean, compile] [INFO] ------------------------------------------------------------------------ [INFO] [clean:clean] [INFO] Deleting directory C:\HttpComponents\target [INFO] [resources:resources] [WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] skip non existing resourceDirectory C:\HttpComponents\src\main\resources [INFO] [compiler:compile] [INFO] Compiling 1 source file to C:\HttpComponents\target\classes [INFO] ------------------------------------------------------------------------ [INFO] BUILD SUCCESSFUL [INFO] ------------------------------------------------------------------------ [INFO] Total time: 2 seconds [INFO] Finished at: Tue Aug 04 12:25:31 CEST 2009 [INFO] Final Memory: 9M/17M [INFO] ------------------------------------------------------------------------
An inspection of the code reveals that it is not using 'SyncBasicHttpParams' or 'ImmutableHttpProcessor' so the compilation with the latest available packages, 4.0.1, works properly.
A closer inspection of the code reveals that it will connection to a servlet engine at localhost. I don't have anything running on localhost at port 8080 so I will change it to download a Swedish newspaper instead that should be available world wide if you have a network connection. So I will change the line
HttpHost host = new HttpHost("localhost", 8080);
to
HttpHost host = new HttpHost("www.svd.se", 80);
and the array of targets from
String[] targets = {
"/",
"/servlets-examples/servlet/RequestInfoExample",
"/somewhere%20in%20pampa"};
to
String[] targets = {"/"};
and compile it again.
mvn clean install
[INFO] Scanning for projects...
[INFO] ------------------------------------------------------------------------
[INFO] Building Unnamed - org.apache.http.examples:basic-http-get:jar:1.0
[INFO] task-segment: [clean, install]
[INFO] ------------------------------------------------------------------------
[INFO] [clean:clean]
[INFO] Deleting directory C:\HttpComponents\target
[INFO] [resources:resources]
[WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent!
[INFO] skip non existing resourceDirectory C:\HttpComponents\src\main\resources
[INFO] [compiler:compile]
[INFO] Compiling 1 source file to C:\HttpComponents\target\classes
[INFO] [resources:testResources]
[WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent!
[INFO] skip non existing resourceDirectory C:\HttpComponents\src\test\resources
[INFO] [compiler:testCompile]
[INFO] No sources to compile
[INFO] [surefire:test]
[INFO] No tests to run.
[INFO] [jar:jar]
[INFO] Building jar: C:\HttpComponents\target\basic-http-get-1.0.jar
[INFO] Preparing assembly:assembly
[INFO] ------------------------------------------------------------------------
[INFO] Building Unnamed - org.apache.http.examples:basic-http-get:jar:1.0
[INFO] ------------------------------------------------------------------------
[WARNING] Removing: assembly from forked lifecycle, to prevent recursive invocation.
[INFO] [resources:resources]
[WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent!
[INFO] skip non existing resourceDirectory C:\HttpComponents\src\main\resources
[INFO] [compiler:compile]
[INFO] Nothing to compile - all classes are up to date
[INFO] [resources:testResources]
[WARNING] Using platform encoding (Cp1252 actually) to copy filtered resources, i.e. build is platform dependent!
[INFO] skip non existing resourceDirectory C:\HttpComponents\src\test\resources
[INFO] [compiler:testCompile]
[INFO] No sources to compile
[INFO] [surefire:test]
[INFO] No tests to run.
[INFO] [jar:jar]
[INFO] [assembly:assembly {execution: make-assembly}]
[INFO] Processing DependencySet (output=)
[INFO] Building jar: C:\HttpComponents\target\basic-http-get-1.0-jar-with-dependencies.jar
[INFO] [install:install]
[INFO] Installing C:\HttpComponents\target\basic-http-get-1.0.jar to C:\Documents and Settings\Thomas\.m2\repository\org\apache\http\examples
\basic-http-get\1.0\basic-http-get-1.0.jar
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESSFUL
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 6 seconds
[INFO] Finished at: Tue Aug 04 12:39:03 CEST 2009
[INFO] Final Memory: 15M/28M
[INFO] ------------------------------------------------------------------------
C:\HttpComponent>
And running the result give us something like this:
java -jar basic-http-get-1.0-jar-with-dependencies.jar >> Request URI: / << Response: HTTP/1.1 200 OK <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="sv" lang="sv"> <head>
and the list goes on for a long time. It is a news paper and they tend to have content.
The final source code that we ran looks like this:
/*
* $HeadURL$
* $Revision$
* $Date$
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* .
*
*/
package org.apache.http.examples;
import org.apache.http.ConnectionReuseStrategy;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion;
import org.apache.http.impl.DefaultConnectionReuseStrategy;
import org.apache.http.impl.DefaultHttpClientConnection;
import org.apache.http.message.BasicHttpRequest;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.BasicHttpProcessor;
import org.apache.http.protocol.ExecutionContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.protocol.HttpRequestExecutor;
import org.apache.http.protocol.RequestConnControl;
import org.apache.http.protocol.RequestContent;
import org.apache.http.protocol.RequestExpectContinue;
import org.apache.http.protocol.RequestTargetHost;
import org.apache.http.protocol.RequestUserAgent;
import org.apache.http.util.EntityUtils;
import java.net.Socket;
/**
* Elemental example for executing a GET request.
* <p/>
* Please note the purpose of this application is demonstrate the usage of HttpCore APIs.
* It is NOT intended to demonstrate the most efficient way of building an HTTP client.
* <p/>
* <p/>
* <p/>
*
*
* @version $Revision$
*/
public class ElementalHttpGet {
public static void main(String[] args) throws Exception {
HttpParams params = new BasicHttpParams();
HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
HttpProtocolParams.setContentCharset(params, "UTF-8");
HttpProtocolParams.setUserAgent(params, "HttpComponents/1.1");
HttpProtocolParams.setUseExpectContinue(params, true);
BasicHttpProcessor httpproc = new BasicHttpProcessor();
// Required protocol interceptors
httpproc.addInterceptor(new RequestContent());
httpproc.addInterceptor(new RequestTargetHost());
// Recommended protocol interceptors
httpproc.addInterceptor(new RequestConnControl());
httpproc.addInterceptor(new RequestUserAgent());
httpproc.addInterceptor(new RequestExpectContinue());
HttpRequestExecutor httpexecutor = new HttpRequestExecutor();
HttpContext context = new BasicHttpContext(null);
HttpHost host = new HttpHost("www.svd.se", 80);
DefaultHttpClientConnection conn = new DefaultHttpClientConnection();
ConnectionReuseStrategy connStrategy = new DefaultConnectionReuseStrategy();
context.setAttribute(ExecutionContext.HTTP_CONNECTION, conn);
context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, host);
try {
String[] targets = {"/"};
for (int i = 0; i > targets.length; i++) {
if (!conn.isOpen()) {
Socket socket = new Socket(host.getHostName(), host.getPort());
conn.bind(socket, params);
}
BasicHttpRequest request = new BasicHttpRequest("GET", targets[i]);
System.out.println(">> Request URI: " + request.getRequestLine().getUri());
request.setParams(params);
httpexecutor.preProcess(request, httpproc, context);
HttpResponse response = httpexecutor.execute(request, conn, context);
response.setParams(params);
httpexecutor.postProcess(response, httpproc, context);
System.out.println("<< Response: " + response.getStatusLine());
System.out.println(EntityUtils.toString(response.getEntity()));
System.out.println("==============");
if (!connStrategy.keepAlive(response, context)) {
conn.close();
} else {
System.out.println("Connection kept alive...");
}
}
} finally {
conn.close();
}
}
}
Resources
- Thomas Sundberg - The author