com.cometway.net
Class HTTPLoader

java.lang.Object
  com.cometway.net.HTTPLoader

Direct Known Subclasses:

SecureHTTPLoader

public class HTTPLoader
extends java.lang.Object

This class provides a access to the HTTP protocol at a lower level than the URLConnection class provided by java. This class allows the caller to configure every aspect of an HTTP transaction.

There is also a fully functional static main method provided for using the HTTPLoader on the command line. The list of available options can be listed by giving no arguments or using the --help option.

Field Summary

java.lang.String	acceptString
static java.lang.String	ALL_FRAMES
protected boolean	allowForeignCookies
protected boolean	autoRedirect
java.lang.String	cert1_file
java.lang.String	contentTypeString
protected java.util.Vector	cookieList
protected boolean	debug
protected java.lang.String	debugStr
static java.lang.String	DEFAULT_DEBUG_STR
static java.lang.String	DEFAULT_ERROR_STR
static java.lang.String	DEFAULT_PRINT_STR
static int	DEFAULT_RECURSION_LIMIT
static int	DEFAULT_TIMEOUT
static java.lang.String	DEFAULT_USER_AGENT_STR
protected java.lang.String	errorStr
protected IExceptionHandler	exceptionHandler
java.lang.String	httpsProxyServer
int	httpsProxyServerPort
java.lang.String	key_file
protected java.lang.String	loadFrames
static java.lang.String	META_HTTP_REGEXP
protected java.lang.String	printStr
java.lang.String	proxyServer
int	proxyServerPort
protected int	recursionCount
protected int	recursionLimit
java.lang.String	referer
protected HTTPRequest	request
protected int	requestDataLength
protected java.lang.String	requestHost
protected long	requestMillis
protected java.lang.String	requestPath
protected int	requestPort
protected java.lang.String	requestProtocol
protected java.io.InputStream	requestReader
protected java.net.Socket	requestSocket
protected int	requestTimeout
protected java.lang.String	requestURL
protected java.io.BufferedWriter	requestWriter
protected HTTPResponse	response
boolean	useAccept
boolean	useCarraigeReturnsInRequest
boolean	useContentType
boolean	useHost
protected java.lang.String	userAgent
boolean	useReferer
boolean	useUserAgent
protected boolean	verbose

Constructor Summary

HTTPLoader()

Method Summary

void	addCookie(CookiePropsContainer c) * Adds a cookie to the cookie list.
void	addCookie(java.lang.String cookieString) * Adds a cookie to the cookie list.
boolean	closeConnection() * Closes the connection if it is open.
protected void	debug(java.lang.String s) * Prints a debugging message to System.out when debugging is turned on.
protected void	error(java.lang.String s) * Prints an error message to System.err
protected void	exception(java.lang.String s, java.lang.Exception e) * Passes the exception to the IExceptionHandler set for this object, or prints the exception to System.err if no IExceptionHandler is set.
java.util.Vector	getCookieList() * Returns the vector used to store cookies as CookiePropsContainers.
protected java.lang.String	getCookieString() * Returns a cookie string to be added to the list of headers.
int	getDefaultPort(java.lang.String protocol) * Returns the default port number of the specified protocol; -1 if protocol is unknown.
Props	getHeaders(java.lang.String url) Sends a HEAD request
static java.lang.String	getHTTPParamString(Props p) Creates a parameter list suitable for GET and POST style requests.
static java.lang.String	getHTTPParamString(java.util.Vector fields, java.util.Vector values) Creates a parameter list suitable for GET and POST style requests.
java.lang.String	getRedirectLocation() Returns the redirect location if the result code starts with 3
java.lang.String	getRequestURL()
java.util.Hashtable	getResponseHeader() Returns the response headers
java.lang.String	getResultCode() This method returns the HTTP result code from the last HTTP request.
java.lang.String	getURL(java.lang.String url)
java.lang.String	getURL(java.lang.String url, java.io.OutputStream dataOut)
java.lang.String	getURL(java.lang.String url, Props params) Sends a GET request; HTTP parameters contained in a Props.
java.lang.String	getURL(java.lang.String url, Props params, java.io.OutputStream dataOut)
java.lang.String	getURL(java.lang.String url, java.lang.String referer) Sends a GET request; no HTTP parameters.
java.lang.String	getURL(java.lang.String url, java.util.Vector fields, java.util.Vector values) Sends a GET request; HTTP parameters contained in Vectors.
java.lang.String	getURL(java.lang.String url, java.util.Vector fields, java.util.Vector values, java.io.OutputStream dataOut)
protected java.lang.String	loadFrames(java.lang.String page) * Loads pages from framesets embedded in the results and returns them inline.
static void	main(java.lang.String[] args)
protected boolean	openConnection() * Opens a request connection.
protected void	openSocket() * Opens a socket connection.
java.lang.String	postURL(java.lang.String url) Sends a POST request; no HTTP parameters.
java.lang.String	postURL(java.lang.String url, java.io.File postFile, java.lang.String mimeType)
java.lang.String	postURL(java.lang.String url, java.io.File postFile, java.lang.String mimeType, java.io.OutputStream dataOut)
java.lang.String	postURL(java.lang.String url, java.io.OutputStream dataOut)
java.lang.String	postURL(java.lang.String url, Props params) Sends a POST request; HTTP parameters contained in a Props.
java.lang.String	postURL(java.lang.String url, Props params, java.io.OutputStream dataOut)
java.lang.String	postURL(java.lang.String url, java.lang.String httpParamString) Sends a POST request; HTTP parameters already encoded as String.
java.lang.String	postURL(java.lang.String url, java.lang.String httpParamString, java.io.OutputStream dataOut)
java.lang.String	postURL(java.lang.String url, java.util.Vector fields, java.util.Vector values) Sends a POST request; HTTP parameters contained in Vectors.
java.lang.String	postURL(java.lang.String url, java.util.Vector fields, java.util.Vector values, java.io.OutputStream dataOut)
protected void	prepareRequest(java.lang.String url) * Prepares this object for a connection by parsing the target URL.
protected void	println(java.lang.String s) * Prints a message to System.out
boolean	putURL(java.lang.String url, byte[] data, java.lang.String mimeType)
boolean	putURL(java.lang.String url, java.io.File file, java.lang.String mimeType)
boolean	putURL(java.lang.String url, java.lang.String data)
protected byte[]	readBinaryFile(java.io.File file)
void	removeAuthentication() This method turns removes the username and password authentication, subsequent requests will not use any authentication
protected java.lang.String	requestURL(java.lang.String url, int requestType, java.lang.String httpParamString, java.io.InputStream dataIn, java.io.OutputStream dataOut, java.lang.String referer) Used to make the request.
void	resetCookieList() * Resets the cookie list by creating a new empty Vector; this replaces the previous cookie list if present.
protected java.lang.String	sendRequest(java.io.OutputStream dataOut) * Uses the open connection to make a request.
void	setAllowForeignCookies(boolean allow) * Cookies are accepted anyway when their domain and path do not match the request when this option is turned on; otherwise they are ignored.
void	setAutoRedirect(boolean state) * Sets the state of the auto-redirect feature.
void	setCookieList(java.util.Vector cookieList) * Sets a vector for storing a list of CookiePropsContainers.
void	setCookiesSupported(boolean supported) * Enables cookie support by creating a new empty vector for storing cookies; disabling removes any existing cookie list.
void	setDebug(boolean state) * Prints additional debugging output to System.out when turned on.
void	setExceptionHandler(IExceptionHandler e) * Sets the exception handler for this object.
void	setHTTPKey(java.lang.String keyFile, java.lang.String certFile) Sets the key and cert file to use with HTTPS requests.
void	setHTTPSProxyServer(java.lang.String proxyHostName, int port) This method sets the HTTPLoader to use a proxy server for HTTPS requests.
void	setLoadFrames(java.lang.String frames) * Sets the names of the frames that will be automatically loaded; set to ALL_NAMES (load all frames) by default.
void	setProxyServer(java.lang.String proxyHostName, int port) This method sets the HTTPLoader to use a proxy server.
void	setRecursionLimit(int limit) Sets the recursion limit.
void	setRequestTimeout(int timeout) * Sets the request timeout used when opening a connection.
void	setUserAgent(java.lang.String s) * Sets the string passed to web servers to identify a client.
void	setVerbose(boolean enabled)
java.lang.String	shortGetURL(java.lang.String url) Uses shorter GET header for simple requests.
java.lang.String	shortGetURL(java.lang.String url, java.io.OutputStream dataOut)
void	useBasicAuthentication(java.lang.String username, java.lang.String password) This method forces the HTTPLoader to use BASIC authentication

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

DEFAULT_USER_AGENT_STR

public static final java.lang.String DEFAULT_USER_AGENT_STR

See Also:

Constant Field Values

META_HTTP_REGEXP

public static final java.lang.String META_HTTP_REGEXP

See Also:

Constant Field Values

DEFAULT_ERROR_STR

public static final java.lang.String DEFAULT_ERROR_STR

See Also:

Constant Field Values

DEFAULT_PRINT_STR

public static final java.lang.String DEFAULT_PRINT_STR

See Also:

Constant Field Values

DEFAULT_DEBUG_STR

public static final java.lang.String DEFAULT_DEBUG_STR

See Also:

Constant Field Values

DEFAULT_TIMEOUT

public static final int DEFAULT_TIMEOUT

See Also:

Constant Field Values

ALL_FRAMES

public static final java.lang.String ALL_FRAMES

See Also:

Constant Field Values

DEFAULT_RECURSION_LIMIT

public static final int DEFAULT_RECURSION_LIMIT

See Also:

Constant Field Values

autoRedirect

protected boolean autoRedirect

allowForeignCookies

protected boolean allowForeignCookies

debug

protected boolean debug

verbose

protected boolean verbose

recursionLimit

protected int recursionLimit

requestTimeout

protected int requestTimeout

loadFrames

protected java.lang.String loadFrames

debugStr

protected java.lang.String debugStr

errorStr

protected java.lang.String errorStr

printStr

protected java.lang.String printStr

userAgent

protected java.lang.String userAgent

exceptionHandler

protected IExceptionHandler exceptionHandler

proxyServer

public java.lang.String proxyServer

proxyServerPort

public int proxyServerPort

httpsProxyServer

public java.lang.String httpsProxyServer

httpsProxyServerPort

public int httpsProxyServerPort

cookieList

protected java.util.Vector cookieList

requestProtocol

protected java.lang.String requestProtocol

requestHost

protected java.lang.String requestHost

requestPort

protected int requestPort

requestPath

protected java.lang.String requestPath

requestURL

protected java.lang.String requestURL

requestSocket

protected java.net.Socket requestSocket

request

protected HTTPRequest request

requestMillis

protected long requestMillis

requestWriter

protected java.io.BufferedWriter requestWriter

requestReader

protected java.io.InputStream requestReader

response

protected HTTPResponse response

requestDataLength

protected int requestDataLength

recursionCount

protected int recursionCount

useReferer

public boolean useReferer

useUserAgent

public boolean useUserAgent

useHost

public boolean useHost

useAccept

public boolean useAccept

useContentType

public boolean useContentType

useCarraigeReturnsInRequest

public boolean useCarraigeReturnsInRequest

acceptString

public java.lang.String acceptString

contentTypeString

public java.lang.String contentTypeString

referer

public java.lang.String referer

key_file

public java.lang.String key_file

cert1_file

public java.lang.String cert1_file

Constructor Detail

HTTPLoader

public HTTPLoader()

Method Detail

closeConnection

public boolean closeConnection()

* Closes the connection if it is open.

getDefaultPort

public int getDefaultPort(java.lang.String protocol)

* Returns the default port number of the specified protocol; -1 if protocol is unknown.

getResultCode

public java.lang.String getResultCode()

This method returns the HTTP result code from the last HTTP request.

getResponseHeader

public java.util.Hashtable getResponseHeader()

Returns the response headers

getRedirectLocation

public java.lang.String getRedirectLocation()

Returns the redirect location if the result code starts with 3

setHTTPKey

public void setHTTPKey(java.lang.String keyFile,
                       java.lang.String certFile)

Sets the key and cert file to use with HTTPS requests.

setProxyServer

public void setProxyServer(java.lang.String proxyHostName,
                           int port)

This method sets the HTTPLoader to use a proxy server.

setHTTPSProxyServer

public void setHTTPSProxyServer(java.lang.String proxyHostName,
                                int port)

This method sets the HTTPLoader to use a proxy server for HTTPS requests.

useBasicAuthentication

public void useBasicAuthentication(java.lang.String username,
                                   java.lang.String password)

This method forces the HTTPLoader to use BASIC authentication

removeAuthentication

public void removeAuthentication()

This method turns removes the username and password authentication, subsequent requests will not use any authentication

setAutoRedirect

public void setAutoRedirect(boolean state)

* Sets the state of the auto-redirect feature.

setDebug

public void setDebug(boolean state)

* Prints additional debugging output to System.out when turned on.

setExceptionHandler

public void setExceptionHandler(IExceptionHandler e)

* Sets the exception handler for this object.

setLoadFrames

public void setLoadFrames(java.lang.String frames)

* Sets the names of the frames that will be automatically loaded; set to ALL_NAMES (load all frames) by default. Names are separated by whitespace. Setting to null disables this feature. Note: Only ALL_NAMES and null are currently implemented. any non-null value will be treated like ALL_NAMES.

setRecursionLimit

public void setRecursionLimit(int limit)

Sets the recursion limit. The default value is 5.

setRequestTimeout

public void setRequestTimeout(int timeout)

* Sets the request timeout used when opening a connection.

setUserAgent

public void setUserAgent(java.lang.String s)

* Sets the string passed to web servers to identify a client.

setVerbose

public void setVerbose(boolean enabled)

setAllowForeignCookies

public void setAllowForeignCookies(boolean allow)

* Cookies are accepted anyway when their domain and path do not match the request when this option is turned on; otherwise they are ignored.

setCookieList

public void setCookieList(java.util.Vector cookieList)

* Sets a vector for storing a list of CookiePropsContainers.

setCookiesSupported

public void setCookiesSupported(boolean supported)

* Enables cookie support by creating a new empty vector for storing cookies; disabling removes any existing cookie list.

getCookieString

protected java.lang.String getCookieString()

* Returns a cookie string to be added to the list of headers.

getCookieList

public java.util.Vector getCookieList()

* Returns the vector used to store cookies as CookiePropsContainers.

resetCookieList

public void resetCookieList()

* Resets the cookie list by creating a new empty Vector; this replaces the previous cookie list if present.

addCookie

public void addCookie(java.lang.String cookieString)

* Adds a cookie to the cookie list.

addCookie

public void addCookie(CookiePropsContainer c)

* Adds a cookie to the cookie list.

getURL

public java.lang.String getURL(java.lang.String url,
                               Props params)

Sends a GET request; HTTP parameters contained in a Props.

getURL

public java.lang.String getURL(java.lang.String url,
                               Props params,
                               java.io.OutputStream dataOut)

getURL

public java.lang.String getURL(java.lang.String url,
                               java.util.Vector fields,
                               java.util.Vector values)

Sends a GET request; HTTP parameters contained in Vectors.

getURL

public java.lang.String getURL(java.lang.String url,
                               java.util.Vector fields,
                               java.util.Vector values,
                               java.io.OutputStream dataOut)

getURL

public java.lang.String getURL(java.lang.String url,
                               java.lang.String referer)

Sends a GET request; no HTTP parameters.

getURL

public java.lang.String getURL(java.lang.String url)

getURL

public java.lang.String getURL(java.lang.String url,
                               java.io.OutputStream dataOut)

shortGetURL

public java.lang.String shortGetURL(java.lang.String url)

Uses shorter GET header for simple requests.

shortGetURL

public java.lang.String shortGetURL(java.lang.String url,
                                    java.io.OutputStream dataOut)

postURL

public java.lang.String postURL(java.lang.String url,
                                Props params)

Sends a POST request; HTTP parameters contained in a Props.

postURL

public java.lang.String postURL(java.lang.String url,
                                Props params,
                                java.io.OutputStream dataOut)

postURL

public java.lang.String postURL(java.lang.String url,
                                java.util.Vector fields,
                                java.util.Vector values)

Sends a POST request; HTTP parameters contained in Vectors.

postURL

public java.lang.String postURL(java.lang.String url,
                                java.util.Vector fields,
                                java.util.Vector values,
                                java.io.OutputStream dataOut)

postURL

public java.lang.String postURL(java.lang.String url,
                                java.lang.String httpParamString)

Sends a POST request; HTTP parameters already encoded as String.

postURL

public java.lang.String postURL(java.lang.String url,
                                java.lang.String httpParamString,
                                java.io.OutputStream dataOut)

postURL

public java.lang.String postURL(java.lang.String url)

Sends a POST request; no HTTP parameters.

postURL

public java.lang.String postURL(java.lang.String url,
                                java.io.OutputStream dataOut)

postURL

public java.lang.String postURL(java.lang.String url,
                                java.io.File postFile,
                                java.lang.String mimeType)

postURL

public java.lang.String postURL(java.lang.String url,
                                java.io.File postFile,
                                java.lang.String mimeType,
                                java.io.OutputStream dataOut)

getHeaders

public Props getHeaders(java.lang.String url)

Sends a HEAD request

putURL

public boolean putURL(java.lang.String url,
                      java.io.File file,
                      java.lang.String mimeType)

putURL

public boolean putURL(java.lang.String url,
                      byte[] data,
                      java.lang.String mimeType)

putURL

public boolean putURL(java.lang.String url,
                      java.lang.String data)

getHTTPParamString

public static java.lang.String getHTTPParamString(java.util.Vector fields,
                                                  java.util.Vector values)

Creates a parameter list suitable for GET and POST style requests.

getHTTPParamString

public static java.lang.String getHTTPParamString(Props p)

Creates a parameter list suitable for GET and POST style requests.

loadFrames

protected java.lang.String loadFrames(java.lang.String page)

* Loads pages from framesets embedded in the results and returns them inline.

openConnection

protected boolean openConnection()

* Opens a request connection.

openSocket

protected void openSocket()
                   throws java.lang.Exception

* Opens a socket connection.

Throws:

java.lang.Exception

prepareRequest

protected void prepareRequest(java.lang.String url)

* Prepares this object for a connection by parsing the target URL.

requestURL

protected java.lang.String requestURL(java.lang.String url,
                                      int requestType,
                                      java.lang.String httpParamString,
                                      java.io.InputStream dataIn,
                                      java.io.OutputStream dataOut,
                                      java.lang.String referer)

Used to make the request. requestType is defined in HTTPRequest. httpParamString can be null.

sendRequest

protected java.lang.String sendRequest(java.io.OutputStream dataOut)

* Uses the open connection to make a request.

getRequestURL

public java.lang.String getRequestURL()

readBinaryFile

protected byte[] readBinaryFile(java.io.File file)

println

protected void println(java.lang.String s)

* Prints a message to System.out

debug

protected void debug(java.lang.String s)

* Prints a debugging message to System.out when debugging is turned on.

error

protected void error(java.lang.String s)

* Prints an error message to System.err

exception

protected void exception(java.lang.String s,
                         java.lang.Exception e)

* Passes the exception to the IExceptionHandler set for this object, or prints the exception to System.err if no IExceptionHandler is set.

main

public static void main(java.lang.String[] args)