com.sun.syndication.fetcher.impl.HttpClientFeedFetcher Maven / Gradle / Ivy

Go to download
/*
 * Copyright 2004 Sun Microsystems, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package com.sun.syndication.fetcher.impl;

import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.zip.GZIPInputStream;

import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpClientParams;

import com.sun.syndication.feed.synd.SyndFeed;
import com.sun.syndication.fetcher.FetcherEvent;
import com.sun.syndication.fetcher.FetcherException;
import com.sun.syndication.io.FeedException;
import com.sun.syndication.io.SyndFeedInput;
import com.sun.syndication.io.XmlReader;

/**
 * @author Nick Lothian
 */
public class HttpClientFeedFetcher extends AbstractFeedFetcher {

	private FeedFetcherCache feedInfoCache;
    private CredentialSupplier credentialSupplier;
    private volatile HttpClientMethodCallbackIntf httpClientMethodCallback;
	private volatile HttpClientParams httpClientParams;

	public HttpClientFeedFetcher() {
		super();
		setHttpClientParams(new HttpClientParams());
	}
	
	/**
	 * @param cache
	 */
	public HttpClientFeedFetcher(FeedFetcherCache cache) {
		this();
		setFeedInfoCache(cache);
	}

	
	public HttpClientFeedFetcher(FeedFetcherCache cache, CredentialSupplier credentialSupplier) {
	    this(cache);
	    setCredentialSupplier(credentialSupplier);
	}


	/**
     * @return Returns the httpClientParams.
     */
    public synchronized HttpClientParams getHttpClientParams() {
        return this.httpClientParams;
    }
    /**
     * @param httpClientParams The httpClientParams to set.
     */
    public synchronized void setHttpClientParams(HttpClientParams httpClientParams) {
        this.httpClientParams = httpClientParams;
    }	

    /**
     * @param timeout Sets the connect timeout for the HttpClient but using the URLConnection method name.
	 * 		  Uses the HttpClientParams method setConnectionManagerTimeout instead of setConnectTimeout
	 * 
	 */
	public synchronized void setConnectTimeout(int timeout) {
		httpClientParams.setConnectionManagerTimeout(timeout);
	}

	/**
	 * @return The currently used connect timeout for the HttpClient but using the URLConnection method name.
	 * 		   Uses the HttpClientParams method getConnectionManagerTimeout instead of getConnectTimeout
	 * 
	 */
	public int getConnectTimeout() {
		return (int) this.getHttpClientParams().getConnectionManagerTimeout();
	}	

	/**
	 * @return The currently used read timeout for the URLConnection, 0 is unlimited, i.e. no timeout 
	 */
	public synchronized void setReadTimeout(int timeout) {
		httpClientParams.setSoTimeout(timeout);
	}

	/**
	 * @param timeout Sets the read timeout for the URLConnection to a specified timeout, in milliseconds.
	 */
	public int getReadTimeout() {
		return (int) this.getHttpClientParams().getSoTimeout();
	}
	
	public HttpClientMethodCallbackIntf getHttpClientMethodCallback() {
		return httpClientMethodCallback;
	}

	public synchronized void setHttpClientMethodCallback(HttpClientMethodCallbackIntf httpClientMethodCallback) {
		this.httpClientMethodCallback = httpClientMethodCallback;
	}	
	
	/**
	 * @return the feedInfoCache.
	 */
	public synchronized FeedFetcherCache getFeedInfoCache() {
		return feedInfoCache;
	}
	
    /**
	 * @param feedInfoCache the feedInfoCache to set
	 */
	public synchronized void setFeedInfoCache(FeedFetcherCache feedInfoCache) {
		this.feedInfoCache = feedInfoCache;
	}

	/**
     * @return Returns the credentialSupplier.
     */
    public synchronized CredentialSupplier getCredentialSupplier() {
        return credentialSupplier;
    }
    /**
     * @param credentialSupplier The credentialSupplier to set.
     */
    public synchronized void setCredentialSupplier(CredentialSupplier credentialSupplier) {
        this.credentialSupplier = credentialSupplier;
    }	
	
	/**
	 * @see com.sun.syndication.fetcher.FeedFetcher#retrieveFeed(java.net.URL)
	 */
	public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
		if (feedUrl == null) {
			throw new IllegalArgumentException("null is not a valid URL");
		}
		// TODO Fix this
		//System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.SimpleLog");
		HttpClient client = new HttpClient(httpClientParams);
		
		if (getCredentialSupplier() != null) {
			client.getState().setAuthenticationPreemptive(true);
			// TODO what should realm be here?
			Credentials credentials = getCredentialSupplier().getCredentials(null, feedUrl.getHost()); 
			if (credentials != null) {
			    client.getState().setCredentials(null, feedUrl.getHost(), credentials);
			}			
		}
		
		
		System.setProperty("httpclient.useragent", getUserAgent());
		String urlStr = feedUrl.toString();
		
		HttpMethod method = new GetMethod(urlStr);
		method.addRequestHeader("Accept-Encoding", "gzip");
		method.addRequestHeader("User-Agent", getUserAgent());		
		method.setFollowRedirects(true);
		
		if (httpClientMethodCallback != null) {
			synchronized (httpClientMethodCallback) {			
				httpClientMethodCallback.afterHttpClientMethodCreate(method);
			}
		}
		
		FeedFetcherCache cache = getFeedInfoCache();
		if (cache != null) {
			// retrieve feed

			try {
				if (isUsingDeltaEncoding()) {
				    method.setRequestHeader("A-IM", "feed");
				}	    

				// get the feed info from the cache
			    // Note that syndFeedInfo will be null if it is not in the cache
				SyndFeedInfo syndFeedInfo = cache.getFeedInfo(feedUrl);			
			    if (syndFeedInfo != null) {
				    method.setRequestHeader("If-None-Match", syndFeedInfo.getETag());
				    
				    if (syndFeedInfo.getLastModified() instanceof String) {
				        method.setRequestHeader("If-Modified-Since", (String)syndFeedInfo.getLastModified());
				    }
			    }
				
				int statusCode = client.executeMethod(method);
				fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);				
				handleErrorCodes(statusCode);			    
			    			    
			    SyndFeed feed = getFeed(syndFeedInfo, urlStr, method, statusCode);
			    				    
				syndFeedInfo = buildSyndFeedInfo(feedUrl, urlStr, method, feed, statusCode);
				
				cache.setFeedInfo(new URL(urlStr), syndFeedInfo);	
				
				// the feed may have been modified to pick up cached values
				// (eg - for delta encoding)
				feed = syndFeedInfo.getSyndFeed();
	
				return feed;
			} finally {
				method.releaseConnection();
				method.recycle();
			}
				
		} else {
		    // cache is not in use		    
			try {
				int statusCode = client.executeMethod(method);
				fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
				handleErrorCodes(statusCode);		
			    
				return getFeed(null, urlStr, method, statusCode);
			} finally {
				method.releaseConnection();
				method.recycle();
			}
		}
	}


	/**
     * @param feedUrl
     * @param urlStr
     * @param method
     * @param feed
     * @return
     * @throws MalformedURLException
     */
    private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr, HttpMethod method, SyndFeed feed, int statusCode) throws MalformedURLException {
        SyndFeedInfo syndFeedInfo;
        syndFeedInfo = new SyndFeedInfo();
        
        // this may be different to feedURL because of 3XX redirects
        syndFeedInfo.setUrl(new URL(urlStr));
        syndFeedInfo.setId(feedUrl.toString());                					
                
        Header imHeader = method.getResponseHeader("IM");
        if (imHeader != null && imHeader.getValue().indexOf("feed") >= 0 && isUsingDeltaEncoding()) {
			FeedFetcherCache cache = getFeedInfoCache();
			if (cache != null && statusCode == 226) {
			    // client is setup to use http delta encoding and the server supports it and has returned a delta encoded response
			    // This response only includes new items
			    SyndFeedInfo cachedInfo = cache.getFeedInfo(feedUrl);
			    if (cachedInfo != null) {
				    SyndFeed cachedFeed = cachedInfo.getSyndFeed();
				    
				    // set the new feed to be the orginal feed plus the new items
				    feed = combineFeeds(cachedFeed, feed);			        
			    }            
			}
		}
        
        Header lastModifiedHeader = method.getResponseHeader("Last-Modified");
        if (lastModifiedHeader != null) {
            syndFeedInfo.setLastModified(lastModifiedHeader.getValue());
        }
        
        Header eTagHeader = method.getResponseHeader("ETag");
        if (eTagHeader != null) {
            syndFeedInfo.setETag(eTagHeader.getValue());
        }
        
        syndFeedInfo.setSyndFeed(feed);
        
        return syndFeedInfo;
    }

    /**
	 * @param client
	 * @param urlStr
	 * @param method
	 * @return
	 * @throws IOException
	 * @throws HttpException
	 * @throws FetcherException
	 * @throws FeedException
	 */
	private static SyndFeed retrieveFeed(String urlStr, HttpMethod method) throws IOException, HttpException, FetcherException, FeedException {
		
		InputStream stream = null;
		if ((method.getResponseHeader("Content-Encoding") != null) && ("gzip".equalsIgnoreCase(method.getResponseHeader("Content-Encoding").getValue()))) {		
		    stream = new GZIPInputStream(method.getResponseBodyAsStream());
		} else {
		    stream = method.getResponseBodyAsStream();
		}		
		try {		
		    XmlReader reader = null;
		    if (method.getResponseHeader("Content-Type") != null) {
		        reader = new XmlReader(stream, method.getResponseHeader("Content-Type").getValue(), true);
		    } else {
		        reader = new XmlReader(stream, true);
		    }
			return new SyndFeedInput().build(reader);
		} finally {
		    if (stream != null) {
		        stream.close();
		    }
		}
	}

	private SyndFeed getFeed(SyndFeedInfo syndFeedInfo, String urlStr, HttpMethod method, int statusCode) throws IOException, HttpException, FetcherException, FeedException {

		if (statusCode == HttpURLConnection.HTTP_NOT_MODIFIED && syndFeedInfo != null) {
		    fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
		    return syndFeedInfo.getSyndFeed();
		}
		
		SyndFeed feed = retrieveFeed(urlStr, method);
		fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr, feed);			
		return feed;
	}
	
    public interface CredentialSupplier {
        public Credentials getCredentials(String realm, String host);
    }
	
    public interface HttpClientMethodCallbackIntf {
    	/**
    	 * Allows access to the underlying HttpClient HttpMethod object. 
    	 * Note that in most cases, method.setRequestHeader(String, String)
    	 * is what you want to do (rather than method.addRequestHeader(String, String))
    	 * 
    	 * @param method
    	 */
    	public void afterHttpClientMethodCreate(HttpMethod method);
    }

}