org.archive.format.gzip.zipnum.ZipNumBlockLoader Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of webarchive-commons Show documentation
The newest version!
package org.archive.format.gzip.zipnum;

import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.archive.util.GeneralURIStreamFactory;
import org.archive.util.HMACSigner;
import org.archive.util.binsearch.SeekableLineReader;
import org.archive.util.binsearch.SeekableLineReaderFactory;
import org.archive.util.binsearch.impl.HTTPSeekableLineReader;
import org.archive.util.binsearch.impl.HTTPSeekableLineReaderFactory;
import org.archive.util.binsearch.impl.HTTPSeekableLineReaderFactory.HttpLibs;
import org.archive.util.io.RuntimeIOException;

public class ZipNumBlockLoader {
		
	private final static Logger LOGGER = Logger.getLogger(ZipNumBlockLoader.class.getName());
	
	protected Map fileFactoryMap = null;
	protected HTTPSeekableLineReaderFactory httpFactory = null;
	
	// Request signing
	final static int DEFAULT_SIG_DURATION_SECS = 10;
	
	protected HMACSigner signer;
	protected int signDurationSecs = DEFAULT_SIG_DURATION_SECS;
	
	protected boolean useNio = false;
	protected String httpLib = HttpLibs.APACHE_31.name();
	
	protected boolean bufferFully = true;
	protected boolean noKeepAlive = true;
	protected String cookie = null;
	
	protected int maxHostConnections = 100000;
	protected int maxTotalConnections = 100000;
	
	protected int connectTimeoutMS = 10000;
	protected int readTimeoutMS = 10000;
	
	protected int numRetries = -1;
	
	protected boolean staleChecking = false;

	
	public ZipNumBlockLoader()
	{

	}
	
	private static ThreadLocal > slrMap = new ThreadLocal >()
	{
		@Override
		protected Map initialValue() {
			return new HashMap();
		}
	};
	
	public static void closeAllReaders()
	{
		for (Entry entry : slrMap.get().entrySet()) {
			try {
				SeekableLineReader reader = entry.getValue();
				if (!reader.isClosed()) {
					LOGGER.warning("Unclosed reader for: " + entry.getKey());
					reader.close();
				}
			} catch (IOException io) {
				
			}
		}
		
		slrMap.get().clear();
	}
	
	public SeekableLineReader createBlockReader(String uri) throws IOException
	{
		SeekableLineReader reader = null;
		
		if (GeneralURIStreamFactory.isHttp(uri)) {
			reader = getHttpReader(uri);
		} else {
			reader = getFileReader(uri);
		}
		
		slrMap.get().put(uri, reader);
				
		return reader;
	}
	
	protected HTTPSeekableLineReader getHttpReader(String url) throws IOException {
		
		if (httpFactory == null) {
			httpFactory = HTTPSeekableLineReaderFactory.getHttpFactory(HttpLibs.valueOf(httpLib), null);
			httpFactory.setMaxHostConnections(maxHostConnections);
			httpFactory.setMaxTotalConnections(maxTotalConnections);
			httpFactory.setConnectionTimeoutMS(connectTimeoutMS);
			httpFactory.setSocketTimeoutMS(readTimeoutMS);
			httpFactory.setStaleChecking(staleChecking);
			
			if (numRetries >= 0) {
				httpFactory.setNumRetries(numRetries);
			}
		}
		
		HTTPSeekableLineReader reader = httpFactory.get(url);
		reader.setBufferFully(bufferFully);
		reader.setNoKeepAlive(noKeepAlive);
		
		String reqCookie = cookie;
		
		if (signer != null) {
			reqCookie = signer.getHMacCookieStr(signDurationSecs);
		}
		
		if (reqCookie != null) {
			reader.setCookie(reqCookie);
		}
		
		return reader;
	}
	
	protected SeekableLineReader getFileReader(String filename) throws IOException {
		
		if (fileFactoryMap == null) {
			fileFactoryMap = new HashMap();
			fileFactoryMap = Collections.synchronizedMap(fileFactoryMap);
		}
		
		SeekableLineReaderFactory factory = fileFactoryMap.get(filename);
		
		if (factory == null) {
			factory = GeneralURIStreamFactory.createSeekableStreamFactory(filename, useNio);
			fileFactoryMap.put(filename, factory);
		}
		
		SeekableLineReader reader = factory.get();
		reader.setBufferFully(bufferFully);
		return reader;
	}
	
	public SeekableLineReader attemptLoadBlock(String location, long startOffset, int totalLength, boolean decompress, boolean isRequired)
	{
		SeekableLineReader currReader = null;
		
		try {
			currReader = createBlockReader(location);
			
	        currReader.seekWithMaxRead(startOffset, decompress, totalLength);
		
		} catch (IOException io) {
			Level level = (isRequired ? Level.SEVERE : Level.WARNING);
			
			String actualLocation = null;
			
			if (currReader instanceof HTTPSeekableLineReader) {
				actualLocation = ((HTTPSeekableLineReader)currReader).getConnectedUrl();
			}
			
			if (actualLocation == null) {
				actualLocation = location;
			}
			
			String msg = io.toString() + " -- -r " + startOffset + "-" + (startOffset + totalLength - 1) + " " + actualLocation;
			
			if (LOGGER.isLoggable(level)) {
				LOGGER.log(level, msg);
			}
			
			if (currReader != null) {
				try {
					currReader.close();
				} catch (IOException e) {
	
				}
				currReader = null;
			}
			
			if (isRequired) {
				throw new RuntimeIOException(msg);
			}
		}
		
		return currReader;
	}
	
	public void closeFileFactory(String filename) throws IOException
	{
		if (fileFactoryMap == null) {
			return;
		}
		
		SeekableLineReaderFactory factory = fileFactoryMap.remove(filename);
		
		if (factory != null) {
			factory.close();
		}
	}
	
	public void close() throws IOException
	{
		if (fileFactoryMap != null) {
			for (SeekableLineReaderFactory factory : fileFactoryMap.values()) {
				factory.close();
			}
			fileFactoryMap = null;
		}
		
		if (httpFactory != null) {
			httpFactory.close();
			httpFactory = null;
		}
	}

	public boolean isUseNio() {
		return useNio;
	}

	public void setUseNio(boolean useNio) {
		this.useNio = useNio;
	}
	
	public boolean isBufferFully() {
		return bufferFully;
	}

	public void setBufferFully(boolean bufferFully) {
		this.bufferFully = bufferFully;
	}
	
	public boolean isNoKeepAlive() {
		return noKeepAlive;
	}

	public void setNoKeepAlive(boolean noKeepAlive) {
		this.noKeepAlive = noKeepAlive;
	}

	public int getMaxHostConnections() {
		return maxHostConnections;
	}

	public void setMaxHostConnections(int maxHostConnections) {
		this.maxHostConnections = maxHostConnections;
	}

	public int getMaxTotalConnections() {
		return maxTotalConnections;
	}

	public void setMaxTotalConnections(int maxTotalConnections) {
		this.maxTotalConnections = maxTotalConnections;
	}

	public int getConnectTimeoutMS() {
		return connectTimeoutMS;
	}

	public void setConnectTimeoutMS(int connectTimeoutMS) {
		this.connectTimeoutMS = connectTimeoutMS;
	}

	public int getReadTimeoutMS() {
		return readTimeoutMS;
	}

	public void setReadTimeoutMS(int readTimeoutMS) {
		this.readTimeoutMS = readTimeoutMS;
	}
	
	public int getNumRetries() {
		return numRetries;
	}

	public void setNumRetries(int numRetries) {
		this.numRetries = numRetries;
	}

	public void setStaleChecking(boolean staleChecking)
	{
		this.staleChecking = staleChecking;
	}
	
	public boolean isStaleChecking()
	{
		return this.staleChecking;
	}

	public String getHttpLib() {
		return httpLib;
	}

	public void setHttpLib(String httpLib) {
		this.httpLib = httpLib;
	}

	public String getCookie() {
		return cookie;
	}

	public void setCookie(String cookie) {
		this.cookie = cookie;
	}

	public HMACSigner getSigner() {
		return signer;
	}

	public void setSigner(HMACSigner signer) {
		this.signer = signer;
	}

	public int getSignDurationSecs() {
		return signDurationSecs;
	}

	public void setSignDurationSecs(int signDurationSecs) {
		this.signDurationSecs = signDurationSecs;
	}
}