org.elasticsearch.util.http.HttpDownloadHelper Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Open Source, Distributed, RESTful Search Engine
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.util.http;
import javax.annotation.Nullable;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
/**
* @author kimchy (shay.banon)
*/
public class HttpDownloadHelper {
private boolean useTimestamp = false;
private boolean skipExisting = false;
private long maxTime = 0;
public boolean download(URL source, File dest, @Nullable DownloadProgress progress) throws IOException {
if (dest.exists() && skipExisting) {
return true;
}
//don't do any progress, unless asked
if (progress == null) {
progress = new NullProgress();
}
//set the timestamp to the file date.
long timestamp = 0;
boolean hasTimestamp = false;
if (useTimestamp && dest.exists()) {
timestamp = dest.lastModified();
hasTimestamp = true;
}
GetThread getThread = new GetThread(source, dest, hasTimestamp, timestamp, progress);
getThread.setDaemon(true);
getThread.start();
try {
getThread.join(maxTime * 1000);
} catch (InterruptedException ie) {
// ignore
}
if (getThread.isAlive()) {
String msg = "The GET operation took longer than " + maxTime
+ " seconds, stopping it.";
getThread.closeStreams();
throw new IOException(msg);
}
return getThread.wasSuccessful();
}
/**
* Interface implemented for reporting
* progress of downloading.
*/
public interface DownloadProgress {
/**
* begin a download
*/
void beginDownload();
/**
* tick handler
*/
void onTick();
/**
* end a download
*/
void endDownload();
}
/**
* do nothing with progress info
*/
public static class NullProgress implements DownloadProgress {
/**
* begin a download
*/
public void beginDownload() {
}
/**
* tick handler
*/
public void onTick() {
}
/**
* end a download
*/
public void endDownload() {
}
}
/**
* verbose progress system prints to some output stream
*/
public static class VerboseProgress implements DownloadProgress {
private int dots = 0;
// CheckStyle:VisibilityModifier OFF - bc
PrintStream out;
// CheckStyle:VisibilityModifier ON
/**
* Construct a verbose progress reporter.
*
* @param out the output stream.
*/
public VerboseProgress(PrintStream out) {
this.out = out;
}
/**
* begin a download
*/
public void beginDownload() {
dots = 0;
}
/**
* tick handler
*/
public void onTick() {
out.print(".");
if (dots++ > 50) {
out.flush();
dots = 0;
}
}
/**
* end a download
*/
public void endDownload() {
out.flush();
}
}
private class GetThread extends Thread {
private final URL source;
private final File dest;
private final boolean hasTimestamp;
private final long timestamp;
private final DownloadProgress progress;
private boolean success = false;
private IOException ioexception = null;
private InputStream is = null;
private OutputStream os = null;
private URLConnection connection;
private int redirections = 0;
GetThread(URL source, File dest, boolean h, long t, DownloadProgress p) {
this.source = source;
this.dest = dest;
hasTimestamp = h;
timestamp = t;
progress = p;
}
public void run() {
try {
success = get();
} catch (IOException ioex) {
ioexception = ioex;
}
}
private boolean get() throws IOException {
connection = openConnection(source);
if (connection == null) {
return false;
}
boolean downloadSucceeded = downloadFile();
//if (and only if) the use file time option is set, then
//the saved file now has its timestamp set to that of the
//downloaded file
if (downloadSucceeded && useTimestamp) {
updateTimeStamp();
}
return downloadSucceeded;
}
private boolean redirectionAllowed(URL aSource, URL aDest) throws IOException {
if (!(aSource.getProtocol().equals(aDest.getProtocol()) || ("http"
.equals(aSource.getProtocol()) && "https".equals(aDest
.getProtocol())))) {
String message = "Redirection detected from "
+ aSource.getProtocol() + " to " + aDest.getProtocol()
+ ". Protocol switch unsafe, not allowed.";
throw new IOException(message);
}
redirections++;
if (redirections > 5) {
String message = "More than " + 5 + " times redirected, giving up";
throw new IOException(message);
}
return true;
}
private URLConnection openConnection(URL aSource) throws IOException {
// set up the URL connection
URLConnection connection = aSource.openConnection();
// modify the headers
// NB: things like user authentication could go in here too.
if (hasTimestamp) {
connection.setIfModifiedSince(timestamp);
}
if (connection instanceof HttpURLConnection) {
((HttpURLConnection) connection).setInstanceFollowRedirects(false);
((HttpURLConnection) connection).setUseCaches(true);
}
// connect to the remote site (may take some time)
connection.connect();
// First check on a 301 / 302 (moved) response (HTTP only)
if (connection instanceof HttpURLConnection) {
HttpURLConnection httpConnection = (HttpURLConnection) connection;
int responseCode = httpConnection.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_MOVED_PERM ||
responseCode == HttpURLConnection.HTTP_MOVED_TEMP ||
responseCode == HttpURLConnection.HTTP_SEE_OTHER) {
String newLocation = httpConnection.getHeaderField("Location");
String message = aSource
+ (responseCode == HttpURLConnection.HTTP_MOVED_PERM ? " permanently"
: "") + " moved to " + newLocation;
URL newURL = new URL(newLocation);
if (!redirectionAllowed(aSource, newURL)) {
return null;
}
return openConnection(newURL);
}
// next test for a 304 result (HTTP only)
long lastModified = httpConnection.getLastModified();
if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED
|| (lastModified != 0 && hasTimestamp && timestamp >= lastModified)) {
// not modified so no file download. just return
// instead and trace out something so the user
// doesn't think that the download happened when it
// didn't
return null;
}
// test for 401 result (HTTP only)
if (responseCode == HttpURLConnection.HTTP_UNAUTHORIZED) {
String message = "HTTP Authorization failure";
throw new IOException(message);
}
}
//REVISIT: at this point even non HTTP connections may
//support the if-modified-since behaviour -we just check
//the date of the content and skip the write if it is not
//newer. Some protocols (FTP) don't include dates, of
//course.
return connection;
}
private boolean downloadFile() throws FileNotFoundException, IOException {
IOException lastEx = null;
for (int i = 0; i < 3; i++) {
// this three attempt trick is to get round quirks in different
// Java implementations. Some of them take a few goes to bind
// property; we ignore the first couple of such failures.
try {
is = connection.getInputStream();
break;
} catch (IOException ex) {
lastEx = ex;
}
}
if (is == null) {
throw new IOException("Can't get " + source + " to " + dest, lastEx);
}
os = new FileOutputStream(dest);
progress.beginDownload();
boolean finished = false;
try {
byte[] buffer = new byte[1024 * 100];
int length;
while (!isInterrupted() && (length = is.read(buffer)) >= 0) {
os.write(buffer, 0, length);
progress.onTick();
}
finished = !isInterrupted();
} finally {
try {
os.close();
} catch (IOException e) {
// ignore
}
try {
is.close();
} catch (IOException e) {
// ignore
}
// we have started to (over)write dest, but failed.
// Try to delete the garbage we'd otherwise leave
// behind.
if (!finished) {
dest.delete();
}
}
progress.endDownload();
return true;
}
private void updateTimeStamp() {
long remoteTimestamp = connection.getLastModified();
if (remoteTimestamp != 0) {
dest.setLastModified(remoteTimestamp);
}
}
/**
* Has the download completed successfully?
*
* Re-throws any exception caught during executaion.
*/
boolean wasSuccessful() throws IOException {
if (ioexception != null) {
throw ioexception;
}
return success;
}
/**
* Closes streams, interrupts the download, may delete the
* output file.
*/
void closeStreams() {
interrupt();
try {
os.close();
} catch (IOException e) {
// ignore
}
try {
is.close();
} catch (IOException e) {
// ignore
}
if (!success && dest.exists()) {
dest.delete();
}
}
}
}