com.jaeksoft.searchlib.crawler.file.process.fileInstances.swift.SwiftProtocol Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of opensearchserver Show documentation
Show all versions of opensearchserver Show documentation
OpenSearchServer is a powerful, enterprise-class, search engine program. Using the web user interface, the crawlers (web, file, database, ...) and the REST/RESTFul API you will be able to integrate quickly and easily advanced full-text search capabilities in your application. OpenSearchServer runs on Windows and Linux/Unix/BSD.
The newest version!
/**
* License Agreement for OpenSearchServer
*
* Copyright (C) 2013 Emmanuel Keller / Jaeksoft
*
* http://www.open-search-server.com
*
* This file is part of OpenSearchServer.
*
* OpenSearchServer is free software: you can redistribute it and/or
* modify it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* OpenSearchServer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with OpenSearchServer.
* If not, see .
**/
package com.jaeksoft.searchlib.crawler.file.process.fileInstances.swift;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import org.apache.commons.io.IOUtils;
import org.apache.http.client.ClientProtocolException;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.jaeksoft.searchlib.SearchLibException;
import com.jaeksoft.searchlib.SearchLibException.WrongStatusCodeException;
import com.jaeksoft.searchlib.crawler.web.database.HeaderItem;
import com.jaeksoft.searchlib.crawler.web.spider.DownloadItem;
import com.jaeksoft.searchlib.crawler.web.spider.HttpDownloader;
import com.jaeksoft.searchlib.util.FormatUtils.ThreadSafeSimpleDateFormat;
import com.jaeksoft.searchlib.util.LinkUtils;
import com.jaeksoft.searchlib.util.RegExpUtils;
import com.jaeksoft.searchlib.util.StringUtils;
import com.jaeksoft.searchlib.util.array.BytesOutputStream;
public class SwiftProtocol {
private final static String APPLICATION_DIRECTORY = "application/directory";
// 2013-02-23T10:53:26.184120
private final static ThreadSafeSimpleDateFormat swiftDateFormat = new ThreadSafeSimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss.SSSSSS");
public static class ObjectMeta {
public final String pathName;
public final String name;
public final Long lastModified;
public final String contentType;
public final Long contentLength;
public final boolean isDirectory;
public final boolean isHidden;
private ObjectMeta(final JSONObject json) throws JSONException, ParseException {
if (json.has("subdir")) {
this.pathName = json.getString("subdir");
this.isDirectory = true;
this.lastModified = null;
this.contentLength = null;
this.contentType = APPLICATION_DIRECTORY;
} else {
this.pathName = json.getString("name");
this.contentType = json.has("content_type") ? json.getString("content_type") : null;
this.lastModified = json.has("last_modified")
? swiftDateFormat.parse(json.getString("last_modified")).getTime() : null;
this.contentLength = json.has("bytes") ? json.getLong("bytes") : null;
this.isDirectory = APPLICATION_DIRECTORY.equals(contentType);
}
name = LinkUtils.lastPart(pathName);
isHidden = name.startsWith(".");
}
public ObjectMeta(final String path, final DownloadItem downloadItem) {
lastModified = downloadItem.getLastModified();
contentType = downloadItem.getContentBaseType();
contentLength = downloadItem.getContentLength();
pathName = path;
isDirectory = APPLICATION_DIRECTORY.equals(contentType);
isHidden = false;
name = null;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder(pathName);
sb.append(' ');
sb.append(contentType);
sb.append(' ');
sb.append(contentLength);
sb.append(' ');
sb.append(lastModified);
return sb.toString();
}
}
final public static List listObjects(HttpDownloader httpDownloader, SwiftToken swiftToken,
String container, String path, boolean withDirectory, boolean ignoreHiddenFiles,
Matcher[] exclusionMatchers)
throws URISyntaxException, ClientProtocolException, IOException, JSONException {
final List headerList = swiftToken.getAuthTokenHeader(null);
final URI uri = swiftToken.getURI(container, path, true);
final DownloadItem downloadItem = httpDownloader.get(uri, null, headerList, null);
try {
downloadItem.checkNoErrorList(200, 204);
} catch (WrongStatusCodeException e) {
throw new IOException(e);
}
final String jsonString = downloadItem.getContentAsString();
final JSONArray jsonArray = new JSONArray(jsonString);
List objectMetaList = new ArrayList(0);
for (int i = 0; i < jsonArray.length(); i++) {
ObjectMeta objectMeta;
try {
objectMeta = new ObjectMeta(jsonArray.getJSONObject(i));
} catch (ParseException e) {
throw new IOException(e);
}
if (ignoreHiddenFiles && objectMeta.isHidden)
continue;
if (!withDirectory && objectMeta.isDirectory)
continue;
if (path != null && path.equals(objectMeta.pathName))
continue;
if (exclusionMatchers != null)
if (RegExpUtils.matches(objectMeta.pathName, exclusionMatchers))
continue;
objectMetaList.add(objectMeta);
}
return objectMetaList;
}
final public static List listObjects(HttpDownloader httpDownloader, SwiftToken swiftToken, String container)
throws URISyntaxException, ClientProtocolException, IllegalStateException, IOException, SearchLibException {
final List headerList = swiftToken.getAuthTokenHeader(null);
final URI uri = swiftToken.getContainerURI(container);
DownloadItem downloadItem = httpDownloader.get(uri, null, headerList, null);
downloadItem.checkNoErrorList(200, 204);
InputStream inputStream = null;
try {
inputStream = downloadItem.getContentInputStream();
if (inputStream != null)
return IOUtils.readLines(inputStream);
return null;
} finally {
IOUtils.closeQuietly(inputStream);
}
}
final public static ObjectMeta headObject(HttpDownloader httpDownloader, SwiftToken swiftToken, String container,
String path) throws URISyntaxException, ClientProtocolException, IllegalStateException, IOException,
SearchLibException {
final List headerList = swiftToken.getAuthTokenHeader(null);
final URI uri = swiftToken.getPathURI(container, path);
final DownloadItem downloadItem = httpDownloader.head(uri, null, headerList, null);
final Integer statusCode = downloadItem.getStatusCode();
if (statusCode != null && statusCode == 404)
return null;
downloadItem.checkNoErrorRange(200, 300);
return new ObjectMeta(path, downloadItem);
}
final public static InputStream getObject(HttpDownloader downloader, String container, SwiftToken swiftToken,
ObjectMeta object) throws URISyntaxException, ClientProtocolException, IllegalStateException, IOException,
SearchLibException {
final List headerList = swiftToken.getAuthTokenHeader(null);
final URI uri = swiftToken.getURI(container, object.pathName, false);
final DownloadItem downloadItem = downloader.get(uri, null, headerList, null);
downloadItem.checkNoErrorRange(200, 204);
return downloadItem.getContentInputStream();
}
final public static void touchObject(HttpDownloader httpDownloader, SwiftToken swiftToken, String container,
String path) throws URISyntaxException, ClientProtocolException, IllegalStateException, IOException,
SearchLibException {
final List headerList = swiftToken.getAuthTokenHeader(new ArrayList(2));
headerList.add(new HeaderItem("Content-Length", "0"));
final URI uri = swiftToken.getPathURI(container, path);
final DownloadItem downloadItem = httpDownloader.put(uri, null, headerList, null, null);
downloadItem.checkNoErrorList(201);
}
final public static DownloadItem writeObject(HttpDownloader httpDownloader, SwiftToken swiftToken, String container,
String path, BytesOutputStream bytes) throws URISyntaxException, ClientProtocolException,
IllegalStateException, IOException, SearchLibException, NoSuchAlgorithmException {
final List headerList = swiftToken.getAuthTokenHeader(new ArrayList(2));
String md5 = bytes.getMD5();
headerList.add(new HeaderItem("ETag", md5));
final URI uri = swiftToken.getPathURI(container, path);
final DownloadItem downloadItem = httpDownloader.put(uri, null, headerList, null, bytes.getHttpEntity());
downloadItem.checkNoErrorList(201);
return downloadItem;
}
final public static void deleteObject(HttpDownloader httpDownloader, SwiftToken swiftToken, String container,
String path) throws URISyntaxException, ClientProtocolException, IllegalStateException, IOException,
SearchLibException {
final List headerList = swiftToken.getAuthTokenHeader(null);
final URI uri = swiftToken.getPathURI(container, path);
final DownloadItem downloadItem = httpDownloader.delete(uri, null, headerList, null);
downloadItem.checkNoErrorList(204, 404);
}
final public static InputStream readObject(HttpDownloader httpDownloader, SwiftToken swiftToken, String container,
String path, long rangeStart, long rangeEnd) throws URISyntaxException, ClientProtocolException,
IllegalStateException, IOException, SearchLibException {
final List headerList = swiftToken.getAuthTokenHeader(new ArrayList(2));
headerList.add(new HeaderItem("Range",
StringUtils.fastConcat(Long.toString(rangeStart), '-', Long.toString(rangeEnd))));
final URI uri = swiftToken.getPathURI(container, path);
final DownloadItem downloadItem = httpDownloader.get(uri, null, headerList, null);
downloadItem.checkNoErrorList(200, 206);
return downloadItem.getContentInputStream();
}
final public static DownloadItem readObject(HttpDownloader httpDownloader, SwiftToken swiftToken, String container,
String path) throws URISyntaxException, ClientProtocolException, IllegalStateException, IOException,
SearchLibException {
final List headerList = swiftToken.getAuthTokenHeader(null);
final URI uri = swiftToken.getPathURI(container, path);
final DownloadItem downloadItem = httpDownloader.get(uri, null, headerList, null);
downloadItem.checkNoErrorList(200, 206);
return downloadItem;
}
}