org.archive.surt.SURTTokenizer Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of heritrix-commons Show documentation
Show all versions of heritrix-commons Show documentation
The Archive Commons Code Libraries project contains general Java utility
libraries, as used by the Heritrix crawler and other projects.
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.surt;
import org.apache.commons.httpclient.URIException;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.ArchiveUtils;
import org.archive.util.SURT;
/**
* provides iterative Url reduction for prefix matching to find ever coarser
* grained URL-specific configuration. Assumes that a prefix binary search is
* being attempted for each returned value. First value is the entire SURT
* url String, with TAB appended. Second removes CGI ARGs. Then each subsequent
* path segment ('/' separated) is removed. Then the login:password, if present
* is removed. Then the port, if not :80 or omitted on the initial URL. Then
* each subsequent authority segment(. separated) is removed.
*
* the nextSearch() method will return null, finally, when no broader searches
* can be attempted on the URL.
*
* @author brad
* @version $Date$, $Revision$
*/
public class SURTTokenizer {
private final static String EXACT_SUFFIX = "\t";
private String remainder;
private boolean triedExact;
private boolean triedFull;
private boolean choppedArgs;
private boolean choppedPath;
private boolean choppedLogin;
private boolean choppedPort;
/**
* constructor
*
* @param url String URL
* @throws URIException
*/
public SURTTokenizer(final String url) throws URIException {
remainder = getKey(url,false);
}
/**
* update internal state and return the next smaller search string
* for the url
*
* @return string to lookup for prefix match for relevant information.
*/
public String nextSearch() {
if(!triedExact) {
triedExact = true;
//remainder = remainder.substring(0,remainder.length()-1);
return remainder + EXACT_SUFFIX;
}
if(!triedFull) {
triedFull = true;
return remainder;
}
if(!choppedArgs) {
choppedArgs = true;
int argStart = remainder.indexOf('?');
if(argStart != -1) {
remainder = remainder.substring(0,argStart);
return remainder;
}
}
if(!choppedPath) {
int lastSlash = remainder.lastIndexOf('/');
if(lastSlash != -1) {
remainder = remainder.substring(0,lastSlash);
if(remainder.endsWith(")")) {
remainder = remainder.substring(0,remainder.length()-1);
}
return remainder;
}
choppedPath = true;
}
if(!choppedLogin) {
choppedLogin = true;
int lastAt = remainder.lastIndexOf('@');
if(lastAt != -1) {
remainder = remainder.substring(0,lastAt);
if(remainder.endsWith(",")) {
remainder = remainder.substring(0,remainder.length()-1);
}
return remainder;
}
}
if(!choppedPort) {
choppedPort = true;
int lastColon = remainder.lastIndexOf(':');
if(lastColon != -1) {
remainder = remainder.substring(0,lastColon);
if(remainder.endsWith(",")) {
remainder = remainder.substring(0,remainder.length()-1);
}
return remainder;
}
}
// now just remove ','s
int lastComma = remainder.lastIndexOf(',');
if(lastComma == -1) {
return null;
}
remainder = remainder.substring(0,lastComma);
return remainder;
}
/**
* @param url
* @return String SURT which will match exactly argument url
* @throws URIException
*/
public static String exactKey(String url) throws URIException {
return getKey(url,false);
}
/**
* @param url
* @return String SURT which will match urls prefixed with the argument url
* @throws URIException
*/
public static String prefixKey(String url) throws URIException {
return getKey(url,true);
}
private static String getKey(String url, boolean prefix)
throws URIException {
String key = ArchiveUtils.addImpliedHttpIfNecessary(url);
UURI uuri = UURIFactory.getInstance(key);
key = uuri.getScheme() + "://" + uuri.getAuthority() +
uuri.getEscapedPathQuery();
key = SURT.fromURI(key);
int hashPos = key.indexOf('#');
if(hashPos != -1) {
key = key.substring(0,hashPos);
}
if(key.startsWith("http://")) {
key = key.substring(7);
}
if(prefix) {
if(key.endsWith(")/")) {
key = key.substring(0,key.length()-2);
}
}
return key;
}
}