com.jaeksoft.searchlib.crawler.web.database.WebPropertyManager Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of opensearchserver Show documentation
OpenSearchServer is a powerful, enterprise-class, search engine program. Using the web user interface, the crawlers (web, file, database, ...) and the REST/RESTFul API you will be able to integrate quickly and easily advanced full-text search capabilities in your application. OpenSearchServer runs on Windows and Linux/Unix/BSD.
The newest version!
/**
 * License Agreement for OpenSearchServer
 * 
 * Copyright (C) 2008-2016 Emmanuel Keller / Jaeksoft
 * 

 * http://www.open-search-server.com
 * 

 * This file is part of OpenSearchServer.
 * 

 * OpenSearchServer is free software: you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 

 * OpenSearchServer is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with OpenSearchServer.
 * If not, see .
 **/

package com.jaeksoft.searchlib.crawler.web.database;

import com.jaeksoft.searchlib.ClientFactory;
import com.jaeksoft.searchlib.crawler.common.database.AbstractPropertyManager;
import com.jaeksoft.searchlib.crawler.web.spider.ProxyHandler;
import com.jaeksoft.searchlib.util.properties.PropertyItem;
import com.jaeksoft.searchlib.util.properties.PropertyItemListener;

import java.io.File;
import java.io.IOException;

public class WebPropertyManager extends AbstractPropertyManager implements PropertyItemListener {

	final private PropertyItem delayBetweenAccesses;
	final private PropertyItem fetchInterval;
	final private PropertyItem fetchIntervalUnit;
	final private PropertyItem maxUrlPerHost;
	final private PropertyItem maxUrlPerSession;
	final private PropertyItem userAgent;
	final private PropertyItem connectionTimeOut;
	final private PropertyItem maxDepth;
	final private PropertyItem exclusionEnabled;
	final private PropertyItem inclusionEnabled;
	final private PropertyItem robotsTxtEnabled;
	final private PropertyItem linkDetectionEnabled;
	final private PropertyItem screenshotBrowser;
	final private PropertyItem screenshotMethod;
	final private PropertyItem screenshotCaptureWidth;
	final private PropertyItem screenshotCaptureHeight;
	final private PropertyItem screenshotResizeWidth;
	final private PropertyItem screenshotResizeHeight;
	final private PropertyItem proxyHost;
	final private PropertyItem proxyPort;
	final private PropertyItem proxyExclusion;
	final private PropertyItem proxyEnabled;
	final private PropertyItem proxyUsername;
	final private PropertyItem proxyPassword;

	private ProxyHandler proxyHandler = null;

	public WebPropertyManager(File file) throws IOException {
		super(file, 1000);
		delayBetweenAccesses = newIntegerProperty("delayBetweenAccesses", 10,
				ClientFactory.INSTANCE.properties.getMinCrawlerDelay(), null);
		fetchInterval = newIntegerProperty("fetchInterval", 30, 1, null);
		fetchIntervalUnit = newStringProperty("fechIntervalUnit", "days");
		maxUrlPerHost = newIntegerProperty("maxUrlPerHost", 100, 1, null);
		maxUrlPerSession = newIntegerProperty("maxUrlPerSession", 10000, 1, null);
		userAgent = newStringProperty("userAgent", "OpenSearchServer_Bot");
		connectionTimeOut = newIntegerProperty("connectionTimeOut", 600, 0, null);
		exclusionEnabled = newBooleanProperty("exclusionEnabled", true);
		maxDepth = newIntegerProperty("maxDepth", null, null, null);
		inclusionEnabled = newBooleanProperty("inclusionEnabled", true);
		robotsTxtEnabled = newBooleanProperty("robotsTxtEnabled", true);
		linkDetectionEnabled = newBooleanProperty("linkDetectionEnabled", true);
		screenshotMethod = newStringProperty("screenshotMethod", "");
		screenshotBrowser = newStringProperty("screenshotBrowser", "");
		screenshotCaptureWidth = newIntegerProperty("screenshotCaptureWidth", 1024, 1, null);
		screenshotCaptureHeight = newIntegerProperty("screenshotCaptureHeight", 768, 1, null);
		screenshotResizeWidth = newIntegerProperty("screenshotResizeWidth", 240, 1, null);
		screenshotResizeHeight = newIntegerProperty("screenshotResizeHeight", 180, 1, null);
		proxyHost = newStringProperty("proxyHost", "");
		proxyPort = newIntegerProperty("proxyPort", 8080, null, null);
		proxyExclusion = newStringProperty("proxyExclusion", "");
		proxyEnabled = newBooleanProperty("proxyEnabled", false);
		proxyUsername = newStringProperty("proxyUsername", "");
		proxyPassword = newStringProperty("proxyPassword", "");
		proxyHost.addListener(this);
		proxyPort.addListener(this);
		proxyExclusion.addListener(this);
		proxyEnabled.addListener(this);
		proxyUsername.addListener(this);
		proxyPassword.addListener(this);
	}

	public PropertyItem getProxyHost() {
		return proxyHost;
	}

	public PropertyItem getProxyExclusion() {
		return proxyExclusion;
	}

	public PropertyItem getProxyEnabled() {
		return proxyEnabled;
	}

	public PropertyItem getProxyPort() {
		return proxyPort;
	}

	public PropertyItem getProxyUsername() {
		return proxyUsername;
	}

	public PropertyItem getProxyPassword() {
		return proxyPassword;
	}

	public PropertyItem getFetchInterval() {
		return fetchInterval;
	}

	public PropertyItem getMaxUrlPerHost() {
		return maxUrlPerHost;
	}

	public PropertyItem getMaxUrlPerSession() {
		return maxUrlPerSession;
	}

	public PropertyItem getUserAgent() {
		return userAgent;
	}

	public PropertyItem getConnectionTimeOut() {
		return connectionTimeOut;
	}

	public PropertyItem getMaxDepth() {
		return maxDepth;
	}

	public PropertyItem getFetchIntervalUnit() {
		return fetchIntervalUnit;
	}

	public PropertyItem getInclusionEnabled() {
		return inclusionEnabled;
	}

	public PropertyItem getExclusionEnabled() {
		return exclusionEnabled;
	}

	public PropertyItem getDelayBetweenAccesses() {
		return delayBetweenAccesses;
	}

	public PropertyItem getRobotsTxtEnabled() {
		return robotsTxtEnabled;
	}

	public PropertyItem getLinkDetectionEnabled() {
		return linkDetectionEnabled;
	}

	public PropertyItem getScreenshotMethod() {
		return screenshotMethod;
	}

	public PropertyItem getScreenshotBrowser() {
		return screenshotBrowser;
	}

	public PropertyItem getScreenshotCaptureWidth() {
		return screenshotCaptureWidth;
	}

	public PropertyItem getScreenshotCaptureHeight() {
		return screenshotCaptureHeight;
	}

	public PropertyItem getScreenshotResizeWidth() {
		return screenshotResizeWidth;
	}

	public PropertyItem getScreenshotResizeHeight() {
		return screenshotResizeHeight;
	}

	@Override
	public void hasBeenSet(PropertyItem prop) {
		synchronized (this) {
			if (prop == proxyHost || prop == proxyPort || prop == proxyExclusion || prop == proxyUsername
					|| prop == proxyPassword)
				proxyHandler = null;
		}
	}

	public ProxyHandler getProxyHandler() throws IOException {
		synchronized (this) {
			if (!proxyEnabled.getValue())
				return null;
			if (proxyHandler != null)
				return proxyHandler;
			proxyHandler = new ProxyHandler(this);
			return proxyHandler;
		}
	}

}