com.jaeksoft.searchlib.scheduler.task.TaskPullFields Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of opensearchserver Show documentation
OpenSearchServer is a powerful, enterprise-class, search engine program. Using the web user interface, the crawlers (web, file, database, ...) and the REST/RESTFul API you will be able to integrate quickly and easily advanced full-text search capabilities in your application. OpenSearchServer runs on Windows and Linux/Unix/BSD.
There is a newer version: 1.5.14
Show newest version
/**   
 * License Agreement for OpenSearchServer
 *
 * Copyright (C) 2012-2013 Emmanuel Keller / Jaeksoft
 * 
 * http://www.open-search-server.com
 * 
 * This file is part of OpenSearchServer.
 *
 * OpenSearchServer is free software: you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 * OpenSearchServer is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with OpenSearchServer. 
 *  If not, see .
 **/

package com.jaeksoft.searchlib.scheduler.task;

import java.io.IOException;
import java.net.URISyntaxException;
import java.security.NoSuchAlgorithmException;
import java.util.List;

import javax.naming.NamingException;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;

import com.jaeksoft.searchlib.Client;
import com.jaeksoft.searchlib.SearchLibException;
import com.jaeksoft.searchlib.analysis.LanguageEnum;
import com.jaeksoft.searchlib.config.Config;
import com.jaeksoft.searchlib.crawler.FieldMap;
import com.jaeksoft.searchlib.index.IndexDocument;
import com.jaeksoft.searchlib.request.AbstractSearchRequest;
import com.jaeksoft.searchlib.request.SearchPatternRequest;
import com.jaeksoft.searchlib.result.AbstractResultSearch;
import com.jaeksoft.searchlib.result.ResultDocument;
import com.jaeksoft.searchlib.scheduler.TaskLog;
import com.jaeksoft.searchlib.scheduler.TaskProperties;
import com.jaeksoft.searchlib.scheduler.TaskPropertyDef;
import com.jaeksoft.searchlib.scheduler.TaskPropertyType;
import com.jaeksoft.searchlib.schema.FieldValueItem;
import com.jaeksoft.searchlib.util.Variables;
import com.jaeksoft.searchlib.util.map.GenericLink;
import com.jaeksoft.searchlib.util.map.SourceField;
import com.jaeksoft.searchlib.util.map.TargetField;

public class TaskPullFields extends TaskPullAbstract {

	final private TaskPropertyDef propSourceQuery = new TaskPropertyDef(
			TaskPropertyType.textBox, "Source query", "Source query",
			"The search query on the source index", 50);

	final private TaskPropertyDef propSourceMappedFields = new TaskPropertyDef(
			TaskPropertyType.multilineTextBox, "Mapped fields on source",
			"Mapped fields on source", null, 80, 5);

	final private TaskPropertyDef[] taskPropertyDefs = { propSourceIndex,
			propLogin, propApiKey, propSourceQuery, propLanguage,
			propSourceField, propTargetField, propSourceMappedFields,
			propTargetMappedFields, propBufferSize };

	@Override
	public String getName() {
		return "Pull fields";
	}

	@Override
	public TaskPropertyDef[] getPropertyList() {
		return taskPropertyDefs;
	}

	@Override
	public String getDefaultValue(Config config, TaskPropertyDef propertyDef) {
		if (propertyDef == propSourceQuery)
			return "*:*";
		return super.getDefaultValue(config, propertyDef);
	}

	@Override
	public void execute(Client client, TaskProperties properties,
			Variables variables, TaskLog taskLog) throws SearchLibException {
		String sourceQuery = properties.getValue(propSourceQuery);
		String sourceMappedFields = properties.getValue(propSourceMappedFields);

		try {

			ExecutionData executionData = new ExecutionData(properties, client);

			FieldMap sourceFieldMap = null;
			if (!StringUtils.isEmpty(sourceMappedFields)) {
				sourceFieldMap = new FieldMap(sourceMappedFields, ',', '|');
				sourceFieldMap.cacheAnalyzers(client.getSchema()
						.getAnalyzerList(), LanguageEnum.UNDEFINED);
			}

			AbstractSearchRequest searchRequest = new SearchPatternRequest(
					executionData.sourceClient);
			searchRequest.setQueryString(sourceQuery);
			searchRequest.addReturnField(executionData.sourceField);
			for (GenericLink link : sourceFieldMap
					.getList())
				link.getSource().addReturnField(searchRequest);
			searchRequest.setRows(executionData.bufferSize);
			int start = 0;

			for (;;) {
				searchRequest.setStart(start);
				AbstractResultSearch result = (AbstractResultSearch) executionData.sourceClient
						.request(searchRequest);

				if (result.getDocumentCount() <= 0)
					break;

				for (ResultDocument document : result) {
					List fieldValueItems = document
							.getValues(executionData.sourceField);
					if (CollectionUtils.isEmpty(fieldValueItems))
						continue;

					IndexDocument mappedDocument = new IndexDocument(
							executionData.lang);
					if (sourceFieldMap != null)
						sourceFieldMap.mapIndexDocument(document,
								mappedDocument);

					for (FieldValueItem fieldValueItem : fieldValueItems) {

						String value = fieldValueItem.getValue();
						if (value == null)
							continue;
						if (value.length() == 0)
							continue;

						executionData.indexDocument(client, mappedDocument,
								value, taskLog);

					}
				}

				searchRequest.reset();
				start += executionData.bufferSize;
			}
			executionData.indexBuffer(client, taskLog);
		} catch (NoSuchAlgorithmException e) {
			throw new SearchLibException(e);
		} catch (IOException e) {
			throw new SearchLibException(e);
		} catch (URISyntaxException e) {
			throw new SearchLibException(e);
		} catch (InstantiationException e) {
			throw new SearchLibException(e);
		} catch (IllegalAccessException e) {
			throw new SearchLibException(e);
		} catch (ClassNotFoundException e) {
			throw new SearchLibException(e);
		} catch (NamingException e) {
			throw new SearchLibException(e);
		}
	}
}