com.jaeksoft.searchlib.crawler.file.process.fileInstances.FtpFileInstance Maven / Gradle / Ivy

Go to download
/**   
 * License Agreement for OpenSearchServer
 *
 * Copyright (C) 2010-2013 Emmanuel Keller / Jaeksoft
 * 
 * http://www.open-search-server.com
 * 
 * This file is part of OpenSearchServer.
 *
 * OpenSearchServer is free software: you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 * OpenSearchServer is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with OpenSearchServer. 
 *  If not, see .
 **/

package com.jaeksoft.searchlib.crawler.file.process.fileInstances;

import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.SocketException;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;

import org.apache.commons.net.ftp.FTP;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPFileFilter;
import org.apache.commons.net.ftp.FTPReply;

import com.jaeksoft.searchlib.Logging;
import com.jaeksoft.searchlib.SearchLibException;
import com.jaeksoft.searchlib.crawler.file.database.FilePathItem;
import com.jaeksoft.searchlib.crawler.file.database.FileTypeEnum;
import com.jaeksoft.searchlib.crawler.file.process.FileInstanceAbstract;
import com.jaeksoft.searchlib.crawler.file.process.SecurityAccess;
import com.jaeksoft.searchlib.util.LinkUtils;
import com.jaeksoft.searchlib.util.RegExpUtils;

public class FtpFileInstance extends FileInstanceAbstract implements
		FileInstanceAbstract.SecurityInterface {

	private FTPFile ftpFile;

	public FtpFileInstance() {
		ftpFile = null;
	}

	protected FtpFileInstance(FilePathItem filePathItem,
			FtpFileInstance parent, FTPFile ftpFile) throws URISyntaxException,
			SearchLibException, UnsupportedEncodingException {
		init(filePathItem, parent,
				LinkUtils.concatPath(parent.getPath(), ftpFile.getName()));
		this.ftpFile = ftpFile;
	}

	@Override
	public URI init() throws URISyntaxException {
		return new URI("ftp", filePathItem.getHost(), getPath(), null);
	}

	@Override
	public FileTypeEnum getFileType() {
		if (ftpFile == null)
			return FileTypeEnum.directory;
		switch (ftpFile.getType()) {
		case FTPFile.DIRECTORY_TYPE:
			return FileTypeEnum.directory;
		case FTPFile.FILE_TYPE:
			return FileTypeEnum.file;
		}
		return null;
	}

	protected FTPFile getFTPFile() {
		return ftpFile;
	}

	protected FTPClient ftpConnect() throws SocketException, IOException,
			NoSuchAlgorithmException {
		FilePathItem fpi = getFilePathItem();
		FTPClient ftp = null;
		try {
			ftp = new FTPClient();
			// For debug
			// f.addProtocolCommandListener(new PrintCommandListener(
			// new PrintWriter(System.out)));
			ftp.setConnectTimeout(120000);
			ftp.setControlKeepAliveTimeout(180);
			ftp.setDataTimeout(120000);
			ftp.connect(fpi.getHost());
			int reply = ftp.getReplyCode();
			if (!FTPReply.isPositiveCompletion(reply))
				throw new IOException("FTP Error Code: " + reply);
			ftp.login(fpi.getUsername(), fpi.getPassword());
			if (!FTPReply.isPositiveCompletion(reply))
				throw new IOException("FTP Error Code: " + reply);
			if (fpi.isFtpUsePassiveMode())
				ftp.enterLocalPassiveMode();
			if (!FTPReply.isPositiveCompletion(reply))
				throw new IOException("FTP Error Code: " + reply);
			FTPClient ftpReturn = ftp;
			ftp = null;
			return ftpReturn;
		} finally {
			if (ftp != null)
				ftpQuietDisconnect(ftp);
		}
	}

	private void ftpQuietDisconnect(FTPClient f) {
		if (f == null)
			return;
		try {
			if (f.isConnected())
				f.disconnect();
		} catch (IOException e) {
			Logging.warn(e);
		}
	}

	protected FtpFileInstance newInstance(FilePathItem filePathItem,
			FtpFileInstance parent, FTPFile ftpFile) throws URISyntaxException,
			SearchLibException, UnsupportedEncodingException {
		return new FtpFileInstance(filePathItem, parent, ftpFile);
	}

	private FileInstanceAbstract[] buildFileInstanceArray(FTPFile[] files)
			throws URISyntaxException, SearchLibException,
			UnsupportedEncodingException {
		if (files == null)
			return null;
		FileInstanceAbstract[] fileInstances = new FileInstanceAbstract[files.length];
		int i = 0;
		for (FTPFile file : files)
			fileInstances[i++] = newInstance(filePathItem, this, file);
		return fileInstances;
	}

	public static class FtpInstanceFileFilter implements FTPFileFilter {

		private final Matcher[] exclusionMatchers;
		private final boolean ignoreHiddenFiles;
		private final boolean fileOnly;

		public FtpInstanceFileFilter(boolean ignoreHiddenFiles,
				boolean fileOnly, Matcher[] exclusionMatchers) {
			this.exclusionMatchers = exclusionMatchers;
			this.ignoreHiddenFiles = ignoreHiddenFiles;
			this.fileOnly = fileOnly;
		}

		@Override
		public boolean accept(FTPFile ff) {
			String name = ff.getName();
			if (name == null)
				return false;
			if (ignoreHiddenFiles)
				if (name.startsWith("."))
					return false;
			if (fileOnly)
				if (ff.getType() != FTPFile.FILE_TYPE)
					return false;
			if (exclusionMatchers != null)
				if (RegExpUtils.matches(ff.getLink(), exclusionMatchers))
					return false;
			return true;
		}
	}

	@Override
	public FileInstanceAbstract[] listFilesOnly() throws SearchLibException {
		FTPClient f = null;
		try {
			f = ftpConnect();
			FTPFile[] files = f.listFiles(
					getPath(),
					new FtpInstanceFileFilter(filePathItem
							.isIgnoreHiddenFiles(), true, filePathItem
							.getExclusionMatchers()));
			return buildFileInstanceArray(files);
		} catch (SocketException e) {
			throw new SearchLibException(e);
		} catch (IOException e) {
			throw new SearchLibException(e);
		} catch (URISyntaxException e) {
			throw new SearchLibException(e);
		} catch (NoSuchAlgorithmException e) {
			throw new SearchLibException(e);
		} finally {
			ftpQuietDisconnect(f);
		}
	}

	@Override
	public FileInstanceAbstract[] listFilesAndDirectories()
			throws SearchLibException {
		FTPClient f = null;
		try {
			f = ftpConnect();
			FTPFile[] files = f.listFiles(
					getPath(),
					new FtpInstanceFileFilter(filePathItem
							.isIgnoreHiddenFiles(), false, filePathItem
							.getExclusionMatchers()));
			return buildFileInstanceArray(files);
		} catch (SocketException e) {
			throw new SearchLibException(e);
		} catch (IOException e) {
			throw new SearchLibException(e);
		} catch (URISyntaxException e) {
			throw new SearchLibException(e);
		} catch (NoSuchAlgorithmException e) {
			throw new SearchLibException(e);
		} finally {
			ftpQuietDisconnect(f);
		}
	}

	@Override
	public Long getLastModified() {
		if (ftpFile == null)
			return null;
		return ftpFile.getTimestamp().getTimeInMillis();
	}

	@Override
	public Long getFileSize() {
		if (ftpFile == null)
			return null;
		return ftpFile.getSize();
	}

	@Override
	public String getFileName() throws SearchLibException {
		if (ftpFile == null)
			return null;
		return ftpFile.getName();
	}

	@Override
	public InputStream getInputStream() throws IOException {
		FTPClient f = null;
		try {
			f = ftpConnect();
			f.setFileType(FTP.BINARY_FILE_TYPE);
			return f.retrieveFileStream(getPath());
		} catch (NoSuchAlgorithmException e) {
			throw new IOException(e);
		} finally {
			ftpQuietDisconnect(f);
		}
	}

	public static List getSecurity(FTPFile ftpFile) {
		List accesses = new ArrayList();
		if (ftpFile == null)
			return accesses;
		if (ftpFile.hasPermission(FTPFile.USER_ACCESS, FTPFile.READ_PERMISSION)) {
			SecurityAccess access = new SecurityAccess();
			access.setGrant(SecurityAccess.Grant.ALLOW);
			access.setType(SecurityAccess.Type.USER);
			access.setId(ftpFile.getUser());
			accesses.add(access);
		}

		if (ftpFile
				.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.READ_PERMISSION)) {
			SecurityAccess access = new SecurityAccess();
			access.setGrant(SecurityAccess.Grant.ALLOW);
			access.setType(SecurityAccess.Type.GROUP);
			access.setId(ftpFile.getGroup());
			accesses.add(access);
		}

		return accesses;
	}

	@Override
	public List getSecurity() throws IOException {
		return getSecurity(getFTPFile());
	}
}