com.dasasian.chok.node.ShardManager Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of chok-core Show documentation
There is a newer version: 1.7
/**
 * Copyright (C) 2014 Dasasian ([email protected])
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.dasasian.chok.node;

import com.dasasian.chok.util.ChokException;
import com.dasasian.chok.util.FileUtil;
import com.dasasian.chok.util.ThrottledInputStream;
import com.dasasian.chok.util.ThrottledInputStream.ThrottleSemaphore;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Progressable;
import org.apache.log4j.Logger;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;

public class ShardManager {

    protected final static Logger LOG = Logger.getLogger(ShardManager.class);
    private final File shardsFolder;
    private final ThrottleSemaphore throttleSemaphore;

    public ShardManager(File shardsFolder) {
        this(shardsFolder, null);
    }

    public ShardManager(File shardsFolder, ThrottleSemaphore throttleSemaphore) {
        this.shardsFolder = shardsFolder;
        this.throttleSemaphore = throttleSemaphore;
        if (!this.shardsFolder.exists()) {
            this.shardsFolder.mkdirs();
        }
        if (!this.shardsFolder.exists()) {
            throw new IllegalStateException("could not create local shard folder '" + this.shardsFolder.getAbsolutePath() + "'");
        }
    }

    public File installShard(String shardName, String shardPath) throws Exception {
        File localShardFolder = getShardFolder(shardName);
        try {
            if (!localShardFolder.exists()) {
                installShard(shardName, shardPath, localShardFolder);
            }
            return localShardFolder;
        } catch (Exception e) {
            FileUtil.deleteFolder(localShardFolder);
            throw e;
        }
    }

    public void uninstallShard(String shardName) {
        File localShardFolder = getShardFolder(shardName);
        FileUtil.deleteFolder(localShardFolder);
    }

    public Collection getInstalledShards() {
        String[] folderList = shardsFolder.list(FileUtil.VISIBLE_FILES_FILTER);
        if (folderList == null) {
            return Collections.EMPTY_LIST;
        }
        return Arrays.asList(folderList);
    }

    public File getShardsFolder() {
        return shardsFolder;
    }

    public File getShardFolder(String shardName) {
        return new File(shardsFolder, shardName);
    }

    /*
     * Loads a shard from the given URI. The uri is handled bye the hadoop file
     * system. So all hadoop support file systems can be used, like local hdfs s3
     * etc. In case the shard is compressed we also unzip the content. If the
     * system property chok.spool.zip.shards is true, the zip file is staged to
     * the local disk before being unzipped.
     */
    private void installShard(String shardName, String shardPath, File localShardFolder) throws ChokException {
        LOG.info("install shard '" + shardName + "' from " + shardPath);
        // TODO sg: to fix HADOOP-4422 we try to download the shard 5 times
        int maxTries = 5;
        for (int i = 0; i < maxTries; i++) {
            URI uri;
            try {
                uri = new URI(shardPath);
                FileSystem fileSystem = FileSystem.get(uri, new Configuration());
                if (throttleSemaphore != null) {
                    fileSystem = new ThrottledFileSystem(fileSystem, throttleSemaphore);
                }
                final Path path = new Path(shardPath);
                boolean isZip = fileSystem.isFile(path) && shardPath.endsWith(".zip");

                File shardTmpFolder = new File(localShardFolder.getAbsolutePath() + "_tmp");
                try {
                    FileUtil.deleteFolder(localShardFolder);
                    FileUtil.deleteFolder(shardTmpFolder);

                    if (isZip) {
                        FileUtil.unzip(path, shardTmpFolder, fileSystem, "true".equalsIgnoreCase(System.getProperty("chok.spool.zip.shards", "false")));
                    } else {
                        fileSystem.copyToLocalFile(path, new Path(shardTmpFolder.getAbsolutePath()));
                    }
                    shardTmpFolder.renameTo(localShardFolder);
                } finally {
                    // Ensure that the tmp folder is deleted on an error
                    FileUtil.deleteFolder(shardTmpFolder);
                }
                // Looks like we were successful.
                if (i > 0) {
                    LOG.error("Loaded shard:" + shardPath);
                }
                return;
            } catch (final URISyntaxException e) {
                throw new ChokException("Can not parse uri for path: " + shardPath, e);
            } catch (final Exception e) {
                LOG.error(String.format("Error loading shard: %s (try %d of %d)", shardPath, i, maxTries), e);
                if (i >= maxTries - 1) {
                    throw new ChokException("Can not load shard: " + shardPath, e);
                }
            }
        }
    }

    @SuppressWarnings("deprecation")
    private static class ThrottledFileSystem extends FileSystem {

        private final FileSystem fileSystemDelegate;
        private final ThrottleSemaphore throttleSemaphore;

        public ThrottledFileSystem(FileSystem fileSystem, ThrottleSemaphore throttleSemaphore) {
            fileSystemDelegate = fileSystem;
            this.throttleSemaphore = throttleSemaphore;
        }

        @Override
        public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException {
            return fileSystemDelegate.append(arg0, arg1, arg2);
        }

        @Override
        public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3, short arg4, long arg5, Progressable arg6) throws IOException {
            return fileSystemDelegate.create(arg0, arg1, arg2, arg3, arg4, arg5, arg6);
        }

        @Override
        public boolean delete(Path arg0) throws IOException {
            return fileSystemDelegate.delete(arg0);
        }

        @Override
        public boolean delete(Path arg0, boolean arg1) throws IOException {
            return fileSystemDelegate.delete(arg0, arg1);
        }

        @Override
        public FileStatus getFileStatus(Path arg0) throws IOException {
            return fileSystemDelegate.getFileStatus(arg0);
        }

        @Override
        public URI getUri() {
            return fileSystemDelegate.getUri();
        }

        @Override
        public Path getWorkingDirectory() {
            return fileSystemDelegate.getWorkingDirectory();
        }

        @Override
        public void setWorkingDirectory(Path arg0) {
            fileSystemDelegate.setWorkingDirectory(arg0);
        }

        @Override
        public FileStatus[] listStatus(Path arg0) throws IOException {
            return fileSystemDelegate.listStatus(arg0);
        }

        @Override
        public boolean mkdirs(Path arg0, FsPermission arg1) throws IOException {
            return fileSystemDelegate.mkdirs(arg0, arg1);
        }

        @Override
        public FSDataInputStream open(Path path, int bufferSize) throws IOException {
            ThrottledInputStream throttledInputStream = new ThrottledInputStream(fileSystemDelegate.open(path, bufferSize), throttleSemaphore);
            return new FSDataInputStream(throttledInputStream);
        }

        @Override
        public boolean rename(Path arg0, Path arg1) throws IOException {
            return fileSystemDelegate.rename(arg0, arg1);
        }

        @Override
        public Configuration getConf() {
            return fileSystemDelegate.getConf();
        }
    }
}