com.staros.filestore.HDFSFileStoreMgr Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of starmanager Show documentation
There is a newer version: 3.4-rc2
// Copyright 2021-present StarRocks, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


package com.staros.filestore;

import com.staros.exception.ExceptionCode;
import com.staros.exception.StarException;
import com.staros.journal.DummyJournalSystem;
import com.staros.journal.JournalSystem;
import com.staros.proto.FileStoreInfo;
import com.staros.proto.FileStoreType;
import com.staros.proto.HDFSFileStoreInfo;
import com.staros.util.Config;
import com.staros.util.LockCloseable;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;

public class HDFSFileStoreMgr implements FileStoreMgr {
    private static final Logger LOG = LogManager.getLogger(HDFSFileStoreMgr.class);

    // HDFS file store key to FileStore object
    private Map fileStores;

    private ReentrantReadWriteLock lock;

    private JournalSystem journalSystem;

    // FOR TEST
    public HDFSFileStoreMgr() {
        this(new DummyJournalSystem());
    }

    public HDFSFileStoreMgr(JournalSystem journalSystem) {
        this.fileStores = new HashMap();
        this.lock = new ReentrantReadWriteLock();
        this.journalSystem = journalSystem;
    }

    public FileStore getFileStore(String fsKey) {
        try (LockCloseable lockCloseable = new LockCloseable(lock.readLock())) {
            HDFSFileStore hdfs = fileStores.get(fsKey);
            return hdfs;
        }
    }

    public void loadFileStoreFromConfig() {
        HDFSFileStore hdfs = new HDFSFileStore(Config.HDFS_URL);
        try {
            addFileStore(hdfs);
            LOG.info("Add hdfs file store with key {} from config", hdfs.key());
        } catch (StarException e) {
            if (e.getExceptionCode() == ExceptionCode.ALREADY_EXIST) {
                try {
                    updateFileStore(hdfs);
                } catch (StarException e2) {
                    LOG.warn(e2.getMessage());
                }
            } else {
                LOG.info("no default hdfs file store configured");
            }
        }
    }

    public void addFileStore(FileStore hdfs) throws StarException {
        if (!hdfs.isValid()) {
            throw new StarException(ExceptionCode.INVALID_ARGUMENT, 
                    "HDFS file store invalid, please check");
        }

        try (LockCloseable lockCloseable = new LockCloseable(lock.writeLock())) {
            if (fileStores.containsKey(hdfs.key())) {
                throw new StarException(ExceptionCode.ALREADY_EXIST,
                        String.format("HDFS file store with key '%s' already exist", hdfs.key()));
            }

            fileStores.put(hdfs.key(), (HDFSFileStore) hdfs);

            LOG.info("Add HDFS file store {}", hdfs);
        }
    }

    public FileStore newFsFromProtobuf(FileStoreInfo fsInfo) {
        assert fsInfo.getFsType() == FileStoreType.HDFS;

        HDFSFileStoreInfo hdfsInfo = fsInfo.getHdfsFsInfo();
        HDFSFileStore hdfs = new HDFSFileStore(hdfsInfo.getUrl());
        return hdfs;
    }

    public void removeFileStore(String fsKey) throws StarException {
        // TODO(jeff.ding)
    }

    public List listFileStore() throws StarException {
        // TODO(jeff.ding)
        return new ArrayList();
    }

    public void updateFileStore(FileStore hdfs) throws StarException {
        // TODO(jeff.ding)
    }

    public FileStore allocFileStore() throws StarException {
        try (LockCloseable lockCloseable = new LockCloseable(lock.readLock())) {
            if (fileStores.isEmpty()) {
                throw new StarException(ExceptionCode.NOT_EXIST, "No any hdfs file store exist");
            }

            // choose the first one now, will add some policy in future
            HDFSFileStore fileStore = fileStores.values().iterator().next();
            return fileStore;
        }
    }
}