com.staros.filestore.S3FileStoreMgr Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of starmanager Show documentation
There is a newer version: 3.4-rc2
// Copyright 2021-present StarRocks, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


package com.staros.filestore;

import com.staros.exception.ExceptionCode;
import com.staros.exception.StarException;
import com.staros.journal.DummyJournalSystem;
import com.staros.journal.JournalSystem;
import com.staros.proto.FileStoreInfo;
import com.staros.proto.FileStoreType;
import com.staros.proto.S3FileStoreInfo;
import com.staros.util.Config;
import com.staros.util.LockCloseable;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;

public class S3FileStoreMgr implements FileStoreMgr {
    private static final Logger LOG = LogManager.getLogger(S3FileStoreMgr.class);

    // File store unit key to S3FileStoreUnit object
    private Map s3FileStores;

    private ReentrantReadWriteLock lock;

    private JournalSystem journalSystem;

    // FOR TEST
    public S3FileStoreMgr() {
        this(new DummyJournalSystem());
    }

    public S3FileStoreMgr(JournalSystem journalSystem) {
        this.s3FileStores = new HashMap();
        this.lock = new ReentrantReadWriteLock();
        this.journalSystem = journalSystem;
    }

    public void loadFileStoreFromConfig() {
        S3FileStore s3fs = new S3FileStore(Config.S3_BUCKET,
                                           Config.S3_REGION,
                                           Config.S3_ENDPOINT,
                                           Config.S3_AK,
                                           Config.S3_SK);
        try {
            addFileStore(s3fs);
            LOG.info("Add s3 file store with key {} from config", s3fs.key());
        } catch (StarException e) {
            if (e.getExceptionCode() == ExceptionCode.ALREADY_EXIST) {
                try {
                    updateFileStore(s3fs);
                } catch (StarException e2) {
                    LOG.warn(e2.getMessage());
                }
            } else {
                LOG.info("no default file store configured");
            }
        }
    }

    public FileStore newFsFromProtobuf(FileStoreInfo fsInfo) {
        assert fsInfo.getFsType() == FileStoreType.S3;

        S3FileStoreInfo s3fsInfo = fsInfo.getS3FsInfo();
        S3FileStore s3fs = new S3FileStore(s3fsInfo.getBucket(),
                                           s3fsInfo.getRegion(),
                                           s3fsInfo.getEndpoint(),
                                           s3fsInfo.getAccessKey(),
                                           s3fsInfo.getAccessKeySecret());
        return s3fs;
    }

    public FileStore getFileStore(String fsKey) {
        try (LockCloseable lockCloseable = new LockCloseable(lock.readLock())) {
            S3FileStore fileStore = s3FileStores.get(fsKey);
            return fileStore;
        }
    }

    public FileStore allocFileStore() throws StarException {
        try (LockCloseable lockCloseable = new LockCloseable(lock.readLock())) {
            if (s3FileStores.isEmpty()) {
                throw new StarException(ExceptionCode.NOT_EXIST, "No any S3 file store exist");
            }

            // choose the first one now, will add some policy in future
            S3FileStore fileStore = s3FileStores.values().iterator().next();
            return fileStore;
        }
    }

    public void addFileStore(FileStore s3fs) throws StarException {
        if (!s3fs.isValid()) {
            throw new StarException(ExceptionCode.INVALID_ARGUMENT, 
                    "Invalid S3 file store, please check");
        }

        try (LockCloseable lockCloseable = new LockCloseable(lock.writeLock())) {
            if (s3FileStores.containsKey(s3fs.key())) {
                throw new StarException(ExceptionCode.ALREADY_EXIST,
                        String.format("S3 file store with key '%s' already exist", s3fs.key()));
            }

            s3FileStores.put(s3fs.key(), (S3FileStore) s3fs);

            LOG.info("Add S3 file store {}", s3fs);
        }
    }

    public void removeFileStore(String fsKey) throws StarException {
        // TODO(jeff.ding)
    }

    public List listFileStore() throws StarException {
        // TODO(jeff.ding)
        return new ArrayList();
    }

    public void updateFileStore(FileStore s3fs) throws StarException {
        // TODO(jeff.ding)
    }
}