com.github.ltsopensource.kv.index.MemIndexSnapshot Maven / Gradle / Ivy

Go to download
package com.github.ltsopensource.kv.index;

import com.github.ltsopensource.core.commons.file.FileUtils;
import com.github.ltsopensource.core.commons.io.UnsafeByteArrayInputStream;
import com.github.ltsopensource.core.commons.io.UnsafeByteArrayOutputStream;
import com.github.ltsopensource.kv.StoreConfig;
import com.github.ltsopensource.kv.replay.TxLogReplay;
import com.github.ltsopensource.kv.serializer.StoreSerializer;
import com.github.ltsopensource.kv.txlog.StoreTxLogPosition;
import com.github.ltsopensource.core.json.TypeReference;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.util.Arrays;
import java.util.Comparator;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;

/**
 * @author Robert HG ([email protected]) on 12/19/15.
 */
public class MemIndexSnapshot extends AbstractIndexSnapshot {

    private TxLogReplay txLogReplay;

    public MemIndexSnapshot(TxLogReplay txLogReplay, Index index, StoreConfig storeConfig, StoreSerializer serializer) {
        super(index, storeConfig, serializer);
        this.txLogReplay = txLogReplay;
    }

    @Override
    protected void loadFromDisk() throws IOException {

        FileUtils.createDirIfNotExist(storeConfig.getIndexPath());

        String[] indexFiles = getIndexFiles();
        if (indexFiles == null || indexFiles.length == 0) {
            return;
        }

        FileChannel fileChannel = null;
        try {
            File lastSnapshot = new File(storeConfig.getIndexPath(), indexFiles[indexFiles.length - 1]);
            fileChannel = FileUtils.newFileChannel(lastSnapshot, "rw");
            IndexSnapshotFileHeader fileHeader = new IndexSnapshotFileHeader();
            fileHeader.read(fileChannel);

            ConcurrentMap> indexMap = null;
            if (fileHeader.getStoreTxLogRecordId() != 0) {
                UnsafeByteArrayOutputStream os = new UnsafeByteArrayOutputStream();
                WritableByteChannel target = Channels.newChannel(os);
                long readLength = fileChannel.size() - fileHeader.getLength();
                if (readLength != 0) {
                    fileChannel.transferTo(fileHeader.getLength(), readLength, target);

                    UnsafeByteArrayInputStream is = new UnsafeByteArrayInputStream(os.toByteArray());

                    indexMap = serializer.deserialize(is,
                            new TypeReference>>() {
                            }.getType());
                }
            }

            if (indexMap == null) {
                indexMap = new ConcurrentSkipListMap>();
            }

            ((MemIndex) index).setIndexMap(indexMap);

            StoreTxLogPosition lastTxLog = new StoreTxLogPosition();
            lastTxLog.setRecordId(fileHeader.getStoreTxLogRecordId());

            ((MemIndex) index).setLastTxLog(lastTxLog);

        } finally {
            if (fileChannel != null) {
                fileChannel.close();
            }
        }
    }

    private String[] getIndexFiles() throws IOException {
        String[] indexFiles = storeConfig.getIndexPath().list(new FilenameFilter() {
            @Override
            public boolean accept(File dir, String name) {
                return name.endsWith(".snapshot");
            }
        });

        if (indexFiles == null) {
            throw new IOException("can't list file in " + storeConfig.getIndexPath());
        }

        if (indexFiles.length == 0) {
            return null;
        }

        Arrays.sort(indexFiles, new Comparator() {
            @Override
            public int compare(String left, String right) {
                return left.compareTo(right);
            }
        });

        return indexFiles;
    }

    @Override
    protected void replayTxLog() {
        // 重放
        StoreTxLogPosition indexTxLog = index.lastTxLog();
        StoreTxLogPosition dataTxLog = storeConfig.getLastTxLogPositionOnDataBlock();

        // 需要重放的位置
        StoreTxLogPosition replayTxLog = null;
        if (dataTxLog == null) {
            replayTxLog = indexTxLog;
        } else if (indexTxLog == null) {
            replayTxLog = dataTxLog;
        } else {
            replayTxLog = (indexTxLog.getRecordId() < dataTxLog.getRecordId() ? indexTxLog : dataTxLog);
        }

        if (replayTxLog == null) {
            return;
        }

        txLogReplay.replay(replayTxLog);
    }

    private StoreTxLogPosition lastStoreTxLogPosition;

    @Override
    public void snapshot() throws IOException {

        StoreTxLogPosition storeTxLogPosition = index.lastTxLog();

        if (storeTxLogPosition == null) {
            return;
        }
        if (lastStoreTxLogPosition != null && lastStoreTxLogPosition.getRecordId() == storeTxLogPosition.getRecordId()) {
            return;
        }

        ConcurrentMap> indexMap = ((MemIndex) index).getIndexMap();

        String name = System.currentTimeMillis() + ".snapshot";
        File snapshot = new File(storeConfig.getIndexPath(), name);
        FileChannel fileChannel = FileUtils.newFileChannel(snapshot, "rw");

        IndexSnapshotFileHeader fileHeader = new IndexSnapshotFileHeader();

        UnsafeByteArrayOutputStream os = new UnsafeByteArrayOutputStream();
        serializer.serialize(indexMap, os);
        byte[] payload = os.toByteArray();
        ReadableByteChannel src = Channels.newChannel(new UnsafeByteArrayInputStream(payload));

        // 先写一个空的文件头
        fileHeader.write(fileChannel);

        // 写内容
        fileChannel.transferFrom(src, fileHeader.getLength(), payload.length);
        fileChannel.force(true);

        // 写真实的文件头
        fileHeader.setStoreTxLogRecordId(storeTxLogPosition.getRecordId());
        fileHeader.write(fileChannel);

        // 删除多余的快照数目
        deleteOverSnapshot();

        LOGGER.info("snapshot index finished: [" + name + "]");

        lastStoreTxLogPosition = storeTxLogPosition;
    }

    /**
     * 删除多余的快照数目
     */
    private void deleteOverSnapshot() throws IOException {
        String[] indexFiles = getIndexFiles();
        if (indexFiles == null || indexFiles.length == 0) {
            return;
        }

        if (storeConfig.getMaxIndexSnapshotSize() > 1 && indexFiles.length > storeConfig.getMaxIndexSnapshotSize()) {

            for (int i = 0; i < indexFiles.length - storeConfig.getMaxIndexSnapshotSize(); i++) {

                FileUtils.delete(new File(storeConfig.getIndexPath(), indexFiles[i]));
                LOGGER.info("delete index snapshot [" + indexFiles[i] + "] succeed");
            }

        }

    }
}