com.alibaba.rocketmq.store.index.IndexFile Maven / Gradle / Ivy

Go to download
/**
 * Copyright (C) 2010-2013 Alibaba Group Holding Limited
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.alibaba.rocketmq.store.index;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.rocketmq.common.constant.LoggerName;
import com.alibaba.rocketmq.store.MapedFile;


/**
 * 存储具体消息索引信息的文件
 * 
 * @author shijia.wxr
 * @since 2013-7-21
 */
public class IndexFile {
    private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName);
    private static int HASH_SLOT_SIZE = 4;
    private static int INDEX_SIZE = 20;
    private static int INVALID_INDEX = 0;
    private final int hashSlotNum;
    private final int indexNum;
    private final MapedFile mapedFile;
    private final FileChannel fileChannel;
    private final MappedByteBuffer mappedByteBuffer;
    private final IndexHeader indexHeader;


    public IndexFile(final String fileName, final int hashSlotNum, final int indexNum,
            final long endPhyOffset, final long endTimestamp) throws IOException {
        int fileTotalSize =
                IndexHeader.INDEX_HEADER_SIZE + (hashSlotNum * HASH_SLOT_SIZE) + (indexNum * INDEX_SIZE);
        this.mapedFile = new MapedFile(fileName, fileTotalSize);
        this.fileChannel = this.mapedFile.getFileChannel();
        this.mappedByteBuffer = this.mapedFile.getMappedByteBuffer();
        this.hashSlotNum = hashSlotNum;
        this.indexNum = indexNum;

        ByteBuffer byteBuffer = this.mappedByteBuffer.slice();
        this.indexHeader = new IndexHeader(byteBuffer);

        if (endPhyOffset > 0) {
            this.indexHeader.setBeginPhyOffset(endPhyOffset);
            this.indexHeader.setEndPhyOffset(endPhyOffset);
        }

        if (endTimestamp > 0) {
            this.indexHeader.setBeginTimestamp(endTimestamp);
            this.indexHeader.setEndTimestamp(endTimestamp);
        }
    }


    public String getFileName() {
        return this.mapedFile.getFileName();
    }


    public void load() {
        this.indexHeader.load();
    }


    public void flush() {
        long beginTime = System.currentTimeMillis();
        if (this.mapedFile.hold()) {
            this.indexHeader.updateByteBuffer();
            this.mappedByteBuffer.force();
            this.mapedFile.release();
            log.info("flush index file eclipse time(ms) " + (System.currentTimeMillis() - beginTime));
        }
    }


    /**
     * 当前索引文件是否写满
     */
    public boolean isWriteFull() {
        return this.indexHeader.getIndexCount() >= this.indexNum;
    }


    public boolean destroy(final long intervalForcibly) {
        return this.mapedFile.destroy(intervalForcibly);
    }


    /**
     * 如果返回false，表示需要创建新的索引文件
     */
    public boolean putKey(final String key, final long phyOffset, final long storeTimestamp) {
        if (this.indexHeader.getIndexCount() < this.indexNum) {
            int keyHash = key.hashCode();
            // Math.abs计算结果依旧为负
            if (Integer.MIN_VALUE == keyHash)
                keyHash = 0;
            int slotPos = Math.abs(keyHash) % this.hashSlotNum;
            int absSlotPos = IndexHeader.INDEX_HEADER_SIZE + slotPos * HASH_SLOT_SIZE;

            FileLock fileLock = null;

            try {
                // TODO 是否是读写锁
                fileLock = this.fileChannel.lock(absSlotPos, HASH_SLOT_SIZE, false);
                int slotValue = this.mappedByteBuffer.getInt(absSlotPos);
                if (slotValue <= INVALID_INDEX || slotValue > this.indexHeader.getIndexCount()) {
                    slotValue = INVALID_INDEX;
                }

                long timeDiff = storeTimestamp - this.indexHeader.getBeginTimestamp();
                if (this.indexHeader.getBeginTimestamp() <= 0) {
                    timeDiff = 0;
                }
                else if (timeDiff > Integer.MAX_VALUE) {
                    timeDiff = Integer.MAX_VALUE;
                }
                else if (timeDiff < 0) {
                    timeDiff = 0;
                }

                // 时间差存储单位由毫秒改为秒
                timeDiff = timeDiff / 1000;

                int absIndexPos =
                        IndexHeader.INDEX_HEADER_SIZE + this.hashSlotNum * HASH_SLOT_SIZE
                                + this.indexHeader.getIndexCount() * INDEX_SIZE;

                // 写入真正索引
                this.mappedByteBuffer.putInt(absIndexPos, keyHash);
                this.mappedByteBuffer.putLong(absIndexPos + 4, phyOffset);
                this.mappedByteBuffer.putInt(absIndexPos + 4 + 8, (int) timeDiff);
                this.mappedByteBuffer.putInt(absIndexPos + 4 + 8 + 4, slotValue);

                // 更新哈希槽
                this.mappedByteBuffer.putInt(absSlotPos, this.indexHeader.getIndexCount());

                // 第一次写入
                if (this.indexHeader.getIndexCount() <= 1) {
                    this.indexHeader.setBeginPhyOffset(phyOffset);
                    this.indexHeader.setBeginTimestamp(storeTimestamp);
                }

                this.indexHeader.incHashSlotCount();
                this.indexHeader.incIndexCount();
                this.indexHeader.setEndPhyOffset(phyOffset);
                this.indexHeader.setEndTimestamp(storeTimestamp);

                return true;
            }
            catch (Exception e) {
                log.error("putKey exception, Key: " + key + " KeyHashCode: " + key.hashCode(), e);
            }
            finally {
                if (fileLock != null) {
                    try {
                        fileLock.release();
                    }
                    catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
        else {
            log.warn("putKey index count " + this.indexHeader.getIndexCount() + " index max num "
                    + this.indexNum);
        }

        return false;
    }


    public long getBeginTimestamp() {
        return this.indexHeader.getBeginTimestamp();
    }


    public long getEndTimestamp() {
        return this.indexHeader.getEndTimestamp();
    }


    public long getEndPhyOffset() {
        return this.indexHeader.getEndPhyOffset();
    }


    /**
     * 时间区间是否匹配
     */
    public boolean isTimeMatched(final long begin, final long end) {
        boolean result =
                begin < this.indexHeader.getBeginTimestamp() && end > this.indexHeader.getEndTimestamp();

        result =
                result
                        || (begin >= this.indexHeader.getBeginTimestamp() && begin <= this.indexHeader
                            .getEndTimestamp());

        result =
                result
                        || (end >= this.indexHeader.getBeginTimestamp() && end <= this.indexHeader
                            .getEndTimestamp());
        return result;
    }


    /**
     * 前提：入参时间区间在调用前已经匹配了当前索引文件的起始结束时间
     */
    public void selectPhyOffset(final List phyOffsets, final String key, final int maxNum,
            final long begin, final long end, boolean lock) {
        if (this.mapedFile.hold()) {
            int keyHash = key.hashCode();
            if (Integer.MIN_VALUE == keyHash)
                keyHash = 0;
            int slotPos = Math.abs(keyHash) % this.hashSlotNum;
            int absSlotPos = IndexHeader.INDEX_HEADER_SIZE + slotPos * HASH_SLOT_SIZE;

            FileLock fileLock = null;
            try {
                if (lock) {
                    fileLock = this.fileChannel.lock(absSlotPos, HASH_SLOT_SIZE, true);
                }

                int slotValue = this.mappedByteBuffer.getInt(absSlotPos);
                if (fileLock != null) {
                    fileLock.release();
                    fileLock = null;
                }

                if (slotValue <= INVALID_INDEX || slotValue > this.indexHeader.getIndexCount()
                        || this.indexHeader.getIndexCount() <= 1) {
                    // TODO NOTFOUND
                }
                else {
                    for (int nextIndexToRead = slotValue;;) {
                        if (phyOffsets.size() >= maxNum) {
                            break;
                        }

                        int absIndexPos =
                                IndexHeader.INDEX_HEADER_SIZE + this.hashSlotNum * HASH_SLOT_SIZE
                                        + nextIndexToRead * INDEX_SIZE;

                        int keyHashRead = this.mappedByteBuffer.getInt(absIndexPos);
                        long phyOffsetRead = this.mappedByteBuffer.getLong(absIndexPos + 4);
                        int timeDiff = this.mappedByteBuffer.getInt(absIndexPos + 4 + 8);
                        int prevIndexRead = this.mappedByteBuffer.getInt(absIndexPos + 4 + 8 + 4);

                        // 读到了未知数据
                        if (timeDiff < 0) {
                            break;
                        }

                        // 时间差存储的是秒，再还原为毫秒
                        timeDiff *= 1000;

                        long timeRead = this.indexHeader.getBeginTimestamp() + timeDiff;
                        boolean timeMatched = (timeRead >= begin) && (timeRead <= end);

                        if (keyHash == keyHashRead && timeMatched) {
                            phyOffsets.add(phyOffsetRead);
                        }

                        if (prevIndexRead <= INVALID_INDEX
                                || prevIndexRead > this.indexHeader.getIndexCount()
                                || prevIndexRead == nextIndexToRead || timeRead < begin) {
                            break;
                        }

                        nextIndexToRead = prevIndexRead;
                    }
                }
            }
            catch (Exception e) {
                log.error("selectPhyOffset exception ", e);
            }
            finally {
                if (fileLock != null) {
                    try {
                        fileLock.release();
                    }
                    catch (IOException e) {
                        e.printStackTrace();
                    }
                }

                this.mapedFile.release();
            }
        }
    }
}