org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata Maven / Gradle / Ivy
Show all versions of cassandra-all Show documentation
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.disk.v1.segment;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Stream;
import com.google.common.collect.ImmutableMap;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.v1.MetadataSource;
import org.apache.cassandra.index.sai.disk.v1.MetadataWriter;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexOutput;
/**
* Multiple {@link SegmentMetadata} are stored in {@link IndexComponent#META} file, each corresponds to an on-disk
* index segment.
*/
public class SegmentMetadata
{
private static final String NAME = "SegmentMetadata";
/**
* Used to retrieve sstableRowId which equals to offset plus segmentRowId.
*/
public final long rowIdOffset;
/**
* Min and max sstable rowId in current segment.
*
* For index generated by compaction, minSSTableRowId is the same as segmentRowIdOffset.
* But for flush, segmentRowIdOffset is taken from previous segment's maxSSTableRowId.
*/
public final long minSSTableRowId;
public final long maxSSTableRowId;
/**
* number of indexed rows (aka. a pair of term and segmentRowId) in the current segment
*/
public final long numRows;
/**
* Ordered by their token position in current segment
*/
public final PrimaryKey minKey;
public final PrimaryKey maxKey;
/**
* Minimum and maximum indexed column value ordered by its {@link org.apache.cassandra.db.marshal.AbstractType}.
*/
public final ByteBuffer minTerm;
public final ByteBuffer maxTerm;
/**
* Root, offset, length for each index structure in the segment.
*
* Note: postings block offsets are stored in terms dictionary, no need to worry about its root.
*/
public final ComponentMetadataMap componentMetadatas;
public SegmentMetadata(long rowIdOffset,
long numRows,
long minSSTableRowId,
long maxSSTableRowId,
PrimaryKey minKey,
PrimaryKey maxKey,
ByteBuffer minTerm,
ByteBuffer maxTerm,
ComponentMetadataMap componentMetadatas)
{
assert numRows < Integer.MAX_VALUE;
Objects.requireNonNull(minKey);
Objects.requireNonNull(maxKey);
Objects.requireNonNull(minTerm);
Objects.requireNonNull(maxTerm);
this.rowIdOffset = rowIdOffset;
this.minSSTableRowId = minSSTableRowId;
this.maxSSTableRowId = maxSSTableRowId;
this.numRows = numRows;
this.minKey = minKey;
this.maxKey = maxKey;
this.minTerm = minTerm;
this.maxTerm = maxTerm;
this.componentMetadatas = componentMetadatas;
}
private SegmentMetadata(DataInput input, PrimaryKey.Factory primaryKeyFactory) throws IOException
{
this.rowIdOffset = input.readLong();
this.numRows = input.readLong();
this.minSSTableRowId = input.readLong();
this.maxSSTableRowId = input.readLong();
this.minKey = primaryKeyFactory.fromComparableBytes(ByteSource.fixedLength(readBytes(input)));
this.maxKey = primaryKeyFactory.fromComparableBytes(ByteSource.fixedLength(readBytes(input)));
this.minTerm = readBytes(input);
this.maxTerm = readBytes(input);
this.componentMetadatas = new ComponentMetadataMap(input);
}
public int toSegmentRowId(long sstableRowId)
{
return Math.toIntExact(sstableRowId - rowIdOffset);
}
public static List load(MetadataSource source, PrimaryKey.Factory primaryKeyFactory) throws IOException
{
DataInput input = source.get(NAME);
int segmentCount = input.readVInt();
List segmentMetadata = new ArrayList<>(segmentCount);
for (int i = 0; i < segmentCount; i++)
{
segmentMetadata.add(new SegmentMetadata(input, primaryKeyFactory));
}
return segmentMetadata;
}
/**
* Writes disk metadata for the given segment list.
*/
public static void write(MetadataWriter writer, List segments) throws IOException
{
try (IndexOutput output = writer.builder(NAME))
{
output.writeVInt(segments.size());
for (SegmentMetadata metadata : segments)
{
output.writeLong(metadata.rowIdOffset);
output.writeLong(metadata.numRows);
output.writeLong(metadata.minSSTableRowId);
output.writeLong(metadata.maxSSTableRowId);
Stream.of(ByteSourceInverse.readBytes(metadata.minKey.asComparableBytes(ByteComparable.Version.OSS50)),
ByteSourceInverse.readBytes(metadata.maxKey.asComparableBytes(ByteComparable.Version.OSS50)))
.forEach(b -> writeBytes(b, output));
Stream.of(metadata.minTerm, metadata.maxTerm).forEach(bb -> writeBytes(bb, output));
metadata.componentMetadatas.write(output);
}
}
}
@Override
public String toString()
{
return "SegmentMetadata{" +
"rowIdOffset=" + rowIdOffset +
", minSSTableRowId=" + minSSTableRowId +
", maxSSTableRowId=" + maxSSTableRowId +
", numRows=" + numRows +
", componentMetadatas=" + componentMetadatas +
'}';
}
private static ByteBuffer readBytes(DataInput input) throws IOException
{
int len = input.readInt();
byte[] bytes = new byte[len];
input.readBytes(bytes, 0, len);
return ByteBuffer.wrap(bytes);
}
private static void writeBytes(ByteBuffer buf, IndexOutput out)
{
try
{
byte[] bytes = ByteBufferUtil.getArray(buf);
out.writeInt(bytes.length);
out.writeBytes(bytes, 0, bytes.length);
}
catch (IOException e)
{
throw new UncheckedIOException(e);
}
}
private static void writeBytes(byte[] bytes, IndexOutput out)
{
try
{
out.writeInt(bytes.length);
out.writeBytes(bytes, 0, bytes.length);
}
catch (IOException ioe)
{
throw new RuntimeException(ioe);
}
}
long getIndexRoot(IndexComponent indexComponent)
{
return componentMetadatas.get(indexComponent).root;
}
public static class ComponentMetadataMap
{
private final Map metas = new EnumMap<>(IndexComponent.class);
ComponentMetadataMap(DataInput input) throws IOException
{
int size = input.readInt();
for (int i = 0; i < size; i++)
{
metas.put(IndexComponent.valueOf(input.readString()), new ComponentMetadata(input));
}
}
public ComponentMetadataMap()
{
}
public void put(IndexComponent indexComponent, long root, long offset, long length)
{
metas.put(indexComponent, new ComponentMetadata(root, offset, length));
}
public void put(IndexComponent indexComponent, long root, long offset, long length, Map additionalMap)
{
metas.put(indexComponent, new ComponentMetadata(root, offset, length, additionalMap));
}
private void write(IndexOutput output) throws IOException
{
output.writeInt(metas.size());
for (Map.Entry entry : metas.entrySet())
{
output.writeString(entry.getKey().name());
entry.getValue().write(output);
}
}
public ComponentMetadata get(IndexComponent indexComponent)
{
if (!metas.containsKey(indexComponent))
throw new IllegalArgumentException(indexComponent + " ComponentMetadata not found");
return metas.get(indexComponent);
}
public Map> asMap()
{
Map> metaAttributes = new HashMap<>();
for (Map.Entry entry : metas.entrySet())
{
String name = entry.getKey().name();
ComponentMetadata metadata = entry.getValue();
Map componentAttributes = metadata.asMap();
assert !metaAttributes.containsKey(name) : "Found duplicate index type: " + name;
metaAttributes.put(name, componentAttributes);
}
return metaAttributes;
}
@Override
public String toString()
{
return "ComponentMetadataMap{" +
"metas=" + metas +
'}';
}
public double indexSize()
{
return metas.values().stream().mapToLong(meta -> meta.length).sum();
}
}
public static class ComponentMetadata
{
public static final String ROOT = "Root";
public static final String OFFSET = "Offset";
public static final String LENGTH = "Length";
public final long root;
public final long offset;
public final long length;
public final Map attributes;
ComponentMetadata(long root, long offset, long length)
{
this.root = root;
this.offset = offset;
this.length = length;
this.attributes = Collections.emptyMap();
}
ComponentMetadata(long root, long offset, long length, Map attributes)
{
this.root = root;
this.offset = offset;
this.length = length;
this.attributes = attributes;
}
ComponentMetadata(DataInput input) throws IOException
{
this.root = input.readLong();
this.offset = input.readLong();
this.length = input.readLong();
int size = input.readInt();
attributes = new HashMap<>(size);
for (int x=0; x < size; x++)
{
String key = input.readString();
String value = input.readString();
attributes.put(key, value);
}
}
public void write(IndexOutput output) throws IOException
{
output.writeLong(root);
output.writeLong(offset);
output.writeLong(length);
output.writeInt(attributes.size());
for (Map.Entry entry : attributes.entrySet())
{
output.writeString(entry.getKey());
output.writeString(entry.getValue());
}
}
@Override
public String toString()
{
return String.format("ComponentMetadata{root=%d, offset=%d, length=%d, attributes=%s}", root, offset, length, attributes.toString());
}
public Map asMap()
{
return ImmutableMap.builder().putAll(attributes).put(OFFSET, Long.toString(offset)).put(LENGTH, Long.toString(length)).put(ROOT, Long.toString(root)).build();
}
}
}