org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch subproject :server
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.snapshots.blobstore;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.FromXContentBuilder;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.store.StoreFileMetaData;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* Shard snapshot metadata
*/
public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuilder {
public static final BlobStoreIndexShardSnapshot PROTO = new BlobStoreIndexShardSnapshot();
/**
* Information about snapshotted file
*/
public static class FileInfo {
private final String name;
private final ByteSizeValue partSize;
private final long partBytes;
private final long numberOfParts;
private final StoreFileMetaData metadata;
/**
* Constructs a new instance of file info
*
* @param name file name as stored in the blob store
* @param metaData the files meta data
* @param partSize size of the single chunk
*/
public FileInfo(String name, StoreFileMetaData metaData, ByteSizeValue partSize) {
this.name = name;
this.metadata = metaData;
long partBytes = Long.MAX_VALUE;
if (partSize != null) {
partBytes = partSize.bytes();
}
long totalLength = metaData.length();
long numberOfParts = totalLength / partBytes;
if (totalLength % partBytes > 0) {
numberOfParts++;
}
if (numberOfParts == 0) {
numberOfParts++;
}
this.numberOfParts = numberOfParts;
this.partSize = partSize;
this.partBytes = partBytes;
}
/**
* Returns the base file name
*
* @return file name
*/
public String name() {
return name;
}
/**
* Returns part name if file is stored as multiple parts
*
* @param part part number
* @return part name
*/
public String partName(long part) {
if (numberOfParts > 1) {
return name + ".part" + part;
} else {
return name;
}
}
/**
* Returns base file name from part name
*
* @param blobName part name
* @return base file name
*/
public static String canonicalName(String blobName) {
if (blobName.contains(".part")) {
return blobName.substring(0, blobName.indexOf(".part"));
}
return blobName;
}
/**
* Returns original file name
*
* @return original file name
*/
public String physicalName() {
return metadata.name();
}
/**
* File length
*
* @return file length
*/
public long length() {
return metadata.length();
}
/**
* Returns part size
*
* @return part size
*/
public ByteSizeValue partSize() {
return partSize;
}
/**
* Returns the size (in bytes) of a given part
*
* @return the size (in bytes) of a given part
*/
public long partBytes(int part) {
if (numberOfParts == 1) {
return length();
}
// First and last-but-one parts have a size equal to partBytes
if (part < (numberOfParts - 1)) {
return partBytes;
}
// Last part size is deducted from the length and the number of parts
return length() - (partBytes * (numberOfParts-1));
}
/**
* Returns number of parts
*
* @return number of parts
*/
public long numberOfParts() {
return numberOfParts;
}
/**
* Returns file md5 checksum provided by {@link org.elasticsearch.index.store.Store}
*
* @return file checksum
*/
@Nullable
public String checksum() {
return metadata.checksum();
}
/**
* Returns the StoreFileMetaData for this file info.
*/
public StoreFileMetaData metadata() {
return metadata;
}
/**
* Checks if a file in a store is the same file
*
* @param md file in a store
* @return true if file in a store this this file have the same checksum and length
*/
public boolean isSame(StoreFileMetaData md) {
return metadata.isSame(md);
}
/**
* Checks if a file in a store is the same file
*
* @param fileInfo file in a store
* @return true if file in a store this this file have the same checksum and length
*/
public boolean isSame(FileInfo fileInfo) {
if (numberOfParts != fileInfo.numberOfParts) return false;
if (partBytes != fileInfo.partBytes) return false;
if (!name.equals(fileInfo.name)) return false;
if (partSize != null) {
if (!partSize.equals(fileInfo.partSize)) return false;
} else {
if (fileInfo.partSize != null) return false;
}
return metadata.isSame(fileInfo.metadata);
}
static final class Fields {
static final XContentBuilderString NAME = new XContentBuilderString("name");
static final XContentBuilderString PHYSICAL_NAME = new XContentBuilderString("physical_name");
static final XContentBuilderString LENGTH = new XContentBuilderString("length");
static final XContentBuilderString CHECKSUM = new XContentBuilderString("checksum");
static final XContentBuilderString PART_SIZE = new XContentBuilderString("part_size");
static final XContentBuilderString WRITTEN_BY = new XContentBuilderString("written_by");
static final XContentBuilderString META_HASH = new XContentBuilderString("meta_hash");
}
/**
* Serializes file info into JSON
*
* @param file file info
* @param builder XContent builder
* @param params parameters
*/
public static void toXContent(FileInfo file, XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject();
builder.field(Fields.NAME, file.name);
builder.field(Fields.PHYSICAL_NAME, file.metadata.name());
builder.field(Fields.LENGTH, file.metadata.length());
if (file.metadata.checksum() != null) {
builder.field(Fields.CHECKSUM, file.metadata.checksum());
}
if (file.partSize != null) {
builder.field(Fields.PART_SIZE, file.partSize.bytes());
}
if (file.metadata.writtenBy() != null) {
builder.field(Fields.WRITTEN_BY, file.metadata.writtenBy());
}
if (file.metadata.hash() != null && file.metadata().hash().length > 0) {
builder.field(Fields.META_HASH, file.metadata.hash());
}
builder.endObject();
}
/**
* Parses JSON that represents file info
*
* @param parser parser
* @return file info
*/
public static FileInfo fromXContent(XContentParser parser) throws IOException {
XContentParser.Token token = parser.currentToken();
String name = null;
String physicalName = null;
long length = -1;
String checksum = null;
ByteSizeValue partSize = null;
Version writtenBy = null;
BytesRef metaHash = new BytesRef();
if (token == XContentParser.Token.START_OBJECT) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
String currentFieldName = parser.currentName();
token = parser.nextToken();
if (token.isValue()) {
if ("name".equals(currentFieldName)) {
name = parser.text();
} else if ("physical_name".equals(currentFieldName)) {
physicalName = parser.text();
} else if ("length".equals(currentFieldName)) {
length = parser.longValue();
} else if ("checksum".equals(currentFieldName)) {
checksum = parser.text();
} else if ("part_size".equals(currentFieldName)) {
partSize = new ByteSizeValue(parser.longValue());
} else if ("written_by".equals(currentFieldName)) {
writtenBy = Lucene.parseVersionLenient(parser.text(), null);
} else if ("meta_hash".equals(currentFieldName)) {
metaHash.bytes = parser.binaryValue();
metaHash.offset = 0;
metaHash.length = metaHash.bytes.length;
} else {
throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
}
} else {
throw new ElasticsearchParseException("unexpected token [{}]", token);
}
} else {
throw new ElasticsearchParseException("unexpected token [{}]",token);
}
}
}
// Verify that file information is complete
if (name == null || Strings.validFileName(name) == false) {
throw new ElasticsearchParseException("missing or invalid file name [" + name + "]");
} else if (physicalName == null || Strings.validFileName(physicalName) == false) {
throw new ElasticsearchParseException("missing or invalid physical file name [" + physicalName + "]");
} else if (length < 0) {
throw new ElasticsearchParseException("missing or invalid file length");
}
return new FileInfo(name, new StoreFileMetaData(physicalName, length, checksum, writtenBy, metaHash), partSize);
}
}
private final String snapshot;
private final long indexVersion;
private final long startTime;
private final long time;
private final int numberOfFiles;
private final long totalSize;
private final List indexFiles;
/**
* Constructs new shard snapshot metadata from snapshot metadata
*
* @param snapshot snapshot id
* @param indexVersion index version
* @param indexFiles list of files in the shard
* @param startTime snapshot start time
* @param time snapshot running time
* @param numberOfFiles number of files that where snapshotted
* @param totalSize total size of all files snapshotted
*/
public BlobStoreIndexShardSnapshot(String snapshot, long indexVersion, List indexFiles, long startTime, long time,
int numberOfFiles, long totalSize) {
assert snapshot != null;
assert indexVersion >= 0;
this.snapshot = snapshot;
this.indexVersion = indexVersion;
this.indexFiles = Collections.unmodifiableList(new ArrayList<>(indexFiles));
this.startTime = startTime;
this.time = time;
this.numberOfFiles = numberOfFiles;
this.totalSize = totalSize;
}
/**
* Special constructor for the prototype
*/
private BlobStoreIndexShardSnapshot() {
this.snapshot = "";
this.indexVersion = 0;
this.indexFiles = Collections.emptyList();
this.startTime = 0;
this.time = 0;
this.numberOfFiles = 0;
this.totalSize = 0;
}
/**
* Returns index version
*
* @return index version
*/
public long indexVersion() {
return indexVersion;
}
/**
* Returns snapshot id
*
* @return snapshot id
*/
public String snapshot() {
return snapshot;
}
/**
* Returns list of files in the shard
*
* @return list of files
*/
public List indexFiles() {
return indexFiles;
}
/**
* Returns snapshot start time
*/
public long startTime() {
return startTime;
}
/**
* Returns snapshot running time
*/
public long time() {
return time;
}
/**
* Returns number of files that where snapshotted
*/
public int numberOfFiles() {
return numberOfFiles;
}
/**
* Returns total size of all files that where snapshotted
*/
public long totalSize() {
return totalSize;
}
static final class Fields {
static final XContentBuilderString NAME = new XContentBuilderString("name");
static final XContentBuilderString INDEX_VERSION = new XContentBuilderString("index_version");
static final XContentBuilderString START_TIME = new XContentBuilderString("start_time");
static final XContentBuilderString TIME = new XContentBuilderString("time");
static final XContentBuilderString NUMBER_OF_FILES = new XContentBuilderString("number_of_files");
static final XContentBuilderString TOTAL_SIZE = new XContentBuilderString("total_size");
static final XContentBuilderString FILES = new XContentBuilderString("files");
}
static final class ParseFields {
static final ParseField NAME = new ParseField("name");
static final ParseField INDEX_VERSION = new ParseField("index_version", "index-version");
static final ParseField START_TIME = new ParseField("start_time");
static final ParseField TIME = new ParseField("time");
static final ParseField NUMBER_OF_FILES = new ParseField("number_of_files");
static final ParseField TOTAL_SIZE = new ParseField("total_size");
static final ParseField FILES = new ParseField("files");
}
/**
* Serializes shard snapshot metadata info into JSON
*
* @param builder XContent builder
* @param params parameters
*/
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(Fields.NAME, snapshot);
builder.field(Fields.INDEX_VERSION, indexVersion);
builder.field(Fields.START_TIME, startTime);
builder.field(Fields.TIME, time);
builder.field(Fields.NUMBER_OF_FILES, numberOfFiles);
builder.field(Fields.TOTAL_SIZE, totalSize);
builder.startArray(Fields.FILES);
for (FileInfo fileInfo : indexFiles) {
FileInfo.toXContent(fileInfo, builder, params);
}
builder.endArray();
return builder;
}
/**
* Parses shard snapshot metadata
*
* @param parser parser
* @return shard snapshot metadata
*/
public BlobStoreIndexShardSnapshot fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
String snapshot = null;
long indexVersion = -1;
long startTime = 0;
long time = 0;
int numberOfFiles = 0;
long totalSize = 0;
List indexFiles = new ArrayList<>();
if (parser.currentToken() == null) { // fresh parser? move to the first token
parser.nextToken();
}
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.START_OBJECT) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
String currentFieldName = parser.currentName();
token = parser.nextToken();
if (token.isValue()) {
if (parseFieldMatcher.match(currentFieldName, ParseFields.NAME)) {
snapshot = parser.text();
} else if (parseFieldMatcher.match(currentFieldName, ParseFields.INDEX_VERSION)) {
// The index-version is needed for backward compatibility with v 1.0
indexVersion = parser.longValue();
} else if (parseFieldMatcher.match(currentFieldName, ParseFields.START_TIME)) {
startTime = parser.longValue();
} else if (parseFieldMatcher.match(currentFieldName, ParseFields.TIME)) {
time = parser.longValue();
} else if (parseFieldMatcher.match(currentFieldName, ParseFields.NUMBER_OF_FILES)) {
numberOfFiles = parser.intValue();
} else if (parseFieldMatcher.match(currentFieldName, ParseFields.TOTAL_SIZE)) {
totalSize = parser.longValue();
} else {
throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (parseFieldMatcher.match(currentFieldName, ParseFields.FILES)) {
while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
indexFiles.add(FileInfo.fromXContent(parser));
}
} else {
throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
}
} else {
throw new ElasticsearchParseException("unexpected token [{}]", token);
}
} else {
throw new ElasticsearchParseException("unexpected token [{}]", token);
}
}
}
return new BlobStoreIndexShardSnapshot(snapshot, indexVersion, Collections.unmodifiableList(indexFiles),
startTime, time, numberOfFiles, totalSize);
}
}