org.apache.cassandra.io.util.DataIntegrityMetadata Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of cassandra-all Show documentation
Show all versions of cassandra-all Show documentation
A fork of the Apache Cassandra Project ready to embed Elasticsearch.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.io.util;
import java.io.BufferedWriter;
import java.io.Closeable;
import java.io.DataOutput;
import java.io.File;
import java.io.IOError;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.zip.CRC32;
import java.util.zip.CheckedInputStream;
import java.util.zip.Checksum;
import com.google.common.base.Charsets;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.utils.Throwables;
public class DataIntegrityMetadata
{
public static ChecksumValidator checksumValidator(Descriptor desc) throws IOException
{
return new ChecksumValidator(desc);
}
public static class ChecksumValidator implements Closeable
{
private final Checksum checksum;
private final RandomAccessReader reader;
public final int chunkSize;
private final String dataFilename;
public ChecksumValidator(Descriptor descriptor) throws IOException
{
this(descriptor.version.uncompressedChecksumType().newInstance(),
RandomAccessReader.open(new File(descriptor.filenameFor(Component.CRC))),
descriptor.filenameFor(Component.DATA));
}
public ChecksumValidator(Checksum checksum, RandomAccessReader reader, String dataFilename) throws IOException
{
this.checksum = checksum;
this.reader = reader;
this.dataFilename = dataFilename;
chunkSize = reader.readInt();
}
public void seek(long offset)
{
long start = chunkStart(offset);
reader.seek(((start / chunkSize) * 4L) + 4); // 8 byte checksum per chunk + 4 byte header/chunkLength
}
public long chunkStart(long offset)
{
long startChunk = offset / chunkSize;
return startChunk * chunkSize;
}
public void validate(byte[] bytes, int start, int end) throws IOException
{
checksum.update(bytes, start, end);
int current = (int) checksum.getValue();
checksum.reset();
int actual = reader.readInt();
if (current != actual)
throw new IOException("Corrupted File : " + dataFilename);
}
public void close()
{
reader.close();
}
}
public static FileDigestValidator fileDigestValidator(Descriptor desc) throws IOException
{
return new FileDigestValidator(desc);
}
public static class FileDigestValidator implements Closeable
{
private final Checksum checksum;
private final RandomAccessReader digestReader;
private final RandomAccessReader dataReader;
private final Descriptor descriptor;
private long storedDigestValue;
public FileDigestValidator(Descriptor descriptor) throws IOException
{
this.descriptor = descriptor;
checksum = descriptor.version.uncompressedChecksumType().newInstance();
digestReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.digestFor(descriptor.version.uncompressedChecksumType()))));
dataReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DATA)));
try
{
storedDigestValue = Long.parseLong(digestReader.readLine());
}
catch (Exception e)
{
close();
// Attempting to create a FileDigestValidator without a DIGEST file will fail
throw new IOException("Corrupted SSTable : " + descriptor.filenameFor(Component.DATA));
}
}
// Validate the entire file
public void validate() throws IOException
{
CheckedInputStream checkedInputStream = new CheckedInputStream(dataReader, checksum);
byte[] chunk = new byte[64 * 1024];
while( checkedInputStream.read(chunk) > 0 ) { }
long calculatedDigestValue = checkedInputStream.getChecksum().getValue();
if (storedDigestValue != calculatedDigestValue) {
throw new IOException("Corrupted SSTable : " + descriptor.filenameFor(Component.DATA));
}
}
public void close()
{
Throwables.perform(digestReader::close,
dataReader::close);
}
}
public static class ChecksumWriter
{
private final CRC32 incrementalChecksum = new CRC32();
private final DataOutput incrementalOut;
private final CRC32 fullChecksum = new CRC32();
public ChecksumWriter(DataOutput incrementalOut)
{
this.incrementalOut = incrementalOut;
}
public void writeChunkSize(int length)
{
try
{
incrementalOut.writeInt(length);
}
catch (IOException e)
{
throw new IOError(e);
}
}
// checksumIncrementalResult indicates if the checksum we compute for this buffer should itself be
// included in the full checksum, translating to if the partial checksum is serialized along with the
// data it checksums (in which case the file checksum as calculated by external tools would mismatch if
// we did not include it), or independently.
// CompressedSequentialWriters serialize the partial checksums inline with the compressed data chunks they
// corroborate, whereas ChecksummedSequentialWriters serialize them to a different file.
public void appendDirect(ByteBuffer bb, boolean checksumIncrementalResult)
{
try
{
ByteBuffer toAppend = bb.duplicate();
toAppend.mark();
incrementalChecksum.update(toAppend);
toAppend.reset();
int incrementalChecksumValue = (int) incrementalChecksum.getValue();
incrementalOut.writeInt(incrementalChecksumValue);
fullChecksum.update(toAppend);
if (checksumIncrementalResult)
{
ByteBuffer byteBuffer = ByteBuffer.allocate(4);
byteBuffer.putInt(incrementalChecksumValue);
fullChecksum.update(byteBuffer.array(), 0, byteBuffer.array().length);
}
incrementalChecksum.reset();
}
catch (IOException e)
{
throw new IOError(e);
}
}
public void writeFullChecksum(Descriptor descriptor)
{
if (descriptor.digestComponent == null)
throw new NullPointerException("Null digest component for " + descriptor.ksname + '.' + descriptor.cfname + " file " + descriptor.baseFilename());
File outFile = new File(descriptor.filenameFor(descriptor.digestComponent));
try (BufferedWriter out =Files.newBufferedWriter(outFile.toPath(), Charsets.UTF_8))
{
out.write(String.valueOf(fullChecksum.getValue()));
}
catch (IOException e)
{
throw new FSWriteError(e, outFile);
}
}
}
}