Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* An abstract raw erasure decoder that's to be inherited by new decoders.
*
* Raw erasure coder is part of erasure codec framework, where erasure coder is
* used to encode/decode a group of blocks (BlockGroup) according to the codec
* specific BlockGroup layout and logic. An erasure coder extracts chunks of
* data from the blocks and can employ various low level raw erasure coders to
* perform encoding/decoding against the chunks.
*
* To distinguish from erasure coder, here raw erasure coder is used to mean the
* low level constructs, since it only takes care of the math calculation with
* a group of byte buffers.
*
* Note it mainly provides decode() calls, which should be stateless and may be
* made thread-safe in future.
*/
@InterfaceAudience.Private
public abstract class RawErasureDecoder {
private final ErasureCoderOptions coderOptions;
public RawErasureDecoder(ErasureCoderOptions coderOptions) {
this.coderOptions = coderOptions;
}
/**
* Decode with inputs and erasedIndexes, generates outputs.
* How to prepare for inputs:
* 1. Create an array containing data units + parity units. Please note the
* data units should be first or before the parity units.
* 2. Set null in the array locations specified via erasedIndexes to indicate
* they're erased and no data are to read from;
* 3. Set null in the array locations for extra redundant items, as they're
* not necessary to read when decoding. For example in RS-6-3, if only 1
* unit is really erased, then we have 2 extra items as redundant. They can
* be set as null to indicate no data will be used from them.
*
* For an example using RS (6, 3), assuming sources (d0, d1, d2, d3, d4, d5)
* and parities (p0, p1, p2), d2 being erased. We can and may want to use only
* 6 units like (d1, d3, d4, d5, p0, p2) to recover d2. We will have:
* inputs = [null(d0), d1, null(d2), d3, d4, d5, p0, null(p1), p2]
* erasedIndexes = [2] // index of d2 into inputs array
* outputs = [a-writable-buffer]
*
* Note, for both inputs and outputs, no mixing of on-heap buffers and direct
* buffers are allowed.
*
* If the coder option ALLOW_CHANGE_INPUTS is set true (false by default), the
* content of input buffers may change after the call, subject to concrete
* implementation.
*
* @param inputs input buffers to read data from. The buffers' remaining will
* be 0 after decoding
* @param erasedIndexes indexes of erased units in the inputs array
* @param outputs output buffers to put decoded data into according to
* erasedIndexes, ready for read after the call
*/
public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
ByteBuffer[] outputs) throws IOException {
ByteBufferDecodingState decodingState = new ByteBufferDecodingState(this,
inputs, erasedIndexes, outputs);
boolean usingDirectBuffer = decodingState.usingDirectBuffer;
int dataLen = decodingState.decodeLength;
if (dataLen == 0) {
return;
}
int[] inputPositions = new int[inputs.length];
for (int i = 0; i < inputPositions.length; i++) {
if (inputs[i] != null) {
inputPositions[i] = inputs[i].position();
}
}
if (usingDirectBuffer) {
doDecode(decodingState);
} else {
ByteArrayDecodingState badState = decodingState.convertToByteArrayState();
doDecode(badState);
}
for (int i = 0; i < inputs.length; i++) {
if (inputs[i] != null) {
// dataLen bytes consumed
inputs[i].position(inputPositions[i] + dataLen);
}
}
}
/**
* Perform the real decoding using Direct ByteBuffer.
* @param decodingState the decoding state
*/
protected abstract void doDecode(ByteBufferDecodingState decodingState)
throws IOException;
/**
* Decode with inputs and erasedIndexes, generates outputs. More see above.
*
* @param inputs input buffers to read data from
* @param erasedIndexes indexes of erased units in the inputs array
* @param outputs output buffers to put decoded data into according to
* erasedIndexes, ready for read after the call
* @throws IOException if the decoder is closed.
*/
public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs)
throws IOException {
ByteArrayDecodingState decodingState = new ByteArrayDecodingState(this,
inputs, erasedIndexes, outputs);
if (decodingState.decodeLength == 0) {
return;
}
doDecode(decodingState);
}
/**
* Perform the real decoding using bytes array, supporting offsets and
* lengths.
* @param decodingState the decoding state
* @throws IOException if the decoder is closed.
*/
protected abstract void doDecode(ByteArrayDecodingState decodingState)
throws IOException;
/**
* Decode with inputs and erasedIndexes, generates outputs. More see above.
*
* Note, for both input and output ECChunks, no mixing of on-heap buffers and
* direct buffers are allowed.
*
* @param inputs input buffers to read data from
* @param erasedIndexes indexes of erased units in the inputs array
* @param outputs output buffers to put decoded data into according to
* erasedIndexes, ready for read after the call
* @throws IOException if the decoder is closed
*/
public void decode(ECChunk[] inputs, int[] erasedIndexes,
ECChunk[] outputs) throws IOException {
ByteBuffer[] newInputs = CoderUtil.toBuffers(inputs);
ByteBuffer[] newOutputs = CoderUtil.toBuffers(outputs);
decode(newInputs, erasedIndexes, newOutputs);
}
public int getNumDataUnits() {
return coderOptions.getNumDataUnits();
}
public int getNumParityUnits() {
return coderOptions.getNumParityUnits();
}
protected int getNumAllUnits() {
return coderOptions.getNumAllUnits();
}
/**
* Tell if direct buffer is preferred or not. It's for callers to
* decide how to allocate coding chunk buffers, using DirectByteBuffer or
* bytes array. It will return false by default.
* @return true if native buffer is preferred for performance consideration,
* otherwise false.
*/
public boolean preferDirectBuffer() {
return false;
}
/**
* Allow change into input buffers or not while perform encoding/decoding.
* @return true if it's allowed to change inputs, false otherwise
*/
public boolean allowChangeInputs() {
return coderOptions.allowChangeInputs();
}
/**
* Allow to dump verbose info during encoding/decoding.
* @return true if it's allowed to do verbose dump, false otherwise.
*/
public boolean allowVerboseDump() {
return coderOptions.allowVerboseDump();
}
/**
* Should be called when release this coder. Good chance to release encoding
* or decoding buffers
*/
public void release() {
// Nothing to do here.
}
}
