org.apache.hadoop.hbase.regionserver.AbstractMultiFileWriter Maven / Gradle / Ivy
Show all versions of hbase-server Show documentation
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.apache.hadoop.fs.Path;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Base class for cell sink that separates the provided cells into multiple files.
*/
@InterfaceAudience.Private
public abstract class AbstractMultiFileWriter implements CellSink, ShipperListener {
private static final Logger LOG = LoggerFactory.getLogger(AbstractMultiFileWriter.class);
/** Factory that is used to produce single StoreFile.Writer-s */
protected WriterFactory writerFactory;
/** Source scanner that is tracking KV count; may be null if source is not StoreScanner */
protected StoreScanner sourceScanner;
public interface WriterFactory {
public StoreFileWriter createWriter() throws IOException;
default StoreFileWriter createWriterWithStoragePolicy(String fileStoragePolicy)
throws IOException {
return createWriter();
};
}
/**
* Initializes multi-writer before usage.
* @param sourceScanner Optional store scanner to obtain the information about read progress.
* @param factory Factory used to produce individual file writers.
*/
public void init(StoreScanner sourceScanner, WriterFactory factory) {
this.writerFactory = factory;
this.sourceScanner = sourceScanner;
}
/**
* Commit all writers.
*
* Notice that here we use the same maxSeqId for all output files since we haven't
* find an easy to find enough sequence ids for different output files in some corner cases. See
* comments in HBASE-15400 for more details.
*/
public List commitWriters(long maxSeqId, boolean majorCompaction) throws IOException {
return commitWriters(maxSeqId, majorCompaction, Collections.EMPTY_SET);
}
public List commitWriters(long maxSeqId, boolean majorCompaction,
Collection storeFiles) throws IOException {
preCommitWriters();
Collection writers = this.writers();
if (LOG.isDebugEnabled()) {
LOG.debug(
"Commit " + writers.size() + " writers, maxSeqId=" + maxSeqId + ", majorCompaction=" +
majorCompaction);
}
List paths = new ArrayList<>();
for (StoreFileWriter writer : writers) {
if (writer == null) {
continue;
}
writer.appendMetadata(maxSeqId, majorCompaction, storeFiles);
preCloseWriter(writer);
paths.add(writer.getPath());
writer.close();
}
return paths;
}
/**
* Close all writers without throwing any exceptions. This is used when compaction failed usually.
*/
public List abortWriters() {
List paths = new ArrayList<>();
for (StoreFileWriter writer : writers()) {
try {
if (writer != null) {
paths.add(writer.getPath());
writer.close();
}
} catch (Exception ex) {
LOG.error("Failed to close the writer after an unfinished compaction.", ex);
}
}
return paths;
}
protected abstract Collection writers();
/**
* Subclasses override this method to be called at the end of a successful sequence of append; all
* appends are processed before this method is called.
*/
protected void preCommitWriters() throws IOException {
}
/**
* Subclasses override this method to be called before we close the give writer. Usually you can
* append extra metadata to the writer.
*/
protected void preCloseWriter(StoreFileWriter writer) throws IOException {
}
@Override
public void beforeShipped() throws IOException {
Collection writers = writers();
if (writers != null) {
for (StoreFileWriter writer : writers) {
if (writer != null) {
writer.beforeShipped();
}
}
}
}
}