All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.hadoop.hbase.wal.WALEdit Maven / Gradle / Ivy

There is a newer version: 3.0.0-beta-1
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.wal;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.codec.Codec;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.yetus.audience.InterfaceAudience;

import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor;

/**
 * Used in HBase's transaction log (WAL) to represent a collection of edits (Cell/KeyValue objects)
 * that came in as a single transaction. All the edits for a given transaction are written out as a
 * single record, in PB format, followed (optionally) by Cells written via the WALCellEncoder.
 * 

This class is LimitedPrivate for CPs to read-only. The {@link #add} methods are * classified as private methods, not for use by CPs.

* *

A particular WALEdit 'type' is the 'meta' type used to mark key operational * events in the WAL such as compaction, flush, or region open. These meta types do not traverse * hbase memstores. They are edits made by the hbase system rather than edit data submitted by * clients. They only show in the WAL. These 'Meta' types have not been formally specified * (or made into an explicit class type). They evolved organically. HBASE-8457 suggests codifying * a WALEdit 'type' by adding a type field to WALEdit that gets serialized into the WAL. TODO. * Would have to work on the consumption-side. Reading WALs on replay we seem to consume * a Cell-at-a-time rather than by WALEdit. We are already in the below going out of our * way to figure particular types -- e.g. if a compaction, replay, or close meta Marker -- during * normal processing so would make sense to do this. Current system is an awkward marking of Cell * columnfamily as {@link #METAFAMILY} and then setting qualifier based off meta edit type. For * replay-time where we read Cell-at-a-time, there are utility methods below for figuring * meta type. See also * {@link #createBulkLoadEvent(RegionInfo, WALProtos.BulkLoadDescriptor)}, etc., for where we * create meta WALEdit instances.

* *

WALEdit will accumulate a Set of all column family names referenced by the Cells * {@link #add(Cell)}'d. This is an optimization. Usually when loading a WALEdit, we have the * column family name to-hand.. just shove it into the WALEdit if available. Doing this, we can * save on a parse of each Cell to figure column family down the line when we go to add the * WALEdit to the WAL file. See the hand-off in FSWALEntry Constructor. * @see WALKey */ // TODO: Do not expose this class to Coprocessors. It has set methods. A CP might meddle. @InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.REPLICATION, HBaseInterfaceAudience.COPROC }) public class WALEdit implements HeapSize { // Below defines are for writing WALEdit 'meta' Cells.. // TODO: Get rid of this system of special 'meta' Cells. See HBASE-8457. It suggests // adding a type to WALEdit itself for use denoting meta Edits and their types. public static final byte [] METAFAMILY = Bytes.toBytes("METAFAMILY"); /** * @deprecated Since 2.3.0. Not used. */ @Deprecated public static final byte [] METAROW = Bytes.toBytes("METAROW"); /** * @deprecated Since 2.3.0. Make it protected, internal-use only. Use * {@link #isCompactionMarker(Cell)} */ @Deprecated @InterfaceAudience.Private public static final byte[] COMPACTION = Bytes.toBytes("HBASE::COMPACTION"); /** * @deprecated Since 2.3.0. Make it protected, internal-use only. */ @Deprecated @InterfaceAudience.Private public static final byte [] FLUSH = Bytes.toBytes("HBASE::FLUSH"); /** * Qualifier for region event meta 'Marker' WALEdits start with the * {@link #REGION_EVENT_PREFIX} prefix ('HBASE::REGION_EVENT::'). After the prefix, * we note the type of the event which we get from the RegionEventDescriptor protobuf * instance type (A RegionEventDescriptor protobuf instance is written as the meta Marker * Cell value). Adding a type suffix means we do not have to deserialize the protobuf to * figure out what type of event this is.. .just read the qualifier suffix. For example, * a close region event descriptor will have a qualifier of HBASE::REGION_EVENT::REGION_CLOSE. * See WAL.proto and the EventType in RegionEventDescriptor protos for all possible * event types. */ private static final String REGION_EVENT_STR = "HBASE::REGION_EVENT"; private static final String REGION_EVENT_PREFIX_STR = REGION_EVENT_STR + "::"; private static final byte [] REGION_EVENT_PREFIX = Bytes.toBytes(REGION_EVENT_PREFIX_STR); /** * @deprecated Since 2.3.0. Remove. Not for external use. Not used. */ @Deprecated public static final byte [] REGION_EVENT = Bytes.toBytes(REGION_EVENT_STR); /** * We use this define figuring if we are carrying a close event. */ private static final byte [] REGION_EVENT_CLOSE = createRegionEventDescriptorQualifier(RegionEventDescriptor.EventType.REGION_CLOSE); @InterfaceAudience.Private public static final byte [] BULK_LOAD = Bytes.toBytes("HBASE::BULK_LOAD"); private final transient boolean replay; private ArrayList cells; /** * All the Cell families in cells. Updated by {@link #add(Cell)} and * {@link #add(Map)}. This Set is passed to the FSWALEntry so it does not have * to recalculate the Set of families in a transaction; makes for a bunch of CPU savings. */ private Set families = null; public WALEdit() { this(1, false); } /** * @deprecated since 2.0.1 and will be removed in 4.0.0. Use {@link #WALEdit(int, boolean)} * instead. * @see #WALEdit(int, boolean) * @see HBASE-20781 */ @Deprecated public WALEdit(boolean replay) { this(1, replay); } /** * @deprecated since 2.0.1 and will be removed in 4.0.0. Use {@link #WALEdit(int, boolean)} * instead. * @see #WALEdit(int, boolean) * @see HBASE-20781 */ @Deprecated public WALEdit(int cellCount) { this(cellCount, false); } /** * @param cellCount Pass so can pre-size the WALEdit. Optimization. */ public WALEdit(int cellCount, boolean isReplay) { this.replay = isReplay; cells = new ArrayList<>(cellCount); } private Set getOrCreateFamilies() { if (this.families == null) { this.families = new TreeSet<>(Bytes.BYTES_COMPARATOR); } return this.families; } /** * For use by FSWALEntry ONLY. An optimization. * @return All families in {@link #getCells()}; may be null. */ public Set getFamilies() { return this.families; } /** * @return True is f is {@link #METAFAMILY} * @deprecated Since 2.3.0. Do not expose. Make protected. */ @Deprecated public static boolean isMetaEditFamily(final byte [] f) { return Bytes.equals(METAFAMILY, f); } /** * Replaying WALs can read Cell-at-a-time so need this method in those cases. */ public static boolean isMetaEditFamily(Cell cell) { return CellUtil.matchingFamily(cell, METAFAMILY); } /** * @return True if this is a meta edit; has one edit only and its columnfamily * is {@link #METAFAMILY}. */ public boolean isMetaEdit() { return this.families != null && this.families.size() == 1 && this.families.contains(METAFAMILY); } /** * @return True when current WALEdit is created by log replay. Replication skips WALEdits from * replay. */ public boolean isReplay() { return this.replay; } @InterfaceAudience.Private public WALEdit add(Cell cell, byte [] family) { getOrCreateFamilies().add(family); return addCell(cell); } @InterfaceAudience.Private public WALEdit add(Cell cell) { // We clone Family each time we add a Cell. Expensive but safe. For CPU savings, use // add(Map) or add(Cell, family). return add(cell, CellUtil.cloneFamily(cell)); } public boolean isEmpty() { return cells.isEmpty(); } public int size() { return cells.size(); } public ArrayList getCells() { return cells; } /** * This is not thread safe. * This will change the WALEdit and shouldn't be used unless you are sure that nothing * else depends on the contents being immutable. * * @param cells the list of cells that this WALEdit now contains. */ @InterfaceAudience.Private // Used by replay. public void setCells(ArrayList cells) { this.cells = cells; this.families = null; } /** * Reads WALEdit from cells. * @param cellDecoder Cell decoder. * @param expectedCount Expected cell count. * @return Number of KVs read. */ public int readFromCells(Codec.Decoder cellDecoder, int expectedCount) throws IOException { cells.clear(); cells.ensureCapacity(expectedCount); while (cells.size() < expectedCount && cellDecoder.advance()) { add(cellDecoder.current()); } return cells.size(); } @Override public long heapSize() { long ret = ClassSize.ARRAYLIST; for (Cell cell : cells) { ret += cell.heapSize(); } return ret; } public long estimatedSerializedSizeOf() { long ret = 0; for (Cell cell: cells) { ret += PrivateCellUtil.estimatedSerializedSizeOf(cell); } return ret; } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("[#edits: ").append(cells.size()).append(" = <"); for (Cell cell : cells) { sb.append(cell); sb.append("; "); } sb.append(">]"); return sb.toString(); } public static WALEdit createFlushWALEdit(RegionInfo hri, FlushDescriptor f) { KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, FLUSH, EnvironmentEdgeManager.currentTime(), f.toByteArray()); return new WALEdit().add(kv, METAFAMILY); } public static FlushDescriptor getFlushDescriptor(Cell cell) throws IOException { return CellUtil.matchingColumn(cell, METAFAMILY, FLUSH)? FlushDescriptor.parseFrom(CellUtil.cloneValue(cell)): null; } /** * @return A meta Marker WALEdit that has a single Cell whose value is the passed in * regionEventDesc serialized and whose row is this region, * columnfamily is {@link #METAFAMILY} and qualifier is * {@link #REGION_EVENT_PREFIX} + {@link RegionEventDescriptor#getEventType()}; * for example HBASE::REGION_EVENT::REGION_CLOSE. */ public static WALEdit createRegionEventWALEdit(RegionInfo hri, RegionEventDescriptor regionEventDesc) { return createRegionEventWALEdit(getRowForRegion(hri), regionEventDesc); } @InterfaceAudience.Private public static WALEdit createRegionEventWALEdit(byte [] rowForRegion, RegionEventDescriptor regionEventDesc) { KeyValue kv = new KeyValue(rowForRegion, METAFAMILY, createRegionEventDescriptorQualifier(regionEventDesc.getEventType()), EnvironmentEdgeManager.currentTime(), regionEventDesc.toByteArray()); return new WALEdit().add(kv, METAFAMILY); } /** * @return Cell qualifier for the passed in RegionEventDescriptor Type; e.g. we'll * return something like a byte array with HBASE::REGION_EVENT::REGION_OPEN in it. */ @InterfaceAudience.Private public static byte [] createRegionEventDescriptorQualifier(RegionEventDescriptor.EventType t) { return Bytes.toBytes(REGION_EVENT_PREFIX_STR + t.toString()); } /** * Public so can be accessed from regionserver.wal package. * @return True if this is a Marker Edit and it is a RegionClose type. */ public boolean isRegionCloseMarker() { return isMetaEdit() && PrivateCellUtil.matchingQualifier(this.cells.get(0), REGION_EVENT_CLOSE, 0, REGION_EVENT_CLOSE.length); } /** * @return Returns a RegionEventDescriptor made by deserializing the content of the * passed in cell, IFF the cell is a RegionEventDescriptor * type WALEdit. */ public static RegionEventDescriptor getRegionEventDescriptor(Cell cell) throws IOException { return CellUtil.matchingColumnFamilyAndQualifierPrefix(cell, METAFAMILY, REGION_EVENT_PREFIX)? RegionEventDescriptor.parseFrom(CellUtil.cloneValue(cell)): null; } /** * @return A Marker WALEdit that has c serialized as its value */ public static WALEdit createCompaction(final RegionInfo hri, final CompactionDescriptor c) { byte [] pbbytes = c.toByteArray(); KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, COMPACTION, EnvironmentEdgeManager.currentTime(), pbbytes); return new WALEdit().add(kv, METAFAMILY); //replication scope null so this won't be replicated } public static byte[] getRowForRegion(RegionInfo hri) { byte[] startKey = hri.getStartKey(); if (startKey.length == 0) { // empty row key is not allowed in mutations because it is both the start key and the end key // we return the smallest byte[] that is bigger (in lex comparison) than byte[0]. return new byte[] {0}; } return startKey; } /** * Deserialized and returns a CompactionDescriptor is the KeyValue contains one. * @param kv the key value * @return deserialized CompactionDescriptor or null. */ public static CompactionDescriptor getCompaction(Cell kv) throws IOException { return isCompactionMarker(kv)? CompactionDescriptor.parseFrom(CellUtil.cloneValue(kv)): null; } /** * Returns true if the given cell is a serialized {@link CompactionDescriptor} * * @see #getCompaction(Cell) */ public static boolean isCompactionMarker(Cell cell) { return CellUtil.matchingColumn(cell, METAFAMILY, COMPACTION); } /** * Create a bulk loader WALEdit * * @param hri The RegionInfo for the region in which we are bulk loading * @param bulkLoadDescriptor The descriptor for the Bulk Loader * @return The WALEdit for the BulkLoad */ public static WALEdit createBulkLoadEvent(RegionInfo hri, WALProtos.BulkLoadDescriptor bulkLoadDescriptor) { KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, BULK_LOAD, EnvironmentEdgeManager.currentTime(), bulkLoadDescriptor.toByteArray()); return new WALEdit().add(kv, METAFAMILY); } /** * Deserialized and returns a BulkLoadDescriptor from the passed in Cell * @param cell the key value * @return deserialized BulkLoadDescriptor or null. */ public static WALProtos.BulkLoadDescriptor getBulkLoadDescriptor(Cell cell) throws IOException { return CellUtil.matchingColumn(cell, METAFAMILY, BULK_LOAD)? WALProtos.BulkLoadDescriptor.parseFrom(CellUtil.cloneValue(cell)): null; } /** * Append the given map of family->edits to a WALEdit data structure. * This does not write to the WAL itself. * Note that as an optimization, we will stamp the Set of column families into the WALEdit * to save on our having to calculate column families subsequently down in the actual WAL * writing. * * @param familyMap map of family->edits */ public void add(Map> familyMap) { for (Map.Entry> e: familyMap.entrySet()) { // 'foreach' loop NOT used. See HBASE-12023 "...creates too many iterator objects." int listSize = e.getValue().size(); // Add all Cells first and then at end, add the family rather than call {@link #add(Cell)} // and have it clone family each time. Optimization! for (int i = 0; i < listSize; i++) { addCell(e.getValue().get(i)); } addFamily(e.getKey()); } } private void addFamily(byte [] family) { getOrCreateFamilies().add(family); } private WALEdit addCell(Cell cell) { this.cells.add(cell); return this; } }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy