All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.hadoop.hbase.io.hfile.BlockCacheUtil Maven / Gradle / Ivy

There is a newer version: 3.0.0-beta-1
Show newest version
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.io.hfile;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.NavigableMap;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.ConcurrentSkipListSet;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram;
import org.apache.hadoop.hbase.util.Bytes;

/**
 * Utilty for aggregating counts in CachedBlocks and toString/toJSON CachedBlocks and BlockCaches.
 * No attempt has been made at making this thread safe.
 */
@InterfaceAudience.Private
public class BlockCacheUtil {


  public static final long NANOS_PER_SECOND = 1000000000;

  /**
   * Needed generating JSON.
   */
  private static final ObjectMapper MAPPER = new ObjectMapper();
  static {
    MAPPER.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    MAPPER.configure(SerializationFeature.FLUSH_AFTER_WRITE_VALUE, true);
    MAPPER.configure(SerializationFeature.INDENT_OUTPUT, true);
  }

  /**
   * @param cb
   * @return The block content as String.
   */
  public static String toString(final CachedBlock cb, final long now) {
    return "filename=" + cb.getFilename() + ", " + toStringMinusFileName(cb, now);
  }

  /**
   * Little data structure to hold counts for a file.
   * Used doing a toJSON.
   */
  static class CachedBlockCountsPerFile {
    private int count = 0;
    private long size = 0;
    private int countData = 0;
    private long sizeData = 0;
    private final String filename;

    CachedBlockCountsPerFile(final String filename) {
      this.filename = filename;
    }

    public int getCount() {
      return count;
    }

    public long getSize() {
      return size;
    }

    public int getCountData() {
      return countData;
    }

    public long getSizeData() {
      return sizeData;
    }

    public String getFilename() {
      return filename;
    }
  }

  /**
   * @param filename
   * @param blocks
   * @return A JSON String of filename and counts of blocks
   * @throws JsonGenerationException
   * @throws JsonMappingException
   * @throws IOException
   */
  public static String toJSON(final String filename, final NavigableSet blocks)
  throws JsonGenerationException, JsonMappingException, IOException {
    CachedBlockCountsPerFile counts = new CachedBlockCountsPerFile(filename);
    for (CachedBlock cb: blocks) {
      counts.count++;
      counts.size += cb.getSize();
      BlockType bt = cb.getBlockType();
      if (bt != null && bt.isData()) {
        counts.countData++;
        counts.sizeData += cb.getSize();
      }
    }
    return MAPPER.writeValueAsString(counts);
  }

  /**
   * @param cbsbf
   * @return JSON string of cbsf aggregated
   * @throws JsonGenerationException
   * @throws JsonMappingException
   * @throws IOException
   */
  public static String toJSON(final CachedBlocksByFile cbsbf)
  throws JsonGenerationException, JsonMappingException, IOException {
    return MAPPER.writeValueAsString(cbsbf);
  }

  /**
   * @param bc
   * @return JSON string of bc content.
   * @throws JsonGenerationException
   * @throws JsonMappingException
   * @throws IOException
   */
  public static String toJSON(final BlockCache bc)
  throws JsonGenerationException, JsonMappingException, IOException {
    return MAPPER.writeValueAsString(bc);
  }

  /**
   * @param cb
   * @return The block content of bc as a String minus the filename.
   */
  public static String toStringMinusFileName(final CachedBlock cb, final long now) {
    return "offset=" + cb.getOffset() +
      ", size=" + cb.getSize() +
      ", age=" + (now - cb.getCachedTime()) +
      ", type=" + cb.getBlockType() +
      ", priority=" + cb.getBlockPriority();
  }

  /**
   * Get a {@link CachedBlocksByFile} instance and load it up by iterating content in
   * {@link BlockCache}.
   * @param conf Used to read configurations
   * @param bc Block Cache to iterate.
   * @return Laoded up instance of CachedBlocksByFile
   */
  public static CachedBlocksByFile getLoadedCachedBlocksByFile(final Configuration conf,
      final BlockCache bc) {
    CachedBlocksByFile cbsbf = new CachedBlocksByFile(conf);
    for (CachedBlock cb: bc) {
      if (cbsbf.update(cb)) break;
    }
    return cbsbf;
  }

  public static int compareCacheBlock(Cacheable left, Cacheable right) {
    ByteBuffer l = ByteBuffer.allocate(left.getSerializedLength());
    left.serialize(l);
    ByteBuffer r = ByteBuffer.allocate(right.getSerializedLength());
    right.serialize(r);
    return Bytes.compareTo(l.array(), l.arrayOffset(), l.limit(),
	      r.array(), r.arrayOffset(), r.limit());
  }

  /**
   * Use one of these to keep a running account of cached blocks by file.  Throw it away when done.
   * This is different than metrics in that it is stats on current state of a cache.
   * See getLoadedCachedBlocksByFile
   */
  @JsonIgnoreProperties({"cachedBlockStatsByFile"})
  public static class CachedBlocksByFile {
    private int count;
    private int dataBlockCount;
    private long size;
    private long dataSize;
    private final long now = System.nanoTime();
    /**
     * How many blocks to look at before we give up.
     * There could be many millions of blocks. We don't want the
     * ui to freeze while we run through 1B blocks... users will
     * think hbase dead. UI displays warning in red when stats
     * are incomplete.
     */
    private final int max;
    public static final int DEFAULT_MAX = 1000000;

    CachedBlocksByFile() {
      this(null);
    }

    CachedBlocksByFile(final Configuration c) {
      this.max = c == null? DEFAULT_MAX: c.getInt("hbase.ui.blockcache.by.file.max", DEFAULT_MAX);
    }

    /**
     * Map by filename. use concurent utils because we want our Map and contained blocks sorted.
     */
    private NavigableMap> cachedBlockByFile = new ConcurrentSkipListMap<>();
    FastLongHistogram hist = new FastLongHistogram();

    /**
     * @param cb
     * @return True if full.... if we won't be adding any more.
     */
    public boolean update(final CachedBlock cb) {
      if (isFull()) return true;
      NavigableSet set = this.cachedBlockByFile.get(cb.getFilename());
      if (set == null) {
        set = new ConcurrentSkipListSet<>();
        this.cachedBlockByFile.put(cb.getFilename(), set);
      }
      set.add(cb);
      this.size += cb.getSize();
      this.count++;
      BlockType bt = cb.getBlockType();
      if (bt != null && bt.isData()) {
        this.dataBlockCount++;
        this.dataSize += cb.getSize();
      }
      long age = (this.now - cb.getCachedTime())/NANOS_PER_SECOND;
      this.hist.add(age, 1);
      return false;
    }

    /**
     * @return True if full; i.e. there are more items in the cache but we only loaded up
     * the maximum set in configuration hbase.ui.blockcache.by.file.max
     * (Default: DEFAULT_MAX).
     */
    public boolean isFull() {
      return this.count >= this.max;
    }

    public NavigableMap> getCachedBlockStatsByFile() {
      return this.cachedBlockByFile;
    }

    /**
     * @return count of blocks in the cache
     */
    public int getCount() {
      return count;
    }

    public int getDataCount() {
      return dataBlockCount;
    }

    /**
     * @return size of blocks in the cache
     */
    public long getSize() {
      return size;
    }

    /**
     * @return Size of data.
     */
    public long getDataSize() {
      return dataSize;
    }

    public AgeSnapshot getAgeInCacheSnapshot() {
      return new AgeSnapshot(this.hist);
    }

    @Override
    public String toString() {
      AgeSnapshot snapshot = getAgeInCacheSnapshot();
      return "count=" + count + ", dataBlockCount=" + dataBlockCount + ", size=" + size +
          ", dataSize=" + getDataSize() +
          ", mean age=" + snapshot.getMean() +
          ", min age=" + snapshot.getMin() +
          ", max age=" + snapshot.getMax() +
          ", 75th percentile age="   + snapshot.get75thPercentile() +
          ", 95th percentile age="   + snapshot.get95thPercentile() +
          ", 98th percentile age="   + snapshot.get98thPercentile() +
          ", 99th percentile age="   + snapshot.get99thPercentile() +
          ", 99.9th percentile age=" + snapshot.get99thPercentile();
    }
  }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy