org.apache.hadoop.hive.llap.cache.FileCacheCleanupThread Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hive-llap-server
There is a newer version: 4.0.1
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hive.llap.cache;

import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
import org.apache.hive.common.util.Ref;

/** Class used to slowly clean up a map of FileCache-s. */
abstract class FileCacheCleanupThread extends Thread {
  private final long approxCleanupIntervalSec;
  private final AtomicInteger newEvictions;
  private final ConcurrentHashMap> fileMap;

  public FileCacheCleanupThread(String name, ConcurrentHashMap> fileMap,
      AtomicInteger newEvictions, long cleanupInterval) {
    super(name);
    this.fileMap = fileMap;
    this.newEvictions = newEvictions;
    this.approxCleanupIntervalSec = cleanupInterval;
    setDaemon(true);
    setPriority(1);
  }

  @Override
  public void run() {
    while (true) {
      try {
        doOneCleanupRound();
      } catch (InterruptedException ex) {
        LlapIoImpl.LOG.warn("Cleanup thread has been interrupted");
        Thread.currentThread().interrupt();
        break;
      } catch (Throwable t) {
        LlapIoImpl.LOG.error("Cleanup has failed; the thread will now exit", t);
        break;
      }
    }
  }

  private void doOneCleanupRound() throws InterruptedException {
    while (true) {
      int evictionsSinceLast = newEvictions.getAndSet(0);
      if (evictionsSinceLast > 0) break;
      synchronized (newEvictions) {
        newEvictions.wait(10000);
      }
    }
    // Duration is an estimate; if the size of the map changes, it can be very different.
    long endTime = System.nanoTime() + approxCleanupIntervalSec * 1000000000L;
    int leftToCheck = 0; // approximate
    for (FileCache fc : fileMap.values()) {
      leftToCheck += getCacheSize(fc);
    }
    // Iterate thru all the filecaches. This is best-effort.
    // If these super-long-lived iterators affect the map in some bad way,
    // we'd need to sleep once per round instead.
    Iterator>> iter = fileMap.entrySet().iterator();
    Ref isPastEndTime = Ref.from(false);
    while (iter.hasNext()) {
      FileCache fc = iter.next().getValue();
      if (!fc.incRef()) {
        throw new AssertionError("Something other than cleanup is removing elements from map");
      }
      leftToCheck = cleanUpOneFileCache(fc, leftToCheck, endTime, isPastEndTime);
      if (getCacheSize(fc) > 0) {
        fc.decRef();
        continue;
      }
      // FileCache might be empty; see if we can remove it. "tryWriteLock"
      if (!fc.startEvicting()) continue;
      if (getCacheSize(fc) == 0) {
        fc.commitEvicting();
        iter.remove();
      } else {
        fc.abortEvicting();
      }
    }
  }

  protected abstract int getCacheSize(FileCache fc);

  protected abstract int cleanUpOneFileCache(FileCache fc, int leftToCheck, long endTime,
      Ref isPastEndTime) throws InterruptedException;
}