org.elasticsearch.index.translog.TranslogDeletionPolicy Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch subproject :server
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.index.translog;
import org.apache.lucene.util.Counter;
import org.elasticsearch.Assertions;
import org.elasticsearch.core.Releasable;
import org.elasticsearch.index.seqno.SequenceNumbers;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
public class TranslogDeletionPolicy {
private final Map openTranslogRef;
public void assertNoOpenTranslogRefs() {
if (openTranslogRef.isEmpty() == false) {
AssertionError e = new AssertionError("not all translog generations have been released");
openTranslogRef.values().forEach(e::addSuppressed);
throw e;
}
}
/**
* Records how many retention locks are held against each
* translog generation
*/
private final Map translogRefCounts = new HashMap<>();
private long localCheckpointOfSafeCommit = SequenceNumbers.NO_OPS_PERFORMED;
private long retentionSizeInBytes;
private long retentionAgeInMillis;
private int retentionTotalFiles;
public TranslogDeletionPolicy(long retentionSizeInBytes, long retentionAgeInMillis, int retentionTotalFiles) {
this.retentionSizeInBytes = retentionSizeInBytes;
this.retentionAgeInMillis = retentionAgeInMillis;
this.retentionTotalFiles = retentionTotalFiles;
if (Assertions.ENABLED) {
openTranslogRef = new ConcurrentHashMap<>();
} else {
openTranslogRef = null;
}
}
public synchronized void setLocalCheckpointOfSafeCommit(long newCheckpoint) {
if (newCheckpoint < this.localCheckpointOfSafeCommit) {
throw new IllegalArgumentException(
"local checkpoint of the safe commit can't go backwards: "
+ "current ["
+ this.localCheckpointOfSafeCommit
+ "] new ["
+ newCheckpoint
+ "]"
);
}
this.localCheckpointOfSafeCommit = newCheckpoint;
}
public synchronized void setRetentionSizeInBytes(long bytes) {
retentionSizeInBytes = bytes;
}
public synchronized void setRetentionAgeInMillis(long ageInMillis) {
retentionAgeInMillis = ageInMillis;
}
synchronized void setRetentionTotalFiles(int retentionTotalFiles) {
this.retentionTotalFiles = retentionTotalFiles;
}
/**
* acquires the basis generation for a new snapshot. Any translog generation above, and including, the returned generation
* will not be deleted until the returned {@link Releasable} is closed.
*/
synchronized Releasable acquireTranslogGen(final long translogGen) {
translogRefCounts.computeIfAbsent(translogGen, l -> Counter.newCounter(false)).addAndGet(1);
final AtomicBoolean closed = new AtomicBoolean();
assert assertAddTranslogRef(closed);
return () -> {
if (closed.compareAndSet(false, true)) {
releaseTranslogGen(translogGen);
assert assertRemoveTranslogRef(closed);
}
};
}
private boolean assertAddTranslogRef(Object reference) {
final RuntimeException existing = openTranslogRef.put(reference, new RuntimeException());
if (existing != null) {
throw new AssertionError("double adding of closing reference", existing);
}
return true;
}
private boolean assertRemoveTranslogRef(Object reference) {
return openTranslogRef.remove(reference) != null;
}
/** returns the number of generations that were acquired for snapshots */
synchronized int pendingTranslogRefCount() {
return translogRefCounts.size();
}
/**
* releases a generation that was acquired by {@link #acquireTranslogGen(long)}
*/
private synchronized void releaseTranslogGen(long translogGen) {
Counter current = translogRefCounts.get(translogGen);
if (current == null || current.get() <= 0) {
throw new IllegalArgumentException("translog gen [" + translogGen + "] wasn't acquired");
}
if (current.addAndGet(-1) == 0) {
translogRefCounts.remove(translogGen);
}
}
/**
* returns the minimum translog generation that is still required by the system. Any generation below
* the returned value may be safely deleted
*
* @param readers current translog readers
* @param writer current translog writer
*/
synchronized long minTranslogGenRequired(List readers, TranslogWriter writer) throws IOException {
long minByLocks = getMinTranslogGenRequiredByLocks();
long minByAge = getMinTranslogGenByAge(readers, writer, retentionAgeInMillis, currentTime());
long minBySize = getMinTranslogGenBySize(readers, writer, retentionSizeInBytes);
final long minByAgeAndSize;
if (minBySize == Long.MIN_VALUE && minByAge == Long.MIN_VALUE) {
// both size and age are disabled;
minByAgeAndSize = Long.MAX_VALUE;
} else {
minByAgeAndSize = Math.max(minByAge, minBySize);
}
long minByNumFiles = getMinTranslogGenByTotalFiles(readers, writer, retentionTotalFiles);
return Math.min(Math.max(minByAgeAndSize, minByNumFiles), minByLocks);
}
static long getMinTranslogGenBySize(List readers, TranslogWriter writer, long retentionSizeInBytes) {
if (retentionSizeInBytes >= 0) {
long totalSize = writer.sizeInBytes();
long minGen = writer.getGeneration();
for (int i = readers.size() - 1; i >= 0 && totalSize < retentionSizeInBytes; i--) {
final TranslogReader reader = readers.get(i);
totalSize += reader.sizeInBytes();
minGen = reader.getGeneration();
}
return minGen;
} else {
return Long.MIN_VALUE;
}
}
static long getMinTranslogGenByAge(List readers, TranslogWriter writer, long maxRetentionAgeInMillis, long now)
throws IOException {
if (maxRetentionAgeInMillis >= 0) {
for (TranslogReader reader : readers) {
if (now - reader.getLastModifiedTime() <= maxRetentionAgeInMillis) {
return reader.getGeneration();
}
}
return writer.getGeneration();
} else {
return Long.MIN_VALUE;
}
}
static long getMinTranslogGenByTotalFiles(List readers, TranslogWriter writer, final int maxTotalFiles) {
long minGen = writer.generation;
int totalFiles = 1; // for the current writer
for (int i = readers.size() - 1; i >= 0 && totalFiles < maxTotalFiles; i--) {
totalFiles++;
minGen = readers.get(i).generation;
}
return minGen;
}
protected long currentTime() {
return System.currentTimeMillis();
}
private long getMinTranslogGenRequiredByLocks() {
return translogRefCounts.keySet().stream().reduce(Math::min).orElse(Long.MAX_VALUE);
}
/**
* Returns the local checkpoint of the safe commit. This value is used to calculate the min required generation for recovery.
*/
public synchronized long getLocalCheckpointOfSafeCommit() {
return localCheckpointOfSafeCommit;
}
synchronized long getTranslogRefCount(long gen) {
final Counter counter = translogRefCounts.get(gen);
return counter == null ? 0 : counter.get();
}
}