com.palantir.atlasdb.blob.generated.DataMetadataCleanupTask Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of atlasdb-ete-tests Show documentation
Palantir open source project
There is a newer version: 0.1152.0
package com.palantir.atlasdb.blob.generated;

import java.util.Iterator;
import java.util.Map;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.palantir.atlasdb.cleaner.api.OnCleanupTask;
import com.palantir.atlasdb.encoding.PtBytes;
import com.palantir.atlasdb.keyvalue.api.BatchColumnRangeSelection;
import com.palantir.atlasdb.keyvalue.api.Cell;
import com.palantir.atlasdb.keyvalue.api.Namespace;
import com.palantir.atlasdb.protos.generated.StreamPersistence.Status;
import com.palantir.atlasdb.protos.generated.StreamPersistence.StreamMetadata;
import com.palantir.atlasdb.table.description.ValueType;
import com.palantir.atlasdb.transaction.api.Transaction;
import com.palantir.common.streams.KeyedStream;
import com.palantir.logsafe.SafeArg;

public class DataMetadataCleanupTask implements OnCleanupTask {

    private static final Logger log = LoggerFactory.getLogger(DataMetadataCleanupTask.class);

    private final BlobSchemaTableFactory tables;

    public DataMetadataCleanupTask(Namespace namespace) {
        tables = BlobSchemaTableFactory.of(namespace);
    }

    @Override
    public boolean cellsCleanedUp(Transaction t, Set cells) {
        DataStreamMetadataTable metaTable = tables.getDataStreamMetadataTable(t);
        Set rows = Sets.newHashSetWithExpectedSize(cells.size());
        for (Cell cell : cells) {
            rows.add(DataStreamMetadataTable.DataStreamMetadataRow.BYTES_HYDRATOR.hydrateFromBytes(cell.getRowName()));
        }
        DataStreamIdxTable indexTable = tables.getDataStreamIdxTable(t);
        Set rowsWithNoIndexEntries =
                        executeUnreferencedStreamDiagnostics(indexTable, rows);
        Set toDelete = new HashSet<>();
        Map currentMetadata =
                metaTable.getMetadatas(rows);
        for (Map.Entry e : currentMetadata.entrySet()) {
            if (e.getValue().getStatus() != Status.STORED || rowsWithNoIndexEntries.contains(e.getKey())) {
                toDelete.add(e.getKey().getId());
            }
        }
        DataStreamStore.of(tables).deleteStreams(t, toDelete);
        return false;
    }

    private static DataStreamMetadataTable.DataStreamMetadataRow convertFromIndexRow(DataStreamIdxTable.DataStreamIdxRow idxRow) {
        return DataStreamMetadataTable.DataStreamMetadataRow.of(idxRow.getId());
    }
    private static Set convertToIdsForLogging(Set iteratorExcess) {
        return iteratorExcess.stream()
                .map(DataStreamMetadataTable.DataStreamMetadataRow::getId)
                .collect(Collectors.toSet());
    }

    private static Set getUnreferencedStreamsByMultimap(DataStreamIdxTable indexTable, Set indexRows) {
        Multimap indexValues = indexTable.getRowsMultimap(indexRows);
        Set presentMetadataRows
                = indexValues.keySet().stream()
                .map(DataMetadataCleanupTask::convertFromIndexRow)
                .collect(Collectors.toSet());
        Set queriedMetadataRows
                = indexRows.stream()
                .map(DataMetadataCleanupTask::convertFromIndexRow)
                .collect(Collectors.toSet());
        return Sets.difference(queriedMetadataRows, presentMetadataRows);
    }

    private static Set getUnreferencedStreamsByIterator(DataStreamIdxTable indexTable, Set indexRows) {
        Map> referenceIteratorByStream
                = indexTable.getRowsColumnRangeIterator(indexRows,
                        BatchColumnRangeSelection.create(PtBytes.EMPTY_BYTE_ARRAY, PtBytes.EMPTY_BYTE_ARRAY, 1));
        Set unreferencedStreamMetadata
                = KeyedStream.stream(referenceIteratorByStream)
                .filter(valueIterator -> !valueIterator.hasNext())
                .keys() // (authorized)
                .map(DataStreamIdxTable.DataStreamIdxRow::getId)
                .map(DataStreamMetadataTable.DataStreamMetadataRow::of)
                .collect(Collectors.toSet());
        return unreferencedStreamMetadata;
    }

    private static Set executeUnreferencedStreamDiagnostics(DataStreamIdxTable indexTable, Set metadataRows) {
        Set indexRows = metadataRows.stream()
                .map(DataStreamMetadataTable.DataStreamMetadataRow::getId)
                .map(DataStreamIdxTable.DataStreamIdxRow::of)
                .collect(Collectors.toSet());
        Set unreferencedStreamsByIterator = getUnreferencedStreamsByIterator(indexTable, indexRows);
        Set unreferencedStreamsByMultimap = getUnreferencedStreamsByMultimap(indexTable, indexRows);
        if (!unreferencedStreamsByIterator.equals(unreferencedStreamsByMultimap)) {
            log.info("We searched for unreferenced streams with methodological inconsistency: iterators claimed we could delete {}, but multimaps {}.",
                    SafeArg.of("unreferencedByIterator", convertToIdsForLogging(unreferencedStreamsByIterator)),
                    SafeArg.of("unreferencedByMultimap", convertToIdsForLogging(unreferencedStreamsByMultimap)));
            return new HashSet<>();
        } else {
            log.info("We searched for unreferenced streams and consistently found {}.",
                    SafeArg.of("unreferencedStreamIds", convertToIdsForLogging(unreferencedStreamsByIterator)));
            return unreferencedStreamsByIterator;
        }
    }
}