All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.accumulo.server.metadata.ServerAmpleImpl Maven / Gradle / Ivy

There is a newer version: 3.0.0
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.accumulo.server.metadata;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET_GC_CANDIDATES;
import static org.apache.accumulo.server.util.MetadataTableUtil.EMPTY_TEXT;

import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.IsolatedScanner;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.fate.FateTxId;
import org.apache.accumulo.core.gc.ReferenceFile;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.ScanServerRefTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.ValidationUtil;
import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.AmpleImpl;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionFinalState;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.BlipSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection.SkewedKeyValue;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.ExternalCompactionSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.ScanServerFileReferenceSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.server.ServerContext;
import org.apache.hadoop.io.Text;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Preconditions;

public class ServerAmpleImpl extends AmpleImpl implements Ample {

  private static Logger log = LoggerFactory.getLogger(ServerAmpleImpl.class);

  private ServerContext context;

  public ServerAmpleImpl(ServerContext context) {
    super(context);
    this.context = context;
  }

  @Override
  public Ample.TabletMutator mutateTablet(KeyExtent extent) {
    TabletsMutator tmi = mutateTablets();
    Ample.TabletMutator tabletMutator = tmi.mutateTablet(extent);
    ((TabletMutatorBase) tabletMutator).setCloseAfterMutate(tmi);
    return tabletMutator;
  }

  @Override
  public TabletsMutator mutateTablets() {
    return new TabletsMutatorImpl(context);
  }

  private void mutateRootGcCandidates(Consumer mutator) {
    String zpath = context.getZooKeeperRoot() + ZROOT_TABLET_GC_CANDIDATES;
    try {
      context.getZooReaderWriter().mutateOrCreate(zpath, new byte[0], currVal -> {
        String currJson = new String(currVal, UTF_8);
        RootGcCandidates rgcc = new RootGcCandidates(currJson);
        log.debug("Root GC candidates before change : {}", currJson);
        mutator.accept(rgcc);
        String newJson = rgcc.toJson();
        log.debug("Root GC candidates after change  : {}", newJson);
        if (newJson.length() > 262_144) {
          log.warn(
              "Root tablet deletion candidates stored in ZK at {} are getting large ({} bytes), is"
                  + " Accumulo GC process running?  Large nodes may cause problems for Zookeeper!",
              zpath, newJson.length());
        }
        return newJson.getBytes(UTF_8);
      });
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public void putGcCandidates(TableId tableId, Collection candidates) {

    if (RootTable.ID.equals(tableId)) {
      mutateRootGcCandidates(rgcc -> rgcc.add(candidates.stream()));
      return;
    }

    try (BatchWriter writer = context.createBatchWriter(DataLevel.of(tableId).metaTable())) {
      for (StoredTabletFile file : candidates) {
        writer.addMutation(createDeleteMutation(file));
      }
    } catch (MutationsRejectedException | TableNotFoundException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public void putGcFileAndDirCandidates(TableId tableId, Collection candidates) {

    if (DataLevel.of(tableId) == DataLevel.ROOT) {
      // Directories are unexpected for the root tablet, so convert to stored tablet file
      mutateRootGcCandidates(rgcc -> rgcc.add(candidates.stream()
          .map(reference -> new StoredTabletFile(reference.getMetadataEntry()))));
      return;
    }

    try (BatchWriter writer = context.createBatchWriter(DataLevel.of(tableId).metaTable())) {
      for (var fileOrDir : candidates) {
        writer.addMutation(createDeleteMutation(fileOrDir));
      }
    } catch (MutationsRejectedException | TableNotFoundException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public void addBulkLoadInProgressFlag(String path, long fateTxid) {

    // Bulk Import operations are not supported on the metadata table, so no entries will ever be
    // required on the root table.
    Mutation m = new Mutation(BlipSection.getRowPrefix() + path);
    m.put(EMPTY_TEXT, EMPTY_TEXT, new Value(FateTxId.formatTid(fateTxid)));

    try (BatchWriter bw = context.createBatchWriter(MetadataTable.NAME)) {
      bw.addMutation(m);
    } catch (MutationsRejectedException | TableNotFoundException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public void removeBulkLoadInProgressFlag(String path) {

    // Bulk Import operations are not supported on the metadata table, so no entries will ever be
    // required on the root table.
    Mutation m = new Mutation(BlipSection.getRowPrefix() + path);
    m.putDelete(EMPTY_TEXT, EMPTY_TEXT);

    try (BatchWriter bw = context.createBatchWriter(MetadataTable.NAME)) {
      bw.addMutation(m);
    } catch (MutationsRejectedException | TableNotFoundException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public void removeBulkLoadEntries(TableId tableId, long tid, Text firstSplit, Text lastSplit) {
    Preconditions.checkArgument(DataLevel.of(tableId) == DataLevel.USER);
    try (
        Scanner mscanner =
            new IsolatedScanner(context.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
        BatchWriter bw = context.createBatchWriter(MetadataTable.NAME)) {
      mscanner.setRange(new KeyExtent(tableId, lastSplit, firstSplit).toMetaRange());
      mscanner.fetchColumnFamily(BulkFileColumnFamily.NAME);

      for (Map.Entry entry : mscanner) {
        log.trace("Looking at entry {} with tid {}", entry, tid);
        long entryTid = BulkFileColumnFamily.getBulkLoadTid(entry.getValue());
        if (tid == entryTid) {
          log.trace("deleting entry {}", entry);
          Key key = entry.getKey();
          Mutation m = new Mutation(key.getRow());
          m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
          bw.addMutation(m);
        }
      }
    } catch (MutationsRejectedException | TableNotFoundException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public void deleteGcCandidates(DataLevel level, Collection paths) {

    if (level == DataLevel.ROOT) {
      mutateRootGcCandidates(rgcc -> rgcc.remove(paths.stream()));
      return;
    }

    try (BatchWriter writer = context.createBatchWriter(level.metaTable())) {
      for (String path : paths) {
        Mutation m = new Mutation(DeletesSection.encodeRow(path));
        m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
        writer.addMutation(m);
      }
    } catch (MutationsRejectedException | TableNotFoundException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public Iterator getGcCandidates(DataLevel level) {
    if (level == DataLevel.ROOT) {
      var zooReader = context.getZooReader();
      byte[] jsonBytes;
      try {
        jsonBytes =
            zooReader.getData(context.getZooKeeperRoot() + RootTable.ZROOT_TABLET_GC_CANDIDATES);
      } catch (KeeperException | InterruptedException e) {
        throw new RuntimeException(e);
      }
      return new RootGcCandidates(new String(jsonBytes, UTF_8)).sortedStream().iterator();
    } else if (level == DataLevel.METADATA || level == DataLevel.USER) {
      Range range = DeletesSection.getRange();

      Scanner scanner;
      try {
        scanner = context.createScanner(level.metaTable(), Authorizations.EMPTY);
      } catch (TableNotFoundException e) {
        throw new RuntimeException(e);
      }
      scanner.setRange(range);
      return scanner.stream().filter(entry -> entry.getValue().equals(SkewedKeyValue.NAME))
          .map(entry -> DeletesSection.decodeRow(entry.getKey().getRow().toString())).iterator();
    } else {
      throw new IllegalArgumentException();
    }
  }

  @Override
  public Mutation createDeleteMutation(ReferenceFile tabletFilePathToRemove) {
    return createDelMutation(ValidationUtil.validate(tabletFilePathToRemove).getMetadataEntry());
  }

  public Mutation createDeleteMutation(StoredTabletFile pathToRemove) {
    return createDelMutation(pathToRemove.getMetaUpdateDelete());
  }

  private Mutation createDelMutation(String path) {
    Mutation delFlag = new Mutation(new Text(DeletesSection.encodeRow(path)));
    delFlag.put(EMPTY_TEXT, EMPTY_TEXT, DeletesSection.SkewedKeyValue.NAME);
    return delFlag;
  }

  @Override
  public void
      putExternalCompactionFinalStates(Collection finalStates) {
    try (BatchWriter writer = context.createBatchWriter(DataLevel.USER.metaTable())) {
      String prefix = ExternalCompactionSection.getRowPrefix();
      for (ExternalCompactionFinalState finalState : finalStates) {
        Mutation m = new Mutation(prefix + finalState.getExternalCompactionId().canonical());
        m.put("", "", finalState.toJson());
        writer.addMutation(m);
      }
    } catch (MutationsRejectedException | TableNotFoundException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public Stream getExternalCompactionFinalStates() {
    Scanner scanner;
    try {
      scanner = context.createScanner(DataLevel.USER.metaTable(), Authorizations.EMPTY);
    } catch (TableNotFoundException e) {
      throw new RuntimeException(e);
    }

    scanner.setRange(ExternalCompactionSection.getRange());
    int pLen = ExternalCompactionSection.getRowPrefix().length();
    return scanner.stream()
        .map(e -> ExternalCompactionFinalState.fromJson(
            ExternalCompactionId.of(e.getKey().getRowData().toString().substring(pLen)),
            e.getValue().toString()));
  }

  @Override
  public void
      deleteExternalCompactionFinalStates(Collection statusesToDelete) {
    try (BatchWriter writer = context.createBatchWriter(DataLevel.USER.metaTable())) {
      String prefix = ExternalCompactionSection.getRowPrefix();
      for (ExternalCompactionId ecid : statusesToDelete) {
        Mutation m = new Mutation(prefix + ecid.canonical());
        m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
        writer.addMutation(m);
      }
      log.debug("Deleted external compaction final state entries for external compactions: {}",
          statusesToDelete);
    } catch (MutationsRejectedException | TableNotFoundException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public void putScanServerFileReferences(Collection scanRefs) {
    try (BatchWriter writer = context.createBatchWriter(DataLevel.USER.metaTable())) {
      String prefix = ScanServerFileReferenceSection.getRowPrefix();
      for (ScanServerRefTabletFile ref : scanRefs) {
        Mutation m = new Mutation(prefix + ref.getRowSuffix());
        m.put(ref.getServerAddress(), ref.getServerLockUUID(), ref.getValue());
        writer.addMutation(m);
      }
    } catch (MutationsRejectedException | TableNotFoundException e) {
      throw new IllegalStateException(
          "Error inserting scan server file references into " + DataLevel.USER.metaTable(), e);
    }
  }

  @Override
  public Stream getScanServerFileReferences() {
    try {
      Scanner scanner = context.createScanner(DataLevel.USER.metaTable(), Authorizations.EMPTY);
      scanner.setRange(ScanServerFileReferenceSection.getRange());
      int pLen = ScanServerFileReferenceSection.getRowPrefix().length();
      return StreamSupport.stream(scanner.spliterator(), false)
          .map(e -> new ScanServerRefTabletFile(e.getKey().getRowData().toString().substring(pLen),
              e.getKey().getColumnFamily(), e.getKey().getColumnQualifier()));
    } catch (TableNotFoundException e) {
      throw new IllegalStateException(DataLevel.USER.metaTable() + " not found!", e);
    }
  }

  @Override
  public void deleteScanServerFileReferences(String serverAddress, UUID scanServerLockUUID) {
    Objects.requireNonNull(serverAddress, "Server address must be supplied");
    Objects.requireNonNull(scanServerLockUUID, "Server uuid must be supplied");
    try (
        Scanner scanner = context.createScanner(DataLevel.USER.metaTable(), Authorizations.EMPTY)) {
      scanner.setRange(ScanServerFileReferenceSection.getRange());
      scanner.fetchColumn(new Text(serverAddress), new Text(scanServerLockUUID.toString()));

      int pLen = ScanServerFileReferenceSection.getRowPrefix().length();
      Set refsToDelete = StreamSupport.stream(scanner.spliterator(), false)
          .map(e -> new ScanServerRefTabletFile(e.getKey().getRowData().toString().substring(pLen),
              e.getKey().getColumnFamily(), e.getKey().getColumnQualifier()))
          .collect(Collectors.toSet());

      if (!refsToDelete.isEmpty()) {
        this.deleteScanServerFileReferences(refsToDelete);
      }
    } catch (TableNotFoundException e) {
      throw new IllegalStateException(DataLevel.USER.metaTable() + " not found!", e);
    }
  }

  @Override
  public void deleteScanServerFileReferences(Collection refsToDelete) {
    try (BatchWriter writer = context.createBatchWriter(DataLevel.USER.metaTable())) {
      String prefix = ScanServerFileReferenceSection.getRowPrefix();
      for (ScanServerRefTabletFile ref : refsToDelete) {
        Mutation m = new Mutation(prefix + ref.getRowSuffix());
        m.putDelete(ref.getServerAddress(), ref.getServerLockUUID());
        writer.addMutation(m);
      }
      log.debug("Deleted scan server file reference entries for files: {}", refsToDelete);
    } catch (MutationsRejectedException | TableNotFoundException e) {
      throw new RuntimeException(e);
    }
  }

}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy