All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.hadoop.fs.s3a.s3guard.S3GuardFsckViolationHandler Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.fs.s3a.s3guard; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.S3AFileStatus; import org.apache.hadoop.fs.s3a.S3AFileSystem; /** * Violation handler for the S3Guard's fsck. */ public class S3GuardFsckViolationHandler { private static final Logger LOG = LoggerFactory.getLogger( S3GuardFsckViolationHandler.class); // The rawFS and metadataStore are here to prepare when the ViolationHandlers // will not just log, but fix the violations, so they will have access. private final S3AFileSystem rawFs; private final DynamoDBMetadataStore metadataStore; private static String newLine = System.getProperty("line.separator"); public enum HandleMode { FIX, LOG } public S3GuardFsckViolationHandler(S3AFileSystem fs, DynamoDBMetadataStore ddbms) { this.metadataStore = ddbms; this.rawFs = fs; } public void logError(S3GuardFsck.ComparePair comparePair) throws IOException { if (!comparePair.containsViolation()) { LOG.debug("There is no violation in the compare pair: {}", comparePair); return; } StringBuilder sB = new StringBuilder(); sB.append(newLine) .append("On path: ").append(comparePair.getPath()).append(newLine); handleComparePair(comparePair, sB, HandleMode.LOG); LOG.error(sB.toString()); } public void doFix(S3GuardFsck.ComparePair comparePair) throws IOException { if (!comparePair.containsViolation()) { LOG.debug("There is no violation in the compare pair: {}", comparePair); return; } StringBuilder sB = new StringBuilder(); sB.append(newLine) .append("On path: ").append(comparePair.getPath()).append(newLine); handleComparePair(comparePair, sB, HandleMode.FIX); LOG.info(sB.toString()); } /** * Create a new instance of the violation handler for all the violations * found in the compare pair and use it. * * @param comparePair the compare pair with violations * @param sB StringBuilder to append error strings from violations. */ protected void handleComparePair(S3GuardFsck.ComparePair comparePair, StringBuilder sB, HandleMode handleMode) throws IOException { for (S3GuardFsck.Violation violation : comparePair.getViolations()) { try { ViolationHandler handler = violation.getHandler() .getDeclaredConstructor(S3GuardFsck.ComparePair.class) .newInstance(comparePair); switch (handleMode) { case FIX: final String errorStr = handler.getError(); sB.append(errorStr); break; case LOG: final String fixStr = handler.fixViolation(rawFs, metadataStore); sB.append(fixStr); break; default: throw new UnsupportedOperationException("Unknown handleMode: " + handleMode); } } catch (NoSuchMethodException e) { LOG.error("Can not find declared constructor for handler: {}", violation.getHandler()); } catch (IllegalAccessException | InstantiationException | InvocationTargetException e) { LOG.error("Can not instantiate handler: {}", violation.getHandler()); } sB.append(newLine); } } /** * Violation handler abstract class. * This class should be extended for violation handlers. */ public static abstract class ViolationHandler { private final PathMetadata pathMetadata; private final S3AFileStatus s3FileStatus; private final S3AFileStatus msFileStatus; private final List s3DirListing; private final DirListingMetadata msDirListing; public ViolationHandler(S3GuardFsck.ComparePair comparePair) { pathMetadata = comparePair.getMsPathMetadata(); s3FileStatus = comparePair.getS3FileStatus(); if (pathMetadata != null) { msFileStatus = pathMetadata.getFileStatus(); } else { msFileStatus = null; } s3DirListing = comparePair.getS3DirListing(); msDirListing = comparePair.getMsDirListing(); } public abstract String getError(); public PathMetadata getPathMetadata() { return pathMetadata; } public S3AFileStatus getS3FileStatus() { return s3FileStatus; } public S3AFileStatus getMsFileStatus() { return msFileStatus; } public List getS3DirListing() { return s3DirListing; } public DirListingMetadata getMsDirListing() { return msDirListing; } public String fixViolation(S3AFileSystem fs, DynamoDBMetadataStore ddbms) throws IOException { return String.format("Fixing of violation: %s is not supported yet.", this.getClass().getSimpleName()); } } /** * The violation handler when there's no matching metadata entry in the MS. */ public static class NoMetadataEntry extends ViolationHandler { public NoMetadataEntry(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return "No PathMetadata for this path in the MS."; } } /** * The violation handler when there's no parent entry. */ public static class NoParentEntry extends ViolationHandler { public NoParentEntry(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return "Entry does not have a parent entry (not root)"; } } /** * The violation handler when the parent of an entry is a file. */ public static class ParentIsAFile extends ViolationHandler { public ParentIsAFile(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return "The entry's parent in the metastore database is a file."; } } /** * The violation handler when the parent of an entry is tombstoned. */ public static class ParentTombstoned extends ViolationHandler { public ParentTombstoned(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return "The entry in the metastore database has a parent entry " + "which is a tombstone marker"; } } /** * The violation handler when there's a directory is a file metadata in MS. */ public static class DirInS3FileInMs extends ViolationHandler { public DirInS3FileInMs(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return "A directory in S3 is a file entry in the MS"; } } /** * The violation handler when a file metadata is a directory in MS. */ public static class FileInS3DirInMs extends ViolationHandler { public FileInS3DirInMs(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return "A file in S3 is a directory entry in the MS"; } } /** * The violation handler when there's a directory listing content mismatch. */ public static class AuthDirContentMismatch extends ViolationHandler { public AuthDirContentMismatch(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { final String str = String.format( "The content of an authoritative directory listing does " + "not match the content of the S3 listing. S3: %s, MS: %s", Arrays.asList(getS3DirListing()), getMsDirListing().getListing()); return str; } } /** * The violation handler when there's a length mismatch. */ public static class LengthMismatch extends ViolationHandler { public LengthMismatch(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return String.format("File length mismatch - S3: %s, MS: %s", getS3FileStatus().getLen(), getMsFileStatus().getLen()); } } /** * The violation handler when there's a modtime mismatch. */ public static class ModTimeMismatch extends ViolationHandler { public ModTimeMismatch(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return String.format("File timestamp mismatch - S3: %s, MS: %s", getS3FileStatus().getModificationTime(), getMsFileStatus().getModificationTime()); } } /** * The violation handler when there's a version id mismatch. */ public static class VersionIdMismatch extends ViolationHandler { public VersionIdMismatch(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return String.format("getVersionId mismatch - S3: %s, MS: %s", getS3FileStatus().getVersionId(), getMsFileStatus().getVersionId()); } } /** * The violation handler when there's an etag mismatch. */ public static class EtagMismatch extends ViolationHandler { public EtagMismatch(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return String.format("Etag mismatch - S3: %s, MS: %s", getS3FileStatus().getETag(), getMsFileStatus().getETag()); } } /** * The violation handler when there's no etag. */ public static class NoEtag extends ViolationHandler { public NoEtag(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return "No etag."; } } /** * The violation handler when there's a tombstoned entry in the ms is * present, but the object is not deleted in S3. */ public static class TombstonedInMsNotDeletedInS3 extends ViolationHandler { public TombstonedInMsNotDeletedInS3(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return "The entry for the path is tombstoned in the MS."; } } /** * The violation handler there's no parent in the MetadataStore. */ public static class OrphanDDBEntry extends ViolationHandler { public OrphanDDBEntry(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return "The DDB entry is orphan - there is no parent in the MS."; } @Override public String fixViolation(S3AFileSystem fs, DynamoDBMetadataStore ddbms) throws IOException { final Path path = getPathMetadata().getFileStatus().getPath(); ddbms.forgetMetadata(path); return String.format( "Fixing violation by removing metadata entry from the " + "MS on path: %s", path); } } /** * The violation handler when there's no last updated field for the entry. */ public static class NoLastUpdatedField extends ViolationHandler { public NoLastUpdatedField(S3GuardFsck.ComparePair comparePair) { super(comparePair); } @Override public String getError() { return "No lastUpdated field provided for the entry."; } } }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy