• Home
• io.snappydata
• gemfire-core
• 1.6.2.1
• source code
• InitialImageOperation.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.gemstone.gemfire.internal.cache.InitialImageOperation Maven / Gradle / Ivy

/*
 * Copyright (c) 2010-2015 Pivotal Software, Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License. See accompanying
 * LICENSE file.
 */

package com.gemstone.gemfire.internal.cache;

import static com.gemstone.gemfire.internal.offheap.annotations.OffHeapIdentifier.ABSTRACT_REGION_ENTRY_FILL_IN_VALUE;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;

import com.gemstone.gemfire.CancelException;
import com.gemstone.gemfire.DataSerializable;
import com.gemstone.gemfire.DataSerializer;
import com.gemstone.gemfire.InternalGemFireError;
import com.gemstone.gemfire.InternalGemFireException;
import com.gemstone.gemfire.LogWriter;
import com.gemstone.gemfire.SystemFailure;
import com.gemstone.gemfire.cache.DiskAccessException;
import com.gemstone.gemfire.cache.RegionDestroyedException;
import com.gemstone.gemfire.cache.TransactionDataNodeHasDepartedException;
import com.gemstone.gemfire.cache.hdfs.internal.AbstractBucketRegionQueue;
import com.gemstone.gemfire.cache.query.internal.CqQueryImpl;
import com.gemstone.gemfire.cache.query.internal.CqService;
import com.gemstone.gemfire.cache.query.internal.CqStateImpl;
import com.gemstone.gemfire.cache.query.internal.DefaultQueryService;
import com.gemstone.gemfire.distributed.DistributedMember;
import com.gemstone.gemfire.distributed.DistributedSystem;
import com.gemstone.gemfire.distributed.internal.*;
import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
import com.gemstone.gemfire.i18n.LogWriterI18n;
import com.gemstone.gemfire.internal.Assert;
import com.gemstone.gemfire.internal.ByteArrayDataInput;
import com.gemstone.gemfire.internal.DataSerializableFixedID;
import com.gemstone.gemfire.internal.HeapDataOutputStream;
import com.gemstone.gemfire.internal.InternalDataSerializer;
import com.gemstone.gemfire.internal.LogWriterImpl;
import com.gemstone.gemfire.internal.NullDataOutputStream;
import com.gemstone.gemfire.internal.cache.InitialImageFlowControl.FlowControlPermitMessage;
import com.gemstone.gemfire.internal.cache.ha.HAContainerWrapper;
import com.gemstone.gemfire.internal.cache.locks.ExclusiveSharedSynchronizer;
import com.gemstone.gemfire.internal.cache.locks.LockingPolicy;
import com.gemstone.gemfire.internal.cache.persistence.DiskStoreID;
import com.gemstone.gemfire.internal.cache.persistence.PersistenceAdvisor;
import com.gemstone.gemfire.internal.cache.tier.InterestType;
import com.gemstone.gemfire.internal.cache.tier.sockets.CacheClientNotifier;
import com.gemstone.gemfire.internal.cache.tier.sockets.CacheClientProxy;
import com.gemstone.gemfire.internal.cache.tier.sockets.ClientProxyMembershipID;
import com.gemstone.gemfire.internal.cache.versions.DiskRegionVersionVector;
import com.gemstone.gemfire.internal.cache.versions.DiskVersionTag;
import com.gemstone.gemfire.internal.cache.versions.RegionVersionHolder;
import com.gemstone.gemfire.internal.cache.versions.RegionVersionVector;
import com.gemstone.gemfire.internal.cache.versions.VersionSource;
import com.gemstone.gemfire.internal.cache.versions.VersionStamp;
import com.gemstone.gemfire.internal.cache.versions.VersionTag;
import com.gemstone.gemfire.internal.cache.vmotion.VMotionObserverHolder;
import com.gemstone.gemfire.internal.cache.wan.serial.SerialGatewaySenderImpl;
import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
import com.gemstone.gemfire.internal.offheap.ByteSource;
import com.gemstone.gemfire.internal.offheap.OffHeapHelper;
import com.gemstone.gemfire.internal.offheap.annotations.Released;
import com.gemstone.gemfire.internal.offheap.annotations.Retained;
import com.gemstone.gemfire.internal.sequencelog.EntryLogger;
import com.gemstone.gemfire.internal.sequencelog.RegionLogger;
import com.gemstone.gemfire.internal.shared.Version;
import com.gemstone.gemfire.internal.util.ArrayUtils;
import com.gemstone.gemfire.internal.util.concurrent.StoppableCountDownLatch;
import com.gemstone.gnu.trove.THashMap;
import com.gemstone.gnu.trove.THashSet;
import com.gemstone.gnu.trove.TObjectIntProcedure;

/**
 * Handles requests for an initial image from a cache peer
 * 
 * @author Eric Zoerner
 */
public class InitialImageOperation  {
  /**
   * internal flag used by unit tests to test early disconnect from distributed system
   */
  public static volatile boolean abortTest = false;
  
  /**
   * maximum number of bytes to put in a single message
   */
  public static int CHUNK_SIZE_IN_BYTES =
    Integer.getInteger("GetInitialImage.chunkSize", 500 * 1024).intValue();
  
  /**
   * Allowed number of in flight GII chunks
   */
  public static int CHUNK_PERMITS =
    Integer.getInteger("gemfire.GetInitialImage.CHUNK_PERMITS", 16).intValue();

  /**
   * maximum number of unfinished operations to be supported by delta GII
   */
  public static int MAXIMUM_UNFINISHED_OPERATIONS =
    Integer.getInteger("gemfire.GetInitialImage.MAXIMUM_UNFINISHED_OPERATIONS", 10000).intValue();

  /**
   * Allowed number GIIs in parallel
   */
  public static int MAX_PARALLEL_GIIS =
    Integer.getInteger("gemfire.GetInitialImage.MAX_PARALLEL_GIIS", 5).intValue();
  
  /**
   * the region we are fetching
   */
  protected final DistributedRegion region;
  
  /**
   * the underlying Map to receive our values
   */
  private final RegionMap entries;

  /**
   * true if we have received the last piece of the image
   */
  protected volatile boolean gotImage = false;
  
  /** 
   * received region version holder for lost member, used by synchronizeWith only 
   */
  protected RegionVersionHolder rcvd_holderToSync;
  
  /**
   * true if this is delta gii
   */
  protected volatile boolean isDeltaGII = false;

  /**
   * for testing purposes
   */
  public static volatile int slowImageProcessing = 0;
  
  /**
   * for testing purposes
   */
  public static volatile int slowImageSleeps = 0;

  /**
   * for testing purposes
   */
  public static boolean VMOTION_DURING_GII = false;

  private boolean isSynchronizing;
  
  /** Creates a new instance of InitalImageOperation */
  InitialImageOperation(DistributedRegion region, RegionMap entries) {
    this.region = region;
    this.entries = entries;
  }

  /** a flag for inhibiting the use of StateFlushOperation before gii */
  private final static ThreadLocal inhibitStateFlush = new ThreadLocal(){
    @Override
      protected Object initialValue() {
        return Boolean.valueOf(false);
      }
    };
  
  
  /** inhibit use of StateFlush for the current thread */
  public static void setInhibitStateFlush(boolean inhibitIt) {
    inhibitStateFlush.set(Boolean.valueOf(inhibitIt));
  }

  public enum GIIStatus {
    NO_GII,
    GOTIMAGE_BY_FULLGII,
    GOTIMAGE_BY_DELTAGII; 

    public final boolean didGII() {
      return this != NO_GII;
    }

    public static boolean didDeltaGII(GIIStatus giiStatus) {
      return giiStatus == GIIStatus.GOTIMAGE_BY_DELTAGII;
    }

    public static boolean didFullGII(GIIStatus giiStatus) {
      return giiStatus == GIIStatus.GOTIMAGE_BY_FULLGII;
    }
  }

  private GIIStatus reportGIIStatus() {
    if (!this.gotImage) {
      return GIIStatus.NO_GII;
    } else {
      // got image
      if (this.isDeltaGII) {
        return GIIStatus.GOTIMAGE_BY_DELTAGII;
      } else {
        return GIIStatus.GOTIMAGE_BY_FULLGII;
      }
    }
  }

  private boolean hasLockedBucket = false;

  private void unlockTargetBucketLock(InternalDistributedMember recipient) {
    if (hasLockedBucket) {
      LogWriterI18n logger = this.region.getDistributionManager().getLoggerI18n();
      logger.info(LocalizedStrings.DEBUG, "Processing SnapshotBucketLockReleaseProcessor to " + recipient);
      try {
        SnapshotBucketLockReleaseMessage.send(recipient,
            this.region.getDistributionManager(), region.getFullPath());
        logger.info(LocalizedStrings.DEBUG, "Processing SnapshotBucketLockReleaseProcessor done " + recipient);
      } catch (Throwable t) {
        logger.warning(LocalizedStrings.DEBUG, t);
      }
    }
  }

  /**
   * Fetch an initial image from a single recipient
   * 
   * @param recipientSet list of candidates to fetch from
   * @param targetReinitialized true if candidate should wait until initialized
   *        before responding
   * @param advice
   * @param recoveredRVV recovered rvv 
   * @return true if succeeded to get image
   * @throws com.gemstone.gemfire.cache.TimeoutException
   */
  GIIStatus getFromOne(
      Set recipientSet,
      boolean targetReinitialized,
      CacheDistributionAdvisor.InitialImageAdvice advice,
      boolean recoveredFromDisk, RegionVersionVector recoveredRVV)
      throws com.gemstone.gemfire.cache.TimeoutException
  {
    if (VMOTION_DURING_GII) {
      /**
       * TODO (ashetkar): recipientSet may contain more than one member. Ensure
       * only the gii-source member is vMotioned. The test hook may need to be
       * placed at another point.
       */
      VMotionObserverHolder.getInstance().vMotionDuringGII(recipientSet, this.region);
    }
    // Make sure that candidates are regarded in random order
    ArrayList recipients = new ArrayList(recipientSet);

    if (this.region.isUsedForSerialGatewaySenderQueue()) {
      SerialGatewaySenderImpl sender = this.region.getSerialGatewaySender();
      if (sender != null) {
        InternalDistributedMember primary = sender.getSenderAdvisor()
            .advisePrimaryGatewaySender();
        if (primary != null) {
          recipients.remove(primary);
          recipients.add(0, primary);
        }
      }
    } else {
      if (recipients.size() > 1) {
        Collections.shuffle(recipients);
      }
    }
    long giiStart = this.region.getCachePerfStats().startGetInitialImage();

    InternalDistributedMember prevRecipient = null;

    this.region.getLogWriterI18n().info(LocalizedStrings.DEBUG,
        "InitialImageOperation starts for " + this.region.getFullPath() + ", Recipients are " + recipients);
    for (Iterator itr = recipients.iterator(); !this.gotImage && itr.hasNext();) {
      // if we got a partial image from the previous recipient, then clear it

      if(prevRecipient != null ){
        unlockTargetBucketLock(prevRecipient);
      }

      InternalDistributedMember recipient = (InternalDistributedMember)itr.next();
      prevRecipient = recipient;
//      log.info("InitialImageOperation for " + this.region.getFullPath() + " targeting " + recipient);
      
      // In case of HARegion, before getting the region snapshot(image) get the filters 
      // registered by the associated client and apply them.
      // As part of bug fix 39014, while creating the secondary HARegion/Queue, the 
      // filters registered by the client is applied first and then the HARegion 
      // initial image is applied. This is to process any events thats arriving while
      // GII is happening and is not part of the GII result.
      if (region instanceof HARegion){
        try {
//          HARegion r = (HARegion)region;
//          if (!r.isPrimaryQueue()) {
            if (!this.requestFilterInfo(recipient)) {
              if (this.region.getLogWriterI18n().infoEnabled()) {
                // TODO add localizedString
                this.region.getLogWriterI18n().info(LocalizedStrings.DEBUG, 
                    "Failed to receive interest and CQ information from " + recipient);
              }
            }
//          }
        } catch (Exception ex){
          if (this.region.getCache().getLoggerI18n().infoEnabled() && !itr.hasNext()) {
            // TODO add localizedString
            this.region.getCache().getLoggerI18n().info(LocalizedStrings.DEBUG, 
                "Failed while getting interest and CQ information from " + recipient);
            // To report exception.
            if (this.region.getCache().getLoggerI18n().fineEnabled()) {
              this.region.getCache().getLoggerI18n().fine("Failed while getting interest and CQ information from " + 
                  recipient, ex);
            }
          }
          continue;
        }
      }
      
      PersistenceAdvisor persistenceAdvisor = this.region.getPersistenceAdvisor();
      if(persistenceAdvisor != null) {
        try {
          persistenceAdvisor.updateMembershipView(recipient, targetReinitialized);
          persistenceAdvisor.setInitializing(this.region.getPersistentID());
        } catch(ReplyException e) {
          this.region.getCache().getLoggerI18n().fine("Failed to get membership view", e);
          continue;
        }
      }
      final DistributionManager dm = (DistributionManager)this.region.getDistributionManager();

      boolean allowDeltaGII = true;
      if (FORCE_FULL_GII) {
        allowDeltaGII = false;
      }
      Set keysOfUnfinishedOps = null;
      RegionVersionVector received_rvv = null;
      RegionVersionVector remote_rvv = null;
      if (this.region.concurrencyChecksEnabled) {
        if (internalBeforeRequestRVV != null && internalBeforeRequestRVV.getRegionName().equals(this.region.getName())) {
          internalBeforeRequestRVV.run();
        }
        RequestRVVProcessor rvvProcessor = getRVVDetailsFromProvider(dm, recipient, targetReinitialized);
        received_rvv = rvvProcessor.received_rvv;
        hasLockedBucket = rvvProcessor.snapshotGIIWriteLock;
        if (received_rvv == null) {
          continue;
        }

        // remote_rvv will be filled with the versions of unfinished keys
        // then if recoveredRVV is still newer than the filled remote_rvv, do fullGII
        remote_rvv = received_rvv.getCloneForTransmission();
        keysOfUnfinishedOps = processReceivedRVV(remote_rvv, recoveredRVV);
        if (internalAfterCalculatedUnfinishedOps != null && internalAfterCalculatedUnfinishedOps.getRegionName().equals(this.region.getName())) {
          internalAfterCalculatedUnfinishedOps.run();
        }
        if (keysOfUnfinishedOps == null) {
          // if got rvv, keysOfUnfinishedOps at least will be empty
          continue;
        }
      }

      Boolean inhibitFlush = (Boolean)inhibitStateFlush.get();
      if (!inhibitFlush.booleanValue() && !this.region.doesNotDistribute()) {
        if (region instanceof AbstractBucketRegionQueue) {
          // get the corresponding userPRs and do state flush on all of them
          // TODO we should be able to do this state flush with a single
          // message, but that will require changing the messaging layer,
          // which has implications for a rolling upgrade.
          Collection userPRBuckets = ((AbstractBucketRegionQueue)(this.region))
              .getCorrespondingUserPRBuckets();
          if (this.region.getLogWriterI18n().fineEnabled()) {
            this.region.getLogWriterI18n().fine(
                "The parent buckets of this shadowPR region are "
                    + userPRBuckets);
          }

          for (BucketRegion parentBucket : userPRBuckets) {
            if (this.region.getLogWriterI18n().fineEnabled()) {
              this.region.getLogWriterI18n().fine(
                  "Going to do state flush operation on the parent bucket.");
            }
            boolean interrupted = false;
            final StateFlushOperation sf;
            sf = new StateFlushOperation(parentBucket);
            final Set r = new HashSet();
            r.addAll(advice.replicates);
            r.addAll(advice.preloaded);
            r.addAll(advice.others);
            r.addAll(advice.empties);
            r.addAll(advice.uninitialized);
            r.addAll(region.activeTXNodes);
            int processorType = targetReinitialized ? DistributionManager.WAITING_POOL_EXECUTOR
                : DistributionManager.HIGH_PRIORITY_EXECUTOR;
            try {
              boolean success = sf.flush(r, recipient, processorType, true);
              if (!success) {
                continue;
              }
            } catch (InterruptedException ie) {
              Thread.currentThread().interrupt();
              region.getCancelCriterion().checkCancelInProgress(ie);
              this.region.getCachePerfStats().endNoGIIDone(giiStart);
              interrupted = true;
              return GIIStatus.NO_GII;
            } finally {
              if (interrupted) {
                unlockTargetBucketLock(prevRecipient);
              }
            }
            if(this.region.getLogWriterI18n().fineEnabled()) {
              this.region.getLogWriterI18n().fine("Completed state flush operation on the parent bucket.");
            }
          }
        }
        boolean interrupted = false;
        final StateFlushOperation sf;
        sf = new StateFlushOperation(this.region);
        final Set r = new HashSet();
        r.addAll(advice.replicates);
        r.addAll(advice.preloaded);
        r.addAll(advice.others);
        r.addAll(advice.empties);
        r.addAll(advice.uninitialized);
        r.addAll(region.activeTXNodes);
        int processorType = targetReinitialized ? DistributionManager.WAITING_POOL_EXECUTOR
            : DistributionManager.HIGH_PRIORITY_EXECUTOR;
        try {

          if (internalNoGII != null && internalNoGII.getRegionName().equals(region.getName())) {
            Thread.currentThread().interrupt();
          }

          boolean success = sf.flush(r, recipient, processorType, false);
          if (!success) {
            continue;
          }
        } catch (InterruptedException ie) {
          Thread.currentThread().interrupt();
          region.getCancelCriterion().checkCancelInProgress(ie);
          this.region.getCachePerfStats().endNoGIIDone(giiStart);
          interrupted = true;
          return GIIStatus.NO_GII;
        } finally {
          if (interrupted) {
            unlockTargetBucketLock(prevRecipient);
          }
        }
      }
      
      RequestImageMessage m = new RequestImageMessage();
      m.regionPath = this.region.getFullPath();
      m.keysOnly = false;
      m.targetReinitialized = targetReinitialized;
      m.setRecipient(recipient);

      if (this.region.concurrencyChecksEnabled) {
        if (allowDeltaGII && recoveredFromDisk) {
          if (!this.region.getDiskRegion().getRVVTrusted()) {
            this.region.getCache().getLoggerI18n().info(LocalizedStrings.DEBUG, "Region " + this.region.getFullPath() + " recovered without EndGII flag, do full GII");
            m.versionVector = null;
          } else if (keysOfUnfinishedOps.size() > MAXIMUM_UNFINISHED_OPERATIONS) {
            this.region.getCache().getLoggerI18n().info(LocalizedStrings.DEBUG, "Region " + this.region.getFullPath() + " has "+keysOfUnfinishedOps.size()
                +" unfinished operations, which exceeded threshold "+MAXIMUM_UNFINISHED_OPERATIONS+", do full GII instead");
            m.versionVector = null;
          } else {
            if (recoveredRVV.isNewerThanOrCanFillExceptionsFor(remote_rvv)) {
              m.versionVector = null;
              this.region.getCache().getLoggerI18n().info(LocalizedStrings.DEBUG, "Region " + this.region.getFullPath() 
                  + ": after filled versions of unfinished keys, recovered rvv is still newer than remote rvv:"
                  +remote_rvv+". recovered rvv is "+recoveredRVV+". Do full GII");
            } else {
              m.versionVector = recoveredRVV;
              m.unfinishedKeys = keysOfUnfinishedOps;
              this.region.getCache().getLoggerI18n().info(LocalizedStrings.DEBUG, "Region " + this.region.getFullPath() + " recovered with EndGII flag, rvv is "
              +m.versionVector+". recovered rvv is "+recoveredRVV+". Do delta GII");
            }
          }
          m.checkTombstoneVersions = true;
        }
        // pack the original RVV, then save the received one
        if (internalBeforeSavedReceivedRVV != null && internalBeforeSavedReceivedRVV.getRegionName().equals(this.region.getName())) {
          internalBeforeSavedReceivedRVV.run();
        }
        saveReceivedRVV(received_rvv);
        if (internalAfterSavedReceivedRVV != null && internalAfterSavedReceivedRVV.getRegionName().equals(this.region.getName())) {
          internalAfterSavedReceivedRVV.run();
        }
      }

      ImageProcessor processor = new ImageProcessor(this.region.getSystem(),
                                                    recipient);
      dm.acquireGIIPermitUninterruptibly();
      try {
        m.processorId = processor.getProcessorId();
        if (region.isUsedForPartitionedRegionBucket() &&
            region.getDistributionConfig().getAckSevereAlertThreshold() > 0) {
          processor.enableSevereAlertProcessing();
          m.severeAlertEnabled = true;
        }
        if (!LocalRegion.isMetaTable(this.region.getFullPath())) {
          // do not remove the following log statement
          this.region.getCache().getLoggerI18n().info(
                  LocalizedStrings.InitialImageOperation_REGION_0_REQUESTING_INITIAL_IMAGE_FROM_1,
                  new Object[]{this.region.getName(), recipient});
        }

        dm.putOutgoing(m);
        this.region.cache.getCancelCriterion().checkCancelInProgress(null);
        if (internalAfterSentRequestImage != null && internalAfterSentRequestImage.getRegionName().equals(this.region.getName())) {
          internalAfterSentRequestImage.run();
        }
        try {
          processor.waitForRepliesUninterruptibly();
          if(!this.gotImage) {
            continue;
          }
          
          // review unfinished keys and remove untouched entries
          if (this.region.getDataPolicy().withPersistence() && keysOfUnfinishedOps != null && !keysOfUnfinishedOps.isEmpty()) {
            LogWriterI18n logger = this.region.getLogWriterI18n();
            final DiskRegion dr = this.region.getDiskRegion();
            assert dr != null;
            for (Object key:keysOfUnfinishedOps) {
              RegionEntry re = this.entries.getEntry(key);
              if (re == null) {
                continue;
              }
              if (TRACE_GII) {
                logger.info(LocalizedStrings.DEBUG, "Processing unfinished operation:entry="+re);
              }
              DiskEntry de = (DiskEntry)re;
              synchronized (de) {
                DiskId id = de.getDiskId();
                if (id != null && EntryBits.isRecoveredFromDisk(id.getUserBits())) {
                  this.region.destroyRecoveredEntry(key);
                  this.region.getLogWriterI18n().info(LocalizedStrings.DEBUG, "Deleted unfinished keys:key="+key);
                }
              }
            }
          }

          int numTimesRequested = 0;
          region.writeLockEnqueueDelta();
          try {
            ImageState imgState = region.getImageState();
            while (requestUnappliedDeltas(dm, recipient)) {
              imgState.setRequestedUnappliedDelta(true);
              LogWriterI18n logger = this.region.getLogWriterI18n();
              if (TRACE_GII || logger.fineEnabled()) {
                region.getLogWriterI18n().info(LocalizedStrings.DEBUG,
                    "Requested the base value for unapplied deltas for region: " +
                        this.region + " for " + (++numTimesRequested) + " time");
              }
            }
          } finally {
            region.writeUnlockEnqueueDelta();
          }
          continue;
        } catch (InternalGemFireException ex) {
          Throwable cause = ex.getCause();
          if (cause instanceof com.gemstone.gemfire.cache.TimeoutException) {
            throw (com.gemstone.gemfire.cache.TimeoutException)cause;
          }
          throw ex;
        } catch (ReplyException e) {
          if(!region.isDestroyed()) {
            e.handleAsUnexpected();
          }
        } finally {
          ImageState imgState = region.getImageState();
          
          if (imgState.getClearRegionFlag()) {
            // Asif : Since the operation has been completed clear flag
            imgState.setClearRegionFlag(false, null);
          }
          
          if (this.gotImage) {
            RegionLogger.logGII(this.region.getFullPath(), recipient,
                region.getDistributionManager().getDistributionManagerId(), region.getPersistentID());
          }
          if (this.region.getLogWriterI18n().infoEnabled()) {
            if (this.gotImage) {
              // TODO add localizedString
              if (!LocalRegion.isMetaTable(this.region.getFullPath())) {
                this.region.getLogWriterI18n().info(LocalizedStrings.DEBUG,
                        this.region.getName() + " is done getting image from "
                        + recipient + ". isDeltaGII is " + this.isDeltaGII
                        + " rvv is " + this.region.getVersionVector());
              }
            } else {
              // TODO add localizedString
              this.region.getLogWriterI18n().info(LocalizedStrings.DEBUG, this.region.getName() + " failed to get image from " + recipient);
            }
          }
          if (this.region.dataPolicy.withPersistence()
              && this.region.getLogWriterI18n().infoEnabled()
              && !LocalRegion.isMetaTable(this.region.getFullPath())) {
            this.region.getLogWriterI18n().info(
                    LocalizedStrings.InitialImageOperation_REGION_0_INITIALIZED_PERSISTENT_REGION_WITH_ID_1_FROM_2,
                    new Object[] {this.region.getName(),
                        this.region.getPersistentID(), recipient});
          }
          // bug 39050 - no partial images after GII when network partition
          // detection is enabled
          if (!this.gotImage) {
            this.region.cleanUpAfterFailedGII(recoveredFromDisk);           
          } else if (received_rvv != null) {
            checkForUnrecordedOperations(recipient);
          }
        }
      } finally {
        dm.releaseGIIPermit();
        processor.cleanup();
      }
    } // for

    if (this.gotImage) {
      this.region.getCachePerfStats().endGetInitialImage(giiStart);
      if (this.isDeltaGII) {
        this.region.getCachePerfStats().incDeltaGIICompleted();
      }
    } else {
      this.region.getCachePerfStats().endNoGIIDone(giiStart);
    }
    return reportGIIStatus();
  }
  
  /**
   * Request the base value for any unapplied ListOfDelta objects in the region.
   * With gemfirexd, a concurrent UPDATE statement can leave a ListOfDeltas object
   * in the region on the GII recipient.
   * 
   * With delta GII, it's possible the base value might never arrive. If that
   * happens, we need to explicitly request the base value from the source.
   * @param dm 
   * @return whether request was sent or not
   */
  private boolean requestUnappliedDeltas(DistributionManager dm, InternalDistributedMember recipient) {
    ImageState imgState = region.getImageState();
    LogWriterI18n logger = region.getLogWriterI18n();
    //See if there are any deltas that have not yet been merged
    Iterator