All Downloads are FREE. Search and download functionalities are using the official Maven repository.
  • Log In
  • Register

    • com.google.appengine.api.memcache.AsyncMemcacheServiceImpl Maven / Gradle / Ivy

    There is a newer version: 2.0.31
    Show newest version
    /*
 * Copyright 2021 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.appengine.api.memcache;

import static com.google.appengine.api.memcache.MemcacheServiceApiHelper.makeAsyncCall;

import com.google.appengine.api.NamespaceManager;
import com.google.appengine.api.memcache.MemcacheSerialization.ValueAndFlags;
import com.google.appengine.api.memcache.MemcacheService.CasValues;
import com.google.appengine.api.memcache.MemcacheService.IdentifiableValue;
import com.google.appengine.api.memcache.MemcacheService.ItemForPeek;
import com.google.appengine.api.memcache.MemcacheService.SetPolicy;
import com.google.appengine.api.memcache.MemcacheServiceApiHelper.Provider;
import com.google.appengine.api.memcache.MemcacheServiceApiHelper.RpcResponseHandler;
import com.google.appengine.api.memcache.MemcacheServiceApiHelper.Transformer;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheBatchIncrementRequest;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheBatchIncrementResponse;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheDeleteRequest;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheDeleteResponse;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheDeleteResponse.DeleteStatusCode;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheFlushRequest;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheFlushResponse;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheGetRequest;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheGetResponse;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheIncrementRequest;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheIncrementRequest.Direction;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheIncrementResponse;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheIncrementResponse.IncrementStatusCode;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheServiceError;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheSetRequest;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheSetResponse;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheSetResponse.SetStatusCode;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheStatsRequest;
import com.google.appengine.api.memcache.MemcacheServicePb.MemcacheStatsResponse;
import com.google.appengine.api.memcache.MemcacheServicePb.MergedNamespaceStats;
import com.google.appengine.api.utils.FutureWrapper;
import com.google.apphosting.api.ApiProxy;
import com.google.common.base.Ascii;
import com.google.common.base.Joiner;
import com.google.common.primitives.Bytes;
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

/**
 * Java bindings for the AsyncMemcache service.
 *
 */
class AsyncMemcacheServiceImpl extends BaseMemcacheServiceImpl implements AsyncMemcacheService {
  private static final ErrorHandler DO_NOTHING_ERROR_HANDLER =
      new ErrorHandler() {
        @Override
        public void handleDeserializationError(InvalidValueException ivx) {}

        @Override
        public void handleServiceError(MemcacheServiceException ex) {}
      };
  private static final ErrorHandler DO_NOTHING_CONSISTENT_ERROR_HANDLER =
      new ConsistentErrorHandler() {
        @Override
        public void handleDeserializationError(InvalidValueException ivx) {}

        @Override
        public void handleServiceError(MemcacheServiceException ex) {}
      };

  static class StatsImpl implements Stats {
    private final long hits;
    private final long misses;
    private final long bytesFetched;
    private final long items;
    private final long bytesStored;
    private final int maxCachedTime;

    StatsImpl(MergedNamespaceStats stats) {
      if (stats != null) {
        hits = stats.getHits();
        misses = stats.getMisses();
        bytesFetched = stats.getByteHits();
        items = stats.getItems();
        bytesStored = stats.getBytes();
        maxCachedTime = stats.getOldestItemAge();
      } else {
        hits = misses = bytesFetched = items = bytesStored = 0;
        maxCachedTime = 0;
      }
    }

    @Override
    public long getHitCount() {
      return hits;
    }

    @Override
    public long getMissCount() {
      return misses;
    }

    @Override
    public long getBytesReturnedForHits() {
      return bytesFetched;
    }

    @Override
    public long getItemCount() {
      return items;
    }

    @Override
    public long getTotalItemBytes() {
      return bytesStored;
    }

    @Override
    public int getMaxTimeWithoutAccess() {
      return maxCachedTime;
    }

    @Override
    public String toString() {
      StringBuilder builder = new StringBuilder();
      builder.append("Hits: ").append(hits).append('\n');
      builder.append("Misses: ").append(misses).append('\n');
      builder.append("Bytes Fetched: ").append(bytesFetched).append('\n');
      builder.append("Bytes Stored: ").append(bytesStored).append('\n');
      builder.append("Items: ").append(items).append('\n');
      builder.append("Max Cached Time: ").append(maxCachedTime).append('\n');
      return builder.toString();
    }
  }
 
  //@VisibleForTesting
  static final class ItemForPeekImpl implements MemcacheService.ItemForPeek {
    private final Object value;
    private final Long expirationTimeSec;
    private final Long lastAccessTimeSec;
    private final Long deleteLockTimeSec;
    ItemForPeekImpl(Object value, Long expirationTimeSec, Long lastAccessTimeSec, Long deleteLockTimeSec ) {
      this.value = value;
      this.expirationTimeSec = expirationTimeSec;
      this.lastAccessTimeSec = lastAccessTimeSec;
      this.deleteLockTimeSec = deleteLockTimeSec;
    }

    @Override
    public Object getValue() {
      return value;
    }

    @Override
    public Long getExpirationTimeSec() {
      return expirationTimeSec;
    }

    @Override
    public Long getLastAccessTimeSec() {
      return lastAccessTimeSec;
    }

    @Override
    public Long getDeleteLockTimeSec() {
      return deleteLockTimeSec;
    }

  }
  
  //@VisibleForTesting
  static final class IdentifiableValueImpl implements IdentifiableValue {
    private final Object value;
    private final long casId;

    IdentifiableValueImpl(Object value, long casId) {
      this.value = value;
      this.casId = casId;
    }

    @Override
    public Object getValue() {
      return value;
    }

    //@VisibleForTesting
    long getCasId() {
      return casId;
    }

    @Override
    public boolean equals(Object otherObj) {
      if (this == otherObj) {
        return true;
      }
      if ((otherObj == null) || (getClass() != otherObj.getClass())) {
        return false;
      }
      IdentifiableValueImpl otherIdentifiableValue = (IdentifiableValueImpl) otherObj;
      return Objects.equals(value, otherIdentifiableValue.value) &&
          (casId == otherIdentifiableValue.casId);
    }

    @Override
    public int hashCode() {
      return Objects.hash(value, casId);
    }
  }

  private static class DefaultValueProviders {

    @SuppressWarnings("rawtypes")
    private static final Provider NULL_PROVIDER = new Provider() {
          @Override public Object get() {
            return null;
          }
        };

    private static final Provider FALSE_PROVIDER = new Provider() {
          @Override public Boolean get() {
            return Boolean.FALSE;
          }
        };

    @SuppressWarnings("rawtypes")
    private static final Provider SET_PROVIDER =
        new Provider>() {
          @Override
          public Set get() {
            return new HashSet<>(0, 1);
          }
        };

    @SuppressWarnings("rawtypes")
    private static final Provider MAP_PROVIDER =
        new Provider>() {
          @Override
          public Map get() {
            return new HashMap<>(0, 1);
          }
        };

    private static final Provider STATS_PROVIDER = new Provider() {
          final Stats emptyStats =  new AsyncMemcacheServiceImpl.StatsImpl(null);

          @Override public Stats get() {
            return emptyStats;
          }
        };

    static Provider falseValue() {
      return FALSE_PROVIDER;
    }

    @SuppressWarnings("unchecked")
    static  Provider nullValue() {
      return NULL_PROVIDER;
    }

    @SuppressWarnings("unchecked")
    static  Provider> emptySet() {
      return SET_PROVIDER;
    }

    @SuppressWarnings("unchecked")
    static  Provider> emptyMap() {
      return MAP_PROVIDER;
    }

    static Provider emptyStats() {
      return STATS_PROVIDER;
    }
  }

  private static class VoidFutureWrapper extends FutureWrapper {

    private VoidFutureWrapper(Future parent) {
      super(parent);
    }

    private static  Future wrap(Future parent) {
      return new VoidFutureWrapper(parent);
    }

    @Override
    protected Void wrap(K value) {
      return null;
    }

    @Override
    protected Throwable convertException(Throwable cause) {
      return cause;
    }
  }

  private static final class KeyValuePair {
    private final K key;
    private final V value;

    private KeyValuePair(K key, V value) {
      this.key = key;
      this.value = value;
    }

    static  KeyValuePair of(K key, V value) {
      return new KeyValuePair(key, value);
    }
  }

  AsyncMemcacheServiceImpl(String namespace) {
    super(namespace);
  }

  static  Map makeMap(Collection keys, V value) {
    Map map = new LinkedHashMap(keys.size(), 1);
    for (T key : keys) {
      map.put(key, value);
    }
    return map;
  }

  private static ByteString makePbKey(Object key) {
    try {
      return ByteString.copyFrom(MemcacheSerialization.makePbKey(key));
    } catch (IOException ex) {
      throw new IllegalArgumentException("Cannot use as a key: '" + key + "'", ex);
    }
  }

  private static ValueAndFlags serializeValue(Object value) {
    try {
      return MemcacheSerialization.serialize(value);
    } catch (IOException ex) {
      throw new IllegalArgumentException("Cannot use as value: '" + value + "'", ex);
    }
  }

  private Object deserializeItem(Object key, MemcacheGetResponse.Item item) {
    try {
      return MemcacheSerialization.deserialize(item.getValue().toByteArray(), item.getFlags());
    } catch (ClassNotFoundException ex) {
      getErrorHandler().handleDeserializationError(
          new InvalidValueException("Can't find class for value of key '" + key + "'", ex));
    } catch (IOException ex) {
      getErrorHandler().handleDeserializationError(
          new InvalidValueException("Failed to parse the value for '" + key + "'", ex));
    }
    return null;
  }

  private  RpcResponseHandler createRpcResponseHandler(
      M response, String errorText, Transformer responseTransformer) {
    return new RpcResponseHandler(
        response, errorText, responseTransformer, getErrorHandler());
  }

  private  RpcResponseHandlerForPut createRpcResponseHandlerForPut(
      Iterable itemBuilders, String namespace,
      MemcacheSetResponse response, String errorText,
      Transformer responseTransformer) {
    return new RpcResponseHandlerForPut(
        itemBuilders, namespace, response, errorText, responseTransformer, getErrorHandler());
  }

  private class RpcResponseHandlerForPut extends RpcResponseHandler {

    /**
     * We remember what was sent to the backend, so that if we get an error response we can test to
     * see if the error was caused by the API being given invalid values.
     */
    private final Iterable itemsSentToBackend;
    private final String namespace;

    RpcResponseHandlerForPut(Iterable itemsSentToBackend,
        String namespace,
        MemcacheSetResponse response, String errorText,
        Transformer responseTransfomer,
        ErrorHandler errorHandler) {
      super(response, errorText, responseTransfomer, errorHandler);
      this.itemsSentToBackend = itemsSentToBackend;
      this.namespace = namespace;
    }

    /**
     * When we get an error from the backend we check to see if it could possibly have been caused
     * by an invalid key or value being passed into the API from the app. If so we test the original
     * key and value passed in and if we find them to be invalid we throw an exception that is more
     * informative to the app writer than the default exception.
     */
    @Override
    void handleApiProxyException(Throwable cause) throws Exception {
      ErrorHandler errorHandler = getErrorHandler();
      if (cause instanceof ApiProxy.ApplicationException) {
        ApiProxy.ApplicationException applicationException = (ApiProxy.ApplicationException) cause;
        if (applicationException.getApplicationError()
            == MemcacheServiceError.ErrorCode.UNSPECIFIED_ERROR_VALUE) {
          errorHandler =
              handleApiProxyException(applicationException.getErrorDetail(), errorHandler);
        }
      } else if (cause instanceof MemcacheServiceException){
        errorHandler = handleApiProxyException(cause.getMessage(), errorHandler);
      }
      super.handleApiProxyException(cause, errorHandler);
    }

    private ErrorHandler handleApiProxyException(String msg,
        ErrorHandler errorHandler) throws MemcacheServiceException {
      boolean errorHandlerCalled = false;
      for (MemcacheSetRequest.Item.Builder itemBuilder : itemsSentToBackend) {
        ByteString pbKey = itemBuilder.getKey();
        ByteString value = itemBuilder.getValue();
        // Do cheaper test first:
        if (value.size() + pbKey.size() > ITEM_SIZE_LIMIT) {
          errorHandlerCalled |=
              maybeThrow(errorHandler, "Key+value is bigger than maximum allowed. " + msg);
        }
        // This does a slightly expensive scanning of a key for null, but it should happen
        // rarely:
        if (Bytes.contains(pbKey.toByteArray(), (byte) 0)) {
          errorHandlerCalled |= maybeThrow(errorHandler, "Key contains embedded null byte. " + msg);
        }
        // Finally, test for the unlikely case of a bad namespace:
        if (namespace != null) {
          try {
            NamespaceManager.validateNamespace(namespace);
          } catch (IllegalArgumentException ex) {
            // Can only happen if the app writer uses deprecated or undocumented calls to bypass
            // the outgoing checks.
            errorHandlerCalled |= maybeThrow(errorHandler, ex.toString());
          }
        }
      }
      if (errorHandlerCalled) {
        // Prevent super.handleApiProxyException from handling the error twice. Since it checks
        // which interface the handler inherits, we must do the same.
        if (errorHandler instanceof ConsistentErrorHandler) {
          return DO_NOTHING_CONSISTENT_ERROR_HANDLER;
        } else {
          return DO_NOTHING_ERROR_HANDLER;
        }
      } else {
        return errorHandler;
      }
    }

    /**
     * Will throw exception or just log it, depending on what error handler is set.
     *
     * @return true if handleServiceError was called.
     */
    private boolean maybeThrow(ErrorHandler errorHandler, String msg) {
      // TODO replace MemcacheServiceException with new MemcachePutException
      errorHandler.handleServiceError(new MemcacheServiceException(msg));
      return true;
    }
  }

  /**
   * Matches limit documented in
   * https://developers.google.com/appengine/docs/python/memcache/#Python_Limits
   *
   * 
    TODO This is too conservative; need to revisit. The docs are conflating item size limit with
   * total cache size accounting.
   */
  static final int ITEM_SIZE_LIMIT = 1024 * 1024 - 96;

  @Override
  public Future contains(Object key) {
    return doGet(
        key,
        false, // forCas
        false, // forPeek
        "Memcache contains: exception testing contains (" + key + ")",
        new Transformer() {
          @Override
          public Boolean transform(MemcacheGetResponse response) {
            return response.getItemCount() > 0;
          }
        },
        DefaultValueProviders.falseValue());
  }

  private  Future doGet(Object key, boolean forCas, boolean forPeek, String errorText,
      final Transformer responseTransfomer, Provider defaultValue) {
    MemcacheGetRequest.Builder requestBuilder = MemcacheGetRequest.newBuilder();
    requestBuilder.addKey(makePbKey(key));
    requestBuilder.setNameSpace(getEffectiveNamespace());
    if (forCas) {
      requestBuilder.setForCas(true);
    }
    if (forPeek) {
      requestBuilder.setForPeek(true);
    }
    return makeAsyncCall("Get", requestBuilder.build(),
        createRpcResponseHandler(MemcacheGetResponse.getDefaultInstance(), errorText,
            responseTransfomer), defaultValue);
  }

  @Override
  public Future get(final Object key) {
    return doGet(
        key,
        false, // forCas
        false, // forPeek
        "Memcache get: exception getting 1 key (" + key + ")",
        new Transformer() {
          @Override
          public Object transform(MemcacheGetResponse response) {
            return response.getItemCount() == 0 ? null : deserializeItem(key, response.getItem(0));
          }
        },
        DefaultValueProviders.nullValue());
  }

  @Override
  public Future getIdentifiable(final Object key) {
    return doGet(
        key,
        true,  // forCas
        false, // forPeek
        "Memcache getIdentifiable: exception getting 1 key (" + key + ")",
        new IdentifiableTransformer(key),
        DefaultValueProviders.nullValue());
  }

  private class IdentifiableTransformer
      implements Transformer {
    private final Object key;

    IdentifiableTransformer(Object key) {
      this.key = key;
    }

    @Override
    public IdentifiableValue transform(MemcacheGetResponse response) {
      if (response.getItemCount() == 0) {
        return null;
      }
      MemcacheGetResponse.Item item = response.getItem(0);
      return new IdentifiableValueImpl(deserializeItem(key, item), item.getCasId());
    }
  }

  @Override
  public  Future> getIdentifiables(Collection keys) {
    return doGetAll(
        keys,
        true,  // forCas
        false, // forKeep
        "Memcache getIdentifiables: exception getting multiple keys",
        new Transformer, IdentifiableValue>() {
          @Override
          public IdentifiableValue transform(KeyValuePair pair) {
            MemcacheGetResponse.Item item = pair.value;
            Object value = deserializeItem(pair.key, item);
            return new IdentifiableValueImpl(value, item.getCasId());
          }
        },
        DefaultValueProviders.emptyMap());
  }

  
  @Override
  public Future getItemForPeek(final Object key) {
    return doGet(
        key,
        false, // forCas
        true,  // forPeek
        "Memcache getItemForPeek: exception getting 1 key (" + key + ")",
        new ItemForPeekTransformer(key),
        DefaultValueProviders.nullValue());
  }

  private class ItemForPeekTransformer
      implements Transformer {
    private final Object key;

    ItemForPeekTransformer(Object key) {
      this.key = key;
    }

    @Override
    public ItemForPeek transform(MemcacheGetResponse response) {
      if (response.getItemCount() == 0) {
        return null;
      }
      MemcacheGetResponse.Item item = response.getItem(0);
      Object value;
      if (item.hasIsDeleteLocked() && item.getIsDeleteLocked()) {
        value = null;
      } else {
        value = deserializeItem(key, item);
      }

      return new ItemForPeekImpl(
          value,
          item.hasTimestamps() ? item.getTimestamps().getExpirationTimeSec() : null,
          item.hasTimestamps() ? item.getTimestamps().getLastAccessTimeSec() : null,
          item.hasTimestamps() ? item.getTimestamps().getDeleteLockTimeSec() : null);
    }
  }

  @Override
  public  Future> getItemsForPeek(Collection keys) {
    return doGetAll(
        keys,
        false, // forCas
        true, // forPeek
        "Memcache getItemsForPeek: exception getting multiple keys",
        new Transformer, MemcacheService.ItemForPeek>() {
          @Override
          public MemcacheService.ItemForPeek transform(
              KeyValuePair pair) {
            MemcacheGetResponse.Item item = pair.value;
            Object value;
            if (item.hasIsDeleteLocked() && item.getIsDeleteLocked()) {
              value = null;
            } else {
              value = deserializeItem(pair.key, item);
            }

            return new ItemForPeekImpl(
                value,
                item.hasTimestamps() ? item.getTimestamps().getExpirationTimeSec() : null,
                item.hasTimestamps() ? item.getTimestamps().getLastAccessTimeSec() : null,
                item.hasTimestamps() ? item.getTimestamps().getDeleteLockTimeSec() : null);
          }
        },
        DefaultValueProviders.emptyMap());
  }

  @Override
  public  Future> getAll(Collection keys) {
    return doGetAll(
        keys,
        false, // forCas
        false, // forPeek
        "Memcache getAll: exception getting multiple keys",
        new Transformer, Object>() {
          @Override
          public Object transform(KeyValuePair pair) {
            return deserializeItem(pair.key, pair.value);
          }
        },
        DefaultValueProviders.emptyMap());
  }

  private  Future> doGetAll(Collection keys, boolean forCas, boolean forPeek,
      String errorText,
      Transformer, V> responseTransformer,
      Provider> defaultValue) {
    MemcacheGetRequest.Builder requestBuilder = MemcacheGetRequest.newBuilder();
    requestBuilder.setNameSpace(getEffectiveNamespace());
    final Map byteStringToKey = new HashMap(keys.size(), 1);
    for (K key : keys) {
      ByteString pbKey = makePbKey(key);
      byteStringToKey.put(pbKey, key);
      requestBuilder.addKey(pbKey);
    }
    if (forCas) {
      requestBuilder.setForCas(forCas);
    }
    if (forPeek) {
      requestBuilder.setForPeek(forPeek);
    }
    Transformer> rpcResponseTransformer =
        new GetAllRpcResponseTransformer<>(byteStringToKey, responseTransformer);
    return makeAsyncCall(
        "Get",
        requestBuilder.build(),
        createRpcResponseHandler(
            MemcacheGetResponse.getDefaultInstance(), errorText, rpcResponseTransformer),
        defaultValue);
  }

  private static class GetAllRpcResponseTransformer
      implements Transformer> {
    private final Map byteStringToKey;
    private final Transformer, V> responseTransformer;

    GetAllRpcResponseTransformer(
        Map byteStringToKey,
        Transformer, V> responseTransformer) {
      this.byteStringToKey = byteStringToKey;
      this.responseTransformer = responseTransformer;
    }

    @Override
    public Map transform(MemcacheGetResponse response) {
      Map result = new HashMap();
      for (MemcacheGetResponse.Item item : response.getItemList()) {
        K key = byteStringToKey.get(item.getKey());
        V obj = responseTransformer.transform(KeyValuePair.of(key, item));
        result.put(key, obj);
      }
      return result;
    }
  }

  /** Use this to make sure we don't write arbitrarily big log messages. */
  private static final int MAX_LOGGED_VALUE_SIZE = 100;

  /**
   * Represents the overhead to an item as it is stored in memcacheg
   * if it has a value bigger than 65535 bytes and all the optional fields are set.
   * This number was determined by fiddling with cacheserving/memcacheg/server:item_test
   * and analyzing CalculateItemSize from cacheserving/memcacheg/server/item.cc.
   *
   * 
    The overhead can be between 61 and 73 bytes depending on whether optional
   * fields (flag, expiration and CAS) are set or not, adding 4 bytes for each of
   * them.
   */
  private static final int MEMCACHEG_OVERHEAD = 73;
  private static final int ONE_MEGABYTE = 1024 * 1024;
  public static final int MAX_ITEM_SIZE = ONE_MEGABYTE - MEMCACHEG_OVERHEAD;

  /**
   * Note: non-null oldValue implies Compare-and-Swap operation.
   */
  private Future doPut(Object key, IdentifiableValue oldValue, Object value,
      Expiration expires, MemcacheSetRequest.SetPolicy policy) {
    MemcacheSetRequest.Builder requestBuilder = MemcacheSetRequest.newBuilder();
    requestBuilder.setNameSpace(getEffectiveNamespace());
    MemcacheSetRequest.Item.Builder itemBuilder = MemcacheSetRequest.Item.newBuilder();
    ValueAndFlags vaf = serializeValue(value);
    itemBuilder.setValue(ByteString.copyFrom(vaf.value));
    itemBuilder.setFlags(vaf.flags.ordinal());
    itemBuilder.setKey(makePbKey(key));
    itemBuilder.setExpirationTime(expires == null ? 0 : expires.getSecondsValue());
    itemBuilder.setSetPolicy(policy);
    if (policy == MemcacheSetRequest.SetPolicy.CAS) {
      if (oldValue == null) {
        throw new IllegalArgumentException("oldValue must not be null.");
      }
      if (!(oldValue instanceof IdentifiableValueImpl)) {
        throw new IllegalArgumentException(
            "oldValue is an instance of an unapproved IdentifiableValue implementation.  " +
            "Perhaps you implemented your own version of IdentifiableValue?  " +
            "If so, don't do this.");
      }
      itemBuilder.setCasId(((IdentifiableValueImpl) oldValue).getCasId());
    }
    final int itemSize = itemBuilder.getKey().size() + itemBuilder.getValue().size();
    requestBuilder.addItem(itemBuilder);

    // When creating string for logging truncate with ellipsis if necessary.
    String valueAsString =
        Ascii.truncate(String.valueOf(value), MAX_LOGGED_VALUE_SIZE, "...");
    return makeAsyncCall(
        "Set",
        requestBuilder.build(),
        createRpcResponseHandlerForPut(
            Arrays.asList(itemBuilder),
            requestBuilder.getNameSpace(),
            MemcacheSetResponse.getDefaultInstance(),
            String.format("Memcache put: exception setting 1 key (%s) to '%s'", key, valueAsString),
            new PutResponseTransformer(key, itemSize)),
        DefaultValueProviders.falseValue());
  }

  private static class PutResponseTransformer implements Transformer {
    private final Object key;
    private final int itemSize;

    PutResponseTransformer(Object key, int itemSize) {
      this.key = key;
      this.itemSize = itemSize;
    }

    @Override public Boolean transform(MemcacheSetResponse response) {
      if (response.getSetStatusCount() != 1) {
        throw new MemcacheServiceException("Memcache put: Set one item, got "
            + response.getSetStatusCount() + " response statuses");
      }
      SetStatusCode status = response.getSetStatus(0);
      if (status == SetStatusCode.ERROR) {
        if (itemSize > MAX_ITEM_SIZE) {
          throw new MemcacheServiceException(
              String.format("Memcache put: Item may not be more than %d bytes in length; "
                  + "received %d bytes.", MAX_ITEM_SIZE, itemSize));
        }
        throw new MemcacheServiceException(
            "Memcache put: Error setting single item (" + key + ")");
      }
      return status == SetStatusCode.STORED;
    }
  }

  private static MemcacheSetRequest.SetPolicy convertSetPolicyToPb(SetPolicy policy) {
    switch (policy) {
      case SET_ALWAYS:
        return MemcacheSetRequest.SetPolicy.SET;
      case ADD_ONLY_IF_NOT_PRESENT:
        return MemcacheSetRequest.SetPolicy.ADD;
      case REPLACE_ONLY_IF_PRESENT:
        return MemcacheSetRequest.SetPolicy.REPLACE;
    }
    throw new IllegalArgumentException("Unknown policy: " + policy);
  }

  @Override
  public Future put(Object key, Object value, Expiration expires, SetPolicy policy) {
    return doPut(key, null, value, expires, convertSetPolicyToPb(policy));
  }

  @Override
  public Future put(Object key, Object value, Expiration expires) {
    return VoidFutureWrapper.wrap(
        doPut(key, null, value, expires, MemcacheSetRequest.SetPolicy.SET));
  }

  @Override
  public Future put(Object key, Object value) {
    return VoidFutureWrapper.wrap(
        doPut(key, null, value, null, MemcacheSetRequest.SetPolicy.SET));
  }

  @Override
  public Future putIfUntouched(Object key, IdentifiableValue oldValue,
      Object newValue, Expiration expires) {
    return doPut(key, oldValue, newValue, expires, MemcacheSetRequest.SetPolicy.CAS);
  }

  @Override
  public Future putIfUntouched(Object key, IdentifiableValue oldValue, Object newValue) {
    return doPut(key, oldValue, newValue, null, MemcacheSetRequest.SetPolicy.CAS);
  }

  @Override
  public  Future> putIfUntouched(Map values) {
    return doPutAll(values, null, MemcacheSetRequest.SetPolicy.CAS, "putIfUntouched");
  }

  @Override
  public  Future> putIfUntouched(Map values, Expiration expiration) {
    return doPutAll(values, expiration, MemcacheSetRequest.SetPolicy.CAS, "putIfUntouched");
  }

  private  Future> doPutAll(Map values, Expiration expires,
      MemcacheSetRequest.SetPolicy policy, String operation) {
    MemcacheSetRequest.Builder requestBuilder = MemcacheSetRequest.newBuilder();
    requestBuilder.setNameSpace(getEffectiveNamespace());

    List requestedKeys = new ArrayList(values.size());
    Set oversized = new HashSet<>();
    int itemIndex = 0;

    for (Map.Entry entry : values.entrySet()) {
      MemcacheSetRequest.Item.Builder itemBuilder = MemcacheSetRequest.Item.newBuilder();
      requestedKeys.add(entry.getKey());
      itemBuilder.setKey(makePbKey(entry.getKey()));
      ValueAndFlags vaf;
      if (policy == MemcacheSetRequest.SetPolicy.CAS) {
        CasValues value = (CasValues) entry.getValue();
        if (value == null) {
          throw new IllegalArgumentException(entry.getKey() + " has a null for CasValues");
        }
        vaf = serializeValue(value.getNewValue());
        if (!(value.getOldValue() instanceof IdentifiableValueImpl)) {
          throw new IllegalArgumentException(
            entry.getKey() + " CasValues has an oldValue instance of an unapproved " +
            "IdentifiableValue implementation.  Perhaps you implemented your own " +
            "version of IdentifiableValue?  If so, don't do this.");
        }
        itemBuilder.setCasId(((IdentifiableValueImpl) value.getOldValue()).getCasId());
        if (value.getExipration() != null) {
          itemBuilder.setExpirationTime(value.getExipration().getSecondsValue());
        } else {
          itemBuilder.setExpirationTime(expires == null ? 0 : expires.getSecondsValue());
        }
      } else {
        vaf = serializeValue(entry.getValue());
        itemBuilder.setExpirationTime(expires == null ? 0 : expires.getSecondsValue());
      }
      itemBuilder.setValue(ByteString.copyFrom(vaf.value));
      itemBuilder.setFlags(vaf.flags.ordinal());
      itemBuilder.setSetPolicy(policy);
      requestBuilder.addItem(itemBuilder);

      int itemSize = itemBuilder.getKey().size() + itemBuilder.getValue().size();
      if (itemSize > MAX_ITEM_SIZE) {
        oversized.add(itemIndex);
      }
      itemIndex++;
    }

    return makeAsyncCall(
        "Set",
        requestBuilder.build(),
        createRpcResponseHandlerForPut(
            requestBuilder.getItemBuilderList(),
            requestBuilder.getNameSpace(),
            MemcacheSetResponse.getDefaultInstance(),
            "Memcache " + operation + ": Unknown exception setting " + values.size() + " keys",
            new PutAllResponseTransformer<>(requestedKeys, oversized)),
        DefaultValueProviders.emptySet());
  }

  private static class PutAllResponseTransformer
      implements Transformer> {
    private final List requestedKeys;
    private final Set oversized;

    PutAllResponseTransformer(List requestedKeys, Set oversized) {
      this.requestedKeys = requestedKeys;
      this.oversized = oversized;
    }

    @Override
    public Set transform(MemcacheSetResponse response) {
      if (response.getSetStatusCount() != requestedKeys.size()) {
        throw new MemcacheServiceException(
            String.format(
                "Memcache put: Set %d items, got %d response statuses",
                requestedKeys.size(), response.getSetStatusCount()));
      }
      HashSet result = new HashSet<>();
      HashSet sizeErrors = new HashSet<>();
      HashSet otherErrors = new HashSet<>();
      Iterator statusIter = response.getSetStatusList().iterator();
      int itemIndex = 0;

      for (T requestedKey : requestedKeys) {
        SetStatusCode status = statusIter.next();
        if (status == MemcacheSetResponse.SetStatusCode.ERROR) {
          if (oversized.contains(itemIndex)) {
            sizeErrors.add(requestedKey);
          } else {
            otherErrors.add(requestedKey);
          }
        } else if (status == MemcacheSetResponse.SetStatusCode.STORED) {
          result.add(requestedKey);
        }
        itemIndex++;
      }
      // Throw exception if there is any error.
      if (!sizeErrors.isEmpty() || !otherErrors.isEmpty()) {
        StringBuilder builder = new StringBuilder("Memcache put: ");
        if (!sizeErrors.isEmpty()) {
          builder.append(sizeErrors.size());
          builder.append(" items failed for exceeding ");
          builder.append(MAX_ITEM_SIZE).append(" bytes; keys: ");
          builder.append(Joiner.on(", ").join(sizeErrors));
          builder.append(". ");
        }
        if (!otherErrors.isEmpty()) {
          builder.append("Set failed to set ");
          builder.append(otherErrors.size()).append(" keys: ");
          builder.append(Joiner.on(", ").join(otherErrors));
        }
        throw new MemcacheServiceException(builder.toString());
      }
      return result;
    }
  }

  @Override
  public  Future> putAll(Map values, Expiration expires, SetPolicy policy) {
    return doPutAll(values, expires, convertSetPolicyToPb(policy), "putAll");
  }

  @Override
  public Future putAll(Map values, Expiration expires) {
    return VoidFutureWrapper.wrap(
        doPutAll(values, expires, MemcacheSetRequest.SetPolicy.SET, "putAll"));
  }

  @Override
  public Future putAll(Map values) {
    return VoidFutureWrapper.wrap(
        doPutAll(values, null, MemcacheSetRequest.SetPolicy.SET, "putAll"));
  }

  @Override
  public Future delete(Object key) {
    return delete(key, 0);
  }

  @Override
  public Future delete(Object key, long millisNoReAdd) {
    MemcacheDeleteRequest request = MemcacheDeleteRequest.newBuilder()
        .setNameSpace(getEffectiveNamespace())
        .addItem(MemcacheDeleteRequest.Item.newBuilder()
            .setKey(makePbKey(key))
            .setDeleteTime((int) TimeUnit.SECONDS.convert(millisNoReAdd, TimeUnit.MILLISECONDS)))
        .build();
    return makeAsyncCall(
        "Delete",
        request,
        createRpcResponseHandler(
            MemcacheDeleteResponse.getDefaultInstance(),
            "Memcache delete: Unknown exception deleting key: " + key,
            new Transformer() {
              @Override
              public Boolean transform(MemcacheDeleteResponse response) {
                return response.getDeleteStatus(0) == DeleteStatusCode.DELETED;
              }
            }),
        DefaultValueProviders.falseValue());
  }

  @Override
  public  Future> deleteAll(Collection keys) {
    return deleteAll(keys, 0);
  }

  @Override
  public  Future> deleteAll(Collection keys, long millisNoReAdd) {
    MemcacheDeleteRequest.Builder requestBuilder =
        MemcacheDeleteRequest.newBuilder().setNameSpace(getEffectiveNamespace());
    List requestedKeys = new ArrayList(keys.size());
    for (T key : keys) {
      requestedKeys.add(key);
      requestBuilder.addItem(MemcacheDeleteRequest.Item.newBuilder()
                                 .setDeleteTime((int) (millisNoReAdd / 1000))
                                 .setKey(makePbKey(key)));
    }
    return makeAsyncCall(
        "Delete",
        requestBuilder.build(),
        createRpcResponseHandler(
            MemcacheDeleteResponse.getDefaultInstance(),
            "Memcache deleteAll: Unknown exception deleting multiple keys",
            new MemcacheDeleteResponseTransformer<>(requestedKeys)),
        DefaultValueProviders.emptySet());
  }

  private static class MemcacheDeleteResponseTransformer
      implements Transformer> {
    private final List requestedKeys;

    MemcacheDeleteResponseTransformer(List requestedKeys) {
      this.requestedKeys = requestedKeys;
    }

    @Override
    public Set transform(MemcacheDeleteResponse response) {
      Set retval = new LinkedHashSet();
      Iterator requestedKeysIter = requestedKeys.iterator();
      for (DeleteStatusCode deleteStatus : response.getDeleteStatusList()) {
        T requestedKey = requestedKeysIter.next();
        if (deleteStatus == DeleteStatusCode.DELETED) {
          retval.add(requestedKey);
        }
      }
      return retval;
    }
  }

  private static MemcacheIncrementRequest.Builder newIncrementRequestBuilder(
      Object key, long delta, Long initialValue) {
    MemcacheIncrementRequest.Builder requestBuilder = MemcacheIncrementRequest.newBuilder();
    requestBuilder.setKey(makePbKey(key));
    if (delta > 0) {
      requestBuilder.setDirection(Direction.INCREMENT);
      requestBuilder.setDelta(delta);
    } else {
      requestBuilder.setDirection(Direction.DECREMENT);
      requestBuilder.setDelta(-delta);
    }
    if (initialValue != null) {
      requestBuilder.setInitialValue(initialValue);
      requestBuilder.setInitialFlags(MemcacheSerialization.Flag.LONG.ordinal());
    }
    return requestBuilder;
  }

  @Override
  public Future increment(Object key, long delta) {
    return increment(key, delta, null);
  }

  @Override
  public Future increment(Object key, long delta, Long initialValue) {
    MemcacheIncrementRequest request = newIncrementRequestBuilder(key, delta, initialValue)
        .setNameSpace(getEffectiveNamespace())
        .build();
    return makeAsyncCall(
        "Increment",
        request,
        new RpcResponseHandlerForIncrement(key, getErrorHandler()),
        DefaultValueProviders.nullValue());
  }

  private static class RpcResponseHandlerForIncrement
      extends RpcResponseHandler {
    private final Object key;
    private final ErrorHandler errorHandler;

    RpcResponseHandlerForIncrement(Object key, ErrorHandler errorHandler) {
      super(
          MemcacheIncrementResponse.getDefaultInstance(),
          "Memcache increment: exception when incrementing key '" + key + "'",
          new IncrementResponseTransformer(),
          errorHandler);
      this.key = key;
      this.errorHandler = errorHandler;
    }

    @Override
    void handleApiProxyException(Throwable cause) throws Exception {
      // Note the custom error handling here.  Typically we delegate to the
      // error handler to deal with exceptions that are thrown from
      // makeSyncCall, but use of the error handler requires that there be
      // some scenario where it is reasonable for the api to swallow the
      // exception and behave as if the cached data simply isn't available
      // (memcache is unavailable for example).
      // In this case, however, the programmer has either asked for the wrong
      // thing or is wrong about the type of data associated with the provided
      // key, so it never makes sense to swallow the exception.
      if (cause instanceof ApiProxy.ApplicationException) {
        ApiProxy.ApplicationException applicationException = (ApiProxy.ApplicationException) cause;
        getLogger().info(applicationException.getErrorDetail());
        if (applicationException.getApplicationError()
            == MemcacheServiceError.ErrorCode.INVALID_VALUE_VALUE) {
          // value is not incrementable
          throw new InvalidValueException("Non-incrementable value for key '" + key + "'");
        }
      }
      super.handleApiProxyException(cause);
    }

    private static class IncrementResponseTransformer
        implements Transformer {
      @Override
      public Long transform(MemcacheIncrementResponse response) {
        return response.hasNewValue() ? response.getNewValue() : null;
      }
    }
  }

  @Override
  public  Future> incrementAll(Collection keys, long delta) {
    return incrementAll(keys, delta, null);
  }

  @Override
  public  Future> incrementAll(Collection keys, long delta, Long initialValue) {
    return incrementAll(makeMap(keys, delta), initialValue);
  }

  @Override
  public  Future> incrementAll(Map offsets) {
    return incrementAll(offsets, null);
  }

  @Override
  public  Future> incrementAll(Map offsets, Long initialValue) {
    MemcacheBatchIncrementRequest.Builder requestBuilder =
        MemcacheBatchIncrementRequest.newBuilder().setNameSpace(getEffectiveNamespace());
    final List requestedKeys = new ArrayList(offsets.size());
    for (Map.Entry entry : offsets.entrySet()) {
      requestedKeys.add(entry.getKey());
      requestBuilder.addItem(
          newIncrementRequestBuilder(entry.getKey(), entry.getValue(), initialValue));
    }
    Provider> defaultValue = new Provider>() {
      @Override
      public Map get() {
        return makeMap(requestedKeys, null);
      }
    };
    return makeAsyncCall(
        "BatchIncrement",
        requestBuilder.build(),
        createRpcResponseHandler(
            MemcacheBatchIncrementResponse.getDefaultInstance(),
            "Memcache incrementAll: exception incrementing multiple keys",
            new MemcacheBatchIncrementResponseTransformer<>(requestedKeys)),
        defaultValue);
  }

  private static class MemcacheBatchIncrementResponseTransformer
      implements Transformer> {
    private final List requestedKeys;

    MemcacheBatchIncrementResponseTransformer(List requestedKeys) {
      this.requestedKeys = requestedKeys;
    }

    @Override
    public Map transform(MemcacheBatchIncrementResponse response) {
      Map result = new LinkedHashMap<>();
      Iterator items = response.getItemList().iterator();
      for (T requestedKey : requestedKeys) {
        MemcacheIncrementResponse item = items.next();
        if (item.getIncrementStatus().equals(IncrementStatusCode.OK) && item.hasNewValue()) {
          result.put(requestedKey, item.getNewValue());
        } else {
          result.put(requestedKey, null);
        }
      }
      return result;
    }
  }

  @Override
  public Future clearAll() {
    return makeAsyncCall(
        "FlushAll",
        MemcacheFlushRequest.getDefaultInstance(),
        createRpcResponseHandler(
            MemcacheFlushResponse.getDefaultInstance(),
            "Memcache clearAll: exception",
            new Transformer() {
              @Override
              public Void transform(MemcacheFlushResponse response) {
                return null;
              }
            }),
        DefaultValueProviders.nullValue());
  }

  @Override
  public Future getStatistics() {
    return makeAsyncCall(
        "Stats",
        MemcacheStatsRequest.getDefaultInstance(),
        createRpcResponseHandler(
            MemcacheStatsResponse.getDefaultInstance(),
            "Memcache getStatistics: exception",
            new Transformer() {
              @Override
              public Stats transform(MemcacheStatsResponse response) {
                return new StatsImpl(response.getStats());
              }
            }),
        DefaultValueProviders.emptyStats());
  }
}
    
    
    
    

    




    
    
    © 2015 - 2025 Weber Informatics LLC | Privacy Policy