cz.seznam.euphoria.spark.SparkStorageProvider Maven / Gradle / Ivy

Go to download
/**
 * Copyright 2016 Seznam.cz, a.s.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package cz.seznam.euphoria.spark;

import cz.seznam.euphoria.core.client.operator.state.ListStorage;
import cz.seznam.euphoria.core.client.operator.state.ListStorageDescriptor;
import cz.seznam.euphoria.core.client.operator.state.StorageProvider;
import cz.seznam.euphoria.core.client.operator.state.ValueStorage;
import cz.seznam.euphoria.core.client.operator.state.ValueStorageDescriptor;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;

/**
 * Storage provider for batch processing. Date are stored in memory.
 * We don't need to worry about checkpointing here, because the
 * storage is used in batch only and can therefore be reconstructed by
 * recalculation of the data.
 */
class SparkStorageProvider implements StorageProvider, Serializable {

  static class MemValueStorage implements ValueStorage {

    private T value;

    MemValueStorage(T defVal) {
      this.value = defVal;
    }

    @Override
    public void set(T value) {
      this.value = value;
    }

    @Override
    public T get() {
      return value;
    }

    @Override
    public void clear() {
      value = null;
    }
  }

  static class MemListStorage implements ListStorage {

    private final List data = new ArrayList<>();

    @Override
    public void add(T element) {
      data.add(element);
    }

    @Override
    public Iterable get() {
      return data;
    }

    @Override
    public final void clear() {
      data.clear();
    }
  }

  @Override
  public  ValueStorage getValueStorage(ValueStorageDescriptor descriptor) {
    return new MemValueStorage<>(descriptor.getDefaultValue());
  }

  @Override
  public  ListStorage getListStorage(ListStorageDescriptor descriptor) {
      return new MemListStorage<>();
  }
}