All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.tencent.angel.model.io.SnapshotLoaderSaver Maven / Gradle / Ivy

There is a newer version: 3.2.0
Show newest version
/*
 * Tencent is pleased to support the open source community by making Angel available.
 *
 * Copyright (C) 2017-2018 THL A29 Limited, a Tencent company. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
 * compliance with the License. You may obtain a copy of the License at
 *
 * https://opensource.org/licenses/Apache-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 *
 */
package com.tencent.angel.model.io;

import com.tencent.angel.ml.math2.vector.IntDoubleVector;
import com.tencent.angel.ml.math2.vector.IntFloatVector;
import com.tencent.angel.ml.math2.vector.IntIntVector;
import com.tencent.angel.ml.math2.vector.IntLongVector;
import com.tencent.angel.ml.math2.vector.LongDoubleVector;
import com.tencent.angel.ml.math2.vector.LongFloatVector;
import com.tencent.angel.ml.math2.vector.LongIntVector;
import com.tencent.angel.ml.math2.vector.LongLongVector;
import com.tencent.angel.model.ModelIOUtils;
import com.tencent.angel.model.PSMatrixLoadContext;
import com.tencent.angel.model.PSMatrixSaveContext;
import com.tencent.angel.model.output.format.MatrixPartitionMeta;
import com.tencent.angel.model.output.format.RowPartitionMeta;
import com.tencent.angel.model.output.format.SnapshotFormat;
import com.tencent.angel.ps.storage.matrix.PartitionState;
import com.tencent.angel.ps.storage.partition.RowBasedPartition;
import com.tencent.angel.ps.storage.partition.ServerPartition;
import com.tencent.angel.ps.storage.partition.storage.ServerRowsStorage;
import com.tencent.angel.ps.storage.vector.ServerIntAnyRow;
import com.tencent.angel.ps.storage.vector.ServerIntDoubleRow;
import com.tencent.angel.ps.storage.vector.ServerIntFloatRow;
import com.tencent.angel.ps.storage.vector.ServerIntIntRow;
import com.tencent.angel.ps.storage.vector.ServerIntLongRow;
import com.tencent.angel.ps.storage.vector.ServerLongAnyRow;
import com.tencent.angel.ps.storage.vector.ServerLongDoubleRow;
import com.tencent.angel.ps.storage.vector.ServerLongFloatRow;
import com.tencent.angel.ps.storage.vector.ServerLongIntRow;
import com.tencent.angel.ps.storage.vector.ServerLongLongRow;
import com.tencent.angel.ps.storage.vector.ServerRow;
import com.tencent.angel.ps.storage.vector.ServerRowUtils;
import it.unimi.dsi.fastutil.ints.Int2DoubleMap;
import it.unimi.dsi.fastutil.ints.Int2FloatMap;
import it.unimi.dsi.fastutil.ints.Int2FloatMap.Entry;
import it.unimi.dsi.fastutil.ints.Int2IntMap;
import it.unimi.dsi.fastutil.ints.Int2LongMap;
import it.unimi.dsi.fastutil.longs.Long2DoubleMap;
import it.unimi.dsi.fastutil.longs.Long2FloatMap;
import it.unimi.dsi.fastutil.longs.Long2IntMap;
import it.unimi.dsi.fastutil.longs.Long2LongMap;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;

public class SnapshotLoaderSaver extends PSMatrixLoaderSaverImpl {
  private final SnapshotFormat format;
  public SnapshotLoaderSaver(SnapshotFormat format, Configuration conf) {
    super(conf);
    this.format = format;
  }

  @Override
  public void save(ServerPartition part, MatrixPartitionMeta partMeta,
      PSMatrixSaveContext saveContext, DataOutputStream output) throws IOException {
    if(part instanceof RowBasedPartition) {
      save((RowBasedPartition) part, partMeta, saveContext, output);
    }
  }

  @Override
  public void load(ServerPartition part, MatrixPartitionMeta partMeta,
      PSMatrixLoadContext loadContext, DataInputStream input) throws IOException {
    if(part instanceof RowBasedPartition) {
      load((RowBasedPartition) part, partMeta, loadContext, input);
    }
  }


  static enum SaveType {
    DENSE(0), SPARSE(1);
    public static Map typeIdToTypeMap;

    static {
      typeIdToTypeMap = new HashMap();
      typeIdToTypeMap.put(DENSE.typeId, DENSE);
      typeIdToTypeMap.put(SPARSE.typeId, SPARSE);
    }

    public static SaveType valueOf(int id) {
      return typeIdToTypeMap.get(id);
    }

    private final int typeId;

    SaveType(int typeId) {
      this.typeId = typeId;
    }

    public int getTypeId() {
      return typeId;
    }
  }

  /**
   * Matrix partition data
   *
   * @param part matrix partition
   * @param partMeta matrix partition data meta
   * @param saveContext save context
   * @param output output stream
   */
  public void save(RowBasedPartition part, MatrixPartitionMeta partMeta,
      PSMatrixSaveContext saveContext, DataOutputStream output) throws IOException {
    List rowIds = saveContext.getRowIndexes();
    ServerRowsStorage rows = part.getRowsStorage();
    if (rowIds == null || rowIds.isEmpty()) {
      int rowStart = part.getPartitionKey().getStartRow();
      int rowEnd = part.getPartitionKey().getEndRow();

      rowIds = new ArrayList<>(rowEnd - rowStart);
      for (int i = rowStart; i < rowEnd; i++) {
        rowIds.add(i);
      }
    } else {
      rowIds = ModelIOUtils.filter(part, rowIds);
    }

    FSDataOutputStream dataOutputStream =
        new FSDataOutputStream(output, null, partMeta != null ? partMeta.getOffset() : 0);

    partMeta.setSaveRowNum(rowIds.size());
    for (int rowId : rowIds) {
      ServerRow row = rows.getRow(rowId);
      RowPartitionMeta rowMeta = new RowPartitionMeta(rowId, 0, 0);
      if (row != null) {
        rowMeta.setElementNum(row.size());
        rowMeta.setOffset(dataOutputStream.getPos());
        if (row.isDense()) {
          rowMeta.setSaveType(SaveType.DENSE.getTypeId());
        } else {
          rowMeta.setSaveType(SaveType.SPARSE.getTypeId());
        }
        //row.getStorage().serialize(dataOutputStream);
        save(rows.getRow(rowId), saveContext, partMeta, output);
      } else {
        rowMeta.setElementNum(0);
        rowMeta.setOffset(dataOutputStream.getPos());
      }
      partMeta.setRowMeta(rowMeta);
    }
  }

  /**
   * Save a row
   *
   * @param row row data
   * @param saveContext save context
   * @param meta partition data meta
   * @param out output stream
   */
  public void save(ServerRow row, PSMatrixSaveContext saveContext, MatrixPartitionMeta meta,
      DataOutputStream out) throws IOException {
    if (saveContext.cloneFirst()) {
      row = (ServerRow) row.adaptiveClone();
    }
    try {
      row.startWrite();
      if (row instanceof ServerIntFloatRow) {
        save((ServerIntFloatRow) row, saveContext, meta, out);
      } else if (row instanceof ServerIntDoubleRow) {
        save((ServerIntDoubleRow) row, saveContext, meta, out);
      } else if (row instanceof ServerIntIntRow) {
        save((ServerIntIntRow) row, saveContext, meta, out);
      } else if (row instanceof ServerIntLongRow) {
        save((ServerIntLongRow) row, saveContext, meta, out);
      } else if (row instanceof ServerLongFloatRow) {
        save((ServerLongFloatRow) row, saveContext, meta, out);
      } else if (row instanceof ServerLongDoubleRow) {
        save((ServerLongDoubleRow) row, saveContext, meta, out);
      } else if (row instanceof ServerLongIntRow) {
        save((ServerLongIntRow) row, saveContext, meta, out);
      } else if (row instanceof ServerLongLongRow) {
        save((ServerLongLongRow) row, saveContext, meta, out);
      } else if (row instanceof ServerIntAnyRow) {
        save((ServerIntAnyRow) row, saveContext, meta, out);
      } else if (row instanceof ServerLongAnyRow) {
        save((ServerLongAnyRow) row, saveContext, meta, out);
      } else {
        throw new IOException("Unknown vector type " + row.getRowType());
      }
    } finally {
      row.endWrite();
    }
  }

  /**
   * Load a matrix partition
   *
   * @param part matrix partition
   * @param partMeta matrix partition data meta
   * @param loadContext load context
   * @param input input stream
   */
  public void load(RowBasedPartition part, MatrixPartitionMeta partMeta,
      PSMatrixLoadContext loadContext, DataInputStream input) throws IOException {
    ServerRowsStorage rows = part.getRowsStorage();
    try {
      Map rowMetas = partMeta.getRowMetas();
      for (RowPartitionMeta rowMeta : rowMetas.values()) {
        ServerRow row = rows.getRow(rowMeta.getRowId());
        load(row, loadContext, partMeta, input);
        //row.getStorage().deserialize(input);
      }
    } finally {
      part.setState(PartitionState.READ_AND_WRITE);
    }
  }

  /**
   * Load a row data
   *
   * @param row row partition
   * @param meta partition meta
   * @param loadContext load context
   * @param in input stream
   */
  public void load(ServerRow row, PSMatrixLoadContext loadContext, MatrixPartitionMeta meta,
      DataInputStream in) throws IOException {
    try {
      row.startWrite();
      if (row instanceof ServerIntFloatRow) {
        load((ServerIntFloatRow) row, loadContext, meta, in);
      } else if (row instanceof ServerIntDoubleRow) {
        load((ServerIntDoubleRow) row, loadContext, meta, in);
      } else if (row instanceof ServerIntIntRow) {
        load((ServerIntIntRow) row, loadContext, meta, in);
      } else if (row instanceof ServerIntLongRow) {
        load((ServerIntLongRow) row, loadContext, meta, in);
      } else if (row instanceof ServerLongFloatRow) {
        load((ServerLongFloatRow) row, loadContext, meta, in);
      } else if (row instanceof ServerLongDoubleRow) {
        load((ServerLongDoubleRow) row, loadContext, meta, in);
      } else if (row instanceof ServerLongIntRow) {
        load((ServerLongIntRow) row, loadContext, meta, in);
      } else if (row instanceof ServerLongLongRow) {
        load((ServerLongLongRow) row, loadContext, meta, in);
      } else if (row instanceof ServerIntAnyRow) {
        load((ServerIntAnyRow) row, loadContext, meta, in);
      } else if (row instanceof ServerLongAnyRow) {
        load((ServerLongAnyRow) row, loadContext, meta, in);
      } else {
        throw new IOException("Unknown vector type " + row.getRowType());
      }
    } finally {
      row.endWrite();
    }
  }

  private void save(ServerIntFloatRow row, PSMatrixSaveContext saveContext,
      MatrixPartitionMeta meta, DataOutputStream out) throws IOException {
    int startCol = (int) meta.getStartCol();
    IntFloatVector vector = ServerRowUtils.getVector(row);
    if (vector.isDense()) {
      float[] data = vector.getStorage().getValues();
      for (int i = 0; i < data.length; i++) {
        out.writeFloat(data[i]);
      }
    } else if (vector.isSorted()) {
      int[] indices = vector.getStorage().getIndices();
      float[] values = vector.getStorage().getValues();
      for (int i = 0; i < indices.length; i++) {
        out.writeInt(indices[i] + startCol);
        out.writeFloat(values[i]);
      }
    } else {
      ObjectIterator iter = vector.getStorage().entryIterator();
      Int2FloatMap.Entry entry;
      while (iter.hasNext()) {
        entry = iter.next();
        out.writeInt(entry.getIntKey() + startCol);
        out.writeFloat(entry.getFloatValue());
      }
    }
  }

  private void save(ServerIntDoubleRow row, PSMatrixSaveContext saveContext,
      MatrixPartitionMeta meta, DataOutputStream out) throws IOException {
    int startCol = (int) meta.getStartCol();
    IntDoubleVector vector = ServerRowUtils.getVector(row);
    if (vector.isDense()) {
      double[] data = vector.getStorage().getValues();
      for (int i = 0; i < data.length; i++) {
        out.writeDouble(data[i]);
      }
    } else if (vector.isSorted()) {
      int[] indices = vector.getStorage().getIndices();
      double[] values = vector.getStorage().getValues();
      for (int i = 0; i < indices.length; i++) {
        out.writeInt(indices[i] + startCol);
        out.writeDouble(values[i]);
      }
    } else {
      ObjectIterator iter = vector.getStorage().entryIterator();
      Int2DoubleMap.Entry entry;
      while (iter.hasNext()) {
        entry = iter.next();
        out.writeInt(entry.getIntKey() + startCol);
        out.writeDouble(entry.getDoubleValue());
      }
    }
  }

  private void save(ServerIntIntRow row, PSMatrixSaveContext saveContext, MatrixPartitionMeta meta,
      DataOutputStream out) throws IOException {
    int startCol = (int) meta.getStartCol();
    IntIntVector vector = ServerRowUtils.getVector(row);
    if (vector.isDense()) {
      int[] data = vector.getStorage().getValues();
      for (int i = 0; i < data.length; i++) {
        out.writeInt(data[i]);
      }
    } else if (vector.isSorted()) {
      int[] indices = vector.getStorage().getIndices();
      int[] values = vector.getStorage().getValues();
      for (int i = 0; i < indices.length; i++) {
        out.writeInt(indices[i] + startCol);
        out.writeInt(values[i]);
      }
    } else {
      ObjectIterator iter = vector.getStorage().entryIterator();
      Int2IntMap.Entry entry;
      while (iter.hasNext()) {
        entry = iter.next();
        out.writeInt(entry.getIntKey() + startCol);
        out.writeInt(entry.getIntValue());
      }
    }
  }

  private void save(ServerIntLongRow row, PSMatrixSaveContext saveContext, MatrixPartitionMeta meta,
      DataOutputStream out) throws IOException {
    int startCol = (int) meta.getStartCol();
    IntLongVector vector = ServerRowUtils.getVector(row);
    if (vector.isDense()) {
      long[] data = vector.getStorage().getValues();
      for (int i = 0; i < data.length; i++) {
        out.writeLong(data[i]);
      }
    } else if (vector.isSorted()) {
      int[] indices = vector.getStorage().getIndices();
      long[] values = vector.getStorage().getValues();
      for (int i = 0; i < indices.length; i++) {
        out.writeInt(indices[i] + startCol);
        out.writeLong(values[i]);
      }
    } else {
      ObjectIterator iter = vector.getStorage().entryIterator();
      Int2LongMap.Entry entry;
      while (iter.hasNext()) {
        entry = iter.next();
        out.writeInt(entry.getIntKey() + startCol);
        out.writeLong(entry.getLongValue());
      }
    }
  }

  private void save(ServerLongDoubleRow row, PSMatrixSaveContext saveContext,
      MatrixPartitionMeta meta, DataOutputStream out) throws IOException {
    long startCol = meta.getStartCol();
    if (ServerRowUtils.getVector(row) instanceof IntDoubleVector) {
      IntDoubleVector vector = (IntDoubleVector) ServerRowUtils.getVector(row);
      if (vector.isDense()) {
        double[] data = vector.getStorage().getValues();
        for (int i = 0; i < data.length; i++) {
          out.writeDouble(data[i]);
        }
      } else if (vector.isSorted()) {
        int[] indices = vector.getStorage().getIndices();
        double[] values = vector.getStorage().getValues();
        for (int i = 0; i < indices.length; i++) {
          out.writeLong(indices[i] + startCol);
          out.writeDouble(values[i]);
        }
      } else {
        ObjectIterator iter = vector.getStorage().entryIterator();
        Int2DoubleMap.Entry entry;
        while (iter.hasNext()) {
          entry = iter.next();
          out.writeLong(entry.getIntKey() + startCol);
          out.writeDouble(entry.getDoubleValue());
        }
      }
    } else {
      LongDoubleVector vector = (LongDoubleVector) ServerRowUtils.getVector(row);
      if (vector.isSorted()) {
        long[] indices = vector.getStorage().getIndices();
        double[] values = vector.getStorage().getValues();
        for (int i = 0; i < indices.length; i++) {
          out.writeLong(indices[i] + startCol);
          out.writeDouble(values[i]);
        }
      } else {
        ObjectIterator iter = vector.getStorage().entryIterator();
        Long2DoubleMap.Entry entry;
        while (iter.hasNext()) {
          entry = iter.next();
          out.writeLong(entry.getLongKey() + startCol);
          out.writeDouble(entry.getDoubleValue());
        }
      }
    }
  }

  private void save(ServerLongFloatRow row, PSMatrixSaveContext saveContext,
      MatrixPartitionMeta meta, DataOutputStream out) throws IOException {
    long startCol = meta.getStartCol();
    if (ServerRowUtils.getVector(row) instanceof IntFloatVector) {
      IntFloatVector vector = (IntFloatVector) ServerRowUtils.getVector(row);
      if (vector.isDense()) {
        float[] data = vector.getStorage().getValues();
        for (int i = 0; i < data.length; i++) {
          out.writeFloat(data[i]);
        }
      } else if (vector.isSorted()) {
        int[] indices = vector.getStorage().getIndices();
        float[] values = vector.getStorage().getValues();
        for (int i = 0; i < indices.length; i++) {
          out.writeLong(indices[i] + startCol);
          out.writeFloat(values[i]);
        }
      } else {
        ObjectIterator iter = vector.getStorage().entryIterator();
        Int2FloatMap.Entry entry;
        while (iter.hasNext()) {
          entry = iter.next();
          out.writeLong(entry.getIntKey() + startCol);
          out.writeFloat(entry.getFloatValue());
        }
      }
    } else {
      LongFloatVector vector = (LongFloatVector) ServerRowUtils.getVector(row);
      if (vector.isSorted()) {
        long[] indices = vector.getStorage().getIndices();
        float[] values = vector.getStorage().getValues();
        for (int i = 0; i < indices.length; i++) {
          out.writeLong(indices[i] + startCol);
          out.writeFloat(values[i]);
        }
      } else {
        ObjectIterator iter = vector.getStorage().entryIterator();
        Long2FloatMap.Entry entry;
        while (iter.hasNext()) {
          entry = iter.next();
          out.writeLong(entry.getLongKey() + startCol);
          out.writeFloat(entry.getFloatValue());
        }
      }
    }
  }


  private void save(ServerLongIntRow row, PSMatrixSaveContext saveContext, MatrixPartitionMeta meta,
      DataOutputStream out) throws IOException {
    long startCol = meta.getStartCol();
    if (ServerRowUtils.getVector(row) instanceof IntIntVector) {
      IntIntVector vector = (IntIntVector) ServerRowUtils.getVector(row);
      if (vector.isDense()) {
        int[] data = vector.getStorage().getValues();
        for (int i = 0; i < data.length; i++) {
          out.writeInt(data[i]);
        }
      } else if (vector.isSorted()) {
        int[] indices = vector.getStorage().getIndices();
        int[] values = vector.getStorage().getValues();
        for (int i = 0; i < indices.length; i++) {
          out.writeLong(indices[i] + startCol);
          out.writeInt(values[i]);
        }
      } else {
        ObjectIterator iter = vector.getStorage().entryIterator();
        Int2IntMap.Entry entry;
        while (iter.hasNext()) {
          entry = iter.next();
          out.writeLong(entry.getIntKey() + startCol);
          out.writeInt(entry.getIntValue());
        }
      }
    } else {
      LongIntVector vector = (LongIntVector) ServerRowUtils.getVector(row);
      if (vector.isSorted()) {
        long[] indices = vector.getStorage().getIndices();
        int[] values = vector.getStorage().getValues();
        for (int i = 0; i < indices.length; i++) {
          out.writeLong(indices[i] + startCol);
          out.writeInt(values[i]);
        }
      } else {
        ObjectIterator iter = vector.getStorage().entryIterator();
        Long2IntMap.Entry entry;
        while (iter.hasNext()) {
          entry = iter.next();
          out.writeLong(entry.getLongKey() + startCol);
          out.writeInt(entry.getIntValue());
        }
      }
    }
  }

  private void save(ServerLongLongRow row, PSMatrixSaveContext saveContext,
      MatrixPartitionMeta meta, DataOutputStream out) throws IOException {
    long startCol = meta.getStartCol();
    if (ServerRowUtils.getVector(row) instanceof IntLongVector) {
      IntLongVector vector = (IntLongVector) ServerRowUtils.getVector(row);
      if (vector.isDense()) {
        long[] data = vector.getStorage().getValues();
        for (int i = 0; i < data.length; i++) {
          out.writeLong(data[i]);
        }
      } else if (vector.isSorted()) {
        int[] indices = vector.getStorage().getIndices();
        long[] values = vector.getStorage().getValues();
        for (int i = 0; i < indices.length; i++) {
          out.writeLong(indices[i] + startCol);
          out.writeLong(values[i]);
        }
      } else {
        ObjectIterator iter = vector.getStorage().entryIterator();
        Int2LongMap.Entry entry;
        while (iter.hasNext()) {
          entry = iter.next();
          out.writeLong(entry.getIntKey() + startCol);
          out.writeLong(entry.getLongValue());
        }
      }
    } else {
      LongLongVector vector = (LongLongVector) ServerRowUtils.getVector(row);
      if (vector.isSorted()) {
        long[] indices = vector.getStorage().getIndices();
        long[] values = vector.getStorage().getValues();
        for (int i = 0; i < indices.length; i++) {
          out.writeLong(indices[i] + startCol);
          out.writeLong(values[i]);
        }
      } else {
        ObjectIterator iter = vector.getStorage().entryIterator();
        Long2LongMap.Entry entry;
        while (iter.hasNext()) {
          entry = iter.next();
          out.writeLong(entry.getLongKey() + startCol);
          out.writeLong(entry.getLongValue());
        }
      }
    }
  }

  private void save(ServerIntAnyRow row, PSMatrixSaveContext saveContext,
      MatrixPartitionMeta meta, DataOutputStream output) throws IOException {
    row.getStorage().serialize(output);
  }

  private void save(ServerLongAnyRow row, PSMatrixSaveContext saveContext,
      MatrixPartitionMeta meta, DataOutputStream output) throws IOException {
    row.getStorage().serialize(output);
  }

  private void load(ServerIntAnyRow row, PSMatrixLoadContext loadContext,
      MatrixPartitionMeta meta, DataInputStream in) throws IOException {
    row.getStorage().deserialize(in);
  }

  private void load(ServerLongAnyRow row, PSMatrixLoadContext loadContext,
      MatrixPartitionMeta meta, DataInputStream in) throws IOException {
    row.getStorage().deserialize(in);
  }

  private void load(ServerIntFloatRow row, PSMatrixLoadContext loadContext,
      MatrixPartitionMeta meta, DataInputStream in) throws IOException {
    RowPartitionMeta rowMeta = meta.getRowMeta(row.getRowId());
    int elemNum = rowMeta.getElementNum();
    SaveType saveType = SaveType.valueOf(rowMeta.getSaveType());
    int startCol = (int) meta.getStartCol();
    if (saveType == SaveType.DENSE) {
      for (int i = 0; i < elemNum; i++) {
        row.set(i + startCol, in.readFloat());
      }
    } else {
      for (int i = 0; i < elemNum; i++) {
        row.set(in.readInt(), in.readFloat());
      }
    }
  }

  private void load(ServerIntDoubleRow row, PSMatrixLoadContext loadContext,
      MatrixPartitionMeta meta, DataInputStream in) throws IOException {
    RowPartitionMeta rowMeta = meta.getRowMeta(row.getRowId());
    int elemNum = rowMeta.getElementNum();
    SaveType saveType = SaveType.valueOf(rowMeta.getSaveType());
    int startCol = (int) meta.getStartCol();
    if (saveType == SaveType.DENSE) {
      for (int i = 0; i < elemNum; i++) {
        row.set(i + startCol, in.readDouble());
      }
    } else {
      for (int i = 0; i < elemNum; i++) {
        row.set(in.readInt(), in.readDouble());
      }
    }
  }

  private void load(ServerIntIntRow row, PSMatrixLoadContext loadContext, MatrixPartitionMeta meta,
      DataInputStream in) throws IOException {
    RowPartitionMeta rowMeta = meta.getRowMeta(row.getRowId());
    int elemNum = rowMeta.getElementNum();
    SaveType saveType = SaveType.valueOf(rowMeta.getSaveType());
    int startCol = (int) meta.getStartCol();
    if (saveType == SaveType.DENSE) {
      for (int i = 0; i < elemNum; i++) {
        row.set(i + startCol, in.readInt());
      }
    } else {
      for (int i = 0; i < elemNum; i++) {
        row.set(in.readInt(), in.readInt());
      }
    }
  }

  private void load(ServerIntLongRow row, PSMatrixLoadContext loadContext, MatrixPartitionMeta meta,
      DataInputStream in) throws IOException {
    RowPartitionMeta rowMeta = meta.getRowMeta(row.getRowId());
    int elemNum = rowMeta.getElementNum();
    SaveType saveType = SaveType.valueOf(rowMeta.getSaveType());
    int startCol = (int) meta.getStartCol();
    if (saveType == SaveType.DENSE) {
      for (int i = 0; i < elemNum; i++) {
        row.set(i + startCol, in.readLong());
      }
    } else {
      for (int i = 0; i < elemNum; i++) {
        row.set(in.readInt(), in.readLong());
      }
    }
  }

  private void load(ServerLongFloatRow row, PSMatrixLoadContext loadContext,
      MatrixPartitionMeta meta, DataInputStream in) throws IOException {
    RowPartitionMeta rowMeta = meta.getRowMeta(row.getRowId());
    int elemNum = rowMeta.getElementNum();
    SaveType saveType = SaveType.valueOf(rowMeta.getSaveType());
    long startCol = meta.getStartCol();
    if (saveType == SaveType.DENSE) {
      for (int i = 0; i < elemNum; i++) {
        row.set(i + startCol, in.readFloat());
      }
    } else {
      for (int i = 0; i < elemNum; i++) {
        row.set(in.readLong(), in.readFloat());
      }
    }
  }

  private void load(ServerLongDoubleRow row, PSMatrixLoadContext loadContext,
      MatrixPartitionMeta meta, DataInputStream in) throws IOException {
    RowPartitionMeta rowMeta = meta.getRowMeta(row.getRowId());
    int elemNum = rowMeta.getElementNum();
    SaveType saveType = SaveType.valueOf(rowMeta.getSaveType());
    long startCol = meta.getStartCol();
    if (saveType == SaveType.DENSE) {
      for (int i = 0; i < elemNum; i++) {
        row.set(i + startCol, in.readDouble());
      }
    } else {
      for (int i = 0; i < elemNum; i++) {
        row.set(in.readLong(), in.readDouble());
      }
    }
  }

  private void load(ServerLongIntRow row, PSMatrixLoadContext loadContext, MatrixPartitionMeta meta,
      DataInputStream in) throws IOException {
    RowPartitionMeta rowMeta = meta.getRowMeta(row.getRowId());
    int elemNum = rowMeta.getElementNum();
    SaveType saveType = SaveType.valueOf(rowMeta.getSaveType());
    long startCol = meta.getStartCol();
    if (saveType == SaveType.DENSE) {
      for (int i = 0; i < elemNum; i++) {
        row.set(i + startCol, in.readInt());
      }
    } else {
      for (int i = 0; i < elemNum; i++) {
        row.set(in.readLong(), in.readInt());
      }
    }
  }

  private void load(ServerLongLongRow row, PSMatrixLoadContext loadContext,
      MatrixPartitionMeta meta, DataInputStream in) throws IOException {
    RowPartitionMeta rowMeta = meta.getRowMeta(row.getRowId());
    int elemNum = rowMeta.getElementNum();
    SaveType saveType = SaveType.valueOf(rowMeta.getSaveType());
    long startCol = meta.getStartCol();
    if (saveType == SaveType.DENSE) {
      for (int i = 0; i < elemNum; i++) {
        row.set(i + startCol, in.readLong());
      }
    } else {
      for (int i = 0; i < elemNum; i++) {
        row.set(in.readLong(), in.readLong());
      }
    }
  }
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy