org.apache.sysml.runtime.util.LocalFileUtils Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of systemml Show documentation
Declarative Machine Learning
There is a newer version: 1.2.0
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.sysml.runtime.util;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashMap;

import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.MatrixValue;
import org.apache.sysml.runtime.matrix.data.Pair;

public class LocalFileUtils 
{
	public static final int BUFFER_SIZE = 8192;
	
	//unique IDs per JVM for tmp files
	private static IDSequence _seq = null;
	private static String _workingDir = null;
	
	//categories of temp files under process-specific working dir
	public static final String CATEGORY_CACHE        = "cache";
	public static final String CATEGORY_PARTITIONING = "partitioning";
	public static final String CATEGORY_RESULTMERGE  = "resultmerge";
	public static final String CATEGORY_WORK         = "work";
	
	static
	{
		_seq = new IDSequence();
	}
	
	/**
	 * 
	 * @param filePathAndName
	 * @return
	 * @throws IOException
	 */
	public static MatrixBlock readMatrixBlockFromLocal(String filePathAndName)
		throws IOException
	{
		return readMatrixBlockFromLocal(filePathAndName, new MatrixBlock());
	}
	
	/**
	 * 
	 * @param filePathAndName
	 * @return
	 * @throws IOException
	 */
	public static MatrixBlock readMatrixBlockFromLocal(String filePathAndName, MatrixBlock ret)
		throws IOException
	{
		FileInputStream fis = new FileInputStream( filePathAndName );
		//BufferedInputStream bis = new BufferedInputStream( fis, BUFFER_SIZE );
		//DataInputStream in = new DataInputStream( bis );
		FastBufferedDataInputStream in = new FastBufferedDataInputStream(fis, BUFFER_SIZE);
		
		try
		{
			ret.readFields(in);
		}
		finally
		{
			if( in != null )
				in.close();
		}
			
		return ret;
	}
	
	/**
	 * 
	 * @param filePathAndName
	 * @param mb
	 * @throws IOException
	 */
	public static void writeMatrixBlockToLocal (String filePathAndName, MatrixBlock mb)
		throws IOException
	{	
		FileOutputStream fos = new FileOutputStream( filePathAndName );
		//BufferedOutputStream bos = new BufferedOutputStream( fos, BUFFER_SIZE );
		//DataOutputStream out = new DataOutputStream( bos );
		FastBufferedDataOutputStream out = new FastBufferedDataOutputStream(fos, BUFFER_SIZE);
		
		try 
		{
			mb.write(out);
		}
		finally
		{
			if( out != null )
				out.close();	
		}	
	}
	
	
	/**
	 * 
	 * @param filePathAndName
	 * @param data
	 * @throws IOException
	 */
	public static void writeByteArrayToLocal( String filePathAndName, byte[] data )
		throws IOException
	{		
		FileOutputStream fos = new FileOutputStream( filePathAndName );
		
		try 
		{
			fos.write( data );
		}
		finally
		{
			if( fos != null )
				fos.close ();	
		}	
	}
	
	/**
	 * 
	 * @param filePathAndName
	 * @param data
	 * @throws IOException
	 */
	public static void writeByteArrayToLocal( String filePathAndName, byte[][] data )
		throws IOException
	{		
		FileOutputStream fos = new FileOutputStream( filePathAndName );
		
		try 
		{
			for( int i=0; i[] outValues, HashMap outMap) 
		throws IOException
	{
		FileInputStream fis = new FileInputStream( filePathAndName );
		FastBufferedDataInputStream in = new FastBufferedDataInputStream( fis, BUFFER_SIZE );
		int bufferSize = 0;
		
		try
		{
			int len = in.readInt();
			for( int i=0; i[] inValues, int len ) 
		throws IOException
	{
		if( len > inValues.length )
			throw new IOException("Invalid length of block sequence: len="+len+" vs data="+inValues.length);
		
		FileOutputStream fos = new FileOutputStream( filePathAndName );
		FastBufferedDataOutputStream out = new FastBufferedDataOutputStream(fos, BUFFER_SIZE);
		
		try 
		{
			out.writeInt(len);
			for( int i=0; i