Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*******************************************************************************
* HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*******************************************************************************/
package org.hpccsystems.dfs.client;
import org.hpccsystems.commons.ecl.FieldDef;
import org.hpccsystems.commons.ecl.RecordDefinitionTranslator;
import org.hpccsystems.dfs.client.RowServiceOutputStream;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.util.Iterator;
/**
* Remote file writer the writes data to the file part represented by @see org.hpccsystems.dfs.client.DataPartition
* and uses the provided @see org.hpccsystems.dfs.client.IRecordAccessor to access records.
*/
public class HPCCRemoteFileWriter
{
private static final Logger log = LogManager.getLogger(HPCCRemoteFileWriter.class);
private FieldDef recordDef = null;
private DataPartition dataPartition = null;
private RowServiceOutputStream outputStream = null;
private BinaryRecordWriter binaryRecordWriter = null;
private IRecordAccessor recordAccessor = null;
private long recordsWritten = 0;
private long openTimeMs = 0;
/**
* A remote file writer.
*
* @param dp
* the part of the file, name and location
* @param recordDef
* the record def
* @param recordAccessor
* the record accessor
* @param fileCompression
* the file compression
* @throws Exception
* the exception
*/
public HPCCRemoteFileWriter(DataPartition dp, FieldDef recordDef, IRecordAccessor recordAccessor, CompressionAlgorithm fileCompression)
throws Exception
{
this(dp,recordDef,recordAccessor,fileCompression,RowServiceOutputStream.DEFAULT_CONNECT_TIMEOUT_MILIS);
}
/**
* A remote file writer.
*
* @param dp
* the part of the file, name and location
* @param recordDef
* the record def
* @param recordAccessor
* the record accessor
* @param fileCompression
* the file compression
* @param connectTimeoutMs
* the socket timeout in ms (default is 5000)
* @throws Exception
* the exception
*/
public HPCCRemoteFileWriter(DataPartition dp, FieldDef recordDef, IRecordAccessor recordAccessor, CompressionAlgorithm fileCompression, int connectTimeoutMs)
throws Exception
{
this(dp,recordDef,recordAccessor,fileCompression,RowServiceOutputStream.DEFAULT_CONNECT_TIMEOUT_MILIS, RowServiceOutputStream.DEFAULT_SOCKET_OP_TIMEOUT_MS);
}
/**
* A remote file writer.
*
* @param dp
* the part of the file, name and location
* @param recordDef
* the record def
* @param recordAccessor
* the record accessor
* @param fileCompression
* the file compression
* @param connectTimeoutMs
* the socket timeout in ms (default is 5000)
* @param socketOpTimeoutMs
* the socket operation timeout in ms (default is 15000)
* @throws Exception
* the exception
*/
public HPCCRemoteFileWriter(DataPartition dp, FieldDef recordDef, IRecordAccessor recordAccessor, CompressionAlgorithm fileCompression, int connectTimeoutMs, int socketOpTimeoutMs)
throws Exception
{
this.recordDef = recordDef;
this.dataPartition = dp;
this.recordAccessor = recordAccessor;
this.outputStream = new RowServiceOutputStream(dataPartition.getCopyIP(0), dataPartition.getPort(), dataPartition.getUseSsl(),
dataPartition.getFileAccessBlob(), this.recordDef, this.dataPartition.getThisPart(), this.dataPartition.getCopyPath(0),
fileCompression, connectTimeoutMs, socketOpTimeoutMs);
this.binaryRecordWriter = new BinaryRecordWriter(this.outputStream);
this.binaryRecordWriter.initialize(this.recordAccessor);
log.info("HPCCRemoteFileWriter: Opening file part: " + dataPartition.getThisPart()
+ " compression: " + fileCompression.name());
log.trace("Record definition:\n"
+ RecordDefinitionTranslator.toJsonRecord(this.recordDef));
openTimeMs = System.currentTimeMillis();
}
/**
* Write record.
*
* @param record
* the record
* @throws Exception
* the exception
*/
public void writeRecord(T record) throws Exception
{
this.binaryRecordWriter.writeRecord(record);
this.recordsWritten++;
}
/**
* Write records.
*
* @param it
* the it
* @throws Exception
* the exception
*/
public void writeRecords(Iterator it) throws Exception
{
while (it.hasNext())
{
this.binaryRecordWriter.writeRecord(it.next());
this.recordsWritten++;
}
}
/**
* Close.
*
* @throws Exception
* the exception
*/
public void close() throws Exception
{
this.report();
this.binaryRecordWriter.finalize();
long closeTimeMs = System.currentTimeMillis();
double writeTimeS = (closeTimeMs - openTimeMs) / 1000.0;
log.info("HPCCRemoteFileWriter: Closing file part: " + dataPartition.getThisPart()
+ " write time: " + writeTimeS + "s "
+ " records written: " + recordsWritten);
}
/**
* Gets the bytes written.
*
* @return the bytes written
*/
public long getBytesWritten()
{
return this.binaryRecordWriter.getTotalBytesWritten();
}
/**
* Gets the records written.
*
* @return the records written
*/
public long getRecordsWritten()
{
return recordsWritten;
}
/**
* flush
* Flush buffered data from IRecordWriter. This is a blocking operation.
*
* @throws Exception
* the exception
*/
public void flush() throws Exception
{
this.binaryRecordWriter.flush();
}
/**
* getBufferCapacity
* Returns the IRecordWriter internal buffer capacity.
*
* @return the buffer capacity
*/
public int getBufferCapacity()
{
return this.binaryRecordWriter.getBufferCapacity();
}
/**
* getRemainingBufferCapacity
* Returns the IRecordWriter remaining buffer capacity.
*
* @return the remaining buffer capacity
*/
public int getRemainingBufferCapacity()
{
return this.binaryRecordWriter.getRemainingBufferCapacity();
}
public int getRemoteWriteMessageCount()
{
int count = 0;
if (this.binaryRecordWriter != null)
count = this.binaryRecordWriter.getStreamMessageCount();
return count;
}
public String getRemoteWriteMessages()
{
String report = "";
if (this.binaryRecordWriter != null)
report = this.binaryRecordWriter.getStreamMessages();
return report;
}
/**
* Reports summary of messages generated during write operation.
*/
public void report()
{
if (getRemoteWriteMessageCount() > 0)
{
log.warn(getRemoteWriteMessages());
}
}
}
