org.apache.hadoop.hive.ql.exec.FooterBuffer Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.exec;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.util.ReflectionUtils;
public class FooterBuffer {
private List> buffer;
private int cur;
public FooterBuffer() {
}
public void setCursor(int cur) {
this.cur = cur;
}
/**
* Initialize footer buffer in order to keep footer records at the end of file.
*
* @param job
* Current job configuration.
*
* @param recordreader
* Record reader.
*
* @param footerCount
* Footer line number of the table files.
*
* @param key
* Key of current reading record.
*
* @param value
* Value of current reading record.
*
* @return Return true if there are 0 or more records left in the file
* after initializing the footer buffer, otherwise return false.
*/
public boolean initializeBuffer(JobConf job, RecordReader recordreader,
int footerCount, WritableComparable key, Writable value) throws IOException {
// Fill the buffer with key value pairs.
this.buffer = new ArrayList<>();
while (buffer.size() < footerCount) {
boolean notEOF = recordreader.next(key, value);
if (!notEOF) {
return false;
}
WritableComparable left = ReflectionUtils.copy(job, key, null);
Writable right = ReflectionUtils.copy(job, value, null);
Pair tem = Pair.of(left, right);
buffer.add(tem);
}
this.cur = 0;
return true;
}
/**
* Enqueue most recent record read, and dequeue earliest result in the queue.
*
* @param job
* Current job configuration.
*
* @param recordreader
* Record reader.
*
* @param key
* Key of current reading record.
*
* @param value
* Value of current reading record.
*
* @return Return false if reaches the end of file, otherwise return true.
*/
public boolean updateBuffer(JobConf job, RecordReader recordreader,
WritableComparable key, Writable value) throws IOException {
key = ReflectionUtils.copy(job, buffer.get(cur).getKey(), key);
value = ReflectionUtils.copy(job, buffer.get(cur).getValue(), value);
boolean notEOF = recordreader.next(buffer.get(cur).getKey(), buffer.get(cur).getValue());
if (notEOF) {
cur = (++cur) % buffer.size();
}
return notEOF;
}
}