org.apache.hadoop.hdfs.server.namenode.FSEditLogOp Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.util.zip.CheckedInputStream;
import java.util.zip.Checksum;
import java.util.EnumMap;
import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.util.PureJavaCrc32;
import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.*;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
import org.apache.hadoop.hdfs.DeprecatedUTF8;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.EOFException;
/**
* Helper classes for reading the ops from an InputStream.
* All ops derive from FSEditLogOp and are only
* instantiated from Reader#readOp()
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public abstract class FSEditLogOp {
final FSEditLogOpCodes opCode;
long txid;
@SuppressWarnings("deprecation")
private static ThreadLocal> opInstances =
new ThreadLocal>() {
@Override
protected EnumMap initialValue() {
EnumMap instances
= new EnumMap(FSEditLogOpCodes.class);
instances.put(OP_ADD, new AddOp());
instances.put(OP_CLOSE, new CloseOp());
instances.put(OP_SET_REPLICATION, new SetReplicationOp());
instances.put(OP_CONCAT_DELETE, new ConcatDeleteOp());
instances.put(OP_RENAME_OLD, new RenameOldOp());
instances.put(OP_DELETE, new DeleteOp());
instances.put(OP_MKDIR, new MkdirOp());
instances.put(OP_SET_GENSTAMP, new SetGenstampOp());
instances.put(OP_DATANODE_ADD, new DatanodeAddOp());
instances.put(OP_DATANODE_REMOVE, new DatanodeRemoveOp());
instances.put(OP_SET_PERMISSIONS, new SetPermissionsOp());
instances.put(OP_SET_OWNER, new SetOwnerOp());
instances.put(OP_SET_NS_QUOTA, new SetNSQuotaOp());
instances.put(OP_CLEAR_NS_QUOTA, new ClearNSQuotaOp());
instances.put(OP_SET_QUOTA, new SetQuotaOp());
instances.put(OP_TIMES, new TimesOp());
instances.put(OP_SYMLINK, new SymlinkOp());
instances.put(OP_RENAME, new RenameOp());
instances.put(OP_REASSIGN_LEASE, new ReassignLeaseOp());
instances.put(OP_GET_DELEGATION_TOKEN, new GetDelegationTokenOp());
instances.put(OP_RENEW_DELEGATION_TOKEN, new RenewDelegationTokenOp());
instances.put(OP_CANCEL_DELEGATION_TOKEN,
new CancelDelegationTokenOp());
instances.put(OP_UPDATE_MASTER_KEY, new UpdateMasterKeyOp());
instances.put(OP_START_LOG_SEGMENT,
new LogSegmentOp(OP_START_LOG_SEGMENT));
instances.put(OP_END_LOG_SEGMENT,
new LogSegmentOp(OP_END_LOG_SEGMENT));
return instances;
}
};
/**
* Constructor for an EditLog Op. EditLog ops cannot be constructed
* directly, but only through Reader#readOp.
*/
private FSEditLogOp(FSEditLogOpCodes opCode) {
this.opCode = opCode;
this.txid = 0;
}
public void setTransactionId(long txid) {
this.txid = txid;
}
abstract void readFields(DataInputStream in, int logVersion)
throws IOException;
abstract void writeFields(DataOutputStream out)
throws IOException;
@SuppressWarnings("unchecked")
static abstract class AddCloseOp extends FSEditLogOp {
int length;
String path;
short replication;
long mtime;
long atime;
long blockSize;
Block[] blocks;
PermissionStatus permissions;
String clientName;
String clientMachine;
//final DatanodeDescriptor[] dataNodeDescriptors; UNUSED
private AddCloseOp(FSEditLogOpCodes opCode) {
super(opCode);
assert(opCode == OP_ADD || opCode == OP_CLOSE);
}
T setPath(String path) {
this.path = path;
return (T)this;
}
T setReplication(short replication) {
this.replication = replication;
return (T)this;
}
T setModificationTime(long mtime) {
this.mtime = mtime;
return (T)this;
}
T setAccessTime(long atime) {
this.atime = atime;
return (T)this;
}
T setBlockSize(long blockSize) {
this.blockSize = blockSize;
return (T)this;
}
T setBlocks(Block[] blocks) {
this.blocks = blocks;
return (T)this;
}
T setPermissionStatus(PermissionStatus permissions) {
this.permissions = permissions;
return (T)this;
}
T setClientName(String clientName) {
this.clientName = clientName;
return (T)this;
}
T setClientMachine(String clientMachine) {
this.clientMachine = clientMachine;
return (T)this;
}
@Override
void writeFields(DataOutputStream out) throws IOException {
FSImageSerialization.writeString(path, out);
FSImageSerialization.writeShort(replication, out);
FSImageSerialization.writeLong(mtime, out);
FSImageSerialization.writeLong(atime, out);
FSImageSerialization.writeLong(blockSize, out);
new ArrayWritable(Block.class, blocks).write(out);
permissions.write(out);
if (this.opCode == OP_ADD) {
FSImageSerialization.writeString(clientName,out);
FSImageSerialization.writeString(clientMachine,out);
}
}
@Override
void readFields(DataInputStream in, int logVersion)
throws IOException {
// versions > 0 support per file replication
// get name and replication
if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
this.length = in.readInt();
}
if (-7 == logVersion && length != 3||
-17 < logVersion && logVersion < -7 && length != 4 ||
(logVersion <= -17 && length != 5 && !LayoutVersion.supports(
Feature.EDITLOG_OP_OPTIMIZATION, logVersion))) {
throw new IOException("Incorrect data format." +
" logVersion is " + logVersion +
" but writables.length is " +
length + ". ");
}
this.path = FSImageSerialization.readString(in);
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
this.replication = FSImageSerialization.readShort(in);
this.mtime = FSImageSerialization.readLong(in);
} else {
this.replication = readShort(in);
this.mtime = readLong(in);
}
if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, logVersion)) {
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
this.atime = FSImageSerialization.readLong(in);
} else {
this.atime = readLong(in);
}
} else {
this.atime = 0;
}
if (logVersion < -7) {
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
this.blockSize = FSImageSerialization.readLong(in);
} else {
this.blockSize = readLong(in);
}
} else {
this.blockSize = 0;
}
// get blocks
this.blocks = readBlocks(in, logVersion);
if (logVersion <= -11) {
this.permissions = PermissionStatus.read(in);
} else {
this.permissions = null;
}
// clientname, clientMachine and block locations of last block.
if (this.opCode == OP_ADD && logVersion <= -12) {
this.clientName = FSImageSerialization.readString(in);
this.clientMachine = FSImageSerialization.readString(in);
if (-13 <= logVersion) {
readDatanodeDescriptorArray(in);
}
} else {
this.clientName = "";
this.clientMachine = "";
}
}
/** This method is defined for compatibility reason. */
private static DatanodeDescriptor[] readDatanodeDescriptorArray(DataInput in)
throws IOException {
DatanodeDescriptor[] locations = new DatanodeDescriptor[in.readInt()];
for (int i = 0; i < locations.length; i++) {
locations[i] = new DatanodeDescriptor();
locations[i].readFieldsFromFSEditLog(in);
}
return locations;
}
private static Block[] readBlocks(
DataInputStream in,
int logVersion) throws IOException {
int numBlocks = in.readInt();
Block[] blocks = new Block[numBlocks];
for (int i = 0; i < numBlocks; i++) {
Block blk = new Block();
if (logVersion <= -14) {
blk.readFields(in);
} else {
BlockTwo oldblk = new BlockTwo();
oldblk.readFields(in);
blk.set(oldblk.blkid, oldblk.len,
GenerationStamp.GRANDFATHER_GENERATION_STAMP);
}
blocks[i] = blk;
}
return blocks;
}
}
static class AddOp extends AddCloseOp {
private AddOp() {
super(OP_ADD);
}
static AddOp getInstance() {
return (AddOp)opInstances.get().get(OP_ADD);
}
}
static class CloseOp extends AddCloseOp {
private CloseOp() {
super(OP_CLOSE);
}
static CloseOp getInstance() {
return (CloseOp)opInstances.get().get(OP_CLOSE);
}
}
static class SetReplicationOp extends FSEditLogOp {
String path;
short replication;
private SetReplicationOp() {
super(OP_SET_REPLICATION);
}
static SetReplicationOp getInstance() {
return (SetReplicationOp)opInstances.get()
.get(OP_SET_REPLICATION);
}
SetReplicationOp setPath(String path) {
this.path = path;
return this;
}
SetReplicationOp setReplication(short replication) {
this.replication = replication;
return this;
}
@Override
void writeFields(DataOutputStream out) throws IOException {
FSImageSerialization.writeString(path, out);
FSImageSerialization.writeShort(replication, out);
}
@Override
void readFields(DataInputStream in, int logVersion)
throws IOException {
this.path = FSImageSerialization.readString(in);
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) {
this.replication = FSImageSerialization.readShort(in);
} else {
this.replication = readShort(in);
}
}
}
static class ConcatDeleteOp extends FSEditLogOp {
int length;
String trg;
String[] srcs;
long timestamp;
private ConcatDeleteOp() {
super(OP_CONCAT_DELETE);
}
static ConcatDeleteOp getInstance() {
return (ConcatDeleteOp)opInstances.get()
.get(OP_CONCAT_DELETE);
}
ConcatDeleteOp setTarget(String trg) {
this.trg = trg;
return this;
}
ConcatDeleteOp setSources(String[] srcs) {
this.srcs = srcs;
return this;
}
ConcatDeleteOp setTimestamp(long timestamp) {
this.timestamp = timestamp;
return this;
}
@Override
void writeFields(DataOutputStream out) throws IOException {
FSImageSerialization.writeString(trg, out);
DeprecatedUTF8 info[] = new DeprecatedUTF8[srcs.length];
int idx = 0;
for(int i=0; i © 2015 - 2025 Weber Informatics LLC | Privacy Policy