gobblin.commit.FsRenameCommitStep Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of gobblin-core Show documentation
Gobblin Ingestion Framework
There is a newer version: 0.8.0
/*
 * Copyright (C) 2014-2016 LinkedIn Corp. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use
 * this file except in compliance with the License. You may obtain a copy of the
 * License at  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed
 * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 * CONDITIONS OF ANY KIND, either express or implied.
 */

package gobblin.commit;

import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import com.google.common.base.Preconditions;

import gobblin.annotation.Alpha;
import gobblin.configuration.ConfigurationKeys;
import gobblin.configuration.State;
import gobblin.util.HadoopUtils;

import lombok.extern.slf4j.Slf4j;


/**
 * A {@link CommitStep} for renaming files within a {@link FileSystem} or between {@link FileSystem}s.
 *
 * @author ziliu
 */
@Alpha
@Slf4j
public class FsRenameCommitStep extends CommitStepBase {

  private final Path srcPath;
  private final Path dstPath;
  private final String srcFsUri;
  private final String dstFsUri;
  private final boolean overwrite;
  private transient FileSystem srcFs;
  private transient FileSystem dstFs;

  private FsRenameCommitStep(Builder> builder) throws IOException {
    super(builder);

    this.srcPath = builder.srcPath;
    this.dstPath = builder.dstPath;
    this.srcFs = builder.srcFs != null ? builder.srcFs
        : getFileSystem(this.props.getProp(ConfigurationKeys.FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI));
    this.srcFsUri = this.srcFs.getUri().toString();
    this.dstFs = builder.dstFs != null ? builder.dstFs
        : getFileSystem(this.props.getProp(ConfigurationKeys.FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI));
    this.dstFsUri = this.dstFs.getUri().toString();
    this.overwrite = builder.overwrite;
  }

  public static class Builder> extends CommitStepBase.Builder {

    public Builder() {
      super();
    }

    public Builder(CommitSequence.Builder commitSequenceBuilder) {
      super(commitSequenceBuilder);
    }

    private Path srcPath;
    private Path dstPath;
    private FileSystem srcFs;
    private FileSystem dstFs;
    private boolean overwrite;

    @Override
    public T withProps(State props) {
      return (T) super.withProps(props);
    }

    @SuppressWarnings("unchecked")
    public T from(Path srcPath) {
      this.srcPath = srcPath;
      return (T) this;
    }

    @SuppressWarnings("unchecked")
    public T to(Path dstPath) {
      this.dstPath = dstPath;
      return (T) this;
    }

    @SuppressWarnings("unchecked")
    public T withSrcFs(FileSystem srcFs) {
      this.srcFs = srcFs;
      return (T) this;
    }

    @SuppressWarnings("unchecked")
    public T withDstFs(FileSystem dstFs) {
      this.dstFs = dstFs;
      return (T) this;
    }

    @SuppressWarnings("unchecked")
    public T overwrite() {
      this.overwrite = true;
      return (T) this;
    }

    @Override
    public CommitStep build() throws IOException {
      Preconditions.checkNotNull(this.srcPath);
      Preconditions.checkNotNull(this.dstPath);

      return new FsRenameCommitStep(this);
    }
  }

  private FileSystem getFileSystem(String fsUri) throws IOException {
    return FileSystem.get(URI.create(fsUri), HadoopUtils.getConfFromState(this.props));
  }

  @Override
  public boolean isCompleted() throws IOException {
    if (this.dstFs == null) {
      this.dstFs = getFileSystem(this.dstFsUri);
    }
    return this.dstFs.exists(this.dstPath);
  }

  @Override
  public void execute() throws IOException {
    if (this.srcFs == null) {
      this.srcFs = getFileSystem(this.srcFsUri);
    }
    if (this.dstFs == null) {
      this.dstFs = getFileSystem(this.dstFsUri);
    }
    log.info(String.format("Moving %s to %s", this.srcPath, this.dstPath));
    HadoopUtils.movePath(this.srcFs, this.srcPath, this.dstFs, this.dstPath, this.overwrite, this.dstFs.getConf());
  }
}