All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.aliyun.odps.tunnel.VolumeTunnel Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package com.aliyun.odps.tunnel;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.aliyun.odps.Odps;
import com.aliyun.odps.OdpsException;
import com.aliyun.odps.commons.transport.Connection;
import com.aliyun.odps.commons.transport.Headers;
import com.aliyun.odps.commons.transport.Response;
import com.aliyun.odps.commons.util.IOUtils;
import com.aliyun.odps.rest.RestClient;
import com.aliyun.odps.tunnel.io.CompressOption;
import com.aliyun.odps.tunnel.io.VolumeInputStream;
import com.aliyun.odps.tunnel.io.VolumeOutputStream;

/**
 * 访问ODPS Volume Tunnel服务的入口类
 * 暂未开放,仅限内部使用
 *
 * @author [email protected]
 */
public class VolumeTunnel {

  private Configuration config;

  /**
   * 构造此类对象
   *
   * @param odps
   *     {@link com.aliyun.odps.Odps}
   */
  public VolumeTunnel(Odps odps) {
    this.config = new Configuration(odps);
  }

  /**
   * 设置TunnelServer地址
   *
   * 

没有设置TunnelServer地址的情况下, 自动选择

* * @param endpoint */ public void setEndpoint(String endpoint) { try { URI u = new URI(endpoint); config.setEndpoint(u); } catch (URISyntaxException e) { throw new IllegalArgumentException("Invalid endpoint."); } } /** * 构造一个新的{@link com.aliyun.odps.tunnel.VolumeTunnel.UploadSession}对象。 * * @param projectName * 上传File所在的project名称 * @param volumeName * 上传File所在的volume名称 * @param partition * 上传File的partition描述,由字母,数字,下划线组成,3-32个字符,举例如下: my_pt_001 */ public UploadSession createUploadSession(String projectName, String volumeName, String partition) throws TunnelException { return new UploadSession(projectName, volumeName, partition); } /** * 根据已有uploadId获取一个{@link com.aliyun.odps.tunnel.VolumeTunnel.UploadSession}对象。 * * @param projectName * 上传File所在的project名称 * @param volumeName * 上传File所在的volume名称 * @param partition * 上传File的partition描述,由字母,数字,下划线组成,3-32个字符,举例如下: my_pt_001 * @param uploadId * Upload的唯一标识符 */ public UploadSession getUploadSession(String projectName, String volumeName, String partition, String uploadId) throws TunnelException { return new UploadSession(projectName, volumeName, partition, uploadId); } /** * 构造一个新的{@link com.aliyun.odps.tunnel.VolumeTunnel.DownloadSession}对象。 * * @param projectName * 下载File所在project名称 * @param volumeName * 下载File所在volume名称 * @param partition * 下载File的partition描述 * @param fileName * 下载File的名称 */ public DownloadSession createDownloadSession(String projectName, String volumeName, String partition, String fileName) throws TunnelException { return new DownloadSession(projectName, volumeName, partition, fileName); } /** * 根据已有downloadId获取一个{@link com.aliyun.odps.tunnel.VolumeTunnel.DownloadSession}对象。 * * @param projectName * 下载File所在project名称 * @param volumeName * 下载File所在volume名称 * @param partition * 下载File的partition描述 * @param fileName * 下载File的名称 * @param downloadId * VolumeTunnel.DownloadSession的唯一标识符 */ public DownloadSession getDownloadSession(String projectName, String volumeName, String partition, String fileName, String downloadId) throws TunnelException { return new DownloadSession(projectName, volumeName, partition, fileName, downloadId); } /** * UploadStatus表示当前Upload的状态 *
UNKNOWN 未知 *
NORMAL 正常 *
CLOSING 关闭中 *
CLOSED 已关闭 *
CANCELED 已取消 *
EXPIRED 已过期 *
CRITICAL 严重错误 */ public static enum UploadStatus { UNKNOWN, NORMAL, CLOSING, CLOSED, CANCELED, EXPIRED, CRITICAL } /** * UploadSession表示一个向ODPS表中上传数据的会话 * *

* 向ODPS表上传数据的流程如下:
* 1) 创建UploadSession
* 2) 上传数据
* 3) 提交 *

*/ public class UploadSession { private String id; private String projectName; private String volumeName; private String partitionSpec; private HashMap fileLists = new HashMap(); private UploadStatus status = UploadStatus.UNKNOWN; private Configuration conf; private RestClient tunnelServiceClient; /** * 构造一个新的{@link UploadSession}对象。 * * @param projectName * 上传File所在的project名称。 * @param volumeName * 上传File所在Volume名称。 * @param partitionSpec * 上传File的partition描述,格式如下: pt=xxx,dt=xxx。 */ public UploadSession(String projectName, String volumeName, String partitionSpec) throws TunnelException { this.conf = VolumeTunnel.this.config; this.projectName = projectName; this.volumeName = volumeName; this.partitionSpec = partitionSpec; tunnelServiceClient = conf.newRestClient(projectName); initiate(); } /** * 根据已有的uploadId构造一个{@link UploadSession}对象。 * * @param projectName * 上传File所在的project名称。 * @param volumeName * 上传File所在Volume名称。 * @param partitionSpec * 上传File的partition描述,格式如下: pt=xxx,dt=xxx。 * @param uploadId * UploadSession的唯一标识符 */ public UploadSession(String projectName, String volumeName, String partitionSpec, String uploadId) throws TunnelException { this.conf = VolumeTunnel.this.config; this.projectName = projectName; this.volumeName = volumeName; this.partitionSpec = partitionSpec; this.id = uploadId; tunnelServiceClient = conf.newRestClient(projectName); reload(); } private void initiate() throws TunnelException { HashMap headers = new HashMap(); headers.put(Headers.CONTENT_LENGTH, String.valueOf(0)); HashMap params = new HashMap(); params.put(TunnelConstants.TYPE, "volumefile"); params .put(TunnelConstants.TARGET, projectName + "/" + volumeName + "/" + partitionSpec + "/"); Connection conn = null; try { conn = tunnelServiceClient.connect(getResource(), "POST", params, headers); Response resp = conn.getResponse(); if (resp.isOK()) { loadFromJson(conn.getInputStream()); } else { TunnelException e = new TunnelException(conn.getInputStream()); e.setRequestId(resp.getHeader(HttpHeaders.HEADER_ODPS_REQUEST_ID)); throw e; } } catch (IOException e) { throw new TunnelException("Failed to create upload session with tunnel endpoint " + tunnelServiceClient.getEndpoint(), e); } catch (TunnelException e) { throw e; } catch (OdpsException e) { throw new TunnelException(e.getMessage(), e); } finally { if (conn != null) { try { conn.disconnect(); } catch (IOException e) { // nothing } } } } /** * 创建{@link java.io.OutputStream}用来将数据流入到指定File。 * * @param fileName * 指定File的名称。 */ public OutputStream openOutputStream(String fileName) throws TunnelException, IOException { return openOutputStream(fileName, false, false); } /** * 创建{@link java.io.OutputStream}用来将数据流入到指定File。 * * @param fileName * 指定File的名称。 * @param compress * 设置压缩参数对传输数据进行压缩 */ public OutputStream openOutputStream(String fileName, boolean compress) throws TunnelException, IOException { return openOutputStream(fileName, compress, false); } /** * 创建{@link java.io.OutputStream}用来将数据流入到指定File。 * * @param fileName * 指定File的名称。 * @param compress * 数据传输是否进行压缩 * @param append * 是否采用断点续传 */ public OutputStream openOutputStream(String fileName, boolean compress, boolean append) throws TunnelException, IOException { HashMap params = new HashMap(); HashMap headers = new HashMap(); headers.put(Headers.CONTENT_TYPE, "text/plain"); headers.put(Headers.TRANSFER_ENCODING, Headers.CHUNKED); headers.put(HttpHeaders.HEADER_ODPS_TUNNEL_VERSION, String.valueOf(TunnelConstants.VERSION)); if (compress) { if (conf.getCompressOption().algorithm.equals(CompressOption.CompressAlgorithm.ODPS_ZLIB)) { headers.put(HttpHeaders.CONTENT_ENCODING, "deflate"); } else { throw new TunnelException("invalid compression option."); } } fileName = formatFileName(fileName); params.put(TunnelConstants.BLOCKID, fileName); if (append) { params.put(TunnelConstants.RESUME_MODE, null); } VolumeOutputStream vout = null; Connection conn = null; try { conn = tunnelServiceClient.connect(getResource() + "/" + id, "POST", params, headers); CompressOption option = compress ? conf.getCompressOption() : null; vout = new VolumeOutputStream(conn, option); } catch (IOException e) { if (conn != null) { conn.disconnect(); } throw new TunnelException(e.getMessage(), e); } catch (TunnelException e) { throw e; } catch (OdpsException e) { if (conn != null) { conn.disconnect(); } throw new TunnelException(e.getMessage(), e); } return vout; } private void reload() throws TunnelException { HashMap params = new HashMap(); HashMap headers = new HashMap(); headers.put(Headers.CONTENT_LENGTH, String.valueOf(0)); Connection conn = null; try { conn = tunnelServiceClient.connect(getResource() + "/" + id, "GET", params, headers); Response resp = conn.getResponse(); if (resp.isOK()) { loadFromJson(conn.getInputStream()); } else { TunnelException e = new TunnelException(conn.getInputStream()); e.setRequestId(resp.getHeader(HttpHeaders.HEADER_ODPS_REQUEST_ID)); throw e; } } catch (IOException e) { throw new TunnelException(e.getMessage(), e); } catch (TunnelException e) { throw e; } catch (OdpsException e) { throw new TunnelException(e.getMessage(), e); } finally { if (conn != null) { try { conn.disconnect(); } catch (IOException e) { // } } } } /** * 提交本次上传的所有file * * @param files * 已经成功上传的Files列表 * @throws com.aliyun.odps.tunnel.TunnelException * 如果提供的file列表与Server端存在的file不一致抛出异常 */ public void commit(String[] files) throws TunnelException, IOException { if (files == null) { throw new IllegalArgumentException("Invalid argument: files."); } List formatFiles = new ArrayList(); for (String fileId : files) { formatFiles.add(formatFileName(fileId)); } files = formatFiles.toArray(new String[0]); HashMap clientFileMap = new HashMap(); for (String fileId : files) { clientFileMap.put(fileId, true); } this.getStatus(); if (fileLists.size() != clientFileMap.size()) { throw new TunnelException( "File number not match, server: " + fileLists.size() + ", client: " + clientFileMap.size()); } for (String fileId : files) { if (!fileLists.containsKey(fileId)) { throw new TunnelException("File not exits on server, file name is " + fileId); } } completeUpload(); } private String formatFileName(String fileName) throws TunnelException { StringBuilder sb = new StringBuilder(); if (fileName.length() > 0 && fileName.charAt(0) == '/') { throw new TunnelException("FileName cann't start with '/', file name is " + fileName); } int pos = 0; boolean preSlash = false; while (pos < fileName.length()) { if (fileName.charAt(pos) == '/') { if (!preSlash) { sb.append(fileName.charAt(pos)); } preSlash = true; } else { sb.append(fileName.charAt(pos)); preSlash = false; } pos++; } return sb.toString(); } private void completeUpload() throws TunnelException, IOException { HashMap params = new HashMap(); HashMap headers = new HashMap(); headers.put(Headers.CONTENT_LENGTH, String.valueOf(0)); int count = 0; while (true) { count++; Connection conn = null; try { conn = tunnelServiceClient.connect(getResource() + "/" + id, "PUT", params, headers); Response resp = conn.getResponse(); if (resp.isOK()) { loadFromJson(conn.getInputStream()); break; } else { if (resp.getStatus() == HttpURLConnection.HTTP_INTERNAL_ERROR && count < 3) { try { Thread.sleep(2 * count * 1000); } catch (InterruptedException e) { throw new TunnelException(e.getMessage(), e); } continue; } throw new TunnelException(conn.getInputStream()); } } catch (IOException e) { throw new TunnelException(e.getMessage(), e); } catch (TunnelException e) { throw e; } catch (OdpsException e) { throw new TunnelException(e.getMessage(), e); } finally { if (conn != null) { conn.disconnect(); } } } } /** * 获取当前{@link UploadSession}的唯一标识符 */ public String getId() { return this.id; } /** * 获取当前{@link UploadSession}的{@link Status} */ public UploadStatus getStatus() throws TunnelException { reload(); return this.status; } /** * 获取当前{@link UploadSession}已成功上传的files */ public String[] getFileList() throws TunnelException, IOException { reload(); return fileLists.keySet().toArray(new String[0]); } /* * 获取(@param fileName)在当前{@link UploadSession}已成功上传的长度 * * @param fileName * 指定File的名称。 * * @throws TunnelException * */ public Long getFileLength(String fileName) throws TunnelException, IOException { fileName = formatFileName(fileName); reload(); if (fileLists.containsKey(fileName)) { return fileLists.get(fileName); } else { throw new TunnelException(fileName + " not exists in tunnel server"); } } private String getResource() { StringBuilder sb = new StringBuilder(); sb.append("/projects/").append(projectName).append("/tunnel").append("/uploads"); return sb.toString(); } /* * Parse session properties from json stream. */ private void loadFromJson(InputStream is) throws TunnelException { try { String json = IOUtils.readStreamAsString(is); JSONObject tree = JSONObject.parseObject(json); // session id String node = tree.getString("UploadID"); if (node != null) { id = node; } // status node = tree.getString("Status"); if (node != null) { status = UploadStatus.valueOf(node.toUpperCase()); } // fileList fileLists.clear(); JSONArray node2 = tree.getJSONArray("FileList"); if (node2 != null) { for (int i = 0; i < node2.size(); ++i) { JSONObject fileNode = node2.getJSONObject(i); String fileName = fileNode.getString("FileName"); Long fileLength = fileNode.getLong("FileLength"); fileLists.put(fileName, fileLength); } } } catch (Exception e) { throw new TunnelException("Invalid json content.", e); } } } /** * 下载会话的状态 *
UNKNOWN 未知 *
NORMAL 正常 *
CLOSED 关闭 *
EXPIRED 过期 */ public static enum DownloadStatus { UNKNOWN, NORMAL, CLOSED, EXPIRED } /** * 此类表示一个向ODPS中下载Volume的会话 * *

* 向ODPS下载Volume的流程如下:
* 1) 创建DownloadSession
* 2) 下载数据
*

*/ public class DownloadSession { private String id; private String projectName; private String volumeName; private String partitionSpec; private String fileName; private long fileLength = (long) -1; private DownloadStatus status = DownloadStatus.UNKNOWN; private Configuration conf; private RestClient tunnelServiceClient; /** * 构造一个新的{@link DownloadSession}。 * * @param projectName * 下载File所在project名称 * @param volumeName * 下载File所在volume名称 * @param partitionSpec * 下载File的partition描述,格式如下: pt=xxx,dt=xxx * @param fileName * 下载File的名称 */ public DownloadSession(String projectName, String volumeName, String partitionSpec, String fileName) throws TunnelException { this.conf = VolumeTunnel.this.config; this.projectName = projectName; this.volumeName = volumeName; this.partitionSpec = partitionSpec; this.fileName = formatFileName(fileName); tunnelServiceClient = conf.newRestClient(projectName); initiate(); } /** * 根据已有downloadId构造一个{@link DownloadSession}对象。 * * @param projectName * 下载File所在project名称 * @param volumeName * 下载File所在volume名称 * @param partitionSpec * 下载File的partition描述,格式如下: pt=xxx,dt=xxx * @param fileName * 下载File的名称 * @param downloadId * DownloadSession的唯一标识符 */ public DownloadSession(String projectName, String volumeName, String partitionSpec, String fileName, String downloadId) throws TunnelException { this.conf = VolumeTunnel.this.config; this.projectName = projectName; this.volumeName = volumeName; this.partitionSpec = partitionSpec; this.fileName = formatFileName(fileName); this.id = downloadId; tunnelServiceClient = conf.newRestClient(projectName); reload(); //check if volumeName, partitionName and fileName consistent with downloadId. if (!this.volumeName.equalsIgnoreCase(volumeName) || !this.partitionSpec.equalsIgnoreCase(partitionSpec) || !this.fileName.equalsIgnoreCase(fileName)) { throw new TunnelException( "volumeName, partitionName or fileName doesn't match whith downloadId."); } } // initiate a new volume download session private void initiate() throws TunnelException { HashMap headers = new HashMap(); headers.put(Headers.CONTENT_LENGTH, String.valueOf(0)); HashMap params = new HashMap(); params.put(TunnelConstants.TYPE, "volumefile"); params.put(TunnelConstants.TARGET, projectName + "/" + volumeName + "/" + partitionSpec + "/" + fileName); Connection conn = null; try { conn = tunnelServiceClient.connect(getResource(), "POST", params, headers); Response resp = conn.getResponse(); if (resp.isOK()) { loadFromJson(conn.getInputStream()); } else { TunnelException e = new TunnelException(conn.getInputStream()); e.setRequestId(resp.getHeader(HttpHeaders.HEADER_ODPS_REQUEST_ID)); throw e; } } catch (IOException e) { throw new TunnelException("Failed to create download session with tunnel endpoint " + tunnelServiceClient.getEndpoint(), e); } catch (TunnelException e) { throw e; } catch (OdpsException e) { throw new TunnelException(e.getMessage(), e); } finally { if (conn != null) { try { conn.disconnect(); } catch (IOException e) { // nothing } } } } // reload volume download session properties private void reload() throws TunnelException { HashMap params = new HashMap(); HashMap headers = new HashMap(); headers.put(Headers.CONTENT_LENGTH, String.valueOf(0)); Connection conn = null; try { conn = tunnelServiceClient.connect(getResource() + "/" + id, "GET", params, headers); Response resp = conn.getResponse(); if (resp.isOK()) { loadFromJson(conn.getInputStream()); } else { TunnelException e = new TunnelException(conn.getInputStream()); e.setRequestId(resp.getHeader(HttpHeaders.HEADER_ODPS_REQUEST_ID)); throw e; } } catch (IOException e) { throw new TunnelException(e.getMessage(), e); } catch (TunnelException e) { throw e; } catch (OdpsException e) { throw new TunnelException(e.getMessage(), e); } finally { if (conn != null) { try { conn.disconnect(); } catch (IOException e) { // } } } } /** * 创建{@link java.io.InputStream}用来将指定File以数据流的形式输出。 */ public InputStream openInputStream() throws TunnelException, IOException { return this.openInputStream(0, Long.MAX_VALUE, false); } /** * 创建{@link java.io.InputStream}用来将指定File以数据流的形式输出。 * * @param compress * 压缩选项,即使设置了压缩选项,如果server * 不支持压缩,传输数据也不会被压缩 */ public InputStream openInputStream(boolean compress) throws TunnelException, IOException { return this.openInputStream(0, Long.MAX_VALUE, compress); } /** * 创建{@link java.io.InputStream}用来将指定File以数据流的形式输出。 * * @param start * 本次要读的起始位置。 * @param length * 本次要读的字节数量。 */ public InputStream openInputStream(long start, long length) throws TunnelException, IOException { return openInputStream(start, length, false); } /** * 创建{@link java.io.InputStream}用来将指定File以数据流的形式输出。 * * @param start * 本次要读的起始位置。 * @param length * 本次要读的字节数量。 * @param compress * 压缩选项,即使设置了压缩选项,如果server * 不支持压缩,传输数据也不会被压缩 */ public InputStream openInputStream(long start, long length, boolean compress) throws TunnelException, IOException { HashMap params = new HashMap(); HashMap headers = new HashMap(); headers.put(Headers.CONTENT_LENGTH, String.valueOf(0)); headers.put(HttpHeaders.HEADER_ODPS_TUNNEL_VERSION, String.valueOf(TunnelConstants.VERSION)); if (compress) { if (conf.getCompressOption().algorithm .equals(CompressOption.CompressAlgorithm.ODPS_ZLIB)) { headers.put(Headers.ACCEPT_ENCODING, "deflate"); } else { throw new TunnelException("invalid compression option."); } } params.put("data", null); params.put(TunnelConstants.RANGE, "(" + start + "," + length + ")"); VolumeInputStream vin = null; Connection conn = null; try { conn = tunnelServiceClient.connect(getResource() + "/" + id, "GET", params, headers); Response resp = conn.getResponse(); if (!resp.isOK()) { TunnelException err = new TunnelException(conn.getInputStream()); err.setRequestId(resp.getHeader(HttpHeaders.HEADER_ODPS_REQUEST_ID)); throw new IOException(err); } String content_encoding = resp.getHeader(Headers.CONTENT_ENCODING); if (content_encoding != null) { if (resp.getHeader(Headers.CONTENT_ENCODING).equals("deflate")) { conf.setCompressOption(new CompressOption( CompressOption.CompressAlgorithm.ODPS_ZLIB, -1, 0)); } else { throw new TunnelException("invalid content encoding"); } compress = true; } else { compress = false; } CompressOption option = compress ? conf.getCompressOption() : null; vin = new VolumeInputStream(conn, option); } catch (IOException e) { if (conn != null) { conn.disconnect(); } throw new TunnelException(e.getMessage(), e); } catch (TunnelException e) { throw e; } catch (OdpsException e) { if (conn != null) { conn.disconnect(); } throw new TunnelException(e.getMessage(), e); } return vin; } /** * 获取当前{@link DownloadSession}操作的File的长度。 */ public long getFileLength() { return this.fileLength; } /** * 获取当前{@link DownloadSession}的唯一标识符 */ public String getId() { return this.id; } /** * 获取当前{@link DownloadSession}的{@link Status} */ public DownloadStatus getStatus() throws TunnelException, IOException { reload(); return status; } private String getResource() { StringBuilder sb = new StringBuilder(); sb.append("/projects/").append(this.projectName).append("/tunnel").append("/downloads"); return sb.toString(); } private void loadFromJson(InputStream is) throws TunnelException { try { String json = IOUtils.readStreamAsString(is); JSONObject tree = JSON.parseObject(json); // session id String node = tree.getString("DownloadID"); if (node != null) { id = node; } // status node = tree.getString("Status"); if (node != null) { status = DownloadStatus.valueOf(node.toUpperCase()); } // file JSONObject node2 = tree.getJSONObject("File"); if (node2 != null) { fileName = node2.getString("FileName"); fileLength = node2.getLong("FileLength"); } // partition node2 = tree.getJSONObject("Partition"); if (node2 != null) { volumeName = node2.getString("Volume"); partitionSpec = node2.getString("Partition"); } } catch (Exception e) { throw new TunnelException("Invalid json content.", e); } } private String formatFileName(String fileName) throws TunnelException { StringBuilder sb = new StringBuilder(); if (fileName.length() > 0 && fileName.charAt(0) == '/') { throw new TunnelException("FileName cann't start with '/', file name is " + fileName); } int pos = 0; boolean preSlash = false; while (pos < fileName.length()) { if (fileName.charAt(pos) == '/') { if (!preSlash) { sb.append(fileName.charAt(pos)); } preSlash = true; } else { sb.append(fileName.charAt(pos)); preSlash = false; } pos++; } return sb.toString(); } } }