All Downloads are FREE. Search and download functionalities are using the official Maven repository.
  • Log In
  • Register

    • org.apache.zeppelin.submarine.hadoop.YarnClient Maven / Gradle / Ivy

    There is a newer version: 0.11.2
    Show newest version
    /*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.zeppelin.submarine.hadoop;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.security.Principal;
import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;

import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonIOException;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSyntaxException;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;

import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.config.Lookup;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.submarine.commons.SubmarineConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.LoginContext;

public class YarnClient {
  private Logger LOGGER = LoggerFactory.getLogger(YarnClient.class);

  private Configuration hadoopConf;
  private String yarnWebHttpAddr;
  private String principal = "";
  private String keytab = "";

  public static final String YARN_REST_APPATTEMPTS = "appAttempts";
  public static final String YARN_REST_CONTAINER = "container";
  public static final String YARN_REST_APPATTEMPT = "appAttempt";
  public static final String YARN_REST_APPATTEMPTID = "appAttemptId";

  public static final String YARN_REST_EXPOSEDPORTS = "EXPOSEDPORTS";
  public static final String CONTAINER_IP = "CONTAINER_IP";
  public static final String CONTAINER_PORT = "CONTAINER_PORT";
  public static final String HOST_IP = "HOST_IP";
  public static final String HOST_PORT = "HOST_PORT";

  String SERVICE_PATH = "/services/{service_name}";

  private boolean hadoopSecurityEnabled = true; // simple or kerberos

  public YarnClient(Properties properties) {
    this.hadoopConf = new Configuration();

    String hadoopAuthType = properties.getProperty(
        SubmarineConstants.ZEPPELIN_SUBMARINE_AUTH_TYPE, "kerberos");
    if (StringUtils.equals(hadoopAuthType, "simple")) {
      hadoopSecurityEnabled = false;
    }

    yarnWebHttpAddr = properties.getProperty(SubmarineConstants.YARN_WEB_HTTP_ADDRESS, "");
    boolean isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
    if (isSecurityEnabled || hadoopSecurityEnabled) {
      String krb5conf = properties.getProperty(SubmarineConstants.SUBMARINE_HADOOP_KRB5_CONF, "");
      if (StringUtils.isEmpty(krb5conf)) {
        krb5conf = "/etc/krb5.conf";
        System.setProperty("java.security.krb5.conf", krb5conf);
      }

      String keytab = properties.getProperty(
          SubmarineConstants.SUBMARINE_HADOOP_KEYTAB, "");
      String principal = properties.getProperty(
          SubmarineConstants.SUBMARINE_HADOOP_PRINCIPAL, "");

      ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
      if (StringUtils.isEmpty(keytab)) {
        keytab = zConf.getString(
            ZeppelinConfiguration.ConfVars.ZEPPELIN_SERVER_KERBEROS_KEYTAB);
      }
      if (StringUtils.isEmpty(principal)) {
        principal = zConf.getString(
            ZeppelinConfiguration.ConfVars.ZEPPELIN_SERVER_KERBEROS_PRINCIPAL);
      }
      if (StringUtils.isBlank(keytab) || StringUtils.isBlank(principal)) {
        throw new RuntimeException("keytab and principal can not be empty, keytab: "
            + keytab + ", principal: " + principal);
      }

      this.principal = principal;
      this.keytab = keytab;
      if (LOGGER.isDebugEnabled()) {
        System.setProperty("sun.security.spnego.debug", "true");
        System.setProperty("sun.security.krb5.debug", "true");
      }
    }
  }

  // http://yarn-web-http-address/app/v1/services/{service_name}
  public void deleteService(String serviceName) {
    String appUrl = this.yarnWebHttpAddr + "/app/v1/services/" + serviceName
        + "?_=" + System.currentTimeMillis();

    InputStream inputStream = null;
    try {
      HttpResponse response = callRestUrl(appUrl, principal, HTTP.DELETE);
      inputStream = response.getEntity().getContent();
      String result = new BufferedReader(new InputStreamReader(inputStream))
          .lines().collect(Collectors.joining(System.lineSeparator()));
      if (response.getStatusLine().getStatusCode() != 200 /*success*/) {
        LOGGER.warn("Status code " + response.getStatusLine().getStatusCode());
        LOGGER.warn("message is :" + Arrays.deepToString(response.getAllHeaders()));
        LOGGER.warn("result：\n" + result);
      }
    } catch (Exception exp) {
      exp.printStackTrace();
    } finally {
      try {
        if (null != inputStream) {
          inputStream.close();
        }
      } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
      }
    }
  }

  // http://yarn-web-http-address/app/v1/services/{appIdOrName}
  // test/resources/app-v1-services-app_name.json
  public Map getAppServices(String appIdOrName) {
    Map mapStatus = new HashMap<>();
    String appUrl = this.yarnWebHttpAddr + "/app/v1/services/" + appIdOrName
        + "?_=" + System.currentTimeMillis();

    InputStream inputStream = null;
    try {
      HttpResponse response = callRestUrl(appUrl, principal, HTTP.GET);
      inputStream = response.getEntity().getContent();
      String result = new BufferedReader(new InputStreamReader(inputStream))
          .lines().collect(Collectors.joining(System.lineSeparator()));
      if (response.getStatusLine().getStatusCode() != 200 /*success*/
          && response.getStatusLine().getStatusCode() != 404 /*Not found*/) {
        LOGGER.warn("Status code " + response.getStatusLine().getStatusCode());
        LOGGER.warn("message is :" + Arrays.deepToString(response.getAllHeaders()));
        LOGGER.warn("result：\n" + result);
      }

      // parse app status json
      mapStatus = parseAppServices(result);
    } catch (Exception exp) {
      exp.printStackTrace();
    } finally {
      try {
        if (null != inputStream) {
          inputStream.close();
        }
      } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
      }
    }

    return mapStatus;
  }

  // http://yarn-web-http-address/ws/v1/cluster/apps/{appId}
  // test/resources/ws-v1-cluster-apps-application_id-failed.json
  // test/resources/ws-v1-cluster-apps-application_id-finished.json
  // test/resources/ws-v1-cluster-apps-application_id-running.json
  public Map getClusterApps(String appId) {
    Map appAttempts = new HashMap<>();
    String appUrl = this.yarnWebHttpAddr + "/ws/v1/cluster/apps/" + appId
        + "?_=" + System.currentTimeMillis();

    InputStream inputStream = null;
    try {
      HttpResponse response = callRestUrl(appUrl, principal, HTTP.GET);
      inputStream = response.getEntity().getContent();
      String result = new BufferedReader(new InputStreamReader(inputStream))
          .lines().collect(Collectors.joining(System.lineSeparator()));
      if (response.getStatusLine().getStatusCode() != 200 /*success*/) {
        LOGGER.warn("Status code " + response.getStatusLine().getStatusCode());
        LOGGER.warn("message is :" + Arrays.deepToString(response.getAllHeaders()));
        LOGGER.warn("result：\n" + result);
      }
      // parse app status json
      appAttempts = parseClusterApps(result);

      return appAttempts;
    } catch (Exception exp) {
      exp.printStackTrace();
    } finally {
      try {
        if (null != inputStream) {
          inputStream.close();
        }
      } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
      }
    }

    return appAttempts;
  }

  public Map parseClusterApps(String jsonContent) {
    Map appAttempts = new HashMap<>();

    try {
      JsonParser jsonParser = new JsonParser();
      JsonObject jsonObject = (JsonObject) jsonParser.parse(jsonContent);

      JsonObject jsonAppAttempts = jsonObject.get("app").getAsJsonObject();
      if (null == jsonAppAttempts) {
        return appAttempts;
      }
      for (Map.Entry entry : jsonAppAttempts.entrySet()) {
        String key = entry.getKey();
        if (null != entry.getValue() && entry.getValue() instanceof JsonPrimitive) {
          Object value = entry.getValue().getAsString();
          appAttempts.put(key, value);
        }
      }
    } catch (JsonIOException e) {
      LOGGER.error(e.getMessage(), e);
    } catch (JsonSyntaxException e) {
      LOGGER.error(e.getMessage(), e);
    }

    return appAttempts;
  }

  // http://yarn-web-http-address/ws/v1/cluster/apps/{appId}/appattempts
  // test/resources/ws-v1-cluster-apps-application_id-appattempts.json
  public List> getAppAttempts(String appId) {
    List> appAttempts = new ArrayList<>();
    String appUrl = this.yarnWebHttpAddr + "/ws/v1/cluster/apps/" + appId
        + "/appattempts?_=" + System.currentTimeMillis();

    InputStream inputStream = null;
    try {
      HttpResponse response = callRestUrl(appUrl, principal, HTTP.GET);
      inputStream = response.getEntity().getContent();
      String result = new BufferedReader(new InputStreamReader(inputStream))
          .lines().collect(Collectors.joining(System.lineSeparator()));
      if (response.getStatusLine().getStatusCode() != 200 /*success*/) {
        LOGGER.warn("Status code " + response.getStatusLine().getStatusCode());
        LOGGER.warn("message is :" + Arrays.deepToString(response.getAllHeaders()));
        LOGGER.warn("result：\n" + result);
      }

      // parse app status json
      appAttempts = parseAppAttempts(result);
    } catch (Exception exp) {
      exp.printStackTrace();
    } finally {
      try {
        if (null != inputStream) {
          inputStream.close();
        }
      } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
      }
    }

    return appAttempts;
  }

  // http://yarn-web-http-address/ws/v1/cluster/apps/{appId}/appattempts/{appAttemptId}/containers
  // test/resources/ws-v1-cluster-apps-application_id-appattempts-appattempt_id-containers.json
  public List> getAppAttemptsContainers(String appId, String appAttemptId) {
    List> appAttemptsContainers = new ArrayList<>();
    String appUrl = this.yarnWebHttpAddr + "/ws/v1/cluster/apps/" + appId
        + "/appattempts/" + appAttemptId + "/containers?_=" + System.currentTimeMillis();

    InputStream inputStream = null;
    try {
      HttpResponse response = callRestUrl(appUrl, principal, HTTP.GET);
      inputStream = response.getEntity().getContent();
      String result = new BufferedReader(new InputStreamReader(inputStream))
          .lines().collect(Collectors.joining(System.lineSeparator()));
      if (response.getStatusLine().getStatusCode() != 200 /*success*/) {
        LOGGER.warn("Status code " + response.getStatusLine().getStatusCode());
        LOGGER.warn("message is :" + Arrays.deepToString(response.getAllHeaders()));
        LOGGER.warn("result：\n" + result);
      }

      // parse app status json
      appAttemptsContainers = parseAppAttemptsContainers(result);
    } catch (Exception exp) {
      exp.printStackTrace();
    } finally {
      try {
        if (null != inputStream) {
          inputStream.close();
        }
      } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
      }
    }

    return appAttemptsContainers;
  }

  public List> getAppAttemptsContainersExportPorts(String appId) {
    List> listExportPorts = new ArrayList<>();

    // appId -> appAttemptId
    List> listAppAttempts = getAppAttempts(appId);
    for (Map mapAppAttempts : listAppAttempts) {
      if (mapAppAttempts.containsKey(YARN_REST_APPATTEMPTID)) {
        String appAttemptId = (String) mapAppAttempts.get(YARN_REST_APPATTEMPTID);
        List> exportPorts = getAppAttemptsContainers(appId, appAttemptId);
        if (exportPorts.size() > 0) {
          listExportPorts.addAll(exportPorts);
        }
      }
    }

    return listExportPorts;
  }

  // Kerberos authentication for simulated curling
  private static HttpClient buildSpengoHttpClient() {
    HttpClientBuilder builder = HttpClientBuilder.create();
    Lookup authSchemeRegistry
        = RegistryBuilder.create().register(
            AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build();
    builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
    BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() {
      @Override
      public Principal getUserPrincipal() {
        return null;
      }

      @Override
      public String getPassword() {
        return null;
      }
    });
    builder.setDefaultCredentialsProvider(credentialsProvider);

    // Avoid output WARN: Cookie rejected
    RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
        .build();
    builder.setDefaultRequestConfig(globalConfig);

    CloseableHttpClient httpClient = builder.build();

    return httpClient;
  }

  public HttpResponse callRestUrl(final String url, final String userId, HTTP operation) {
    if (LOGGER.isDebugEnabled()) {
      LOGGER.debug(String.format("Calling YarnClient %s %s %s",
          this.principal, this.keytab, url));
    }
    javax.security.auth.login.Configuration config = new javax.security.auth.login.Configuration() {
      @SuppressWarnings("serial")
      @Override
      public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
        return new AppConfigurationEntry[]{new AppConfigurationEntry(
            "com.sun.security.auth.module.Krb5LoginModule",
            AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
            new HashMap() {
              {
                put("useTicketCache", "false");
                put("useKeyTab", "true");
                put("keyTab", keytab);
                // Krb5 in GSS API needs to be refreshed so it does not throw the error
                // Specified version of key is not available
                put("refreshKrb5Config", "true");
                put("principal", principal);
                put("storeKey", "true");
                put("doNotPrompt", "true");
                put("isInitiator", "true");
                if (LOGGER.isDebugEnabled()) {
                  put("debug", "true");
                }
              }
            })};
      }
    };

    Set principals = new HashSet(1);
    principals.add(new KerberosPrincipal(userId));
    Subject sub = new Subject(false, principals, new HashSet(), new HashSet());
    try {
      // Authentication module: Krb5Login
      LoginContext loginContext = new LoginContext("Krb5Login", sub, null, config);
      loginContext.login();
      Subject serviceSubject = loginContext.getSubject();
      return Subject.doAs(serviceSubject, new PrivilegedAction() {
        HttpResponse httpResponse = null;

        @Override
        public HttpResponse run() {
          try {
            HttpUriRequest request = null;
            switch (operation) {
              case DELETE:
                request = new HttpDelete(url);
                break;
              case POST:
                request = new HttpPost(url);
                break;
              default:
                request = new HttpGet(url);
                break;
            }

            HttpClient spengoClient = buildSpengoHttpClient();
            httpResponse = spengoClient.execute(request);
            return httpResponse;
          } catch (IOException e) {
            LOGGER.error(e.getMessage(), e);
          }
          return httpResponse;
        }
      });
    } catch (Exception e) {
      LOGGER.error(e.getMessage(), e);
    }
    return null;
  }

  private Map parseAppServices(String appJson) {
    Map mapStatus = new HashMap<>();

    try {
      JsonParser jsonParser = new JsonParser();
      JsonObject jsonObject = (JsonObject) jsonParser.parse(appJson);

      JsonElement elementAppId = jsonObject.get("id");
      JsonElement elementAppState = jsonObject.get("state");
      JsonElement elementAppName = jsonObject.get("name");

      String appId = (elementAppId == null) ? "" : elementAppId.getAsString();
      String appState = (elementAppState == null) ? "" : elementAppState.getAsString();
      String appName = (elementAppName == null) ? "" : elementAppName.getAsString();

      if (!StringUtils.isEmpty(appId)) {
        mapStatus.put(SubmarineConstants.YARN_APPLICATION_ID, appId);
      }
      if (!StringUtils.isEmpty(appName)) {
        mapStatus.put(SubmarineConstants.YARN_APPLICATION_NAME, appName);
      }
      if (!StringUtils.isEmpty(appState)) {
        mapStatus.put(SubmarineConstants.YARN_APPLICATION_STATUS, appState);
      }
    } catch (JsonIOException e) {
      LOGGER.error(e.getMessage(), e);
    } catch (JsonSyntaxException e) {
      LOGGER.error(e.getMessage(), e);
    }

    return mapStatus;
  }

  // appJson format : submarine/src/test/resources/appAttempts.json
  public List> parseAppAttempts(String jsonContent) {
    List> appAttempts = new ArrayList<>();

    try {
      JsonParser jsonParser = new JsonParser();
      JsonObject jsonObject = (JsonObject) jsonParser.parse(jsonContent);

      JsonObject jsonAppAttempts = jsonObject.get(YARN_REST_APPATTEMPTS).getAsJsonObject();
      if (null == jsonAppAttempts) {
        return appAttempts;
      }
      JsonArray jsonAppAttempt = jsonAppAttempts.get(YARN_REST_APPATTEMPT).getAsJsonArray();
      if (null == jsonAppAttempt) {
        return appAttempts;
      }
      for (int i = 0; i < jsonAppAttempt.size(); i++) {
        Map mapAppAttempt = new HashMap<>();

        JsonObject jsonParagraph = jsonAppAttempt.get(i).getAsJsonObject();

        JsonElement jsonElement = jsonParagraph.get("id");
        String id = (jsonElement == null) ? "" : jsonElement.getAsString();
        mapAppAttempt.put("id", id);

        jsonElement = jsonParagraph.get(YARN_REST_APPATTEMPTID);
        String appAttemptId = (jsonElement == null) ? "" : jsonElement.getAsString();
        mapAppAttempt.put(YARN_REST_APPATTEMPTID, appAttemptId);

        appAttempts.add(mapAppAttempt);
      }
    } catch (JsonIOException e) {
      LOGGER.error(e.getMessage(), e);
    } catch (JsonSyntaxException e) {
      LOGGER.error(e.getMessage(), e);
    }

    return appAttempts;
  }

  // appJson format : submarine/src/test/resources/appAttempts.json
  public List> parseAppAttemptsContainers(String jsonContent) {
    List> appContainers = new ArrayList<>();

    try {
      JsonParser jsonParser = new JsonParser();
      JsonObject jsonObject = (JsonObject) jsonParser.parse(jsonContent);

      JsonArray jsonContainers = jsonObject.get(YARN_REST_CONTAINER).getAsJsonArray();
      for (int i = 0; i < jsonContainers.size(); i++) {
        String hostIp = "";

        JsonObject jsonContainer = jsonContainers.get(i).getAsJsonObject();

        JsonElement jsonElement = jsonContainer.get("nodeId");
        String nodeId = (jsonElement == null) ? "" : jsonElement.getAsString();
        String[] nodeIdParts = nodeId.split(":");
        if (nodeIdParts.length == 2) {
          hostIp = nodeIdParts[0];
        }

        jsonElement = jsonContainer.get("exposedPorts");
        String exposedPorts = (jsonElement == null) ? "" : jsonElement.getAsString();

        Gson gson = new Gson();
        Map>> listExposedPorts = gson.fromJson(exposedPorts,
            new TypeToken>>>() {
            }.getType());
        if (null == listExposedPorts) {
          continue;
        }
        for (Map.Entry>> entry : listExposedPorts.entrySet()) {
          String containerPort = entry.getKey();
          String[] containerPortParts = containerPort.split("/");
          if (containerPortParts.length == 2) {
            List> hostIps = entry.getValue();
            for (Map hostAttrib : hostIps) {
              Map containerExposedPort = new HashMap<>();
              String hostPort = hostAttrib.get("HostPort");
              containerExposedPort.put(HOST_IP, hostIp);
              containerExposedPort.put(HOST_PORT, hostPort);
              containerExposedPort.put(CONTAINER_PORT, containerPortParts[0]);
              appContainers.add(containerExposedPort);
            }
          }
        }
      }
    } catch (JsonIOException e) {
      LOGGER.error(e.getMessage(), e);
    } catch (JsonSyntaxException e) {
      LOGGER.error(e.getMessage(), e);
    }

    return appContainers;
  }

  public List> getAppExportPorts(String name) {
    // Query the IP and port of the submarine interpreter process through the yarn client
    Map mapAppStatus = getAppServices(name);
    if (mapAppStatus.containsKey(SubmarineConstants.YARN_APPLICATION_ID)
        && mapAppStatus.containsKey(SubmarineConstants.YARN_APPLICATION_NAME)
        && mapAppStatus.containsKey(SubmarineConstants.YARN_APPLICATION_STATUS)) {
      String appId = mapAppStatus.get(SubmarineConstants.YARN_APPLICATION_ID).toString();
      String appStatus = mapAppStatus.get(SubmarineConstants.YARN_APPLICATION_STATUS).toString();

      // if (StringUtils.equals(appStatus, SubmarineJob.YarnApplicationState.RUNNING.toString())) {
      List> mapAppAttempts = getAppAttemptsContainersExportPorts(appId);
      return mapAppAttempts;
      //}
    }

    return new ArrayList>() {
    };
  }

  public enum HTTP {
    GET,
    POST,
    DELETE;
  }
}
    
    
    
    

    




    
    
    © 2015 - 2024 Weber Informatics LLC | Privacy Policy