com.nflabs.zeppelin.zengine.stmt.L Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of zeppelin-zengine Show documentation
Show all versions of zeppelin-zengine Show documentation
Zengine is java framework for data analysis on Hadoop. see http://nflabs.github.io/zeppelin/#/zengine
package com.nflabs.zeppelin.zengine.stmt;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.nflabs.zeppelin.conf.ZeppelinConfiguration;
import com.nflabs.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import com.nflabs.zeppelin.driver.ZeppelinConnection;
import com.nflabs.zeppelin.driver.ZeppelinDriver;
import com.nflabs.zeppelin.zengine.ParamInfo;
import com.nflabs.zeppelin.zengine.ZContext;
import com.nflabs.zeppelin.zengine.ZException;
import com.nflabs.zeppelin.zengine.ZWebContext;
import com.nflabs.zeppelin.zengine.Zengine;
/**
* L stands for Library(aks UDF). This class load and execute Zeppelin User Defined Functions
* @author moon
*
*/
public class L extends Q {
private String libName;
transient private URI libUri;
transient private File dir;
transient private File webDir;
transient private File erbFile;
transient private boolean initialized = false;
/**
* @param libName library name to load and run
* @throws ZException
*/
public L(String libName) throws ZException{
this(libName, null);
}
/**
* @param libName library name to laod and run
* @param arg
* @param currentDriver
* @throws ZException
*/
public L(String libName, String arg) throws ZException{
super(arg);
this.libName = libName;
initialize();
}
private Logger logger(){
return LoggerFactory.getLogger(L.class);
}
protected void initialize() throws ZException{
super.initialize();
if(initialized==true){
return ;
}
if(libName==null) return;
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
String libUri = conf.getString(ConfVars.ZEPPELIN_ZAN_LOCAL_REPO)+"/"+libName;
// search for library dir
this.dir = new File(libUri);
if (dir.exists()==false || dir.isDirectory()==false) {
throw new ZException("Directory "+dir.getAbsolutePath()+" does not exist");
}
this.erbFile = new File(libUri.toString()+"/zql.erb");
if(erbFile.isFile() == false){
erbFile = null;
}
this.webDir = new File(libUri+"/web");
initialized = true;
}
/**
* Get query to be executed.
* Each library has query template file. and returns evaluated value.
*
* @return query to be executed
*/
@Override
public String getQuery() throws ZException {
initialize();
if(erbFile==null){
return null;
}
String q;
try {
FileInputStream ins = new FileInputStream(erbFile);
BufferedReader erb = new BufferedReader(new InputStreamReader(ins));
ZContext zContext = new ZContext( (prev()==null) ? null : prev().name(), name(), query, params);
q = getQuery(erb, zContext);
ins.close();
} catch (IOException e1) {
throw new ZException(e1);
}
return q;
}
/**
* access web resources
*
*
* @param path related path from this libraries 'web' dir
* @return
* @throws ZException
*/
public InputStream readWebResource(String path) throws ZException{
initialize();
if(path==null || path.compareTo("/")==0){
path = "index.erb";
}
try {
if (webDir.exists()==false || webDir.isDirectory()==false ) {
return super.readWebResource(path);
}
File resourcePath = new File(webDir, path);
if (resourcePath.isFile()==false) {
return null;
}
if(resourcePath.getName().endsWith(".erb")){
String q;
try {
ZWebContext zWebContext = null;
try{
zWebContext = new ZWebContext(params, result());
} catch(ZException e){
//TODO explain
}
FileInputStream ins = new FileInputStream(resourcePath);
BufferedReader erb = new BufferedReader(new InputStreamReader(ins));
q = evalWebTemplate(erb, zWebContext);
ins.close();
return new ByteArrayInputStream(q.getBytes());
} catch (IOException e1) {
throw new ZException(e1);
}
} else {
return new FileInputStream(resourcePath);
}
} catch (IOException e) {
throw new ZException(e);
}
}
/**
* Check if this library has web capabilities.
*
*/
@Override
public boolean isWebEnabled(){
File resourcePath = new File(webDir, "index.erb");
if(resourcePath.isFile()==false){
return super.isWebEnabled();
} else {
return true;
}
}
/**
* Get resources to be required to run this library
*/
@Override
public List getResources() throws ZException {
List resources = new LinkedList();
File[] files = dir.listFiles();
if(files!=null){
for(File f : files){
if(f.isFile()==false){
continue;
} else if(f.getName().startsWith(".")){ // ignore hidden file
continue;
} else if(f.getName().startsWith(libName+"_")==false){
continue;
} else if(f.getName().equals("zql.erb")){
continue;
} else {
resources.add(f.toURI());
}
}
}
resources.addAll(super.getResources());
return resources;
}
@Override
public String getReleaseQuery() throws ZException {
return super.getReleaseQuery();
}
@Override
public Z pipe(Z z){
if(erbFile==null){
if(prev()!=null){
withName(z.name());
} else {
assert false; // should never reach here
}
setNext(z);
z.setPrev(this);
return z;
} else {
return super.pipe(z);
}
}
@Override
protected Map extractParams() throws ZException {
initialize();
Map paramInfos = new HashMap();
ZContext zContext = null;
if(erbFile!=null){
InputStream ins;
try {
ins = new FileInputStream(erbFile);
BufferedReader erb = new BufferedReader(new InputStreamReader(ins));
zContext = new ZContext( (prev()==null) ? null : prev().name(), name(), query, params);
getQuery(erb, zContext);
ins.close();
} catch (IOException e1) {
logger().debug("dry run error", e1);
} catch (ZException e) {
logger().debug("dry run error", e);
}
}
ZWebContext zWebContext = null;
try {
File resourcePath = new File(webDir, "index.erb");
if(resourcePath.isFile()==true){
zWebContext = new ZWebContext(params, null);
FileInputStream ins = new FileInputStream(resourcePath);
BufferedReader erb = new BufferedReader(new InputStreamReader(ins));
evalWebTemplate(erb, zWebContext);
ins.close();
}
} catch (IOException e) {
logger().debug("dry run error", e);
} catch (ZException e) {
logger().debug("dry run error", e);
}
if(zWebContext!=null){
paramInfos.putAll(zWebContext.getParamInfos());
}
if(zContext!=null){
paramInfos.putAll(zContext.getParamInfos());
}
return paramInfos;
}
}