• Home
• org.semoss
• semoss
• 4.3.1
• source code
• Utility.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

prerna.util.Utility Maven / Gradle / Ivy

/*******************************************************************************
 * Copyright 2015 Defense Health Agency (DHA)
 *
 * If your use of this software does not include any GPLv2 components:
 * 	Licensed under the Apache License, Version 2.0 (the "License");
 * 	you may not use this file except in compliance with the License.
 * 	You may obtain a copy of the License at
 *
 * 	  http://www.apache.org/licenses/LICENSE-2.0
 *
 * 	Unless required by applicable law or agreed to in writing, software
 * 	distributed under the License is distributed on an "AS IS" BASIS,
 * 	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * 	See the License for the specific language governing permissions and
 * 	limitations under the License.
 * ----------------------------------------------------------------------------
 * If your use of this software includes any GPLv2 components:
 * 	This program is free software; you can redistribute it and/or
 * 	modify it under the terms of the GNU General Public License
 * 	as published by the Free Software Foundation; either version 2
 * 	of the License, or (at your option) any later version.
 *
 * 	This program is distributed in the hope that it will be useful,
 * 	but WITHOUT ANY WARRANTY; without even the implied warranty of
 * 	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * 	GNU General Public License for more details.
 *******************************************************************************/
package prerna.util;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.InvocationTargetException;
import java.math.BigDecimal;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
import java.net.URLConnection;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.text.DateFormat;
import java.text.DecimalFormat;
import java.text.Normalizer;
import java.text.Normalizer.Form;
import java.text.NumberFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.DayOfWeek;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeMap;
import java.util.Vector;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import javax.swing.JFrame;
import javax.swing.JList;
import javax.swing.JOptionPane;
import javax.tools.Diagnostic;
import javax.tools.DiagnosticCollector;
import javax.tools.JavaCompiler;
import javax.tools.JavaFileObject;
import javax.tools.StandardJavaFileManager;
import javax.tools.ToolProvider;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.SystemUtils;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLContextBuilder;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.poi.ss.usermodel.Workbook;
import org.json.JSONArray;
import org.json.JSONObject;
import org.owasp.encoder.Encode;
import org.owasp.esapi.ESAPI;
import org.owasp.esapi.codecs.MySQLCodec;
import org.owasp.html.PolicyFactory;
import org.owasp.html.Sanitizers;
import org.quartz.CronExpression;
import org.xeustechnologies.jcl.JarClassLoader;
import org.xeustechnologies.jcl.JclObjectFactory;

import com.google.common.base.Strings;
import com.google.common.net.InternetDomainName;
import com.google.gson.GsonBuilder;

import io.github.classgraph.ClassGraph;
import io.github.classgraph.ClassInfo;
import io.github.classgraph.ClassInfoList;
import io.github.classgraph.ScanResult;
import javassist.CannotCompileException;
import javassist.ClassPool;
import javassist.CtClass;
import javassist.NotFoundException;
import prerna.algorithm.api.SemossDataType;
import prerna.auth.User;
import prerna.auth.utils.SecurityEngineUtils;
import prerna.auth.utils.SecurityProjectUtils;
import prerna.cluster.util.ClusterUtil;
import prerna.cluster.util.ZKClient;
import prerna.date.SemossDate;
import prerna.engine.api.IDatabaseEngine;
import prerna.engine.api.IEngine;
import prerna.engine.api.IFunctionEngine;
import prerna.engine.api.IHeadersDataRow;
import prerna.engine.api.IModelEngine;
import prerna.engine.api.IRawSelectWrapper;
import prerna.engine.api.IReactorFunctionEngine;
import prerna.engine.api.ISelectStatement;
import prerna.engine.api.ISelectWrapper;
import prerna.engine.api.IStorageEngine;
import prerna.engine.api.IVectorDatabaseEngine;
import prerna.engine.api.IVenvEngine;
import prerna.engine.impl.CaseInsensitiveProperties;
import prerna.engine.impl.SmssUtilities;
import prerna.masterdatabase.AddToMasterDB;
import prerna.masterdatabase.DeleteFromMasterDB;
import prerna.masterdatabase.utility.MasterDatabaseUtility;
import prerna.om.IStringExportProcessor;
import prerna.project.api.IProject;
import prerna.rdf.engine.wrappers.WrapperManager;
import prerna.reactor.AbstractReactor;
import prerna.reactor.IReactor;
import prerna.reactor.frame.AbstractFrameReactor;
import prerna.reactor.frame.py.AbstractPyFrameReactor;
import prerna.reactor.frame.r.AbstractRFrameReactor;
import prerna.sablecc2.om.task.ITask;
import prerna.sablecc2.om.task.TaskUtility;
import prerna.tcp.PayloadStruct;
import prerna.tcp.SocketServerHandler;
import prerna.tcp.workers.EngineSocketWrapper;
import prerna.ui.components.api.IPlaySheet;
import prerna.ui.components.playsheets.datamakers.IDataMaker;
import prerna.ui.components.playsheets.datamakers.ISEMOSSAction;
import prerna.ui.components.playsheets.datamakers.ISEMOSSTransformation;
import prerna.util.git.GitAssetUtils;

/**
 * The Utility class contains a variety of miscellaneous functions implemented
 * extensively throughout SEMOSS. Some of these functionalities include getting
 * concept names, printing messages, loading engines, and writing Excel
 * workbooks.
 */
public final class Utility {

	public static int id = 0;
	private static final Logger classLogger = LogManager.getLogger(Utility.class);
	private static final String SPECIFIED_PATTERN = "[@]{1}\\w+[-]*[\\w/.:]+[@]";
	
	/**
	 * Matches the given query against a specified pattern. While the next substring
	 * of the query matches a part of the pattern, set substring as the key with
	 * EMPTY constants (@@) as the value
	 * 
	 * @param Query.
	 * 
	 * @return Hashtable of queries to be replaced
	 */
	public static Hashtable getParams(String query) {
		Hashtable paramHash = new Hashtable();
		Pattern pattern = Pattern.compile(SPECIFIED_PATTERN);

		Matcher matcher = pattern.matcher(query);
		while (matcher.find()) {
			String data = matcher.group();
			data = data.substring(1, data.length() - 1);
			classLogger.debug(data);
			// put something to strip the @
			paramHash.put(data, Constants.EMPTY);
		}

		return paramHash;
	}
	
	/**
	 * This is to remove scripts from being passed
	 * 
	 * @param stringToNormalize
	 * @return
	 */
	public static String inputSanitizer(String stringToNormalize) {
		if (stringToNormalize == null) {
			classLogger.debug("input to sanitzer is null, returning null");
			return stringToNormalize;
		}

		PolicyFactory policy = Sanitizers.FORMATTING.and(Sanitizers.LINKS).and(Sanitizers.BLOCKS).and(Sanitizers.STYLES)
				.and(Sanitizers.IMAGES).and(Sanitizers.TABLES);
		MySQLCodec mySQLCodec = new MySQLCodec(MySQLCodec.Mode.ANSI);
		return ESAPI.encoder().encodeForSQL(mySQLCodec, policy.sanitize(stringToNormalize));
	}


	/**
	 * This is to remove sql injection from strings
	 * 
	 * @param stringToNormalize
	 * @return
	 */
	public static String inputSQLSanitizer(String stringToNormalize) {
		if (stringToNormalize == null) {
			classLogger.debug("input to sanitzer is null, returning null");
			return stringToNormalize;
		}

		MySQLCodec mySQLCodec = new MySQLCodec(MySQLCodec.Mode.ANSI);
		return ESAPI.encoder().encodeForSQL(mySQLCodec, (stringToNormalize));
	}
	
	/**
	 * Matches the given query against a specified pattern. While the next substring
	 * of the query matches a part of the pattern, set substring as the key with
	 * EMPTY constants (@@) as the value
	 * 
	 * @param Query.
	 * 
	 * @return Hashtable of queries to be replaced
	 * @return JSON format of vector 
	 * 
	 */
	public static Hashtable getParamTypeHash(String query) {
		Hashtable paramHash = new Hashtable();
		Pattern pattern = Pattern.compile(SPECIFIED_PATTERN);

		Matcher matcher = pattern.matcher(query);
		while (matcher.find()) {
			String data = matcher.group();
			data = data.substring(1, data.length() - 1);
			String paramName = data.substring(0, data.indexOf("-"));
			String paramValue = data.substring(data.indexOf("-") + 1);

			classLogger.debug(data);
			// put something to strip the @
			paramHash.put(paramName, paramValue);
		}
		
		return paramHash;
	}  
	
	/**
	 * Get the Base Folder
	 * @return
	 */
	public static String getBaseFolder() {
		return getDIHelperProperty(Constants.BASE_FOLDER);
	}
	
	/**
	 * 
	 * @return
	 */
	public static String getInsightCacheDir() {
		return getDIHelperProperty(Constants.INSIGHT_CACHE_DIR);
	}
	
	/**
	 * 
	 * @return
	 */
	public static String getCsvInsightCacheDir() {
		return getDIHelperProperty(Constants.CSV_INSIGHT_CACHE_FOLDER);
	}
	
	/**
	 * Get any property from DIHelper
	 * @param prop
	 * @return
	 */
	public static String getDIHelperProperty(String prop) {
		if(DIHelper.getInstance() == null) {
			return null;
		}
		return DIHelper.getInstance().getProperty(prop);
	}
	
	public static Object getDIHelperLocalProperty(String prop) {
		if(DIHelper.getInstance() == null) {
			return null;
		}
		return DIHelper.getInstance().getLocalProp(prop);
	}

	/**
	 * 
	 * @param s
	 * @return
	 */
	public static String unescapeHTML(String s) {
		s = s.replace(">", ">");
		s = s.replace("<", "<");
		s = s.replace("=", "=");
		s = s.replace("!", "!");
		return s;
	}

	/**
	 * Matches the given query against a specified pattern. While the next substring
	 * of the query matches a part of the pattern, set substring as the key with
	 * EMPTY constants (@@) as the value
	 * 
	 * @param Query.
	 * 
	 * @return Hashtable of queries to be replaced
	 */
	public static String normalizeParam(String query) {
		Map> paramHash = new Hashtable<>();
		Pattern pattern = Pattern.compile(SPECIFIED_PATTERN);

		Matcher matcher = pattern.matcher(query);
		while (matcher.find()) {
			String data = matcher.group();
			data = data.substring(1, data.length() - 1);
			if (data.contains("-")) {
				String paramName = data.substring(0, data.indexOf('-'));

				classLogger.debug(data);
				// put something to strip the @
				List