com.olapdb.obase.utils.Util Maven / Gradle / Ivy
The newest version!
package com.olapdb.obase.utils;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.olapdb.obase.data.Bytez;
import org.apache.commons.lang.time.FastDateFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.*;
import java.lang.reflect.Method;
import java.net.*;
import java.security.CodeSource;
import java.security.ProtectionDomain;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
public class Util {
private static ExecutorService executer = Executors.newFixedThreadPool(5);
public static ExecutorService threadPool(){
return executer;
}
private static long serialId = 0;
synchronized public static long getSerialLong(){
return ++serialId;
}
private final static Object lockInt = new Object();
private static int serialInt = 0;
public static int getSerialInt(){
synchronized(lockInt){
serialInt ++;
if(serialInt == Integer.MAX_VALUE)
serialInt = 0;
return serialInt;
}
}
public static long getLeftTime(){
return Long.MAX_VALUE - System.currentTimeMillis();
}
public static long getLeftTime(long time){
return Long.MAX_VALUE - time;
}
public static long reverse(long i) {
// HD, Figure 7-1
i = (i & 0x5555555555555555L) << 1 | (i >>> 1) & 0x5555555555555555L;
i = (i & 0x3333333333333333L) << 2 | (i >>> 2) & 0x3333333333333333L;
i = (i & 0x0f0f0f0f0f0f0f0fL) << 4 | (i >>> 4) & 0x0f0f0f0f0f0f0f0fL;
i = (i & 0x00ff00ff00ff00ffL) << 8 | (i >>> 8) & 0x00ff00ff00ff00ffL;
i = (i & 0x0000ffff0000ffffL) << 16 | (i >>> 16) & 0x0000ffff0000ffffL;
i = (i & 0x00000000ffffffffL) << 32 | (i >>> 32) & 0x00000000ffffffffL;
return i;
}
public static int reverse(int i) {
return (int)(reverse((long)i)>>32);
}
public static long reverseOld(long i) {
// HD, Figure 7-1
i = (i & 0x5555555555555555L) << 1 | (i >>> 1) & 0x5555555555555555L;
i = (i & 0x0f0f0f0f0f0f0f0fL) << 4 | (i >>> 4) & 0x0f0f0f0f0f0f0f0fL;
i = (i & 0x00ff00ff00ff00ffL) << 8 | (i >>> 8) & 0x00ff00ff00ff00ffL;
i = (i << 48) | ((i & 0xffff0000L) << 16) |
((i >>> 16) & 0xffff0000L) | (i >>> 48);
return i;
}
private static Map locks = new LruMap(10000);
private static Object getLock(String name){
if(!locks.containsKey(name)){
locks.put(name, name);
}
return locks.get(name);
}
public static int getDiscreteIntID(Table table, String counter){
counter = table.getName().getNameWithNamespaceInclAsString()+"|"+counter;
synchronized(getLock(counter)){
try{
Increment inc = new Increment(Bytes.toBytes(counter));
inc.addColumn(Bytes.toBytes(Obase.FAMILY_ATTR),Bytes.toBytes("discrete") ,1);
Result res = Obase.getIncrementTable().increment(inc);
List cells = res.listCells();
long uid = Bytes.toLong(CellUtil.cloneValue(cells.get(0)));
return (int)Util.reverse(uid<<32);
}catch(Exception e){
e.printStackTrace();
return 0;
}
}
}
public static long getDiscreteID(Table table, String counter){
counter = table.getName().getNameWithNamespaceInclAsString()+"|"+counter;
synchronized(getLock(counter)){
try{
Increment inc = new Increment(Bytes.toBytes(counter));
inc.addColumn(Bytes.toBytes(Obase.FAMILY_ATTR),Bytes.toBytes("discrete") ,1);
Result res = Obase.getIncrementTable().increment(inc);
List cells = res.listCells();
long uid = Bytes.toLong(CellUtil.cloneValue(cells.get(0)));
return Util.reverse(uid<<1);
}catch(Exception e){
e.printStackTrace();
return 0;
}
}
}
public static int parseInt(String text){
if(text == null)
return 0;
text = text.replace(",", "");
text = text.toLowerCase();
if(text.endsWith("k")){
return (int)(Double.parseDouble(text.substring(0, text.length()-1))*1000);
}
return Integer.parseInt(text);
}
public static void clearTableCounters(Table table){
try{
String prefix = table.getName().getNameWithNamespaceInclAsString()+"|";
Scan scan = new Scan();
Filter pf = new PrefixFilter(Bytez.from(prefix));
scan.setFilter(pf);
List deletes = new Vector();
ResultScanner rs = Obase.getIncrementTable().getScanner(scan);
for(Result r : rs){
deletes.add(new Delete(r.getRow()));
}
Obase.getIncrementTable().delete(deletes);
}catch(Exception e){
e.printStackTrace();
}
}
public static long[] getDiscreteIDs(Table table, String counter, int count){
counter = table.getName().getNameWithNamespaceInclAsString()+"|"+counter;
synchronized(getLock(counter)){
if(count <= 0)return new long[0];
try{
Increment inc = new Increment(Bytes.toBytes(counter));
inc.addColumn(Bytes.toBytes(Obase.FAMILY_ATTR),Bytes.toBytes("discrete") ,count);
Result res = Obase.getIncrementTable().increment(inc);
List cells = res.listCells();
long uid = Bytes.toLong(CellUtil.cloneValue(cells.get(0)));
long[] ids = new long[count];
for(int i=0;i cells = res.listCells();
long uid = Bytes.toLong(CellUtil.cloneValue(cells.get(0)));
return uid;
}catch(Exception e){
e.printStackTrace();
return 0;
}
}
}
public static long getOrderID(String counter){
try{
return Obase.getIncrementTable().incrementColumnValue(Bytez.from(counter),Bytes.toBytes(Obase.FAMILY_ATTR),Bytes.toBytes("order") ,1);
}catch(Exception e){
e.printStackTrace();
return 0;
}
}
public static long[] getOrderIDs(Table table, String counter, int count){
counter = table.getName().getNameWithNamespaceInclAsString()+"|"+counter;
synchronized(getLock(counter)){
if(count <= 0)return new long[0];
try{
Increment inc = new Increment(Bytes.toBytes(counter));
inc.addColumn(Bytes.toBytes(Obase.FAMILY_ATTR),Bytes.toBytes("order") ,count);
Result res = Obase.getIncrementTable().increment(inc);
List cells = res.listCells();
long uid = Bytes.toLong(CellUtil.cloneValue(cells.get(0)));
long[] ids = new long[count];
for(int i=0;i cells = r.listCells();
return CellUtil.cloneValue(cells.get(0));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return null;
}
}
public static byte[][] calcSplitKeys(int partition) {
byte[][] splitKeys = new byte[partition][];
for(int i=0; i 0){
baos.write(buffer, 0, len);
}
bis.close();
return baos.toByteArray();
}
public static byte[] getIndexRow(byte[] startRow){
return Bytes.add(startRow, Bytes.toBytes("i"));
}
public static byte[] invoke(Object o, byte[] bytes) throws Exception{
ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
ObjectInputStream ois = new ObjectInputStream(bais);
return invoke(o, ois);
}
public static byte[] invoke(Object o, ObjectInputStream ois) throws Exception{
String method = ois.readUTF();
int pnum = ois.readInt();
Class[] pts = new Class[pnum];
for(int i=0; i0) {
oos.writeObject(splitPoint);
}
oos.close();
Get get = new Get(startKey);
get.addColumn(Bytez.from(Obase.FAMILY_EXEC_REGION), baos.toByteArray());
Result r = Obase.getTable(entityClass).get(get);
if(r==null || r.isEmpty())
throw new Exception("exec region operation failed, maybe this table not implemented RegionOperationObserver");
boolean ret = false;
String cause = "";
List cells = r.listCells();
if(cells == null)return false;
for(Cell c : cells){
String column = Bytez.toString(CellUtil.cloneQualifier(c));
if("ret".equals(column)) {
ret = Bytez.toBoolean(CellUtil.cloneValue(c));
}
else if("cause".equals(column)){
cause = Bytez.toString(CellUtil.cloneValue(c));
}
}
if(ret)return true;
if(cause != null && cause.length() > 0){
throw new Exception(cause);
}
throw new Exception("exec region operation failed, unknown reson.");
}
public static boolean execRegionOperation(Class entityClass, byte[] startKey, String method, Map paras) throws Exception{
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(baos);
oos.writeUTF(method);
oos.writeObject(paras);
oos.close();
if(startKey==null || startKey.length == 0){
startKey = Bytez.from((byte)0);
}
Get get = new Get(startKey);
get.addColumn(Bytez.from(Obase.FAMILY_EXEC_REGION), baos.toByteArray());
Result r = Obase.getTable(entityClass).get(get);
if(r==null || r.isEmpty())
throw new Exception("exec region operation failed, maybe this table not implemented RegionOperationObserver");
boolean ret = false;
String cause = "";
List cells = r.listCells();
if(cells == null)return false;
for(Cell c : cells){
String column = Bytez.toString(CellUtil.cloneQualifier(c));
if("ret".equals(column)) {
ret = Bytez.toBoolean(CellUtil.cloneValue(c));
}
else if("cause".equals(column)){
cause = Bytez.toString(CellUtil.cloneValue(c));
}
}
if(ret)return true;
if(cause != null && cause.length() > 0){
throw new Exception(cause);
}
throw new Exception("exec region operation failed, unknown reson.");
}
public static boolean checkEmail(String email){
if (!email.matches("[\\w\\.\\-]+@([\\w\\-]+\\.)+[\\w\\-]+")) {
return false;
}
return true;
}
public static List searchList(String request){
List rets = new Vector();
if(request == null)return rets;
String[] segs = request.split(" ");
Hashtable maps = new Hashtable();
for(String w:segs){
if(w == null || w.equals(""))
continue;
maps.put(w, 1);
}
rets.addAll(maps.keySet());
return rets;
}
public static List searchListSmart(String request){
List rets = new Vector();
if(request == null)return rets;
Hashtable maps = new Hashtable();
try {
maps = Fenci.text2words(request);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
rets.addAll(maps.keySet());
return rets;
}
public final static int TEXT_CHINESE = 1;
public final static int TEXT_ENGLISH = 2;
public final static int TEXT_MIXTURE = 3;
public final static int textStyle(String text){
if(text == null || text.equals(""))
return TEXT_MIXTURE;
int chineseCount = 0;
int spaceCount = 0;
for(char c : text.toCharArray()){
if(c == ' ')
spaceCount++;
else if(isChinese(c))
chineseCount++;
}
if(chineseCount==0)
return TEXT_ENGLISH;
if(chineseCount+spaceCount == text.length())
return TEXT_CHINESE;
return TEXT_MIXTURE;
}
public final static boolean isChineseString(String text){
return textStyle(text) != TEXT_ENGLISH;
}
private static boolean isChinese(char c) {
boolean result = false;
if (c >= 19968 && c <= 171941) {
result = true;
}
return result;
}
public static String readTxtFile(String filePath){
try{
File file=new File(filePath);
if(file.isFile() && file.exists()){
InputStreamReader read = new InputStreamReader(
new FileInputStream(file));
BufferedReader bufferedReader = new BufferedReader(read);
StringBuffer sb = new StringBuffer();
String lineTxt = null;
while((lineTxt = bufferedReader.readLine()) != null){
sb.append(lineTxt);
}
read.close();
return sb.toString();
}
}catch(Exception e){
e.printStackTrace();
}
return "";
}
public static void saveTxtFile(String filePath, String text){
try{
File file=new File(filePath);
OutputStreamWriter writer = new OutputStreamWriter(
new FileOutputStream(file));
BufferedWriter bufferedWriter = new BufferedWriter(writer);
bufferedWriter.write(text);
bufferedWriter.close();
}catch(Exception e){
e.printStackTrace();
}
}
public static String pureString(String text){
if(text == null)
return null;
text = text.trim();
text = text.replace("\t", " ");
for(int i=0; i<100; i++){
text = text.replace(" ", " ");
}
return text;
}
public static String requestPage(String url){
StringBuffer buffer = new StringBuffer();
try {
URL newUrl = new URL(url);
System.setProperty("sun.net.client.defaultConnectTimeout", "60000");
System.setProperty("sun.net.client.defaultReadTimeout", "60000");
HttpURLConnection hConnect = (HttpURLConnection) newUrl.openConnection();
hConnect.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)");
hConnect.setConnectTimeout(60000);
hConnect.setRequestMethod("GET");
hConnect.connect();
BufferedReader rd = new BufferedReader(new InputStreamReader(
hConnect.getInputStream()));
int ch;
while ((ch = rd.read()) > -1)
buffer.append((char) ch);
String s = buffer.toString();
rd.close();
hConnect.disconnect();
return s;
} catch (Exception e) {
e.printStackTrace();
return "";
}
}
public static String timeToStr(long time){
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
return sdf.format(new Date(time));
}
public static boolean isNumber(String str)
{
java.util.regex.Pattern pattern=java.util.regex.Pattern.compile("[0-9]*");
java.util.regex.Matcher match=pattern.matcher(str);
return match.matches();
}
public static long scanTableRowCounts(Table table) throws IOException{
Scan scan = new Scan();
Filter kof = new FirstKeyOnlyFilter();
scan.setFilter(kof);
ResultScanner rs = table.getScanner(scan);
long count = 0;
for(Result r: rs){
if(r == null)break;
count++;
}
return count;
}
// public static String apacheGet(String url){
// HttpClient client = new HttpClient();
// String response = null;
// String keyword = null;
// GetMethod postMethod = new GetMethod(url);
//
// try {
// client.executeMethod(postMethod);
// response = new String(postMethod.getResponseBodyAsString()
// .getBytes("ISO-8859-1"));
// } catch (Exception e) {
//
// e.printStackTrace();
// }
// return response;
// }
public static String getDomain(String url){
if(url == null || url.length() == 0)
return null;
url = url.toLowerCase();
if("null".equals(url))
return null;
try {
URL hurl = new URL(url);
return hurl.getHost();
} catch (MalformedURLException e) {
e.printStackTrace();
return null;
}
}
public static String where(String className){
try {
Class theClazz = Class.forName(className);
return where(theClazz);
} catch (ClassNotFoundException e) {
return "CLASS_NOT_FOUND:"+className;
}
}
public static String where(@SuppressWarnings("rawtypes") final Class cls) {
if (cls == null)throw new IllegalArgumentException("null input: cls");
URL result = null;
final String clsAsResource = cls.getName().replace('.', '/').concat(".class");
final ProtectionDomain pd = cls.getProtectionDomain();
if (pd != null) {
final CodeSource cs = pd.getCodeSource();
if (cs != null) result = cs.getLocation();
if (result != null) {
if ("file".equals(result.getProtocol())) {
try {
if (result.toExternalForm().endsWith(".jar") ||
result.toExternalForm().endsWith(".zip"))
result = new URL("jar:".concat(result.toExternalForm())
.concat("!/").concat(clsAsResource));
else if (new File(result.getFile()).isDirectory())
result = new URL(result, clsAsResource);
}
catch (MalformedURLException ignore) {}
}
}
}
if (result == null) {
final ClassLoader clsLoader = cls.getClassLoader();
result = clsLoader != null ?
clsLoader.getResource(clsAsResource) :
ClassLoader.getSystemResource(clsAsResource);
}
return result.toString();
}
public static void clearNameSpace(String server, String namespace) throws Exception{
Configuration conf = new Configuration();
conf.set("hbase.zookeeper.quorum", server);
Connection conn = ConnectionFactory.createConnection(conf);
Admin admin = conn.getAdmin();
for(TableName tn : admin.listTableNames()){
// System.out.println(tn.getNamespaceAsString());
if(tn.getNamespaceAsString().equals(namespace)){
if(admin.tableExists(tn)) {
if(admin.isTableEnabled(tn)){
admin.disableTable(tn);
}
admin.deleteTable(tn);
}
}
}
}
public static String combineSearchTag(String destination, String words){
if(destination != null)destination = destination.toLowerCase();
if(words != null)words = words.toLowerCase();
String tag = "";
if(destination != null){
tag = "destination:"+destination;
}
tag+="|language:en";
if(words == null)return tag;
// List sws = Util.searchList(words);
List sws = Util.searchListSmart(words);
Collections.sort(sws);
for(String word : sws){
if(tag.length() > 0){
tag += "|";
}
tag += "word:"+word;
}
return tag;
}
public static boolean isEmpty(String text){
return (text== null || text.length()==0);
}
private static Gson _gson = null;
public static Gson gson(){
if(_gson == null){
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.disableHtmlEscaping();
// _gson = gsonBuilder.setPrettyPrinting().create();
_gson = gsonBuilder.create();
}
return _gson;
}
private static Gson _gsonPretty = null;
public static Gson gsonPretty(){
if(_gsonPretty == null){
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.disableHtmlEscaping();
gsonBuilder.setPrettyPrinting();
_gsonPretty = gsonBuilder.setPrettyPrinting().create();
}
return _gsonPretty;
}
public static java.sql.Date pastDate(){
java.sql.Date nowTime = new java.sql.Date(System.currentTimeMillis());
return java.sql.Date.valueOf(nowTime+"");
}
public static java.sql.Date pastDate(long time){
java.sql.Date nowTime = new java.sql.Date(time);
return java.sql.Date.valueOf(nowTime+"");
}
/**
* nano time time scope < 2262
* Obase.nanoTime()/1000000 = System.currentTimeMillis()
* from1970-01-01 00:00:00 calc nano seconds
*/
private static long atomicMillis = 0;
private static long atomicNanos = 0;
synchronized public static long nanoTime(){
long millis = System.currentTimeMillis();
if(atomicMillis == millis){
atomicNanos += 1;
}else{
atomicMillis = millis;
atomicNanos = (long)(Math.random()*100000);
}
return millis*1000000 + atomicNanos;
}
public static long millis2Nano(long millis){
return millis*1000000;
}
public static long nano2millis(long nanos){
return nanos/1000000;
}
private static byte[] localAddress;
public static byte[] getLocalIPV4(){
if(localAddress == null){
try {
localAddress = InetAddress.getLocalHost().getAddress();
}catch (Exception e){
localAddress = Bytez.from((int)(Math.random()*Integer.MAX_VALUE));
}
}
return localAddress;
}
private static FastDateFormat fastDateFormat = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss");
public static String formatTimeString(long timeMilliSeconds){
return fastDateFormat.format(new Date((timeMilliSeconds)));
}
public static Stream> convertToCoarseStream(Stream fineStream, int batchSize) {
Spliterator spliterator = fineStream.spliterator();
int characteristics = spliterator.characteristics();
long estimatedSize = spliterator.estimateSize();
return StreamSupport.stream(new Spliterators.AbstractSpliterator>(estimatedSize, characteristics) {
List buffer = new ArrayList<>(batchSize);
@Override
public boolean tryAdvance(Consumer> action) {
boolean hasMore = spliterator.tryAdvance(buffer::add);
if (!hasMore || buffer.size() == batchSize) {
action.accept(new ArrayList<>(buffer));
buffer.clear();
}
return hasMore;
}
@Override
public void forEachRemaining(Consumer> action) {
spliterator.forEachRemaining(e -> {
buffer.add(e);
if (buffer.size() == batchSize) {
action.accept(new ArrayList<>(buffer));
buffer.clear();
}
});
if (!buffer.isEmpty()) {
action.accept(new ArrayList<>(buffer));
buffer.clear();
}
}
}, false);
}
}
| | | | | | © 2015 - 2025 Weber Informatics LLC | Privacy Policy