• Home
• com.scudata.esproc
• esproc
• 20240731
• source code
• CursorUtil.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.scudata.util.CursorUtil Maven / Gradle / Ivy

package com.scudata.util;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.Iterator;
import java.util.TreeMap;

import com.scudata.array.IArray;
import com.scudata.common.IntArrayList;
import com.scudata.common.Logger;
import com.scudata.common.MessageManager;
import com.scudata.common.RQException;
import com.scudata.dm.BFileReader;
import com.scudata.dm.BFileWriter;
import com.scudata.dm.BaseRecord;
import com.scudata.dm.ComputeStack;
import com.scudata.dm.Context;
import com.scudata.dm.Current;
import com.scudata.dm.DataStruct;
import com.scudata.dm.Env;
import com.scudata.dm.FileObject;
import com.scudata.dm.HashArraySet;
import com.scudata.dm.IndexTable;
import com.scudata.dm.ListBase1;
import com.scudata.dm.Record;
import com.scudata.dm.Sequence;
import com.scudata.dm.Table;
import com.scudata.dm.comparator.*;
import com.scudata.dm.cursor.*;
import com.scudata.dm.op.DiffJoin;
import com.scudata.dm.op.FilterJoin;
import com.scudata.dm.op.IGroupsResult;
import com.scudata.dm.op.Join;
import com.scudata.dm.op.Operation;
import com.scudata.dw.ColPhyTable;
import com.scudata.dw.IDWCursor;
import com.scudata.dw.PhyTable;
import com.scudata.expression.CurrentSeq;
import com.scudata.expression.Expression;
import com.scudata.expression.Node;
import com.scudata.expression.fn.gather.ICount.ICountBitSet;
import com.scudata.expression.fn.gather.ICount.ICountPositionSet;
import com.scudata.parallel.ClusterCursor;
import com.scudata.resources.EngineMessage;
import com.scudata.thread.GroupsJob;
import com.scudata.thread.GroupsJob2;
import com.scudata.thread.GroupxJob;
import com.scudata.thread.MultithreadUtil;
import com.scudata.thread.ThreadPool;

public final class CursorUtil {
	/**
	 * ȡ?α?????ݽṹ
	 * @param cs
	 * @return
	 */
	public static DataStruct getDataStruct(ICursor cs) {
		DataStruct ds = cs.getDataStruct();
		if (ds == null) {
			Sequence seq = cs.peek(ICursor.FETCHCOUNT_M);
			ds = cs.getDataStruct();
			if (ds == null && seq != null) {
				ds = seq.dataStruct();
			}
		}
		
		return ds;
	}

	/**
	 * ?????н??в??з???
	 * @param src ????
	 * @param exps ?????ֶα???ʽ????
	 * @param names ?????ֶ???????
	 * @param calcExps ?????ֶα???ʽ????
	 * @param calcNames ?????ֶ???????
	 * @param opt ѡ??
	 * @param ctx ??????????
	 * @returns
	 */
	public static Table groups_m(Sequence src, Expression[] exps, String[] names,
			Expression[] calcExps, String[] calcNames, String opt, Context ctx) {
		int len = src.length();
		int parallelNum = Env.getParallelNum();
		if (len <= MultithreadUtil.SINGLE_PROSS_COUNT || parallelNum < 2) {
			return src.groups(exps, names, calcExps, calcNames, opt, ctx);
		}
		
		int threadCount = (len - 1) / MultithreadUtil.SINGLE_PROSS_COUNT + 1;
		if (threadCount > parallelNum) {
			threadCount = parallelNum;
		}
		
		int singleCount = len / threadCount;
		int keyCount = exps == null ? 0 : exps.length;
		
		// ???ɷ????????ύ???̳߳?
		ThreadPool pool = ThreadPool.instance();
		GroupsJob []jobs = new GroupsJob[threadCount];
		
		int start = 1;
		int end; // ??????
		for (int i = 0; i < threadCount; ++i) {
			if (i + 1 == threadCount) {
				end = len + 1;
			} else {
				end = start + singleCount;
			}

			Context tmpCtx = ctx.newComputeContext();
			Expression []tmpExps = Operation.dupExpressions(exps, tmpCtx);
			Expression []tmpCalcExps = Operation.dupExpressions(calcExps, tmpCtx);
			
			Sequence seq = src.get(start, end);
			jobs[i] = new GroupsJob(seq, tmpExps, names, tmpCalcExps, calcNames, opt, tmpCtx);
			pool.submit(jobs[i]);
			start = end;
		}
		
		// ?ȴ?????????ִ????ϣ????ѽ?????ӵ?һ?????
		Table result = null;
		IGroupsResult groupsResult = null;
		
		for (int i = 0; i < threadCount; ++i) {
			jobs[i].join();
			groupsResult = jobs[i].getGroupsResult();
			
			if (result == null) {
				result =  groupsResult.getTempResult();
			} else {
				result.addAll(groupsResult.getTempResult());
			}
		}
		
		if (result == null || result.length() == 0) {
			return result;
		}
		
		// ???ɶ??η?????????ʽ
		Expression []keyExps = null;
		if (keyCount > 0) {
			keyExps = new Expression[keyCount];
			for (int i = 0, q = 1; i < keyCount; ++i, ++q) {
				keyExps[i] = new Expression(ctx, "#" + q);
			}
		}

		// ???ɶ??η?????ܱ???ʽ??avg???ܱ??ֳ?sum??count???н??м???
		Expression []valExps = groupsResult.getRegatherExpressions();
		DataStruct tempDs = groupsResult.getRegatherDataStruct();
		int tempFieldCount = tempDs.getFieldCount();
		
		if (keyCount > 0) {
			if (names == null) {
				names = new String[keyCount];
			}
			
			for (int i = 0; i < keyCount; ++i) {
				names[i] = tempDs.getFieldName(i);
			}
		}

		if (tempFieldCount > keyCount) {
			int gatherCount = tempFieldCount - keyCount;
			calcNames = new String[gatherCount];
			for (int i = 0; i < gatherCount; ++i) {
				calcNames[i] = tempDs.getFieldName(keyCount + i);
			}
		}
		
		// ???ж??η???
		result = result.groups(keyExps, names, valExps, calcNames, opt, ctx);
		Expression []newExps = groupsResult.getResultExpressions();
		if (newExps != null) {
			return result.newTable(groupsResult.getResultDataStruct(), newExps, null, ctx);
		} else {
			return result;
		}
	}
	/**
	 * ?????н??в??з???
	 * @param src ????
	 * @param exps ?????ֶα???ʽ????
	 * @param names ?????ֶ???????
	 * @param calcExps ?????ֶα???ʽ????
	 * @param calcNames ?????ֶ???????
	 * @param opt ѡ??
	 * @param ctx ??????????
	 * @param hashCapacity ??ϣ??????
	 * @returns
	 */
	public static Table groups_z(Sequence src, Expression[] exps, String[] names, Expression[] calcExps,
			String[] calcNames, String opt, Context ctx, int hashCapacity) {
		int capacity = hashCapacity > 0 ? hashCapacity :Env.getDefaultHashCapacity();
		HashUtil hashUtil = new HashUtil(capacity);
		capacity = hashUtil.getCapacity();
		
		// ???ɷ????????ύ???̳߳?
		int parallelNum = Env.getParallelNum();
		ThreadPool pool = ThreadPool.newInstance(parallelNum);
		GroupsJob2 []jobs = new GroupsJob2[parallelNum];
		Table groupsResult = null;

		try {
			for (int i = 0; i < parallelNum; ++i) {
				Context tmpCtx = ctx.newComputeContext();
				Expression []tmpExps = Operation.dupExpressions(exps, tmpCtx);
				Expression []tmpCalcExps = Operation.dupExpressions(calcExps, tmpCtx);
				
				GroupsJob2 job = new GroupsJob2(src, hashUtil, null, tmpExps, names, tmpCalcExps, calcNames, opt, tmpCtx, capacity);
				job.setHashStart(i);
				job.setHashEnd(parallelNum);
				jobs[i] = job;
				
				pool.submit(jobs[i]);
			}
			
			// ?ȴ?????????ִ????ϣ????ѽ?????ӵ?һ?????
			for (int i = 0; i < parallelNum; ++i) {
				jobs[i].join();
				
				if (i == 0) {
					groupsResult = jobs[i].getGroupsResult().getResultTable();
				} else {
					Table t = jobs[i].getGroupsResult().getResultTable();
					groupsResult.addAll(t);
				}
			}
		} finally {
			pool.shutdown();
		}
		
		if (opt == null || opt.indexOf('u') == -1) {
			int keyCount = exps.length;
			int []fields = new int[keyCount];
			for (int i = 0; i < keyCount; ++i) {
				fields[i] = i;
			}

			groupsResult.sortFields(fields);
		}
		return groupsResult;
	
	}
	
	/**
	 * ?趨???????????????????ﵽ???ֵ??ֹͣ????
	 * @param cursor ?α?
	 * @param exps ?????ֶα???ʽ????
	 * @param names ?????ֶ???????
	 * @param calcExps ?????ֶα???ʽ????
	 * @param calcNames ?????ֶ???????
	 * @param maxGroupCount
	 * @param opt ѡ??
	 * @param ctx ??????????
	 * @return ??????
	 */
	public static Table fuzzyGroups(ICursor cursor, Expression[] exps, String[] names, 
			Expression[] calcExps, String[] calcNames, String opt, Context ctx, int maxGroupCount) {
		DataStruct ds = cursor.getDataStruct();
		int count = exps.length;
		if (names == null) names = new String[count];
		for (int i = 0; i < count; ++i) {
			if (names[i] == null || names[i].length() == 0) {
				names[i] = exps[i].getFieldName(ds);
			}
		}

		if (calcExps != null) {
			count = calcExps.length;
			if (calcNames == null) calcNames = new String[count];
			for (int i = 0; i < count; ++i) {
				if (calcNames[i] == null || calcNames[i].length() == 0) {
					calcNames[i] = calcExps[i].getFieldName(ds);
				}
			}
		}

		int keyCount = exps.length;
		int valCount = calcExps == null ? 0 : calcExps.length;

		final int INIT_GROUPSIZE = HashUtil.getInitGroupSize();
		HashUtil hashUtil = new HashUtil(maxGroupCount);
		ListBase1 []groups = new ListBase1[hashUtil.getCapacity()];
		Object []keys = new Object[keyCount];
		String[] colNames = new String[keyCount + valCount];
		System.arraycopy(names, 0, colNames, 0, keyCount);

		if (calcNames != null) {
			System.arraycopy(calcNames, 0, colNames, keyCount, valCount);
		}

		Table table = new Table(colNames, hashUtil.getCapacity());
		table.setPrimary(names);
		Node []gathers = Sequence.prepareGatherMethods(calcExps, ctx);

		ComputeStack stack = ctx.getComputeStack();
		while (true) {
			Sequence src = cursor.fetch(ICursor.FETCHCOUNT);
			if (src == null || src.length() == 0) break;

			Current current = new Current(src);
			stack.push(current);

			try {
				for (int i = 1, len = src.length(); i <= len; ++i) {
					current.setCurrent(i);
					for (int k = 0; k < keyCount; ++k) {
						keys[k] = exps[k].calculate(ctx);
					}

					BaseRecord r;
					int hash = hashUtil.hashCode(keys);
					if (groups[hash] == null) {
						groups[hash] = new ListBase1(INIT_GROUPSIZE);
						r = table.newLast(keys);
						groups[hash].add(r);
						for (int v = 0, f = keyCount; v < valCount; ++v, ++f) {
							Object val = gathers[v].gather(ctx);
							r.setNormalFieldValue(f, val);
						}
					} else {
						int index = HashUtil.bsearch_r(groups[hash], keys);
						if (index < 1) {
							r = table.newLast(keys);
							groups[hash].add(-index, r);
							for (int v = 0, f = keyCount; v < valCount; ++v, ++f) {
								Object val = gathers[v].gather(ctx);
								r.setNormalFieldValue(f, val);
							}
						} else {
							r = (BaseRecord)groups[hash].get(index);
							for (int v = 0, f = keyCount; v < valCount; ++v, ++f) {
								Object val = gathers[v].gather(r.getNormalFieldValue(f), ctx);
								r.setNormalFieldValue(f, val);
							}
						}
					}
				}
			} finally {
				stack.pop();
			}

			if (table.length() >= maxGroupCount) {
				break;
			}
		}

		if (opt == null || opt.indexOf('u') == -1) {
			int []fields = new int[keyCount];
			for (int i = 0; i < keyCount; ++i) {
				fields[i] = i;
			}

			table.sortFields(fields);
		}

		table.finishGather(gathers);
		return table;
	}

	/**
	 * ?????н??й?ϣ????
	 * @param src ????
	 * @param exps ???????ʽ????
	 * @param opt ѡ??
	 * @param ctx ??????????
	 * @return ??????
	 */
	public static Sequence hashGroup(Sequence src, Expression[] exps, String opt, Context ctx) {
		if (src == null || src.length() == 0) {
			return new Sequence(0);
		}
		
		int srcLen = src.length();
		boolean isAll = true, isSort = true, isPos = false, isConj = false;
		if (opt != null) {
			if (opt.indexOf('1') != -1) isAll = false;
			if (opt.indexOf('u') != -1) isSort = false;
			if (opt.indexOf('p') != -1) isPos = true;
			if (opt.indexOf('s') != -1) isConj = true;
		}

		int keyCount = exps.length;
		final int INIT_GROUPSIZE = HashUtil.getInitGroupSize();
		HashUtil hashUtil = new HashUtil(srcLen / 2);
		ListBase1 []groups = new ListBase1[hashUtil.getCapacity()];
		Sequence result = new Sequence(hashUtil.getCapacity());
		
		boolean isSingleField = keyCount == 1 && !isAll;
		ListBase1 keyList = null;
		if (isSort) {
			keyList = new ListBase1(hashUtil.getCapacity());
		}

		ComputeStack stack = ctx.getComputeStack();
		Current current = new Current(src);
		stack.push(current);

		try {
			if (isSingleField) {
				Expression exp = exps[0];
				for (int i = 1; i <= srcLen; ++i) {
					current.setCurrent(i);
					Object key = exp.calculate(ctx);
					int hash = hashUtil.hashCode(key);

					if (groups[hash] == null) {
						result.add(isPos ? new Integer(i):current.getCurrent());
						groups[hash] = new ListBase1(INIT_GROUPSIZE);
						groups[hash].add(key);
						
						if (isSort) {
							keyList.add(key);
						}
					} else {
						int index = groups[hash].binarySearch(key);
						if (index < 1) {
							result.add(isPos ? new Integer(i):current.getCurrent());
							groups[hash].add(-index, key);
							
							if (isSort) {
								keyList.add(key);
							}
						}
					}
				}
			} else {
				int count = isAll ? keyCount + 1 : keyCount;
				for (int i = 1; i <= srcLen; ++i) {
					current.setCurrent(i);
					Object []keys = new Object[count];
					for (int k = 0; k < keyCount; ++k) {
						keys[k] = exps[k].calculate(ctx);
					}

					int hash = hashUtil.hashCode(keys, keyCount);
					if (groups[hash] == null) {
						if (isAll) {
							Sequence group = new Sequence(INIT_GROUPSIZE);
							group.add(isPos ? new Integer(i):current.getCurrent());
							keys[keyCount] = group;
							result.add(group);
						} else {
							result.add(isPos ? new Integer(i):current.getCurrent());
						}

						groups[hash] = new ListBase1(INIT_GROUPSIZE);
						groups[hash].add(keys);
						if (isSort) {
							keyList.add(keys);
						}
					} else {
						int index = HashUtil.bsearch_a(groups[hash], keys, keyCount);
						if (index < 1) {
							if (isAll) {
								Sequence group = new Sequence(INIT_GROUPSIZE);
								group.add(isPos ? new Integer(i):current.getCurrent());
								keys[keyCount] = group;
								result.add(group);
							} else {
								result.add(isPos ? new Integer(i):current.getCurrent());
							}

							groups[hash].add(-index, keys);
							if (isSort) {
								keyList.add(keys);
							}
						} else {
							if (isAll) {
								Object []tmps = (Object[])groups[hash].get(index);
								((Sequence)tmps[keyCount]).add(isPos ? new Integer(i):current.getCurrent());
							}
						}
					}
				}
			}
		} finally {
			stack.pop();
		}

		if (isSort) {
			int len = result.length();
			PSortItem []infos = new PSortItem[len + 1];
			for (int i = 1; i <= len; ++i) {
				infos[i] = new PSortItem(i, keyList.get(i));
			}

			Comparator