• Home
• com.scudata.esproc
• esproc
• 20240731
• source code
• Cuboid.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.scudata.dw.Cuboid Maven / Gradle / Ivy

package com.scudata.dw;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;

import com.scudata.common.DateFactory;
import com.scudata.common.MessageManager;
import com.scudata.common.RQException;
import com.scudata.dm.BaseRecord;
import com.scudata.dm.Context;
import com.scudata.dm.DataStruct;
import com.scudata.dm.Env;
import com.scudata.dm.FileObject;
import com.scudata.dm.ObjectReader;
import com.scudata.dm.Sequence;
import com.scudata.dm.Table;
import com.scudata.dm.cursor.ICursor;
import com.scudata.dm.cursor.MemoryCursor;
import com.scudata.dm.cursor.MergesCursor;
import com.scudata.dm.cursor.MultipathCursors;
import com.scudata.expression.Constant;
import com.scudata.expression.Expression;
import com.scudata.expression.IParam;
import com.scudata.expression.Node;
import com.scudata.expression.ParamInfo2;
import com.scudata.expression.UnknownSymbol;
import com.scudata.expression.fn.Between;
import com.scudata.expression.fn.datetime.Month;
import com.scudata.expression.fn.datetime.Year;
import com.scudata.expression.mfn.sequence.Contain;
import com.scudata.expression.operator.And;
import com.scudata.expression.operator.DotOperator;
import com.scudata.expression.operator.Equals;
import com.scudata.expression.operator.Greater;
import com.scudata.expression.operator.NotGreater;
import com.scudata.expression.operator.NotSmaller;
import com.scudata.expression.operator.Or;
import com.scudata.expression.operator.Smaller;
import com.scudata.parallel.ClusterCursor;
import com.scudata.parallel.ClusterPhyTable;
import com.scudata.resources.EngineMessage;
import com.scudata.util.EnvUtil;
import com.scudata.util.Variant;

/**
 * ????????
 * ?̳????д?????????ڴ洢һ??????Ԥ???ܽ??
 * @author runqian
 *
 */
public class Cuboid extends RowComTable {
	public static final String CUBE_PREFIX = "_CUBOID@";//???????ļ?ǰ׺
	private static final int FIXED_OBJ_LEN = 9;//?????ֶεij???
	//?ļ?ͷ????????????
	protected String[] exps;//???????ʽ
	protected String[] newExps;//???ܱ???ʽ
	private long srcCount;//ԭ?????ѷ??????? 

	protected CuboidTable baseTable;
	
	/**
	 * ??,???鱸?ݱ?־
	 * @param file ???????ļ?
	 * @param ctx
	 * @throws IOException
	 */
	public Cuboid(File file, Context ctx) throws IOException {
		super();
		this.file = file;
		this.raf = new RandomAccessFile(file, "rw");
		this.ctx = ctx;
		if (ctx != null)
			ctx.addResource(this);
		readHeader();
	}

	/**
	 * ?½?һ????????
	 * @param file ?ļ?????
	 * @param colNames ??????
	 * @param serialBytesLen ÿ???е??źų???
	 * @param ctx
	 * @param writePsw д????
	 * @param readPsw ??????
	 * @param exps ???????ʽ
	 * @param newExps ???ܱ???ʽ
	 * @throws IOException
	 */
	public Cuboid(File file, String []colNames, int []serialBytesLen, Context ctx, 
			String writePsw, String readPsw, String[] exps, String[] newExps) throws IOException {
		super(file, colNames, null, null, ctx);
		this.exps = exps;
		this.newExps = newExps;
		srcCount = 0;
	}
	
	/**
	 * ???ļ?ͷ
	 */
	protected void readHeader() throws IOException {
		Object syncObj = getSyncObject();
		synchronized(syncObj) {
			restoreTransaction();
			raf.seek(0);
			byte []bytes = new byte[32];
			raf.read(bytes);
			if (bytes[0] != 'r' || bytes[1] != 'q' || bytes[2] != 'd' || bytes[3] != 'w' || bytes[4] != 'g' || bytes[5] != 't' || bytes[6] != 'r') {
				MessageManager mm = EngineMessage.get();
				throw new RQException(mm.getMessage("license.fileFormatError"));
			}
			
			BufferReader reader = new BufferReader(structManager, bytes, 7, 25);
			setBlockSize(reader.readInt32());
			headerBlockLink = new BlockLink(this);
			headerBlockLink.readExternal(reader);
			
			BlockLinkReader headerReader = new BlockLinkReader(headerBlockLink);
			bytes = headerReader.readBlocks();
			headerReader.close();
			reader = new BufferReader(structManager, bytes);
			reader.read(); // r
			reader.read(); // q
			reader.read(); // d
			reader.read(); // w
			reader.read(); // g
			reader.read(); // t
			reader.read(); // r
			
			blockSize = reader.readInt32();
			headerBlockLink.readExternal(reader);
			
			reader.read(reserve); // ????λ
			freePos = reader.readLong40();
			fileSize = reader.readLong40();
			
			if (reserve[0] > 0) {
				writePswHash = reader.readString();
				readPswHash = reader.readString();
				checkPassword(null);
				
				if (reserve[0] > 1) {
					distribute = reader.readString();
				}
			}
	
			int dsCount = reader.readInt();
			if (dsCount > 0) {
				ArrayList dsList = new ArrayList(dsCount);
				for (int i = 0; i < dsCount; ++i) {
					String []fieldNames = reader.readStrings();
					DataStruct ds = new DataStruct(fieldNames);
					dsList.add(ds);
				}
				
				structManager = new StructManager(dsList);
			} else {
				structManager = new StructManager();
			}

			baseTable = new CuboidTable(this, null);
			baseTable.readExternal(reader);
			super.baseTable = baseTable;
			
			exps = reader.readStrings();
			newExps = reader.readStrings();
			srcCount = reader.readLong40();
			
		}
	}
	
	/**
	 * д???ļ?ͷ
	 */
	public void writeHeader() throws IOException {
		Object syncObj = getSyncObject();
		synchronized(syncObj) {
			beginTransaction(null);
			BufferWriter writer = new BufferWriter(structManager);
			writer.write('r');
			writer.write('q');
			writer.write('d');
			writer.write('w');
			writer.write('g');
			writer.write('t');
			writer.write('r');
			
			writer.writeInt32(blockSize);
			headerBlockLink.writeExternal(writer);
			
			reserve[0] = 3; // 1???????룬2???ӷֲ???????3????Ԥ????
			writer.write(reserve); // ????λ
			
			writer.writeLong40(freePos);
			writer.writeLong40(fileSize);
			
			// ??????????Ա?汾1???ӵ?
			writer.writeString(writePswHash);
			writer.writeString(readPswHash);
			
			writer.writeString(distribute); // ?汾2????
	
			ArrayList dsList = structManager.getStructList();
			if (dsList != null) {
				writer.writeInt(dsList.size());
				for (DataStruct ds : dsList) {
					String []fieldNames = ds.getFieldNames();
					writer.writeStrings(fieldNames);
				}
			} else {
				writer.writeInt(0);
			}
			super.baseTable.writeExternal(writer);
			
			writer.writeStrings(exps);
			writer.writeStrings(newExps);
			writer.writeLong40(srcCount);
			
			BlockLinkWriter headerWriter = new BlockLinkWriter(headerBlockLink, false);
			headerWriter.rewriteBlocks(writer.finish());
			headerWriter.close();
			
			// ??дheaderBlockLink
			writer.write('r');
			writer.write('q');
			writer.write('d');
			writer.write('w');
			writer.write('g');
			writer.write('t');
			writer.write('r');
			
			writer.writeInt32(blockSize);
			headerBlockLink.writeExternal(writer);
			raf.seek(0);
			raf.write(writer.finish());
			raf.getChannel().force(true);
			commitTransaction(0);
		}
	}

	public void save() throws IOException {
		super.save();
	}
	
	public void setCount(long count) {
		this.setSrcCount(count);
	}

	/**
	 * ??ʽ??????ʽ???????Ԥ?????ֶ?ƥ?????''???????ƥ??????Ϊ?DZ???ʽ
	 * @param exps
	 * @param ds
	 */
	private static void formatExps(Expression exps[], DataStruct ds) {
		int expsLen = exps == null ? 0 : exps.length;
		for (int i = 0; i < expsLen; i++) {
			if (ds.getFieldIndex(exps[i].getIdentifierName()) >= 0) {
				//???ֱ?Ӿ???Ԥ??????ֶ?
				exps[i] = new Expression("'" + exps[i].getIdentifierName() + "'");
			}
		}
	}
	
	/**
	 * ȡ?÷??????ʽexps?;ۺϱ???ʽnewExps???õ????ֶΣ???????ԭ???α?
	 * @param srcTable ԭ??
	 * @param exps ???????ʽ
	 * @param newExps ?ۺϱ???ʽ
	 * @param w ???˱???ʽ
	 * @param hasM ?Ƿ??·
	 * @param n ????·??
	 * @param ctx ??????
	 * @return
	 */
	private static ICursor makeCursor(PhyTable srcTable, Expression []exps, Expression []newExps,
			Expression w, boolean hasM, int n, Context ctx) {
		List list = new ArrayList();
		if (exps != null) {
			for (Expression exp : exps) {
				exp.getUsedFields(ctx, list);
			}
		}
		if (newExps != null) {
			for (Expression exp : newExps) {
				exp.getUsedFields(ctx, list);
			}
		}
		String fields[] = new String[list.size()];
		list.toArray(fields);
		
		if (hasM) {
			return srcTable.cursor(null, fields, w, null, null, null, n, null, ctx);
		} else {
			return srcTable.cursor(fields, w, ctx);
		}
	}
	
	/**
	 * ????
	 * @param exps ???????ʽ
	 * @param names ???????ʽ??????
	 * @param newExps ???ܱ???ʽ
	 * @param newNames ???ܱ???ʽ??????
	 * @param srcTable Ҫ????????
	 * @param w ????????
	 * @param hasM ?Ƿ???
	 * @param n ??????
	 * @param option
	 * @param ctx
	 * @return
	 */
	public static Sequence cgroups(String []expNames, String []names, String []newExpNames, String []newNames,
			PhyTable srcTable, Expression w, boolean hasM, int n, String option,  Context ctx) {
		//ע?⣬cgroups_?õ????ǰ??????ļ?cuboid??
		
		return cgroups_(expNames, names, newExpNames, newNames, srcTable, w, hasM, n, option, null, ctx);
		
		/*TableMetaData tmd = srcTable.getSupplementTable(false);
		if (tmd == null) {
			return seq;
		} else {			
			//??֯???????ʽ
			int count = names.length;
			Expression exps[] = new Expression[count];
			for (int i = 0; i < count; i++) {
				exps[i] = new Expression(names[i]);
			}
			
			//??֯?پۺϱ???ʽ
			Expression newExps[] = new Expression[count = newNames.length];//?????洢Ҫ?ۺϵ?ԭ?ֶ?
			for (int i = 0, len = count; i < len; i++) {
				String str = newExpNames[i];
				//??count?پۺϣ?Ҫ??Ϊ?ۼ?
				if (str.indexOf("count(") != -1) {
					str = str.replaceFirst("count", "sum");
				}
				
				//?پۺ?ʱҪ?滻һ???ֶ???
				String sub = str.substring(str.indexOf('(') + 1, str.indexOf(')'));
				str = str.replaceAll(sub, "'" + newNames[i] + "'");
				newExps[i] = new Expression(str);
			}
			return seq.groups(exps, names, newExps, newNames, option, ctx);
		}*/
	}
	
	public static Sequence cgroups(String []expNames, String []names, String []newExpNames, String []newNames,
			PhyTable srcTable, Expression w, boolean hasM, int n, String option, FileObject[] files,  Context ctx) {
		return cgroups_(expNames, names, newExpNames, newNames, srcTable, w, hasM, n, option, files, ctx);
	}
	
	private static Sequence cgroups_(String []expNames, String []names, String []newExpNames, String []newNames,
			PhyTable srcTable,	Expression w, boolean hasM, int n, String option, FileObject[] files, Context ctx) {
		Expression []exps = null;
		if (expNames != null) {
			int len = expNames.length;
			exps = new Expression[len];
			for (int i = 0; i < len; i++) {
				exps[i] = new Expression(expNames[i]);
			}
		}
		
		Expression []newExps = null;
		Expression []newExpsBak = null;
		if (newExpNames != null) {
			int len = newExpNames.length;
			newExps = new Expression[len];
			newExpsBak = new Expression[len];
			for (int i = 0; i < len; i++) {
				newExps[i] = new Expression(newExpNames[i]);
				newExpsBak[i] = new Expression(newExpNames[i]);
			}
		}
		
		int fcount = exps == null ? 0 : exps.length;
		if (newExps != null) fcount += newExps.length;
		if (newExpNames != null) {
			for (int i = 0; i < newExpNames.length; i++) {
				String s = new String(newExpNames[i]);
				newExpNames[i] = s;// s.replaceAll(regex, "_");
			}
		}

		Object obj;
		if (files == null) {
			obj= findCuboid(srcTable, expNames, newExpNames, w, ctx);
		} else {
			obj= findCuboid(srcTable, expNames, newExpNames, w, files, ctx);
		}
		try {
			if (obj instanceof ArrayList) {
				//?ж??Ԥ??????????
				@SuppressWarnings("unchecked")
				ArrayList tableList = (ArrayList) obj;
				int size = tableList.size();
				if (size == 0) {
					//û??Ԥ??????????
					ICursor cs = makeCursor(srcTable, exps, newExps, w, hasM, n, ctx);
					return cs.groups(exps, names, newExps, newNames, option, ctx);
				} else {
					CuboidTable table = null;
					if (size == 1) {
						table = (CuboidTable) tableList.get(0);
					} else {
						//ѡһ????¼???????ٵ?
						int idx = 0;
						long cnt = tableList.get(0).getTotalRecordCount();
						for (int i = 1; i < size; i++) {
							long l = tableList.get(i).getTotalRecordCount();
							if (cnt < l) {
								idx = i;
								cnt = l;
							}
						}
						for (int i = 0; i < size; i++) {
							if (i != idx) {
								tableList.get(i).close();
							}
						}
						table = (CuboidTable) tableList.get(idx);
					}

					
					//Ԥ???鲻?ٸ?????????£?????ɾ??
//					???Ԥ?????Ƿ??????µ?
//					if (((Cuboid) table.groupTable).getSrcCount() != srcTable.getActualRecordCount()) {
//						try {
//							((Cuboid) table.groupTable).update(srcTable);
//						} catch (Exception e) {
//							if (table != null)
//								table.close();
//							for (PhyTable tbl : tableList) {
//								tbl.close();
//							}
//							throw new RQException(e.getMessage(), e);
//						}
//					}
					

					ArrayList fieldExpList = new ArrayList();
					Expression[] fieldExps;//?پۺ?ʱҪȡ???????ֶ?
					int expsLen = exps == null ? 0 : exps.length;
					DataStruct ds = table.getDataStruct();
					for (int i = 0; i < expsLen; i++) {
						if (ds.getFieldIndex(expNames[i]) >= 0) {
							//???ֱ?Ӿ???Ԥ??????ֶ?
							fieldExpList.add(new Expression("'" + expNames[i] + "'"));
						} else {
							//??ȡ?漰?????ֶ?
							Expression exp = new Expression(expNames[i]);
							ArrayList strList = new ArrayList();
							exp.getUsedFields(ctx, strList);
							for (String str : strList) {
								fieldExpList.add(new Expression(str));
							}
						}
					}
					for (int i = expsLen; i < fcount; i++) {
						fieldExpList.add(new Expression("'" + newExpNames[i - expsLen] + "'"));
					}
					fieldExps = new Expression[fieldExpList.size()];
					fieldExpList.toArray(fieldExps);

					Expression newSrcExps[] = new Expression[newExps.length];//?????洢Ҫ?ۺϵ?ԭ?ֶ?
					for (int i = 0, len = newExps.length; i < len; i++) {
						String str = new String(newExps[i].getIdentifierName());
						if (newNames[i] == null) {
							newNames[i] = str;
						}

						//??count?پۺϣ?Ҫ??Ϊ?ۼ?
						if (str.indexOf("count(") != -1) {
							str = str.replaceFirst("count", "sum");
						}

						//??Ԥ???????????پۺ?ʱҪ?滻һ???ֶ???
						String sub = str.substring(str.indexOf('(') + 1, str.indexOf(')'));
						str = str.replaceAll(sub, "'" + newExpNames[i] + "'");
						newExps[i] = new Expression(str);
						if (str.indexOf("top(") == 0) {
							sub = sub.substring(sub.indexOf(',') + 1);
						}
						newSrcExps[i] = new Expression(sub);
					}

					//????ʱ???????Ż?
					while (w != null) {
						//??????????ȡ???????ڷ?????ֶΣ??????ж??
						ArrayList