src.com.ibm.as400.access.GenerateConverterTable Maven / Gradle / Ivy
///////////////////////////////////////////////////////////////////////////////
//
// JTOpen (IBM Toolbox for Java - OSS version)
//
// Filename: AS400.java
//
// The source code contained herein is licensed under the IBM Public License
// Version 1.0, which has been approved by the Open Source Initiative.
// Copyright (C) 1997-2016 International Business Machines Corporation and
// others. All rights reserved.
//
///////////////////////////////////////////////////////////////////////////////
package com.ibm.as400.access;
import java.io.*;
import java.lang.reflect.Field;
import java.util.*;
public class GenerateConverterTable {
private static final String copyright = "Copyright (C) 1997-2016 International Business Machines Corporation and others.";
static AS400 sys = null;
static boolean compress_ = true; // Compress the conversion table
// Note: turn this off for debugging purposes
static boolean codePointPerLine_ = false; // Should only 1 code point be
// printed per line
static boolean ascii_ = false; // Indicates if listed ccsids are ascii tables
// or not
static boolean bidi_ = false; // Indicates if listed ccsids are bidi tables or
// not
// Note: bidi_ and ascii_ cannot both be true
static boolean showOffsets_ = false; // Indicates of the offsets should be
// printed in the tables
public static void main(String[] args) {
if (args.length < 4) {
System.out
.println("Usage: java com.ibm.as400.access.GenerateConverterTable system uid pwd [-nocompress] [-ascii] [-bidi] [-showOffsets] [-codePointPerLine] ccsid [ccsid2] [ccsid3] [ccsid4] ...");
System.exit(0);
}
try {
sys = new AS400(args[0], args[1], args[2]);
sys.connectService(AS400.CENTRAL);
} catch (Exception e) {
e.printStackTrace();
System.exit(0);
}
int start = 3;
if (args[start].equals("-nocompress")) {
compress_ = false;
++start;
}
if (args[start].equals("-ascii")) {
ascii_ = true;
++start;
}
if (args[start].equals("-bidi")) {
bidi_ = true;
++start;
}
if (args[start].equals("-showOffsets")) {
showOffsets_ = true;
++start;
}
if (args[start].equals("-codePointPerLine")) {
codePointPerLine_ = true;
++start;
}
for (int i = start; i < args.length; ++i) {
go((new Integer(args[i])).intValue());
}
}
static String formattedChar(char x) {
int num = 0xFFFF & (int) x;
String s = "\\u";
if (num < 16)
s += "0";
if (num < 256)
s += "0";
if (num < 4096)
s += "0";
s += Integer.toHexString(num).toUpperCase();
return s;
}
static void go(int ccsid) {
char[] tableToUnicode = new char[0];
char[] tableToEbcdic = new char[0];
char[][] surrogateTable = null;
// int numTables1 = 1;
// int numTables2 = 1;
boolean ebcdicIsDBCS = false;
int doubleByteFormat = NLSTableDownload.DOUBLE_BYTE_FROM_CCSID;
int originalCcsid = ccsid;
try {
AS400ImplRemote impl = (AS400ImplRemote) sys.getImpl();
NLSTableDownload down = new NLSTableDownload(impl);
down.connect();
if (ccsid == 1089) // There are currently no tables for 1089->13488->1089;
// use 61952 instead, since it would be the same
// anyway.
{
System.out.println("Special case for ccsid 1089.");
System.out.println("Retrieving " + ccsid + "->61952 table...");
tableToUnicode = down.download(ccsid, 61952,
NLSTableDownload.SINGLE_BYTE_FROM_CCSID);
} else if (ccsid == 61175) {
System.out.println("Special case for ccsid 61175.");
System.out.println("Retrieving 1026->13488 table and adjusting...");
tableToUnicode = down.download(1026, 13488,
NLSTableDownload.SINGLE_BYTE_FROM_CCSID);
tableToUnicode[0xFC] = tableToUnicode[0x7F];
tableToUnicode[0x7F] = '"';
} else if (ccsid == 1376) {
// This is double byte so fall into bottom path
tableToUnicode = null;
} else if (ccsid == 1371) {
tableToUnicode = null;
doubleByteFormat = NLSTableDownload.MIXED_BYTE_FROM_CCSID;
} else {
System.out.println("Retrieving " + ccsid + "->13488 table...");
tableToUnicode = down.download(ccsid, 13488 ,
NLSTableDownload.SINGLE_BYTE_FROM_CCSID);
}
if (tableToUnicode == null || tableToUnicode.length == 0) {
String reason = "";
if (tableToUnicode == null) {
reason ="tableToUnicode is null";
} else {
reason ="tableToUnicode.length is 0";
}
if (ccsid == 1175) {
System.out.println("Aborting since CCSD 1175 failed to download");
throw new Exception("Aborting since CCSD 1175 failed to download");
}
System.out
.println(ccsid
+ " must be double-byte because download failed ("+reason+"). Performing secondary retrieve of "
+ ccsid + "->1200 table...");
ebcdicIsDBCS = true;
down.disconnect();
down.connect();
tableToUnicode = down.download(ccsid, 1200,
doubleByteFormat);
}
System.out.println(" Size: " + tableToUnicode.length);
if (tableToUnicode.length > 65536) {
System.out.println("Size is > 65536. Fixing table");
int next = 0;
int from = 0;
char[] newTable = new char[65536];
while (from < tableToUnicode.length && next < 65536) {
int c = 0xFFFF & (int) tableToUnicode[from];
if (next > 0xECAA && next <= 0xECD0) {
System.out.println("Next=0x" + Integer.toHexString(next) + " to="
+ Integer.toHexString(c));
}
int nextchar = 0;
if (from + 1 < tableToUnicode.length) {
nextchar = 0xFFFF & (int) tableToUnicode[from + 1];
}
if (
// in surrogate range
((c >= 0xD800) && (c <= 0xDFFF))
||
// Uses combining character
((nextchar == 0x309A) && (c != 0x3099)) || /* In 835 there are two combining characters next to each other */
/* In that case, we do not combine */
(c != 0xFFfd && nextchar == 0x300)
|| (c != 0xffd && c != 0x300 && nextchar == 0x301)
||
// Weird cases..
(c == 0x2e5 && nextchar == 0x2e9)
|| (c == 0x2e9 && nextchar == 0x2e5)) {
// Mark as surrogate
newTable[next] = (char) 0xD800;
// add to surrogate table
if (surrogateTable == null) {
surrogateTable = new char[65536][];
}
char[] pair = new char[2];
surrogateTable[next] = pair;
pair[0] = (char) (0xFFFF & (int) tableToUnicode[from]);
pair[1] = (char) (0xFFFF & (int) tableToUnicode[from + 1]);
/*
* System.out.println("Warning: Sub at offset "+Integer.toHexString(next
* )+" for "+Integer.toHexString(0xFFFF & (int)
* table1[from])+" "+Integer.toHexString(0xFFFF & (int)
* table1[from+1]));
*/
from += 2;
} else {
newTable[next] = (char) c;
from++;
}
next++;
}
tableToUnicode = newTable;
}
down.disconnect();
down.connect();
if (ccsid == 1089) {
System.out.println("Special case for ccsid 1089.");
System.out.println("Retrieving 61952->" + ccsid + " table...");
tableToEbcdic = down.download(61952, ccsid,
NLSTableDownload.DOUBLE_BYTE_FROM_CCSID);
} else {
/* Use 1200 instead of 13488 */
System.out.println("Retrieving 1200->" + ccsid + " table...");
tableToEbcdic = down.download(1200, ccsid,
NLSTableDownload.DOUBLE_BYTE_FROM_CCSID);
}
System.out.println(" Size: " + tableToEbcdic.length);
// If a mixed CCSID, then we need to fix the tableToEbcdic
// Convert the table to a byte array
// Go through table and process SI/I)
if (doubleByteFormat == NLSTableDownload.MIXED_BYTE_FROM_CCSID) {
char[] newTableToEbcdic = new char[65536];
byte[] oldTableToEbcdic = new byte[tableToEbcdic.length * 2];
for (int i = 0; i < tableToEbcdic.length; i++) {
oldTableToEbcdic[2*i] = (byte) ( 0xFF & (tableToEbcdic[i] >> 8));
oldTableToEbcdic[2*i+1] = (byte)( tableToEbcdic[i] & 0xFF);
}
boolean singleByte = true;
int newTableOffset = 0;
for (int i = 0; i < oldTableToEbcdic.length; i++) {
int b = 0xFF & oldTableToEbcdic[i];
while ((i > 0x0F) && (i < oldTableToEbcdic.length) && ((b == 0x0E) || (b == 0x0F)) ) {
if (b == 0x0E) {
singleByte = false;
i++;
} else {
singleByte = true;
i++;
}
if (i < oldTableToEbcdic.length) {
b = 0xFF & oldTableToEbcdic[i];
}
}
if (i < oldTableToEbcdic.length) {
if (singleByte) {
if (newTableOffset < newTableToEbcdic.length) {
newTableToEbcdic[newTableOffset] = (char) b;
newTableOffset++;
}
} else {
i++;
if (i < oldTableToEbcdic.length) {
int c = 0xFF & oldTableToEbcdic[i];
if (newTableOffset < newTableToEbcdic.length) {
if (newTableOffset == 0x431) {
newTableToEbcdic[newTableOffset] = (char) ((b << 8) + c);
newTableOffset++;
} else {
newTableToEbcdic[newTableOffset] = (char) ((b << 8) + c);
newTableOffset++;
}
}
}
}
}
} /* for i */
while (newTableOffset < newTableToEbcdic.length) {
newTableToEbcdic[newTableOffset]=(char) 0xFEFE;
newTableOffset++;
}
tableToEbcdic=newTableToEbcdic;
} /* If Mixed byte */
sys.disconnectAllServices();
} catch (Exception e) {
e.printStackTrace(System.out);
}
// Do any necessary fixup
if (ccsid == 290) {
tableToUnicode[0xE1] = '\u20ac';
char toEbcdic = tableToEbcdic[0x20ac / 2];
toEbcdic = (char)((0xE1 << 8) | (toEbcdic & 0xFF));
tableToEbcdic[0x20ac / 2] = toEbcdic;
}
// Verify the mapping
verifyRoundTrip(tableToUnicode, tableToEbcdic, ebcdicIsDBCS);
System.out.println("****************************************");
System.out.println("Verify round 2 ");
System.out.println("****************************************");
verifyRoundTrip(tableToUnicode, tableToEbcdic, ebcdicIsDBCS);
// Compress the ccsid table
if (ebcdicIsDBCS) {
if (compress_) {
System.out.println("Compressing " + ccsid
+ "->13488 conversion table...");
char[] arr = compress(tableToUnicode);
System.out.println("Old compression length: " + arr.length
+ " characters.");
char[] temparr = compressBetter(tableToUnicode);
System.out.println("New compression length: " + temparr.length
+ " characters.");
if (temparr.length > arr.length) {
System.out
.println("WARNING: New algorithm WORSE than old algorithm!");
}
System.out.println("Verifying compressed table...");
arr = decompressBetter(temparr);
if (arr.length != tableToUnicode.length) {
System.out.println("Verification failed, lengths not equal: "
+ arr.length + " != " + tableToUnicode.length);
int c = 0;
while (c < arr.length && arr[c] == tableToUnicode[c])
++c;
System.out.println("First mismatch at index " + c + ": "
+ (int) arr[c] + " != " + (int) tableToUnicode[c]);
} else {
boolean bad = false;
for (int c = 0; c < arr.length; ++c) {
if (arr[c] != tableToUnicode[c]) {
bad = true;
System.out.println(c + ": " + Integer.toHexString((int) arr[c])
+ " != " + Integer.toHexString((int) tableToUnicode[c]));
}
}
if (bad) {
System.out.println("Mismatches found in table.");
} else {
tableToUnicode = temparr;
System.out.println("Table verified.");
}
}
}
}
int fileCcsid = ccsid;
if ( doubleByteFormat == NLSTableDownload.MIXED_BYTE_FROM_CCSID) {
fileCcsid = ccsid + 1100000;
System.out.println("Create file using "+fileCcsid+" since MIXED CCSID ");
}
// Write out the ccsid table
try {
String fName = "ConvTable" + fileCcsid + ".java";
FileWriter f = new FileWriter(fName);
writeHeader(f, fileCcsid, sys.getSystemName());
if (ascii_) {
f.write("class ConvTable" + fileCcsid + " extends ConvTableAsciiMap\n{\n");
} else if (bidi_) {
f.write("class ConvTable" + fileCcsid + " extends ConvTableBidiMap\n{\n");
} else if (ebcdicIsDBCS) {
f.write("class ConvTable" + fileCcsid + " extends ConvTableDoubleMap\n{\n");
} else {
f.write("class ConvTable" + fileCcsid + " extends ConvTableSingleMap\n{\n");
}
f.write(" private static char[] toUnicodeArray_; \n");
f.write(" private static final String copyright = \"Copyright (C) 1997-2016 International Business Machines Corporation and others.\";\n");
f.write(" private static final String toUnicode_ = \n");
System.out.print("Writing table for conversion from " + ccsid
+ " to 13488... to " + fName + "\n");
for (int i = 0; i < tableToUnicode.length; i = i + 16) {
if (showOffsets_) {
f.write("/* " + Integer.toHexString(i) + " */ \"");
} else {
f.write(" \"");
}
for (int j = 0; j < 16 && (i + j) < tableToUnicode.length; ++j) {
int num = (int) tableToUnicode[i + j];
if (num == 0x0008)
f.write("\\b");
else if (num == 0x0009)
f.write("\\t");
// else if(num == 0x000A) f.write("\\r");
else if (num == 0x000A)
f.write("\\n");
else if (num == 0x000C)
f.write("\\f");
// else if(num == 0x000D) f.write("\\n");
else if (num == 0x000D)
f.write("\\r");
else if (num == 0x0022)
f.write("\\\"");
else if (num == 0x0027)
f.write("\\'");
else if (num == 0x005C)
f.write("\\\\");
else {
String s = "\\u";
if (num < 16)
s += "0";
if (num < 256)
s += "0";
if (num < 4096)
s += "0";
s += Integer.toHexString(num).toUpperCase();
f.write(s);
}
if (codePointPerLine_) {
if (j < 15) {
if (showOffsets_) {
f.write("\" +\n/* " + Integer.toHexString(i + j + 1) + " */ \"");
} else {
f.write("\" +\n \"");
}
}
}
}
if (i + 16 < tableToUnicode.length)
f.write("\" +\n");
else
f.write("\";\n");
} /* for i */
f.write("\n");
f.write("\n");
// Write out the surrogateTable if it exists
if (surrogateTable != null) {
f.write("\n");
f.write(" private static final char[][] toUnicodeSurrogateMappings = { \n");
System.out.print("Writing surrogate table for conversion from " + ccsid
+ " to 13488... to " + fName + "\n");
for (int i = 0; i < surrogateTable.length; i++) {
char[] pair = surrogateTable[i];
if (pair != null) {
f.write("{'" + formattedChar((char) i) + "','"
+ formattedChar(pair[0]) + "','" + formattedChar(pair[1])
+ "'},\n");
}
} /* for i */
f.write("};\n");
f.write("\n");
f.write("\n");
} /* if surrogate table */
f.close();
} catch (Exception e) {
e.printStackTrace();
}
// Compress the Unicode table
if (compress_) {
System.out
.println("Compressing 13488->" + ccsid + " conversion table...");
char[] arr = compress(tableToEbcdic);
System.out.println("Old compression length: " + arr.length
+ " characters.");
char[] temparr = compressBetter(tableToEbcdic);
System.out.println("New compression length: " + temparr.length
+ " characters.");
if (temparr.length > arr.length) {
System.out.println("WARNING: New algorithm WORSE than old algorithm!");
}
System.out.println("Verifying compressed table...");
arr = decompressBetter(temparr);
if (arr.length != tableToEbcdic.length) {
System.out.println("Verification failed, lengths not equal: "
+ arr.length + " != " + tableToEbcdic.length);
int c = 0;
while (c < arr.length && arr[c] == tableToEbcdic[c])
++c;
System.out.println("First mismatch at index " + c + ": " + (int) arr[c]
+ " != " + (int) tableToEbcdic[c]);
tableToEbcdic = temparr;
} else {
boolean bad = false;
for (int c = 0; c < arr.length; ++c) {
if (arr[c] != tableToEbcdic[c]) {
bad = true;
System.out.println(c + ": " + Integer.toHexString((int) arr[c])
+ " != " + Integer.toHexString((int) tableToEbcdic[c]));
}
}
if (bad) {
System.out.println("Mismatches found in table.");
} else {
tableToEbcdic = temparr;
System.out.println("Table verified.");
}
}
}
// Write out the Unicode table
try {
String fName = "ConvTable" + fileCcsid + ".java";
FileWriter f = new FileWriter(fName, true);
f.write(" private static char[] fromUnicodeArray_; \n");
f.write(" private static final String fromUnicode_ = \n");
System.out.print("Writing table for conversion from 13488 to " + ccsid
+ "... to " + fName + "\n");
for (int i = 0; i < tableToEbcdic.length; i = i + 16) {
if (showOffsets_) {
f.write("/* " + Integer.toHexString(i) + " */ \"");
} else {
f.write(" \"");
}
for (int j = 0; j < 16 && (i + j) < tableToEbcdic.length; ++j) {
int num = (int) tableToEbcdic[i + j]; // these each contain 2 single
// byte chars, but we write it
// like this to save space
if (num == 0x0008)
f.write("\\b");
else if (num == 0x0009)
f.write("\\t");
// else if(num == 0x000A) f.write("\\r");
else if (num == 0x000A)
f.write("\\n");
else if (num == 0x000C)
f.write("\\f");
// else if(num == 0x000D) f.write("\\n");
else if (num == 0x000D)
f.write("\\r");
else if (num == 0x0022)
f.write("\\\"");
else if (num == 0x0027)
f.write("\\'");
else if (num == 0x005C)
f.write("\\\\");
else {
String s = "\\u";
if (num < 16)
s += "0";
if (num < 256)
s += "0";
if (num < 4096)
s += "0";
s += Integer.toHexString(num).toUpperCase();
f.write(s);
}
if (codePointPerLine_) {
if (j < 15) {
if (showOffsets_) {
f.write("\" +\n/* " + Integer.toHexString(i + j + 1) + " */ \"");
} else {
f.write("\" +\n \"");
}
}
}
}
if (i + 16 < tableToEbcdic.length)
f.write("\" +\n");
else
f.write("\";\n");
}
f.write("\n");
f.write(" static {\n");
f.write(" toUnicodeArray_ = toUnicode_.toCharArray();\n");
f.write(" fromUnicodeArray_ = fromUnicode_.toCharArray();\n");
f.write(" }\n");
f.write("\n ConvTable" + fileCcsid + "()\n {\n");
f.write(" super(" + fileCcsid + ", ");
f.write("toUnicodeArray_, ");
if (surrogateTable != null) {
f.write("fromUnicodeArray_,");
f.write("toUnicodeSurrogateMappings);\n");
} else {
f.write("fromUnicodeArray_);\n");
}
f.write(" }\n\n");
f.write("\n ConvTable" + fileCcsid + "(int ccsid)\n {\n");
f.write(" super(ccsid, ");
f.write("toUnicodeArray_, ");
if (surrogateTable != null) {
f.write("fromUnicodeArray_,");
f.write("toUnicodeSurrogateMappings);\n");
} else {
f.write("fromUnicodeArray_);\n");
}
f.write(" }\n}\n");
f.close();
} catch (Exception e) {
e.printStackTrace();
}
System.out.print("Done.\n");
}
private static boolean verifyRoundTrip(char[] tableToUnicode,
char[] tableToEbcdic, boolean ebcdicIsDBCS) {
String ebcdicPrefix = "X";
if (ebcdicIsDBCS) {
ebcdicPrefix = "GX";
}
// Make sure substitution is correct for Unicode to ebcdic
if (!ebcdicIsDBCS) {
int piece = 0xFFFF & tableToEbcdic[0x1a / 2];
if ((piece >> 8) != 0x3f) {
System.out.println("Fixing sub char in tableToEbcdic == sub was 0x"
+ Integer.toHexString(piece >> 8));
piece = (0x3f << 8) | (0xFF & piece);
tableToEbcdic[0x1a / 2] = (char) piece;
}
}
System.out.println("Checking round trip");
boolean passed = true;
StringBuffer sb1 = new StringBuffer();
StringBuffer sb2 = new StringBuffer();
StringBuffer sb3 = new StringBuffer();
for (int i = 0; i < tableToUnicode.length; i++) {
int unicodeChar = 0xFFFF & tableToUnicode[i];
if (unicodeChar != 0xfffd && unicodeChar != 0xD800) {
int ebcdicChar;
if (ebcdicIsDBCS) {
ebcdicChar = 0xFFFF & tableToEbcdic[unicodeChar];
} else {
int piece = 0xFFFF & tableToEbcdic[unicodeChar / 2];
if (unicodeChar % 2 == 0) {
ebcdicChar = piece >> 8;
} else {
ebcdicChar = piece & 0xFF;
}
}
if (i != ebcdicChar) {
if ((unicodeChar != 0x1a)
&& ((ebcdicChar == 0xFEFE) || (ebcdicChar == 0x3F))) {
sb1.append("Fixing up EBCDIC RoundTrip Failure " + ebcdicPrefix
+ "'" + Integer.toHexString(i) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar) + "'" + " -> "
+ ebcdicPrefix + "'" + Integer.toHexString(ebcdicChar) + "'\n");
if (ebcdicIsDBCS) {
tableToEbcdic[unicodeChar] = (char) i;
} else {
int piece = 0xFFFF & tableToEbcdic[unicodeChar / 2];
if (unicodeChar % 2 == 0) {
piece = (i << 8) | (piece & 0x00FF);
} else {
piece = (piece & 0xFF00) | i;
}
tableToEbcdic[unicodeChar / 2] = (char) piece;
}
passed = false;
} else {
int unicodeChar2 = 0;
try {
unicodeChar2 = 0xFFFF & tableToUnicode[ebcdicChar];
} catch (ArrayIndexOutOfBoundsException e) {
System.out.println("ERROR.. ArrayIndexOutOfBounds");
System.out.println("ebcdicChar=0x"
+ Integer.toHexString((int) ebcdicChar));
System.out.println("i=" + i);
System.out.println("unicodeChar=0x"
+ Integer.toHexString((int) unicodeChar));
throw e;
}
if (unicodeChar2 == unicodeChar) {
sb2.append("Secondary EBCDIC mapping " + ebcdicPrefix + "'"
+ Integer.toHexString(i) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar) + "'" + " -> "
+ ebcdicPrefix + "'" + Integer.toHexString(ebcdicChar) + "'"
+ " -> UX'" + Integer.toHexString(unicodeChar2) + "'\n");
} else {
sb3.append("EBCDIC RoundTrip Failure2 " + ebcdicPrefix + "'"
+ Integer.toHexString(i) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar) + "'" + " -> "
+ ebcdicPrefix + "'" + Integer.toHexString(ebcdicChar) + "'"
+ " -> UX'" + Integer.toHexString(unicodeChar2) + "'\n");
passed = false;
}
}
}
}
}
System.out.println(sb2);
System.out.println(sb1);
System.out.println(sb3);
sb1.setLength(0);
sb2.setLength(0);
sb3.setLength(0);
for (int i = 0; i < tableToEbcdic.length; i++) {
int ebcdicChar;
if (ebcdicIsDBCS) {
ebcdicChar = 0xFFFF & tableToEbcdic[i];
} else {
int piece = 0xFFFF & tableToEbcdic[i / 2];
if (i % 2 == 0) {
ebcdicChar = piece >> 8;
} else {
ebcdicChar = piece & 0xFF;
}
}
if ((ebcdicChar != 0xfefe) && (ebcdicChar != 0x3f)) {
int unicodeChar = 0xFFFF & tableToUnicode[ebcdicChar];
if (i != unicodeChar) {
if (unicodeChar == 0xFFFD) {
sb1.append("Unicode RoundTrip Failure UX'" + Integer.toHexString(i)
+ "'" + " -> " + ebcdicPrefix + "'"
+ Integer.toHexString(ebcdicChar) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar) + "'\n");
passed = false;
} else {
int ebcdicChar2;
if (ebcdicIsDBCS) {
ebcdicChar2 = 0xFFFF & tableToEbcdic[unicodeChar];
} else {
int piece = 0xFFFF & tableToEbcdic[unicodeChar / 2];
if (unicodeChar % 2 == 0) {
ebcdicChar2 = piece >> 8;
} else {
ebcdicChar2 = piece & 0xFF;
}
}
if (ebcdicChar2 == ebcdicChar) {
sb2.append("Secondary Unicode mapping UX'"
+ Integer.toHexString(i) + "'" + " -> " + ebcdicPrefix + "'"
+ Integer.toHexString(ebcdicChar) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar) + "'" + " -> "
+ ebcdicPrefix + "'" + Integer.toHexString(ebcdicChar2)
+ "'\n");
} else {
sb3.append("Unicode RoundTrip Failure2 UX'"
+ Integer.toHexString(i) + "'" + " -> " + ebcdicPrefix + "'"
+ Integer.toHexString(ebcdicChar) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar) + "'" + " -> "
+ ebcdicPrefix + "'" + Integer.toHexString(ebcdicChar2)
+ "'\n");
passed = false;
}
}
}
}
}
System.out.println(sb2);
System.out.println(sb1);
System.out.println(sb3);
return passed;
}
private static final char repSig = '\uFFFF'; // compression indication
// character
private static final char cic_ = repSig;
private static final char rampSig = '\uFFFE'; // ramp indication character
private static final char ric_ = rampSig;
private static final char hbSig = '\u0000'; // high-byte compression
// indication character
private static final char pad = '\u0000'; // pad character
static int repeatCheck(char[] arr, int startingIndex) {
int index = startingIndex + 1;
while (index < arr.length && arr[index] == arr[index - 1]) {
++index;
}
return (index - startingIndex);
}
static final int rampCheck(char[] arr, int startingIndex) {
int index = startingIndex + 1;
while (index < arr.length && arr[index] == arr[index - 1] + 1) {
++index;
}
return (index - startingIndex);
}
static int hbCheck(char[] arr, int startingIndex) {
int index = startingIndex + 1;
while (index < arr.length) {
// check for repeat
// for 6 repeated chars, we'd need either 3 hb-compressed chars or 3
// repeatsig chars, so it's a toss up
if (repeatCheck(arr, index) > 6)
return (index - startingIndex); // at this point though, it's better to
// stop and do the repeat
// check for ramp, same reason
if (rampCheck(arr, index) > 6)
return (index - startingIndex);
// OK, finally check for hb
if ((arr[index] & 0xFF00) != (arr[index - 1] & 0xFF00))
return (index - startingIndex);
++index;
}
return (index - startingIndex);
}
static int numRepeats;
static int numRamps;
static int hbRepeats;
static int charRepeats;
// This is the new way - 05/04/2000.
static char[] compressBetter(char[] arr) {
numRepeats = 0;
numRamps = 0;
hbRepeats = 0;
charRepeats = 0;
// This uses the "correct" compression scheme from my invention disclosure
// It also employs high-byte compression, something that I did not include
// in my disclosure.
StringBuffer buf = new StringBuffer();
for (int i = 0; i < arr.length; ++i) {
int repNum = repeatCheck(arr, i);
if (repNum > 3) // had enough repeats
{
numRepeats++;
buf.append(repSig);
buf.append((char) repNum);
buf.append(arr[i]);
i += repNum - 1;
} else {
int rampNum = rampCheck(arr, i);
if (rampNum > 3) // had enough in the ramp
{
numRamps++;
buf.append(rampSig);
buf.append((char) rampNum);
buf.append(arr[i]);
i += rampNum - 1;
} else {
int hbNum = hbCheck(arr, i);
--hbNum; // don't include the first char, since we always append it.
if (hbNum >= 6) {
// System.out.print("HBNUM is "+Integer.toHexString((int)hbNum)+"; ");
hbRepeats++;
// pattern is this: ss ss nn nn hh tt xx xx xx xx ...
// where ss ss is hbSig
// nn nn is hbNum
// hh tt is the first char (hh is the repeated high byte)
// xx is the lower byte of the next char in the sequence
// xx repeats hbNum/2 times so that
// hbNum is the total number of repeated db chars in the ciphertext,
// not including the first char.
// Note that there may be, in actuality, hbNum*2 +1 chars in the
// cleartext that fit into the
// conversion, but since we'd have to fill out the last char with an
// empty byte, there's no point
// in doing it anyway. Besides, it might be able to be compressed
// via another scheme with itself as
// the starting character.
// int start = i;
buf.append(hbSig);
if (hbNum % 2 == 1) // odd number
{
--hbNum; // no point in doing the last char
}
// System.out.println("Appending "+Integer.toHexString((int)((char)(hbNum/2))));
buf.append((char) (hbNum / 2)); // hbNum is always even, so this
// comes out.
// System.out.print("hb comp: "+Integer.toHexString(hbNum)+": ");
// for (int b=0; b>> 8));
char c2 = (char) (highByteMask + (0x00FF & both));
buf.append(c1);
buf.append(c2);
// System.out.print(Integer.toHexString((int)c1)+" "+Integer.toHexString((int)c2)+" ");
}
// System.out.println(Integer.toHexString((int)arr[i+hbNum]));
i = i + hbNum - 1;
}
} else {
buf.append(arr[i]);
charRepeats++;
}
}
System.out.println("Decompression stats: " + numRepeats + " repeats, "
+ numRamps + " ramps, " + hbRepeats + " highbytes, " + charRepeats
+ " regular.");
numRepeats = 0;
numRamps = 0;
hbRepeats = 0;
charRepeats = 0;
return buf.toString().toCharArray();
}
// This is the old way
static char[] compress(char[] arr) {
if (arr.length < 3)
return arr;
StringBuffer buf = new StringBuffer();
char oldold = arr[0];
char old = arr[1];
int count = 0;
boolean inCompression = false; // this flags if we are repeating the same
// character
boolean inRamp = false; // this flags if each subsequent characters is the
// previous character + 1
for (int i = 2; i < arr.length; ++i) {
if (!inCompression && !inRamp) {
if (arr[i] == old && arr[i] == oldold) // Check for repeating
{
inCompression = true;
buf.append(cic_);
buf.append(oldold);
count = 3;
} else if (arr[i] == old + 1 && arr[i] == oldold + 2) // Check for ramp
{
inRamp = true;
buf.append(ric_);
buf.append(oldold);
} else if (oldold == cic_) // Check for duplicate cic
{
buf.append(cic_);
} else if (oldold == ric_) // Check for duplicate ric
{
buf.append(ric_);
} else // Just copy it normal
{
buf.append(oldold);
}
oldold = old;
old = arr[i];
} else if (inCompression) {
if (arr[i] == old && arr[i] == oldold) // Still repeating?
{
++count;
oldold = old;
old = arr[i];
} else // Not repeating anymore
{
inCompression = false;
char c = (char) count;
if (count == 0x0008)
c = '\b';
else if (count == 0x0009)
c = '\t';
// else if(count == 0x000A) c = '\r'; // Had trouble with 0x0A and
// 0x0D getting messed up.
else if (count == 0x000A)
c = '\n'; // Had trouble with 0x0A and 0x0D getting messed up.
else if (count == 0x000C)
c = '\f';
// else if(count == 0x000D) c = '\n';
else if (count == 0x000D)
c = '\r';
else if (count == 0x0022)
c = '\"';
else if (count == 0x0027)
c = '\'';
else if (count == 0x005C)
c = '\\';
buf.append(c);
oldold = arr[i++]; // yes this is right... think about it.
old = arr[i];
}
} else { // must be in ramp
if (arr[i] == old + 1 && arr[i] == oldold + 2) {
oldold = old;
old = arr[i];
} else {
inRamp = false;
buf.append(old);
oldold = arr[i++];
old = arr[i];
}
}
}
if (inCompression) {
char c = (char) count;
if (count == 0x0008)
c = '\b';
else if (count == 0x0009)
c = '\t';
else if (count == 0x000A)
c = '\n';
else if (count == 0x000C)
c = '\f';
// else if(count == 0x000D) c = '\n';
else if (count == 0x000D)
c = '\r';
else if (count == 0x0022)
c = '\"';
else if (count == 0x0027)
c = '\'';
else if (count == 0x005C)
c = '\\';
buf.append(c);
}
if (inRamp) {
buf.append(old);
}
return buf.toString().toCharArray();
}
static void writeHeader(FileWriter f, int ccsid, String system)
throws Exception {
int dotIndex = system.indexOf('.');
if (dotIndex > 0) {
system = system.substring(0, dotIndex);
}
Date currentDate = new Date();
// Look up the version dynamically
Class copyrightClass = Copyright.class;
Field field = copyrightClass.getField("version");
String jtopenVersion = (String) field.get(null);
f.write("///////////////////////////////////////////////////////////////////////////////\n");
f.write("//\n");
f.write("// JTOpen (IBM Toolbox for Java - OSS version)\n");
f.write("//\n");
f.write("// Filename: ConvTable" + ccsid + ".java\n");
f.write("//\n");
f.write("// The source code contained herein is licensed under the IBM Public License\n");
f.write("// Version 1.0, which has been approved by the Open Source Initiative.\n");
f.write("// Copyright (C) 1997-2016 International Business Machines Corporation and\n");
f.write("// others. All rights reserved.\n");
f.write("//\n");
f.write("// Generated " + currentDate + " from " + system + "\n");
f.write("// Using " + jtopenVersion + "\n");
f.write("///////////////////////////////////////////////////////////////////////////////\n\n");
f.write("package com.ibm.as400.access;\n\n");
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy