src.com.ibm.as400.access.GenerateConverterTable Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of jt400 Show documentation
Show all versions of jt400 Show documentation
The Open Source version of the IBM Toolbox for Java
///////////////////////////////////////////////////////////////////////////////
//
// JTOpen (IBM Toolbox for Java - OSS version)
//
// Filename: AS400.java
//
// The source code contained herein is licensed under the IBM Public License
// Version 1.0, which has been approved by the Open Source Initiative.
// Copyright (C) 1997-2016 International Business Machines Corporation and
// others. All rights reserved.
//
///////////////////////////////////////////////////////////////////////////////
package com.ibm.as400.access;
import java.io.*;
import java.lang.reflect.Field;
import java.util.*;
public class GenerateConverterTable
{
private static final String copyright = "Copyright (C) 1997-2016 International Business Machines Corporation and others.";
static AS400 sys = null;
static boolean compress_ = true; // Compress the conversion table
// Note: turn this off for debugging purposes
static boolean codePointPerLine_ = false; // Should only 1 code point be printed per line
static boolean ascii_ = false; // Indicates if listed ccsids are ascii tables or not
static boolean bidi_ = false; // Indicates if listed ccsids are bidi tables or not
// Note: bidi_ and ascii_ cannot both be true
static boolean showOffsets_ = false; // Indicates of the offsets should be printed in the tables
public static void main(String[] args)
{
if (args.length < 4)
{
System.out.println("Usage: java com.ibm.as400.access.GenerateConverterTable system uid pwd [-nocompress] [-ascii] [-bidi] [-showOffsets] [-codePointPerLine] ccsid [ccsid2] [ccsid3] [ccsid4] ...");
System.exit(0);
}
try
{
sys = new AS400(args[0], args[1], args[2]);
sys.connectService(AS400.CENTRAL);
}
catch (Exception e)
{
e.printStackTrace();
System.exit(0);
}
int start = 3;
if (args[start].equals("-nocompress"))
{
compress_ = false;
++start;
}
if (args[start].equals("-ascii"))
{
ascii_ = true;
++start;
}
if (args[start].equals("-bidi"))
{
bidi_ = true;
++start;
}
if (args[start].equals("-showOffsets"))
{
showOffsets_ = true;
++start;
}
if (args[start].equals("-codePointPerLine"))
{
codePointPerLine_ = true;
++start;
}
for (int i=start; i13488->1089; use 61952 instead, since it would be the same anyway.
{
System.out.println("Special case for ccsid 1089.");
System.out.println("Retrieving "+ccsid+"->61952 table...");
tableToUnicode = down.download(ccsid, 61952, false);
}
else
{
System.out.println("Retrieving "+ccsid+"->13488 table...");
tableToUnicode = down.download(ccsid, 13488, false);
}
if (tableToUnicode == null || tableToUnicode.length == 0)
{
System.out.println(ccsid+" must be double-byte. Performing secondary retrieve of "+ccsid+"->1200 table...");
table1IsDBCS = true;
down.disconnect();
down.connect();
tableToUnicode = down.download(ccsid, 1200, true);
}
System.out.println(" Size: "+tableToUnicode.length);
if (tableToUnicode.length > 65536) {
System.out.println("Size is > 65536. Fixing table");
int next = 0;
int from = 0;
char[] newTable = new char[65536];
while (from < tableToUnicode.length && next < 65536) {
int c = 0xFFFF & (int) tableToUnicode[from];
if (next > 0xECAA && next <= 0xECD0) {
System.out.println("Next=0x"+Integer.toHexString(next)+" to="+Integer.toHexString(c));
}
int nextchar = 0;
if (from +1 < tableToUnicode.length) {
nextchar = 0xFFFF & (int) tableToUnicode[from+1];
}
if (
// in surrogate range
((c >= 0xD800) && (c <=0xDFFF)) ||
// Uses combining character
(nextchar == 0x309A) ||
(c != 0xFFfd && nextchar == 0x300) ||
(c != 0xffd && c != 0x300 && nextchar == 0x301) ||
// Weird cases..
(c == 0x2e5 && nextchar == 0x2e9) ||
(c == 0x2e9 && nextchar == 0x2e5)) {
// Mark as surrogate
newTable[next]=(char) 0xD800;
// add to surrogate table
if (surrogateTable == null) {
surrogateTable = new char[65536][];
}
char[] pair = new char[2];
surrogateTable[next] = pair;
pair[0] = (char) (0xFFFF & (int) tableToUnicode[from]);
pair[1] = (char) (0xFFFF & (int) tableToUnicode[from+1]);
/* System.out.println("Warning: Sub at offset "+Integer.toHexString(next)+" for "+Integer.toHexString(0xFFFF & (int) table1[from])+" "+Integer.toHexString(0xFFFF & (int) table1[from+1])); */
from +=2;
} else {
newTable[next]=(char) c;
from++;
}
next++;
}
tableToUnicode = newTable;
}
down.disconnect();
down.connect();
if (ccsid == 1089)
{
System.out.println("Special case for ccsid 1089.");
System.out.println("Retrieving 61952->"+ccsid+" table...");
tableToEbcdic = down.download(61952, ccsid, true);
}
else
{
/* Use 1200 instead of 13488 */
System.out.println("Retrieving 1200->"+ccsid+" table...");
tableToEbcdic = down.download(1200, ccsid, true);
}
System.out.println(" Size: "+tableToEbcdic.length);
sys.disconnectAllServices();
}
catch (Exception e)
{
e.printStackTrace();
}
// Verify the mapping
verifyRoundTrip(tableToUnicode, tableToEbcdic, table1IsDBCS);
System.out.println("****************************************");
System.out.println("Verify round 2 ");
System.out.println("****************************************");
verifyRoundTrip(tableToUnicode, tableToEbcdic, table1IsDBCS);
// Compress the ccsid table
if (table1IsDBCS)
{
if (compress_)
{
System.out.println("Compressing "+ccsid+"->13488 conversion table...");
char[] arr = compress(tableToUnicode);
System.out.println("Old compression length: "+arr.length+" characters.");
char[] temparr = compressBetter(tableToUnicode);
System.out.println("New compression length: "+temparr.length+" characters.");
if (temparr.length > arr.length)
{
System.out.println("WARNING: New algorithm WORSE than old algorithm!");
}
System.out.println("Verifying compressed table...");
arr = decompressBetter(temparr);
if (arr.length != tableToUnicode.length)
{
System.out.println("Verification failed, lengths not equal: "+arr.length+" != "+tableToUnicode.length);
int c = 0;
while (c < arr.length && arr[c] == tableToUnicode[c]) ++c;
System.out.println("First mismatch at index "+c+": "+(int)arr[c]+" != "+(int)tableToUnicode[c]);
}
else
{
boolean bad = false;
for (int c=0; c"+ccsid+" conversion table...");
char[] arr = compress(tableToEbcdic);
System.out.println("Old compression length: "+arr.length+" characters.");
char[] temparr = compressBetter(tableToEbcdic);
System.out.println("New compression length: "+temparr.length+" characters.");
if (temparr.length > arr.length)
{
System.out.println("WARNING: New algorithm WORSE than old algorithm!");
}
System.out.println("Verifying compressed table...");
arr = decompressBetter(temparr);
if (arr.length != tableToEbcdic.length)
{
System.out.println("Verification failed, lengths not equal: "+arr.length+" != "+tableToEbcdic.length);
int c = 0;
while (c < arr.length && arr[c] == tableToEbcdic[c]) ++c;
System.out.println("First mismatch at index "+c+": "+(int)arr[c]+" != "+(int)tableToEbcdic[c]);
tableToEbcdic = temparr;
}
else
{
boolean bad = false;
for (int c=0; c UX'"+Integer.toHexString(unicodeChar)+"'"+
" -> GX'"+Integer.toHexString(ebcdicChar)+"'\n");
tableToEbcdic[unicodeChar] = (char) i;
passed = false;
} else {
if (ebcdicChar >= tableToUnicode.length) {
sb2.append("Invalid ebcdic char "+Integer.toHexString(ebcdicChar)+" unicodeChar="+Integer.toHexString(unicodeChar)+ " originalEbcdic="+Integer.toHexString(i)+"\n");
passed =false;
} else {
int unicodeChar2 = 0xFFFF & tableToUnicode[ebcdicChar];
if (unicodeChar2 == unicodeChar) {
sb2.append("Secondary EBCDIC mapping GX'"
+ Integer.toHexString(i) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar) + "'" + " -> GX'"
+ Integer.toHexString(ebcdicChar) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar2) + "'\n");
} else {
sb3.append("EBCDIC RoundTrip Failure2 GX'"
+ Integer.toHexString(i) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar) + "'" + " -> GX'"
+ Integer.toHexString(ebcdicChar) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar2) + "'\n");
passed = false;
}
}
}
}
}
}
System.out.println(sb2);
System.out.println(sb1);
System.out.println(sb3);
sb1.setLength(0);
sb2.setLength(0);
sb3.setLength(0);
for (int i = 0; i < tableToEbcdic.length; i++) {
int ebcdicChar = 0xFFFF & tableToEbcdic[i];
if (ebcdicChar != 0xfefe && ebcdicChar != 0x3f ) {
if (ebcdicChar > tableToUnicode.length) {
sb1.append("Unicode RoundTrip Failure UX'"
+ Integer.toHexString(i) + "'" + " -> GX'"
+ Integer.toHexString(ebcdicChar) + "' -> IndxOutOfBounds\n");
} else {
int unicodeChar = 0xFFFF & tableToUnicode[ebcdicChar];
if (i != unicodeChar) {
if (unicodeChar == 0xFFFD) {
sb1.append("Unicode RoundTrip Failure UX'"
+ Integer.toHexString(i) + "'" + " -> GX'"
+ Integer.toHexString(ebcdicChar) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar) + "'\n");
passed = false;
} else {
int ebcdicChar2 = 0xFFFF & tableToEbcdic[unicodeChar];
if (ebcdicChar2 == ebcdicChar) {
sb2.append("Secondary Unicode mapping UX'"
+ Integer.toHexString(i) + "'" + " -> GX'"
+ Integer.toHexString(ebcdicChar) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar) + "'" + " -> GX'"
+ Integer.toHexString(ebcdicChar2) + "'\n");
} else {
sb3.append("Unicode RoundTrip Failure2 UX'"
+ Integer.toHexString(i) + "'" + " -> GX'"
+ Integer.toHexString(ebcdicChar) + "'" + " -> UX'"
+ Integer.toHexString(unicodeChar) + "'" + " -> GX'"
+ Integer.toHexString(ebcdicChar2) + "'\n");
passed = false;
}
}
}
}
}
}
System.out.println(sb2);
System.out.println(sb1);
System.out.println(sb3);
return passed;
}
private static final char repSig = '\uFFFF'; // compression indication character
private static final char cic_ = repSig;
private static final char rampSig = '\uFFFE'; // ramp indication character
private static final char ric_ = rampSig;
private static final char hbSig = '\u0000'; // high-byte compression indication character
private static final char pad = '\u0000'; // pad character
static int repeatCheck(char[] arr, int startingIndex)
{
int index = startingIndex+1;
while (index < arr.length && arr[index] == arr[index-1])
{
++index;
}
return(index-startingIndex);
}
static final int rampCheck(char[] arr, int startingIndex)
{
int index = startingIndex+1;
while (index < arr.length && arr[index] == arr[index-1]+1)
{
++index;
}
return(index-startingIndex);
}
static int hbCheck(char[] arr, int startingIndex)
{
int index = startingIndex+1;
while (index < arr.length)
{
// check for repeat
// for 6 repeated chars, we'd need either 3 hb-compressed chars or 3 repeatsig chars, so it's a toss up
if (repeatCheck(arr, index) > 6) return(index-startingIndex); // at this point though, it's better to stop and do the repeat
// check for ramp, same reason
if (rampCheck(arr, index) > 6) return(index-startingIndex);
// OK, finally check for hb
if ((arr[index] & 0xFF00) != (arr[index-1] & 0xFF00)) return(index-startingIndex);
++index;
}
return(index-startingIndex);
}
static int numRepeats;
static int numRamps;
static int hbRepeats;
static int charRepeats;
// This is the new way - 05/04/2000.
static char[] compressBetter(char[] arr)
{
numRepeats = 0;
numRamps = 0;
hbRepeats = 0;
charRepeats = 0;
// This uses the "correct" compression scheme from my invention disclosure
// It also employs high-byte compression, something that I did not include in my disclosure.
StringBuffer buf = new StringBuffer();
for (int i=0; i 3) // had enough repeats
{
numRepeats++;
buf.append(repSig);
buf.append((char)repNum);
buf.append(arr[i]);
i += repNum-1;
}
else
{
int rampNum = rampCheck(arr, i);
if (rampNum > 3) // had enough in the ramp
{
numRamps++;
buf.append(rampSig);
buf.append((char)rampNum);
buf.append(arr[i]);
i += rampNum-1;
}
else
{
int hbNum = hbCheck(arr, i);
--hbNum; // don't include the first char, since we always append it.
if (hbNum >= 6)
{
// System.out.print("HBNUM is "+Integer.toHexString((int)hbNum)+"; ");
hbRepeats++;
// pattern is this: ss ss nn nn hh tt xx xx xx xx ...
// where ss ss is hbSig
// nn nn is hbNum
// hh tt is the first char (hh is the repeated high byte)
// xx is the lower byte of the next char in the sequence
// xx repeats hbNum/2 times so that
// hbNum is the total number of repeated db chars in the ciphertext, not including the first char.
// Note that there may be, in actuality, hbNum*2 +1 chars in the cleartext that fit into the
// conversion, but since we'd have to fill out the last char with an empty byte, there's no point
// in doing it anyway. Besides, it might be able to be compressed via another scheme with itself as
// the starting character.
// int start = i;
buf.append(hbSig);
if (hbNum % 2 == 1) // odd number
{
--hbNum; // no point in doing the last char
}
// System.out.println("Appending "+Integer.toHexString((int)((char)(hbNum/2))));
buf.append((char)(hbNum/2)); // hbNum is always even, so this comes out.
// System.out.print("hb comp: "+Integer.toHexString(hbNum)+": ");
// for (int b=0; b>> 8));
char c2 = (char)(highByteMask + (0x00FF & both));
buf.append(c1);
buf.append(c2);
// System.out.print(Integer.toHexString((int)c1)+" "+Integer.toHexString((int)c2)+" ");
}
// System.out.println(Integer.toHexString((int)arr[i+hbNum]));
i = i + hbNum - 1;
}
}
else
{
buf.append(arr[i]);
charRepeats++;
}
}
System.out.println("Decompression stats: "+numRepeats+" repeats, "+numRamps+" ramps, "+hbRepeats+" highbytes, "+charRepeats+" regular.");
numRepeats = 0;
numRamps = 0;
hbRepeats = 0;
charRepeats = 0;
return buf.toString().toCharArray();
}
// This is the old way
static char[] compress(char[] arr)
{
if (arr.length < 3) return arr;
StringBuffer buf = new StringBuffer();
char oldold = arr[0];
char old = arr[1];
int count = 0;
boolean inCompression = false; // this flags if we are repeating the same character
boolean inRamp = false; // this flags if each subsequent characters is the previous character + 1
for (int i=2; i 0) {
system=system.substring(0,dotIndex);
}
Date currentDate = new Date();
// Look up the version dynamically
Class copyrightClass = Copyright.class;
Field field = copyrightClass.getField("version");
String jtopenVersion = (String) field.get(null);
f.write("///////////////////////////////////////////////////////////////////////////////\n");
f.write("//\n");
f.write("// JTOpen (IBM Toolbox for Java - OSS version)\n");
f.write("//\n");
f.write("// Filename: ConvTable"+ccsid+".java\n");
f.write("//\n");
f.write("// The source code contained herein is licensed under the IBM Public License\n");
f.write("// Version 1.0, which has been approved by the Open Source Initiative.\n");
f.write("// Copyright (C) 1997-2016 International Business Machines Corporation and\n");
f.write("// others. All rights reserved.\n");
f.write("//\n");
f.write("// Generated "+currentDate+" from "+system+"\n");
f.write("// Using "+jtopenVersion+"\n");
f.write("///////////////////////////////////////////////////////////////////////////////\n\n");
f.write("package com.ibm.as400.access;\n\n");
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy