View Javadoc
1   /*
2   Copyright (c) 2008 Health Market Science, Inc.
3   
4   Licensed under the Apache License, Version 2.0 (the "License");
5   you may not use this file except in compliance with the License.
6   You may obtain a copy of the License at
7   
8       http://www.apache.org/licenses/LICENSE-2.0
9   
10  Unless required by applicable law or agreed to in writing, software
11  distributed under the License is distributed on an "AS IS" BASIS,
12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  See the License for the specific language governing permissions and
14  limitations under the License.
15  */
16  
17  package com.healthmarketscience.jackcess.impl;
18  
19  import java.io.File;
20  import java.lang.reflect.Field;
21  import java.nio.ByteBuffer;
22  import java.util.Arrays;
23  import java.util.HashMap;
24  import java.util.Map;
25  import java.util.TreeMap;
26  import java.util.regex.Matcher;
27  import java.util.regex.Pattern;
28  
29  import com.healthmarketscience.jackcess.ColumnBuilder;
30  import com.healthmarketscience.jackcess.Cursor;
31  import com.healthmarketscience.jackcess.CursorBuilder;
32  import com.healthmarketscience.jackcess.DataType;
33  import com.healthmarketscience.jackcess.Database;
34  import com.healthmarketscience.jackcess.DateTimeType;
35  import com.healthmarketscience.jackcess.Index;
36  import com.healthmarketscience.jackcess.Row;
37  import com.healthmarketscience.jackcess.Table;
38  import com.healthmarketscience.jackcess.TableBuilder;
39  import static com.healthmarketscience.jackcess.impl.JetFormatTest.*;
40  import junit.framework.TestCase;
41  import static com.healthmarketscience.jackcess.TestUtil.*;
42  
43  /**
44   * @author James Ahlborn
45   */
46  public class IndexCodesTest extends TestCase {
47  
48    private static final Map<Character,String> SPECIAL_CHARS =
49      new HashMap<Character,String>();
50    static {
51      SPECIAL_CHARS.put('\b', "\\b");
52      SPECIAL_CHARS.put('\t', "\\t");
53      SPECIAL_CHARS.put('\n', "\\n");
54      SPECIAL_CHARS.put('\f', "\\f");
55      SPECIAL_CHARS.put('\r', "\\r");
56      SPECIAL_CHARS.put('\"', "\\\"");
57      SPECIAL_CHARS.put('\'', "\\'");
58      SPECIAL_CHARS.put('\\', "\\\\");
59    }
60  
61    public IndexCodesTest(String name) throws Exception {
62      super(name);
63    }
64  
65    public void testIndexCodes() throws Exception
66    {
67      doTestDb(Basename.INDEX_CODES);
68    }
69  
70    public void testEmoticons() throws Exception
71    {
72      doTestDb(Basename.EMOTICONS);
73    }
74  
75    private static void doTestDb(Basename dbBaseName) throws Exception
76    {
77      for (final TestDB testDB : TestDB.getSupportedForBasename(dbBaseName, true)) {
78        Database db = openMem(testDB);
79        db.setDateTimeType(DateTimeType.DATE);
80  
81        for(Table t : db) {
82          for(Index index : t.getIndexes()) {
83    //         System.out.println("Checking " + t.getName() + "." + index.getName());
84            checkIndexEntries(testDB, t, index);
85          }
86        }
87  
88        db.close();
89      }
90    }
91  
92    public static void checkIndexEntries(final TestDB testDB, Table t, Index index) throws Exception
93    {
94  //         index.initialize();
95  //         System.out.println("Ind " + index);
96  
97      Cursor cursor = CursorBuilder.createCursor(index);
98      while(cursor.moveToNextRow()) {
99  
100       Row row = cursor.getCurrentRow();
101 
102       Object data = row.get("data");
103       if((testDB.getExpectedFileFormat() == Database.FileFormat.V1997) &&
104          (data instanceof String) && ((String)data).contains("\uFFFD")) {
105         // this row has a character not supported in the v1997 charset
106         continue;
107       }
108 
109       Cursor.Position curPos = cursor.getSavepoint().getCurrentPosition();
110       boolean success = false;
111       try {
112         findRow(testDB, t, index, row, curPos);
113         success = true;
114       } finally {
115         if(!success) {
116           System.out.println("CurPos: " + curPos);
117           System.out.println("Value: " + row + ": " +
118                              toUnicodeStr(row.get("data")));
119         }
120       }
121     }
122 
123   }
124 
125   private static void findRow(final TestDB testDB, Table t, Index index,
126                               Row expectedRow,
127                               Cursor.Position expectedPos)
128     throws Exception
129   {
130     Object[] idxRow = ((IndexImpl)index).constructIndexRow(expectedRow);
131     Cursor cursor = CursorBuilder.createCursor(index, idxRow, idxRow);
132 
133     Cursor.Position startPos = cursor.getSavepoint().getCurrentPosition();
134 
135     cursor.beforeFirst();
136     while(cursor.moveToNextRow()) {
137       Row row = cursor.getCurrentRow();
138       if(expectedRow.equals(row)) {
139         // verify that the entries are indeed equal
140         Cursor.Position curPos = cursor.getSavepoint().getCurrentPosition();
141         assertEquals(entryToString(expectedPos), entryToString(curPos));
142         return;
143       }
144     }
145 
146     // TODO long rows not handled completely yet in V2010
147     // seems to truncate entry at 508 bytes with some trailing 2 byte seq
148     if((testDB != null) &&
149        (testDB.getExpectedFileFormat() == Database.FileFormat.V2010)) {
150       String rowId = expectedRow.getString("name");
151       String tName = t.getName();
152       if(("Table11".equals(tName) || "Table11_desc".equals(tName)) &&
153          ("row10".equals(rowId) || "row11".equals(rowId) ||
154           "row12".equals(rowId))) {
155         System.out.println(
156             "TODO long rows not handled completely yet in V2010: " + tName +
157                            ", " + rowId);
158         return;
159       }
160     }
161 
162     fail("testDB: " + testDB + ";\nCould not find expected row " + expectedRow + " starting at " +
163          entryToString(startPos));
164   }
165 
166 
167   //////
168   //
169   // The code below is for use in reverse engineering index entries.
170   //
171   //////
172 
173   public void testNothing() throws Exception {
174     // keep this so build doesn't fail if other tests are disabled
175   }
176 
177   public void x_testCreateIsoFile() throws Exception
178   {
179     Database db = create(Database.FileFormat.V2000, true);
180 
181     Table t = new TableBuilder("test")
182       .addColumn(new ColumnBuilder("row", DataType.TEXT))
183       .addColumn(new ColumnBuilder("data", DataType.TEXT))
184       .toTable(db);
185 
186     for(int i = 0; i < 256; ++i) {
187       String str = "AA" + ((char)i) + "AA";
188       t.addRow("row" + i, str);
189     }
190 
191     db.close();
192   }
193 
194   public void x_testCreateAltIsoFile() throws Exception
195   {
196     Database db = openCopy(Database.FileFormat.V2000, new File("/tmp/test_ind.mdb"), true);
197 
198     Table t = db.getTable("Table1");
199 
200     for(int i = 0; i < 256; ++i) {
201       String str = "AA" + ((char)i) + "AA";
202       t.addRow("row" + i, str,
203                (byte)42 + i, (short)53 + i, 13 * i,
204                (6.7d / i), null, null, true);
205     }
206 
207     db.close();
208   }
209 
210   public void x_testWriteAllCodesMdb() throws Exception
211   {
212     Database db = create(Database.FileFormat.V2000, true);
213 
214 //     Table t = new TableBuilder("Table1")
215 //       .addColumn(new ColumnBuilder("key", DataType.TEXT))
216 //       .addColumn(new ColumnBuilder("data", DataType.TEXT))
217 //       .toTable(db);
218 
219 //     for(int i = 0; i <= 0xFFFF; ++i) {
220 //       // skip non-char chars
221 //       char c = (char)i;
222 //       if(Character.isHighSurrogate(c) || Character.isLowSurrogate(c)) {
223 //         continue;
224 //       }
225 //       String key = toUnicodeStr(c);
226 //       String str = "AA" + c + "AA";
227 //       t.addRow(key, str);
228 //     }
229 
230     Table t = new TableBuilder("Table5")
231       .addColumn(new ColumnBuilder("name", DataType.TEXT))
232       .addColumn(new ColumnBuilder("data", DataType.TEXT))
233       .toTable(db);
234 
235     char c = (char)0x3041;   // crazy 7F 02 ... A0
236     char c2 = (char)0x30A2;  // crazy 7F 02 ...
237     char c3 = (char)0x2045;  // inat 27 ... 1C
238     char c4 = (char)0x3043;  // crazy 7F 03 ... A0
239     char c5 = (char)0x3046;  // crazy 7F 04 ...
240     char c6 = (char)0x30F6;  // crazy 7F 0D ... A0
241     char c7 = (char)0x3099;  // unprint 03
242     char c8 = (char)0x0041;  // A
243     char c9 = (char)0x002D;  // - (unprint)
244     char c10 = (char)0x20E1; // unprint F2
245     char c11 = (char)0x309A; // unprint 04
246     char c12 = (char)0x01C4; // (long extra)
247     char c13 = (char)0x005F; // _ (long inline)
248     char c14 = (char)0xFFFE; // removed
249 
250     char[] cs = new char[]{c7, c8, c3, c12, c13, c14, c, c2, c9};
251     addCombos(t, 0, "", cs, 5);
252 
253 //     t = new TableBuilder("Table2")
254 //       .addColumn(new ColumnBuilder("data", DataType.TEXT))
255 //       .toTable(db);
256 
257 //     writeChars(0x0000, t);
258 
259 //     t = new TableBuilder("Table3")
260 //       .addColumn(new ColumnBuilder("data", DataType.TEXT))
261 //       .toTable(db);
262 
263 //     writeChars(0x0400, t);
264 
265 
266     db.close();
267   }
268 
269   public void x_testReadAllCodesMdb() throws Exception
270   {
271 //     Database db = openCopy(new File("/data2/jackcess_test/testAllIndexCodes.mdb"));
272 //     Database db = openCopy(new File("/data2/jackcess_test/testAllIndexCodes_orig.mdb"));
273 //     Database db = openCopy(new File("/data2/jackcess_test/testSomeMoreCodes.mdb"));
274     Database db = openCopy(Database.FileFormat.V2000, new File("/data2/jackcess_test/testStillMoreCodes.mdb"));
275     Table t = db.getTable("Table5");
276 
277     Index ind = t.getIndexes().iterator().next();
278     ((IndexImpl)ind).initialize();
279 
280     System.out.println("Ind " + ind);
281 
282     Cursor cursor = CursorBuilder.createCursor(ind);
283     while(cursor.moveToNextRow()) {
284       System.out.println("=======");
285       String entryStr =
286         entryToString(cursor.getSavepoint().getCurrentPosition());
287       System.out.println("Entry Bytes: " + entryStr);
288       System.out.println("Value: " + cursor.getCurrentRow() + "; " +
289                          toUnicodeStr(cursor.getCurrentRow().get("data")));
290     }
291 
292     db.close();
293   }
294 
295   private int addCombos(Table t, int rowNum, String s, char[] cs, int len)
296     throws Exception
297   {
298     if(s.length() >= len) {
299       return rowNum;
300     }
301 
302     for(int i = 0; i < cs.length; ++i) {
303       String name = "row" + (rowNum++);
304       String ss = s + cs[i];
305       t.addRow(name, ss);
306       rowNum = addCombos(t, rowNum, ss, cs, len);
307     }
308 
309     return rowNum;
310   }
311 
312   private void writeChars(int hibyte, Table t) throws Exception
313   {
314     char other = (char)(hibyte | 0x41);
315     for(int i = 0; i < 0xFF; ++i) {
316       char c = (char)(hibyte | i);
317       String str = "" + other + c + other;
318       t.addRow(str);
319     }
320   }
321 
322   public void x_testReadIsoMdb() throws Exception
323   {
324 //     Database db = open(new File("/tmp/test_ind.mdb"));
325 //     Database db = open(new File("/tmp/test_ind2.mdb"));
326     Database db = open(Database.FileFormat.V2000, new File("/tmp/test_ind3.mdb"));
327 //     Database db = open(new File("/tmp/test_ind4.mdb"));
328 
329     Table t = db.getTable("Table1");
330     Index index = t.getIndex("B");
331     ((IndexImpl)index).initialize();
332     System.out.println("Ind " + index);
333 
334     Cursor cursor = CursorBuilder.createCursor(index);
335     while(cursor.moveToNextRow()) {
336       System.out.println("=======");
337       System.out.println("Savepoint: " + cursor.getSavepoint());
338       System.out.println("Value: " + cursor.getCurrentRow());
339     }
340 
341     db.close();
342   }
343 
344   public void x_testReverseIsoMdb2010() throws Exception
345   {
346     Database db = open(Database.FileFormat.V2010, new File("/data2/jackcess_test/testAllIndexCodes3_2010.accdb"));
347 
348     Table t = db.getTable("Table1");
349     Index index = t.getIndexes().iterator().next();
350     ((IndexImpl)index).initialize();
351     System.out.println("Ind " + index);
352 
353     Pattern inlinePat = Pattern.compile("7F 0E 02 0E 02 (.*)0E 02 0E 02 01 00");
354     Pattern unprintPat = Pattern.compile("01 01 01 80 (.+) 06 (.+) 00");
355     Pattern unprint2Pat = Pattern.compile("0E 02 0E 02 0E 02 0E 02 01 02 (.+) 00");
356     Pattern inatPat = Pattern.compile("7F 0E 02 0E 02 (.*)0E 02 0E 02 01 02 02 (.+) 00");
357     Pattern inat2Pat = Pattern.compile("7F 0E 02 0E 02 (.*)0E 02 0E 02 01 (02 02 (.+))?01 01 (.*)FF 02 80 FF 80 00");
358 
359     Map<Character,String[]> inlineCodes = new TreeMap<Character,String[]>();
360     Map<Character,String[]> unprintCodes = new TreeMap<Character,String[]>();
361     Map<Character,String[]> unprint2Codes = new TreeMap<Character,String[]>();
362     Map<Character,String[]> inatInlineCodes = new TreeMap<Character,String[]>();
363     Map<Character,String[]> inatExtraCodes = new TreeMap<Character,String[]>();
364     Map<Character,String[]> inat2Codes = new TreeMap<Character,String[]>();
365     Map<Character,String[]> inat2ExtraCodes = new TreeMap<Character,String[]>();
366     Map<Character,String[]> inat2CrazyCodes = new TreeMap<Character,String[]>();
367 
368 
369     Cursor cursor = CursorBuilder.createCursor(index);
370     while(cursor.moveToNextRow()) {
371 //       System.out.println("=======");
372 //       System.out.println("Savepoint: " + cursor.getSavepoint());
373 //       System.out.println("Value: " + cursor.getCurrentRow());
374       Cursor.Savepoint savepoint = cursor.getSavepoint();
375       String entryStr = entryToString(savepoint.getCurrentPosition());
376 
377       Row row = cursor.getCurrentRow();
378       String value = row.getString("data");
379       String key = row.getString("key");
380       char c = value.charAt(2);
381 
382       System.out.println("=======");
383       System.out.println("RowId: " +
384                          savepoint.getCurrentPosition().getRowId());
385       System.out.println("Entry: " + entryStr);
386 //         System.out.println("Row: " + row);
387       System.out.println("Value: (" + key + ")" + value);
388       System.out.println("Char: " + c + ", " + (int)c + ", " +
389                          toUnicodeStr(c));
390 
391       String type = null;
392       if(entryStr.endsWith("01 00")) {
393 
394         // handle inline codes
395         type = "INLINE";
396         Matcher m = inlinePat.matcher(entryStr);
397         m.find();
398         handleInlineEntry(m.group(1), c, inlineCodes);
399 
400       } else if(entryStr.contains("01 01 01 80")) {
401 
402         // handle most unprintable codes
403         type = "UNPRINTABLE";
404         Matcher m = unprintPat.matcher(entryStr);
405         m.find();
406         handleUnprintableEntry(m.group(2), c, unprintCodes);
407 
408       } else if(entryStr.contains("01 02 02") &&
409                 !entryStr.contains("FF 02 80 FF 80")) {
410 
411         // handle chars w/ symbols
412         type = "CHAR_WITH_SYMBOL";
413         Matcher m = inatPat.matcher(entryStr);
414         m.find();
415         handleInternationalEntry(m.group(1), m.group(2), c,
416                                  inatInlineCodes, inatExtraCodes);
417 
418       } else if(entryStr.contains("0E 02 0E 02 0E 02 0E 02 01 02")) {
419 
420         // handle chars w/ symbols
421         type = "UNPRINTABLE_2";
422         Matcher m = unprint2Pat.matcher(entryStr);
423         m.find();
424         handleUnprintable2Entry(m.group(1), c, unprint2Codes);
425 
426       } else if(entryStr.contains("FF 02 80 FF 80")) {
427 
428         type = "CRAZY_INAT";
429         Matcher m = inat2Pat.matcher(entryStr);
430         m.find();
431         handleInternational2Entry(m.group(1), m.group(3), m.group(4), c,
432                                   inat2Codes, inat2ExtraCodes,
433                                   inat2CrazyCodes);
434 
435       } else {
436 
437         // throw new RuntimeException("unhandled " + entryStr);
438         System.out.println("unhandled " + entryStr);
439       }
440 
441       System.out.println("Type: " + type);
442     }
443 
444     System.out.println("\n***CODES");
445     for(int i = 0; i <= 0xFFFF; ++i) {
446 
447       if(i == 256) {
448         System.out.println("\n***EXTENDED CODES");
449       }
450 
451       // skip non-char chars
452       char c = (char)i;
453       if(Character.isHighSurrogate(c) || Character.isLowSurrogate(c)) {
454         continue;
455       }
456 
457       if(c == (char)0xFFFE) {
458         // this gets replaced with FFFD, treat it the same
459         c = (char)0xFFFD;
460       }
461 
462       Character cc = c;
463       String[] chars = inlineCodes.get(cc);
464       if(chars != null) {
465         if((chars.length == 1) && (chars[0].length() == 0)) {
466           System.out.println("X");
467         } else {
468           System.out.println("S" + toByteString(chars));
469         }
470         continue;
471       }
472 
473       chars = inatInlineCodes.get(cc);
474       if(chars != null) {
475         String[] extra = inatExtraCodes.get(cc);
476         System.out.println("I" + toByteString(chars) + "," +
477                            toByteString(extra));
478         continue;
479       }
480 
481       chars = unprintCodes.get(cc);
482       if(chars != null) {
483         System.out.println("U" + toByteString(chars));
484         continue;
485       }
486 
487       chars = unprint2Codes.get(cc);
488       if(chars != null) {
489         if(chars.length > 1) {
490           throw new RuntimeException("long unprint codes");
491         }
492         int val = Integer.parseInt(chars[0], 16) - 2;
493         String valStr = ByteUtil.toHexString(new byte[]{(byte)val}).trim();
494         System.out.println("P" + valStr);
495         continue;
496       }
497 
498       chars = inat2Codes.get(cc);
499       if(chars != null) {
500         String [] crazyCodes = inat2CrazyCodes.get(cc);
501         String crazyCode = "";
502         if(crazyCodes != null) {
503           if((crazyCodes.length != 1) || !"A0".equals(crazyCodes[0])) {
504             throw new RuntimeException("CC " + Arrays.asList(crazyCodes));
505           }
506           crazyCode = "1";
507         }
508 
509         String[] extra = inat2ExtraCodes.get(cc);
510         System.out.println("Z" + toByteString(chars) + "," +
511                            toByteString(extra) + "," +
512                            crazyCode);
513         continue;
514       }
515 
516       throw new RuntimeException("Unhandled char " + toUnicodeStr(c));
517     }
518     System.out.println("\n***END CODES");
519 
520     db.close();
521   }
522 
523   public void x_testReverseIsoMdb() throws Exception
524   {
525     Database db = open(Database.FileFormat.V2000, new File("/data2/jackcess_test/testAllIndexCodes3.mdb"));
526 
527     Table t = db.getTable("Table1");
528     Index index = t.getIndexes().iterator().next();
529     ((IndexImpl)index).initialize();
530     System.out.println("Ind " + index);
531 
532     Pattern inlinePat = Pattern.compile("7F 4A 4A (.*)4A 4A 01 00");
533     Pattern unprintPat = Pattern.compile("01 01 01 80 (.+) 06 (.+) 00");
534     Pattern unprint2Pat = Pattern.compile("4A 4A 4A 4A 01 02 (.+) 00");
535     Pattern inatPat = Pattern.compile("7F 4A 4A (.*)4A 4A 01 02 02 (.+) 00");
536     Pattern inat2Pat = Pattern.compile("7F 4A 4A (.*)4A 4A 01 (02 02 (.+))?01 01 (.*)FF 02 80 FF 80 00");
537 
538     Map<Character,String[]> inlineCodes = new TreeMap<Character,String[]>();
539     Map<Character,String[]> unprintCodes = new TreeMap<Character,String[]>();
540     Map<Character,String[]> unprint2Codes = new TreeMap<Character,String[]>();
541     Map<Character,String[]> inatInlineCodes = new TreeMap<Character,String[]>();
542     Map<Character,String[]> inatExtraCodes = new TreeMap<Character,String[]>();
543     Map<Character,String[]> inat2Codes = new TreeMap<Character,String[]>();
544     Map<Character,String[]> inat2ExtraCodes = new TreeMap<Character,String[]>();
545     Map<Character,String[]> inat2CrazyCodes = new TreeMap<Character,String[]>();
546 
547 
548     Cursor cursor = CursorBuilder.createCursor(index);
549     while(cursor.moveToNextRow()) {
550 //       System.out.println("=======");
551 //       System.out.println("Savepoint: " + cursor.getSavepoint());
552 //       System.out.println("Value: " + cursor.getCurrentRow());
553       Cursor.Savepoint savepoint = cursor.getSavepoint();
554       String entryStr = entryToString(savepoint.getCurrentPosition());
555 
556       Row row = cursor.getCurrentRow();
557       String value = row.getString("data");
558       String key = row.getString("key");
559       char c = value.charAt(2);
560       System.out.println("=======");
561       System.out.println("RowId: " +
562                          savepoint.getCurrentPosition().getRowId());
563       System.out.println("Entry: " + entryStr);
564 //         System.out.println("Row: " + row);
565       System.out.println("Value: (" + key + ")" + value);
566       System.out.println("Char: " + c + ", " + (int)c + ", " +
567                          toUnicodeStr(c));
568 
569       String type = null;
570       if(entryStr.endsWith("01 00")) {
571 
572         // handle inline codes
573         type = "INLINE";
574         Matcher m = inlinePat.matcher(entryStr);
575         m.find();
576         handleInlineEntry(m.group(1), c, inlineCodes);
577 
578       } else if(entryStr.contains("01 01 01 80")) {
579 
580         // handle most unprintable codes
581         type = "UNPRINTABLE";
582         Matcher m = unprintPat.matcher(entryStr);
583         m.find();
584         handleUnprintableEntry(m.group(2), c, unprintCodes);
585 
586       } else if(entryStr.contains("01 02 02") &&
587                 !entryStr.contains("FF 02 80 FF 80")) {
588 
589         // handle chars w/ symbols
590         type = "CHAR_WITH_SYMBOL";
591         Matcher m = inatPat.matcher(entryStr);
592         m.find();
593         handleInternationalEntry(m.group(1), m.group(2), c,
594                                  inatInlineCodes, inatExtraCodes);
595 
596       } else if(entryStr.contains("4A 4A 4A 4A 01 02")) {
597 
598         // handle chars w/ symbols
599         type = "UNPRINTABLE_2";
600         Matcher m = unprint2Pat.matcher(entryStr);
601         m.find();
602         handleUnprintable2Entry(m.group(1), c, unprint2Codes);
603 
604       } else if(entryStr.contains("FF 02 80 FF 80")) {
605 
606         type = "CRAZY_INAT";
607         Matcher m = inat2Pat.matcher(entryStr);
608         m.find();
609         handleInternational2Entry(m.group(1), m.group(3), m.group(4), c,
610                                   inat2Codes, inat2ExtraCodes,
611                                   inat2CrazyCodes);
612 
613       } else {
614 
615         throw new RuntimeException("unhandled " + entryStr);
616       }
617 
618       System.out.println("Type: " + type);
619     }
620 
621     System.out.println("\n***CODES");
622     for(int i = 0; i <= 0xFFFF; ++i) {
623 
624       if(i == 256) {
625         System.out.println("\n***EXTENDED CODES");
626       }
627 
628       // skip non-char chars
629       char c = (char)i;
630       if(Character.isHighSurrogate(c) || Character.isLowSurrogate(c)) {
631         continue;
632       }
633 
634       if(c == (char)0xFFFE) {
635         // this gets replaced with FFFD, treat it the same
636         c = (char)0xFFFD;
637       }
638 
639       Character cc = c;
640       String[] chars = inlineCodes.get(cc);
641       if(chars != null) {
642         if((chars.length == 1) && (chars[0].length() == 0)) {
643           System.out.println("X");
644         } else {
645           System.out.println("S" + toByteString(chars));
646         }
647         continue;
648       }
649 
650       chars = inatInlineCodes.get(cc);
651       if(chars != null) {
652         String[] extra = inatExtraCodes.get(cc);
653         System.out.println("I" + toByteString(chars) + "," +
654                            toByteString(extra));
655         continue;
656       }
657 
658       chars = unprintCodes.get(cc);
659       if(chars != null) {
660         System.out.println("U" + toByteString(chars));
661         continue;
662       }
663 
664       chars = unprint2Codes.get(cc);
665       if(chars != null) {
666         if(chars.length > 1) {
667           throw new RuntimeException("long unprint codes");
668         }
669         int val = Integer.parseInt(chars[0], 16) - 2;
670         String valStr = ByteUtil.toHexString(new byte[]{(byte)val}).trim();
671         System.out.println("P" + valStr);
672         continue;
673       }
674 
675       chars = inat2Codes.get(cc);
676       if(chars != null) {
677         String [] crazyCodes = inat2CrazyCodes.get(cc);
678         String crazyCode = "";
679         if(crazyCodes != null) {
680           if((crazyCodes.length != 1) || !"A0".equals(crazyCodes[0])) {
681             throw new RuntimeException("CC " + Arrays.asList(crazyCodes));
682           }
683           crazyCode = "1";
684         }
685 
686         String[] extra = inat2ExtraCodes.get(cc);
687         System.out.println("Z" + toByteString(chars) + "," +
688                            toByteString(extra) + "," +
689                            crazyCode);
690         continue;
691       }
692 
693       throw new RuntimeException("Unhandled char " + toUnicodeStr(c));
694     }
695     System.out.println("\n***END CODES");
696 
697     db.close();
698   }
699 
700   private static String toByteString(String[] chars)
701   {
702     String str = join(chars, "", "");
703     if(str.length() > 0 && str.charAt(0) == '0') {
704       str = str.substring(1);
705     }
706     return str;
707   }
708 
709   private static void handleInlineEntry(
710       String entryCodes, char c, Map<Character,String[]> inlineCodes)
711     throws Exception
712   {
713     inlineCodes.put(c, entryCodes.trim().split(" "));
714   }
715 
716   private static void handleUnprintableEntry(
717       String entryCodes, char c, Map<Character,String[]> unprintCodes)
718     throws Exception
719   {
720     unprintCodes.put(c, entryCodes.trim().split(" "));
721   }
722 
723   private static void handleUnprintable2Entry(
724       String entryCodes, char c, Map<Character,String[]> unprintCodes)
725     throws Exception
726   {
727     unprintCodes.put(c, entryCodes.trim().split(" "));
728   }
729 
730   private static void handleInternationalEntry(
731       String inlineCodes, String entryCodes, char c,
732       Map<Character,String[]> inatInlineCodes,
733       Map<Character,String[]> inatExtraCodes)
734     throws Exception
735   {
736     inatInlineCodes.put(c, inlineCodes.trim().split(" "));
737     inatExtraCodes.put(c, entryCodes.trim().split(" "));
738   }
739 
740   private static void handleInternational2Entry(
741       String inlineCodes, String entryCodes, String crazyCodes, char c,
742       Map<Character,String[]> inatInlineCodes,
743       Map<Character,String[]> inatExtraCodes,
744       Map<Character,String[]> inatCrazyCodes)
745     throws Exception
746   {
747     inatInlineCodes.put(c, inlineCodes.trim().split(" "));
748     if(entryCodes != null) {
749       inatExtraCodes.put(c, entryCodes.trim().split(" "));
750     }
751     if((crazyCodes != null) && (crazyCodes.length() > 0)) {
752       inatCrazyCodes.put(c, crazyCodes.trim().split(" "));
753     }
754   }
755 
756   public static String toUnicodeStr(Object obj) throws Exception {
757     StringBuilder sb = new StringBuilder();
758     for(char c : obj.toString().toCharArray()) {
759       sb.append(toUnicodeStr(c)).append(" ");
760     }
761     return sb.toString();
762   }
763 
764   private static String toUnicodeStr(char c) throws Exception {
765     String specialStr = SPECIAL_CHARS.get(c);
766     if(specialStr != null) {
767       return specialStr;
768     }
769 
770     String digits = Integer.toHexString(c).toUpperCase();
771     while(digits.length() < 4) {
772       digits = "0" + digits;
773     }
774     return "\\u" + digits;
775   }
776 
777   private static String join(String[] strs, String joinStr, String prefixStr) {
778     if(strs == null) {
779       return "";
780     }
781     StringBuilder builder = new StringBuilder();
782     for(int i = 0; i < strs.length; ++i) {
783       if(strs[i].length() == 0) {
784         continue;
785       }
786       builder.append(prefixStr).append(strs[i]);
787       if(i < (strs.length - 1)) {
788         builder.append(joinStr);
789       }
790     }
791     return builder.toString();
792   }
793 
794   public static String entryToString(Cursor.Position curPos)
795     throws Exception
796   {
797     Field eField = curPos.getClass().getDeclaredField("_entry");
798     eField.setAccessible(true);
799     IndexData.Entry entry = (IndexData.Entry)eField.get(curPos);
800     Field ebField = entry.getClass().getDeclaredField("_entryBytes");
801     ebField.setAccessible(true);
802     byte[] entryBytes = (byte[])ebField.get(entry);
803 
804     return ByteUtil.toHexString(ByteBuffer.wrap(entryBytes),
805                                 0, entryBytes.length, false);
806   }
807 
808 }