/*
   Copyright (c) 2005 Health Market Science, Inc.
   
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
   
       http://www.apache.org/licenses/LICENSE-2.0
   
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
  */
  
  package com.healthmarketscience.jackcess.impl;
  
  import java.io.BufferedWriter;
  import java.io.IOException;
  import java.io.StringWriter;
  import java.lang.System.Logger;
  import java.nio.BufferOverflowException;
  import java.nio.ByteBuffer;
  import java.nio.charset.Charset;
  import java.time.LocalDateTime;
  import java.util.AbstractMap;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collection;
  import java.util.Collections;
  import java.util.Comparator;
  import java.util.HashMap;
  import java.util.Iterator;
  import java.util.LinkedHashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  import java.util.TreeSet;
  
  import com.healthmarketscience.jackcess.BatchUpdateException;
  import com.healthmarketscience.jackcess.Column;
  import com.healthmarketscience.jackcess.ColumnBuilder;
  import com.healthmarketscience.jackcess.ConstraintViolationException;
  import com.healthmarketscience.jackcess.CursorBuilder;
  import com.healthmarketscience.jackcess.Index;
  import com.healthmarketscience.jackcess.IndexBuilder;
  import com.healthmarketscience.jackcess.InvalidValueException;
  import com.healthmarketscience.jackcess.JackcessException;
  import com.healthmarketscience.jackcess.PropertyMap;
  import com.healthmarketscience.jackcess.Row;
  import com.healthmarketscience.jackcess.RowId;
  import com.healthmarketscience.jackcess.Table;
  import com.healthmarketscience.jackcess.expr.Identifier;
  import com.healthmarketscience.jackcess.util.ErrorHandler;
  import com.healthmarketscience.jackcess.util.ExportUtil;
  
  /**
   * A single database table
   * <p>
   * Is not thread-safe.
   *
   * @author Tim McCune
   * @usage _intermediate_class_
   */
  public class TableImpl implements Table, PropertyMaps.Owner
  {
    private static final Logger LOG = System.getLogger(TableImpl.class.getName());
  
    private static final short OFFSET_MASK = (short)0x1FFF;
  
    private static final short DELETED_ROW_MASK = (short)0x8000;
  
    private static final short OVERFLOW_ROW_MASK = (short)0x4000;
  
    static final int MAGIC_TABLE_NUMBER = 1625;
  
    private static final int MAX_BYTE = 256;
  
    /**
     * Table type code for system tables
     * @usage _intermediate_class_
     */
    public static final byte TYPE_SYSTEM = 0x53;
    /**
     * Table type code for user tables
     * @usage _intermediate_class_
     */
    public static final byte TYPE_USER = 0x4e;
  
    public enum IndexFeature {
      EXACT_MATCH, EXACT_UNIQUE_ONLY, ANY_MATCH;
    }
  
    /** comparator which sorts variable length columns based on their index into
        the variable length offset table */
    private static final Comparator<ColumnImpl> VAR_LEN_COLUMN_COMPARATOR =
      Comparator.comparingInt(ColumnImpl::getVarLenTableIndex);
  
   /** owning database */
   private final DatabaseImpl _database;
   /** additional table flags from the catalog entry */
   private final int _flags;
   /** Type of the table (either TYPE_SYSTEM or TYPE_USER) */
   private final byte _tableType;
   /** Number of actual indexes on the table */
   private int _indexCount;
   /** Number of logical indexes for the table */
   private int _logicalIndexCount;
   /** page number of the definition of this table */
   private final int _tableDefPageNumber;
   /** max Number of columns in the table (includes previous deletions) */
   private short _maxColumnCount;
   /** max Number of variable columns in the table (includes deletions) */
   private short _maxVarColumnCount;
   /** List of columns in this table, ordered by column number */
   private final List<ColumnImpl> _columns = new ArrayList<>();
   /** List of variable length columns in this table, ordered by offset */
   private final List<ColumnImpl> _varColumns = new ArrayList<>();
   /** List of autonumber columns in this table, ordered by column number */
   private final List<ColumnImpl> _autoNumColumns = new ArrayList<>(1);
   /** handler for calculated columns */
   private final CalcColEvaluator _calcColEval = new CalcColEvaluator();
   /** List of indexes on this table (multiple logical indexes may be backed by
       the same index data) */
   private final List<IndexImpl> _indexes = new ArrayList<>();
   /** List of index datas on this table (the actual backing data for an
       index) */
   private final List<IndexData> _indexDatas = new ArrayList<>();
   /** List of columns in this table which are in one or more indexes */
   private final Set<ColumnImpl> _indexColumns = new LinkedHashSet<>();
   /** Table name as stored in Database */
   private final String _name;
   /** Usage map of pages that this table owns */
   private final UsageMap _ownedPages;
   /** Usage map of pages that this table owns with free space on them */
   private final UsageMap _freeSpacePages;
   /** Number of rows in the table */
   private int _rowCount;
   /** last long auto number for the table */
   private int _lastLongAutoNumber;
   /** last complex type auto number for the table */
   private int _lastComplexTypeAutoNumber;
   /** modification count for the table, keeps row-states up-to-date */
   private int _modCount;
   /** page buffer used to update data pages when adding rows */
   private final TempPageHolder _addRowBufferH =
     TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);
   /** page buffer used to update the table def page */
   private final TempPageHolder _tableDefBufferH =
     TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);
   /** buffer used to writing rows of data */
   private final TempBufferHolder _writeRowBufferH =
     TempBufferHolder.newHolder(TempBufferHolder.Type.SOFT, true);
   /** page buffer used to write out-of-row "long value" data */
   private final TempPageHolder _longValueBufferH =
     TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);
   /** optional error handler to use when row errors are encountered */
   private ErrorHandler _tableErrorHandler;
   /** properties for this table */
   private PropertyMap _props;
   /** properties group for this table (and columns) */
   private PropertyMaps _propertyMaps;
   /** name map for this table, read on first use */
   private TableNameMapImpl _nameMap;
   /** optional flag indicating whether or not auto numbers can be directly
       inserted by the user */
   private Boolean _allowAutoNumInsert;
   /** foreign-key enforcer for this table */
   private final FKEnforcer _fkEnforcer;
   /** table validator if any (and enabled) */
   private RowValidatorEvalContext _rowValidator;
   /** description of an inconsistency between the table definition and the
       column definitions, {@code null} if they agree.  rows cannot be written
       to such a table, but they can still be read */
   private String _writeDefError;
 
   /** default cursor for iterating through the table, kept here for basic
       table traversal */
   private CursorImpl _defaultCursor;
 
   /**
    * Only used by unit tests
    * @usage _advanced_method_
    */
   protected TableImpl(boolean testing, List<ColumnImpl> columns)
   {
     if(!testing) {
       throw new IllegalArgumentException();
     }
     _database = null;
     _tableDefPageNumber = PageChannel.INVALID_PAGE_NUMBER;
     _name = null;
 
     _columns.addAll(columns);
     for(ColumnImpl col : _columns) {
       if(col.getType().isVariableLength()) {
         _varColumns.add(col);
       }
     }
     _maxColumnCount = (short)_columns.size();
     _maxVarColumnCount = (short)_varColumns.size();
     initAutoNumberColumns();
 
     _fkEnforcer = null;
     _flags = 0;
     _tableType = TYPE_USER;
     _indexCount = 0;
     _logicalIndexCount = 0;
     _ownedPages = null;
     _freeSpacePages = null;
   }
 
   /**
    * @param database database which owns this table
    * @param tableBuffer Buffer to read the table with
    * @param pageNumber Page number of the table definition
    * @param name Table name
    */
   protected TableImpl(DatabaseImpl database, ByteBuffer tableBuffer,
                       int pageNumber, String name, int flags)
     throws IOException
   {
     _database = database;
     _tableDefPageNumber = pageNumber;
     _name = name;
     _flags = flags;
 
     // read table definition
     tableBuffer = loadCompleteTableDefinitionBuffer(tableBuffer, null);
 
     _rowCount = tableBuffer.getInt(getFormat().OFFSET_NUM_ROWS);
     _lastLongAutoNumber = tableBuffer.getInt(getFormat().OFFSET_NEXT_AUTO_NUMBER);
     if(getFormat().OFFSET_NEXT_COMPLEX_AUTO_NUMBER >= 0) {
       _lastComplexTypeAutoNumber = tableBuffer.getInt(
           getFormat().OFFSET_NEXT_COMPLEX_AUTO_NUMBER);
     }
     _tableType = tableBuffer.get(getFormat().OFFSET_TABLE_TYPE);
     _maxColumnCount = tableBuffer.getShort(getFormat().OFFSET_MAX_COLS);
     _maxVarColumnCount = tableBuffer.getShort(getFormat().OFFSET_NUM_VAR_COLS);
     short columnCount = tableBuffer.getShort(getFormat().OFFSET_NUM_COLS);
     _logicalIndexCount = tableBuffer.getInt(getFormat().OFFSET_NUM_INDEX_SLOTS);
     _indexCount = tableBuffer.getInt(getFormat().OFFSET_NUM_INDEXES);
 
     tableBuffer.position(getFormat().OFFSET_OWNED_PAGES);
     _ownedPages = UsageMap.read(getDatabase(), tableBuffer);
     tableBuffer.position(getFormat().OFFSET_FREE_SPACE_PAGES);
     _freeSpacePages = UsageMap.read(getDatabase(), tableBuffer);
 
     for (int i = 0; i < _indexCount; i++) {
       _indexDatas.add(IndexData.create(this, tableBuffer, i, getFormat()));
     }
 
     readColumnDefinitions(tableBuffer, columnCount);
 
     validateColumnDefs();
 
     readIndexDefinitions(tableBuffer);
 
     // read column usage map info
     while((tableBuffer.remaining() >= 2) &&
           readColumnUsageMaps(tableBuffer)) {
       // keep reading ...
     }
 
     // re-sort columns if necessary
     if(getDatabase().getColumnOrder() != ColumnOrder.DATA) {
       Collections.sort(_columns, ColumnImpl.DISPLAY_ORDER_COMPARATOR);
     }
 
     for(ColumnImpl col : _columns) {
       // some columns need to do extra work after the table is completely
       // loaded
       col.postTableLoadInit();
     }
 
     _fkEnforcer = new FKEnforcer(this);
 
     if(!isSystem()) {
       // after fully constructed, allow column/row validators to be configured
       // (but only for user tables)
       for(ColumnImpl col : _columns) {
         col.initColumnValidator();
       }
 
       reloadRowValidator();
     }
   }
 
   private void reloadRowValidator() throws IOException {
 
     // reset table row validator before proceeding
     _rowValidator = null;
 
     if(!getDatabase().isEvaluateExpressions()) {
       return;
     }
 
     PropertyMap props = getProperties();
 
     String exprStr = PropertyMaps.getTrimmedStringProperty(
         props, PropertyMap.VALIDATION_RULE_PROP);
 
     if(exprStr != null) {
       String helpStr = PropertyMaps.getTrimmedStringProperty(
           props, PropertyMap.VALIDATION_TEXT_PROP);
 
       _rowValidator = new RowValidatorEvalContext(this)
         .setExpr(exprStr, helpStr);
     }
   }
 
   @Override
   public String getName() {
     return _name;
   }
 
   @Override
   public boolean isHidden() {
     return((_flags & DatabaseImpl.HIDDEN_OBJECT_FLAG) != 0);
   }
 
   @Override
   public boolean isSystem() {
     return(_tableType != TYPE_USER);
   }
 
   /**
    * @usage _advanced_method_
    */
   public int getMaxColumnCount() {
     return _maxColumnCount;
   }
 
   @Override
   public int getColumnCount() {
     return _columns.size();
   }
 
   @Override
   public DatabaseImpl getDatabase() {
     return _database;
   }
 
   /**
    * @usage _advanced_method_
    */
   public JetFormat getFormat() {
     return getDatabase().getFormat();
   }
 
   /**
    * @usage _advanced_method_
    */
   public PageChannel getPageChannel() {
     return getDatabase().getPageChannel();
   }
 
   @Override
   public ErrorHandler getErrorHandler() {
     return((_tableErrorHandler != null) ? _tableErrorHandler :
            getDatabase().getErrorHandler());
   }
 
   @Override
   public void setErrorHandler(ErrorHandler newErrorHandler) {
     _tableErrorHandler = newErrorHandler;
   }
 
   public int getTableDefPageNumber() {
     return _tableDefPageNumber;
   }
 
   @Override
   public boolean isAllowAutoNumberInsert() {
     return ((_allowAutoNumInsert != null) ? (boolean)_allowAutoNumInsert :
             getDatabase().isAllowAutoNumberInsert());
   }
 
   @Override
   public void setAllowAutoNumberInsert(Boolean allowAutoNumInsert) {
     _allowAutoNumInsert = allowAutoNumInsert;
   }
 
   /**
    * @usage _advanced_method_
    */
   public RowState createRowState() {
     return new RowState(TempBufferHolder.Type.HARD);
   }
 
   /**
    * @usage _advanced_method_
    */
   public UsageMap.PageCursor getOwnedPagesCursor() {
     return _ownedPages.cursor();
   }
 
   /**
    * Returns the <i>approximate</i> number of database pages owned by this
    * table and all related indexes (this number does <i>not</i> take into
    * account pages used for large OLE/MEMO fields).
    * <p>
    * To calculate the approximate number of bytes owned by a table:
    * <code>
    * int approxTableBytes = (table.getApproximateOwnedPageCount() *
    *                         table.getFormat().PAGE_SIZE);
    * </code>
    * @usage _intermediate_method_
    */
   public int getApproximateOwnedPageCount() {
 
     // add a page for the table def (although that might actually be more than
     // one page)
     int count = _ownedPages.getPageCount() + 1;
 
     for(ColumnImpl col : _columns) {
       count += col.getOwnedPageCount();
     }
 
     // note, we count owned pages from _physical_ indexes, not logical indexes
     // (otherwise we could double count pages)
     for(IndexData indexData : _indexDatas) {
       count += indexData.getOwnedPageCount();
     }
 
     return count;
   }
 
   protected TempPageHolder getLongValueBuffer() {
     return _longValueBufferH;
   }
 
   @Override
   public List<ColumnImpl> getColumns() {
     return Collections.unmodifiableList(_columns);
   }
 
   @Override
   public ColumnImpl getColumn(String name) {
     for(ColumnImpl column : _columns) {
       if(column.getName().equalsIgnoreCase(name)) {
         return column;
       }
     }
     throw new IllegalArgumentException(withErrorContext(
             "Column with name " + name + " does not exist in this table"));
   }
 
   public boolean hasColumn(String name) {
     for(ColumnImpl column : _columns) {
       if(column.getName().equalsIgnoreCase(name)) {
         return true;
       }
     }
     return false;
   }
 
   @Override
   public PropertyMap getProperties() throws IOException {
     if(_props == null) {
       _props = getPropertyMaps().getDefault();
     }
     return _props;
   }
 
   @Override
   public TableNameMapImpl getNameMap() throws IOException {
     if(_nameMap == null) {
       byte[] bytes = (byte[])getProperties().getValue(
           PropertyMap.NAME_MAP_PROP);
       if(bytes != null) {
         _nameMap = TableNameMapImpl.read(bytes, this);
       }
     }
     return _nameMap;
   }
 
   @Override
   public LocalDateTime getCreatedDate() throws IOException {
     return getDatabase().getCreateDateForObject(_tableDefPageNumber);
   }
 
   @Override
   public LocalDateTime getUpdatedDate() throws IOException {
     return getDatabase().getUpdateDateForObject(_tableDefPageNumber);
   }
 
   /**
    * @return all PropertyMaps for this table (and columns)
    * @usage _advanced_method_
    */
   public PropertyMaps getPropertyMaps() throws IOException {
     if(_propertyMaps == null) {
       _propertyMaps = getDatabase().getPropertiesForObject(
           _tableDefPageNumber, this);
     }
     return _propertyMaps;
   }
 
   @Override
   public void propertiesUpdated() throws IOException {
     // the name map property may have been replaced
     _nameMap = null;
 
     // propagate update to columns
     for(ColumnImpl col : _columns) {
       col.propertiesUpdated();
     }
 
     reloadRowValidator();
 
     // calculated columns will need to be re-sorted (their expressions may
     // have changed when their properties were updated)
     _calcColEval.reSort();
   }
 
   @Override
   public List<IndexImpl> getIndexes() {
     return Collections.unmodifiableList(_indexes);
   }
 
   @Override
   public IndexImpl getIndex(String name) {
     for(IndexImpl index : _indexes) {
       if(index.getName().equalsIgnoreCase(name)) {
         return index;
       }
     }
     throw new IllegalArgumentException(withErrorContext(
             "Index with name " + name + " does not exist on this table"));
   }
 
   @Override
   public IndexImpl getPrimaryKeyIndex() {
     for(IndexImpl index : _indexes) {
       if(index.isPrimaryKey()) {
         return index;
       }
     }
     throw new IllegalArgumentException(withErrorContext(
             "No primary key index found"));
   }
 
   @Override
   public IndexImpl getForeignKeyIndex(Table otherTable) {
     for(IndexImpl index : _indexes) {
       if(index.isForeignKey() && (index.getReference() != null) &&
          (index.getReference().getOtherTablePageNumber() ==
           ((TableImpl)otherTable).getTableDefPageNumber())) {
         return index;
       }
     }
     throw new IllegalArgumentException(withErrorContext(
         "No foreign key reference to " +
         otherTable.getName() + " found"));
   }
 
   /**
    * @return All of the IndexData on this table (unmodifiable List)
    * @usage _advanced_method_
    */
   public List<IndexData> getIndexDatas() {
     return Collections.unmodifiableList(_indexDatas);
   }
 
   /**
    * Only called by unit tests
    * @usage _advanced_method_
    */
   public int getLogicalIndexCount() {
     return _logicalIndexCount;
   }
 
   int getIndexCount() {
     return _indexCount;
   }
 
   public IndexImpl findIndexForColumns(Collection<String> searchColumns,
                                        IndexFeature feature) {
 
     IndexImpl partialIndex = null;
     for(IndexImpl index : _indexes) {
 
       Collection<? extends Index.Column> indexColumns = index.getColumns();
       if(indexColumns.size() < searchColumns.size()) {
         continue;
       }
       boolean exactMatch = (indexColumns.size() == searchColumns.size());
 
       Iterator<String> sIter = searchColumns.iterator();
       Iterator<? extends Index.Column> iIter = indexColumns.iterator();
       boolean searchMatches = true;
       while(sIter.hasNext()) {
         String sColName = sIter.next();
         String iColName = iIter.next().getName();
         if((sColName != iColName) &&
            ((sColName == null) || !sColName.equalsIgnoreCase(iColName))) {
           searchMatches = false;
           break;
         }
       }
 
       if(searchMatches) {
 
         if(exactMatch && ((feature != IndexFeature.EXACT_UNIQUE_ONLY) ||
                           index.isUnique())) {
           return index;
         }
 
         if(!exactMatch && (feature == IndexFeature.ANY_MATCH) &&
            ((partialIndex == null) ||
             (indexColumns.size() < partialIndex.getColumnCount()))) {
           // this is a better partial index match
           partialIndex = index;
         }
       }
     }
 
     return partialIndex;
   }
 
   List<ColumnImpl> getAutoNumberColumns() {
     return _autoNumColumns;
   }
 
   @Override
   public CursorImpl getDefaultCursor() {
     if(_defaultCursor == null) {
       _defaultCursor = CursorImpl.createCursor(this);
     }
     return _defaultCursor;
   }
 
   @Override
   public CursorBuilder newCursor() {
     return new CursorBuilder(this);
   }
 
   @Override
   public void reset() {
     getDefaultCursor().reset();
   }
 
   @Override
   public Rowf="../../../../com/healthmarketscience/jackcess/Row.html#Row">Row deleteRow(Row row) throws IOException {
     deleteRow(row.getId());
     return row;
   }
 
   /**
    * Delete the row with the given id.  Provided RowId must have previously
    * been returned from this Table.
    * @return the given rowId
    * @throws IllegalStateException if the given row is not valid
    * @usage _intermediate_method_
    */
   public RowId"../../../../com/healthmarketscience/jackcess/RowId.html#RowId">RowId deleteRow(RowId rowId) throws IOException {
     deleteRow(getDefaultCursor().getRowState(), (RowIdImpl)rowId);
     return rowId;
   }
 
   /**
    * Delete the row for the given rowId.
    * @usage _advanced_method_
    */
   public void deleteRow(RowState rowState, RowIdImpl rowId)
     throws IOException
   {
     requireValidRowId(rowId);
 
     getPageChannel().startWrite();
     try {
 
       // ensure that the relevant row state is up-to-date
       ByteBuffer rowBuffer = positionAtRowHeader(rowState, rowId);
 
       if(rowState.isDeleted()) {
         // don't care about duplicate deletion
         return;
       }
       requireNonDeletedRow(rowState, rowId);
 
       // delete flag always gets set in the "header" row (even if data is on
       // overflow row)
       int pageNumber = rowState.getHeaderRowId().getPageNumber();
       int rowNumber = rowState.getHeaderRowId().getRowNumber();
 
       // attempt to fill in index column values
       Object[] rowValues = null;
       if(!_indexDatas.isEmpty()) {
 
         // move to row data to get index values
         rowBuffer = positionAtRowData(rowState, rowId);
 
         for(ColumnImpl idxCol : _indexColumns) {
           getRowColumn(getFormat(), rowBuffer, idxCol, rowState, null);
         }
 
         // use any read rowValues to help update the indexes
         rowValues = rowState.getRowCacheValues();
 
         // check foreign keys before proceeding w/ deletion
         _fkEnforcer.deleteRow(rowValues);
 
         // move back to the header
         rowBuffer = positionAtRowHeader(rowState, rowId);
       }
 
       // finally, pull the trigger
       int rowIndex = getRowStartOffset(rowNumber, getFormat());
       rowBuffer.putShort(rowIndex, (short)(rowBuffer.getShort(rowIndex)
                                            | DELETED_ROW_MASK | OVERFLOW_ROW_MASK));
       writeDataPage(rowBuffer, pageNumber);
 
       // update the indexes
       for(IndexData indexData : _indexDatas) {
         indexData.deleteRow(rowValues, rowId);
       }
 
       // make sure table def gets updated
       updateTableDefinition(-1);
 
     } finally {
       getPageChannel().finishWrite();
     }
   }
 
   @Override
   public Row getNextRow() throws IOException {
     return getDefaultCursor().getNextRow();
   }
 
   /**
    * Reads a single column from the given row.
    * @usage _advanced_method_
    */
   public Object getRowValue(RowState rowState, RowIdImpl rowId,
                             ColumnImpl column)
     throws IOException
   {
     if(this != column.getTable()) {
       throw new IllegalArgumentException(withErrorContext(
           "Given column " + column + " is not from this table"));
     }
     requireValidRowId(rowId);
 
     // position at correct row
     ByteBuffer rowBuffer = positionAtRowData(rowState, rowId);
     requireNonDeletedRow(rowState, rowId);
 
     return getRowColumn(getFormat(), rowBuffer, column, rowState, null);
   }
 
   /**
    * Reads some columns from the given row.
    * @param columnNames Only column names in this collection will be returned
    * @usage _advanced_method_
    */
   public RowImpl getRow(
       RowState rowState, RowIdImpl rowId, Collection<String> columnNames)
     throws IOException
   {
     requireValidRowId(rowId);
 
     // position at correct row
     ByteBuffer rowBuffer = positionAtRowData(rowState, rowId);
     requireNonDeletedRow(rowState, rowId);
 
     return getRow(getFormat(), rowState, rowBuffer, _columns, columnNames);
   }
 
   /**
    * Reads the row data from the given row buffer.  Leaves limit unchanged.
    * Saves parsed row values to the given rowState.
    */
   private static RowImpl getRow(
       JetFormat format,
       RowState rowState,
       ByteBuffer rowBuffer,
       Collection<ColumnImpl> columns,
       Collection<String> columnNames)
     throws IOException
   {
     RowImplckcess/impl/RowImpl.html#RowImpl">RowImpl rtn = new RowImpl(rowState.getHeaderRowId(), columns.size());
     for(ColumnImpl column : columns) {
 
       if((columnNames == null) || (columnNames.contains(column.getName()))) {
         // Add the value to the row data
         column.setRowValue(
             rtn, getRowColumn(format, rowBuffer, column, rowState, null));
       }
     }
     return rtn;
   }
 
   /**
    * Reads the column data from the given row buffer.  Leaves limit unchanged.
    * Caches the returned value in the rowState.
    */
   private static Object getRowColumn(JetFormat format,
                                      ByteBuffer rowBuffer,
                                      ColumnImpl column,
                                      RowState rowState,
                                      Map<ColumnImpl,byte[]> rawVarValues)
     throws IOException
   {
     byte[] columnData = null;
     try {
 
       NullMask nullMask = rowState.getNullMask(rowBuffer);
       boolean isNull = nullMask.isNull(column);
       if(column.storeInNullMask()) {
           // Boolean values are stored in the null mask.  see note about
           // caching below
         return rowState.setRowCacheValue(column.getColumnIndex(),
                                          column.readFromNullMask(isNull));
       } else if(isNull) {
         // well, that's easy! (no need to update cache w/ null)
         return null;
       }
 
       Object cachedValue = rowState.getRowCacheValue(column.getColumnIndex());
       if(cachedValue != null) {
         // we already have it, use it
         return cachedValue;
       }
 
       // reset position to row start
       rowBuffer.reset();
 
       // locate the column data bytes
       int rowStart = rowBuffer.position();
       int colDataPos = 0;
       int colDataLen = 0;
       if(!column.isVariableLength()) {
 
         // read fixed length value (non-boolean at this point)
         int dataStart = rowStart + format.OFFSET_COLUMN_FIXED_DATA_ROW_OFFSET;
         colDataPos = dataStart + column.getFixedDataOffset();
         colDataLen = column.getFixedDataSize();
 
       } else {
         int varDataStart;
         int varDataEnd;
 
         if(format.SIZE_ROW_VAR_COL_OFFSET == 2) {
 
           // read simple var length value
           int varColumnOffsetPos =
             (rowBuffer.limit() - nullMask.byteSize() - 4) -
             (column.getVarLenTableIndex() * 2);
 
           varDataStart = rowBuffer.getShort(varColumnOffsetPos);
           varDataEnd = rowBuffer.getShort(varColumnOffsetPos - 2);
 
         } else {
 
           // read jump-table based var length values
           short[] varColumnOffsets = readJumpTableVarColOffsets(
               rowState, rowBuffer, rowStart, nullMask);
 
           varDataStart = varColumnOffsets[column.getVarLenTableIndex()];
           varDataEnd = varColumnOffsets[column.getVarLenTableIndex() + 1];
         }
 
         colDataPos = rowStart + varDataStart;
         colDataLen = varDataEnd - varDataStart;
       }
 
       // grab the column data
       rowBuffer.position(colDataPos);
       columnData = ByteUtil.getBytes(rowBuffer, colDataLen);
 
       if((rawVarValues != null) && column.isVariableLength()) {
         // caller wants raw value as well
         rawVarValues.put(column, columnData);
       }
 
       // parse the column data.  we cache the row values in order to be able
       // to update the index on row deletion.  note, most of the returned
       // values are immutable, except for binary data (returned as byte[]),
       // but binary data shouldn't be indexed anyway.
       return rowState.setRowCacheValue(column.getColumnIndex(),
                                        column.read(columnData));
 
     } catch(Exception e) {
 
       // cache "raw" row value.  see note about caching above
       rowState.setRowCacheValue(column.getColumnIndex(),
                                 ColumnImpl.rawDataWrapper(columnData));
 
       return rowState.handleRowError(column, columnData, e);
     }
   }
 
   private static short[] readJumpTableVarColOffsets(
       RowState rowState, ByteBuffer rowBuffer, int rowStart,
       NullMask nullMask)
   {
     short[] varColOffsets = rowState.getVarColOffsets();
     if(varColOffsets != null) {
       return varColOffsets;
     }
 
     // calculate offsets using jump-table info
     int nullMaskSize = nullMask.byteSize();
     int rowEnd = rowStart + rowBuffer.remaining() - 1;
     int numVarCols = ByteUtil.getUnsignedByte(rowBuffer,
                                               rowEnd - nullMaskSize);
     varColOffsets = new short[numVarCols + 1];
 
     int rowLen = rowEnd - rowStart + 1;
     int numJumps = (rowLen - 1) / MAX_BYTE;
     int colOffset = rowEnd - nullMaskSize - numJumps - 1;
 
     // If last jump is a dummy value, ignore it
     if(((colOffset - rowStart - numVarCols) / MAX_BYTE) < numJumps) {
       numJumps--;
     }
 
     int jumpsUsed = 0;
     for(int i = 0; i < numVarCols + 1; i++) {
 
       while((jumpsUsed < numJumps) &&
          (i == ByteUtil.getUnsignedByte(
               rowBuffer, rowEnd - nullMaskSize-jumpsUsed - 1))) {
         jumpsUsed++;
       }
 
       varColOffsets[i] = (short)
         (ByteUtil.getUnsignedByte(rowBuffer, colOffset - i)
          + (jumpsUsed * MAX_BYTE));
     }
 
     rowState.setVarColOffsets(varColOffsets);
     return varColOffsets;
   }
 
   /**
    * Reads the null mask from the given row buffer.  Leaves limit unchanged.
    */
   private NullMask getRowNullMask(ByteBuffer rowBuffer)
   {
     // reset position to row start
     rowBuffer.reset();
 
     // Number of columns in this row
     int columnCount = ByteUtil.getUnsignedVarInt(
         rowBuffer, getFormat().SIZE_ROW_COLUMN_COUNT);
 
     // read null mask
     NullMask/impl/NullMask.html#NullMask">NullMask nullMask = new NullMask(columnCount);
     rowBuffer.position(rowBuffer.limit() - nullMask.byteSize());  //Null mask at end
     nullMask.read(rowBuffer);
 
     return nullMask;
   }
 
   /**
    * Sets a new buffer to the correct row header page using the given rowState
    * according to the given rowId.  Deleted state is
    * determined, but overflow row pointers are not followed.
    *
    * @return a ByteBuffer of the relevant page, or null if row was invalid
    * @usage _advanced_method_
    */
   public static ByteBuffer positionAtRowHeader(RowState rowState,
                                                RowIdImpl rowId)
     throws IOException
   {
     ByteBuffer rowBuffer = rowState.setHeaderRow(rowId);
 
     if(rowState.isAtHeaderRow()) {
       // this task has already been accomplished
       return rowBuffer;
     }
 
     if(!rowState.isValid()) {
       // this was an invalid page/row
       rowState.setStatus(RowStateStatus.AT_HEADER);
       return null;
     }
 
     // note, we don't use findRowStart here cause we need the unmasked value
     short rowStart = rowBuffer.getShort(
         getRowStartOffset(rowId.getRowNumber(),
                           rowState.getTable().getFormat()));
 
     // check the deleted, overflow flags for the row (the "real" flags are
     // always set on the header row)
     RowStatus rowStatus = RowStatus.NORMAL;
     if(isDeletedRow(rowStart)) {
       rowStatus = RowStatus.DELETED;
     } else if(isOverflowRow(rowStart)) {
       rowStatus = RowStatus.OVERFLOW;
     }
 
     rowState.setRowStatus(rowStatus);
     rowState.setStatus(RowStateStatus.AT_HEADER);
     return rowBuffer;
   }
 
   /**
    * Sets the position and limit in a new buffer using the given rowState
    * according to the given row number and row end, following overflow row
    * pointers as necessary.
    *
    * @return a ByteBuffer narrowed to the actual row data, or null if row was
    *         invalid or deleted
    * @usage _advanced_method_
    */
   public static ByteBuffer positionAtRowData(RowState rowState,
                                              RowIdImpl rowId)
     throws IOException
   {
     positionAtRowHeader(rowState, rowId);
     if(!rowState.isValid() || rowState.isDeleted()) {
       // row is invalid or deleted
       rowState.setStatus(RowStateStatus.AT_FINAL);
       return null;
     }
 
     ByteBuffer rowBuffer = rowState.getFinalPage();
     int rowNum = rowState.getFinalRowId().getRowNumber();
     JetFormat format = rowState.getTable().getFormat();
 
     if(rowState.isAtFinalRow()) {
       // we've already found the final row data
       return PageChannel.narrowBuffer(
           rowBuffer,
           findRowStart(rowBuffer, rowNum, format),
           findRowEnd(rowBuffer, rowNum, format));
     }
 
     while(true) {
 
       // note, we don't use findRowStart here cause we need the unmasked value
       short rowStart = rowBuffer.getShort(getRowStartOffset(rowNum, format));
       short rowEnd = findRowEnd(rowBuffer, rowNum, format);
 
       // note, at this point we know the row is not deleted, so ignore any
       // subsequent deleted flags (as overflow rows are always marked deleted
       // anyway)
       boolean overflowRow = isOverflowRow(rowStart);
 
       // now, strip flags from rowStart offset
       rowStart = (short)(rowStart & OFFSET_MASK);
 
       if (overflowRow) {
 
         if((rowEnd - rowStart) < 4) {
           throw new IOException(rowState.getTable().withErrorContext(
                                     "invalid overflow row info"));
         }
 
         // Overflow page.  the "row" data in the current page points to
         // another page/row
         int overflowRowNum = ByteUtil.getUnsignedByte(rowBuffer, rowStart);
         int overflowPageNum = ByteUtil.get3ByteInt(rowBuffer, rowStart + 1);
         rowBuffer = rowState.setOverflowRow(
             new RowIdImpl(overflowPageNum, overflowRowNum));
         rowNum = overflowRowNum;
 
       } else {
 
         rowState.setStatus(RowStateStatus.AT_FINAL);
         return PageChannel.narrowBuffer(rowBuffer, rowStart, rowEnd);
       }
     }
   }
 
   @Override
   public Iterator<Row> iterator() {
     return getDefaultCursor().iterator();
   }
 
   /**
    * Writes a new table defined by the given TableCreator to the database.
    * @usage _advanced_method_
    */
   protected static void writeTableDefinition(TableCreator creator)
     throws IOException
   {
     // first, create the usage map page
     createUsageMapDefinitionBuffer(creator);
 
     // next, determine how big the table def will be (in case it will be more
     // than one page)
     JetFormat format = creator.getFormat();
     int idxDataLen = (creator.getIndexCount() *
                       (format.SIZE_INDEX_DEFINITION +
                        format.SIZE_INDEX_COLUMN_BLOCK)) +
       (creator.getLogicalIndexCount() * format.SIZE_INDEX_INFO_BLOCK);
     int colUmapLen = creator.getLongValueColumns().size() * 10;
     int totalTableDefSize = format.SIZE_TDEF_HEADER +
       (format.SIZE_COLUMN_DEF_BLOCK * creator.getColumns().size()) +
       idxDataLen + colUmapLen + format.SIZE_TDEF_TRAILER;
 
     // total up the amount of space used by the column and index names (2
     // bytes per char + 2 bytes for the length)
     for(ColumnBuilder col : creator.getColumns()) {
       totalTableDefSize += DBMutator.calculateNameLength(col.getName());
     }
 
     for(IndexBuilder idx : creator.getIndexes()) {
       totalTableDefSize += DBMutator.calculateNameLength(idx.getName());
     }
 
 
     // now, create the table definition
     ByteBuffer buffer = PageChannel.createBuffer(Math.max(totalTableDefSize,
                                                           format.PAGE_SIZE));
     writeTableDefinitionHeader(creator, buffer, totalTableDefSize);
 
     if(creator.hasIndexes()) {
       // index row counts
       IndexData.writeRowCountDefinitions(creator, buffer);
     }
 
     // column definitions
     ColumnImpl.writeDefinitions(creator, buffer);
 
     if(creator.hasIndexes()) {
       // index and index data definitions
       IndexData.writeDefinitions(creator, buffer);
       IndexImpl.writeDefinitions(creator, buffer);
     }
 
     // column usage map references
     ColumnImpl.writeColUsageMapDefinitions(creator, buffer);
 
     //End of tabledef
     buffer.put((byte) 0xff);
     buffer.put((byte) 0xff);
     buffer.flip();
 
     // write table buffer to database
     writeTableDefinitionBuffer(buffer, creator.getTdefPageNumber(), creator,
                                Collections.<Integer>emptyList());
   }
 
   private static void writeTableDefinitionBuffer(
       ByteBuffer buffer, int tdefPageNumber,
       TableMutator mutator, List<Integer> reservedPages)
     throws IOException
   {
     buffer.rewind();
     int totalTableDefSize = buffer.remaining();
     JetFormat format = mutator.getFormat();
     PageChannel pageChannel = mutator.getPageChannel();
 
     // write table buffer to database
     if(totalTableDefSize <= format.PAGE_SIZE) {
 
       // easy case, fits on one page
 
       // overwrite page free space
       buffer.putShort(format.OFFSET_FREE_SPACE,
                       (short)(Math.max(
                                 format.PAGE_SIZE - totalTableDefSize - 8, 0)));
       // Write the tdef page to disk.
       buffer.clear();
       pageChannel.writePage(buffer, tdefPageNumber);
 
     } else {
 
       // need to split across multiple pages
 
       ByteBuffer partialTdef = pageChannel.createPageBuffer();
       buffer.rewind();
       int nextTdefPageNumber = PageChannel.INVALID_PAGE_NUMBER;
       while(buffer.hasRemaining()) {
 
         // reset for next write
         partialTdef.clear();
 
         if(nextTdefPageNumber == PageChannel.INVALID_PAGE_NUMBER) {
 
           // this is the first page.  note, the first page already has the
           // page header, so no need to write it here
           nextTdefPageNumber = tdefPageNumber;
 
         } else {
 
           // write page header
           writeTablePageHeader(partialTdef);
         }
 
         // copy the next page of tdef bytes
         int curTdefPageNumber = nextTdefPageNumber;
         int writeLen = Math.min(partialTdef.remaining(), buffer.remaining());
         partialTdef.put(buffer.array(), buffer.position(), writeLen);
         ByteUtil.forward(buffer, writeLen);
 
         if(buffer.hasRemaining()) {
           // need a next page
           if(reservedPages.isEmpty()) {
             nextTdefPageNumber = pageChannel.allocateNewPage();
           } else {
             nextTdefPageNumber = reservedPages.remove(0);
           }
           partialTdef.putInt(format.OFFSET_NEXT_TABLE_DEF_PAGE,
                              nextTdefPageNumber);
         }
 
         // update page free space
         partialTdef.putShort(format.OFFSET_FREE_SPACE,
                              (short)(Math.max(
                                        partialTdef.remaining() - 8, 0)));
 
         // write partial page to disk
         pageChannel.writePage(partialTdef, curTdefPageNumber);
       }
 
     }
 
   }
 
   /**
    * Writes a column defined by the given TableUpdater to this table.
    * @usage _advanced_method_
    */
   protected ColumnImpl mutateAddColumn(TableUpdater mutator) throws IOException
   {
     ColumnBuilder column = mutator.getColumn();
     JetFormat format = mutator.getFormat();
     boolean isVarCol = column.isVariableLength();
     boolean isLongVal = column.getType().isLongValue();
 
     ////
     // calculate how much more space we need in the table def
     if(isLongVal) {
       mutator.addTdefLen(10);
     }
 
     mutator.addTdefLen(format.SIZE_COLUMN_DEF_BLOCK);
 
     int nameByteLen = DBMutator.calculateNameLength(column.getName());
     mutator.addTdefLen(nameByteLen);
 
     ////
     // load current table definition and add space for new info
     ByteBuffer tableBuffer = loadCompleteTableDefinitionBufferForUpdate(
         mutator);
 
     ColumnImpl newCol = null;
     int umapPos = -1;
     boolean success = false;
     try {
 
       ////
       // update various bits of the table def
       ByteUtil.forward(tableBuffer, 29);
       tableBuffer.putShort((short)(_maxColumnCount + 1));
       short varColCount = (short)(_maxVarColumnCount + (isVarCol ? 1 : 0));
       tableBuffer.putShort(varColCount);
       tableBuffer.putShort((short)(_columns.size() + 1));
 
       // move to end of column def blocks
       tableBuffer.position(format.SIZE_TDEF_HEADER +
                            (_indexCount * format.SIZE_INDEX_DEFINITION) +
                            (_columns.size() * format.SIZE_COLUMN_DEF_BLOCK));
 
       // figure out the data offsets for the new column
       int varOffset = 0;
       // find the variable offset
       for(ColumnImpl col : _varColumns) {
         if(col.isVariableLength() && (col.getVarLenTableIndex() >= varOffset)) {
           varOffset = col.getVarLenTableIndex() + 1;
         }
       }
 
       int fixedOffset = 0;
       if(!column.isVariableLength() && !column.storeInNullMask()) {
         // find the fixed offset
         for(ColumnImpl col : _columns) {
           if(!col.isVariableLength() &&
              (col.getFixedDataOffset() >= fixedOffset)) {
             fixedOffset = col.getFixedDataOffset() + col.getFixedDataSize();
           }
         }
       }
 
       mutator.setColumnOffsets(fixedOffset, varOffset, varOffset);
 
       // insert space for the column definition and write it
       int colDefPos = tableBuffer.position();
       ByteUtil.insertEmptyData(tableBuffer, format.SIZE_COLUMN_DEF_BLOCK);
       ColumnImpl.writeDefinition(mutator, column, tableBuffer);
 
       // skip existing column names and write new name
       skipNames(tableBuffer, _columns.size());
       ByteUtil.insertEmptyData(tableBuffer, nameByteLen);
       writeName(tableBuffer, column.getName(), mutator.getCharset());
 
       if(isLongVal) {
 
         // allocate usage maps for the long value col
         Map.Entry<Integer,Integer> umapInfo = addUsageMaps(2, null);
         TableMutator.ColumnState colState = mutator.getColumnState(column);
         colState.setUmapPageNumber(umapInfo.getKey());
         byte rowNum = umapInfo.getValue().byteValue();
         colState.setUmapOwnedRowNumber(rowNum);
         colState.setUmapFreeRowNumber((byte)(rowNum + 1));
 
         // skip past index defs
         ByteUtil.forward(tableBuffer, (_indexCount *
                                        format.SIZE_INDEX_COLUMN_BLOCK));
         ByteUtil.forward(tableBuffer,
                          (_logicalIndexCount * format.SIZE_INDEX_INFO_BLOCK));
         skipNames(tableBuffer, _logicalIndexCount);
 
         // skip existing usage maps
         while(tableBuffer.remaining() >= 2) {
           if(tableBuffer.getShort() == IndexData.COLUMN_UNUSED) {
             // found end of tdef, we want to insert before this
             ByteUtil.forward(tableBuffer, -2);
             break;
           }
 
           ByteUtil.forward(tableBuffer, 8);
 
           // keep reading ...
         }
 
         // write new column usage map info
         umapPos = tableBuffer.position();
         ByteUtil.insertEmptyData(tableBuffer, 10);
         ColumnImpl.writeColUsageMapDefinition(
             mutator, column, tableBuffer);
       }
 
       // sanity check the updates
       validateTableDefUpdate(mutator, tableBuffer);
 
       // before writing the new table def, create the column
       newCol = ColumnImpl.create(this, tableBuffer, colDefPos,
                                  column.getName(), _columns.size());
       newCol.setColumnIndex(_columns.size());
 
       ////
       // write updated table def back to the database
       writeTableDefinitionBuffer(tableBuffer, _tableDefPageNumber, mutator,
                                  mutator.getNextPages());
       success = true;
 
     } finally {
       if(!success) {
         // need to discard modified table buffer
         _tableDefBufferH.invalidate();
       }
     }
 
     ////
     // now, update current TableImpl
 
     _columns.add(newCol);
     ++_maxColumnCount;
     if(newCol.isVariableLength()) {
       _varColumns.add(newCol);
       ++_maxVarColumnCount;
     }
     if(newCol.isAutoNumber()) {
       _autoNumColumns.add(newCol);
     }
     if(newCol.isCalculated()) {
       _calcColEval.add(newCol);
     }
 
     if(umapPos >= 0) {
       // read column usage map
       tableBuffer.position(umapPos);
       readColumnUsageMaps(tableBuffer);
     }
 
     newCol.postTableLoadInit();
 
     if(!isSystem()) {
       // after fully constructed, allow column validator to be configured (but
       // only for user tables)
       newCol.initColumnValidator();
     }
 
     // save any column properties
     Map<String,PropertyMap.Property> colProps = column.getProperties();
     if(colProps != null) {
       newCol.getProperties().putAll(colProps.values());
       getProperties().save();
     }
 
     completeTableMutation(tableBuffer);
 
     return newCol;
   }
 
   /**
    * Writes a index defined by the given TableUpdater to this table.
    * @usage _advanced_method_
    */
   protected IndexData mutateAddIndexData(TableUpdater mutator) throws IOException
   {
     IndexBuilder index = mutator.getIndex();
     JetFormat format = mutator.getFormat();
 
     ////
     // calculate how much more space we need in the table def
     mutator.addTdefLen(format.SIZE_INDEX_DEFINITION +
                        format.SIZE_INDEX_COLUMN_BLOCK);
 
     ////
     // load current table definition and add space for new info
     ByteBuffer tableBuffer = loadCompleteTableDefinitionBufferForUpdate(
         mutator);
 
     IndexData newIdxData = null;
     boolean success = false;
     try {
 
       ////
       // update various bits of the table def
       ByteUtil.forward(tableBuffer, 39);
       tableBuffer.putInt(_indexCount + 1);
 
       // move to end of index data def blocks
       tableBuffer.position(format.SIZE_TDEF_HEADER +
                            (_indexCount * format.SIZE_INDEX_DEFINITION));
 
       // write index row count definition (empty initially)
       ByteUtil.insertEmptyData(tableBuffer, format.SIZE_INDEX_DEFINITION);
       IndexData.writeRowCountDefinitions(mutator, tableBuffer, 1);
 
       // skip columns and column names
       ByteUtil.forward(tableBuffer,
                        (_columns.size() * format.SIZE_COLUMN_DEF_BLOCK));
       skipNames(tableBuffer, _columns.size());
 
       // move to end of current index datas
       ByteUtil.forward(tableBuffer, (_indexCount *
                                      format.SIZE_INDEX_COLUMN_BLOCK));
 
       // allocate usage maps and root page
       TableMutator.IndexDataState idxDataState = mutator.getIndexDataState(index);
       int rootPageNumber = getPageChannel().allocateNewPage();
       Map.Entry<Integer,Integer> umapInfo = addUsageMaps(1, rootPageNumber);
       idxDataState.setRootPageNumber(rootPageNumber);
       idxDataState.setUmapPageNumber(umapInfo.getKey());
       idxDataState.setUmapRowNumber(umapInfo.getValue().byteValue());
 
       // write index data def
       int idxDataDefPos = tableBuffer.position();
       ByteUtil.insertEmptyData(tableBuffer, format.SIZE_INDEX_COLUMN_BLOCK);
       IndexData.writeDefinition(mutator, tableBuffer, idxDataState, null);
 
       // sanity check the updates
       validateTableDefUpdate(mutator, tableBuffer);
 
       // before writing the new table def, create the index data
       tableBuffer.position(0);
       newIdxData = IndexData.create(
           this, tableBuffer, idxDataState.getIndexDataNumber(), format);
       tableBuffer.position(idxDataDefPos);
       newIdxData.read(tableBuffer, _columns);
 
       ////
       // write updated table def back to the database
       writeTableDefinitionBuffer(tableBuffer, _tableDefPageNumber, mutator,
                                  mutator.getNextPages());
       success = true;
 
     } finally {
       if(!success) {
         // need to discard modified table buffer
         _tableDefBufferH.invalidate();
       }
     }
 
     ////
     // now, update current TableImpl
 
     for(IndexData.ColumnDescriptor iCol : newIdxData.getColumns()) {
       _indexColumns.add(iCol.getColumn());
     }
 
     ++_indexCount;
     _indexDatas.add(newIdxData);
 
     completeTableMutation(tableBuffer);
 
     // don't forget to populate the new index
     populateIndexData(newIdxData);
 
     return newIdxData;
   }
 
   private void populateIndexData(IndexData idxData)
     throws IOException
   {
     // grab the columns involved in this index
     List<ColumnImpl> idxCols = new ArrayList<>();
     for(IndexData.ColumnDescriptor col : idxData.getColumns()) {
       idxCols.add(col.getColumn());
     }
 
     // iterate through all the rows and add them to the index
     Object[] rowVals = new Object[_columns.size()];
     for(Row row : getDefaultCursor().newIterable().addColumns(idxCols)) {
       for(Column col : idxCols) {
         col.setRowValue(rowVals, col.getRowValue(row));
       }
 
       IndexData.commitAll(
           idxData.prepareAddRow(rowVals, (RowIdImpl)row.getId(), null));
     }
 
     updateTableDefinition(0);
   }
 
   /**
    * Writes a index defined by the given TableUpdater to this table.
    * @usage _advanced_method_
    */
   protected IndexImpl mutateAddIndex(TableUpdater mutator) throws IOException
   {
     IndexBuilder index = mutator.getIndex();
     JetFormat format = mutator.getFormat();
 
     ////
     // calculate how much more space we need in the table def
     mutator.addTdefLen(format.SIZE_INDEX_INFO_BLOCK);
 
     int nameByteLen = DBMutator.calculateNameLength(index.getName());
     mutator.addTdefLen(nameByteLen);
 
     ////
     // load current table definition and add space for new info
     ByteBuffer tableBuffer = loadCompleteTableDefinitionBufferForUpdate(
         mutator);
 
     IndexImpl newIdx = null;
     boolean success = false;
     try {
 
       ////
       // update various bits of the table def
       ByteUtil.forward(tableBuffer, 35);
       tableBuffer.putInt(_logicalIndexCount + 1);
 
       // move to end of index data def blocks
       tableBuffer.position(format.SIZE_TDEF_HEADER +
                            (_indexCount * format.SIZE_INDEX_DEFINITION));
 
       // skip columns and column names
       ByteUtil.forward(tableBuffer,
                        (_columns.size() * format.SIZE_COLUMN_DEF_BLOCK));
       skipNames(tableBuffer, _columns.size());
 
       // move to end of current index datas
       ByteUtil.forward(tableBuffer, (_indexCount *
                                      format.SIZE_INDEX_COLUMN_BLOCK));
       // move to end of current indexes
       ByteUtil.forward(tableBuffer, (_logicalIndexCount *
                                      format.SIZE_INDEX_INFO_BLOCK));
 
       int idxDefPos = tableBuffer.position();
       ByteUtil.insertEmptyData(tableBuffer, format.SIZE_INDEX_INFO_BLOCK);
       IndexImpl.writeDefinition(mutator, index, tableBuffer);
 
       // skip existing index names and write new name
       skipNames(tableBuffer, _logicalIndexCount);
       ByteUtil.insertEmptyData(tableBuffer, nameByteLen);
       writeName(tableBuffer, index.getName(), mutator.getCharset());
 
       // sanity check the updates
       validateTableDefUpdate(mutator, tableBuffer);
 
       // before writing the new table def, create the index
       tableBuffer.position(idxDefPos);
       newIdx = new IndexImpl(tableBuffer, _indexDatas, format);
       newIdx.setName(index.getName());
 
       ////
       // write updated table def back to the database
       writeTableDefinitionBuffer(tableBuffer, _tableDefPageNumber, mutator,
                                  mutator.getNextPages());
       success = true;
 
     } finally {
       if(!success) {
         // need to discard modified table buffer
         _tableDefBufferH.invalidate();
       }
     }
 
     ////
     // now, update current TableImpl
 
     ++_logicalIndexCount;
     _indexes.add(newIdx);
 
     completeTableMutation(tableBuffer);
 
     return newIdx;
   }
 
   private void validateTableDefUpdate(TableUpdater mutator, ByteBuffer tableBuffer)
   {
     if(!mutator.validateUpdatedTdef(tableBuffer)) {
       throw new IllegalStateException(
           withErrorContext("Failed updating table definition (unexpected length)"));
     }
   }
 
   private void completeTableMutation(ByteBuffer tableBuffer)
   {
     // lastly, may need to clear table def buffer
     _tableDefBufferH.possiblyInvalidate(_tableDefPageNumber, tableBuffer);
 
     // update any foreign key enforcing
     _fkEnforcer.reset();
 
     // update modification count so any active RowStates can keep themselves
     // up-to-date
     ++_modCount;
   }
 
   /**
    * Skips the given number of names in the table buffer.
    */
   private static void skipNames(ByteBuffer tableBuffer, int count) {
     for(int i = 0; i < count; ++i) {
       ByteUtil.forward(tableBuffer, tableBuffer.getShort());
     }
   }
 
   private ByteBuffer loadCompleteTableDefinitionBufferForUpdate(
       TableUpdater mutator)
     throws IOException
   {
     // load complete table definition
     ByteBuffer tableBuffer = _tableDefBufferH.setPage(getPageChannel(),
                                                       _tableDefPageNumber);
     tableBuffer = loadCompleteTableDefinitionBuffer(
         tableBuffer, mutator.getNextPages());
 
     // make sure the table buffer has enough room for the new info
     int addedLen = mutator.getAddedTdefLen();
     int origTdefLen = tableBuffer.getInt(8);
     mutator.setOrigTdefLen(origTdefLen);
     int newTdefLen = origTdefLen + addedLen;
     while(newTdefLen > tableBuffer.capacity()) {
       tableBuffer = expandTableBuffer(tableBuffer);
       tableBuffer.flip();
     }
 
     tableBuffer.limit(origTdefLen);
 
     // set new tdef length
     tableBuffer.position(8);
     tableBuffer.putInt(newTdefLen);
 
     return tableBuffer;
   }
 
   /**
    * Adds some usage maps for use with this table.  This method is expected to
    * be called with a small-ish number of requested usage maps.
    */
   private Map.Entry<Integer,Integer> addUsageMaps(
       int numMaps, Integer firstUsedPage)
     throws IOException
   {
     JetFormat format = getFormat();
     PageChannel pageChannel = getPageChannel();
     int umapRowLength = format.OFFSET_USAGE_MAP_START +
       format.USAGE_MAP_TABLE_BYTE_LENGTH;
     int totalUmapSpaceUsage = getRowSpaceUsage(umapRowLength, format) * numMaps;
     int umapPageNumber = PageChannel.INVALID_PAGE_NUMBER;
     int firstRowNum = -1;
     int freeSpace = 0;
 
     // search currently known usage map buffers to find one with enough free
     // space (the numMaps should always be small enough to put them all on one
     // page).  pages will free space will probaby be newer pages (higher
     // numbers), so we sort in reverse order.
     Set<Integer> knownPages = new TreeSet<>(Collections.reverseOrder());
     collectUsageMapPages(knownPages);
 
     ByteBuffer umapBuf = pageChannel.createPageBuffer();
     for(Integer pageNum : knownPages) {
       pageChannel.readPage(umapBuf, pageNum);
       freeSpace = umapBuf.getShort(format.OFFSET_FREE_SPACE);
       if(freeSpace >= totalUmapSpaceUsage) {
         // found a page!
         umapPageNumber = pageNum;
         firstRowNum = getRowsOnDataPage(umapBuf, format);
         break;
       }
     }
 
     if(umapPageNumber == PageChannel.INVALID_PAGE_NUMBER) {
 
       // didn't find any existing pages, need to create a new one
       umapPageNumber = pageChannel.allocateNewPage();
       freeSpace = format.DATA_PAGE_INITIAL_FREE_SPACE;
       firstRowNum = 0;
       umapBuf = createUsageMapDefPage(pageChannel, freeSpace);
     }
 
     // write the actual usage map defs
     int rowStart = findRowEnd(umapBuf, firstRowNum, format) - umapRowLength;
     int umapRowNum = firstRowNum;
     for(int i = 0; i < numMaps; ++i) {
       umapBuf.putShort(getRowStartOffset(umapRowNum, format), (short)rowStart);
       umapBuf.put(rowStart, UsageMap.MAP_TYPE_INLINE);
 
       int dataOffset = rowStart + 1;
       if(firstUsedPage != null) {
         // fill in the first used page of the usage map
         umapBuf.putInt(dataOffset, firstUsedPage);
         dataOffset += 4;
         umapBuf.put(dataOffset, (byte)1);
         dataOffset++;
       }
 
       // zero remaining row data
       ByteUtil.clearRange(umapBuf, dataOffset, (rowStart + umapRowLength));
 
       rowStart -= umapRowLength;
       ++umapRowNum;
     }
 
     // finish the page
     freeSpace -= totalUmapSpaceUsage;
     umapBuf.putShort(format.OFFSET_FREE_SPACE, (short)freeSpace);
     umapBuf.putShort(format.OFFSET_NUM_ROWS_ON_DATA_PAGE,
                      (short)umapRowNum);
     pageChannel.writePage(umapBuf, umapPageNumber);
 
     return new AbstractMap.SimpleImmutableEntry<Integer,Integer>(
         umapPageNumber, firstRowNum);
   }
 
   void collectUsageMapPages(Collection<Integer> pages) {
     pages.add(_ownedPages.getTablePageNumber());
     pages.add(_freeSpacePages.getTablePageNumber());
 
     for(IndexData idx : _indexDatas) {
       idx.collectUsageMapPages(pages);
     }
 
     for(ColumnImpl col : _columns) {
       col.collectUsageMapPages(pages);
     }
   }
 
   /**
    * @param buffer Buffer to write to
    */
   private static void writeTableDefinitionHeader(
       TableCreator creator, ByteBuffer buffer, int totalTableDefSize)
   {
     List<ColumnBuilder> columns = creator.getColumns();
 
     //Start writing the tdef
     writeTablePageHeader(buffer);
     buffer.putInt(totalTableDefSize);  //Length of table def
     buffer.putInt(MAGIC_TABLE_NUMBER); // seemingly constant magic value
     buffer.putInt(0);  //Number of rows
     buffer.putInt(0); //Last Autonumber
     buffer.put((byte) 1); // this makes autonumbering work in access
     // 3 constant zero bytes, the next complex autonumber value, then 8 bytes
     // which hold whatever the page held before the definition was written
     for (int i = 0; i < 15; i++) {
       buffer.put((byte) 0);
     }
     buffer.put(TYPE_USER); //Table type
     buffer.putShort((short) columns.size()); //Max columns a row will have
     buffer.putShort(ColumnImpl.countVariableLength(columns));  //Number of variable columns in table
     buffer.putShort((short) columns.size()); //Number of columns in table
     buffer.putInt(creator.getLogicalIndexCount());  //Number of logical indexes in table
     buffer.putInt(creator.getIndexCount());  //Number of indexes in table
     buffer.put((byte) 0); //Usage map row number
     ByteUtil.put3ByteInt(buffer, creator.getUmapPageNumber());  //Usage map page number
     buffer.put((byte) 1); //Free map row number
     ByteUtil.put3ByteInt(buffer, creator.getUmapPageNumber());  //Free map page number
   }
 
   /**
    * Writes the page header for a table definition page
    * @param buffer Buffer to write to
    */
   private static void writeTablePageHeader(ByteBuffer buffer)
   {
     buffer.put(PageTypes.TABLE_DEF);  //Page type
     buffer.put((byte) 0x01); // constant 1 on every page type
     // free space on this page, which writeTableDefinitionBuffer fills in once
     // the length of the definition is known
     buffer.putShort((short) 0);
     buffer.putInt(0);  //Next TDEF page pointer
   }
 
   /**
    * Writes the given name into the given buffer in the format as expected by
    * {@link #readName}.
    */
   static void writeName(ByteBuffer buffer, String name, Charset charset)
   {
       ByteBuffer encName = ColumnImpl.encodeUncompressedText(name, charset);
       buffer.putShort((short) encName.remaining());
       buffer.put(encName);
   }
 
   /**
    * Create the usage map definition page buffer.  The "used pages" map is in
    * row 0, the "pages with free space" map is in row 1.  Index usage maps are
    * in subsequent rows.
    */
   private static void createUsageMapDefinitionBuffer(TableCreator creator)
     throws IOException
   {
     List<ColumnBuilder> lvalCols = creator.getLongValueColumns();
 
     // 2 table usage maps plus 1 for each index and 2 for each lval col
     int indexUmapEnd = 2 + creator.getIndexCount();
     int umapNum = indexUmapEnd + (lvalCols.size() * 2);
 
     JetFormat format = creator.getFormat();
     int umapRowLength = format.OFFSET_USAGE_MAP_START +
       format.USAGE_MAP_TABLE_BYTE_LENGTH;
     int umapSpaceUsage = getRowSpaceUsage(umapRowLength, format);
     PageChannel pageChannel = creator.getPageChannel();
     int umapPageNumber = PageChannel.INVALID_PAGE_NUMBER;
     ByteBuffer umapBuf = null;
     int freeSpace = 0;
     int rowStart = 0;
     int umapRowNum = 0;
 
     for(int i = 0; i < umapNum; ++i) {
 
       if(umapBuf == null) {
 
         // need new page for usage maps
         if(umapPageNumber == PageChannel.INVALID_PAGE_NUMBER) {
           // first umap page has already been reserved
           umapPageNumber = creator.getUmapPageNumber();
         } else {
           // need another umap page
           umapPageNumber = creator.reservePageNumber();
         }
 
         freeSpace = format.DATA_PAGE_INITIAL_FREE_SPACE;
 
         umapBuf = createUsageMapDefPage(pageChannel, freeSpace);
 
         rowStart = findRowEnd(umapBuf, 0, format) - umapRowLength;
         umapRowNum = 0;
       }
 
       umapBuf.putShort(getRowStartOffset(umapRowNum, format), (short)rowStart);
 
       if(i == 0) {
 
         // table "owned pages" map definition
         umapBuf.put(rowStart, UsageMap.MAP_TYPE_REFERENCE);
 
       } else if(i == 1) {
 
         // table "free space pages" map definition
         umapBuf.put(rowStart, UsageMap.MAP_TYPE_INLINE);
 
       } else if(i < indexUmapEnd) {
 
         // index umap
         int indexIdx = i - 2;
         TableMutator.IndexDataState idxDataState =
           creator.getIndexDataStates().get(indexIdx);
 
         // allocate root page for the index
         int rootPageNumber = pageChannel.allocateNewPage();
 
         // stash info for later use
         idxDataState.setRootPageNumber(rootPageNumber);
         idxDataState.setUmapRowNumber((byte)umapRowNum);
         idxDataState.setUmapPageNumber(umapPageNumber);
 
         // index map definition, including initial root page
         umapBuf.put(rowStart, UsageMap.MAP_TYPE_INLINE);
         umapBuf.putInt(rowStart + 1, rootPageNumber);
         umapBuf.put(rowStart + 5, (byte)1);
 
       } else {
 
         // long value column umaps
         int lvalColIdx = i - indexUmapEnd;
         int umapType = lvalColIdx % 2;
         lvalColIdx /= 2;
 
         ColumnBuilder lvalCol = lvalCols.get(lvalColIdx);
         TableMutator.ColumnState colState =
           creator.getColumnState(lvalCol);
 
         umapBuf.put(rowStart, UsageMap.MAP_TYPE_INLINE);
 
         if((umapType == 1) &&
            (umapPageNumber != colState.getUmapPageNumber())) {
           // we want to force both usage maps for a column to be on the same
           // data page, so just discard the previous one we wrote
           --i;
           umapType = 0;
         }
 
         if(umapType == 0) {
           // lval column "owned pages" usage map
           colState.setUmapOwnedRowNumber((byte)umapRowNum);
           colState.setUmapPageNumber(umapPageNumber);
         } else {
           // lval column "free space pages" usage map (always on same page)
           colState.setUmapFreeRowNumber((byte)umapRowNum);
         }
       }
 
       rowStart -= umapRowLength;
       freeSpace -= umapSpaceUsage;
       ++umapRowNum;
 
       if((freeSpace <= umapSpaceUsage) || (i == (umapNum - 1))) {
         // finish current page
         umapBuf.putShort(format.OFFSET_FREE_SPACE, (short)freeSpace);
         umapBuf.putShort(format.OFFSET_NUM_ROWS_ON_DATA_PAGE,
                          (short)umapRowNum);
         pageChannel.writePage(umapBuf, umapPageNumber);
         umapBuf = null;
       }
     }
   }
 
   private static ByteBuffer createUsageMapDefPage(
       PageChannel pageChannel, int freeSpace)
   {
     ByteBuffer umapBuf = pageChannel.createPageBuffer();
     umapBuf.put(PageTypes.DATA);
     umapBuf.put((byte) 0x1);  // constant 1 on every page type
     umapBuf.putShort((short)freeSpace);  //Free space in page
     umapBuf.putInt(0); //Table definition
     umapBuf.putInt(0); // write stamp, see newDataPage
     umapBuf.putShort((short)0); //Number of records on this page
     return umapBuf;
   }
 
   /**
    * Returns a single ByteBuffer which contains the entire table definition
    * (which may span multiple database pages).
    */
   private ByteBuffer loadCompleteTableDefinitionBuffer(
       ByteBuffer tableBuffer, List<Integer> pages)
     throws IOException
   {
     int nextPage = tableBuffer.getInt(getFormat().OFFSET_NEXT_TABLE_DEF_PAGE);
     ByteBuffer nextPageBuffer = null;
     while (nextPage != 0) {
       if(pages != null) {
         pages.add(nextPage);
       }
       if (nextPageBuffer == null) {
         nextPageBuffer = getPageChannel().createPageBuffer();
       }
       getPageChannel().readPage(nextPageBuffer, nextPage);
       nextPage = nextPageBuffer.getInt(getFormat().OFFSET_NEXT_TABLE_DEF_PAGE);
       tableBuffer = expandTableBuffer(tableBuffer);
       tableBuffer.put(nextPageBuffer.array(), 8, getFormat().PAGE_SIZE - 8);
       tableBuffer.flip();
     }
     return tableBuffer;
   }
 
   private ByteBuffer expandTableBuffer(ByteBuffer tableBuffer) {
       ByteBuffer newBuffer = PageChannel.createBuffer(
           tableBuffer.capacity() + getFormat().PAGE_SIZE - 8);
       newBuffer.put(tableBuffer);
       return newBuffer;
   }
 
   private void readColumnDefinitions(ByteBuffer tableBuffer, short columnCount)
     throws IOException
   {
     int colOffset = getFormat().OFFSET_INDEX_DEF_BLOCK +
         _indexCount * getFormat().SIZE_INDEX_DEFINITION;
 
     tableBuffer.position(colOffset +
                          (columnCount * getFormat().SIZE_COLUMN_HEADER));
     List<String> colNames = new ArrayList<>(columnCount);
     for (int i = 0; i < columnCount; i++) {
       colNames.add(readName(tableBuffer));
     }
 
     int dispIndex = 0;
     for (int i = 0; i < columnCount; i++) {
       ColumnImpl column = ColumnImpl.create(this, tableBuffer,
           colOffset + (i * getFormat().SIZE_COLUMN_HEADER), colNames.get(i),
           dispIndex++);
       _columns.add(column);
       if(column.isVariableLength()) {
         // also shove it in the variable columns list, which is ordered
         // differently from the _columns list
         _varColumns.add(column);
       }
     }
 
     Collections.sort(_columns, ColumnImpl.DEFAULT_ORDER_COMPARATOR);
     initAutoNumberColumns();
     initCalculatedColumns();
 
     // setup the data index for the columns
     int colIdx = 0;
     for(ColumnImpl col : _columns) {
       col.setColumnIndex(colIdx++);
     }
 
     // sort variable length columns based on their index into the variable
     // length offset table, because we will write the columns in this order
     Collections.sort(_varColumns, VAR_LEN_COLUMN_COMPARATOR);
   }
 
   /**
    * Verifies that the column counts and sizes in the table definition cover
    * the columns which are actually defined.  A damaged table definition can
    * disagree, in which case writing a row would run off the end of the null
    * mask, the variable length offset table or the row buffer.
    * <p>
    * Rows can still be read from such a table, so this only records the
    * problem, which is reported when a row write is attempted.
    */
   private void validateColumnDefs() {
     int maxRowSize = getFormat().MAX_ROW_SIZE;
 
     for(ColumnImpl col : _columns) {
 
       if(col.getColumnNumber() >= _maxColumnCount) {
         setWriteDefError("column " + col.getName() + " has number " +
                          col.getColumnNumber() + ", which is outside the " +
                          "table column count " + _maxColumnCount);
       } else if(col.isVariableLength()) {
         if(col.getVarLenTableIndex() >= _maxVarColumnCount) {
           setWriteDefError("variable length column " + col.getName() +
                            " has offset index " + col.getVarLenTableIndex() +
                            ", which is outside the table variable length " +
                            "column count " + _maxVarColumnCount);
         }
       } else if((col.getFixedDataOffset() + col.getLength()) > maxRowSize) {
         setWriteDefError("fixed length column " + col.getName() +
                          " ends at offset " +
                          (col.getFixedDataOffset() + col.getLength()) +
                          ", which is beyond the maximum row size " +
                          maxRowSize);
       }
     }
   }
 
   /**
    * Records the first inconsistency found by {@link #validateColumnDefs}.
    */
   private void setWriteDefError(String reason) {
     if(_writeDefError != null) {
       return;
     }
     _writeDefError = "Table definition is corrupt, " + reason;
     LOG.log(Logger.Level.WARNING, withErrorContext(
                  _writeDefError + ".  Table is read-only"));
   }
 
   private void readIndexDefinitions(ByteBuffer tableBuffer) throws IOException
   {
     // read index column information
     for (int i = 0; i < _indexCount; i++) {
       IndexData idxData = _indexDatas.get(i);
       idxData.read(tableBuffer, _columns);
       // keep track of all columns involved in indexes
       for(IndexData.ColumnDescriptor iCol : idxData.getColumns()) {
         _indexColumns.add(iCol.getColumn());
       }
     }
 
     // read logical index info (may be more logical indexes than index datas)
     for (int i = 0; i < _logicalIndexCount; i++) {
       _indexes.add(new IndexImpl(tableBuffer, _indexDatas, getFormat()));
     }
 
     // read logical index names
     for (int i = 0; i < _logicalIndexCount; i++) {
       _indexes.get(i).setName(readName(tableBuffer));
     }
 
     Collections.sort(_indexes, IndexImpl.DEFAULT_ORDER_COMPARATOR);
   }
 
   private boolean readColumnUsageMaps(ByteBuffer tableBuffer)
     throws IOException
   {
     short umapColNum = tableBuffer.getShort();
     if(umapColNum == IndexData.COLUMN_UNUSED) {
       return false;
     }
 
     int pos = tableBuffer.position();
     UsageMap colOwnedPages = null;
     UsageMap colFreeSpacePages = null;
     try {
       colOwnedPages = UsageMap.read(getDatabase(), tableBuffer);
       colFreeSpacePages = UsageMap.read(getDatabase(), tableBuffer);
     } catch(IllegalStateException e) {
       // ignore invalid usage map info
       colOwnedPages = null;
       colFreeSpacePages = null;
       tableBuffer.position(pos + 8);
       LOG.log(Logger.Level.WARNING, withErrorContext("Invalid column " + umapColNum +
                                 " usage map definition: " + e));
     }
 
     for(ColumnImpl col : _columns) {
       if(col.getColumnNumber() == umapColNum) {
         col.setUsageMaps(colOwnedPages, colFreeSpacePages);
         break;
       }
     }
 
     return true;
   }
 
   /**
    * Writes the given page data to the given page number, clears any other
    * relevant buffers.
    */
   private void writeDataPage(ByteBuffer pageBuffer, int pageNumber)
     throws IOException
   {
     // write the page data
     getPageChannel().writePage(pageBuffer, pageNumber);
 
     // possibly invalidate the add row buffer if a different data buffer is
     // being written (e.g. this happens during deleteRow)
     _addRowBufferH.possiblyInvalidate(pageNumber, pageBuffer);
 
     // update modification count so any active RowStates can keep themselves
     // up-to-date
     ++_modCount;
   }
 
   /**
    * Returns a name read from the buffer at the current position. The
    * expected name format is the name length followed by the name
    * encoded using the {@link JetFormat#CHARSET}
    */
   private String readName(ByteBuffer buffer) {
     int nameLength = readNameLength(buffer);
     byte[] nameBytes = ByteUtil.getBytes(buffer, nameLength);
     return ColumnImpl.decodeUncompressedText(nameBytes,
                                          getDatabase().getCharset());
   }
 
   /**
    * Returns a name length read from the buffer at the current position.
    */
   private int readNameLength(ByteBuffer buffer) {
     return ByteUtil.getUnsignedVarInt(buffer, getFormat().SIZE_NAME_LENGTH);
   }
 
   @Override
   public Object[] asRow(Map<String,?> rowMap) {
     return asRow(rowMap, null, false);
   }
 
   /**
    * Converts a map of columnName -&gt; columnValue to an array of row values
    * appropriate for a call to {@link #addRow(Object...)}, where the generated
    * RowId will be an extra value at the end of the array.
    * @see ColumnImpl#RETURN_ROW_ID
    * @usage _intermediate_method_
    */
   public Object[] asRowWithRowId(Map<String,?> rowMap) {
     return asRow(rowMap, null, true);
   }
 
   @Override
   public Object[] asUpdateRow(Map<String,?> rowMap) {
     return asRow(rowMap, Column.KEEP_VALUE, false);
   }
 
   /**
    * @return the generated RowId added to a row of values created via {@link
    *         #asRowWithRowId}
    * @usage _intermediate_method_
    */
   public RowId getRowId(Object[] row) {
     return (RowId)row[_columns.size()];
   }
 
   /**
    * Converts a map of columnName -&gt; columnValue to an array of row values.
    */
   private Object[] asRow(Map<String,?> rowMap, Object defaultValue,
                          boolean returnRowId)
   {
     int len = _columns.size();
     if(returnRowId) {
       ++len;
     }
     Object[] row = new Object[len];
     if(defaultValue != null) {
       Arrays.fill(row, defaultValue);
     }
     if(returnRowId) {
       row[len - 1] = ColumnImpl.RETURN_ROW_ID;
     }
     if(rowMap == null) {
       return row;
     }
     for(ColumnImpl col : _columns) {
       if(rowMap.containsKey(col.getName())) {
         col.setRowValue(row, col.getRowValue(rowMap));
       }
     }
     return row;
   }
 
   @Override
   public Object[] addRow(Object... row) throws IOException {
     return addRows(Collections.singletonList(row), false).get(0);
   }
 
   @Override
   public <M extends Map<String,Object>> M addRowFromMap(M row)
     throws IOException
   {
     Object[] rowValues = asRow(row);
 
     addRow(rowValues);
 
     returnRowValues(row, rowValues, _columns);
     return row;
   }
 
   @Override
   public List<? extends Object[]> addRows(List<? extends Object[]> rows)
     throws IOException
   {
     return addRows(rows, true);
   }
 
   @Override
   public <M extends Map<String,Object>> List<M> addRowsFromMaps(List<M> rows)
     throws IOException
   {
     List<Object[]> rowValuesList = new ArrayList<>(rows.size());
     for(Map<String,Object> row : rows) {
       rowValuesList.add(asRow(row));
     }
 
     addRows(rowValuesList);
 
     for(int i = 0; i < rowValuesList.size(); ++i) {
       Map<String,Object> row = rows.get(i);
       Object[] rowValues = rowValuesList.get(i);
       returnRowValues(row, rowValues, _columns);
     }
     return rows;
   }
 
   private static void returnRowValues(Map<String,Object> row, Object[] rowValues,
                                       List<ColumnImpl> cols)
   {
     for(ColumnImpl col : cols) {
       col.setRowValue(row, col.getRowValue(rowValues));
     }
   }
 
   /**
    * Add multiple rows to this table, only writing to disk after all
    * rows have been written, and every time a data page is filled.
    * @param rows List of Object[] row values
    */
   protected List<? extends Object[]> addRows(List<? extends Object[]> rows,
                                              final boolean isBatchWrite)
     throws IOException
   {
     if(rows.isEmpty()) {
       return rows;
     }
 
     getPageChannel().startWrite();
     try {
 
       ByteBuffer dataPage = null;
       int pageNumber = PageChannel.INVALID_PAGE_NUMBER;
       int updateCount = 0;
       int autoNumAssignCount = 0;
       WriteRowState writeRowState =
         (!_autoNumColumns.isEmpty() ? new WriteRowState() : null);
       try {
 
         List<Object[]> dupeRows = null;
         final int numCols = _columns.size();
         for (int i = 0; i < rows.size(); i++) {
 
           // we need to make sure the row is the right length and is an
           // Object[] (fill with null if too short).  note, if the row is
           // copied the caller will not be able to access any generated
           // auto-number value, but if they need that info they should use a
           // row array of the right size/type!
           Object[] row = rows.get(i);
           if((row.length < numCols) || (row.getClass() != Object[].class)) {
             row = dupeRow(row, numCols);
             // copy the input rows to a modifiable list so we can update the
             // elements
             if(dupeRows == null) {
               dupeRows = new ArrayList<>(rows);
               rows = dupeRows;
             }
             // we copied the row, so put the copy back into the rows list
             dupeRows.set(i, row);
           }
 
           // handle various value massaging activities
           for(ColumnImpl column : _columns) {
             if(!column.isAutoNumber()) {
               Object val = column.getRowValue(row);
               if(val == null) {
                 val = column.generateDefaultValue();
               }
               // pass input value through column validator
               column.setRowValue(row, column.validate(val));
             }
           }
 
           // fill in autonumbers
           handleAutoNumbersForAdd(row, writeRowState);
           ++autoNumAssignCount;
 
           // need to assign calculated values after all the other fields are
           // filled in but before final validation
           _calcColEval.calculate(row);
 
           // run row validation if enabled
           if(_rowValidator != null) {
             _rowValidator.validate(row);
           }
 
           // write the row of data to a temporary buffer
           ByteBuffer rowData = createRow(
               row, _writeRowBufferH.getPageBuffer(getPageChannel()));
 
           int rowSize = rowData.remaining();
           if (rowSize > getFormat().MAX_ROW_SIZE) {
             throw createTooLargeException(rowSize);
           }
 
           // get page with space
           dataPage = findFreeRowSpace(rowSize, dataPage, pageNumber);
           pageNumber = _addRowBufferH.getPageNumber();
 
           // determine where this row will end up on the page
           int rowNum = getRowsOnDataPage(dataPage, getFormat());
 
           RowIdImplss/impl/RowIdImpl.html#RowIdImpl">RowIdImpl rowId = new RowIdImpl(pageNumber, rowNum);
 
           // before we actually write the row data, we verify all the database
           // constraints.
           if(!_indexDatas.isEmpty()) {
 
             IndexData.PendingChange idxChange = null;
             try {
 
               // handle foreign keys before adding to table
               _fkEnforcer.addRow(row);
 
               // prepare index updates
               for(IndexData indexData : _indexDatas) {
                 idxChange = indexData.prepareAddRow(row, rowId, idxChange);
               }
 
               // complete index updates
               IndexData.commitAll(idxChange);
 
             } catch(ConstraintViolationException ce) {
               IndexData.rollbackAll(idxChange);
               throw ce;
             }
           }
 
           // we have satisfied all the constraints, write the row
           addDataPageRow(dataPage, rowSize, getFormat(), 0);
           dataPage.put(rowData);
 
           // return rowTd if desired
           if((row.length > numCols) &&
              (row[numCols] == ColumnImpl.RETURN_ROW_ID)) {
             row[numCols] = rowId;
           }
 
           ++updateCount;
         }
 
         writeDataPage(dataPage, pageNumber);
 
         // Update tdef page
         updateTableDefinition(rows.size());
 
       } catch(Exception rowWriteFailure) {
 
         boolean isWriteFailure = isWriteFailure(rowWriteFailure);
 
         if(!isWriteFailure && (autoNumAssignCount > updateCount)) {
           // we assigned some autonumbers which won't get written.  attempt to
           // recover them so we don't get ugly "holes"
           restoreAutoNumbersFromAdd(rows.get(autoNumAssignCount - 1));
         }
 
         if(!isBatchWrite) {
           // just re-throw the original exception
           if(rowWriteFailure instanceof IOException) {
             throw (IOException)rowWriteFailure;
           }
           throw (RuntimeException)rowWriteFailure;
         }
 
         // attempt to resolve a partial batch write
         if(isWriteFailure) {
 
           // we don't really know the status of any of the rows, so clear the
           // update count
           updateCount = 0;
 
         } else if(updateCount > 0) {
 
           // attempt to flush the rows already written to disk
           try {
 
             writeDataPage(dataPage, pageNumber);
 
             // Update tdef page
             updateTableDefinition(updateCount);
 
           } catch(Exception flushFailure) {
             // the flush failure is "worse" as it implies possible database
             // corruption (failed write vs. a row failure which was not a
             // write failure).  we don't know the status of any rows at this
             // point (and the original failure is probably irrelevant)
             LOG.log(Logger.Level.WARNING, withErrorContext(
                     "Secondary row failure which preceded the write failure"),
                      rowWriteFailure);
             updateCount = 0;
             rowWriteFailure = flushFailure;
           }
         }
 
         throw new BatchUpdateException(
             updateCount, withErrorContext("Failed adding rows"),
             rowWriteFailure);
       }
 
     } finally {
       getPageChannel().finishWrite();
     }
 
     return rows;
   }
 
   private static boolean isWriteFailure(Throwable t) {
     while(t != null) {
       if((t instanceof IOException) && !(t instanceof JackcessException)) {
         return true;
       }
       t = t.getCause();
     }
     // some other sort of exception which is not a write failure
     return false;
   }
 
   @Override
   public Rowf="../../../../com/healthmarketscience/jackcess/Row.html#Row">Row updateRow(Row row) throws IOException {
     return updateRowFromMap(
         getDefaultCursor().getRowState(), (RowIdImpl)row.getId(), row);
   }
 
   /**
    * Update the row with the given id.  Provided RowId must have previously
    * been returned from this Table.
    * @return the given row, updated with the current row values
    * @throws IllegalStateException if the given row is not valid, or deleted.
    * @usage _intermediate_method_
    */
   public Object[] updateRow(RowId rowId, Object... row) throws IOException {
     return updateRow(
         getDefaultCursor().getRowState(), (RowIdImpl)rowId, row);
   }
 
   /**
    * Update the given column's value for the given row id.  Provided RowId
    * must have previously been returned from this Table.
    * @throws IllegalStateException if the given row is not valid, or deleted.
    * @usage _intermediate_method_
    */
   public void updateValue(Column column, RowId rowId, Object value)
     throws IOException
   {
     Object[] row = new Object[_columns.size()];
     Arrays.fill(row, Column.KEEP_VALUE);
     column.setRowValue(row, value);
 
     updateRow(rowId, row);
   }
 
   public <M extends Map<String,Object>> M updateRowFromMap(
       RowState rowState, RowIdImpl rowId, M row)
      throws IOException
   {
     Object[] rowValues = updateRow(rowState, rowId, asUpdateRow(row));
     returnRowValues(row, rowValues, _columns);
     return row;
   }
 
   /**
    * Update the row for the given rowId.
    * @usage _advanced_method_
    */
   public Object[] updateRow(RowState rowState, RowIdImpl rowId, Object... row)
     throws IOException
   {
     requireValidRowId(rowId);
 
     getPageChannel().startWrite();
     try {
 
       // ensure that the relevant row state is up-to-date
       ByteBuffer rowBuffer = positionAtRowData(rowState, rowId);
       int oldRowSize = rowBuffer.remaining();
 
       requireNonDeletedRow(rowState, rowId);
 
       // we need to make sure the row is the right length & type (fill with
       // null if too short).
       if((row.length < _columns.size()) || (row.getClass() != Object[].class)) {
         row = dupeRow(row, _columns.size());
       }
 
       // hang on to the raw values of var length columns we are "keeping".  this
       // will allow us to re-use pre-written var length data, which can save
       // space for things like long value columns.
       Map<ColumnImpl,byte[]> keepRawVarValues =
         (!_varColumns.isEmpty() ? new HashMap<>() : null);
 
       // handle various value massaging activities
       for(ColumnImpl column : _columns) {
 
         if(column.isAutoNumber()) {
           // handle these separately (below)
           continue;
         }
 
         Object rowValue = column.getRowValue(row);
         if(rowValue == Column.KEEP_VALUE) {
 
           // fill in any "keep value" fields (restore old value)
           rowValue = getRowColumn(getFormat(), rowBuffer, column, rowState,
                                   keepRawVarValues);
 
         } else {
 
           // set oldValue to something that could not possibly be a real value
           Object oldValue = Column.KEEP_VALUE;
           if(_indexColumns.contains(column)) {
             // read (old) row value to help update indexes
             oldValue = getRowColumn(getFormat(), rowBuffer, column, rowState,
                                     null);
           } else {
             oldValue = rowState.getRowCacheValue(column.getColumnIndex());
           }
 
           // if the old value was passed back in, we don't need to validate
           if(oldValue != rowValue) {
             // pass input value through column validator
             rowValue = column.validate(rowValue);
           }
         }
 
         column.setRowValue(row, rowValue);
       }
 
       // fill in autonumbers
       handleAutoNumbersForUpdate(row, rowBuffer, rowState);
 
       // need to assign calculated values after all the other fields are
       // filled in but before final validation
       _calcColEval.calculate(row);
 
       // run row validation if enabled
       if(_rowValidator != null) {
         _rowValidator.validate(row);
       }
 
       // generate new row bytes
       ByteBuffer newRowData = createRow(
           row, _writeRowBufferH.getPageBuffer(getPageChannel()), oldRowSize,
           keepRawVarValues);
 
       if (newRowData.limit() > getFormat().MAX_ROW_SIZE) {
         throw createTooLargeException(newRowData.limit());
       }
 
       if(!_indexDatas.isEmpty()) {
 
         IndexData.PendingChange idxChange = null;
         try {
 
           Object[] oldRowValues = rowState.getRowCacheValues();
 
           // check foreign keys before actually updating
           _fkEnforcer.updateRow(oldRowValues, row);
 
           // prepare index updates
           for(IndexData indexData : _indexDatas) {
             idxChange = indexData.prepareUpdateRow(oldRowValues, rowId, row,
                                                    idxChange);
           }
 
           // complete index updates
           IndexData.commitAll(idxChange);
 
         } catch(ConstraintViolationException ce) {
           IndexData.rollbackAll(idxChange);
           throw ce;
         }
       }
 
       // see if we can squeeze the new row data into the existing row
       rowBuffer.reset();
       int rowSize = newRowData.remaining();
 
       ByteBuffer dataPage = null;
       int pageNumber = PageChannel.INVALID_PAGE_NUMBER;
 
       if(oldRowSize >= rowSize) {
 
         // awesome, slap it in!
         rowBuffer.put(newRowData);
 
         // grab the page we just updated
         dataPage = rowState.getFinalPage();
         pageNumber = rowState.getFinalRowId().getPageNumber();
 
       } else {
 
         // bummer, need to find a new page for the data
         dataPage = findFreeRowSpace(rowSize, null,
                                     PageChannel.INVALID_PAGE_NUMBER);
         pageNumber = _addRowBufferH.getPageNumber();
 
         RowIdImpl headerRowId = rowState.getHeaderRowId();
         ByteBuffer headerPage = rowState.getHeaderPage();
         if(pageNumber == headerRowId.getPageNumber()) {
           // new row is on the same page as header row, share page
           dataPage = headerPage;
         }
 
         // write out the new row data (set the deleted flag on the new data row
         // so that it is ignored during normal table traversal)
         int rowNum = addDataPageRow(dataPage, rowSize, getFormat(),
                                     DELETED_ROW_MASK);
         dataPage.put(newRowData);
 
         // write the overflow info into the header row and clear out the
         // remaining header data
         rowBuffer = PageChannel.narrowBuffer(
             headerPage,
             findRowStart(headerPage, headerRowId.getRowNumber(), getFormat()),
             findRowEnd(headerPage, headerRowId.getRowNumber(), getFormat()));
         rowBuffer.put((byte)rowNum);
         ByteUtil.put3ByteInt(rowBuffer, pageNumber);
         ByteUtil.clearRemaining(rowBuffer);
 
         // set the overflow flag on the header row
         int headerRowIndex = getRowStartOffset(headerRowId.getRowNumber(),
                                                getFormat());
         headerPage.putShort(headerRowIndex,
                             (short)(headerPage.getShort(headerRowIndex)
                                     | OVERFLOW_ROW_MASK));
         if(pageNumber != headerRowId.getPageNumber()) {
           writeDataPage(headerPage, headerRowId.getPageNumber());
         }
       }
 
       writeDataPage(dataPage, pageNumber);
 
       updateTableDefinition(0);
 
     } finally {
       getPageChannel().finishWrite();
     }
 
     return row;
   }
 
   private ByteBuffer findFreeRowSpace(int rowSize, ByteBuffer dataPage,
                                       int pageNumber)
     throws IOException
   {
     // assume incoming page is modified
     boolean modifiedPage = true;
 
     if(dataPage == null) {
 
       // find owned page w/ free space
       dataPage = findFreeRowSpace(_ownedPages, _freeSpacePages,
                                   _addRowBufferH);
 
       if(dataPage == null) {
         // No data pages exist (with free space).  Create a new one.
         return newDataPage();
       }
 
       // found a page, see if it will work
       pageNumber = _addRowBufferH.getPageNumber();
       // since we just loaded this page, it is not yet modified
       modifiedPage = false;
     }
 
     if(!rowFitsOnDataPage(rowSize, dataPage, getFormat())) {
 
       // Last data page is full.  Write old one and create a new one.
       if(modifiedPage) {
         writeDataPage(dataPage, pageNumber);
       }
       _freeSpacePages.removePageNumber(pageNumber);
 
       dataPage = newDataPage();
     }
 
     return dataPage;
   }
 
   static ByteBuffer findFreeRowSpace(
       UsageMap./../../com/healthmarketscience/jackcess/impl/UsageMap.html#UsageMap">UsageMap ownedPages, UsageMap freeSpacePages,
       TempPageHolder rowBufferH)
     throws IOException
   {
     // find last data page (Not bothering to check other pages for free
     // space.)
     UsageMap.PageCursor revPageCursor = ownedPages.cursor();
     revPageCursor.afterLast();
     while(true) {
       int tmpPageNumber = revPageCursor.getPreviousPage();
       if(tmpPageNumber < 0) {
         break;
       }
       // only use if actually listed in free space pages
       if(!freeSpacePages.containsPageNumber(tmpPageNumber)) {
         continue;
       }
       ByteBuffer dataPage = rowBufferH.setPage(ownedPages.getPageChannel(),
                                                tmpPageNumber);
       if(dataPage.get() == PageTypes.DATA) {
         // found last data page with free space
         return dataPage;
       }
     }
 
     return null;
   }
 
   /**
    * Updates the table definition after rows are modified.
    */
   private void updateTableDefinition(int rowCountInc) throws IOException
   {
     // load table definition
     ByteBuffer tdefPage = _tableDefBufferH.setPage(getPageChannel(),
                                                    _tableDefPageNumber);
 
     // make sure rowcount and autonumber are up-to-date
     _rowCount += rowCountInc;
     tdefPage.putInt(getFormat().OFFSET_NUM_ROWS, _rowCount);
     tdefPage.putInt(getFormat().OFFSET_NEXT_AUTO_NUMBER, _lastLongAutoNumber);
     int ctypeOff = getFormat().OFFSET_NEXT_COMPLEX_AUTO_NUMBER;
     if(ctypeOff >= 0) {
       tdefPage.putInt(ctypeOff, _lastComplexTypeAutoNumber);
     }
 
     // write any index changes
     for (IndexData indexData : _indexDatas) {
       // write the unique entry count for the index to the table definition
       // page
       tdefPage.putInt(indexData.getUniqueEntryCountOffset(),
                       indexData.getUniqueEntryCount());
       // write the entry page for the index
       indexData.update();
     }
 
     // write modified table definition
     getPageChannel().writePage(tdefPage, _tableDefPageNumber);
   }
 
   /**
    * Create a new data page
    * @return Page number of the new page
    */
   private ByteBuffer newDataPage() throws IOException {
     ByteBuffer dataPage = _addRowBufferH.setNewPage(getPageChannel());
     dataPage.put(PageTypes.DATA); //Page type
     dataPage.put((byte) 1); // constant 1 on every page type
     dataPage.putShort((short)getFormat().DATA_PAGE_INITIAL_FREE_SPACE); //Free space in this page
     dataPage.putInt(_tableDefPageNumber); //Page pointer to table definition
     // ms access stamps GetTickCount() here when it writes the page, and
     // writes zero when it initializes one
     dataPage.putInt(0);
     dataPage.putShort((short)0); //Number of rows on this page
     int pageNumber = _addRowBufferH.getPageNumber();
     getPageChannel().writePage(dataPage, pageNumber);
     _ownedPages.addPageNumber(pageNumber);
     _freeSpacePages.addPageNumber(pageNumber);
     return dataPage;
   }
 
   // exposed for unit tests
   protected ByteBuffer createRow(Object[] rowArray, ByteBuffer buffer)
     throws IOException
   {
     return createRow(rowArray, buffer, 0,
                      Collections.<ColumnImpl,byte[]>emptyMap());
   }
 
   /**
    * Serialize a row of Objects into a byte buffer.
    *
    * @param rowArray row data, expected to be correct length for this table
    * @param buffer buffer to which to write the row data
    * @param minRowSize min size for result row
    * @param rawVarValues optional, pre-written values for var length columns
    *                     (enables re-use of previously written values).
    * @return the given buffer, filled with the row data
    */
   private ByteBuffer createRow(Object[] rowArray, ByteBuffer buffer,
                                int minRowSize,
                                Map<ColumnImpl,byte[]> rawVarValues)
     throws IOException
   {
     if(_writeDefError != null) {
       // we cannot lay out a row without a table definition which matches the
       // columns
       throw new JackcessException(withErrorContext(_writeDefError));
     }
 
     buffer.putShort(_maxColumnCount);
     NullMask/impl/NullMask.html#NullMask">NullMask nullMask = new NullMask(_maxColumnCount);
 
     //Fixed length column data comes first
     int fixedDataStart = buffer.position();
     int fixedDataEnd = fixedDataStart;
     for (ColumnImpl col : _columns) {
 
       if(col.isVariableLength()) {
         continue;
       }
 
       Object rowValue = col.getRowValue(rowArray);
 
       if (col.storeInNullMask()) {
 
         if(col.writeToNullMask(rowValue)) {
           nullMask.markNotNull(col);
         }
         rowValue = null;
       }
 
       if(rowValue != null) {
 
         // we have a value to write
         nullMask.markNotNull(col);
 
         // remainingRowLength is ignored when writing fixed length data
         buffer.position(fixedDataStart + col.getFixedDataOffset());
         buffer.put(col.write(rowValue, 0));
       }
 
       // always insert space for the entire fixed data column length
       // (including null values), access expects the row to always be at least
       // big enough to hold all fixed values
       buffer.position(fixedDataStart + col.getFixedDataOffset() +
                       col.getLength());
 
       // keep track of the end of fixed data
       if(buffer.position() > fixedDataEnd) {
         fixedDataEnd = buffer.position();
       }
 
     }
 
     // reposition at end of fixed data
     buffer.position(fixedDataEnd);
 
     // only need this info if this table contains any var length data
     if(_maxVarColumnCount > 0) {
 
       int maxRowSize = getFormat().MAX_ROW_SIZE;
 
       // figure out how much space remains for var length data.  first,
       // account for already written space
       maxRowSize -= buffer.position();
       // now, account for trailer space
       int trailerSize = (nullMask.byteSize() + 4 + (_maxVarColumnCount * 2));
       maxRowSize -= trailerSize;
 
       // for each non-null long value column we need to reserve a small
       // amount of space so that we don't end up running out of row space
       // later by being too greedy
       for (ColumnImpl varCol : _varColumns) {
         if((varCol.getType().isLongValue()) &&
            (varCol.getRowValue(rowArray) != null)) {
           maxRowSize -= getFormat().SIZE_LONG_VALUE_DEF;
         }
       }
 
       //Now write out variable length column data
       short[] varColumnOffsets = new short[_maxVarColumnCount];
       int varColumnOffsetsIndex = 0;
       for (ColumnImpl varCol : _varColumns) {
         short offset = (short) buffer.position();
         Object rowValue = varCol.getRowValue(rowArray);
         if (rowValue != null) {
           // we have a value
           nullMask.markNotNull(varCol);
 
           byte[] rawValue = null;
           ByteBuffer varDataBuf = null;
           if(((rawValue = rawVarValues.get(varCol)) != null) &&
              (rawValue.length <= maxRowSize)) {
             // save time and potentially db space, re-use raw value
             varDataBuf = ByteBuffer.wrap(rawValue);
           } else {
             // write column value
             varDataBuf = varCol.write(rowValue, maxRowSize);
           }
 
           maxRowSize -= varDataBuf.remaining();
           if(varCol.getType().isLongValue()) {
             // we already accounted for some amount of the long value data
             // above.  add that space back so we don't double count
             maxRowSize += getFormat().SIZE_LONG_VALUE_DEF;
           }
           try {
             buffer.put(varDataBuf);
           } catch(BufferOverflowException e) {
             // if the data is too big for the buffer, then we have gone over
             // the max row size
             throw createTooLargeException(buffer.limit());
           }
         }
 
         // we do a loop here so that we fill in offsets for deleted columns
         while(varColumnOffsetsIndex <= varCol.getVarLenTableIndex()) {
           varColumnOffsets[varColumnOffsetsIndex++] = offset;
         }
       }
 
       // fill in offsets for any remaining deleted columns
       while(varColumnOffsetsIndex < varColumnOffsets.length) {
         varColumnOffsets[varColumnOffsetsIndex++] = (short) buffer.position();
       }
 
       // record where we stopped writing
       int eod = buffer.position();
 
       if(buffer.remaining() < trailerSize) {
         // we don't have enough space left for required trailer
         throw createTooLargeException(eod + trailerSize);
       }
 
       // insert padding if necessary
       padRowBuffer(buffer, minRowSize, trailerSize);
 
       buffer.putShort((short) eod); //EOD marker
 
       //Now write out variable length offsets
       //Offsets are stored in reverse order
       for (int i = _maxVarColumnCount - 1; i >= 0; i--) {
         buffer.putShort(varColumnOffsets[i]);
       }
       buffer.putShort(_maxVarColumnCount);  //Number of var length columns
 
     } else {
 
       // insert padding for row w/ no var cols
       padRowBuffer(buffer, minRowSize, nullMask.byteSize());
     }
 
     nullMask.write(buffer);  //Null mask
     buffer.flip();
     return buffer;
   }
 
   private InvalidValueException createTooLargeException(int size) {
     return new InvalidValueException(
         withErrorContext(
             "Row size " + size + " is too large (max " +
             getFormat().MAX_ROW_SIZE + ")"));
   }
 
   /**
    * Fill in all autonumber column values for add.
    */
   private void handleAutoNumbersForAdd(Object[] row, WriteRowState writeRowState)
     throws IOException
   {
     if(_autoNumColumns.isEmpty()) {
       return;
     }
 
     boolean enableInsert = isAllowAutoNumberInsert();
     writeRowState.resetAutoNumber();
     for(ColumnImpl col : _autoNumColumns) {
 
       // ignore input row value, use original row value (unless explicitly
       // enabled)
       Object inRowValue = getInputAutoNumberRowValue(enableInsert, col, row);
 
       ColumnImpl.AutoNumberGenerator autoNumGen = col.getAutoNumberGenerator();
       Object rowValue = ((inRowValue == null) ?
                          autoNumGen.getNext(writeRowState) :
                          autoNumGen.handleInsert(writeRowState, inRowValue));
 
       col.setRowValue(row, rowValue);
     }
   }
 
   /**
    * Fill in all autonumber column values for update.
    */
   private void handleAutoNumbersForUpdate(Object[] row, ByteBuffer rowBuffer,
                                           RowState rowState)
     throws IOException
   {
     if(_autoNumColumns.isEmpty()) {
       return;
     }
 
     boolean enableInsert = isAllowAutoNumberInsert();
     rowState.resetAutoNumber();
     for(ColumnImpl col : _autoNumColumns) {
 
       // ignore input row value, use original row value (unless explicitly
       // enabled)
       Object inRowValue = getInputAutoNumberRowValue(enableInsert, col, row);
 
       Object rowValue =
         ((inRowValue == null) ?
          getRowColumn(getFormat(), rowBuffer, col, rowState, null) :
          col.getAutoNumberGenerator().handleInsert(rowState, inRowValue));
 
       col.setRowValue(row, rowValue);
     }
   }
 
   /**
    * Optionally get the input autonumber row value for the given column from
    * the given row if one was provided.
    */
   private static Object getInputAutoNumberRowValue(
       boolean enableInsert, ColumnImpl col, Object[] row)
   {
     if(!enableInsert) {
       return null;
     }
 
     Object inRowValue = col.getRowValue(row);
     if((inRowValue == Column.KEEP_VALUE) || (inRowValue == Column.AUTO_NUMBER)) {
       // these "special" values both behave like nothing was given
       inRowValue = null;
     }
     return inRowValue;
   }
 
   /**
    * Restores all autonumber column values from a failed add row.
    */
   private void restoreAutoNumbersFromAdd(Object[] row)
   {
     if(_autoNumColumns.isEmpty()) {
       return;
     }
 
     for(ColumnImpl col : _autoNumColumns) {
       // restore the last value from the row
       col.getAutoNumberGenerator().restoreLast(col.getRowValue(row));
     }
   }
 
   private static void padRowBuffer(ByteBuffer buffer, int minRowSize,
                                    int trailerSize)
   {
     int pos = buffer.position();
     if((pos + trailerSize) < minRowSize) {
       // pad the row to get to the min byte size
       int padSize = minRowSize - (pos + trailerSize);
       ByteUtil.clearRange(buffer, pos, pos + padSize);
       ByteUtil.forward(buffer, padSize);
     }
   }
 
   @Override
   public int getRowCount() {
     return _rowCount;
   }
 
   int getNextLongAutoNumber() {
     // note, the saved value is the last one handed out, so pre-increment
     return ++_lastLongAutoNumber;
   }
 
   int getLastLongAutoNumber() {
     // gets the last used auto number (does not modify)
     return _lastLongAutoNumber;
   }
 
   void adjustLongAutoNumber(int inLongAutoNumber) {
     if(inLongAutoNumber > _lastLongAutoNumber) {
       _lastLongAutoNumber = inLongAutoNumber;
     }
   }
 
   void restoreLastLongAutoNumber(int lastLongAutoNumber) {
     // restores the last used auto number
     _lastLongAutoNumber = lastLongAutoNumber - 1;
   }
 
   int getNextComplexTypeAutoNumber() {
     // note, the saved value is the last one handed out, so pre-increment
     return ++_lastComplexTypeAutoNumber;
   }
 
   int getLastComplexTypeAutoNumber() {
     // gets the last used auto number (does not modify)
     return _lastComplexTypeAutoNumber;
   }
 
   void adjustComplexTypeAutoNumber(int inComplexTypeAutoNumber) {
     if(inComplexTypeAutoNumber > _lastComplexTypeAutoNumber) {
       _lastComplexTypeAutoNumber = inComplexTypeAutoNumber;
     }
   }
 
   void restoreLastComplexTypeAutoNumber(int lastComplexTypeAutoNumber) {
     // restores the last used auto number
     _lastComplexTypeAutoNumber = lastComplexTypeAutoNumber - 1;
   }
 
   @Override
   public String toString() {
     return ToStringBuilder.builder(this)
       .append("type", (_tableType + (!isSystem() ? " (USER)" : " (SYSTEM)")))
       .append("name", _name)
       .append("rowCount", _rowCount)
       .append("columnCount", _columns.size())
       .append("indexCount(data)", _indexCount)
       .append("logicalIndexCount", _logicalIndexCount)
       .appendIfNotNull("validator", _rowValidator)
       .append("columns", _columns)
       .append("indexes", _indexes)
       .append("ownedPages", _ownedPages)
       .toString();
   }
 
   /**
    * @return A simple String representation of the entire table in
    *         tab-delimited format
    * @usage _general_method_
    */
   public String display() throws IOException {
     return display(Long.MAX_VALUE);
   }
 
   /**
    * @param limit Maximum number of rows to display
    * @return A simple String representation of the entire table in
    *         tab-delimited format
    * @usage _general_method_
    */
   public String display(long limit) throws IOException {
     reset();
     StringWriter rtn = new StringWriter();
     new ExportUtil.Builder(getDefaultCursor()).setDelimiter("\t").setHeader(true)
       .exportWriter(new BufferedWriter(rtn));
     return rtn.toString();
   }
 
   /**
    * Updates free space and row info for a new row of the given size in the
    * given data page.  Positions the page for writing the row data.
    * @return the row number of the new row
    * @usage _advanced_method_
    */
   public static int addDataPageRow(ByteBuffer dataPage,
                                    int rowSize,
                                    JetFormat format,
                                    int rowFlags)
   {
     int rowSpaceUsage = getRowSpaceUsage(rowSize, format);
 
     // Decrease free space record.
     short freeSpaceInPage = dataPage.getShort(format.OFFSET_FREE_SPACE);
     dataPage.putShort(format.OFFSET_FREE_SPACE, (short) (freeSpaceInPage -
                                                          rowSpaceUsage));
 
     // Increment row count record.
     short rowCount = dataPage.getShort(format.OFFSET_NUM_ROWS_ON_DATA_PAGE);
     dataPage.putShort(format.OFFSET_NUM_ROWS_ON_DATA_PAGE,
                       (short) (rowCount + 1));
 
     // determine row position
     short rowLocation = findRowEnd(dataPage, rowCount, format);
     rowLocation -= rowSize;
 
     // write row position
     dataPage.putShort(getRowStartOffset(rowCount, format),
                       (short)(rowLocation | rowFlags));
 
     // set position for row data
     dataPage.position(rowLocation);
 
     return rowCount;
   }
 
   /**
    * Returns the row count for the current page.  If the page is invalid
    * ({@code null}) or the page is not a DATA page, 0 is returned.
    */
   static int getRowsOnDataPage(ByteBuffer rowBuffer, JetFormat format)
   {
     int rowsOnPage = 0;
     if((rowBuffer != null) && (rowBuffer.get(0) == PageTypes.DATA)) {
       rowsOnPage = rowBuffer.getShort(format.OFFSET_NUM_ROWS_ON_DATA_PAGE);
     }
     return rowsOnPage;
   }
 
   /**
    * @throws IllegalStateException if the given rowId is invalid
    */
   private void requireValidRowId(RowIdImpl rowId) {
     if(!rowId.isValid()) {
       throw new IllegalArgumentException(withErrorContext(
               "Given rowId is invalid: " + rowId));
     }
   }
 
   /**
    * @throws IllegalStateException if the given row is invalid or deleted
    */
   private void requireNonDeletedRow(RowState rowState, RowIdImpl rowId)
   {
     if(!rowState.isValid()) {
       throw new IllegalArgumentException(withErrorContext(
           "Given rowId is invalid for this table: " + rowId));
     }
     if(rowState.isDeleted()) {
       throw new IllegalStateException(withErrorContext(
           "Row is deleted: " + rowId));
     }
   }
 
   /**
    * @usage _advanced_method_
    */
   public static boolean isDeletedRow(short rowStart) {
     return ((rowStart & DELETED_ROW_MASK) != 0);
   }
 
   /**
    * @usage _advanced_method_
    */
   public static boolean isOverflowRow(short rowStart) {
     return ((rowStart & OVERFLOW_ROW_MASK) != 0);
   }
 
   /**
    * @usage _advanced_method_
    */
   public static short cleanRowStart(short rowStart) {
     return (short)(rowStart & OFFSET_MASK);
   }
 
   /**
    * @usage _advanced_method_
    */
   public static short findRowStart(ByteBuffer buffer, int rowNum,
                                    JetFormat format)
   {
     return cleanRowStart(
         buffer.getShort(getRowStartOffset(rowNum, format)));
   }
 
   /**
    * @usage _advanced_method_
    */
   public static int getRowStartOffset(int rowNum, JetFormat format)
   {
     return format.OFFSET_ROW_START + (format.SIZE_ROW_LOCATION * rowNum);
   }
 
   /**
    * @usage _advanced_method_
    */
   public static short findRowEnd(ByteBuffer buffer, int rowNum,
                                  JetFormat format)
   {
     return (short)((rowNum == 0) ?
                    format.PAGE_SIZE :
                    cleanRowStart(
                        buffer.getShort(getRowEndOffset(rowNum, format))));
   }
 
   /**
    * @usage _advanced_method_
    */
   public static int getRowEndOffset(int rowNum, JetFormat format)
   {
     return format.OFFSET_ROW_START + (format.SIZE_ROW_LOCATION * (rowNum - 1));
   }
 
   /**
    * @usage _advanced_method_
    */
   public static int getRowSpaceUsage(int rowSize, JetFormat format)
   {
     return rowSize + format.SIZE_ROW_LOCATION;
   }
 
   private void initAutoNumberColumns() {
     for(ColumnImpl c : _columns) {
       if(c.isAutoNumber()) {
         _autoNumColumns.add(c);
       }
     }
   }
 
   private void initCalculatedColumns() {
     for(ColumnImpl c : _columns) {
       if(c.isCalculated()) {
         _calcColEval.add(c);
       }
     }
   }
 
   boolean isThisTable(Identifier identifier) {
     String collectionName = identifier.getCollectionName();
     return ((collectionName == null) ||
             collectionName.equalsIgnoreCase(getName()));
   }
 
   /**
    * Returns {@code true} if a row of the given size will fit on the given
    * data page, {@code false} otherwise.
    * @usage _advanced_method_
    */
   public static boolean rowFitsOnDataPage(
       int rowLength, ByteBuffer dataPage, JetFormat format)
   {
     int rowSpaceUsage = getRowSpaceUsage(rowLength, format);
     short freeSpaceInPage = dataPage.getShort(format.OFFSET_FREE_SPACE);
     int rowsOnPage = getRowsOnDataPage(dataPage, format);
     return ((rowSpaceUsage <= freeSpaceInPage) &&
             (rowsOnPage < format.MAX_NUM_ROWS_ON_DATA_PAGE));
   }
 
   /**
    * Duplicates and returns a row of data, optionally with a longer length
    * filled with {@code null}.
    */
   static Object[] dupeRow(Object[] row, int newRowLength) {
     Object[] copy = new Object[newRowLength];
     System.arraycopy(row, 0, copy, 0, Math.min(row.length, newRowLength));
     return copy;
   }
 
   String withErrorContext(String msg) {
     return withErrorContext(msg, getDatabase(), getName());
   }
 
   private static String withErrorContext(String msg, DatabaseImpl db,
                                          String tableName) {
     return msg + " (Db=" + db.getName() + ";Table=" + tableName + ")";
   }
 
   /** various statuses for the row data */
   private enum RowStatus {
     INIT, INVALID_PAGE, INVALID_ROW, VALID, DELETED, NORMAL, OVERFLOW;
   }
 
   /** the phases the RowState moves through as the data is parsed */
   private enum RowStateStatus {
     INIT, AT_HEADER, AT_FINAL;
   }
 
   /**
    * Maintains state for writing a new row of data.
    */
   protected static class WriteRowState
   {
     private int _complexAutoNumber = ColumnImpl.INVALID_AUTO_NUMBER;
 
     public int getComplexAutoNumber() {
       return _complexAutoNumber;
     }
 
     public void setComplexAutoNumber(int complexAutoNumber) {
       _complexAutoNumber = complexAutoNumber;
     }
 
     public void resetAutoNumber() {
       _complexAutoNumber = ColumnImpl.INVALID_AUTO_NUMBER;
     }
   }
 
   /**
    * Maintains the state of reading/updating a row of data.
    * @usage _advanced_class_
    */
   public final class RowState extends WriteRowState
     implements ErrorHandler.Location
   {
     /** Buffer used for reading the header row data pages */
     private final TempPageHolder _headerRowBufferH;
     /** the header rowId */
     private RowIdImpl _headerRowId = RowIdImpl.FIRST_ROW_ID;
     /** the number of rows on the header page */
     private int _rowsOnHeaderPage;
     /** the rowState status */
     private RowStateStatus _status = RowStateStatus.INIT;
     /** the row status */
     private RowStatus _rowStatus = RowStatus.INIT;
     /** buffer used for reading overflow pages */
     private final TempPageHolder _overflowRowBufferH =
       TempPageHolder.newHolder(TempBufferHolder.Type.SOFT);
     /** the row buffer which contains the final data (after following any
         overflow pointers) */
     private ByteBuffer _finalRowBuffer;
     /** the rowId which contains the final data (after following any overflow
         pointers) */
     private RowIdImpl _finalRowId = null;
     /** true if the row values array has data */
     private boolean _haveRowValues;
     /** values read from the last row */
     private Object[] _rowValues;
     /** null mask for the last row */
     private NullMask _nullMask;
     /** last modification count seen on the table we track this so that the
         rowState can detect updates to the table and re-read any buffered
         data */
     private int _lastModCount;
     /** optional error handler to use when row errors are encountered */
     private ErrorHandler _errorHandler;
     /** cached variable column offsets for jump-table based rows */
     private short[] _varColOffsets;
 
     private RowState(TempBufferHolder.Type headerType) {
       _headerRowBufferH = TempPageHolder.newHolder(headerType);
       _rowValues = new Object[TableImpl.this.getColumnCount()];
       _lastModCount = TableImpl.this._modCount;
     }
 
     @Override
     public TableImpl getTable() {
       return TableImpl.this;
     }
 
     public ErrorHandler getErrorHandler() {
       return((_errorHandler != null) ? _errorHandler :
              getTable().getErrorHandler());
     }
 
     public void setErrorHandler(ErrorHandler newErrorHandler) {
       _errorHandler = newErrorHandler;
     }
 
     public void reset() {
       resetAutoNumber();
       _finalRowId = null;
       _finalRowBuffer = null;
       _rowsOnHeaderPage = 0;
       _status = RowStateStatus.INIT;
       _rowStatus = RowStatus.INIT;
       _varColOffsets = null;
       _nullMask = null;
       if(_haveRowValues) {
         Arrays.fill(_rowValues, null);
         _haveRowValues = false;
       }
     }
 
     public boolean isUpToDate() {
       return(TableImpl.this._modCount == _lastModCount);
     }
 
     private void checkForModification() {
       if(!isUpToDate()) {
         reset();
         _headerRowBufferH.invalidate();
         _overflowRowBufferH.invalidate();
         int colCount = TableImpl.this.getColumnCount();
         if(colCount != _rowValues.length) {
           // columns added or removed from table
           _rowValues = new Object[colCount];
         }
         _lastModCount = TableImpl.this._modCount;
       }
     }
 
     private ByteBuffer getFinalPage()
       throws IOException
     {
       if(_finalRowBuffer == null) {
         // (re)load current page
         _finalRowBuffer = getHeaderPage();
       }
       return _finalRowBuffer;
     }
 
     public RowIdImpl getFinalRowId() {
       if(_finalRowId == null) {
         _finalRowId = getHeaderRowId();
       }
       return _finalRowId;
     }
 
     private void setRowStatus(RowStatus rowStatus) {
       _rowStatus = rowStatus;
     }
 
     public boolean isValid() {
       return(_rowStatus.ordinal() >= RowStatus.VALID.ordinal());
     }
 
     public boolean isDeleted() {
       return(_rowStatus == RowStatus.DELETED);
     }
 
     public boolean isOverflow() {
       return(_rowStatus == RowStatus.OVERFLOW);
     }
 
     public boolean isHeaderPageNumberValid() {
       return(_rowStatus.ordinal() > RowStatus.INVALID_PAGE.ordinal());
     }
 
     public boolean isHeaderRowNumberValid() {
       return(_rowStatus.ordinal() > RowStatus.INVALID_ROW.ordinal());
     }
 
     private void setStatus(RowStateStatus status) {
       _status = status;
     }
 
     public boolean isAtHeaderRow() {
       return(_status.ordinal() >= RowStateStatus.AT_HEADER.ordinal());
     }
 
     public boolean isAtFinalRow() {
       return(_status.ordinal() >= RowStateStatus.AT_FINAL.ordinal());
     }
 
     private Object setRowCacheValue(int idx, Object value) {
       _haveRowValues = true;
       _rowValues[idx] = value;
       return value;
     }
 
     private Object getRowCacheValue(int idx) {
       Object value = _rowValues[idx];
       // only return immutable values.  mutable values could have been
       // modified externally and therefore could return an incorrect value
       return(ColumnImpl.isImmutableValue(value) ? value : null);
     }
 
     public Object[] getRowCacheValues() {
       return dupeRow(_rowValues, _rowValues.length);
     }
 
     public NullMask getNullMask(ByteBuffer rowBuffer) {
       if(_nullMask == null) {
         _nullMask = getRowNullMask(rowBuffer);
       }
       return _nullMask;
     }
 
     private short[] getVarColOffsets() {
       return _varColOffsets;
     }
 
     private void setVarColOffsets(short[] varColOffsets) {
       _varColOffsets = varColOffsets;
     }
 
     public RowIdImpl getHeaderRowId() {
       return _headerRowId;
     }
 
     public int getRowsOnHeaderPage() {
       return _rowsOnHeaderPage;
     }
 
     private ByteBuffer getHeaderPage()
       throws IOException
     {
       checkForModification();
       return _headerRowBufferH.getPage(getPageChannel());
     }
 
     private ByteBuffer setHeaderRow(RowIdImpl rowId)
       throws IOException
     {
       checkForModification();
 
       // don't do any work if we are already positioned correctly
       if(isAtHeaderRow() && (getHeaderRowId().equals(rowId))) {
         return(isValid() ? getHeaderPage() : null);
       }
 
       // rejigger everything
       reset();
       _headerRowId = rowId;
       _finalRowId = rowId;
 
       int pageNumber = rowId.getPageNumber();
       int rowNumber = rowId.getRowNumber();
       if((pageNumber < 0) || !_ownedPages.containsPageNumber(pageNumber)) {
         setRowStatus(RowStatus.INVALID_PAGE);
         return null;
       }
 
       _finalRowBuffer = _headerRowBufferH.setPage(getPageChannel(),
                                                   pageNumber);
       _rowsOnHeaderPage = getRowsOnDataPage(_finalRowBuffer, getFormat());
 
       if((rowNumber < 0) || (rowNumber >= _rowsOnHeaderPage)) {
         setRowStatus(RowStatus.INVALID_ROW);
         return null;
       }
 
       setRowStatus(RowStatus.VALID);
       return _finalRowBuffer;
     }
 
     private ByteBuffer setOverflowRow(RowIdImpl rowId)
       throws IOException
     {
       // this should never see modifications because it only happens within
       // the positionAtRowData method
       if(!isUpToDate()) {
         throw new IllegalStateException(getTable().withErrorContext(
                                             "Table modified while searching?"));
       }
       if(_rowStatus != RowStatus.OVERFLOW) {
         throw new IllegalStateException(getTable().withErrorContext(
                                             "Row is not an overflow row?"));
       }
       _finalRowId = rowId;
       _finalRowBuffer = _overflowRowBufferH.setPage(getPageChannel(),
                                                     rowId.getPageNumber());
       return _finalRowBuffer;
     }
 
     private Object handleRowError(ColumnImpl column, byte[] columnData,
                                   Exception error)
       throws IOException
     {
       return getErrorHandler().handleRowError(column, columnData,
                                               this, error);
     }
 
     @Override
     public String toString() {
       return ToStringBuilder.valueBuilder(this)
         .append("headerRowId", _headerRowId)
         .append("finalRowId", _finalRowId)
         .toString();
     }
   }
 
   /**
    * Utility for managing calculated columns.  Calculated columns need to be
    * evaluated in dependency order.
    */
   private class CalcColEvaluator
   {
     /** List of calculated columns in this table, ordered by calculation
         dependency */
     private final List<ColumnImpl> _calcColumns = new ArrayList<>(1);
     private boolean _sorted;
 
     public void add(ColumnImpl col) {
       if(!getDatabase().isEvaluateExpressions()) {
         return;
       }
       _calcColumns.add(col);
       // whenever we add new columns, we need to re-sort
       _sorted = false;
     }
 
     public void reSort() {
       // mark columns for re-sort on next use
       _sorted = false;
     }
 
     public void calculate(Object[] row) throws IOException {
       if(!_sorted) {
         sortColumnsByDeps();
         _sorted = true;
       }
 
       for(ColumnImpl col : _calcColumns) {
         Object rowValue = col.getCalculationContext().eval(row);
         col.setRowValue(row, rowValue);
       }
     }
 
     private void sortColumnsByDeps() {
 
       // a topological sort sorts nodes where A -> B such that A ends up in
       // the list before B (assuming that we are working with a DAG).  In our
       // case, we return "descendent" info as Field1 -> Field2 (where Field1
       // uses Field2 in its calculation).  This means that in order to
       // correctly calculate Field1, we need to calculate Field2 first, and
       // hence essentially need the reverse topo sort (a list where Field2
       // comes before Field1).
       (new TopoSorter<ColumnImpl>(_calcColumns, TopoSorter.REVERSE) {
         @Override
         protected void getDescendents(ColumnImpl from,
                                       List<ColumnImpl> descendents) {
 
           Set<Identifier> identifiers = new LinkedHashSet<>();
           from.getCalculationContext().collectIdentifiers(identifiers);
 
           for(Identifier identifier : identifiers) {
             if(isThisTable(identifier)) {
               String colName = identifier.getObjectName();
               for(ColumnImpl calcCol : _calcColumns) {
                 // we only care if the identifier is another calc field
                 if(calcCol.getName().equalsIgnoreCase(colName)) {
                   descendents.add(calcCol);
                 }
               }
             }
           }
         }
       }).sort();
     }
   }
 }