/* $Id: IncrementalIngester.java 988245 2010-08-23 18:39:35Z kwright $ */

/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.manifoldcf.agents.incrementalingest;

import org.apache.manifoldcf.core.interfaces.*;
import org.apache.manifoldcf.agents.interfaces.*;
import org.apache.manifoldcf.agents.system.Logging;
import org.apache.manifoldcf.agents.system.ManifoldCF;
import java.util.*;
import java.io.*;

/** Incremental ingestion API implementation.
* This class is responsible for keeping track of what has been sent where, and also the corresponding version of
* each document so indexed.  The space over which this takes place is defined by the individual output connection - that is, the output connection
* seems to "remember" what documents were handed to it.
*
* A secondary purpose of this module is to provide a mapping between the key by which a document is described internally (by an
* identifier hash, plus the name of an identifier space), and the way the document is identified in the output space (by the name of an
* output connection, plus a URI which is considered local to that output connection space).
*
* <br><br>
* <b>ingeststatus</b>
* <table border="1" cellpadding="3" cellspacing="0" summary="">
* <tr class="TableHeadingColor">
* <th>Field</th><th>Type</th><th>Description&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</th>
* <tr><td>id</td><td>BIGINT</td><td>Primary Key</td></tr>
* <tr><td>connectionname</td><td>VARCHAR(32)</td><td>Reference:outputconnections.connectionname</td></tr>
* <tr><td>dockey</td><td>VARCHAR(73)</td><td></td></tr>
* <tr><td>componenthash</td><td>VARCHAR(40)</td><td></td></tr>
* <tr><td>docuri</td><td>LONGTEXT</td><td></td></tr>
* <tr><td>urihash</td><td>VARCHAR(40)</td><td></td></tr>
* <tr><td>lastversion</td><td>LONGTEXT</td><td></td></tr>
* <tr><td>lastoutputversion</td><td>LONGTEXT</td><td></td></tr>
* <tr><td>lasttransformationversion</td><td>LONGTEXT</td><td></td></tr>
* <tr><td>changecount</td><td>BIGINT</td><td></td></tr>
* <tr><td>firstingest</td><td>BIGINT</td><td></td></tr>
* <tr><td>lastingest</td><td>BIGINT</td><td></td></tr>
* <tr><td>authorityname</td><td>VARCHAR(32)</td><td></td></tr>
* </table>
* <br><br>
* 
*/
public class IncrementalIngester extends org.apache.manifoldcf.core.database.BaseTable implements IIncrementalIngester
{
  public static final String _rcsid = "@(#)$Id: IncrementalIngester.java 988245 2010-08-23 18:39:35Z kwright $";

  // Fields
  protected final static String idField = "id";
  protected final static String outputConnNameField = "connectionname";
  protected final static String docKeyField = "dockey";
  protected final static String componentHashField = "componenthash";
  protected final static String docURIField = "docuri";
  protected final static String uriHashField = "urihash";
  protected final static String lastVersionField = "lastversion";
  protected final static String lastOutputVersionField = "lastoutputversion";
  protected final static String lastTransformationVersionField = "lasttransformationversion";
  protected final static String changeCountField = "changecount";
  protected final static String firstIngestField = "firstingest";
  protected final static String lastIngestField = "lastingest";
  protected final static String authorityNameField = "authorityname";

  // Thread context.
  protected final IThreadContext threadContext;
  // Lock manager.
  protected final ILockManager lockManager;
  // Output connection manager
  protected final IOutputConnectionManager connectionManager;
  // Output connector pool manager
  protected final IOutputConnectorPool outputConnectorPool;
  // Transformation connector pool manager
  protected final ITransformationConnectorPool transformationConnectorPool;
  
  /** Constructor.
  */
  public IncrementalIngester(IThreadContext threadContext, IDBInterface database)
    throws ManifoldCFException
  {
    super(database,"ingeststatus");
    this.threadContext = threadContext;
    lockManager = LockManagerFactory.make(threadContext);
    connectionManager = OutputConnectionManagerFactory.make(threadContext);
    outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
    transformationConnectorPool = TransformationConnectorPoolFactory.make(threadContext);
  }

  /** Install the incremental ingestion manager.
  */
  @Override
  public void install()
    throws ManifoldCFException
  {
    String outputConnectionTableName = connectionManager.getTableName();
    String outputConnectionNameField = connectionManager.getConnectionNameColumn();

    // We always include an outer loop, because some upgrade conditions require retries.
    while (true)
    {
      // Postgresql has a limitation on the number of characters that can be indexed in a column.  So we use hashes instead.
      Map existing = getTableSchema(null,null);
      if (existing == null)
      {
        HashMap map = new HashMap();
        map.put(idField,new ColumnDescription("BIGINT",true,false,null,null,false));
        map.put(outputConnNameField,new ColumnDescription("VARCHAR(32)",false,false,outputConnectionTableName,outputConnectionNameField,false));
        map.put(docKeyField,new ColumnDescription("VARCHAR(73)",false,false,null,null,false));
        map.put(componentHashField,new ColumnDescription("VARCHAR(40)",false,true,null,null,false));
        // The document URI field, if null, indicates that the document was not actually ingested!
        // This happens when a connector wishes to keep track of a version string, but not actually ingest the doc.
        map.put(docURIField,new ColumnDescription("LONGTEXT",false,true,null,null,false));
        map.put(uriHashField,new ColumnDescription("VARCHAR(40)",false,true,null,null,false));
        map.put(lastVersionField,new ColumnDescription("LONGTEXT",false,true,null,null,false));
        map.put(lastOutputVersionField,new ColumnDescription("LONGTEXT",false,true,null,null,false));
        map.put(lastTransformationVersionField,new ColumnDescription("LONGTEXT",false,true,null,null,false));
        map.put(changeCountField,new ColumnDescription("BIGINT",false,false,null,null,false));
        map.put(firstIngestField,new ColumnDescription("BIGINT",false,false,null,null,false));
        map.put(lastIngestField,new ColumnDescription("BIGINT",false,false,null,null,false));
        map.put(authorityNameField,new ColumnDescription("VARCHAR(32)",false,true,null,null,false));
        performCreate(map,null);
      }
      else
      {
        // Upgrades from 2.0 onward go here
      }

      // Now, do indexes
      IndexDescription keyIndex = new IndexDescription(true,new String[]{docKeyField,outputConnNameField,componentHashField});
      IndexDescription uriHashIndex = new IndexDescription(false,new String[]{uriHashField,outputConnNameField});
      IndexDescription outputConnIndex = new IndexDescription(false,new String[]{outputConnNameField});

      // Get rid of indexes that shouldn't be there
      Map indexes = getTableIndexes(null,null);
      Iterator iter = indexes.keySet().iterator();
      while (iter.hasNext())
      {
        String indexName = (String)iter.next();
        IndexDescription id = (IndexDescription)indexes.get(indexName);

        if (keyIndex != null && id.equals(keyIndex))
          keyIndex = null;
        else if (uriHashIndex != null && id.equals(uriHashIndex))
          uriHashIndex = null;
        else if (outputConnIndex != null && id.equals(outputConnIndex))
          outputConnIndex = null;
        else if (indexName.indexOf("_pkey") == -1)
          // This index shouldn't be here; drop it
          performRemoveIndex(indexName);
      }

      // Add the ones we didn't find
      if (uriHashIndex != null)
        performAddIndex(null,uriHashIndex);

      if (keyIndex != null)
        performAddIndex(null,keyIndex);

      if (outputConnIndex != null)
        performAddIndex(null,outputConnIndex);
      
      // All done; break out of loop
      break;
    }

  }

  /** Uninstall the incremental ingestion manager.
  */
  @Override
  public void deinstall()
    throws ManifoldCFException
  {
    performDrop(null);
  }

  /** Flush all knowledge of what was ingested before.
  */
  @Override
  public void clearAll()
    throws ManifoldCFException
  {
    performDelete("",null,null);
  }

  /** From a pipeline specification, get the name of the output connection that will be indexed last
  * in the pipeline.
  *@param pipelineSpecificationBasic is the basic pipeline specification.
  *@return the last indexed output connection name.
  */
  @Override
  public String getLastIndexedOutputConnectionName(IPipelineSpecificationBasic pipelineSpecificationBasic)
  {
    // It's always the last in the sequence.
    int count = pipelineSpecificationBasic.getOutputCount();
    if (count == 0)
      return null;
    return pipelineSpecificationBasic.getStageConnectionName(pipelineSpecificationBasic.getOutputStage(count-1));
  }

  /** From a pipeline specification, get the name of the output connection that will be indexed first
  * in the pipeline.
  *@param pipelineSpecificationBasic is the basic pipeline specification.
  *@return the first indexed output connection name.
  */
  @Override
  public String getFirstIndexedOutputConnectionName(IPipelineSpecificationBasic pipelineSpecificationBasic)
  {
    if (pipelineSpecificationBasic.getOutputCount() == 0)
      return null;
    return pipelineSpecificationBasic.getStageConnectionName(pipelineSpecificationBasic.getOutputStage(0));
  }

  /** Check if a date is indexable.
  *@param pipelineSpecification is the IPipelineSpecification object for this pipeline.
  *@param date is the date to check.
  *@param activity are the activities available to this method.
  *@return true if the mimeType is indexable.
  */
  @Override
  public boolean checkDateIndexable(
    IPipelineSpecification pipelineSpecification,
    Date date,
    IOutputCheckActivity activity)
    throws ManifoldCFException, ServiceInterruption
  {
    PipelineObject pipeline = pipelineGrab(pipelineSpecification);
    if (pipeline == null)
      // A connector is not installed; treat this as a service interruption.
      throw new ServiceInterruption("One or more connectors are not installed",0L);
    try
    {
      return pipeline.checkDateIndexable(date,activity);
    }
    finally
    {
      pipeline.release();
    }
  }

  /** Check if a mime type is indexable.
  *@param pipelineSpecification is the IPipelineSpecification object for this pipeline.
  *@param mimeType is the mime type to check.
  *@param activity are the activities available to this method.
  *@return true if the mimeType is indexable.
  */
  @Override
  public boolean checkMimeTypeIndexable(
    IPipelineSpecification pipelineSpecification,
    String mimeType,
    IOutputCheckActivity activity)
    throws ManifoldCFException, ServiceInterruption
  {
    PipelineObject pipeline = pipelineGrab(pipelineSpecification);
    if (pipeline == null)
      // A connector is not installed; treat this as a service interruption.
      throw new ServiceInterruption("One or more connectors are not installed",0L);
    try
    {
      return pipeline.checkMimeTypeIndexable(mimeType,activity);
    }
    finally
    {
      pipeline.release();
    }
  }

  /** Check if a file is indexable.
  *@param pipelineSpecification is the IPipelineSpecification object for this pipeline.
  *@param localFile is the local file to check.
  *@param activity are the activities available to this method.
  *@return true if the local file is indexable.
  */
  @Override
  public boolean checkDocumentIndexable(
    IPipelineSpecification pipelineSpecification,
    File localFile,
    IOutputCheckActivity activity)
    throws ManifoldCFException, ServiceInterruption
  {
    PipelineObject pipeline = pipelineGrab(pipelineSpecification);
    if (pipeline == null)
      // A connector is not installed; treat this as a service interruption.
      throw new ServiceInterruption("One or more connectors are not installed",0L);
    try
    {
      return pipeline.checkDocumentIndexable(localFile,activity);
    }
    finally
    {
      pipeline.release();
    }
  }

  /** Pre-determine whether a document's length is indexable by this connector.  This method is used by participating repository connectors
  * to help filter out documents that are too long to be indexable.
  *@param pipelineSpecification is the IPipelineSpecification object for this pipeline.
  *@param length is the length of the document.
  *@param activity are the activities available to this method.
  *@return true if the file is indexable.
  */
  @Override
  public boolean checkLengthIndexable(
    IPipelineSpecification pipelineSpecification,
    long length,
    IOutputCheckActivity activity)
    throws ManifoldCFException, ServiceInterruption
  {
    PipelineObject pipeline = pipelineGrab(pipelineSpecification);
    if (pipeline == null)
      // A connector is not installed; treat this as a service interruption.
      throw new ServiceInterruption("One or more connectors are not installed",0L);
    try
    {
      return pipeline.checkLengthIndexable(length,activity);
    }
    finally
    {
      pipeline.release();
    }
  }

  /** Pre-determine whether a document's URL is indexable by this connector.  This method is used by participating repository connectors
  * to help filter out documents that not indexable.
  *@param pipelineSpecification is the IPipelineSpecification object for this pipeline.
  *@param url is the url of the document.
  *@param activity are the activities available to this method.
  *@return true if the file is indexable.
  */
  @Override
  public boolean checkURLIndexable(
    IPipelineSpecification pipelineSpecification,
    String url,
    IOutputCheckActivity activity)
    throws ManifoldCFException, ServiceInterruption
  {
    PipelineObject pipeline = pipelineGrab(pipelineSpecification);
    if (pipeline == null)
      // A connector is not installed; treat this as a service interruption.
      throw new ServiceInterruption("One or more connectors are not installed",0L);
    try
    {
      return pipeline.checkURLIndexable(url,activity);
    }
    finally
    {
      pipeline.release();
    }
  }

  /** Grab the entire pipeline.
  *@param pipelineConnections - the pipeline specification with version information
  *@return the pipeline description, or null if any part of the pipeline cannot be grabbed.
  */
  protected PipelineObjectWithVersions pipelineGrabWithVersions(IPipelineSpecificationWithVersions pipelineConnections)
    throws ManifoldCFException
  {
    // Pick up all needed transformation connectors
    ITransformationConnector[] transformationConnectors = transformationConnectorPool.grabMultiple(pipelineConnections.getTransformationConnectionNames(),pipelineConnections.getTransformationConnections());
    for (ITransformationConnector c : transformationConnectors)
    {
      if (c == null)
      {
        transformationConnectorPool.releaseMultiple(pipelineConnections.getTransformationConnections(),transformationConnectors);
        return null;
      }
    }
    
    // Pick up all needed output connectors.  If this fails we have to release the transformation connectors.
    try
    {
      IOutputConnector[] outputConnectors = outputConnectorPool.grabMultiple(pipelineConnections.getOutputConnectionNames(),pipelineConnections.getOutputConnections());
      for (IOutputConnector c : outputConnectors)
      {
        if (c == null)
        {
          outputConnectorPool.releaseMultiple(pipelineConnections.getOutputConnections(),outputConnectors);
          transformationConnectorPool.releaseMultiple(pipelineConnections.getTransformationConnections(),transformationConnectors);
          return null;
        }
      }
      return new PipelineObjectWithVersions(pipelineConnections,transformationConnectors,outputConnectors);
    }
    catch (Throwable e)
    {
      transformationConnectorPool.releaseMultiple(pipelineConnections.getTransformationConnections(),transformationConnectors);
      if (e instanceof ManifoldCFException)
        throw (ManifoldCFException)e;
      else if (e instanceof RuntimeException)
        throw (RuntimeException)e;
      else if (e instanceof Error)
        throw (Error)e;
      else
        throw new RuntimeException("Unexpected exception type: "+e.getClass().getName()+": "+e.getMessage(),e);
    }
  }

  /** Grab the entire pipeline.
  *@param pipelineConnections - the pipeline specification
  *@return the pipeline description, or null if any part of the pipeline cannot be grabbed.
  */
  protected PipelineObject pipelineGrab(IPipelineSpecification pipelineConnections)
    throws ManifoldCFException
  {
    // Pick up all needed transformation connectors
    ITransformationConnector[] transformationConnectors = transformationConnectorPool.grabMultiple(pipelineConnections.getTransformationConnectionNames(),pipelineConnections.getTransformationConnections());
    for (ITransformationConnector c : transformationConnectors)
    {
      if (c == null)
      {
        transformationConnectorPool.releaseMultiple(pipelineConnections.getTransformationConnections(),transformationConnectors);
        return null;
      }
    }
    
    // Pick up all needed output connectors.  If this fails we have to release the transformation connectors.
    try
    {
      IOutputConnector[] outputConnectors = outputConnectorPool.grabMultiple(pipelineConnections.getOutputConnectionNames(),pipelineConnections.getOutputConnections());
      for (IOutputConnector c : outputConnectors)
      {
        if (c == null)
        {
          outputConnectorPool.releaseMultiple(pipelineConnections.getOutputConnections(),outputConnectors);
          transformationConnectorPool.releaseMultiple(pipelineConnections.getTransformationConnections(),transformationConnectors);
          return null;
        }
      }
      return new PipelineObject(pipelineConnections,transformationConnectors,outputConnectors);
    }
    catch (Throwable e)
    {
      transformationConnectorPool.releaseMultiple(pipelineConnections.getTransformationConnections(),transformationConnectors);
      if (e instanceof ManifoldCFException)
        throw (ManifoldCFException)e;
      else if (e instanceof RuntimeException)
        throw (RuntimeException)e;
      else if (e instanceof Error)
        throw (Error)e;
      else
        throw new RuntimeException("Unexpected exception type: "+e.getClass().getName()+": "+e.getMessage(),e);
    }
  }

  /** Get an output version string for a document.
  *@param outputConnection is the output connection associated with this action.
  *@param spec is the output specification.
  *@return the description string.
  */
  @Override
  public VersionContext getOutputDescription(IOutputConnection outputConnection, Specification spec)
    throws ManifoldCFException, ServiceInterruption
  {
    IOutputConnector connector = outputConnectorPool.grab(outputConnection);
    if (connector == null)
      // The connector is not installed; treat this as a service interruption.
      throw new ServiceInterruption("Output connector not installed",0L);
    try
    {
      return connector.getPipelineDescription(spec);
    }
    finally
    {
      outputConnectorPool.release(outputConnection,connector);
    }

  }

  /** Get transformation version string for a document.
  *@param transformationConnection is the transformation connection associated with this action.
  *@param spec is the transformation specification.
  *@return the description string.
  */
  @Override
  public VersionContext getTransformationDescription(ITransformationConnection transformationConnection, Specification spec)
    throws ManifoldCFException, ServiceInterruption
  {
    ITransformationConnector connector = transformationConnectorPool.grab(transformationConnection);
    if (connector == null)
      // The connector is not installed; treat this as a service interruption.
      throw new ServiceInterruption("Transformation connector not installed",0L);
    try
    {
      return connector.getPipelineDescription(spec);
    }
    finally
    {
      transformationConnectorPool.release(transformationConnection,connector);
    }
  }

  /** Determine whether we need to fetch or refetch a document.
  * Pass in information including the pipeline specification with existing version info, plus new document and parameter version strings.
  * If no outputs need to be updated, then this method will return false.  If any outputs need updating, then true is returned.
  *@param pipelineSpecificationWithVersions is the pipeline specification including new version info for all transformation and output
  *  connections.
  *@param newDocumentVersion is the newly-determined document version.
  *@param newAuthorityNameString is the newly-determined authority name.
  *@return true if the document needs to be refetched.
  */
  @Override
  public boolean checkFetchDocument(
    IPipelineSpecificationWithVersions pipelineSpecificationWithVersions,
    String newDocumentVersion,
    String newAuthorityNameString)
  {
    if (newAuthorityNameString == null)
      newAuthorityNameString = "";
    // Cycle through the outputs
    for (int i = 0; i < pipelineSpecificationWithVersions.getOutputCount(); i++)
    {
      int stage = pipelineSpecificationWithVersions.getOutputStage(i);
      String oldDocumentVersion = pipelineSpecificationWithVersions.getOutputDocumentVersionString(i);
      String oldOutputVersion = pipelineSpecificationWithVersions.getOutputVersionString(i);
      String oldAuthorityName = pipelineSpecificationWithVersions.getAuthorityNameString(i);
      // If it looks like we never indexed this output before, we need to do it now.
      if (oldDocumentVersion == null)
        return true;
      // Look first at the version strings that aren't pipeline dependent
      if (!oldDocumentVersion.equals(newDocumentVersion) ||
        !oldAuthorityName.equals(newAuthorityNameString) ||
        !oldOutputVersion.equals(pipelineSpecificationWithVersions.getStageDescriptionString(stage).getVersionString()))
        return true;
      
      // Everything matches so far.  Next step is to compute a transformation path an corresponding version string.
      String newTransformationVersion = computePackedTransformationVersion(pipelineSpecificationWithVersions,stage);
      if (!pipelineSpecificationWithVersions.getOutputTransformationVersionString(i).equals(newTransformationVersion))
        return true;
    }
    // Everything matches, so no reindexing is needed.
    return false;
  }

  /** Compute a transformation version given a pipeline specification and starting output stage.
  *@param pipelineSpecification is the pipeline specification.
  *@param stage is the stage number of the output stage.
  *@return the transformation version string, which will be a composite of all the transformations applied.
  */
  protected static String computePackedTransformationVersion(IPipelineSpecification pipelineSpecification, int stage)
  {
    // First, count the stages we need to represent
    int stageCount = 0;
    int currentStage = stage;
    while (true)
    {
      int newStage = pipelineSpecification.getStageParent(currentStage);
      if (newStage == -1)
        break;
      stageCount++;
      currentStage = newStage;
    }
    // Doesn't matter how we pack it; I've chosen to do it in reverse for convenience
    String[] stageNames = new String[stageCount];
    String[] stageDescriptions = new String[stageCount];
    stageCount = 0;
    currentStage = stage;
    while (true)
    {
      int newStage = pipelineSpecification.getStageParent(currentStage);
      if (newStage == -1)
        break;
      stageNames[stageCount] = pipelineSpecification.getStageConnectionName(newStage);
      stageDescriptions[stageCount] = pipelineSpecification.getStageDescriptionString(newStage).getVersionString();
      stageCount++;
      currentStage = newStage;
    }
    // Finally, do the packing.
    StringBuilder sb = new StringBuilder();
    packList(sb,stageNames,'+');
    packList(sb,stageDescriptions,'!');
    return sb.toString();
  }
  
  protected static void packList(StringBuilder output, String[] values, char delimiter)
  {
    pack(output,Integer.toString(values.length),delimiter);
    int i = 0;
    while (i < values.length)
    {
      pack(output,values[i++],delimiter);
    }
  }

  protected static void pack(StringBuilder sb, String value, char delim)
  {
    for (int i = 0; i < value.length(); i++)
    {
      char x = value.charAt(i);
      if (x == delim || x == '\\')
      {
        sb.append('\\');
      }
      sb.append(x);
    }
    sb.append(delim);
  }

  /** Record a document version, but don't ingest it.
  * The purpose of this method is to update document version information without reindexing the document.
  *@param pipelineSpecificationBasic is the basic pipeline specification needed.
  *@param identifierClass is the name of the space in which the identifier hash should be interpreted.
  *@param identifierHash is the hashed document identifier.
  *@param componentHash is the hashed component identifier, if any.
  *@param documentVersion is the document version.
  *@param recordTime is the time at which the recording took place, in milliseconds since epoch.
  */
  @Override
  public void documentRecord(
    IPipelineSpecificationBasic pipelineSpecificationBasic,
    String identifierClass, String identifierHash, String componentHash,
    String documentVersion, long recordTime)
    throws ManifoldCFException
  {
    // This method is called when a connector decides that the last indexed version of the document is in fact just fine,
    // but the document version information should be updated.
    // The code pathway is therefore similar to that of document indexing, EXCEPT that no indexing will ever
    // take place.  This has some interesting side effects.  For example:
    // (1) In the case of a document collision with another job using the same repository connection, the last document
    //    indexed cannot be changed.  Updating the version string for the document would therefore be misleading.  This
    //    case should be detected and prevented from occurring, by refusing to perform the update.
    //    On the other hand, only one thread at a time can be processing the document at a given time, and therefore
    //    since the connector detected "no change", we are safe to presume we can just update the version info.
    // (2) In the case of a URL conflict with another job, since nothing changes and no new URL is recorded, no cleanup
    //    of conflicting records sharing the same URL should be needed.
    
    String docKey = makeKey(identifierClass,identifierHash);

    String[] outputConnectionNames = extractOutputConnectionNames(pipelineSpecificationBasic);

    if (Logging.ingest.isDebugEnabled())
    {
      Logging.ingest.debug("Recording document '"+docKey+"' component hash "+((componentHash==null)?"(None)":("'"+componentHash+"'"))+" for output connections '"+outputConnectionNames+"'");
    }

    for (int k = 0; k < outputConnectionNames.length; k++)
    {
      String outputConnectionName = outputConnectionNames[k];

      // If we get here, it means we are noting that the document was examined, but that no change was required.  This is signaled
      // to noteDocumentIngest by having the null documentURI.
      noteDocumentIngest(outputConnectionName,docKey,componentHash,documentVersion,null,null,null,recordTime,null,null);
    }
  }

  /** Remove a document from specified indexes, just as if an empty document
  * was indexed, and record the necessary version information.
  * This method is conceptually similar to documentIngest(), but does not actually take
  * a document or allow it to be transformed.  If there is a document already
  * indexed, it is removed from the index.
  *@param pipelineSpecificationWithVersions is the pipeline specification with already-fetched output versioning information.
  *@param identifierClass is the name of the space in which the identifier hash should be interpreted.
  *@param identifierHash is the hashed document identifier.
  *@param componentHash is the hashed component identifier, if any.
  *@param documentVersion is the document version.
  *@param authorityName is the name of the authority associated with the document, if any.
  *@param recordTime is the time at which the recording took place, in milliseconds since epoch.
  *@param activities is an object providing a set of methods that the implementer can use to perform the operation.
  */
  @Override
  public void documentNoData(
    IPipelineSpecificationWithVersions pipelineSpecificationWithVersions,
    String identifierClass, String identifierHash, String componentHash,
    String documentVersion,
    String authorityName,
    long recordTime,
    IOutputActivity activities)
    throws ManifoldCFException, ServiceInterruption
  {
    String docKey = makeKey(identifierClass,identifierHash);

    if (Logging.ingest.isDebugEnabled())
    {
      Logging.ingest.debug("Logging empty document '"+docKey+"' component hash "+((componentHash==null)?"(None)":("'"+componentHash+"'"))+" into output connections '"+extractOutputConnectionNames(pipelineSpecificationWithVersions)+"'");
    }

    // Set up a pipeline
    PipelineObjectWithVersions pipeline = pipelineGrabWithVersions(pipelineSpecificationWithVersions);
    if (pipeline == null)
      // A connector is not installed; treat this as a service interruption.
      throw new ServiceInterruption("Pipeline connector not installed",0L);
    try
    {
      pipeline.noDocument(docKey,componentHash,documentVersion,authorityName,activities,recordTime);
    }
    finally
    {
      pipeline.release();
    }
  }

  /** Ingest a document.
  * This ingests the document, and notes it.  If this is a repeat ingestion of the document, this
  * method also REMOVES ALL OLD METADATA.  When complete, the index will contain only the metadata
  * described by the RepositoryDocument object passed to this method.
  * ServiceInterruption is thrown if the document ingestion must be rescheduled.
  *@param pipelineSpecificationWithVersions is the pipeline specification with already-fetched output versioning information.
  *@param identifierClass is the name of the space in which the identifier hash should be interpreted.
  *@param identifierHash is the hashed document identifier.
  *@param componentHash is the hashed component identifier, if any.
  *@param documentVersion is the document version.
  *@param authorityName is the name of the authority associated with the document, if any.
  *@param data is the document data.  The data is closed after ingestion is complete.
  *@param ingestTime is the time at which the ingestion took place, in milliseconds since epoch.
  *@param documentURI is the URI of the document, which will be used as the key of the document in the index.
  *@param activities is an object providing a set of methods that the implementer can use to perform the operation.
  *@return true if the ingest was ok, false if the ingest is illegal (and should not be repeated).
  *@throws IOException only if data stream throws an IOException.
  */
  @Override
  public boolean documentIngest(
    IPipelineSpecificationWithVersions pipelineSpecificationWithVersions,
    String identifierClass, String identifierHash, String componentHash,
    String documentVersion,
    String authorityName,
    RepositoryDocument data,
    long ingestTime, String documentURI,
    IOutputActivity activities)
    throws ManifoldCFException, ServiceInterruption, IOException
  {
    String docKey = makeKey(identifierClass,identifierHash);

    if (Logging.ingest.isDebugEnabled())
    {
      Logging.ingest.debug("Ingesting document '"+docKey+"' component hash "+((componentHash==null)?"(None)":("'"+componentHash+"'"))+" into output connections '"+extractOutputConnectionNames(pipelineSpecificationWithVersions)+"'");
    }

    // Set indexing date
    data.setIndexingDate(new Date());
    
    // Set up a pipeline
    PipelineObjectWithVersions pipeline = pipelineGrabWithVersions(pipelineSpecificationWithVersions);
    if (pipeline == null)
      // A connector is not installed; treat this as a service interruption.
      throw new ServiceInterruption("Pipeline connector not installed",0L);
    try
    {
      return pipeline.addOrReplaceDocumentWithException(docKey,componentHash,documentURI,data,documentVersion,authorityName,activities,ingestTime) == IPipelineConnector.DOCUMENTSTATUS_ACCEPTED;
    }
    finally
    {
      pipeline.release();
    }
  }

  /** Remove a document component from the search engine index.
  *@param pipelineConnections is the pipeline specification.
  *@param identifierClass is the name of the space in which the identifier hash should be interpreted.
  *@param identifierHash is the hash of the id of the document.
  *@param componentHash is the hashed component identifier, if any.
  *@param activities is the object to use to log the details of the ingestion attempt.  May be null.
  */
  @Override
  public void documentRemove(
    IPipelineConnections pipelineConnections,
    String identifierClass, String identifierHash, String componentHash,
    IOutputRemoveActivity activities)
    throws ManifoldCFException, ServiceInterruption
  {
    documentRemoveMultiple(pipelineConnections,
      new String[]{identifierClass},
      new String[]{identifierHash},
      componentHash,
      activities);
  }

  protected static String[] extractOutputConnectionNames(IPipelineSpecificationBasic pipelineSpecificationBasic)
  {
    String[] rval = new String[pipelineSpecificationBasic.getOutputCount()];
    for (int i = 0; i < rval.length; i++)
    {
      rval[i] = pipelineSpecificationBasic.getStageConnectionName(pipelineSpecificationBasic.getOutputStage(i));
    }
    return rval;
  }
  
  /** Note the fact that we checked a document (and found that it did not need to be ingested, because the
  * versions agreed).
  *@param pipelineSpecificationBasic is a pipeline specification.
  *@param identifierClasses are the names of the spaces in which the identifier hashes should be interpreted.
  *@param identifierHashes are the set of document identifier hashes.
  *@param checkTime is the time at which the check took place, in milliseconds since epoch.
  */
  @Override
  public void documentCheckMultiple(
    IPipelineSpecificationBasic pipelineSpecificationBasic,
    String[] identifierClasses, String[] identifierHashes,
    long checkTime)
    throws ManifoldCFException
  {
    // Extract output connection names from pipeline spec
    String[] outputConnectionNames = extractOutputConnectionNames(pipelineSpecificationBasic);
    beginTransaction();
    try
    {
      int maxClauses;
      
      Set<String> docIDValues = new HashSet<String>();
      for (int j = 0; j < identifierHashes.length; j++)
      {
        String docDBString = makeKey(identifierClasses[j],identifierHashes[j]);
        docIDValues.add(docDBString);
      }

      // Now, perform n queries, each of them no larger the maxInClause in length.
      // Create a list of row id's from this.
      Set<Long> rowIDSet = new HashSet<Long>();
      Iterator<String> iter = docIDValues.iterator();
      int j = 0;
      List<String> list = new ArrayList<String>();
      maxClauses = maxClausesRowIdsForDocIds(outputConnectionNames);
      while (iter.hasNext())
      {
        if (j == maxClauses)
        {
          findRowIdsForDocIds(outputConnectionNames,rowIDSet,list);
          list.clear();
          j = 0;
        }
        list.add(iter.next());
        j++;
      }

      if (j > 0)
        findRowIdsForDocIds(outputConnectionNames,rowIDSet,list);

      // Now, break row id's into chunks too; submit one chunk at a time
      j = 0;
      List<Long> list2 = new ArrayList<Long>();
      Iterator<Long> iter2 = rowIDSet.iterator();
      maxClauses = maxClausesUpdateRowIds();
      while (iter2.hasNext())
      {
        if (j == maxClauses)
        {
          updateRowIds(list2,checkTime);
          list2.clear();
          j = 0;
        }
        list2.add(iter2.next());
        j++;
      }

      if (j > 0)
        updateRowIds(list2,checkTime);
    }
    catch (ManifoldCFException e)
    {
      signalRollback();
      throw e;
    }
    catch (Error e)
    {
      signalRollback();
      throw e;
    }
    finally
    {
      endTransaction();
    }
  }

  /** Note the fact that we checked a document (and found that it did not need to be ingested, because the
  * versions agreed).
  *@param pipelineSpecificationBasic is a basic pipeline specification.
  *@param identifierClass is the name of the space in which the identifier hash should be interpreted.
  *@param identifierHash is the hashed document identifier.
  *@param checkTime is the time at which the check took place, in milliseconds since epoch.
  */
  @Override
  public void documentCheck(
    IPipelineSpecificationBasic pipelineSpecificationBasic,
    String identifierClass, String identifierHash,
    long checkTime)
    throws ManifoldCFException
  {
    documentCheckMultiple(pipelineSpecificationBasic,new String[]{identifierClass},new String[]{identifierHash},checkTime);
  }

  /** Calculate the number of clauses.
  */
  protected int maxClausesUpdateRowIds()
  {
    return findConjunctionClauseMax(new ClauseDescription[]{});
  }
  
  /** Update a chunk of row ids.
  */
  protected void updateRowIds(List<Long> list, long checkTime)
    throws ManifoldCFException
  {
    ArrayList newList = new ArrayList();
    String query = buildConjunctionClause(newList,new ClauseDescription[]{
      new MultiClause(idField,list)});
      
    HashMap map = new HashMap();
    map.put(lastIngestField,new Long(checkTime));
    performUpdate(map,"WHERE "+query,newList,null);
  }


  /** Delete multiple documents from the search engine index.
  *@param pipelineConnections are the pipeline specifications associated with the documents.
  *@param identifierClasses are the names of the spaces in which the identifier hashes should be interpreted.
  *@param identifierHashes is tha array of document identifier hashes if the documents.
  *@param activities is the object to use to log the details of the ingestion attempt.  May be null.
  */
  @Override
  public void documentDeleteMultiple(
    IPipelineConnections[] pipelineConnections,
    String[] identifierClasses, String[] identifierHashes,
    IOutputRemoveActivity activities)
    throws ManifoldCFException, ServiceInterruption
  {
    // Segregate request by pipeline spec instance address.  Not perfect but works in the
    // environment it is used it.
    Map<IPipelineConnections,List<Integer>> keyMap = new HashMap<IPipelineConnections,List<Integer>>();
    for (int i = 0; i < pipelineConnections.length; i++)
    {
      IPipelineConnections spec = pipelineConnections[i];
      List<Integer> list = keyMap.get(spec);
      if (list == null)
      {
        list = new ArrayList<Integer>();
        keyMap.put(spec,list);
      }
      list.add(new Integer(i));
    }

    // Create the return array.
    Iterator<IPipelineConnections> iter = keyMap.keySet().iterator();
    while (iter.hasNext())
    {
      IPipelineConnections spec = iter.next();
      List<Integer> list = keyMap.get(spec);
      String[] localIdentifierClasses = new String[list.size()];
      String[] localIdentifierHashes = new String[list.size()];
      for (int i = 0; i < localIdentifierClasses.length; i++)
      {
        int index = list.get(i).intValue();
        localIdentifierClasses[i] = identifierClasses[index];
        localIdentifierHashes[i] = identifierHashes[index];
      }
      documentDeleteMultiple(spec,localIdentifierClasses,localIdentifierHashes,activities);
    }
  }

  protected static String createURILockName(String outputConnectionName, String uriHash)
  {
    // The lock name needs to be constrained to some acceptably small number in order to avoid
    // a lot of zookeeper locks.  See CONNECTORS-1123.
    int hashCode = outputConnectionName.hashCode() + uriHash.hashCode();
    hashCode &= 0xffff;
    return "URILOCK-"+hashCode;
  }
  
  /** Delete multiple documents from the search engine index.
  *@param pipelineConnections is the pipeline specification.
  *@param identifierClasses are the names of the spaces in which the identifier hashes should be interpreted.
  *@param identifierHashes is tha array of document identifier hashes if the documents.
  *@param originalActivities is the object to use to log the details of the ingestion attempt.  May be null.
  */
  @Override
  public void documentDeleteMultiple(
    IPipelineConnections pipelineConnections,
    String[] identifierClasses, String[] identifierHashes,
    IOutputRemoveActivity originalActivities)
    throws ManifoldCFException, ServiceInterruption
  {
    String[] outputConnectionNames = pipelineConnections.getOutputConnectionNames();
    IOutputConnection[] outputConnections = pipelineConnections.getOutputConnections();
    
    // No transactions here, so we can cycle through the connection names one at a time
    for (int z = 0; z < outputConnectionNames.length; z++)
    {
      String outputConnectionName = outputConnectionNames[z];
      IOutputConnection connection = outputConnections[z];

      IOutputRemoveActivity activities = new OutputRemoveActivitiesWrapper(originalActivities,outputConnectionName);

      if (Logging.ingest.isDebugEnabled())
      {
        for (int i = 0; i < identifierHashes.length; i++)
        {
          Logging.ingest.debug("Request to delete document '"+makeKey(identifierClasses[i],identifierHashes[i])+"' from output connection '"+outputConnectionName+"'");
        }
      }

      // No transactions.  Time for the operation may exceed transaction timeout.

      // Obtain the current URIs of all of these.
      List<DeleteInfo> uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes);

      // Grab critical section locks so that we can't attempt to ingest at the same time we are deleting.
      // (This guarantees that when this operation is complete the database reflects reality.)
      int validURIcount = 0;
      for (DeleteInfo uri : uris)
      {
        if (uri.getURI() != null)
          validURIcount++;
      }
      String[] lockArray = new String[validURIcount];
      String[] validURIArray = new String[validURIcount];
      String[] validURIHashArray = new String[validURIcount];
      validURIcount = 0;
      for (DeleteInfo uri : uris)
      {
        if (uri.getURI() != null)
        {
          validURIArray[validURIcount] = uri.getURI();
          validURIHashArray[validURIcount] = uri.getURIHash();
          lockArray[validURIcount] = createURILockName(outputConnectionName,validURIHashArray[validURIcount]);
          validURIcount++;
        }
      }

      lockManager.enterLocks(null,null,lockArray);
      try
      {
        // Fetch the document URIs for the listed documents
        for (DeleteInfo uri : uris)
        {
          if (uri.getURI() != null)
          {
            removeDocument(connection,uri.getURI(),uri.getOutputVersion(),activities);
          }
        }

        // Now, get rid of all rows that match the given uris.
        // Do the queries together, then the deletes
        beginTransaction();
        try
        {
          // The basic process is this:
          // 1) Come up with a set of urihash values
          // 2) Find the matching, corresponding id values
          // 3) Delete the rows corresponding to the id values, in sequence

          // Process (1 & 2) has to be broken down into chunks that contain the maximum
          // number of doc hash values each.  We need to avoid repeating doc hash values,
          // so the first step is to come up with ALL the doc hash values before looping
          // over them.

          int maxClauses;
          
          // Find all the documents that match this set of URIs
          Set<String> docURIHashValues = new HashSet<String>();
          Set<String> docURIValues = new HashSet<String>();
          for (String docDBString : validURIArray)
          {
            docURIValues.add(docDBString);
          }
          for (String docDBString : validURIHashArray)
          {
            docURIHashValues.add(docDBString);
          }

          // Now, perform n queries, each of them no larger the maxInClause in length.
          // Create a list of row id's from this.
          Set<Long> rowIDSet = new HashSet<Long>();
          Iterator<String> iter = docURIHashValues.iterator();
          int j = 0;
          List<String> hashList = new ArrayList<String>();
          maxClauses = maxClausesRowIdsForURIs(outputConnectionName);
          while (iter.hasNext())
          {
            if (j == maxClauses)
            {
              findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList);
              hashList.clear();
              j = 0;
            }
            hashList.add(iter.next());
            j++;
          }

          if (j > 0)
            findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList);

          // Next, go through the list of row IDs, and delete them in chunks
          j = 0;
          List<Long> list = new ArrayList<Long>();
          Iterator<Long> iter2 = rowIDSet.iterator();
          maxClauses = maxClausesDeleteRowIds();
          while (iter2.hasNext())
          {
            if (j == maxClauses)
            {
              deleteRowIds(list);
              list.clear();
              j = 0;
            }
            list.add(iter2.next());
            j++;
          }

          if (j > 0)
            deleteRowIds(list);

          // Now, find the set of documents that remain that match the document identifiers.
          Set<String> docIdValues = new HashSet<String>();
          for (int i = 0; i < identifierHashes.length; i++)
          {
            String docDBString = makeKey(identifierClasses[i],identifierHashes[i]);
            docIdValues.add(docDBString);
          }

          // Now, perform n queries, each of them no larger the maxInClause in length.
          // Create a list of row id's from this.
          rowIDSet.clear();
          iter = docIdValues.iterator();
          j = 0;
          List<String> list2 = new ArrayList<String>();
          maxClauses = maxClausesRowIdsForDocIds(outputConnectionName);
          while (iter.hasNext())
          {
            if (j == maxClauses)
            {
              findRowIdsForDocIds(outputConnectionName,rowIDSet,list2);
              list2.clear();
              j = 0;
            }
            list2.add(iter.next());
            j++;
          }

          if (j > 0)
            findRowIdsForDocIds(outputConnectionName,rowIDSet,list2);

          // Next, go through the list of row IDs, and delete them in chunks
          j = 0;
          list.clear();
          iter2 = rowIDSet.iterator();
          maxClauses = maxClausesDeleteRowIds();
          while (iter2.hasNext())
          {
            if (j == maxClauses)
            {
              deleteRowIds(list);
              list.clear();
              j = 0;
            }
            list.add(iter2.next());
            j++;
          }

          if (j > 0)
            deleteRowIds(list);

        }
        catch (ManifoldCFException e)
        {
          signalRollback();
          throw e;
        }
        catch (Error e)
        {
          signalRollback();
          throw e;
        }
        finally
        {
          endTransaction();
        }
      }
      finally
      {
        lockManager.leaveLocks(null,null,lockArray);
      }
    }
  }

  /** Remove multiple document components from the search engine index.
  *@param pipelineConnections is the pipeline specification.
  *@param identifierClasses are the names of the spaces in which the identifier hash should be interpreted.
  *@param identifierHashes are the hashes of the ids of the documents.
  *@param componentHash is the hashed component identifier, if any.
  *@param activities is the object to use to log the details of the ingestion attempt.  May be null.
  */
  @Override
  public void documentRemoveMultiple(
    IPipelineConnections pipelineConnections,
    String[] identifierClasses, String[] identifierHashes, String componentHash,
    IOutputRemoveActivity activities)
    throws ManifoldCFException, ServiceInterruption
  {
    String[] outputConnectionNames = pipelineConnections.getOutputConnectionNames();
    IOutputConnection[] outputConnections = pipelineConnections.getOutputConnections();
    
    // No transactions here, so we can cycle through the connection names one at a time
    for (int z = 0; z < outputConnectionNames.length; z++)
    {
      String outputConnectionName = outputConnectionNames[z];
      IOutputConnection connection = outputConnections[z];

      activities = new OutputRemoveActivitiesWrapper(activities,outputConnectionName);

      if (Logging.ingest.isDebugEnabled())
      {
        for (int i = 0; i < identifierHashes.length; i++)
        {
          Logging.ingest.debug("Request to remove document '"+makeKey(identifierClasses[i],identifierHashes[i])+"' component hash "+((componentHash==null)?"(None)":("'"+componentHash+"'"))+" from output connection '"+outputConnectionName+"'");
        }
      }

      // No transactions.  Time for the operation may exceed transaction timeout.

      // Obtain the current URIs of all of these.
      List<DeleteInfo> uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes,componentHash);

      // Grab critical section locks so that we can't attempt to ingest at the same time we are deleting.
      // (This guarantees that when this operation is complete the database reflects reality.)
      int validURIcount = 0;
      for (DeleteInfo uri : uris)
      {
        if (uri.getURI() != null)
          validURIcount++;
      }
      String[] lockArray = new String[validURIcount];
      String[] validURIArray = new String[validURIcount];
      String[] validURIHashArray = new String[validURIcount];
      validURIcount = 0;
      for (DeleteInfo uri : uris)
      {
        if (uri.getURI() != null)
        {
          validURIArray[validURIcount] = uri.getURI();
          validURIHashArray[validURIcount] = uri.getURIHash();
          lockArray[validURIcount] = createURILockName(outputConnectionName,validURIHashArray[validURIcount]);
          validURIcount++;
        }
      }

      lockManager.enterLocks(null,null,lockArray);
      try
      {
        // Fetch the document URIs for the listed documents
        for (DeleteInfo uri : uris)
        {
          if (uri.getURI() != null)
            removeDocument(connection,uri.getURI(),uri.getOutputVersion(),activities);
        }

        // Now, get rid of all rows that match the given uris.
        // Do the queries together, then the deletes
        beginTransaction();
        try
        {
          // The basic process is this:
          // 1) Come up with a set of urihash values
          // 2) Find the matching, corresponding id values
          // 3) Delete the rows corresponding to the id values, in sequence

          // Process (1 & 2) has to be broken down into chunks that contain the maximum
          // number of doc hash values each.  We need to avoid repeating doc hash values,
          // so the first step is to come up with ALL the doc hash values before looping
          // over them.

          int maxClauses;
          
          // Find all the documents that match this set of URIs
          Set<String> docURIHashValues = new HashSet<String>();
          Set<String> docURIValues = new HashSet<String>();
          for (String docDBString : validURIArray)
          {
            docURIValues.add(docDBString);
          }
          for (String docDBString : validURIHashArray)
          {
            docURIHashValues.add(docDBString);
          }

          // Now, perform n queries, each of them no larger the maxInClause in length.
          // Create a list of row id's from this.
          Set<Long> rowIDSet = new HashSet<Long>();
          Iterator<String> iter = docURIHashValues.iterator();
          int j = 0;
          List<String> hashList = new ArrayList<String>();
          maxClauses = maxClausesRowIdsForURIs(outputConnectionName);
          while (iter.hasNext())
          {
            if (j == maxClauses)
            {
              findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList);
              hashList.clear();
              j = 0;
            }
            hashList.add(iter.next());
            j++;
          }

          if (j > 0)
            findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList);

          // Next, go through the list of row IDs, and delete them in chunks
          j = 0;
          List<Long> list = new ArrayList<Long>();
          Iterator<Long> iter2 = rowIDSet.iterator();
          maxClauses = maxClausesDeleteRowIds();
          while (iter2.hasNext())
          {
            if (j == maxClauses)
            {
              deleteRowIds(list);
              list.clear();
              j = 0;
            }
            list.add(iter2.next());
            j++;
          }

          if (j > 0)
            deleteRowIds(list);

          // Now, find the set of documents that remain that match the document identifiers.
          Set<String> docIdValues = new HashSet<String>();
          for (int i = 0; i < identifierHashes.length; i++)
          {
            String docDBString = makeKey(identifierClasses[i],identifierHashes[i]);
            docIdValues.add(docDBString);
          }

          // Now, perform n queries, each of them no larger the maxInClause in length.
          // Create a list of row id's from this.
          rowIDSet.clear();
          iter = docIdValues.iterator();
          j = 0;
          List<String> list2 = new ArrayList<String>();
          maxClauses = maxClausesRowIdsForDocIds(outputConnectionName,componentHash);
          while (iter.hasNext())
          {
            if (j == maxClauses)
            {
              findRowIdsForDocIds(outputConnectionName,rowIDSet,list2,componentHash);
              list2.clear();
              j = 0;
            }
            list2.add(iter.next());
            j++;
          }

          if (j > 0)
            findRowIdsForDocIds(outputConnectionName,rowIDSet,list2,componentHash);

          // Next, go through the list of row IDs, and delete them in chunks
          j = 0;
          list.clear();
          iter2 = rowIDSet.iterator();
          maxClauses = maxClausesDeleteRowIds();
          while (iter2.hasNext())
          {
            if (j == maxClauses)
            {
              deleteRowIds(list);
              list.clear();
              j = 0;
            }
            list.add(iter2.next());
            j++;
          }

          if (j > 0)
            deleteRowIds(list);

        }
        catch (ManifoldCFException e)
        {
          signalRollback();
          throw e;
        }
        catch (Error e)
        {
          signalRollback();
          throw e;
        }
        finally
        {
          endTransaction();
        }
      }
      finally
      {
        lockManager.leaveLocks(null,null,lockArray);
      }
    }
  }

  /** Calculate the clauses.
  */
  protected int maxClausesRowIdsForURIs(String outputConnectionName)
  {
    return findConjunctionClauseMax(new ClauseDescription[]{
      new UnitaryClause(outputConnNameField,outputConnectionName)});
  }
  
  /** Given values and parameters corresponding to a set of hash values, add corresponding
  * table row id's to the output map.
  */
  protected void findRowIdsForURIs(String outputConnectionName, Set<Long> rowIDSet, Set<String> uris, List<String> hashParamValues)
    throws ManifoldCFException
  {
    ArrayList list = new ArrayList();
    String query = buildConjunctionClause(list,new ClauseDescription[]{
      new MultiClause(uriHashField,hashParamValues),
      new UnitaryClause(outputConnNameField,outputConnectionName)});
      
    IResultSet set = performQuery("SELECT "+idField+","+docURIField+" FROM "+
      getTableName()+" WHERE "+query,list,null,null);
    
    for (int i = 0; i < set.getRowCount(); i++)
    {
      IResultRow row = set.getRow(i);
      String docURI = (String)row.getValue(docURIField);
      if (docURI != null && docURI.length() > 0)
      {
        if (uris.contains(docURI))
        {
          Long rowID = (Long)row.getValue(idField);
          rowIDSet.add(rowID);
        }
      }
    }
  }

  /** Calculate the maximum number of doc ids we should use.
  */
  protected int maxClausesRowIdsForDocIds(String outputConnectionName)
  {
    return findConjunctionClauseMax(new ClauseDescription[]{
      new UnitaryClause(outputConnNameField,outputConnectionName)});
  }

    /** Calculate the maximum number of doc ids we should use.
  */
  protected int maxClausesRowIdsForDocIds(String outputConnectionName, String componentHash)
  {
    return findConjunctionClauseMax(new ClauseDescription[]{
      new UnitaryClause(outputConnNameField,outputConnectionName),
      (componentHash == null || componentHash.length() == 0)?new NullCheckClause(componentHashField,true):new UnitaryClause(componentHashField,componentHash)});
  }

  /** Calculate the maximum number of doc ids we should use.
  */
  protected int maxClausesRowIdsForDocIds(String[] outputConnectionNames)
  {
    return findConjunctionClauseMax(new ClauseDescription[]{
      new MultiClause(outputConnNameField,outputConnectionNames)});
  }
  
  /** Given values and parameters corresponding to a set of hash values, add corresponding
  * table row id's to the output map.
  */
  protected void findRowIdsForDocIds(String outputConnectionName, Set<Long> rowIDSet, List<String> paramValues)
    throws ManifoldCFException
  {
    ArrayList list = new ArrayList();
    String query = buildConjunctionClause(list,new ClauseDescription[]{
      new MultiClause(docKeyField,paramValues),
      new UnitaryClause(outputConnNameField,outputConnectionName)});
      
    IResultSet set = performQuery("SELECT "+idField+" FROM "+
      getTableName()+" WHERE "+query,list,null,null);
    
    for (int i = 0; i < set.getRowCount(); i++)
    {
      IResultRow row = set.getRow(i);
      Long rowID = (Long)row.getValue(idField);
      rowIDSet.add(rowID);
    }
  }

  /** Given values and parameters corresponding to a set of hash values, add corresponding
  * table row id's to the output map.
  */
  protected void findRowIdsForDocIds(String outputConnectionName, Set<Long> rowIDSet, List<String> paramValues, String componentHash)
    throws ManifoldCFException
  {
    ArrayList list = new ArrayList();
    String query = buildConjunctionClause(list,new ClauseDescription[]{
      new MultiClause(docKeyField,paramValues),
      new UnitaryClause(outputConnNameField,outputConnectionName),
      (componentHash==null || componentHash.length() == 0)?new NullCheckClause(componentHashField,true):new UnitaryClause(componentHashField,componentHash)});
      
    IResultSet set = performQuery("SELECT "+idField+" FROM "+
      getTableName()+" WHERE "+query,list,null,null);
    
    for (int i = 0; i < set.getRowCount(); i++)
    {
      IResultRow row = set.getRow(i);
      Long rowID = (Long)row.getValue(idField);
      rowIDSet.add(rowID);
    }
  }

  /** Given values and parameters corresponding to a set of hash values, add corresponding
  * table row id's to the output map.
  */
  protected void findRowIdsForDocIds(String[] outputConnectionNames, Set<Long> rowIDSet, List<String> paramValues)
    throws ManifoldCFException
  {
    ArrayList list = new ArrayList();
    String query = buildConjunctionClause(list,new ClauseDescription[]{
      new MultiClause(docKeyField,paramValues),
      new MultiClause(outputConnNameField,outputConnectionNames)});
      
    IResultSet set = performQuery("SELECT "+idField+" FROM "+
      getTableName()+" WHERE "+query,list,null,null);
    
    for (int i = 0; i < set.getRowCount(); i++)
    {
      IResultRow row = set.getRow(i);
      Long rowID = (Long)row.getValue(idField);
      rowIDSet.add(rowID);
    }
  }

  /** Calculate the maximum number of clauses.
  */
  protected int maxClausesDeleteRowIds()
  {
    return findConjunctionClauseMax(new ClauseDescription[]{});
  }
    
  /** Delete a chunk of row ids.
  */
  protected void deleteRowIds(List<Long> list)
    throws ManifoldCFException
  {
    ArrayList newList = new ArrayList();
    String query = buildConjunctionClause(newList,new ClauseDescription[]{
      new MultiClause(idField,list)});
    performDelete("WHERE "+query,newList,null);
  }

  /** Delete a document from the search engine index.
  *@param pipelineConnections is the pipeline specification.
  *@param identifierClass is the name of the space in which the identifier hash should be interpreted.
  *@param identifierHash is the hash of the id of the document.
  *@param activities is the object to use to log the details of the ingestion attempt.  May be null.
  */
  @Override
  public void documentDelete(
    IPipelineConnections pipelineConnections,
    String identifierClass, String identifierHash,
    IOutputRemoveActivity activities)
    throws ManifoldCFException, ServiceInterruption
  {
    documentDeleteMultiple(pipelineConnections,new String[]{identifierClass},new String[]{identifierHash},activities);
  }

  /** Find out what URIs a SET of document URIs are currently ingested.
  *@param identifierHashes is the array of document id's to check.
  *@return the array of current document uri's.  Null returned for identifiers
  * that don't exist in the index.
  */
  protected List<DeleteInfo> getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses, String[] identifierHashes)
    throws ManifoldCFException
  {
    List<DeleteInfo> rval = new ArrayList<DeleteInfo>();
    beginTransaction();
    try
    {
      List<String> list = new ArrayList<String>();
      int maxCount = maxClauseDocumentURIChunk(outputConnectionName);
      int j = 0;
      for (int i = 0; i < identifierHashes.length; i++)
      {
        if (j == maxCount)
        {
          getDocumentURIChunk(rval,outputConnectionName,list);
          j = 0;
          list.clear();
        }
        list.add(makeKey(identifierClasses[i],identifierHashes[i]));
        j++;
      }
      if (j > 0)
        getDocumentURIChunk(rval,outputConnectionName,list);
      return rval;
    }
    catch (ManifoldCFException e)
    {
      signalRollback();
      throw e;
    }
    catch (Error e)
    {
      signalRollback();
      throw e;
    }
    finally
    {
      endTransaction();
    }
  }

  /** Find out what URIs a SET of document URIs are currently ingested.
  *@param outputConnectionName is the output connection name.
  *@param identifierClasses is the array of identifier classes.
  *@param identifierHashes is the array of document id's to check.
  *@param componentHash is the component hash to check.
  *@return the array of current document uri's.  Null returned for identifiers
  * that don't exist in the index.
  */
  protected List<DeleteInfo> getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses, String[] identifierHashes, String componentHash)
    throws ManifoldCFException
  {
    List<DeleteInfo> rval = new ArrayList<DeleteInfo>();
    beginTransaction();
    try
    {
      List<String> list = new ArrayList<String>();
      int maxCount = maxClauseDocumentURIChunk(outputConnectionName,componentHash);
      int j = 0;
      for (int i = 0; i < identifierHashes.length; i++)
      {
        if (j == maxCount)
        {
          getDocumentURIChunk(rval,outputConnectionName,list,componentHash);
          j = 0;
          list.clear();
        }
        list.add(makeKey(identifierClasses[i],identifierHashes[i]));
        j++;
      }
      if (j > 0)
        getDocumentURIChunk(rval,outputConnectionName,list,componentHash);
      return rval;
    }
    catch (ManifoldCFException e)
    {
      signalRollback();
      throw e;
    }
    catch (Error e)
    {
      signalRollback();
      throw e;
    }
    finally
    {
      endTransaction();
    }
  }

  /** Look up ingestion data for a set of documents.
  *@param rval is a map of output key to document data, in no particular order, which will be loaded with all matching results.
  *@param pipelineSpecificationBasics are the pipeline specifications corresponding to the identifier classes and hashes.
  *@param identifierClasses are the names of the spaces in which the identifier hashes should be interpreted.
  *@param identifierHashes is the array of document identifier hashes to look up.
  */
  @Override
  public void getPipelineDocumentIngestDataMultiple(
    IngestStatuses rval,
    IPipelineSpecificationBasic[] pipelineSpecificationBasics,
    String[] identifierClasses, String[] identifierHashes)
    throws ManifoldCFException
  {
    // Organize by pipeline spec.
    Map<IPipelineSpecificationBasic,List<Integer>> keyMap = new HashMap<IPipelineSpecificationBasic,List<Integer>>();
    for (int i = 0; i < pipelineSpecificationBasics.length; i++)
    {
      IPipelineSpecificationBasic spec = pipelineSpecificationBasics[i];
      List<Integer> list = keyMap.get(spec);
      if (list == null)
      {
        list = new ArrayList<Integer>();
        keyMap.put(spec,list);
      }
      list.add(new Integer(i));
    }

    // Create the return array.
    Iterator<IPipelineSpecificationBasic> iter = keyMap.keySet().iterator();
    while (iter.hasNext())
    {
      IPipelineSpecificationBasic spec = iter.next();
      List<Integer> list = keyMap.get(spec);
      String[] localIdentifierClasses = new String[list.size()];
      String[] localIdentifierHashes = new String[list.size()];
      for (int i = 0; i < localIdentifierClasses.length; i++)
      {
        int index = list.get(i).intValue();
        localIdentifierClasses[i] = identifierClasses[index];
        localIdentifierHashes[i] = identifierHashes[index];
      }
      getPipelineDocumentIngestDataMultiple(rval,spec,localIdentifierClasses,localIdentifierHashes);
    }
  }

  /** Look up ingestion data for a SET of documents.
  *@param rval is a map of output key to document data, in no particular order, which will be loaded with all matching results.
  *@param pipelineSpecificationBasic is the pipeline specification for all documents.
  *@param identifierClasses are the names of the spaces in which the identifier hashes should be interpreted.
  *@param identifierHashes is the array of document identifier hashes to look up.
  */
  @Override
  public void getPipelineDocumentIngestDataMultiple(
    IngestStatuses rval,
    IPipelineSpecificationBasic pipelineSpecificationBasic,
    String[] identifierClasses, String[] identifierHashes)
    throws ManifoldCFException
  {
    String[] outputConnectionNames = extractOutputConnectionNames(pipelineSpecificationBasic);

    // Build a map, so we can convert an identifier into an array index.
    Map<String,Integer> indexMap = new HashMap<String,Integer>();
    for (int i = 0; i < identifierHashes.length; i++)
    {
      indexMap.put(makeKey(identifierClasses[i],identifierHashes[i]),new Integer(i));
    }

    beginTransaction();
    try
    {
      List<String> list = new ArrayList<String>();
      int maxCount = maxClausePipelineDocumentIngestDataChunk(outputConnectionNames);
      int j = 0;
      Iterator<String> iter = indexMap.keySet().iterator();
      while (iter.hasNext())
      {
        if (j == maxCount)
        {
          getPipelineDocumentIngestDataChunk(rval,indexMap,outputConnectionNames,list,identifierClasses,identifierHashes);
          j = 0;
          list.clear();
        }
        list.add(iter.next());
        j++;
      }
      if (j > 0)
        getPipelineDocumentIngestDataChunk(rval,indexMap,outputConnectionNames,list,identifierClasses,identifierHashes);
    }
    catch (ManifoldCFException e)
    {
      signalRollback();
      throw e;
    }
    catch (Error e)
    {
      signalRollback();
      throw e;
    }
    finally
    {
      endTransaction();
    }

  }

  /** Get a chunk of document ingest data records.
  *@param rval is the document ingest status array where the data should be put.
  *@param map is the map from id to index.
  *@param list is the parameter list for the query.
  */
  protected void getPipelineDocumentIngestDataChunk(IngestStatuses rval, Map<String,Integer> map, String[] outputConnectionNames, List<String> list,
    String[] identifierClasses, String[] identifierHashes)
    throws ManifoldCFException
  {
    ArrayList newList = new ArrayList();
    String query = buildConjunctionClause(newList,new ClauseDescription[]{
      new MultiClause(docKeyField,list),
      new MultiClause(outputConnNameField,outputConnectionNames)});
      
    // Get the primary records associated with this hash value
    IResultSet set = performQuery("SELECT "+idField+","+outputConnNameField+","+docKeyField+","+componentHashField+","+lastVersionField+","+lastOutputVersionField+","+authorityNameField+","+lastTransformationVersionField+
      " FROM "+getTableName()+" WHERE "+query,newList,null,null);

    // Now, go through the original request once more, this time building the result
    for (int i = 0; i < set.getRowCount(); i++)
    {
      IResultRow row = set.getRow(i);
      String docHash = row.getValue(docKeyField).toString();
      Integer position = map.get(docHash);
      if (position != null)
      {
        Long id = (Long)row.getValue(idField);
        String outputConnectionName = (String)row.getValue(outputConnNameField);
        String componentHash = (String)row.getValue(componentHashField);
        String lastVersion = (String)row.getValue(lastVersionField);
        if (lastVersion == null)
          lastVersion = "";
        String lastTransformationVersion = (String)row.getValue(lastTransformationVersionField);
        if (lastTransformationVersion == null)
          lastTransformationVersion = "";
        String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
        if (lastOutputVersion == null)
          lastOutputVersion = "";
        String authorityName = (String)row.getValue(authorityNameField);
        if (authorityName == null)
          authorityName = "";
        int indexValue = position.intValue();
        rval.addStatus(identifierClasses[indexValue],identifierHashes[indexValue],outputConnectionName,
          componentHash,new DocumentIngestStatus(lastVersion,lastTransformationVersion,lastOutputVersion,authorityName));
      }
    }
  }
  
  /** Look up ingestion data for a document.
  *@param rval is a map of output key to document data, in no particular order, which will be loaded with all matching results.
  *@param pipelineSpecificationBasic is the pipeline specification for the document.
  *@param identifierClass is the name of the space in which the identifier hash should be interpreted.
  *@param identifierHash is the hash of the id of the document.
  */
  @Override
  public void getPipelineDocumentIngestData(
    IngestStatuses rval,
    IPipelineSpecificationBasic pipelineSpecificationBasic,
    String identifierClass, String identifierHash)
    throws ManifoldCFException
  {
    getPipelineDocumentIngestDataMultiple(rval,pipelineSpecificationBasic,
      new String[]{identifierClass},new String[]{identifierHash});
  }

  /** Calculate the average time interval between changes for a document.
  * This is based on the data gathered for the document.
  *@param pipelineSpecificationBasic is the basic pipeline specification.
  *@param identifierClasses are the names of the spaces in which the identifier hashes should be interpreted.
  *@param identifierHashes is the hashes of the ids of the documents.
  *@return the number of milliseconds between changes, or 0 if this cannot be calculated.
  */
  @Override
  public long[] getDocumentUpdateIntervalMultiple(
    IPipelineSpecificationBasic pipelineSpecificationBasic,
    String[] identifierClasses, String[] identifierHashes)
    throws ManifoldCFException
  {
    // Get the output connection names
    String[] outputConnectionNames = extractOutputConnectionNames(pipelineSpecificationBasic);

    // Do these all at once!!
    // First, create a return array
    long[] rval = new long[identifierHashes.length];
    // Also create a map from identifier to return index.
    Map<String,Integer> returnMap = new HashMap<String,Integer>();
    // Finally, need the set of hash codes
    Set<String> idCodes = new HashSet<String>();
    for (int j = 0; j < identifierHashes.length; j++)
    {
      String key = makeKey(identifierClasses[j],identifierHashes[j]);
      rval[j] = Long.MAX_VALUE;
      returnMap.put(key,new Integer(j));
      idCodes.add(key);
    }

    // Get the chunk size
    int maxClause = maxClauseGetIntervals(outputConnectionNames);

    // Loop through the hash codes
    Iterator<String> iter = idCodes.iterator();
    List<String> list = new ArrayList<String>();
    int j = 0;
    while (iter.hasNext())
    {
      if (j == maxClause)
      {
        getIntervals(rval,outputConnectionNames,list,returnMap);
        list.clear();
        j = 0;
      }

      list.add(iter.next());
      j++;
    }

    if (j > 0)
      getIntervals(rval,outputConnectionNames,list,returnMap);

    for (int i = 0; i < rval.length; i++)
    {
      if (rval[i] == Long.MAX_VALUE)
        rval[i] = 0;
    }
    
    return rval;

  }

  /** Calculate the average time interval between changes for a document.
  * This is based on the data gathered for the document.
  *@param pipelineSpecificationBasic is the basic pipeline specification.
  *@param identifierClass is the name of the space in which the identifier hash should be interpreted.
  *@param identifierHash is the hash of the id of the document.
  *@return the number of milliseconds between changes, or 0 if this cannot be calculated.
  */
  @Override
  public long getDocumentUpdateInterval(
    IPipelineSpecificationBasic pipelineSpecificationBasic,
    String identifierClass, String identifierHash)
    throws ManifoldCFException
  {
    return getDocumentUpdateIntervalMultiple(
      pipelineSpecificationBasic,
      new String[]{identifierClass},new String[]{identifierHash})[0];
  }

  /** Calculate the number of clauses.
  */
  protected int maxClauseGetIntervals(String[] outputConnectionNames)
  {
    return findConjunctionClauseMax(new ClauseDescription[]{
      new MultiClause(outputConnNameField,outputConnectionNames)});
  }
  
  /** Query for and calculate the interval for a bunch of hashcodes.
  *@param rval is the array to stuff calculated return values into.
  *@param list is the list of parameters.
  *@param returnMap is a mapping from document id to rval index.
  */
  protected void getIntervals(long[] rval, String[] outputConnectionNames, List<String> list, Map<String,Integer> returnMap)
    throws ManifoldCFException
  {
    ArrayList newList = new ArrayList();
    String query = buildConjunctionClause(newList,new ClauseDescription[]{
      new MultiClause(docKeyField,list),
      new MultiClause(outputConnNameField,outputConnectionNames)});
      
    IResultSet set = performQuery("SELECT "+docKeyField+","+changeCountField+","+firstIngestField+","+lastIngestField+
      " FROM "+getTableName()+" WHERE "+query,newList,null,null);

    for (int i = 0; i < set.getRowCount(); i++)
    {
      IResultRow row = set.getRow(i);
      String docHash = (String)row.getValue(docKeyField);
      Integer index = (Integer)returnMap.get(docHash);
      if (index != null)
      {
        // Calculate the return value
        long changeCount = ((Long)row.getValue(changeCountField)).longValue();
        long firstIngest = ((Long)row.getValue(firstIngestField)).longValue();
        long lastIngest = ((Long)row.getValue(lastIngestField)).longValue();
        int indexValue = index.intValue();
        long newValue = (long)(((double)(lastIngest-firstIngest))/(double)changeCount);
        if (newValue < rval[indexValue])
          rval[indexValue] = newValue;
      }
    }
  }

  /** Reset all documents belonging to a specific output connection, because we've got information that
  * that system has been reconfigured.  This will force all such documents to be reindexed the next time
  * they are checked.
  *@param outputConnection is the output connection associated with this action.
  */
  @Override
  public void resetOutputConnection(IOutputConnection outputConnection)
    throws ManifoldCFException
  {
    if (outputConnection == null)
      return;

    // We're not going to blow away the records, but we are going to set their versions to mean, "reindex required"
    HashMap map = new HashMap();
    map.put(lastVersionField,null);
    ArrayList list = new ArrayList();
    String query = buildConjunctionClause(list,new ClauseDescription[]{
      new UnitaryClause(outputConnNameField,outputConnection.getName())});
      
    performUpdate(map,"WHERE "+query,list,null);
  }

  /** Remove all knowledge of an output index from the system.  This is appropriate
  * when the output index no longer exists and you wish to delete the associated job.
  *@param outputConnection is the output connection associated with this action.
  */
  @Override
  public void removeOutputConnection(IOutputConnection outputConnection)
    throws ManifoldCFException
  {
    if (outputConnection == null)
      return;

    ArrayList list = new ArrayList();
    String query = buildConjunctionClause(list,new ClauseDescription[]{
      new UnitaryClause(outputConnNameField,outputConnection.getName())});
      
    performDelete("WHERE "+query,list,null);
      
    // Notify the output connection of the removal of all the records for the connection
    IOutputConnector connector = outputConnectorPool.grab(outputConnection);
    if (connector == null)
      return;
    try
    {
      connector.noteAllRecordsRemoved();
    }
    finally
    {
      outputConnectorPool.release(outputConnection,connector);
    }

  }
  
  /** Note the ingestion of a document, or the "update" of a document.
  *@param outputConnectionName is the name of the output connection.
  *@param docKey is the key string describing the document.
  *@param componentHash is the component identifier hash for this document.
  *@param documentVersion is a string describing the new version of the document.
  *@param transformationVersion is a string describing all current transformations for the document.
  *@param outputVersion is the version string calculated for the output connection.
  *@param authorityNameString is the name of the relevant authority connection.
  *@param ingestTime is the time at which the ingestion took place, in milliseconds since epoch.
  *@param documentURI is the uri the document can be accessed at, or null (which signals that we are to record the version, but no
  * ingestion took place).
  *@param documentURIHash is the hash of the document uri.
  */
  protected void noteDocumentIngest(String outputConnectionName,
    String docKey, String componentHash, String documentVersion, String transformationVersion,
    String outputVersion,
    String authorityNameString,
    long ingestTime, String documentURI, String documentURIHash)
    throws ManifoldCFException
  {
    HashMap map = new HashMap();
    while (true)
    {
      // The table can have at most one row per URI, for non-null URIs.  It can also have at most one row per document identifier.
      // However, for null URI's, multiple rows are allowed.  Null URIs have a special meaning, which is that
      // the document was not actually ingested.

      // To make sure the constraints are enforced, we cannot simply look for the row and insert one if not found.  This is because
      // postgresql does not cause a lock to be created on rows that don't yet exist, so multiple transactions of the kind described
      // can lead to multiple rows with the same key.  Instead, we *could* lock the whole table down, but that would interfere with
      // parallelism.  The lowest-impact approach is to make sure an index constraint is in place, and first attempt to do an INSERT.
      // That attempt will fail if a record already exists.  Then, an update can be attempted.
      //
      // In the situation where the INSERT fails, the current transaction is aborted and a new transaction must be performed.
      // This means that it is impossible to structure things so that the UPDATE is guaranteed to succeed.  So, on the event of an
      // INSERT failure, the UPDATE is tried, but if that fails too, then the INSERT is tried again.  This should also handle the
      // case where a DELETE in another transaction removes the database row before it can be UPDATEd.
      //
      // If the UPDATE does not appear to modify any rows, this is also a signal that the INSERT must be retried.
      //

      // Try the update first.  Typically this succeeds except in the case where a doc is indexed for the first time.
      map.clear();
      if (componentHash != null)
        map.put(componentHashField,componentHash);
      map.put(lastVersionField,documentVersion);
      map.put(lastTransformationVersionField,transformationVersion);
      map.put(lastOutputVersionField,outputVersion);
      map.put(lastIngestField,new Long(ingestTime));
      if (documentURI != null)
      {
        map.put(docURIField,documentURI);
        map.put(uriHashField,documentURIHash);
      }
      if (authorityNameString != null)
        map.put(authorityNameField,authorityNameString);
      else
        map.put(authorityNameField,"");
      
      // Transaction abort due to deadlock should be retried here.
      while (true)
      {
        long sleepAmt = 0L;

        beginTransaction();
        try
        {
          // Look for existing row.
          ArrayList list = new ArrayList();
          String query = buildConjunctionClause(list,new ClauseDescription[]{
            new UnitaryClause(docKeyField,docKey),
            new UnitaryClause(outputConnNameField,outputConnectionName),
            ((componentHash==null)?new NullCheckClause(componentHashField,true):new UnitaryClause(componentHashField,componentHash))});
          IResultSet set = performQuery("SELECT "+idField+","+changeCountField+" FROM "+getTableName()+" WHERE "+
            query+" FOR UPDATE",list,null,null);
          IResultRow row = null;
          if (set.getRowCount() > 0)
            row = set.getRow(0);

          if (row != null)
          {
            // Update the record
            list.clear();
            query = buildConjunctionClause(list,new ClauseDescription[]{
              new UnitaryClause(idField,row.getValue(idField))});
            long changeCount = ((Long)row.getValue(changeCountField)).longValue();
            changeCount++;
            map.put(changeCountField,new Long(changeCount));
            performUpdate(map,"WHERE "+query,list,null);
            // Update successful!
            performCommit();
            return;
          }

          // Update failed to find a matching record, so try the insert
          break;
        }
        catch (ManifoldCFException e)
        {
          signalRollback();
          if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
          {
            if (Logging.perf.isDebugEnabled())
              Logging.perf.debug("Aborted transaction noting ingestion: "+e.getMessage());
            sleepAmt = getSleepAmt();
            continue;
          }

          throw e;
        }
        catch (Error e)
        {
          signalRollback();
          throw e;
        }
        finally
        {
          endTransaction();
          sleepFor(sleepAmt);
        }
      }

      // Set up for insert
      map.clear();
      if (componentHash != null)
        map.put(componentHashField,componentHash);
      map.put(lastVersionField,documentVersion);
      map.put(lastTransformationVersionField,transformationVersion);
      map.put(lastOutputVersionField,outputVersion);
      map.put(lastIngestField,new Long(ingestTime));
      if (documentURI != null)
      {
        map.put(docURIField,documentURI);
        map.put(uriHashField,documentURIHash);
      }
      if (authorityNameString != null)
        map.put(authorityNameField,authorityNameString);
      else
        map.put(authorityNameField,"");

      Long id = new Long(IDFactory.make(threadContext));
      map.put(idField,id);
      map.put(outputConnNameField,outputConnectionName);
      map.put(docKeyField,docKey);
      map.put(changeCountField,new Long(1));
      map.put(firstIngestField,map.get(lastIngestField));
      beginTransaction();
      try
      {
        performInsert(map,null);
        noteModifications(1,0,0);
        performCommit();
        return;
      }
      catch (ManifoldCFException e)
      {
        signalRollback();
        // If this is simply a constraint violation, we just want to fall through and try the update!
        if (e.getErrorCode() != ManifoldCFException.DATABASE_TRANSACTION_ABORT)
          throw e;
        // Otherwise, exit transaction and fall through to 'update' attempt
      }
      catch (Error e)
      {
        signalRollback();
        throw e;
      }
      finally
      {
        endTransaction();
      }

      // Insert must have failed.  Attempt an update.
    }
  }

  /** Calculate how many clauses at a time
  */
  protected int maxClauseDocumentURIChunk(String outputConnectionName)
  {
    return findConjunctionClauseMax(new ClauseDescription[]{
      new UnitaryClause(outputConnNameField,outputConnectionName)});
  }
  
  /** Get a chunk of document uris.
  *@param rval is the string array where the uris should be put.
  *@param list are the doc keys for the query.
  */
  protected void getDocumentURIChunk(List<DeleteInfo> rval, String outputConnectionName,
    List<String> list)
    throws ManifoldCFException
  {
    ArrayList newList = new ArrayList();
    String query = buildConjunctionClause(newList,new ClauseDescription[]{
      new MultiClause(docKeyField,list),
      new UnitaryClause(outputConnNameField,outputConnectionName)});
      
    IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+uriHashField+","+lastOutputVersionField+" FROM "+getTableName()+" WHERE "+
      query,newList,null,null);

    // Go through list and put into buckets.
    for (int i = 0; i < set.getRowCount(); i++)
    {
      IResultRow row = set.getRow(i);
      //String docHash = row.getValue(docKeyField).toString();
      String lastURI = (String)row.getValue(docURIField);
      if (lastURI != null && lastURI.length() == 0)
        lastURI = null;
      String lastURIHash = (String)row.getValue(uriHashField);
      if (lastURIHash != null && lastURIHash.length() == 0)
        lastURIHash = null;
      String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
      rval.add(new DeleteInfo(lastURI,lastURIHash,lastOutputVersion));
    }
  }

  /** Calculate how many clauses at a time
  */
  protected int maxClauseDocumentURIChunk(String outputConnectionName, String componentHash)
  {
    return findConjunctionClauseMax(new ClauseDescription[]{
      new UnitaryClause(outputConnNameField,outputConnectionName),
      ((componentHash==null)?new NullCheckClause(componentHashField,true):new UnitaryClause(componentHashField,componentHash))});
  }

  /** Get a chunk of document uris.
  *@param rval is the string array where the uris should be put.
  *@param list are the doc keys for the query.
  *@param componentHash is the component hash, if any, for the query.
  */
  protected void getDocumentURIChunk(List<DeleteInfo> rval, String outputConnectionName,
    List<String> list, String componentHash)
    throws ManifoldCFException
  {
    ArrayList newList = new ArrayList();
    String query = buildConjunctionClause(newList,new ClauseDescription[]{
      new MultiClause(docKeyField,list),
      new UnitaryClause(outputConnNameField,outputConnectionName),
      ((componentHash==null)?new NullCheckClause(componentHashField,true):new UnitaryClause(componentHashField,componentHash))});
      
    IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+uriHashField+","+lastOutputVersionField+" FROM "+getTableName()+" WHERE "+
      query,newList,null,null);

    // Go through list and put into buckets.
    for (int i = 0; i < set.getRowCount(); i++)
    {
      IResultRow row = set.getRow(i);
      //String docHash = row.getValue(docKeyField).toString();
      String lastURI = (String)row.getValue(docURIField);
      if (lastURI != null && lastURI.length() == 0)
        lastURI = null;
      String lastURIHash = (String)row.getValue(uriHashField);
      if (lastURIHash != null && lastURIHash.length() == 0)
        lastURIHash = null;
      String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
      rval.add(new DeleteInfo(lastURI,lastURIHash,lastOutputVersion));
    }
  }

  /** Count the clauses
  */
  protected int maxClauseDocumentIngestDataChunk(String outputConnectionName)
  {
    return findConjunctionClauseMax(new ClauseDescription[]{
      new UnitaryClause(outputConnNameField,outputConnectionName)});
  }

  /** Count the clauses
  */
  protected int maxClausePipelineDocumentIngestDataChunk(String[] outputConnectionNames)
  {
    return findConjunctionClauseMax(new ClauseDescription[]{
      new MultiClause(outputConnNameField,outputConnectionNames)});
  }
  

  // Protected methods

  /** Remove document, using the specified output connection, via the standard pool.
  */
  protected void removeDocument(IOutputConnection connection, String documentURI, String outputDescription, IOutputRemoveActivity activities)
    throws ManifoldCFException, ServiceInterruption
  {
    IOutputConnector connector = outputConnectorPool.grab(connection);
    if (connector == null)
      // The connector is not installed; treat this as a service interruption.
      throw new ServiceInterruption("Output connector not installed",0L);
    try
    {
      connector.removeDocument(documentURI,outputDescription,activities);
    }
    finally
    {
      outputConnectorPool.release(connection,connector);
    }
  }

  /** Make a key from a document class and a hash */
  protected static String makeKey(String documentClass, String documentHash)
  {
    return documentClass + ":" + documentHash;
  }

  /** This class contains the information necessary to delete a document */
  protected static class DeleteInfo
  {
    protected String uriValue;
    protected String uriHashValue;
    protected String outputVersion;

    public DeleteInfo(String uriValue, String uriHashValue, String outputVersion)
    {
      this.uriValue = uriValue;
      this.uriHashValue = uriHashValue;
      this.outputVersion = outputVersion;
    }

    public String getURI()
    {
      return uriValue;
    }

    public String getURIHash()
    {
      return uriHashValue;
    }

    public String getOutputVersion()
    {
      return outputVersion;
    }
  }
  
  /** Wrapper class for add activity.  This handles conversion of output connector activity logging to 
  * qualified activity names */
  protected static class OutputRecordingActivity implements IOutputHistoryActivity
  {
    protected final IOutputHistoryActivity activityProvider;
    protected final String outputConnectionName;
    
    public OutputRecordingActivity(IOutputHistoryActivity activityProvider, String outputConnectionName)
    {
      this.activityProvider = activityProvider;
      this.outputConnectionName = outputConnectionName;
    }
    
    /** Record time-stamped information about the activity of the output connector.
    *@param startTime is either null or the time since the start of epoch in milliseconds (Jan 1, 1970).  Every
    *       activity has an associated time; the startTime field records when the activity began.  A null value
    *       indicates that the start time and the finishing time are the same.
    *@param activityType is a string which is fully interpretable only in the context of the connector involved, which is
    *       used to categorize what kind of activity is being recorded.  For example, a web connector might record a
    *       "fetch document" activity.  Cannot be null.
    *@param dataSize is the number of bytes of data involved in the activity, or null if not applicable.
    *@param entityURI is a (possibly long) string which identifies the object involved in the history record.
    *       The interpretation of this field will differ from connector to connector.  May be null.
    *@param resultCode contains a terse description of the result of the activity.  The description is limited in
    *       size to 255 characters, and can be interpreted only in the context of the current connector.  May be null.
    *@param resultDescription is a (possibly long) human-readable string which adds detail, if required, to the result
    *       described in the resultCode field.  This field is not meant to be queried on.  May be null.
    */
    @Override
    public void recordActivity(Long startTime, String activityType, Long dataSize,
      String entityURI, String resultCode, String resultDescription)
      throws ManifoldCFException
    {
      activityProvider.recordActivity(startTime,ManifoldCF.qualifyOutputActivityName(activityType,outputConnectionName),
        dataSize,entityURI,resultCode,resultDescription);
    }

  }
  
  /** Wrapper class for add activity.  This handles conversion of transformation connector activity logging to 
  * qualified activity names */
  protected static class TransformationRecordingActivity implements IOutputHistoryActivity
  {
    protected final IOutputHistoryActivity activityProvider;
    protected final String transformationConnectionName;
    
    public TransformationRecordingActivity(IOutputHistoryActivity activityProvider, String transformationConnectionName)
    {
      this.activityProvider = activityProvider;
      this.transformationConnectionName = transformationConnectionName;
    }
    
    /** Record time-stamped information about the activity of the output connector.
    *@param startTime is either null or the time since the start of epoch in milliseconds (Jan 1, 1970).  Every
    *       activity has an associated time; the startTime field records when the activity began.  A null value
    *       indicates that the start time and the finishing time are the same.
    *@param activityType is a string which is fully interpretable only in the context of the connector involved, which is
    *       used to categorize what kind of activity is being recorded.  For example, a web connector might record a
    *       "fetch document" activity.  Cannot be null.
    *@param dataSize is the number of bytes of data involved in the activity, or null if not applicable.
    *@param entityURI is a (possibly long) string which identifies the object involved in the history record.
    *       The interpretation of this field will differ from connector to connector.  May be null.
    *@param resultCode contains a terse description of the result of the activity.  The description is limited in
    *       size to 255 characters, and can be interpreted only in the context of the current connector.  May be null.
    *@param resultDescription is a (possibly long) human-readable string which adds detail, if required, to the result
    *       described in the resultCode field.  This field is not meant to be queried on.  May be null.
    */
    @Override
    public void recordActivity(Long startTime, String activityType, Long dataSize,
      String entityURI, String resultCode, String resultDescription)
      throws ManifoldCFException
    {
      activityProvider.recordActivity(startTime,ManifoldCF.qualifyTransformationActivityName(activityType,transformationConnectionName),
        dataSize,entityURI,resultCode,resultDescription);
    }

  }

  protected static class OutputRemoveActivitiesWrapper extends OutputRecordingActivity implements IOutputRemoveActivity
  {
    protected final IOutputRemoveActivity removeActivities;
    
    public OutputRemoveActivitiesWrapper(IOutputRemoveActivity removeActivities, String outputConnectionName)
    {
      super(removeActivities,outputConnectionName);
      this.removeActivities = removeActivities;
    }

  }
  
  protected static class OutputAddActivitiesWrapper extends OutputRecordingActivity implements IOutputAddActivity
  {
    protected final IOutputAddActivity addActivities;
    
    public OutputAddActivitiesWrapper(IOutputAddActivity addActivities, String outputConnectionName)
    {
      super(addActivities,outputConnectionName);
      this.addActivities = addActivities;
    }
    
    /** Qualify an access token appropriately, to match access tokens as returned by mod_aa.  This method
    * includes the authority name with the access token, if any, so that each authority may establish its own token space.
    *@param authorityNameString is the name of the authority to use to qualify the access token.
    *@param accessToken is the raw, repository access token.
    *@return the properly qualified access token.
    */
    @Override
    public String qualifyAccessToken(String authorityNameString, String accessToken)
      throws ManifoldCFException
    {
      return addActivities.qualifyAccessToken(authorityNameString,accessToken);
    }

    /** Send a document via the pipeline to the next output connection.
    *@param documentURI is the document's URI.
    *@param document is the document data to be processed (handed to the output data store).
    *@return the document status (accepted or permanently rejected); return codes are listed in IPipelineConnector.
    *@throws IOException only if there's an IO error reading the data from the document.
    */
    @Override
    public int sendDocument(String documentURI, RepositoryDocument document)
      throws ManifoldCFException, ServiceInterruption, IOException
    {
      return addActivities.sendDocument(documentURI,document);
    }

    /** Send NO document via the pipeline to the next output connection.  This is equivalent
    * to sending an empty document placeholder.
    */
    @Override
    public void noDocument()
      throws ManifoldCFException, ServiceInterruption
    {
      addActivities.noDocument();
    }

    /** Detect if a date is acceptable downstream or not.  This method is used to determine whether it makes sense to fetch a document
    * in the first place.
    *@param date is the mime type of the document.
    *@return true if the date can be accepted by the downstream connection.
    */
    @Override
    public boolean checkDateIndexable(Date date)
      throws ManifoldCFException, ServiceInterruption
    {
      return addActivities.checkDateIndexable(date);
    }

    /** Detect if a mime type is acceptable downstream or not.  This method is used to determine whether it makes sense to fetch a document
    * in the first place.
    *@param mimeType is the mime type of the document.
    *@return true if the mime type can be accepted by the downstream connection.
    */
    @Override
    public boolean checkMimeTypeIndexable(String mimeType)
      throws ManifoldCFException, ServiceInterruption
    {
      return addActivities.checkMimeTypeIndexable(mimeType);
    }

    /** Pre-determine whether a document (passed here as a File object) is acceptable downstream.  This method is
    * used to determine whether a document needs to be actually transferred.  This hook is provided mainly to support
    * search engines that only handle a small set of accepted file types.
    *@param localFile is the local file to check.
    *@return true if the file is acceptable by the downstream connection.
    */
    @Override
    public boolean checkDocumentIndexable(File localFile)
      throws ManifoldCFException, ServiceInterruption
    {
      return addActivities.checkDocumentIndexable(localFile);
    }

    /** Pre-determine whether a document's length is acceptable downstream.  This method is used
    * to determine whether to fetch a document in the first place.
    *@param length is the length of the document.
    *@return true if the file is acceptable by the downstream connection.
    */
    @Override
    public boolean checkLengthIndexable(long length)
      throws ManifoldCFException, ServiceInterruption
    {
      return addActivities.checkLengthIndexable(length);
    }

    /** Pre-determine whether a document's URL is acceptable downstream.  This method is used
    * to help filter out documents that cannot be indexed in advance.
    *@param url is the URL of the document.
    *@return true if the file is acceptable by the downstream connection.
    */
    @Override
    public boolean checkURLIndexable(String url)
      throws ManifoldCFException, ServiceInterruption
    {
      return addActivities.checkURLIndexable(url);
    }

  }
  
  protected static class OutputActivitiesWrapper extends OutputAddActivitiesWrapper implements IOutputActivity
  {
    protected final IOutputActivity activities;
    
    public OutputActivitiesWrapper(IOutputActivity activities, String outputConnectionName)
    {
      super(activities,outputConnectionName);
      this.activities = activities;
    }
  }
  
  protected class PipelineObject
  {
    public final IPipelineSpecification pipelineConnections;
    public final IOutputConnector[] outputConnectors;
    public final ITransformationConnector[] transformationConnectors;
    
    public PipelineObject(
      IPipelineSpecification pipelineConnections,
      ITransformationConnector[] transformationConnectors,
      IOutputConnector[] outputConnectors)
    {
      this.pipelineConnections = pipelineConnections;
      this.transformationConnectors = transformationConnectors;
      this.outputConnectors = outputConnectors;
    }

    public boolean checkDateIndexable(Date date, IOutputCheckActivity finalActivity)
      throws ManifoldCFException, ServiceInterruption
    {
      PipelineCheckFanout entryPoint = buildCheckPipeline(finalActivity);
      return entryPoint.checkDateIndexable(date);
    }

    public boolean checkMimeTypeIndexable(String mimeType, IOutputCheckActivity finalActivity)
      throws ManifoldCFException, ServiceInterruption
    {
      PipelineCheckFanout entryPoint = buildCheckPipeline(finalActivity);
      return entryPoint.checkMimeTypeIndexable(mimeType);
    }

    public boolean checkDocumentIndexable(File localFile, IOutputCheckActivity finalActivity)
      throws ManifoldCFException, ServiceInterruption
    {
      PipelineCheckFanout entryPoint = buildCheckPipeline(finalActivity);
      return entryPoint.checkDocumentIndexable(localFile);
    }

    public boolean checkLengthIndexable(long length, IOutputCheckActivity finalActivity)
      throws ManifoldCFException, ServiceInterruption
    {
      PipelineCheckFanout entryPoint = buildCheckPipeline(finalActivity);
      return entryPoint.checkLengthIndexable(length);
    }
    
    public boolean checkURLIndexable(String uri, IOutputCheckActivity finalActivity)
      throws ManifoldCFException, ServiceInterruption
    {
      PipelineCheckFanout entryPoint = buildCheckPipeline(finalActivity);
      return entryPoint.checkURLIndexable(uri);
    }

    public void release()
      throws ManifoldCFException
    {
      outputConnectorPool.releaseMultiple(pipelineConnections.getOutputConnections(),outputConnectors);
      transformationConnectorPool.releaseMultiple(pipelineConnections.getTransformationConnections(),transformationConnectors);
    }
    
    protected PipelineCheckFanout buildCheckPipeline(IOutputCheckActivity finalActivity)
    {
      // Algorithm for building a pipeline:
      // (1) We start with the set of final output connection stages, and build an entry point for each one.  That's our "current set".
      // (2) We cycle through the "current set".  For each member, we attempt to go upstream a level.
      // (3) Before we can build the pipeline activity class for the next upstream stage, we need to have present ALL of the children that share that
      //   parent.  If we don't have that yet, we throw the stage back into the list.
      // (4) We continue until there is one stage left that has no parent, and that's what we return.
      
      // Create the current set
      Map<Integer,PipelineCheckEntryPoint> currentSet = new HashMap<Integer,PipelineCheckEntryPoint>();
      // First, locate all the output stages, and enter them into the set
      int count = pipelineConnections.getOutputCount();
      for (int i = 0; i < count; i++)
      {
        int outputStage = pipelineConnections.getOutputStage(i);
        PipelineCheckEntryPoint outputStageEntryPoint = new PipelineCheckEntryPoint(
          outputConnectors[pipelineConnections.getOutputConnectionIndex(outputStage).intValue()],
          pipelineConnections.getStageDescriptionString(outputStage),finalActivity);
        currentSet.put(new Integer(outputStage), outputStageEntryPoint);
      }
      // Cycle through the "current set"
      while (true)
      {
        int parent = -1;
        int[] siblings = null;
        for (Integer outputStage : currentSet.keySet())
        {
          parent = pipelineConnections.getStageParent(outputStage.intValue());
          // Look up the children
          siblings = pipelineConnections.getStageChildren(parent);
          // Are all the siblings in the current set yet?  If not, we can't proceed with this entry.
          boolean skipToNext = false;
          for (int sibling : siblings)
          {
            if (currentSet.get(new Integer(sibling)) == null)
            {
              skipToNext = true;
              break;
            }
          }
          if (skipToNext)
          {
            siblings = null;
            continue;
          }
          // All siblings are present!
          break;
        }
        
        // Siblings will be set if there's a stage we can do.  If not, we're done, but this should already have been detected.
        if (siblings == null)
          throw new IllegalStateException("Not at root but can't progress");
        
        PipelineCheckEntryPoint[] siblingEntryPoints = new PipelineCheckEntryPoint[siblings.length];
        for (int j = 0; j < siblings.length; j++)
        {
          siblingEntryPoints[j] = currentSet.remove(new Integer(siblings[j]));
        }
        // Wrap the entry points in a fan-out class, which has pipe connector-like methods that fire across all the connectors.
        PipelineCheckFanout pcf = new PipelineCheckFanout(siblingEntryPoints);
        if (parent == -1)
          return pcf;
        PipelineCheckEntryPoint newEntry = new PipelineCheckEntryPoint(
          transformationConnectors[pipelineConnections.getTransformationConnectionIndex(parent).intValue()],
          pipelineConnections.getStageDescriptionString(parent),pcf);
        currentSet.put(new Integer(parent), newEntry);
      }
    }
  }
  
  protected class PipelineObjectWithVersions extends PipelineObject
  {
    protected final IPipelineSpecificationWithVersions pipelineSpecificationWithVersions;
    
    public PipelineObjectWithVersions(
      IPipelineSpecificationWithVersions pipelineSpecificationWithVersions,
      ITransformationConnector[] transformationConnectors,
      IOutputConnector[] outputConnectors)
    {
      super(pipelineSpecificationWithVersions,transformationConnectors,outputConnectors);
      this.pipelineSpecificationWithVersions = pipelineSpecificationWithVersions;
    }

    public int addOrReplaceDocumentWithException(String docKey, String componentHash, String documentURI, RepositoryDocument document, String newDocumentVersion, String authorityNameString, IOutputActivity finalActivity, long ingestTime)
      throws ManifoldCFException, ServiceInterruption, IOException
    {
      PipelineAddFanout entryPoint = buildAddPipeline(finalActivity,newDocumentVersion,authorityNameString,ingestTime,docKey,componentHash);
      return entryPoint.sendDocument(documentURI,document);
    }

    public void noDocument(String docKey, String componentHash, String newDocumentVersion, String authorityNameString, IOutputActivity finalActivity, long ingestTime)
      throws ManifoldCFException, ServiceInterruption
    {
      PipelineAddFanout entryPoint = buildAddPipeline(finalActivity,newDocumentVersion,authorityNameString,ingestTime,docKey,componentHash);
      entryPoint.noDocument();
    }

    protected PipelineAddFanout buildAddPipeline(IOutputActivity finalActivity,
      String newDocumentVersion, String newAuthorityNameString,
      long ingestTime, String docKey, String componentHash)
    {
      // Algorithm for building a pipeline:
      // (1) We start with the set of final output connection stages, and build an entry point for each one.  That's our "current set".
      // (2) We cycle through the "current set".  For each member, we attempt to go upstream a level.
      // (3) Before we can build the pipeline activity class for the next upstream stage, we need to have present ALL of the children that share that
      //   parent.  If we don't have that yet, we throw the stage back into the list.
      // (4) We continue until there is one stage left that has no parent, and that's what we return.
      
      // Create the current set
      Map<Integer,PipelineAddEntryPoint> currentSet = new HashMap<Integer,PipelineAddEntryPoint>();
      // First, locate all the output stages, and enter them into the set
      IPipelineSpecificationWithVersions fullSpec = pipelineSpecificationWithVersions;
      
      int outputCount = fullSpec.getOutputCount();
      for (int i = 0; i < outputCount; i++)
      {
        int outputStage = fullSpec.getOutputStage(i);
        
        // Compute whether we need to reindex this record to this output or not, based on spec.
        String oldDocumentVersion = fullSpec.getOutputDocumentVersionString(i);
        String oldOutputVersion = fullSpec.getOutputVersionString(i);
        String oldTransformationVersion = fullSpec.getOutputTransformationVersionString(i);
        String oldAuthorityName = fullSpec.getAuthorityNameString(i);

        // Compute the transformation version string.  Must always be computed if we're going to reindex, since we save it.
        String newTransformationVersion = computePackedTransformationVersion(fullSpec,outputStage);
        
        boolean needToReindex = (oldDocumentVersion == null);
        if (needToReindex == false)
        {
          // We need a way to signal that a document has no valid version string.
          // That way is when the new document version string is empty.
          needToReindex = (newDocumentVersion.length() == 0 ||
            !oldDocumentVersion.equals(newDocumentVersion) ||
            !oldOutputVersion.equals(fullSpec.getStageDescriptionString(outputStage).getVersionString()) ||
            !oldAuthorityName.equals((newAuthorityNameString==null)?"":newAuthorityNameString));
        }
        if (needToReindex == false)
        {
          needToReindex = (!oldTransformationVersion.equals(newTransformationVersion));
        }

        int connectionIndex = fullSpec.getOutputConnectionIndex(outputStage).intValue();
        PipelineAddEntryPoint outputStageEntryPoint = new OutputAddEntryPoint(
          outputConnectors[connectionIndex],
          fullSpec.getStageDescriptionString(outputStage),
          new OutputActivitiesWrapper(finalActivity,fullSpec.getStageConnectionName(outputStage)),
          needToReindex,
          fullSpec.getStageConnectionName(outputStage),
          newTransformationVersion,
          ingestTime,
          newDocumentVersion,
          docKey,
          componentHash,
          newAuthorityNameString);
        currentSet.put(new Integer(outputStage), outputStageEntryPoint);
      }
      // Cycle through the "current set"
      while (true)
      {
        int parent = -1;
        int[] siblings = null;
        for (Integer outputStage : currentSet.keySet())
        {
          parent = fullSpec.getStageParent(outputStage.intValue());
          // Look up the children
          siblings = fullSpec.getStageChildren(parent);
          // Are all the siblings in the current set yet?  If not, we can't proceed with this entry.
          boolean skipToNext = false;
          for (int sibling : siblings)
          {
            if (currentSet.get(new Integer(sibling)) == null)
            {
              skipToNext = true;
              break;
            }
          }
          if (skipToNext)
          {
            siblings = null;
            continue;
          }
          // All siblings are present!
          break;
        }
        
        // Siblings will be set if there's a stage we can do.  If not, we're done, but this should already have been detected.
        if (siblings == null)
          throw new IllegalStateException("Not at root but can't progress");
        
        PipelineAddEntryPoint[] siblingEntryPoints = new PipelineAddEntryPoint[siblings.length];
        for (int j = 0; j < siblings.length; j++)
        {
          siblingEntryPoints[j] = currentSet.remove(new Integer(siblings[j]));
        }
        // Wrap the entry points in a fan-out class, which has pipe connector-like methods that fire across all the connectors.
        PipelineAddFanout pcf = new PipelineAddFanout(siblingEntryPoints,
          (parent==-1)?null:new TransformationRecordingActivity(finalActivity,
            fullSpec.getStageConnectionName(parent)),
          finalActivity);
        if (parent == -1)
          return pcf;
        PipelineAddEntryPoint newEntry = new PipelineAddEntryPoint(
          transformationConnectors[fullSpec.getTransformationConnectionIndex(parent).intValue()],
          fullSpec.getStageDescriptionString(parent),newAuthorityNameString,pcf,pcf.checkNeedToReindex());
        currentSet.put(new Integer(parent), newEntry);
      }

    }

  }

  /** This class describes the entry stage of multiple siblings in a check pipeline.
  */
  public static class PipelineCheckFanout implements IOutputCheckActivity
  {
    protected final PipelineCheckEntryPoint[] entryPoints;
    
    public PipelineCheckFanout(PipelineCheckEntryPoint[] entryPoints)
    {
      this.entryPoints = entryPoints;
    }
    
    @Override
    public boolean checkDateIndexable(Date date)
      throws ManifoldCFException, ServiceInterruption
    {
      // OR all results
      for (PipelineCheckEntryPoint p : entryPoints)
      {
        if (p.checkDateIndexable(date))
          return true;
      }
      return false;
    }

    @Override
    public boolean checkMimeTypeIndexable(String mimeType)
      throws ManifoldCFException, ServiceInterruption
    {
      // OR all results
      for (PipelineCheckEntryPoint p : entryPoints)
      {
        if (p.checkMimeTypeIndexable(mimeType))
          return true;
      }
      return false;
    }
    
    @Override
    public boolean checkDocumentIndexable(File localFile)
      throws ManifoldCFException, ServiceInterruption
    {
      // OR all results
      for (PipelineCheckEntryPoint p : entryPoints)
      {
        if (p.checkDocumentIndexable(localFile))
          return true;
      }
      return false;
    }

    @Override
    public boolean checkLengthIndexable(long length)
      throws ManifoldCFException, ServiceInterruption
    {
      // OR all results
      for (PipelineCheckEntryPoint p : entryPoints)
      {
        if (p.checkLengthIndexable(length))
          return true;
      }
      return false;
    }

    @Override
    public boolean checkURLIndexable(String uri)
      throws ManifoldCFException, ServiceInterruption
    {
      // OR all results
      for (PipelineCheckEntryPoint p : entryPoints)
      {
        if (p.checkURLIndexable(uri))
          return true;
      }
      return false;
    }
  }
  
  /** This class describes the entry stage of a check pipeline.
  */
  public static class PipelineCheckEntryPoint
  {
    protected final IPipelineConnector pipelineConnector;
    protected final VersionContext pipelineDescriptionString;
    protected final IOutputCheckActivity checkActivity;
    
    public PipelineCheckEntryPoint(
      IPipelineConnector pipelineConnector,
      VersionContext pipelineDescriptionString,
      IOutputCheckActivity checkActivity)
    {
      this.pipelineConnector= pipelineConnector;
      this.pipelineDescriptionString = pipelineDescriptionString;
      this.checkActivity = checkActivity;
    }
    
    public boolean checkDateIndexable(Date date)
      throws ManifoldCFException, ServiceInterruption
    {
      return pipelineConnector.checkDateIndexable(pipelineDescriptionString,date,checkActivity);
    }

    public boolean checkMimeTypeIndexable(String mimeType)
      throws ManifoldCFException, ServiceInterruption
    {
      return pipelineConnector.checkMimeTypeIndexable(pipelineDescriptionString,mimeType,checkActivity);
    }
    
    public boolean checkDocumentIndexable(File localFile)
      throws ManifoldCFException, ServiceInterruption
    {
      return pipelineConnector.checkDocumentIndexable(pipelineDescriptionString,localFile,checkActivity);
    }
    
    public boolean checkLengthIndexable(long length)
      throws ManifoldCFException, ServiceInterruption
    {
      return pipelineConnector.checkLengthIndexable(pipelineDescriptionString,length,checkActivity);
    }

    public boolean checkURLIndexable(String uri)
      throws ManifoldCFException, ServiceInterruption
    {
      return pipelineConnector.checkURLIndexable(pipelineDescriptionString,uri,checkActivity);
    }
    
  }
  
  /** This class describes the entry stage of multiple siblings in an add pipeline.
  */
  public static class PipelineAddFanout implements IOutputAddActivity
  {
    protected final PipelineAddEntryPoint[] entryPoints;
    protected final IOutputHistoryActivity finalHistoryActivity;
    protected final IOutputQualifyActivity finalQualifyActivity;

    public PipelineAddFanout(PipelineAddEntryPoint[] entryPoints, IOutputHistoryActivity finalHistoryActivity,
      IOutputQualifyActivity finalQualifyActivity)
    {
      this.entryPoints = entryPoints;
      this.finalHistoryActivity = finalHistoryActivity;
      this.finalQualifyActivity = finalQualifyActivity;
    }
    
    public boolean checkNeedToReindex()
    {
      // Look at the entry points, and make sure they're not all disabled.
      for (PipelineAddEntryPoint p : entryPoints)
      {
        if (p.isActive())
          return true;
      }
      return false;
    }
    
    @Override
    public boolean checkDateIndexable(Date date)
      throws ManifoldCFException, ServiceInterruption
    {
      // OR all results
      for (PipelineAddEntryPoint p : entryPoints)
      {
        if (p.checkDateIndexable(date))
          return true;
      }
      return false;
    }

    @Override
    public boolean checkMimeTypeIndexable(String mimeType)
      throws ManifoldCFException, ServiceInterruption
    {
      // OR all results
      for (PipelineAddEntryPoint p : entryPoints)
      {
        if (p.checkMimeTypeIndexable(mimeType))
          return true;
      }
      return false;
    }
    
    @Override
    public boolean checkDocumentIndexable(File localFile)
      throws ManifoldCFException, ServiceInterruption
    {
      // OR all results
      for (PipelineAddEntryPoint p : entryPoints)
      {
        if (p.checkDocumentIndexable(localFile))
          return true;
      }
      return false;
    }

    @Override
    public boolean checkLengthIndexable(long length)
      throws ManifoldCFException, ServiceInterruption
    {
      // OR all results
      for (PipelineAddEntryPoint p : entryPoints)
      {
        if (p.checkLengthIndexable(length))
          return true;
      }
      return false;
    }

    @Override
    public boolean checkURLIndexable(String uri)
      throws ManifoldCFException, ServiceInterruption
    {
      // OR all results
      for (PipelineAddEntryPoint p : entryPoints)
      {
        if (p.checkURLIndexable(uri))
          return true;
      }
      return false;
    }
    
    /** Send a document via the pipeline to the next output connection.
    *@param documentURI is the document's URI.
    *@param document is the document data to be processed (handed to the output data store).
    *@return the document status (accepted or permanently rejected); return codes are listed in IPipelineConnector.
    *@throws IOException only if there's an IO error reading the data from the document.
    */
    @Override
    public int sendDocument(String documentURI, RepositoryDocument document)
      throws ManifoldCFException, ServiceInterruption, IOException
    {
      // First, count the number of active entry points.
      int activeCount = 0;
      for (PipelineAddEntryPoint p : entryPoints)
      {
        if (p.isActive())
          activeCount++;
      }
      if (activeCount <= 1)
      {
        // No need to copy anything.
        int rval = IPipelineConnector.DOCUMENTSTATUS_REJECTED;
        for (PipelineAddEntryPoint p : entryPoints)
        {
          if (!p.isActive())
            continue;
          if (p.addOrReplaceDocumentWithException(documentURI,document) == IPipelineConnector.DOCUMENTSTATUS_ACCEPTED)
            rval = IPipelineConnector.DOCUMENTSTATUS_ACCEPTED;
        }
        return rval;
      }
      else
      {
        // Create a RepositoryDocumentFactory, which we'll need to clean up at the end.
        RepositoryDocumentFactory factory = new RepositoryDocumentFactory(document);
        try
        {
          // If any of them accept the document, we return "accept".
          int rval = IPipelineConnector.DOCUMENTSTATUS_REJECTED;
          for (PipelineAddEntryPoint p : entryPoints)
          {
            if (!p.isActive())
              continue;
            if (p.addOrReplaceDocumentWithException(documentURI,factory.createDocument()) == IPipelineConnector.DOCUMENTSTATUS_ACCEPTED)
              rval = IPipelineConnector.DOCUMENTSTATUS_ACCEPTED;
          }
          return rval;
        }
        finally
        {
          factory.close();
        }
      }
    }

    /** Send NO document via the pipeline to the next output connection.  This is equivalent
    * to sending an empty document placeholder.
    */
    @Override
    public void noDocument()
      throws ManifoldCFException, ServiceInterruption
    {
      for (PipelineAddEntryPoint p : entryPoints)
      {
        if (p.isActive())
        {
          // Invoke the addEntryPoint method for handling "noDocument"
          p.noDocument();
        }
      }
    }

    /** Qualify an access token appropriately, to match access tokens as returned by mod_aa.  This method
    * includes the authority name with the access token, if any, so that each authority may establish its own token space.
    *@param authorityNameString is the name of the authority to use to qualify the access token.
    *@param accessToken is the raw, repository access token.
    *@return the properly qualified access token.
    */
    @Override
    public String qualifyAccessToken(String authorityNameString, String accessToken)
      throws ManifoldCFException
    {
      // This functionality does not need to be staged; we just want to vector through to the final stage directly.
      return finalQualifyActivity.qualifyAccessToken(authorityNameString,accessToken);
    }

    /** Record time-stamped information about the activity of the output connector.
    *@param startTime is either null or the time since the start of epoch in milliseconds (Jan 1, 1970).  Every
    *       activity has an associated time; the startTime field records when the activity began.  A null value
    *       indicates that the start time and the finishing time are the same.
    *@param activityType is a string which is fully interpretable only in the context of the connector involved, which is
    *       used to categorize what kind of activity is being recorded.  For example, a web connector might record a
    *       "fetch document" activity.  Cannot be null.
    *@param dataSize is the number of bytes of data involved in the activity, or null if not applicable.
    *@param entityURI is a (possibly long) string which identifies the object involved in the history record.
    *       The interpretation of this field will differ from connector to connector.  May be null.
    *@param resultCode contains a terse description of the result of the activity.  The description is limited in
    *       size to 255 characters, and can be interpreted only in the context of the current connector.  May be null.
    *@param resultDescription is a (possibly long) human-readable string which adds detail, if required, to the result
    *       described in the resultCode field.  This field is not meant to be queried on.  May be null.
    */
    @Override
    public void recordActivity(Long startTime, String activityType, Long dataSize,
      String entityURI, String resultCode, String resultDescription)
      throws ManifoldCFException
    {
      // Each stage of the pipeline uses a specific activity for recording history, but it's not fundamentally
      // pipelined
      finalHistoryActivity.recordActivity(startTime,activityType,dataSize,entityURI,resultCode,resultDescription);
    }
  }

  /** This class describes the entry stage of an add pipeline.
  */
  public static class PipelineAddEntryPoint
  {
    protected final IPipelineConnector pipelineConnector;
    protected final VersionContext pipelineDescriptionString;
    protected final String authorityNameString;
    protected final IOutputAddActivity addActivity;
    protected final boolean isActive;
    
    public PipelineAddEntryPoint(IPipelineConnector pipelineConnector,
      VersionContext pipelineDescriptionString,
      String authorityNameString,
      IOutputAddActivity addActivity,
      boolean isActive)
    {
      this.pipelineConnector = pipelineConnector;
      this.pipelineDescriptionString = pipelineDescriptionString;
      this.authorityNameString = authorityNameString;
      this.addActivity = addActivity;
      this.isActive = isActive;
    }
    
    public boolean isActive()
    {
      return isActive;
    }

    public boolean checkDateIndexable(Date date)
      throws ManifoldCFException, ServiceInterruption
    {
      return pipelineConnector.checkDateIndexable(pipelineDescriptionString,date,addActivity);
    }
    
    public boolean checkMimeTypeIndexable(String mimeType)
      throws ManifoldCFException, ServiceInterruption
    {
      return pipelineConnector.checkMimeTypeIndexable(pipelineDescriptionString,mimeType,addActivity);
    }
    
    public boolean checkDocumentIndexable(File localFile)
      throws ManifoldCFException, ServiceInterruption
    {
      return pipelineConnector.checkDocumentIndexable(pipelineDescriptionString,localFile,addActivity);
    }
    
    public boolean checkLengthIndexable(long length)
      throws ManifoldCFException, ServiceInterruption
    {
      return pipelineConnector.checkLengthIndexable(pipelineDescriptionString,length,addActivity);
    }

    public boolean checkURLIndexable(String uri)
      throws ManifoldCFException, ServiceInterruption
    {
      return pipelineConnector.checkURLIndexable(pipelineDescriptionString,uri,addActivity);
    }

    public int addOrReplaceDocumentWithException(String documentURI, RepositoryDocument document)
      throws ManifoldCFException, ServiceInterruption, IOException
    {
      // If the transformation connector doesn't do what it should, compensate!
      MonitoredAddActivityWrapper wrapper = new MonitoredAddActivityWrapper(addActivity);
      int rval = pipelineConnector.addOrReplaceDocumentWithException(
        documentURI,pipelineDescriptionString,
        document,authorityNameString,wrapper);
      // The wrapper detects activity by the connector, so if we don't see either sendDocument() or
      // noDocument(), we issue noDocument() ourselves.  If the connector was an output connector,
      // this will wind up being a no-op, but otherwise it will guarantee that recording takes place.
      if (!wrapper.wasDocumentActedUpon())
        addActivity.noDocument();
      return rval;
    }
    
    public void noDocument()
      throws ManifoldCFException, ServiceInterruption
    {
      // Call the addActivity method for handling no document
      addActivity.noDocument();
    }
  }
  
  public class OutputAddEntryPoint extends PipelineAddEntryPoint
  {
    protected final IOutputConnector outputConnector;
    protected final String outputConnectionName;
    protected final String transformationVersion;
    protected final long ingestTime;
    protected final String documentVersion;
    protected final String docKey;
    protected final String componentHash;
    protected final IOutputActivity activity;
    
    public OutputAddEntryPoint(IOutputConnector outputConnector,
      VersionContext outputDescriptionString,
      IOutputActivity activity,
      boolean isActive,
      String outputConnectionName,
      String transformationVersion,
      long ingestTime,
      String documentVersion,
      String docKey,
      String componentHash,
      String authorityNameString)
    {
      super(outputConnector,outputDescriptionString,authorityNameString,activity,isActive);
      this.outputConnector = outputConnector;
      this.outputConnectionName = outputConnectionName;
      this.transformationVersion = transformationVersion;
      this.ingestTime = ingestTime;
      this.documentVersion = documentVersion;
      this.docKey = docKey;
      this.componentHash = componentHash;
      this.activity = activity;
    }
    
    @Override
    public void noDocument()
      throws ManifoldCFException, ServiceInterruption
    {
      try
      {
        addOrReplaceDocumentWithException(null,null);
      }
      catch (IOException e)
      {
        throw new RuntimeException("Unexpected IOException: "+e.getMessage(),e);
      }
    }
    
    @Override
    public int addOrReplaceDocumentWithException(String documentURI, RepositoryDocument document)
      throws ManifoldCFException, ServiceInterruption, IOException
    {
      // No transactions; not safe because post may take too much time

      // First, calculate a document uri hash value
      String documentURIHash = null;
      if (documentURI != null)
        documentURIHash = ManifoldCF.hash(documentURI);

      String oldURI = null;
      String oldURIHash = null;
      String oldOutputVersion = null;

      // Repeat if needed
      while (true)
      {
        long sleepAmt = 0L;
        try
        {
          // See what uri was used before for this doc, if any
          ArrayList list = new ArrayList();
          String query = buildConjunctionClause(list,new ClauseDescription[]{
            new UnitaryClause(docKeyField,docKey),
            new UnitaryClause(outputConnNameField,outputConnectionName),
            (componentHash == null || componentHash.length() == 0)?new NullCheckClause(componentHashField,true):new UnitaryClause(componentHashField,componentHash)});
            
          IResultSet set = performQuery("SELECT "+docURIField+","+uriHashField+","+lastOutputVersionField+" FROM "+getTableName()+
            " WHERE "+query,list,null,null);

          if (set.getRowCount() > 0)
          {
            IResultRow row = set.getRow(0);
            oldURI = (String)row.getValue(docURIField);
            oldURIHash = (String)row.getValue(uriHashField);
            oldOutputVersion = (String)row.getValue(lastOutputVersionField);
          }
          
          break;
        }
        catch (ManifoldCFException e)
        {
          // Look for deadlock and retry if so
          if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
          {
            if (Logging.perf.isDebugEnabled())
              Logging.perf.debug("Aborted select looking for status: "+e.getMessage());
            sleepAmt = getSleepAmt();
            continue;
          }
          throw e;
        }
        finally
        {
          sleepFor(sleepAmt);
        }
      }

      // If uri hashes collide, then we must be sure to eliminate only the *correct* records from the table, or we will leave
      // dangling documents around.  So, all uri searches and comparisons MUST compare the actual uri as well.

      // But, since we need to insure that any given URI is only worked on by one thread at a time, use critical sections
      // to block the rare case that multiple threads try to work on the same URI.
      
      String[] lockArray = computeLockArray(documentURIHash,oldURIHash,outputConnectionName);
      lockManager.enterLocks(null,null,lockArray);
      try
      {

        ArrayList list = new ArrayList();
        
        if (oldURI != null && (documentURI == null || !oldURI.equals(documentURI)))
        {
          // Delete all records from the database that match the old URI, except for THIS record.
          list.clear();
          String query = buildConjunctionClause(list,new ClauseDescription[]{
            new UnitaryClause(uriHashField,"=",oldURIHash),
            new UnitaryClause(outputConnNameField,outputConnectionName)});
          list.add(docKey);
          performDelete("WHERE "+query+" AND "+docKeyField+"!=?",list,null);
          outputConnector.removeDocument(oldURI,oldOutputVersion,activity);
        }

        if (documentURI != null)
        {
          // Get rid of all records that match the NEW uri, except for this record.
          list.clear();
          String query = buildConjunctionClause(list,new ClauseDescription[]{
            new UnitaryClause(uriHashField,"=",documentURIHash),
            new UnitaryClause(outputConnNameField,outputConnectionName)});
          list.add(docKey);
          performDelete("WHERE "+query+" AND "+ docKeyField+"!=?",list,null);

          // Now, we know we are ready for the ingest.
        
          // Here are the cases:
          // 1) There was a service interruption before the upload started.
          // (In that case, we don't need to log anything, just reschedule).
          // 2) There was a service interruption after the document was transmitted.
          // (In that case, we should presume that the document was ingested, but
          //  reschedule another import anyway.)
          // 3) Everything went OK
          // (need to log the ingestion.)
          // 4) Everything went OK, but we were told we have an illegal document.
          // (We note the ingestion because if we don't we will be forced to repeat ourselves.
          //  In theory, document doesn't need to be deleted, but there is no way to signal
          //  that at the moment.)

          // Note an ingestion before we actually try it.
          // This is a marker that says "something is there"; it has an empty version, which indicates
          // that we don't know anything about it.  That means it will be reingested when the
          // next version comes along, and will be deleted if called for also.
          noteDocumentIngest(outputConnectionName,docKey,componentHash,null,null,null,null,ingestTime,documentURI,documentURIHash);
          int result = super.addOrReplaceDocumentWithException(documentURI, document);
          noteDocumentIngest(outputConnectionName,docKey,componentHash,documentVersion,transformationVersion,pipelineDescriptionString.getVersionString(),authorityNameString,ingestTime,documentURI,documentURIHash);
          return result;
        }

        // If we get here, it means we are noting that the document was examined, but that no change was required.  This is signaled
        // to noteDocumentIngest by having the null documentURI.
        noteDocumentIngest(outputConnectionName,docKey,componentHash,documentVersion,transformationVersion,pipelineDescriptionString.getVersionString(),authorityNameString,ingestTime,null,null);
        return IPipelineConnector.DOCUMENTSTATUS_ACCEPTED;
      }
      finally
      {
        lockManager.leaveLocks(null,null,lockArray);
      }
    }
  }

  protected static String[] computeLockArray(String documentURIHash, String oldURIHash, String outputConnectionName)
  {
    int uriCount = 0;
    if (documentURIHash != null)
      uriCount++;
    if (oldURIHash != null && (documentURIHash == null || !documentURIHash.equals(oldURIHash)))
      uriCount++;
    String[] lockArray = new String[uriCount];
    uriCount = 0;
    if (documentURIHash != null)
      lockArray[uriCount++] = createURILockName(outputConnectionName,documentURIHash);
    if (oldURIHash != null && (documentURIHash == null || !documentURIHash.equals(oldURIHash)))
      lockArray[uriCount++] = createURILockName(outputConnectionName,oldURIHash);
    return lockArray;
  }
  
  /** This class passes everything through, and monitors what happens so that the
  * framework can compensate for any transformation connector coding errors.
  */
  protected static class MonitoredAddActivityWrapper implements IOutputAddActivity
  {
    protected final IOutputAddActivity activities;
    
    protected boolean documentProcessed = false;
    
    public MonitoredAddActivityWrapper(IOutputAddActivity activities)
    {
      this.activities = activities;
    }
    
    public boolean wasDocumentActedUpon()
    {
      return documentProcessed;
    }

    /** Send a document via the pipeline to the next output connection.
    *@param documentURI is the document's URI.
    *@param document is the document data to be processed (handed to the output data store).
    *@return the document status (accepted or permanently rejected); return codes are listed in IPipelineConnector.
    *@throws IOException only if there's an IO error reading the data from the document.
    */
    @Override
    public int sendDocument(String documentURI, RepositoryDocument document)
      throws ManifoldCFException, ServiceInterruption, IOException
    {
      if (documentProcessed)
        throw new IllegalStateException("Document cannot have multiple dispositions");
      int rval = activities.sendDocument(documentURI,document);
      documentProcessed = true;
      return rval;
    }

    /** Send NO document via the pipeline to the next output connection.  This is equivalent
    * to sending an empty document placeholder.
    */
    @Override
    public void noDocument()
      throws ManifoldCFException, ServiceInterruption
    {
      if (documentProcessed)
        throw new IllegalStateException("Document cannot have multiple dispositions");
      activities.noDocument();
      documentProcessed = true;
    }

    /** Qualify an access token appropriately, to match access tokens as returned by mod_aa.  This method
    * includes the authority name with the access token, if any, so that each authority may establish its own token space.
    *@param authorityNameString is the name of the authority to use to qualify the access token.
    *@param accessToken is the raw, repository access token.
    *@return the properly qualified access token.
    */
    @Override
    public String qualifyAccessToken(String authorityNameString, String accessToken)
      throws ManifoldCFException
    {
      return activities.qualifyAccessToken(authorityNameString,accessToken);
    }

    /** Record time-stamped information about the activity of the output connector.
    *@param startTime is either null or the time since the start of epoch in milliseconds (Jan 1, 1970).  Every
    *       activity has an associated time; the startTime field records when the activity began.  A null value
    *       indicates that the start time and the finishing time are the same.
    *@param activityType is a string which is fully interpretable only in the context of the connector involved, which is
    *       used to categorize what kind of activity is being recorded.  For example, a web connector might record a
    *       "fetch document" activity.  Cannot be null.
    *@param dataSize is the number of bytes of data involved in the activity, or null if not applicable.
    *@param entityURI is a (possibly long) string which identifies the object involved in the history record.
    *       The interpretation of this field will differ from connector to connector.  May be null.
    *@param resultCode contains a terse description of the result of the activity.  The description is limited in
    *       size to 255 characters, and can be interpreted only in the context of the current connector.  May be null.
    *@param resultDescription is a (possibly long) human-readable string which adds detail, if required, to the result
    *       described in the resultCode field.  This field is not meant to be queried on.  May be null.
    */
    @Override
    public void recordActivity(Long startTime, String activityType, Long dataSize,
      String entityURI, String resultCode, String resultDescription)
      throws ManifoldCFException
    {
      activities.recordActivity(startTime,activityType,dataSize,entityURI,resultCode,resultDescription);
    }

    /** Detect if a date is acceptable downstream or not.  This method is used to determine whether it makes sense to fetch a document
    * in the first place.
    *@param date is the date of the document.
    *@return true if the document described by the date can be accepted by the downstream connection.
    */
    @Override
    public boolean checkDateIndexable(Date date)
      throws ManifoldCFException, ServiceInterruption
    {
      return activities.checkDateIndexable(date);
    }

    /** Detect if a mime type is acceptable downstream or not.  This method is used to determine whether it makes sense to fetch a document
    * in the first place.
    *@param mimeType is the mime type of the document.
    *@return true if the mime type can be accepted by the downstream connection.
    */
    @Override
    public boolean checkMimeTypeIndexable(String mimeType)
      throws ManifoldCFException, ServiceInterruption
    {
      return activities.checkMimeTypeIndexable(mimeType);
    }

    /** Pre-determine whether a document (passed here as a File object) is acceptable downstream.  This method is
    * used to determine whether a document needs to be actually transferred.  This hook is provided mainly to support
    * search engines that only handle a small set of accepted file types.
    *@param localFile is the local file to check.
    *@return true if the file is acceptable by the downstream connection.
    */
    @Override
    public boolean checkDocumentIndexable(File localFile)
      throws ManifoldCFException, ServiceInterruption
    {
      return activities.checkDocumentIndexable(localFile);
    }

    /** Pre-determine whether a document's length is acceptable downstream.  This method is used
    * to determine whether to fetch a document in the first place.
    *@param length is the length of the document.
    *@return true if the file is acceptable by the downstream connection.
    */
    @Override
    public boolean checkLengthIndexable(long length)
      throws ManifoldCFException, ServiceInterruption
    {
      return activities.checkLengthIndexable(length);
    }

    /** Pre-determine whether a document's URL is acceptable downstream.  This method is used
    * to help filter out documents that cannot be indexed in advance.
    *@param url is the URL of the document.
    *@return true if the file is acceptable by the downstream connection.
    */
    @Override
    public boolean checkURLIndexable(String url)
      throws ManifoldCFException, ServiceInterruption
    {
      return activities.checkURLIndexable(url);
    }

  }
  
}
