I have a large v4.11.10 instance hosting 7 different language sites. We have some issues which I think are related at a corrupt or incomplete search index and so trawling the logs table I keep finging this exception reported which appears to stop the index process dead in it's tracks:

[UmbracoExamine] (InternalIndexer)Error indexing queue items,An item with the same key has already been added., IndexSet: InternalIndexSet

The config for this index is the out of the box config as follows:

<IndexSet SetName="InternalIndexSet" IndexPath="~/App_Data/TEMP/ExamineIndexes/Internal/"/>

We have a couple of hooks into Examine/Lucene events which are as follows:

private void LuceneDocumentWriting(object sender, Examine.LuceneEngine.DocumentWritingEventArgs e)
{
  try
  {
    // Grab the Lucene document from the event arguments
    var doc = e.Document;

    // Iterate over each of the fields in the collection and add a copy of the field that is not analyzed 
    // and therefore remains in it's original state
    foreach (var f in e.Fields.Where(f => !f.Key.StartsWith("__")))
    {
      var key = string.Concat("__", f.Key);

      if (!e.Fields.ContainsKey(key))
      {
        doc.Add(new Field(key, f.Value, Field.Store.YES, Field.Index.NOT_ANALYZED));
      }
    }
  }
  catch (Exception ex)
  {
    Log.Add(LogTypes.Error, -1, string.Concat("Failed to write new field(s) to Lucene index: ", ex));
  }
}

private void SearchIndexerGatheringNodeData(object sender, IndexingNodeDataEventArgs e)
{
  if (e.IndexType == IndexTypes.Content)
  {
    try
    {
      if (e.Fields.ContainsKey("path"))
      {
        // Split the path to the current node and extract the homepage id
        string rootNodeId = e.Fields["path"].Split(',')[2];
        e.Fields.Add("siteId", rootNodeId);

        // Split the path into a new searchable field
        e.Fields.Add("searchablePath", e.Fields["path"].Replace(",", " "));
      }

      if (e.Fields.ContainsKey("tags"))
      {
        // Remove the comma delimeters on the tags field so that they 
        // can all be indexed individually
        if (!string.IsNullOrEmpty(e.Fields["tags"]))
        {
          e.Fields["tags"] = e.Fields["tags"].Replace(" ", "_").Replace(",", " ");
        }
      }
    }
    catch (Exception ex)
    {
      Log.Add(LogTypes.Error, -1, string.Concat("Failed to update search index node data: ", ex));
    }
  }
}

Unfortunately this issue impacted on my time at CodeGarden this year and continues to do so - so any thoughts would be appreciated.

Cheers, Simon