875 lines
26 KiB
C#
875 lines
26 KiB
C#
/*
|
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
* contributor license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright ownership.
|
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
* (the "License"); you may not use this file except in compliance with
|
|
* the License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using NUnit.Framework;
|
|
|
|
using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
|
|
using Document = Lucene.Net.Documents.Document;
|
|
using Field = Lucene.Net.Documents.Field;
|
|
using Directory = Lucene.Net.Store.Directory;
|
|
using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
|
|
using RAMDirectory = Lucene.Net.Store.RAMDirectory;
|
|
using IndexSearcher = Lucene.Net.Search.IndexSearcher;
|
|
using Query = Lucene.Net.Search.Query;
|
|
using ScoreDoc = Lucene.Net.Search.ScoreDoc;
|
|
using TermQuery = Lucene.Net.Search.TermQuery;
|
|
using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
|
|
|
|
namespace Lucene.Net.Index
|
|
{
|
|
|
|
/*
|
|
Verify we can read the pre-2.1 file format, do searches
|
|
against it, and add documents to it.*/
|
|
|
|
[TestFixture]
|
|
public class TestDeletionPolicy : LuceneTestCase
|
|
{
|
|
private void VerifyCommitOrder<T>(IList<T> commits) where T : IndexCommit
|
|
{
|
|
IndexCommit firstCommit = commits[0];
|
|
long last = SegmentInfos.GenerationFromSegmentsFileName(firstCommit.SegmentsFileName);
|
|
Assert.AreEqual(last, firstCommit.Generation);
|
|
long lastVersion = firstCommit.Version;
|
|
long lastTimestamp = firstCommit.Timestamp;
|
|
for (int i = 1; i < commits.Count; i++)
|
|
{
|
|
IndexCommit commit = commits[i];
|
|
long now = SegmentInfos.GenerationFromSegmentsFileName(commit.SegmentsFileName);
|
|
long nowVersion = commit.Version;
|
|
long nowTimestamp = commit.Timestamp;
|
|
Assert.IsTrue(now > last, "SegmentInfos commits are out-of-order");
|
|
Assert.IsTrue(nowVersion > lastVersion, "SegmentInfos versions are out-of-order");
|
|
Assert.IsTrue(nowTimestamp >= lastTimestamp, "SegmentInfos timestamps are out-of-order: now=" + nowTimestamp + " vs last=" + lastTimestamp);
|
|
Assert.AreEqual(now, commit.Generation);
|
|
last = now;
|
|
lastVersion = nowVersion;
|
|
lastTimestamp = nowTimestamp;
|
|
}
|
|
}
|
|
|
|
internal class KeepAllDeletionPolicy : IndexDeletionPolicy
|
|
{
|
|
public KeepAllDeletionPolicy(TestDeletionPolicy enclosingInstance)
|
|
{
|
|
InitBlock(enclosingInstance);
|
|
}
|
|
private void InitBlock(TestDeletionPolicy enclosingInstance)
|
|
{
|
|
this.enclosingInstance = enclosingInstance;
|
|
}
|
|
private TestDeletionPolicy enclosingInstance;
|
|
public TestDeletionPolicy Enclosing_Instance
|
|
{
|
|
get
|
|
{
|
|
return enclosingInstance;
|
|
}
|
|
|
|
}
|
|
internal int numOnInit;
|
|
internal int numOnCommit;
|
|
internal Directory dir;
|
|
public virtual void OnInit<T>(IList<T> commits) where T : IndexCommit
|
|
{
|
|
Enclosing_Instance.VerifyCommitOrder(commits);
|
|
numOnInit++;
|
|
}
|
|
public virtual void OnCommit<T>(IList<T> commits) where T : IndexCommit
|
|
{
|
|
IndexCommit lastCommit = (IndexCommit) commits[commits.Count - 1];
|
|
IndexReader r = IndexReader.Open(dir, true);
|
|
Assert.AreEqual(r.IsOptimized(), lastCommit.IsOptimized, "lastCommit.isOptimized()=" + lastCommit.IsOptimized + " vs IndexReader.isOptimized=" + r.IsOptimized());
|
|
r.Close();
|
|
Enclosing_Instance.VerifyCommitOrder(commits);
|
|
numOnCommit++;
|
|
}
|
|
}
|
|
|
|
/// <summary> This is useful for adding to a big index when you know
|
|
/// readers are not using it.
|
|
/// </summary>
|
|
internal class KeepNoneOnInitDeletionPolicy : IndexDeletionPolicy
|
|
{
|
|
public KeepNoneOnInitDeletionPolicy(TestDeletionPolicy enclosingInstance)
|
|
{
|
|
InitBlock(enclosingInstance);
|
|
}
|
|
private void InitBlock(TestDeletionPolicy enclosingInstance)
|
|
{
|
|
this.enclosingInstance = enclosingInstance;
|
|
}
|
|
private TestDeletionPolicy enclosingInstance;
|
|
public TestDeletionPolicy Enclosing_Instance
|
|
{
|
|
get
|
|
{
|
|
return enclosingInstance;
|
|
}
|
|
|
|
}
|
|
internal int numOnInit;
|
|
internal int numOnCommit;
|
|
public virtual void OnInit<T>(IList<T> commits) where T : IndexCommit
|
|
{
|
|
Enclosing_Instance.VerifyCommitOrder(commits);
|
|
numOnInit++;
|
|
// On init, delete all commit points:
|
|
System.Collections.IEnumerator it = commits.GetEnumerator();
|
|
while (it.MoveNext())
|
|
{
|
|
IndexCommit commit = (IndexCommit) it.Current;
|
|
commit.Delete();
|
|
Assert.IsTrue(commit.IsDeleted);
|
|
}
|
|
}
|
|
public virtual void OnCommit<T>(IList<T> commits) where T : IndexCommit
|
|
{
|
|
Enclosing_Instance.VerifyCommitOrder(commits);
|
|
int size = commits.Count;
|
|
// Delete all but last one:
|
|
for (int i = 0; i < size - 1; i++)
|
|
{
|
|
((IndexCommit) commits[i]).Delete();
|
|
}
|
|
numOnCommit++;
|
|
}
|
|
}
|
|
|
|
internal class KeepLastNDeletionPolicy : IndexDeletionPolicy
|
|
{
|
|
private void InitBlock(TestDeletionPolicy enclosingInstance)
|
|
{
|
|
this.enclosingInstance = enclosingInstance;
|
|
}
|
|
private TestDeletionPolicy enclosingInstance;
|
|
public TestDeletionPolicy Enclosing_Instance
|
|
{
|
|
get
|
|
{
|
|
return enclosingInstance;
|
|
}
|
|
|
|
}
|
|
internal int numOnInit;
|
|
internal int numOnCommit;
|
|
internal int numToKeep;
|
|
internal int numDelete;
|
|
internal System.Collections.Hashtable seen = new System.Collections.Hashtable();
|
|
|
|
public KeepLastNDeletionPolicy(TestDeletionPolicy enclosingInstance, int numToKeep)
|
|
{
|
|
InitBlock(enclosingInstance);
|
|
this.numToKeep = numToKeep;
|
|
}
|
|
|
|
public virtual void OnInit<T>(IList<T> commits) where T : IndexCommit
|
|
{
|
|
Enclosing_Instance.VerifyCommitOrder(commits);
|
|
numOnInit++;
|
|
// do no deletions on init
|
|
DoDeletes(commits, false);
|
|
}
|
|
|
|
public virtual void OnCommit<T>(IList<T> commits) where T : IndexCommit
|
|
{
|
|
Enclosing_Instance.VerifyCommitOrder(commits);
|
|
DoDeletes(commits, true);
|
|
}
|
|
|
|
private void DoDeletes<T>(IList<T> commits, bool isCommit) where T : IndexCommit
|
|
{
|
|
|
|
// Assert that we really are only called for each new
|
|
// commit:
|
|
if (isCommit)
|
|
{
|
|
System.String fileName = commits[commits.Count - 1].SegmentsFileName;
|
|
if (seen.Contains(fileName))
|
|
{
|
|
throw new System.SystemException("onCommit was called twice on the same commit point: " + fileName);
|
|
}
|
|
seen.Add(fileName, fileName);
|
|
numOnCommit++;
|
|
}
|
|
int size = commits.Count;
|
|
for (int i = 0; i < size - numToKeep; i++)
|
|
{
|
|
((IndexCommit) commits[i]).Delete();
|
|
numDelete++;
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Delete a commit only when it has been obsoleted by N
|
|
* seconds.
|
|
*/
|
|
internal class ExpirationTimeDeletionPolicy : IndexDeletionPolicy
|
|
{
|
|
private void InitBlock(TestDeletionPolicy enclosingInstance)
|
|
{
|
|
this.enclosingInstance = enclosingInstance;
|
|
}
|
|
private TestDeletionPolicy enclosingInstance;
|
|
public TestDeletionPolicy Enclosing_Instance
|
|
{
|
|
get
|
|
{
|
|
return enclosingInstance;
|
|
}
|
|
|
|
}
|
|
|
|
internal Directory dir;
|
|
internal double expirationTimeSeconds;
|
|
internal int numDelete;
|
|
|
|
public ExpirationTimeDeletionPolicy(TestDeletionPolicy enclosingInstance, Directory dir, double seconds)
|
|
{
|
|
InitBlock(enclosingInstance);
|
|
this.dir = dir;
|
|
this.expirationTimeSeconds = seconds;
|
|
}
|
|
|
|
public virtual void OnInit<T>(IList<T> commits) where T : IndexCommit
|
|
{
|
|
Enclosing_Instance.VerifyCommitOrder(commits);
|
|
OnCommit(commits);
|
|
}
|
|
|
|
public virtual void OnCommit<T>(IList<T> commits) where T : IndexCommit
|
|
{
|
|
Enclosing_Instance.VerifyCommitOrder(commits);
|
|
|
|
IndexCommit lastCommit = commits[commits.Count - 1];
|
|
|
|
// Any commit older than expireTime should be deleted:
|
|
double expireTime = dir.FileModified(lastCommit.SegmentsFileName) / 1000.0 - expirationTimeSeconds;
|
|
|
|
System.Collections.IEnumerator it = commits.GetEnumerator();
|
|
|
|
while (it.MoveNext())
|
|
{
|
|
IndexCommit commit = (IndexCommit) it.Current;
|
|
double modTime = dir.FileModified(commit.SegmentsFileName) / 1000.0;
|
|
if (commit != lastCommit && modTime < expireTime)
|
|
{
|
|
commit.Delete();
|
|
numDelete += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Test "by time expiration" deletion policy:
|
|
*/
|
|
[Test]
|
|
public virtual void TestExpirationTimeDeletionPolicy()
|
|
{
|
|
double SECONDS = 2.0;
|
|
|
|
bool useCompoundFile = true;
|
|
|
|
Directory dir = new RAMDirectory();
|
|
ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(this, dir, SECONDS);
|
|
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
writer.Close();
|
|
|
|
long lastDeleteTime = 0;
|
|
for (int i = 0; i < 7; i++)
|
|
{
|
|
// Record last time when writer performed deletes of
|
|
// past commits
|
|
lastDeleteTime = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond);
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
for (int j = 0; j < 17; j++)
|
|
{
|
|
AddDoc(writer);
|
|
}
|
|
writer.Close();
|
|
|
|
// Make sure to sleep long enough so that some commit
|
|
// points will be deleted:
|
|
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * (int) (1000.0 * (SECONDS / 5.0))));
|
|
}
|
|
|
|
// First, make sure the policy in fact deleted something:
|
|
Assert.IsTrue(policy.numDelete > 0, "no commits were deleted");
|
|
|
|
// Then simplistic check: just verify that the
|
|
// segments_N's that still exist are in fact within SECONDS
|
|
// seconds of the last one's mod time, and, that I can
|
|
// open a reader on each:
|
|
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
|
|
|
|
System.String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
|
|
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
|
|
while (gen > 0)
|
|
{
|
|
try
|
|
{
|
|
IndexReader reader = IndexReader.Open(dir, true);
|
|
reader.Close();
|
|
fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
|
|
long modTime = dir.FileModified(fileName);
|
|
Assert.IsTrue(lastDeleteTime - modTime <= (SECONDS * 1000), "commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted");
|
|
}
|
|
catch (System.IO.IOException)
|
|
{
|
|
// OK
|
|
break;
|
|
}
|
|
|
|
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
|
|
gen--;
|
|
}
|
|
|
|
dir.Close();
|
|
}
|
|
|
|
/*
|
|
* Test a silly deletion policy that keeps all commits around.
|
|
*/
|
|
[Test]
|
|
public virtual void TestKeepAllDeletionPolicy()
|
|
{
|
|
|
|
for (int pass = 0; pass < 2; pass++)
|
|
{
|
|
bool useCompoundFile = (pass % 2) != 0;
|
|
|
|
// Never deletes a commit
|
|
KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(this);
|
|
|
|
Directory dir = new RAMDirectory();
|
|
policy.dir = dir;
|
|
|
|
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.SetMaxBufferedDocs(10);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
writer.SetMergeScheduler(new SerialMergeScheduler());
|
|
for (int i = 0; i < 107; i++)
|
|
{
|
|
AddDoc(writer);
|
|
}
|
|
writer.Close();
|
|
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
writer.Optimize();
|
|
writer.Close();
|
|
|
|
Assert.AreEqual(2, policy.numOnInit);
|
|
|
|
// If we are not auto committing then there should
|
|
// be exactly 2 commits (one per close above):
|
|
Assert.AreEqual(2, policy.numOnCommit);
|
|
|
|
// Test listCommits
|
|
ICollection<IndexCommit> commits = IndexReader.ListCommits(dir);
|
|
// 1 from opening writer + 2 from closing writer
|
|
Assert.AreEqual(3, commits.Count);
|
|
|
|
System.Collections.IEnumerator it = commits.GetEnumerator();
|
|
// Make sure we can open a reader on each commit:
|
|
while (it.MoveNext())
|
|
{
|
|
IndexCommit commit = (IndexCommit) it.Current;
|
|
IndexReader r = IndexReader.Open(commit, null, false);
|
|
r.Close();
|
|
}
|
|
|
|
// Simplistic check: just verify all segments_N's still
|
|
// exist, and, I can open a reader on each:
|
|
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
|
|
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
|
|
while (gen > 0)
|
|
{
|
|
IndexReader reader = IndexReader.Open(dir, true);
|
|
reader.Close();
|
|
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
|
|
gen--;
|
|
|
|
if (gen > 0)
|
|
{
|
|
// Now that we've removed a commit point, which
|
|
// should have orphan'd at least one index file.
|
|
// Open & close a writer and assert that it
|
|
// actually removed something:
|
|
int preCount = dir.ListAll().Length;
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.LIMITED);
|
|
writer.Close();
|
|
int postCount = dir.ListAll().Length;
|
|
Assert.IsTrue(postCount < preCount);
|
|
}
|
|
}
|
|
|
|
dir.Close();
|
|
}
|
|
}
|
|
|
|
/* Uses KeepAllDeletionPolicy to keep all commits around,
|
|
* then, opens a new IndexWriter on a previous commit
|
|
* point. */
|
|
[Test]
|
|
public virtual void TestOpenPriorSnapshot()
|
|
{
|
|
|
|
// Never deletes a commit
|
|
KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(this);
|
|
|
|
Directory dir = new MockRAMDirectory();
|
|
policy.dir = dir;
|
|
|
|
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED);
|
|
writer.SetMaxBufferedDocs(2);
|
|
for (int i = 0; i < 10; i++)
|
|
{
|
|
AddDoc(writer);
|
|
if ((1 + i) % 2 == 0)
|
|
writer.Commit();
|
|
}
|
|
writer.Close();
|
|
|
|
ICollection<IndexCommit> commits = IndexReader.ListCommits(dir);
|
|
Assert.AreEqual(6, commits.Count);
|
|
IndexCommit lastCommit = null;
|
|
System.Collections.IEnumerator it = commits.GetEnumerator();
|
|
while (it.MoveNext())
|
|
{
|
|
IndexCommit commit = (IndexCommit) it.Current;
|
|
if (lastCommit == null || commit.Generation > lastCommit.Generation)
|
|
lastCommit = commit;
|
|
}
|
|
Assert.IsTrue(lastCommit != null);
|
|
|
|
// Now add 1 doc and optimize
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED);
|
|
AddDoc(writer);
|
|
Assert.AreEqual(11, writer.NumDocs());
|
|
writer.Optimize();
|
|
writer.Close();
|
|
|
|
Assert.AreEqual(7, IndexReader.ListCommits(dir).Count);
|
|
|
|
// Now open writer on the commit just before optimize:
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED, lastCommit);
|
|
Assert.AreEqual(10, writer.NumDocs());
|
|
|
|
// Should undo our rollback:
|
|
writer.Rollback();
|
|
|
|
IndexReader r = IndexReader.Open(dir, true);
|
|
// Still optimized, still 11 docs
|
|
Assert.IsTrue(r.IsOptimized());
|
|
Assert.AreEqual(11, r.NumDocs());
|
|
r.Close();
|
|
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED, lastCommit);
|
|
Assert.AreEqual(10, writer.NumDocs());
|
|
// Commits the rollback:
|
|
writer.Close();
|
|
|
|
// Now 8 because we made another commit
|
|
Assert.AreEqual(8, IndexReader.ListCommits(dir).Count);
|
|
|
|
r = IndexReader.Open(dir, true);
|
|
// Not optimized because we rolled it back, and now only
|
|
// 10 docs
|
|
Assert.IsTrue(!r.IsOptimized());
|
|
Assert.AreEqual(10, r.NumDocs());
|
|
r.Close();
|
|
|
|
// Reoptimize
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED);
|
|
writer.Optimize();
|
|
writer.Close();
|
|
|
|
r = IndexReader.Open(dir, true);
|
|
Assert.IsTrue(r.IsOptimized());
|
|
Assert.AreEqual(10, r.NumDocs());
|
|
r.Close();
|
|
|
|
// Now open writer on the commit just before optimize,
|
|
// but this time keeping only the last commit:
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), new KeepOnlyLastCommitDeletionPolicy(), IndexWriter.MaxFieldLength.LIMITED, lastCommit);
|
|
Assert.AreEqual(10, writer.NumDocs());
|
|
|
|
// Reader still sees optimized index, because writer
|
|
// opened on the prior commit has not yet committed:
|
|
r = IndexReader.Open(dir, true);
|
|
Assert.IsTrue(r.IsOptimized());
|
|
Assert.AreEqual(10, r.NumDocs());
|
|
r.Close();
|
|
|
|
writer.Close();
|
|
|
|
// Now reader sees unoptimized index:
|
|
r = IndexReader.Open(dir, true);
|
|
Assert.IsTrue(!r.IsOptimized());
|
|
Assert.AreEqual(10, r.NumDocs());
|
|
r.Close();
|
|
|
|
dir.Close();
|
|
}
|
|
|
|
|
|
/* Test keeping NO commit points. This is a viable and
|
|
* useful case eg where you want to build a big index and
|
|
* you know there are no readers.
|
|
*/
|
|
[Test]
|
|
public virtual void TestKeepNoneOnInitDeletionPolicy()
|
|
{
|
|
for (int pass = 0; pass < 2; pass++)
|
|
{
|
|
bool useCompoundFile = (pass % 2) != 0;
|
|
|
|
KeepNoneOnInitDeletionPolicy policy = new KeepNoneOnInitDeletionPolicy(this);
|
|
|
|
Directory dir = new RAMDirectory();
|
|
|
|
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.SetMaxBufferedDocs(10);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
for (int i = 0; i < 107; i++)
|
|
{
|
|
AddDoc(writer);
|
|
}
|
|
writer.Close();
|
|
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
writer.Optimize();
|
|
writer.Close();
|
|
|
|
Assert.AreEqual(2, policy.numOnInit);
|
|
// If we are not auto committing then there should
|
|
// be exactly 2 commits (one per close above):
|
|
Assert.AreEqual(2, policy.numOnCommit);
|
|
|
|
// Simplistic check: just verify the index is in fact
|
|
// readable:
|
|
IndexReader reader = IndexReader.Open(dir, true);
|
|
reader.Close();
|
|
|
|
dir.Close();
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Test a deletion policy that keeps last N commits.
|
|
*/
|
|
[Test]
|
|
public virtual void TestKeepLastNDeletionPolicy()
|
|
{
|
|
int N = 5;
|
|
|
|
for (int pass = 0; pass < 2; pass++)
|
|
{
|
|
bool useCompoundFile = (pass % 2) != 0;
|
|
|
|
Directory dir = new RAMDirectory();
|
|
|
|
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
|
|
|
|
for (int j = 0; j < N + 1; j++)
|
|
{
|
|
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.SetMaxBufferedDocs(10);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
for (int i = 0; i < 17; i++)
|
|
{
|
|
AddDoc(writer);
|
|
}
|
|
writer.Optimize();
|
|
writer.Close();
|
|
}
|
|
|
|
Assert.IsTrue(policy.numDelete > 0);
|
|
Assert.AreEqual(N + 1, policy.numOnInit);
|
|
Assert.AreEqual(N + 1, policy.numOnCommit);
|
|
|
|
// Simplistic check: just verify only the past N segments_N's still
|
|
// exist, and, I can open a reader on each:
|
|
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
|
|
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
|
|
for (int i = 0; i < N + 1; i++)
|
|
{
|
|
try
|
|
{
|
|
IndexReader reader = IndexReader.Open(dir, true);
|
|
reader.Close();
|
|
if (i == N)
|
|
{
|
|
Assert.Fail("should have failed on commits prior to last " + N);
|
|
}
|
|
}
|
|
catch (System.IO.IOException e)
|
|
{
|
|
if (i != N)
|
|
{
|
|
throw e;
|
|
}
|
|
}
|
|
if (i < N)
|
|
{
|
|
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
|
|
}
|
|
gen--;
|
|
}
|
|
|
|
dir.Close();
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Test a deletion policy that keeps last N commits
|
|
* around, with reader doing deletes.
|
|
*/
|
|
[Test]
|
|
public virtual void TestKeepLastNDeletionPolicyWithReader()
|
|
{
|
|
int N = 10;
|
|
|
|
for (int pass = 0; pass < 2; pass++)
|
|
{
|
|
bool useCompoundFile = (pass % 2) != 0;
|
|
|
|
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
|
|
|
|
Directory dir = new RAMDirectory();
|
|
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
writer.Close();
|
|
Term searchTerm = new Term("content", "aaa");
|
|
Query query = new TermQuery(searchTerm);
|
|
|
|
for (int i = 0; i < N + 1; i++)
|
|
{
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
for (int j = 0; j < 17; j++)
|
|
{
|
|
AddDoc(writer);
|
|
}
|
|
// this is a commit
|
|
writer.Close();
|
|
IndexReader reader = IndexReader.Open(dir, policy, false);
|
|
reader.DeleteDocument(3 * i + 1);
|
|
reader.SetNorm(4 * i + 1, "content", 2.0F);
|
|
IndexSearcher searcher = new IndexSearcher(reader);
|
|
ScoreDoc[] hits = searcher.Search(query, null, 1000).ScoreDocs;
|
|
Assert.AreEqual(16 * (1 + i), hits.Length);
|
|
// this is a commit
|
|
reader.Close();
|
|
searcher.Close();
|
|
}
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
writer.Optimize();
|
|
// this is a commit
|
|
writer.Close();
|
|
|
|
Assert.AreEqual(2 * (N + 2), policy.numOnInit);
|
|
Assert.AreEqual(2 * (N + 2) - 1, policy.numOnCommit);
|
|
|
|
IndexSearcher searcher2 = new IndexSearcher(dir, false);
|
|
ScoreDoc[] hits2 = searcher2.Search(query, null, 1000).ScoreDocs;
|
|
Assert.AreEqual(176, hits2.Length);
|
|
|
|
// Simplistic check: just verify only the past N segments_N's still
|
|
// exist, and, I can open a reader on each:
|
|
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
|
|
|
|
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
|
|
int expectedCount = 176;
|
|
|
|
for (int i = 0; i < N + 1; i++)
|
|
{
|
|
try
|
|
{
|
|
IndexReader reader = IndexReader.Open(dir, true);
|
|
|
|
// Work backwards in commits on what the expected
|
|
// count should be.
|
|
searcher2 = new IndexSearcher(reader);
|
|
hits2 = searcher2.Search(query, null, 1000).ScoreDocs;
|
|
if (i > 1)
|
|
{
|
|
if (i % 2 == 0)
|
|
{
|
|
expectedCount += 1;
|
|
}
|
|
else
|
|
{
|
|
expectedCount -= 17;
|
|
}
|
|
}
|
|
Assert.AreEqual(expectedCount, hits2.Length);
|
|
searcher2.Close();
|
|
reader.Close();
|
|
if (i == N)
|
|
{
|
|
Assert.Fail("should have failed on commits before last 5");
|
|
}
|
|
}
|
|
catch (System.IO.IOException e)
|
|
{
|
|
if (i != N)
|
|
{
|
|
throw e;
|
|
}
|
|
}
|
|
if (i < N)
|
|
{
|
|
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
|
|
}
|
|
gen--;
|
|
}
|
|
|
|
dir.Close();
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Test a deletion policy that keeps last N commits
|
|
* around, through creates.
|
|
*/
|
|
[Test]
|
|
public virtual void TestKeepLastNDeletionPolicyWithCreates()
|
|
{
|
|
int N = 10;
|
|
|
|
for (int pass = 0; pass < 2; pass++)
|
|
{
|
|
bool useCompoundFile = (pass % 2) != 0;
|
|
|
|
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
|
|
|
|
Directory dir = new RAMDirectory();
|
|
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.SetMaxBufferedDocs(10);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
writer.Close();
|
|
Term searchTerm = new Term("content", "aaa");
|
|
Query query = new TermQuery(searchTerm);
|
|
|
|
for (int i = 0; i < N + 1; i++)
|
|
{
|
|
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
writer.SetMaxBufferedDocs(10);
|
|
writer.UseCompoundFile = useCompoundFile;
|
|
for (int j = 0; j < 17; j++)
|
|
{
|
|
AddDoc(writer);
|
|
}
|
|
// this is a commit
|
|
writer.Close();
|
|
IndexReader reader = IndexReader.Open(dir, policy, false);
|
|
reader.DeleteDocument(3);
|
|
reader.SetNorm(5, "content", 2.0F);
|
|
IndexSearcher searcher = new IndexSearcher(reader);
|
|
ScoreDoc[] hits = searcher.Search(query, null, 1000).ScoreDocs;
|
|
Assert.AreEqual(16, hits.Length);
|
|
// this is a commit
|
|
reader.Close();
|
|
searcher.Close();
|
|
|
|
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, policy, IndexWriter.MaxFieldLength.UNLIMITED);
|
|
// This will not commit: there are no changes
|
|
// pending because we opened for "create":
|
|
writer.Close();
|
|
}
|
|
|
|
Assert.AreEqual(1 + 3 * (N + 1), policy.numOnInit);
|
|
Assert.AreEqual(3 * (N + 1), policy.numOnCommit);
|
|
|
|
IndexSearcher searcher2 = new IndexSearcher(dir, false);
|
|
ScoreDoc[] hits2 = searcher2.Search(query, null, 1000).ScoreDocs;
|
|
Assert.AreEqual(0, hits2.Length);
|
|
|
|
// Simplistic check: just verify only the past N segments_N's still
|
|
// exist, and, I can open a reader on each:
|
|
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
|
|
|
|
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
|
|
int expectedCount = 0;
|
|
|
|
for (int i = 0; i < N + 1; i++)
|
|
{
|
|
try
|
|
{
|
|
IndexReader reader = IndexReader.Open(dir, true);
|
|
|
|
// Work backwards in commits on what the expected
|
|
// count should be.
|
|
searcher2 = new IndexSearcher(reader);
|
|
hits2 = searcher2.Search(query, null, 1000).ScoreDocs;
|
|
Assert.AreEqual(expectedCount, hits2.Length);
|
|
searcher2.Close();
|
|
if (expectedCount == 0)
|
|
{
|
|
expectedCount = 16;
|
|
}
|
|
else if (expectedCount == 16)
|
|
{
|
|
expectedCount = 17;
|
|
}
|
|
else if (expectedCount == 17)
|
|
{
|
|
expectedCount = 0;
|
|
}
|
|
reader.Close();
|
|
if (i == N)
|
|
{
|
|
Assert.Fail("should have failed on commits before last " + N);
|
|
}
|
|
}
|
|
catch (System.IO.IOException e)
|
|
{
|
|
if (i != N)
|
|
{
|
|
throw e;
|
|
}
|
|
}
|
|
if (i < N)
|
|
{
|
|
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
|
|
}
|
|
gen--;
|
|
}
|
|
|
|
dir.Close();
|
|
}
|
|
}
|
|
|
|
private void AddDoc(IndexWriter writer)
|
|
{
|
|
Document doc = new Document();
|
|
doc.Add(new Field("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
|
|
writer.AddDocument(doc);
|
|
}
|
|
}
|
|
} |