285 lines
9.0 KiB
C#
285 lines
9.0 KiB
C#
/*
|
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
* contributor license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright ownership.
|
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
* (the "License"); you may not use this file except in compliance with
|
|
* the License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
using System;
|
|
using System.IO;
|
|
using Lucene.Net.Analysis.Tokenattributes;
|
|
using Lucene.Net.Util;
|
|
using NUnit.Framework;
|
|
using Attribute = Lucene.Net.Util.Attribute;
|
|
using Payload = Lucene.Net.Index.Payload;
|
|
using TestSimpleAttributeImpls = Lucene.Net.Analysis.Tokenattributes.TestSimpleAttributeImpls;
|
|
using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
|
|
|
|
namespace Lucene.Net.Analysis
|
|
{
|
|
|
|
[TestFixture]
|
|
public class TestToken:LuceneTestCase
|
|
{
|
|
public TestToken()
|
|
{
|
|
}
|
|
|
|
public TestToken(System.String name):base(name)
|
|
{
|
|
}
|
|
|
|
[Test]
|
|
public virtual void TestCtor()
|
|
{
|
|
Token t = new Token();
|
|
char[] content = "hello".ToCharArray();
|
|
t.SetTermBuffer(content, 0, content.Length);
|
|
char[] buf = t.TermBuffer();
|
|
Assert.AreNotEqual(t.TermBuffer(), content);
|
|
Assert.AreEqual("hello", t.Term);
|
|
Assert.AreEqual("word", t.Type);
|
|
Assert.AreEqual(0, t.Flags);
|
|
|
|
t = new Token(6, 22);
|
|
t.SetTermBuffer(content, 0, content.Length);
|
|
Assert.AreEqual("hello", t.Term);
|
|
Assert.AreEqual("(hello,6,22)", t.ToString());
|
|
Assert.AreEqual("word", t.Type);
|
|
Assert.AreEqual(0, t.Flags);
|
|
|
|
t = new Token(6, 22, 7);
|
|
t.SetTermBuffer(content, 0, content.Length);
|
|
Assert.AreEqual("hello", t.Term);
|
|
Assert.AreEqual("(hello,6,22)", t.ToString());
|
|
Assert.AreEqual(7, t.Flags);
|
|
|
|
t = new Token(6, 22, "junk");
|
|
t.SetTermBuffer(content, 0, content.Length);
|
|
Assert.AreEqual("hello", t.Term);
|
|
Assert.AreEqual("(hello,6,22,type=junk)", t.ToString());
|
|
Assert.AreEqual(0, t.Flags);
|
|
}
|
|
|
|
[Test]
|
|
public virtual void TestResize()
|
|
{
|
|
Token t = new Token();
|
|
char[] content = "hello".ToCharArray();
|
|
t.SetTermBuffer(content, 0, content.Length);
|
|
for (int i = 0; i < 2000; i++)
|
|
{
|
|
t.ResizeTermBuffer(i);
|
|
Assert.IsTrue(i <= t.TermBuffer().Length);
|
|
Assert.AreEqual("hello", t.Term);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public virtual void TestGrow()
|
|
{
|
|
Token t = new Token();
|
|
System.Text.StringBuilder buf = new System.Text.StringBuilder("ab");
|
|
for (int i = 0; i < 20; i++)
|
|
{
|
|
char[] content = buf.ToString().ToCharArray();
|
|
t.SetTermBuffer(content, 0, content.Length);
|
|
Assert.AreEqual(buf.Length, t.TermLength());
|
|
Assert.AreEqual(buf.ToString(), t.Term);
|
|
buf.Append(buf.ToString());
|
|
}
|
|
Assert.AreEqual(1048576, t.TermLength());
|
|
Assert.AreEqual(1179654, t.TermBuffer().Length);
|
|
|
|
// now as a string, first variant
|
|
t = new Token();
|
|
buf = new System.Text.StringBuilder("ab");
|
|
for (int i = 0; i < 20; i++)
|
|
{
|
|
System.String content = buf.ToString();
|
|
t.SetTermBuffer(content, 0, content.Length);
|
|
Assert.AreEqual(content.Length, t.TermLength());
|
|
Assert.AreEqual(content, t.Term);
|
|
buf.Append(content);
|
|
}
|
|
Assert.AreEqual(1048576, t.TermLength());
|
|
Assert.AreEqual(1179654, t.TermBuffer().Length);
|
|
|
|
// now as a string, second variant
|
|
t = new Token();
|
|
buf = new System.Text.StringBuilder("ab");
|
|
for (int i = 0; i < 20; i++)
|
|
{
|
|
System.String content = buf.ToString();
|
|
t.SetTermBuffer(content);
|
|
Assert.AreEqual(content.Length, t.TermLength());
|
|
Assert.AreEqual(content, t.Term);
|
|
buf.Append(content);
|
|
}
|
|
Assert.AreEqual(1048576, t.TermLength());
|
|
Assert.AreEqual(1179654, t.TermBuffer().Length);
|
|
|
|
// Test for slow growth to a long term
|
|
t = new Token();
|
|
buf = new System.Text.StringBuilder("a");
|
|
for (int i = 0; i < 20000; i++)
|
|
{
|
|
System.String content = buf.ToString();
|
|
t.SetTermBuffer(content);
|
|
Assert.AreEqual(content.Length, t.TermLength());
|
|
Assert.AreEqual(content, t.Term);
|
|
buf.Append("a");
|
|
}
|
|
Assert.AreEqual(20000, t.TermLength());
|
|
Assert.AreEqual(20167, t.TermBuffer().Length);
|
|
|
|
// Test for slow growth to a long term
|
|
t = new Token();
|
|
buf = new System.Text.StringBuilder("a");
|
|
for (int i = 0; i < 20000; i++)
|
|
{
|
|
System.String content = buf.ToString();
|
|
t.SetTermBuffer(content);
|
|
Assert.AreEqual(content.Length, t.TermLength());
|
|
Assert.AreEqual(content, t.Term);
|
|
buf.Append("a");
|
|
}
|
|
Assert.AreEqual(20000, t.TermLength());
|
|
Assert.AreEqual(20167, t.TermBuffer().Length);
|
|
}
|
|
|
|
[Test]
|
|
public virtual void TestToString()
|
|
{
|
|
char[] b = new char[]{'a', 'l', 'o', 'h', 'a'};
|
|
Token t = new Token("", 0, 5);
|
|
t.SetTermBuffer(b, 0, 5);
|
|
Assert.AreEqual("(aloha,0,5)", t.ToString());
|
|
|
|
t.SetTermBuffer("hi there");
|
|
Assert.AreEqual("(hi there,0,5)", t.ToString());
|
|
}
|
|
|
|
[Test]
|
|
public virtual void TestTermBufferEquals()
|
|
{
|
|
Token t1a = new Token();
|
|
char[] content1a = "hello".ToCharArray();
|
|
t1a.SetTermBuffer(content1a, 0, 5);
|
|
Token t1b = new Token();
|
|
char[] content1b = "hello".ToCharArray();
|
|
t1b.SetTermBuffer(content1b, 0, 5);
|
|
Token t2 = new Token();
|
|
char[] content2 = "hello2".ToCharArray();
|
|
t2.SetTermBuffer(content2, 0, 6);
|
|
Assert.IsTrue(t1a.Equals(t1b));
|
|
Assert.IsFalse(t1a.Equals(t2));
|
|
Assert.IsFalse(t2.Equals(t1b));
|
|
}
|
|
|
|
[Test]
|
|
public virtual void TestMixedStringArray()
|
|
{
|
|
Token t = new Token("hello", 0, 5);
|
|
Assert.AreEqual(t.TermLength(), 5);
|
|
Assert.AreEqual(t.Term, "hello");
|
|
t.SetTermBuffer("hello2");
|
|
Assert.AreEqual(t.TermLength(), 6);
|
|
Assert.AreEqual(t.Term, "hello2");
|
|
t.SetTermBuffer("hello3".ToCharArray(), 0, 6);
|
|
Assert.AreEqual(t.Term, "hello3");
|
|
|
|
char[] buffer = t.TermBuffer();
|
|
buffer[1] = 'o';
|
|
Assert.AreEqual(t.Term, "hollo3");
|
|
}
|
|
|
|
[Test]
|
|
public virtual void TestClone()
|
|
{
|
|
Token t = new Token(0, 5);
|
|
char[] content = "hello".ToCharArray();
|
|
t.SetTermBuffer(content, 0, 5);
|
|
char[] buf = t.TermBuffer();
|
|
Token copy = (Token) TestSimpleAttributeImpls.AssertCloneIsEqual(t);
|
|
Assert.AreEqual(t.Term, copy.Term);
|
|
Assert.AreNotSame(buf, copy.TermBuffer());
|
|
|
|
Payload pl = new Payload(new byte[]{1, 2, 3, 4});
|
|
t.Payload = pl;
|
|
copy = (Token) TestSimpleAttributeImpls.AssertCloneIsEqual(t);
|
|
Assert.AreEqual(pl, copy.Payload);
|
|
Assert.AreNotSame(pl, copy.Payload);
|
|
}
|
|
|
|
[Test]
|
|
public virtual void TestCopyTo()
|
|
{
|
|
Token t = new Token();
|
|
Token copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t);
|
|
Assert.AreEqual("", t.Term);
|
|
Assert.AreEqual("", copy.Term);
|
|
|
|
t = new Token(0, 5);
|
|
char[] content = "hello".ToCharArray();
|
|
t.SetTermBuffer(content, 0, 5);
|
|
char[] buf = t.TermBuffer();
|
|
copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t);
|
|
Assert.AreEqual(t.Term, copy.Term);
|
|
Assert.AreNotSame(buf, copy.TermBuffer());
|
|
|
|
Payload pl = new Payload(new byte[]{1, 2, 3, 4});
|
|
t.Payload = pl;
|
|
copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t);
|
|
Assert.AreEqual(pl, copy.Payload);
|
|
Assert.AreNotSame(pl, copy.Payload);
|
|
}
|
|
|
|
public interface ISenselessAttribute : IAttribute {}
|
|
|
|
public class SenselessAttribute : Attribute, ISenselessAttribute
|
|
{
|
|
public override void CopyTo(Attribute target)
|
|
{ }
|
|
|
|
public override void Clear()
|
|
{ }
|
|
|
|
public override bool Equals(object other)
|
|
{
|
|
return other is SenselessAttribute;
|
|
}
|
|
|
|
public override int GetHashCode()
|
|
{
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public void TestTokenAttributeFactory()
|
|
{
|
|
TokenStream ts = new WhitespaceTokenizer(Token.TOKEN_ATTRIBUTE_FACTORY, new StringReader("foo, bar"));
|
|
|
|
Assert.IsTrue(ts.AddAttribute<ISenselessAttribute>() is SenselessAttribute,
|
|
"TypeAttribute is not implemented by SenselessAttributeImpl");
|
|
|
|
Assert.IsTrue(ts.AddAttribute<ITermAttribute>() is Token, "TermAttribute is not implemented by Token");
|
|
Assert.IsTrue(ts.AddAttribute<IOffsetAttribute>() is Token, "OffsetAttribute is not implemented by Token");
|
|
Assert.IsTrue(ts.AddAttribute<IFlagsAttribute>() is Token, "FlagsAttribute is not implemented by Token");
|
|
Assert.IsTrue(ts.AddAttribute<IPayloadAttribute>() is Token, "PayloadAttribute is not implemented by Token");
|
|
Assert.IsTrue(ts.AddAttribute<IPositionIncrementAttribute>() is Token, "PositionIncrementAttribute is not implemented by Token");
|
|
Assert.IsTrue(ts.AddAttribute<ITypeAttribute>() is Token, "TypeAttribute is not implemented by Token");
|
|
}
|
|
}
|
|
} |