adding in PlainTokenizer to unpooled SB changes

This commit is contained in:
Plaisted
2021-01-19 18:52:14 -06:00
parent feb6117e1e
commit 4c807691b7
2 changed files with 5 additions and 4 deletions

View File

@@ -1,11 +1,12 @@
namespace UglyToad.PdfPig.Tokenization
{
using Core;
using System.Text;
using Tokens;
internal class PlainTokenizer : ITokenizer
{
private static readonly StringBuilderPool StringBuilderPool = new StringBuilderPool(10);
private readonly StringBuilder stringBuilder = new();
public bool ReadsNextByte { get; } = true;
@@ -18,7 +19,7 @@
return false;
}
var builder = StringBuilderPool.Borrow();
var builder = stringBuilder;
builder.Append((char)currentByte);
while (inputBytes.MoveNext())
{
@@ -39,7 +40,7 @@
}
var text = builder.ToString();
StringBuilderPool.Return(builder);
builder.Clear();
switch (text)
{

View File

@@ -15,10 +15,10 @@
private static readonly DictionaryTokenizer DictionaryTokenizer = new DictionaryTokenizer();
private static readonly HexTokenizer HexTokenizer = new HexTokenizer();
private static readonly NameTokenizer NameTokenizer = new NameTokenizer();
private static readonly PlainTokenizer PlainTokenizer = new PlainTokenizer();
// NOTE: these are not thread safe so should not be static. Each instance includes a
// StringBuilder it re-uses.
private readonly PlainTokenizer PlainTokenizer = new PlainTokenizer();
private readonly NumericTokenizer NumericTokenizer = new NumericTokenizer();
private readonly StringTokenizer StringTokenizer = new StringTokenizer();