move tokenizers to their own project

since both pdfs and Adobe Type1 fonts use postscript type objects, tokenization is needed by the main project and the fonts project

src/UglyToad.PdfPig.Fonts/Type1/Type1ArrayTokenizer.cs

@@ -0,0 +1,80 @@
+namespace UglyToad.PdfPig.Fonts.Type1
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Globalization;
+    using System.Text;
+    using Core;
+    using Tokens;
+    using Tokenization;
+
+    /// <inheritdoc />
+    public class Type1ArrayTokenizer : ITokenizer
+    {
+        /// <inheritdoc />
+        public bool ReadsNextByte { get; } = false;
+
+        /// <inheritdoc />
+        public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token)
+        {
+            token = null;
+
+            if (currentByte != '{')
+            {
+                return false;
+            }
+
+            var builder = new StringBuilder();
+
+            while (inputBytes.MoveNext())
+            {
+                if (inputBytes.CurrentByte == '}')
+                {
+                    break;
+                }
+
+                builder.Append((char) inputBytes.CurrentByte);
+            }
+
+            var parts = builder.ToString().Split(new[] {" "}, StringSplitOptions.RemoveEmptyEntries);
+
+            var tokens = new List<IToken>();
+
+            foreach (var part in parts)
+            {
+                if (char.IsNumber(part[0]) || part[0] == '-')
+                {
+                    if (decimal.TryParse(part, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var value))
+                    {
+                        tokens.Add(new NumericToken(value));
+                    }
+                    else
+                    {
+                        tokens.Add(OperatorToken.Create(part));
+                    }
+
+                    continue;
+                }
+
+                if (part[0] == '/')
+                {
+                    tokens.Add(NameToken.Create(part.Substring(1)));
+                    continue;
+                }
+
+                if (part[0] == '(' && part[part.Length - 1] == ')')
+                {
+                    tokens.Add(new StringToken(part));
+                    continue;
+                }
+
+                tokens.Add(OperatorToken.Create(part));
+
+            }
+
+            token = new ArrayToken(tokens);
+
+            return true;
+        }
+    }
+}

src/UglyToad.PdfPig.Fonts/Type1/Type1NameTokenizer.cs

@@ -0,0 +1,45 @@
+namespace UglyToad.PdfPig.Fonts.Type1
+{
+    using System.Text;
+    using Core;
+    using Tokens;
+    using Tokenization;
+
+    /// <inheritdoc />
+    public class Type1NameTokenizer : ITokenizer
+    {
+        /// <inheritdoc />
+        public bool ReadsNextByte { get; } = true;
+
+        /// <inheritdoc />
+        public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token)
+        {
+            token = null;
+
+            if (currentByte != '/')
+            {
+                return false;
+            }
+
+            var builder = new StringBuilder();
+            while (inputBytes.MoveNext())
+            {
+                if (ReadHelper.IsWhitespace(inputBytes.CurrentByte)
+                    || inputBytes.CurrentByte == '{'
+                    || inputBytes.CurrentByte == '<'
+                    || inputBytes.CurrentByte == '/'
+                    || inputBytes.CurrentByte == '['
+                    || inputBytes.CurrentByte == '(')
+                {
+                    break;
+                }
+
+                builder.Append((char)inputBytes.CurrentByte);
+            }
+
+            token = NameToken.Create(builder.ToString());
+
+            return true;
+        }
+    }
+}

src/UglyToad.PdfPig.Fonts/UglyToad.PdfPig.Fonts.csproj

@@ -27,6 +27,7 @@
   
   <ItemGroup>
     <ProjectReference Include="..\UglyToad.PdfPig.Core\UglyToad.PdfPig.Core.csproj" />
+    <ProjectReference Include="..\UglyToad.PdfPig.Tokenization\UglyToad.PdfPig.Tokenization.csproj" />
     <ProjectReference Include="..\UglyToad.PdfPig.Tokens\UglyToad.PdfPig.Tokens.csproj" />
   </ItemGroup>