move tokens to new project

src/UglyToad.PdfPig.Core/IndirectReference.cs

@@ -1,4 +1,4 @@
-namespace UglyToad.PdfPig
+namespace UglyToad.PdfPig.Core
 {
     using System.Diagnostics;
 

src/UglyToad.PdfPig.Fonts/UglyToad.PdfPig.Fonts.csproj

@@ -27,6 +27,7 @@
   
   <ItemGroup>
     <ProjectReference Include="..\UglyToad.PdfPig.Core\UglyToad.PdfPig.Core.csproj" />
+    <ProjectReference Include="..\UglyToad.PdfPig.Tokens\UglyToad.PdfPig.Tokens.csproj" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFramework)'=='net45' OR '$(TargetFramework)'=='net451' OR '$(TargetFramework)'=='net452' OR '$(TargetFramework)'=='net46' OR '$(TargetFramework)'=='net461' OR '$(TargetFramework)'=='net462' OR '$(TargetFramework)'=='net47'">

src/UglyToad.PdfPig.Tests/ContentStream/IndirectReferenceTests.cs

@@ -1,5 +1,6 @@
 namespace UglyToad.PdfPig.Tests.ContentStream
 {
+    using PdfPig.Core;
     using Xunit;
 
     public class IndirectReferenceTests

src/UglyToad.PdfPig.Tests/PublicApiScannerTests.cs

@@ -179,7 +179,6 @@
                 "UglyToad.PdfPig.Graphics.Operations.TextState.Type3SetGlyphWidth",
                 "UglyToad.PdfPig.Graphics.Operations.TextState.Type3SetGlyphWidthAndBoundingBox",
                 "UglyToad.PdfPig.Graphics.TextMatrices",
-                "UglyToad.PdfPig.IndirectReference",
                 "UglyToad.PdfPig.Logging.ILog",
                 "UglyToad.PdfPig.Outline.Bookmarks",
                 "UglyToad.PdfPig.Outline.BookmarkNode",
@@ -191,21 +190,6 @@
                 "UglyToad.PdfPig.ParsingOptions",
                 "UglyToad.PdfPig.PdfDocument",
                 "UglyToad.PdfPig.Structure",
-                "UglyToad.PdfPig.Tokens.ArrayToken",
-                "UglyToad.PdfPig.Tokens.BooleanToken",
-                "UglyToad.PdfPig.Tokens.CommentToken",
-                "UglyToad.PdfPig.Tokens.DictionaryToken",
-                "UglyToad.PdfPig.Tokens.HexToken",
-                "UglyToad.PdfPig.Tokens.IDataToken`1",
-                "UglyToad.PdfPig.Tokens.IndirectReferenceToken",
-                "UglyToad.PdfPig.Tokens.InlineImageDataToken",
-                "UglyToad.PdfPig.Tokens.IToken",
-                "UglyToad.PdfPig.Tokens.NameToken",
-                "UglyToad.PdfPig.Tokens.NullToken",
-                "UglyToad.PdfPig.Tokens.NumericToken",
-                "UglyToad.PdfPig.Tokens.ObjectToken",
-                "UglyToad.PdfPig.Tokens.StreamToken",
-                "UglyToad.PdfPig.Tokens.StringToken",
                 "UglyToad.PdfPig.Util.Adler32Checksum",
                 "UglyToad.PdfPig.Util.IWordExtractor",
                 "UglyToad.PdfPig.Util.DefaultWordExtractor",

src/UglyToad.PdfPig.Tests/TestObjectLocationProvider.cs

@@ -1,6 +1,7 @@
 namespace UglyToad.PdfPig.Tests
 {
     using System.Collections.Generic;
+    using PdfPig.Core;
     using PdfPig.Tokenization.Scanner;
     using PdfPig.Tokens;
 

src/UglyToad.PdfPig.Tests/Tokenization/DictionaryTokenizerTests.cs

@@ -3,6 +3,7 @@ namespace UglyToad.PdfPig.Tests.Tokenization
 {
     using System;
     using System.Collections.Generic;
+    using PdfPig.Core;
     using PdfPig.Tokenization;
     using PdfPig.Tokens;
     using Xunit;

src/UglyToad.PdfPig.Tests/Tokens/TestPdfTokenScanner.cs

@@ -3,14 +3,14 @@ namespace UglyToad.PdfPig.Tests.Tokens
 {
     using System;
     using System.Collections.Generic;
+    using PdfPig.Core;
     using PdfPig.Tokenization;
     using PdfPig.Tokenization.Scanner;
     using PdfPig.Tokens;
 
     internal class TestPdfTokenScanner : IPdfTokenScanner
     {
-        public Dictionary<IndirectReference, ObjectToken> Objects { get; }
-         = new Dictionary<IndirectReference, ObjectToken>();
+        public Dictionary<IndirectReference, ObjectToken> Objects { get; } = new Dictionary<IndirectReference, ObjectToken>();
         public bool MoveNext()
         {
             throw new NotImplementedException();

src/UglyToad.PdfPig.Tests/UglyToad.PdfPig.Tests.csproj

@@ -100,6 +100,7 @@
   <ItemGroup>
     <ProjectReference Include="..\UglyToad.PdfPig.Core\UglyToad.PdfPig.Core.csproj" />
     <ProjectReference Include="..\UglyToad.PdfPig.Fonts\UglyToad.PdfPig.Fonts.csproj" />
+    <ProjectReference Include="..\UglyToad.PdfPig.Tokens\UglyToad.PdfPig.Tokens.csproj" />
     <ProjectReference Include="..\UglyToad.PdfPig\UglyToad.PdfPig.csproj" />
   </ItemGroup>
 

src/UglyToad.PdfPig.Tokens/ArrayToken.cs

@@ -3,7 +3,7 @@
     using System;
     using System.Collections.Generic;
     using System.Text;
-    using Util.JetBrains.Annotations;
+    using Core;
 
     /// <summary>
     /// An array object is a one-dimensional collection of objects arranged sequentially.
@@ -15,7 +15,6 @@
         /// <summary>
         /// The tokens contained in this array.
         /// </summary>
-        [NotNull]
         public IReadOnlyList<IToken> Data { get; }
 
         /// <summary>
@@ -32,7 +31,7 @@
         /// Create a new <see cref="ArrayToken"/>.
         /// </summary>
         /// <param name="data">The tokens contained by this array.</param>
-        public ArrayToken([NotNull] IReadOnlyList<IToken> data)
+        public ArrayToken(IReadOnlyList<IToken> data)
         {
             if (data == null)
             {

src/UglyToad.PdfPig.Tokens/BooleanToken.cs

@@ -1,7 +1,5 @@
 namespace UglyToad.PdfPig.Tokens
 {
-    using Util.JetBrains.Annotations;
-
     /// <summary>
     /// The boolean object either <see cref="True"/> (<see langword="true"/>) or <see cref="False"/> (<see langword="true"/>).
     /// </summary>
@@ -10,13 +8,11 @@
         /// <summary>
         /// The boolean token corresponding to <see langword="true"/>.
         /// </summary>
-        [NotNull]
         public static BooleanToken True { get; } = new BooleanToken(true);
 
         /// <summary>
         /// The boolean token corresponding to <see langword="false"/> 
         /// </summary>
-        [NotNull]
         public static BooleanToken False { get; } = new BooleanToken(false);
 
         /// <summary>

src/UglyToad.PdfPig.Tokens/CommentToken.cs

@@ -1,7 +1,5 @@
 namespace UglyToad.PdfPig.Tokens
 {
-    using Util.JetBrains.Annotations;
-
     /// <summary>
     /// A comment from a PDF document. Any occurrence of the percent sign character (%) outside a string or stream
     /// introduces a comment. The comment consists of all characters between the percent sign and the end of the line.
@@ -11,14 +9,13 @@
         /// <summary>
         /// The text of the comment (excluding the initial percent '%' sign).
         /// </summary>
-        [NotNull]
         public string Data { get; }
 
         /// <summary>
         /// Create a new <see cref="CommentToken"/>.
         /// </summary>
         /// <param name="data">The text of the comment.</param>
-        public CommentToken([CanBeNull]string data)
+        public CommentToken(string data)
         {
             Data = data ?? string.Empty;
         }

src/UglyToad.PdfPig.Tokens/DictionaryToken.cs

@@ -3,10 +3,6 @@
     using System;
     using System.Collections.Generic;
     using System.Linq;
-    using Core;
-    using Parser.Parts;
-    using Tokenization.Scanner;
-    using Util.JetBrains.Annotations;
 
     /// <summary>
     /// A dictionary object is an associative table containing pairs of objects, known as the dictionary's entries. 
@@ -17,14 +13,13 @@
         /// <summary>
         /// The key value pairs in this dictionary.
         /// </summary>
-        [NotNull]
         public IReadOnlyDictionary<string, IToken> Data { get; }
 
         /// <summary>
         /// Create a new <see cref="DictionaryToken"/>.
         /// </summary>
         /// <param name="data">The data this dictionary will contain.</param>
-        public DictionaryToken([NotNull]IReadOnlyDictionary<NameToken, IToken> data)
+        public DictionaryToken(IReadOnlyDictionary<NameToken, IToken> data)
         {
             if (data == null)
             {
@@ -46,21 +41,6 @@
             Data = data;
         }
 
-        internal T Get<T>(NameToken name, IPdfTokenScanner scanner) where T : IToken
-        {
-            if (!TryGet(name, out var token) || !(token is T typedToken))
-            {
-                if (!(token is IndirectReferenceToken indirectReference))
-                {
-                    throw new PdfDocumentFormatException($"Dictionary does not contain token with name {name} of type {typeof(T).Name}.");
-                }
-
-                typedToken = DirectObjectFinder.Get<T>(indirectReference, scanner);
-            }
-
-            return typedToken;
-        }
-
         /// <summary>
         /// Try and get the entry with a given name.
         /// </summary>
@@ -77,26 +57,6 @@
             return Data.TryGetValue(name.Data, out token);
         }
 
-        /// <summary>
-        /// Try and get the entry with a given name and type or look-up the object if it's an indirect reference.
-        /// </summary>
-        internal bool TryGet<T>(NameToken name, IPdfTokenScanner tokenScanner, out T token) where T : IToken
-        {
-            token = default(T);
-            if (!TryGet(name, out var t) || !(t is T typedToken))
-            {
-                if (t is IndirectReferenceToken reference)
-                {
-                    return DirectObjectFinder.TryGet(reference, tokenScanner, out token);
-                }
-
-                return false;
-            }
-
-            token = typedToken;
-            return true;
-        }
-
         /// <summary>
         /// Try and get the entry with a given name and a specific data type.
         /// </summary>

src/UglyToad.PdfPig.Tokens/EndOfLineToken.cs

@@ -1,9 +1,9 @@
 namespace UglyToad.PdfPig.Tokens
 {
     /// <summary>
-    /// Represents an End Of Line marker found in Type 1 font files. This is not used by the main PDF tokenizer.
+    /// Represents an End Of Line marker found in Adobe Type 1 font files. This is not used by the main PDF tokenizer.
     /// </summary>
-    internal class EndOfLineToken : IToken
+    public class EndOfLineToken : IToken
     {
         /// <summary>
         /// The instance of the end of line token.

src/UglyToad.PdfPig.Tokens/HexToken.cs

@@ -4,7 +4,6 @@ namespace UglyToad.PdfPig.Tokens
     using System.Collections.Generic;
     using System.Linq;
     using System.Text;
-    using Util.JetBrains.Annotations;
 
     /// <summary>
     /// A token containing string data where the string is encoded as hexadecimal.
@@ -41,20 +40,18 @@ namespace UglyToad.PdfPig.Tokens
         /// <summary>
         /// The string contained in the hex data.
         /// </summary>
-        [NotNull]
         public string Data { get; }
 
         /// <summary>
         /// The bytes of the hex data.
         /// </summary>
-        [NotNull]
         public IReadOnlyList<byte> Bytes { get; }
 
         /// <summary>
         /// Create a new <see cref="HexToken"/> from the provided hex characters.
         /// </summary>
         /// <param name="characters">A set of hex characters 0-9, A - F, a - f representing a string.</param>
-        public HexToken([NotNull] IReadOnlyList<char> characters)
+        public HexToken(IReadOnlyList<char> characters)
         {
             if (characters == null)
             {
@@ -122,7 +119,7 @@ namespace UglyToad.PdfPig.Tokens
         /// </summary>
         /// <param name="token">The token containing the data to convert.</param>
         /// <returns>The integer corresponding to the bytes.</returns>
-        public static int ConvertHexBytesToInt([NotNull] HexToken token)
+        public static int ConvertHexBytesToInt(HexToken token)
         {
             if (token == null)
             {

src/UglyToad.PdfPig.Tokens/IndirectReferenceToken.cs

@@ -1,10 +1,9 @@
 namespace UglyToad.PdfPig.Tokens
 {
-    using CrossReference;
+    using Core;
 
     /// <summary>
-    /// A reference to an indirect object (see <see cref="ObjectToken"/>). This object may be located by using the
-    /// <see cref="CrossReferenceTable"/>.
+    /// A reference to an indirect object (see <see cref="ObjectToken"/>).
     /// </summary>
     public class IndirectReferenceToken : IDataToken<IndirectReference>
     {

src/UglyToad.PdfPig.Tokens/NameToken.cs

@@ -1,7 +1,6 @@
 namespace UglyToad.PdfPig.Tokens
 {
     using System;
-    using Util.JetBrains.Annotations;
 
     /// <inheritdoc />
     /// <summary>
@@ -15,7 +14,6 @@
         /// <summary>
         /// The string representation of the name.
         /// </summary>
-        [NotNull]
         public string Data { get; }
         
         private NameToken(string text)
@@ -31,7 +29,7 @@
         /// </summary>
         /// <param name="name">The string representation of the name for the token to create.</param>
         /// <returns>The created or existing token.</returns>
-        public static NameToken Create([NotNull] string name)
+        public static NameToken Create(string name)
         {
             if (name == null)
             {

src/UglyToad.PdfPig.Tokens/NumericToken.cs

@@ -0,0 +1,121 @@
+namespace UglyToad.PdfPig.Tokens
+{
+    using System.Globalization;
+
+    /// <inheritdoc />
+    /// <summary>
+    /// PDF supports integer and real numbers. Integer objects represent mathematical integers within a certain interval centered at 0. 
+    /// Real objects  approximate mathematical real numbers, but with limited range and precision.
+    /// This token represents both types and they are used interchangeably in the specification.
+    /// </summary>
+    public class NumericToken : IDataToken<decimal>
+    {
+        /// <summary>
+        /// Single instance of numeric token for 0.
+        /// </summary>
+        public static readonly NumericToken Zero = new NumericToken(0);
+
+        /// <summary>
+        /// Single instance of numeric token for 1.
+        /// </summary>
+        public static readonly NumericToken One = new NumericToken(1);
+
+        /// <summary>
+        /// Single instance of numeric token for 2.
+        /// </summary>
+        public static readonly NumericToken Two = new NumericToken(2);
+
+        /// <summary>
+        /// Single instance of numeric token for 3.
+        /// </summary>
+        public static readonly NumericToken Three = new NumericToken(3);
+
+        /// <summary>
+        /// Single instance of numeric token for 4.
+        /// </summary>
+        public static readonly NumericToken Four = new NumericToken(4);
+        /// <summary>
+        /// Single instance of numeric token for 5.
+        /// </summary>
+        public static readonly NumericToken Five = new NumericToken(5);
+
+        /// <summary>
+        /// Single instance of numeric token for 6.
+        /// </summary>
+        public static readonly NumericToken Six = new NumericToken(6);
+
+        /// <summary>
+        /// Single instance of numeric token for 7.
+        /// </summary>
+        public static readonly NumericToken Seven = new NumericToken(7);
+
+        /// <summary>
+        /// Single instance of numeric token for 8.
+        /// </summary>
+        public static readonly NumericToken Eight = new NumericToken(8);
+
+        /// <summary>
+        /// Single instance of numeric token for 9.
+        /// </summary>
+        public static readonly NumericToken Nine = new NumericToken(9);
+
+        /// <summary>
+        /// Single instance of numeric token for 10.
+        /// </summary>
+        public static readonly NumericToken Ten = new NumericToken(10);
+
+        /// <summary>
+        /// Single instance of numeric token for 100.
+        /// </summary>
+        public static readonly NumericToken OneHundred = new NumericToken(100);
+
+        /// <summary>
+        /// Single instance of numeric token for 1000.
+        /// </summary>
+        public static readonly NumericToken OneThousand = new NumericToken(1000);
+
+        /// <inheritdoc />
+        public decimal Data { get; }
+
+        /// <summary>
+        /// Whether the number represented has a non-zero decimal part.
+        /// </summary>
+        public bool HasDecimalPlaces => decimal.Floor(Data) != Data;
+
+        /// <summary>
+        /// The value of this number as an <see langword="int"/>.
+        /// </summary>
+        public int Int => (int)Data;
+
+        /// <summary>
+        /// The value of this number as a <see langword="long"/>.
+        /// </summary>
+        public long Long => (long)Data;
+
+        /// <summary>
+        /// The value of this number as a <see langword="double"/>.
+        /// </summary>
+        public double Double => (double)Data;
+
+        /// <summary>
+        /// Create a <see cref="NumericToken"/>.
+        /// </summary>
+        /// <param name="value">The number to represent.</param>
+        public NumericToken(decimal value)
+        {
+            Data = value;
+        }
+
+        /// <inheritdoc />
+        public override int GetHashCode()
+        {
+            return Data.GetHashCode();
+        }
+
+        /// <inheritdoc />
+        public override string ToString()
+        {
+            return Data.ToString(NumberFormatInfo.InvariantInfo);
+        }
+    }
+}

src/UglyToad.PdfPig.Tokens/ObjectToken.cs

@@ -1,7 +1,7 @@
 namespace UglyToad.PdfPig.Tokens
 {
     using System;
-    using Util.JetBrains.Annotations;
+    using Core;
 
     /// <summary>
     /// Any object in a PDF file may be labeled as an indirect object. This gives the object a unique object identifier by which other objects can refer to it.
@@ -22,7 +22,6 @@
         /// <summary>
         /// The inner data of the object.
         /// </summary>
-        [NotNull]
         public IToken Data { get; }
 
         /// <summary>

src/UglyToad.PdfPig.Tokens/OperatorToken.cs

@@ -1,34 +1,123 @@
 namespace UglyToad.PdfPig.Tokens
 {
+    using System;
     using System.Collections.Generic;
 
-    internal class OperatorToken : IDataToken<string>
+    /// <inheritdoc />
+    /// <summary>
+    /// An operator token encountered in a page content or Adobe Type 1 font stream.
+    /// </summary>
+    public class OperatorToken : IDataToken<string>
     {
         private static readonly object Lock = new object();
         private static readonly Dictionary<string, string> PooledNames = new Dictionary<string, string>();
 
+        /// <summary>
+        /// Begin text.
+        /// </summary>
         public static readonly OperatorToken Bt = new OperatorToken("BT");
+
+        /// <summary>
+        /// Def.
+        /// </summary>
         public static readonly OperatorToken Def = new OperatorToken("def");
+
+        /// <summary>
+        /// Dict.
+        /// </summary>
         public static readonly OperatorToken Dict = new OperatorToken("dict");
+
+        /// <summary>
+        /// Dup.
+        /// </summary>
         public static readonly OperatorToken Dup = new OperatorToken("dup");
+
+        /// <summary>
+        /// Eexec.
+        /// </summary>
         public static readonly OperatorToken Eexec = new OperatorToken("eexec");
+
+        /// <summary>
+        /// End object.
+        /// </summary>
         public static readonly OperatorToken EndObject = new OperatorToken("endobj");
+
+        /// <summary>
+        /// End stream.
+        /// </summary>
         public static readonly OperatorToken EndStream = new OperatorToken("endstream");
+
+        /// <summary>
+        /// End text.
+        /// </summary>
         public static readonly OperatorToken Et = new OperatorToken("ET");
+
+        /// <summary>
+        /// For.
+        /// </summary>
         public static readonly OperatorToken For = new OperatorToken("for");
+
+        /// <summary>
+        /// N.
+        /// </summary>
         public static readonly OperatorToken N = new OperatorToken("n");
+
+        /// <summary>
+        /// Put.
+        /// </summary>
         public static readonly OperatorToken Put = new OperatorToken("put");
+
+        /// <summary>
+        /// Pop.
+        /// </summary>
         public static readonly OperatorToken QPop = new OperatorToken("Q");
+
+        /// <summary>
+        /// Push.
+        /// </summary>
         public static readonly OperatorToken QPush = new OperatorToken("q");
+
+        /// <summary>
+        /// R.
+        /// </summary>
         public static readonly OperatorToken R = new OperatorToken("R");
+
+        /// <summary>
+        /// Rectangle.
+        /// </summary>
         public static readonly OperatorToken Re = new OperatorToken("re");
+
+        /// <summary>
+        /// Readonly.
+        /// </summary>
         public static readonly OperatorToken Readonly = new OperatorToken("readonly");
+
+        /// <summary>
+        /// Object.
+        /// </summary>
         public static readonly OperatorToken StartObject = new OperatorToken("obj");
+
+        /// <summary>
+        /// Stream.
+        /// </summary>
         public static readonly OperatorToken StartStream = new OperatorToken("stream");
+
+        /// <summary>
+        /// Set font and size.
+        /// </summary>
         public static readonly OperatorToken Tf = new OperatorToken("Tf");
+
+        /// <summary>
+        /// Modify clipping.
+        /// </summary>
         public static readonly OperatorToken WStar = new OperatorToken("W*");
+
+        /// <summary>
+        /// Cross reference.
+        /// </summary>
         public static readonly OperatorToken Xref = new OperatorToken("xref");
 
+        /// <inheritdoc />
         public string Data { get; }
 
         private OperatorToken(string data)
@@ -47,8 +136,16 @@
             Data = stored;
         }
 
+        /// <summary>
+        /// Create a new <see cref="OperatorToken"/>.
+        /// </summary>
         public static OperatorToken Create(string data)
         {
+            if (data == null)
+            {
+                throw new ArgumentNullException(nameof(data));
+            }
+
             switch (data)
             {
                 case "BT":

src/UglyToad.PdfPig.Tokens/StreamToken.cs

@@ -2,8 +2,6 @@
 {
     using System;
     using System.Collections.Generic;
-    using Filters;
-    using Util.JetBrains.Annotations;
 
     /// <summary>
     /// A stream consists of a dictionary followed by zero or more bytes bracketed between the keywords stream and endstream.
@@ -11,20 +9,14 @@
     /// </summary>
     public class StreamToken : IDataToken<IReadOnlyList<byte>>
     {
-        private readonly object lockObject = new object();
-
-        private IReadOnlyList<byte> decodedBytes;
-
         /// <summary>
         /// The dictionary specifying the length of the stream, any applied compression filters and additional information.
         /// </summary>
-        [NotNull]
         public DictionaryToken StreamDictionary { get; }
 
         /// <summary>
         /// The compressed byte data of the stream.
         /// </summary>
-        [NotNull]
         public IReadOnlyList<byte> Data { get; }
 
         /// <summary>
@@ -32,35 +24,12 @@
         /// </summary>
         /// <param name="streamDictionary">The stream dictionary.</param>
         /// <param name="data">The stream data.</param>
-        public StreamToken([NotNull] DictionaryToken streamDictionary, [NotNull] IReadOnlyList<byte> data)
+        public StreamToken(DictionaryToken streamDictionary, IReadOnlyList<byte> data)
         {
             StreamDictionary = streamDictionary ?? throw new ArgumentNullException(nameof(streamDictionary));
             Data = data ?? throw new ArgumentNullException(nameof(data));
         }
-
-        internal IReadOnlyList<byte> Decode(IFilterProvider filterProvider)
-        {
-            lock (lockObject)
-            {
-                if (decodedBytes != null)
-                {
-                    return decodedBytes;
-                }
-                
-                var filters = filterProvider.GetFilters(StreamDictionary);
-
-                var transform = Data;
-                for (var i = 0; i < filters.Count; i++)
-                {
-                    transform = filters[i].Decode(transform, StreamDictionary, i);
-                }
-
-                decodedBytes = transform;
-
-                return transform;
-            }
-        }
-
+        
         /// <inheritdoc />
         public override string ToString()
         {

src/UglyToad.PdfPig.Tokens/StringToken.cs

@@ -1,7 +1,6 @@
 namespace UglyToad.PdfPig.Tokens
 {
     using System;
-    using Util.JetBrains.Annotations;
 
     /// <summary>
     /// Represents a string of text contained in a PDF document.
@@ -11,14 +10,13 @@ namespace UglyToad.PdfPig.Tokens
         /// <summary>
         /// The string in the token.
         /// </summary>
-        [NotNull]
         public string Data { get; }
 
         /// <summary>
         /// Create a new <see cref="StringToken"/>.
         /// </summary>
         /// <param name="data">The string data for the token to contain.</param>
-        public StringToken([NotNull] string data)
+        public StringToken(string data)
         {
             Data = data ?? throw new ArgumentNullException(nameof(data));
         }

src/UglyToad.PdfPig.Tokens/UglyToad.PdfPig.Tokens.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>netstandard2.0;net45;net451;net452;net46;net461;net462;net47</TargetFrameworks>
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(TargetFramework)|$(Platform)'=='Debug|netstandard2.0|AnyCPU'">
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <WarningsAsErrors />
+    <DocumentationFile>obj\Debug\netstandard2.0\UglyToad.PdfPig.Tokens.xml</DocumentationFile>
+  </PropertyGroup>
+
+  <ItemGroup Condition="'$(TargetFramework)'=='net45' OR '$(TargetFramework)'=='net451' OR '$(TargetFramework)'=='net452' OR '$(TargetFramework)'=='net46' OR '$(TargetFramework)'=='net461' OR '$(TargetFramework)'=='net462' OR '$(TargetFramework)'=='net47'">
+    <PackageReference Include="System.ValueTuple" Version="4.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\UglyToad.PdfPig.Core\UglyToad.PdfPig.Core.csproj" />
+  </ItemGroup>
+</Project>

src/UglyToad.PdfPig.sln

@@ -16,7 +16,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "UglyToad.PdfPig.Fonts", "Ug
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "UglyToad.PdfPig.Core", "UglyToad.PdfPig.Core\UglyToad.PdfPig.Core.csproj", "{A7E1DD9F-92B7-4061-9BA5-6E06C06175F9}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UglyToad.PdfPig.DocumentLayoutAnalysis", "UglyToad.PdfPig.DocumentLayoutAnalysis\UglyToad.PdfPig.DocumentLayoutAnalysis.csproj", "{60126BCA-6C52-48A9-A0A6-51796C8B0BE7}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "UglyToad.PdfPig.DocumentLayoutAnalysis", "UglyToad.PdfPig.DocumentLayoutAnalysis\UglyToad.PdfPig.DocumentLayoutAnalysis.csproj", "{60126BCA-6C52-48A9-A0A6-51796C8B0BE7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UglyToad.PdfPig.Tokens", "UglyToad.PdfPig.Tokens\UglyToad.PdfPig.Tokens.csproj", "{D840FF69-4250-4B05-9829-5ABEC43EC82C}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -44,6 +46,10 @@ Global
 		{60126BCA-6C52-48A9-A0A6-51796C8B0BE7}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{60126BCA-6C52-48A9-A0A6-51796C8B0BE7}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{60126BCA-6C52-48A9-A0A6-51796C8B0BE7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D840FF69-4250-4B05-9829-5ABEC43EC82C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D840FF69-4250-4B05-9829-5ABEC43EC82C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D840FF69-4250-4B05-9829-5ABEC43EC82C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D840FF69-4250-4B05-9829-5ABEC43EC82C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/UglyToad.PdfPig/AcroForms/AcroForm.cs

@@ -3,6 +3,7 @@
     using System;
     using System.Collections.Generic;
     using System.Linq;
+    using Core;
     using Fields;
     using Tokens;
     using Util.JetBrains.Annotations;

src/UglyToad.PdfPig/AcroForms/Fields/AcroFieldCommonInformation.cs

@@ -1,5 +1,6 @@
 namespace UglyToad.PdfPig.AcroForms.Fields
 {
+    using Core;
     using Util.JetBrains.Annotations;
 
     /// <summary>

src/UglyToad.PdfPig/Content/Catalog.cs

@@ -2,6 +2,7 @@
 {
     using System;
     using System.Collections.Generic;
+    using Core;
     using Tokens;
     using Util.JetBrains.Annotations;
 

src/UglyToad.PdfPig/Content/PageTreeNode.cs

@@ -2,6 +2,7 @@
 {
     using System;
     using System.Collections.Generic;
+    using Core;
     using Tokens;
     using Util.JetBrains.Annotations;
 

src/UglyToad.PdfPig/CrossReference/CrossReferenceTable.cs

@@ -2,6 +2,7 @@
 {
     using System;
     using System.Collections.Generic;
+    using Core;
     using Util.JetBrains.Annotations;
 
     /// <summary>

src/UglyToad.PdfPig/CrossReference/CrossReferenceTableBuilder.cs

@@ -3,6 +3,7 @@
     using System;
     using System.Collections.Generic;
     using System.Linq;
+    using Core;
     using Logging;
     using Tokens;
 

src/UglyToad.PdfPig/CrossReference/CrossReferenceTablePart.cs

@@ -1,6 +1,7 @@
 namespace UglyToad.PdfPig.CrossReference
 {
     using System.Collections.Generic;
+    using Core;
     using Tokens;
 
     /// <summary>

src/UglyToad.PdfPig/CrossReference/CrossReferenceTablePartBuilder.cs

@@ -1,6 +1,7 @@
 namespace UglyToad.PdfPig.CrossReference
 {
     using System.Collections.Generic;
+    using Core;
     using Tokens;
 
     internal class CrossReferenceTablePartBuilder

src/UglyToad.PdfPig/Encryption/IEncryptionHandler.cs

@@ -1,5 +1,6 @@
 namespace UglyToad.PdfPig.Encryption
 {
+    using Core;
     using Tokens;
 
     /// <summary>

src/UglyToad.PdfPig/Encryption/NoOpEncryptionHandler.cs

@@ -1,5 +1,6 @@
 namespace UglyToad.PdfPig.Encryption
 {
+    using Core;
     using Tokens;
 
     internal class NoOpEncryptionHandler : IEncryptionHandler

src/UglyToad.PdfPig/PdfExtensions.cs

@@ -0,0 +1,60 @@
+namespace UglyToad.PdfPig
+{
+    using System.Collections.Generic;
+    using Core;
+    using Filters;
+    using Parser.Parts;
+    using Tokenization.Scanner;
+    using Tokens;
+
+    internal static class PdfExtensions
+    {
+        /// <summary>
+        /// Try and get the entry with a given name and type or look-up the object if it's an indirect reference.
+        /// </summary>
+        internal static bool TryGet<T>(this DictionaryToken dictionary, NameToken name, IPdfTokenScanner tokenScanner, out T token) where T : IToken
+        {
+            token = default(T);
+            if (!dictionary.TryGet(name, out var t) || !(t is T typedToken))
+            {
+                if (t is IndirectReferenceToken reference)
+                {
+                    return DirectObjectFinder.TryGet(reference, tokenScanner, out token);
+                }
+
+                return false;
+            }
+
+            token = typedToken;
+            return true;
+        }
+
+        internal static T Get<T>(this DictionaryToken dictionary, NameToken name, IPdfTokenScanner scanner) where T : IToken
+        {
+            if (!dictionary.TryGet(name, out var token) || !(token is T typedToken))
+            {
+                if (!(token is IndirectReferenceToken indirectReference))
+                {
+                    throw new PdfDocumentFormatException($"Dictionary does not contain token with name {name} of type {typeof(T).Name}.");
+                }
+
+                typedToken = DirectObjectFinder.Get<T>(indirectReference, scanner);
+            }
+
+            return typedToken;
+        }
+        
+        internal static IReadOnlyList<byte> Decode(this StreamToken stream, IFilterProvider filterProvider)
+        {
+            var filters = filterProvider.GetFilters(stream.StreamDictionary);
+
+            var transform = stream.Data;
+            for (var i = 0; i < filters.Count; i++)
+            {
+                transform = filters[i].Decode(transform, stream.StreamDictionary, i);
+            }
+
+            return transform;
+        }
+    }
+}

src/UglyToad.PdfPig/Structure.cs

@@ -2,6 +2,7 @@
 {
     using System;
     using Content;
+    using Core;
     using CrossReference;
     using Tokenization.Scanner;
     using Tokens;

src/UglyToad.PdfPig/Tokenization/Scanner/IObjectLocationProvider.cs

@@ -2,6 +2,7 @@
 {
     using System;
     using System.Collections.Generic;
+    using Core;
     using CrossReference;
     using Parser.Parts;
     using Tokens;

src/UglyToad.PdfPig/Tokenization/Scanner/IPdfTokenScanner.cs

@@ -1,6 +1,7 @@
 namespace UglyToad.PdfPig.Tokenization.Scanner
 {
     using System;
+    using Core;
     using Tokens;
 
     /// <summary>

src/UglyToad.PdfPig/Tokens/NumericToken.cs

@@ -1,71 +0,0 @@
-namespace UglyToad.PdfPig.Tokens
-{
-    using System.Globalization;
-
-    /// <inheritdoc />
-    /// <summary>
-    /// PDF supports integer and real numbers. Integer objects represent mathematical integers within a certain interval centered at 0. 
-    /// Real objects  approximate mathematical real numbers, but with limited range and precision.
-    /// This token represents both types and they are used interchangeably in the specification.
-    /// </summary>
-    public class NumericToken : IDataToken<decimal>
-    {
-        internal static readonly NumericToken Zero = new NumericToken(0);
-        internal static readonly NumericToken One = new NumericToken(1);
-        internal static readonly NumericToken Two = new NumericToken(2);
-        internal static readonly NumericToken Three = new NumericToken(3);
-        internal static readonly NumericToken Four = new NumericToken(4);
-        internal static readonly NumericToken Five = new NumericToken(5);
-        internal static readonly NumericToken Six = new NumericToken(6);
-        internal static readonly NumericToken Seven = new NumericToken(7);
-        internal static readonly NumericToken Eight = new NumericToken(8);
-        internal static readonly NumericToken Nine = new NumericToken(9);
-        internal static readonly NumericToken Ten = new NumericToken(10);
-        internal static readonly NumericToken OneHundred = new NumericToken(100);
-        internal static readonly NumericToken OneThousand = new NumericToken(1000);
-
-        /// <inheritdoc />
-        public decimal Data { get; }
-
-        /// <summary>
-        /// Whether the number represented has a non-zero decimal part.
-        /// </summary>
-        public bool HasDecimalPlaces => decimal.Floor(Data) != Data;
-
-        /// <summary>
-        /// The value of this number as an <see langword="int"/>.
-        /// </summary>
-        public int Int => (int)Data;
-
-        /// <summary>
-        /// The value of this number as a <see langword="long"/>.
-        /// </summary>
-        public long Long => (long)Data;
-
-        /// <summary>
-        /// The value of this number as a <see langword="double"/>.
-        /// </summary>
-        public double Double => (double)Data;
-
-        /// <summary>
-        /// Create a <see cref="NumericToken"/>.
-        /// </summary>
-        /// <param name="value">The number to represent.</param>
-        public NumericToken(decimal value)
-        {
-            Data = value;
-        }
-
-        /// <inheritdoc />
-        public override int GetHashCode()
-        {
-            return Data.GetHashCode();
-        }
-
-        /// <inheritdoc />
-        public override string ToString()
-        {
-            return Data.ToString(NumberFormatInfo.InvariantInfo);
-        }
-    }
-}

src/UglyToad.PdfPig/UglyToad.PdfPig.csproj

@@ -52,6 +52,7 @@
   <ItemGroup>
     <ProjectReference Include="..\UglyToad.PdfPig.Core\UglyToad.PdfPig.Core.csproj" />
     <ProjectReference Include="..\UglyToad.PdfPig.Fonts\UglyToad.PdfPig.Fonts.csproj" />
+    <ProjectReference Include="..\UglyToad.PdfPig.Tokens\UglyToad.PdfPig.Tokens.csproj" />
   </ItemGroup>
 
   <ItemGroup>