Make IToken implement IEquatable<IToken>

This would allow us to deduplicate tokens, by comparing their content

src/UglyToad.PdfPig.Tokens/ArrayToken.cs

@@ -88,5 +88,26 @@
 
             return builder.ToString();
         }
+        
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is ArrayToken other))
+                return false;
+
+            if (other.Length != Length)
+                return false;
+
+            for (var index = 0; index < Length; ++index)
+            {
+                if (!Data[index].Equals(other[index]))
+                    return false;
+            }
+
+            return true;
+        }
     }
 }

src/UglyToad.PdfPig.Tokens/BooleanToken.cs

@@ -29,17 +29,6 @@
             Data = data;
         }
 
-        /// <inheritdoc />
-        public override bool Equals(object obj)
-        {
-            if (!(obj is BooleanToken other))
-            {
-                return false;
-            }
-
-            return other.Data == Data;
-        }
-        
         /// <inheritdoc />
         public override int GetHashCode()
         {
@@ -51,5 +40,19 @@
         {
             return Data.ToString();
         }
+
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is BooleanToken other))
+            {
+                return false;
+            }
+
+            return other.Data == Data;
+        }
     }
 }

src/UglyToad.PdfPig.Tokens/CommentToken.cs

@@ -25,5 +25,16 @@
         {
             return Data;
         }
+
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (!(obj is CommentToken other))
+            {
+                return false;
+            }
+
+            return other.Data == Data;
+        }
     }
 }

src/UglyToad.PdfPig.Tokens/DictionaryToken.cs

@@ -114,6 +114,26 @@
             return new DictionaryToken(result);
         }
 
+
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is DictionaryToken other))
+            {
+                return false;
+            }
+
+            if (Data.Count != other.Data.Count)
+                return false;
+
+            // TODO: Maybe consider using a sorted dictionary?
+            return Data.OrderBy(kvp => kvp.Key)
+           .SequenceEqual(other.Data.OrderBy(kvp => kvp.Key));
+        }
+
         /// <inheritdoc />
         public override string ToString()
         {

src/UglyToad.PdfPig.Tokens/EndOfLineToken.cs

@@ -13,5 +13,19 @@
         private EndOfLineToken()
         {
         }
+
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is EndOfLineToken other))
+            {
+                return false;
+            }
+
+            return true;
+        }
     }
 }

src/UglyToad.PdfPig.Tokens/HexToken.cs

@@ -138,6 +138,20 @@ namespace UglyToad.PdfPig.Tokens
             return value;
         }
 
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is HexToken other))
+            {
+                return false;
+            }
+
+            return Data == other.Data;
+        }
+
         /// <summary>
         /// Converts the binary data back to a hex string representation.
         /// </summary>

src/UglyToad.PdfPig.Tokens/IToken.cs

@@ -1,9 +1,11 @@
-namespace UglyToad.PdfPig.Tokens
+using System;
+
+namespace UglyToad.PdfPig.Tokens
 {
     /// <summary>
     /// A marker interface for tokens from the PDF file contents.
     /// </summary>
-    public interface IToken
+    public interface IToken : IEquatable<IToken>
     {
     }
 }

src/UglyToad.PdfPig.Tokens/IndirectReferenceToken.cs

@@ -21,6 +21,20 @@
             Data = data;
         }
 
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is IndirectReferenceToken other))
+            {
+                return false;
+            }
+
+            return Data.Equals(other.Data);
+        }
+
         /// <inheritdoc />
         public override string ToString()
         {

src/UglyToad.PdfPig.Tokens/InlineImageDataToken.cs

@@ -18,5 +18,30 @@
         {
             Data = data;
         }
+
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is InlineImageDataToken other))
+            {
+                return false;
+            }
+
+            if (Data.Count != other.Data.Count)
+                return false;
+
+            // Note: This maybe slow?
+            // Maybe pre calculate some sort of hash and compare that?
+            for (var index = 0; index < Data.Count; ++index)
+            {
+                if (Data[index] != other.Data[index])
+                    return false;
+            }
+
+            return true;
+        }
     }
 }

src/UglyToad.PdfPig.Tokens/NameToken.cs

@@ -58,6 +58,12 @@
             return string.Equals(Data, other?.Data);
         }
 
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            return Equals(obj as NameToken);
+        }
+
         /// <inheritdoc />
         public override int GetHashCode()
         {

src/UglyToad.PdfPig.Tokens/NullToken.cs

@@ -34,6 +34,12 @@
             return Equals(Data, other.Data);
         }
 
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            return obj is NullToken;
+        }
+
         /// <inheritdoc />
         public override int GetHashCode()
         {

src/UglyToad.PdfPig.Tokens/NumericToken.cs

@@ -106,6 +106,20 @@
             Data = value;
         }
 
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is NumericToken other))
+            {
+                return false;
+            }
+
+            return Data == other.Data;
+        }
+
         /// <inheritdoc />
         public override int GetHashCode()
         {

src/UglyToad.PdfPig.Tokens/ObjectToken.cs

@@ -37,6 +37,39 @@
             Data = data ?? throw new ArgumentNullException(nameof(data));
         }
 
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is ObjectToken other))
+            {
+                return false;
+            }
+
+            // I don't think we should compare this value...
+            // If this values are the same but this object are not the same (reference)
+            // It means that one of the obj is not from stream or one of the object have been overwritten
+            // So maybe putting an assert instead, would be a good idea?
+            if (Position != other.Position)
+            {
+                return false;
+            }
+
+            if (!Number.Equals(other.Number))
+            {
+                return false;
+            }
+
+            if (!Data.Equals(other.Data))
+            {
+                return false;
+            }
+
+            return true;
+        }
+
         /// <inheritdoc />
         public override string ToString()
         {

src/UglyToad.PdfPig.Tokens/OperatorToken.cs

@@ -195,6 +195,20 @@
             }
         }
 
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is OperatorToken other))
+            {
+                return false;
+            }
+
+            return Data == other.Data;
+        }
+
         /// <inheritdoc />
         public override int GetHashCode()
         {

src/UglyToad.PdfPig.Tokens/StreamToken.cs

@@ -29,7 +29,34 @@
             StreamDictionary = streamDictionary ?? throw new ArgumentNullException(nameof(streamDictionary));
             Data = data ?? throw new ArgumentNullException(nameof(data));
         }
-        
+
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is StreamToken other))
+            {
+                return false;
+            }
+
+            if (!StreamDictionary.Equals(other.StreamDictionary))
+            {
+                return false;
+            }
+
+            // Note: This maybe slow?
+            // Maybe pre calculate some sort of hash and compare that?
+            for (var index = 0; index < Data.Count; ++index)
+            {
+                if (Data[index] != other.Data[index])
+                    return false;
+            }
+
+            return true;
+        }
+
         /// <inheritdoc />
         public override string ToString()
         {

src/UglyToad.PdfPig.Tokens/StringToken.cs

@@ -58,6 +58,25 @@ namespace UglyToad.PdfPig.Tokens
             }
         }
 
+        /// <inheritdoc />
+        public bool Equals(IToken obj)
+        {
+            if (this == obj)
+                return true;
+
+            if (!(obj is StringToken other))
+            {
+                return false;
+            }
+
+            if (!EncodedWith.Equals(other.EncodedWith))
+            {
+                return false;
+            }
+
+            return Data.Equals(other.Data);
+        }
+
         /// <inheritdoc />
         public override string ToString()
         {