quick initial draft of the hex parser. very inefficient

src/UglyToad.Pdf.Tests/Tokenization/HexStringTokenizerTests.cs

@@ -1,6 +1,7 @@
 namespace UglyToad.Pdf.Tests.Tokenization
 {
     using Pdf.Tokenization;
+    using Pdf.Tokenization.Tokens;
     using Xunit;
 
     public class HexStringTokenizerTests
@@ -21,5 +22,28 @@
             Assert.False(result);
             Assert.Null(token);
         }
+
+        [Theory]
+        [InlineData("<00>", "\0")]
+        [InlineData("<A1>", "¡")]
+        public void TokenizesHexStringsCorrectly(string s, string expected)
+        {
+            var input = StringBytesTestConverter.Convert(s);
+
+            var result = tokenizer.TryTokenize(input.First, input.Bytes, out var token);
+
+            Assert.True(result);
+
+            Assert.Equal(expected, AssertHexToken(token).Data);
+        }
+
+        private static HexToken AssertHexToken(IToken token)
+        {
+            Assert.NotNull(token);
+
+            var hexToken = Assert.IsType<HexToken>(token);
+
+            return hexToken;
+        }
     }
 }

src/UglyToad.Pdf/Tokenization/HexStringTokenizer.cs

@@ -1,5 +1,6 @@
 namespace UglyToad.Pdf.Tokenization
 {
+    using System.Text;
     using IO;
     using Parser.Parts;
     using Tokens;
@@ -15,6 +16,8 @@
                 return false;
             }
 
+            var characters = new StringBuilder();
+
             while (inputBytes.MoveNext())
             {
                 var current = inputBytes.CurrentByte;
@@ -24,18 +27,22 @@
                     continue;
                 }
 
+                if (current == '>')
+                {
+                    break;
+                }
+
                 if (!IsValidHexCharacter(current))
                 {
                     return false;
                 }
 
-                if (current == '>')
-                {
-                    return true;
-                }
+                characters.Append((char)current);
             }
 
-            return false;
+            token = new HexToken(characters.ToString());
+
+            return true;
         }
 
         private static bool IsValidHexCharacter(byte b)

src/UglyToad.Pdf/Tokenization/Tokens/HexToken.cs

@@ -0,0 +1,31 @@
+namespace UglyToad.Pdf.Tokenization.Tokens
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Text;
+
+    public class HexToken : IDataToken<string>
+    {
+        public string Data { get; }
+
+        public IReadOnlyList<byte> Bytes { get; }
+
+        public HexToken(string characters)
+        {
+            if (characters.Length % 2 != 0)
+            {
+                characters += "0";
+            }
+
+            var builder = new StringBuilder();
+            byte[] raw = new byte[characters.Length / 2];
+            for (int i = 0; i < raw.Length; i++)
+            {
+               builder.Append((char)Convert.ToByte(characters.Substring(i * 2, 2), 16));
+            }
+
+            Bytes = raw;
+            Data = builder.ToString();
+        }
+    }
+}