add flate filter to compress truetype font in output and fix some structure issues for pdf verification

src/UglyToad.PdfPig.Tests/Filters/FlateFilterTests.cs

@@ -0,0 +1,28 @@
+namespace UglyToad.PdfPig.Tests.Filters
+{
+    using System.Collections.Generic;
+    using System.IO;
+    using PdfPig.Filters;
+    using PdfPig.Tokens;
+    using Xunit;
+
+    public class FlateFilterTests
+    {
+        private readonly FlateFilter filter = new FlateFilter(new DecodeParameterResolver(new TestingLog()), new PngPredictor(), new TestingLog());
+
+        [Fact]
+        public void EncodeAndDecodePreservesInput()
+        {
+            var parameters = new DictionaryToken(new Dictionary<NameToken, IToken>());
+            var input = new byte[] {67, 69, 69, 10, 4, 20, 6, 19, 120, 64, 64, 64, 32};
+
+            using (var inputStream = new MemoryStream(input))
+            {
+                inputStream.Seek(0, SeekOrigin.Begin);
+                var result = filter.Encode(inputStream, parameters, 0);
+                var decoded = filter.Decode(result, parameters, 0);
+                Assert.Equal(input, decoded);
+            }
+        }
+    }
+}

src/UglyToad.PdfPig/Filters/FlateFilter.cs

@@ -109,5 +109,20 @@
                 throw;
             }
         }
+
+        public byte[] Encode(Stream input, DictionaryToken streamDictionary, int index)
+        {
+            var resx = new List<byte>{ 120, 156 };
+            using (var output = new MemoryStream())
+            using (var flater = new DeflateStream(output, CompressionMode.Compress, true))
+            {
+                input.CopyTo(flater);
+                flater.Close();
+
+                resx.AddRange(output.ToArray());
+            }
+
+            return resx.ToArray();
+        }
     }
 }

src/UglyToad.PdfPig/Tokens/HexToken.cs

@@ -2,6 +2,7 @@ namespace UglyToad.PdfPig.Tokens
 {
     using System;
     using System.Collections.Generic;
+    using System.Linq;
     using System.Text;
     using Util.JetBrains.Annotations;
 
@@ -126,5 +127,13 @@ namespace UglyToad.PdfPig.Tokens
 
             return value;
         }
+
+        /// <summary>
+        /// Converts the binary data back to a hex string representation.
+        /// </summary>
+        public string GetHexString()
+        {
+            return BitConverter.ToString(Bytes.ToArray()).Replace("-", string.Empty);
+        }
     }
 }

src/UglyToad.PdfPig/Writer/PdfDocumentBuilder.cs

@@ -20,7 +20,6 @@
     /// </summary>
     internal class PdfDocumentBuilder
     {
-        private const byte Break = (byte)'\n';
         private static readonly TrueTypeFontParser Parser = new TrueTypeFontParser();
 
         private readonly Dictionary<int, PdfPageBuilder> pages = new Dictionary<int, PdfPageBuilder>();
@@ -188,6 +187,14 @@
                 // Header
                 WriteString("%PDF-1.7", memory);
 
+                // Files with binary data should contain a 2nd comment line followed by 4 bytes with values > 127
+                memory.WriteText("%");
+                memory.WriteByte(169);
+                memory.WriteByte(205);
+                memory.WriteByte(196);
+                memory.WriteByte(210);
+                memory.WriteNewLine();
+
                 // Body
                 foreach (var font in fonts)
                 {
@@ -210,6 +217,9 @@
                     resources.Add(NameToken.Font, new IndirectReferenceToken(fontsDictionaryRef.Number));
                 }
 
+                var reserved = context.ReserveNumber();
+                var parentIndirect = new IndirectReferenceToken(new IndirectReference(reserved, 0));
+
                 var pageReferences = new List<IndirectReferenceToken>();
                 foreach (var page in pages)
                 {
@@ -220,7 +230,8 @@
                             NameToken.Resources,
                             new DictionaryToken(resources)
                         },
-                        {NameToken.MediaBox, RectangleToArray(page.Value.PageSize)}
+                        {NameToken.MediaBox, RectangleToArray(page.Value.PageSize)},
+                        {NameToken.Parent, parentIndirect}
                     };
 
                     if (page.Value.Operations.Count > 0)
@@ -244,7 +255,7 @@
                     { NameToken.Count, new NumericToken(1) }
                 });
 
-                var pagesRef = context.WriteObject(memory, pagesDictionary);
+                var pagesRef = context.WriteObject(memory, pagesDictionary, reserved);
 
                 var catalog = new DictionaryToken(new Dictionary<NameToken, IToken>
                 {
@@ -310,7 +321,7 @@
             stream.Write(bytes, 0, bytes.Length);
             if (appendBreak)
             {
-                stream.WriteByte(Break);
+                stream.WriteNewLine();
             }
         }
 

src/UglyToad.PdfPig/Writer/Standard14WritingFont.cs

@@ -1,5 +1,6 @@
 namespace UglyToad.PdfPig.Writer
 {
+    using System;
     using System.Collections.Generic;
     using System.IO;
     using Core;
@@ -75,18 +76,43 @@
 
     internal class BuilderContext
     {
+        private readonly List<int> reservedNumbers = new List<int>();
+
         public int CurrentNumber { get; private set; } = 1;
 
-        private Dictionary<IndirectReference, long> objectOffsets = new Dictionary<IndirectReference, long>();
+        private readonly Dictionary<IndirectReference, long> objectOffsets = new Dictionary<IndirectReference, long>();
         public IReadOnlyDictionary<IndirectReference, long> ObjectOffsets => objectOffsets;
 
-        public ObjectToken WriteObject(Stream stream, IToken token)
+        public ObjectToken WriteObject(Stream stream, IToken token, int? reservedNumber = null)
         {
-            var reference = new IndirectReference(CurrentNumber++, 0);
+            int number;
+            if (reservedNumber.HasValue)
+            {
+                if (!reservedNumbers.Remove(reservedNumber.Value))
+                {
+                    throw new InvalidOperationException();
+                }
+
+                number = reservedNumber.Value;
+            }
+            else
+            {
+                number = CurrentNumber++;
+            }
+
+            var reference = new IndirectReference(number, 0);
             var obj = new ObjectToken(stream.Position, reference, token);
             objectOffsets.Add(reference, obj.Position);
             TokenWriter.WriteToken(obj, stream);
             return obj;
         }
+
+        public int ReserveNumber()
+        {
+            var reserved = CurrentNumber;
+            reservedNumbers.Add(reserved);
+            CurrentNumber++;
+            return reserved;
+        }
     }
 }

src/UglyToad.PdfPig/Writer/TokenWriter.cs

@@ -2,9 +2,9 @@
 {
     using System;
     using System.Collections.Generic;
-    using System.Collections.ObjectModel;
     using System.IO;
     using System.Linq;
+    using Graphics.Operations;
     using Tokens;
     using Util;
 
@@ -17,13 +17,14 @@
         private static readonly byte[] DictionaryEnd = OtherEncodings.StringAsLatin1Bytes(">>");
 
         private static readonly byte Comment = GetByte("%");
-
-        private static readonly byte EndOfLine = OtherEncodings.StringAsLatin1Bytes("\n")[0];
-
+        
         private static readonly byte[] Eof = OtherEncodings.StringAsLatin1Bytes("%%EOF");
 
         private static readonly byte[] FalseBytes = OtherEncodings.StringAsLatin1Bytes("false");
 
+        private static readonly byte HexStart = GetByte("<");
+        private static readonly byte HexEnd = GetByte(">");
+
         private static readonly byte InUseEntry = GetByte("n");
 
         private static readonly byte NameStart = GetByte("/");
@@ -67,8 +68,9 @@
                 case DictionaryToken dictionary:
                     WriteDictionary(dictionary, outputStream);
                     break;
-                case HexToken _:
-                    throw new NotImplementedException();
+                case HexToken hex:
+                    WriteHex(hex, outputStream);
+                    break;
                 case IndirectReferenceToken reference:
                     WriteIndirectReference(reference, outputStream);
                     break;
@@ -94,6 +96,13 @@
             }
         }
 
+        private static void WriteHex(HexToken hex, Stream stream)
+        {
+            stream.WriteByte(HexStart);
+            stream.WriteText(hex.GetHexString());
+            stream.WriteByte(HexEnd);
+        }
+
         public static void WriteCrossReferenceTable(IReadOnlyDictionary<IndirectReference, long> objectOffsets, 
             ObjectToken catalogToken,
             Stream outputStream,
@@ -117,12 +126,15 @@
                 throw new NotSupportedException("Object numbers must form a contiguous range");
             }
 
-            WriteLong(min, outputStream);
+            WriteLong(0, outputStream);
             WriteWhitespace(outputStream);
-            WriteLong(max, outputStream);
+            // 1 extra for the free entry.
+            WriteLong(objectOffsets.Count + 1, outputStream);
             WriteWhitespace(outputStream);
             WriteLineBreak(outputStream);
 
+            WriteFirstXrefEmptyEntry(outputStream);
+            
             foreach (var keyValuePair in objectOffsets.OrderBy(x => x.Key.ObjectNumber))
             {
                 /*
@@ -152,10 +164,18 @@
             outputStream.Write(Trailer, 0, Trailer.Length);
             WriteLineBreak(outputStream);
 
+            var identifier = new ArrayToken(new IToken[]
+            {
+                new HexToken(Guid.NewGuid().ToString("N").ToCharArray()), 
+                new HexToken(Guid.NewGuid().ToString("N").ToCharArray()) 
+            });
+
             var trailerDictionaryData = new Dictionary<NameToken, IToken>
             {
-                {NameToken.Size, new NumericToken(objectOffsets.Count)},
-                {NameToken.Root, new IndirectReferenceToken(catalogToken.Number)}
+                // 1 for the free entry.
+                {NameToken.Size, new NumericToken(objectOffsets.Count + 1)},
+                {NameToken.Root, new IndirectReferenceToken(catalogToken.Number)},
+                {NameToken.Id, identifier}
             };
 
             if (documentInformationReference.HasValue)
@@ -310,7 +330,7 @@
 
         private static void WriteLineBreak(Stream outputStream)
         {
-            outputStream.WriteByte(EndOfLine);
+            outputStream.WriteNewLine();
         }
 
         private static void WriteLong(long value, Stream outputStream)
@@ -324,6 +344,22 @@
             outputStream.WriteByte(Whitespace);
         }
 
+        private static void WriteFirstXrefEmptyEntry(Stream outputStream)
+        {
+            /*
+             *  The first entry in the table (object number 0) is always free and has a generation number of 65,535;
+             * it is the head of the linked list of free objects. 
+             */
+
+            outputStream.WriteText(new string('0', 10));
+            outputStream.WriteWhiteSpace();
+            outputStream.WriteText("65535");
+            outputStream.WriteWhiteSpace();
+            outputStream.WriteText("f");
+            outputStream.WriteWhiteSpace();
+            outputStream.WriteNewLine();
+        }
+
         private static byte GetByte(string value)
         {
             var bytes = OtherEncodings.StringAsLatin1Bytes(value);

src/UglyToad.PdfPig/Writer/TrueTypeWritingFont.cs

@@ -5,13 +5,16 @@
     using System.IO;
     using System.Linq;
     using Core;
+    using Filters;
     using Fonts;
     using Fonts.Encodings;
     using Fonts.Exceptions;
     using Fonts.TrueType;
     using Fonts.TrueType.Tables;
     using Geometry;
+    using Logging;
     using Tokens;
+    using Util;
 
     internal class TrueTypeWritingFont : IWritingFont
     {
@@ -46,10 +49,12 @@
 
         public ObjectToken WriteFont(NameToken fontKeyName, Stream outputStream, BuilderContext context)
         {
-            var bytes = fontFileBytes;
+            var bytes = CompressBytes();
             var embeddedFile = new StreamToken(new DictionaryToken(new Dictionary<NameToken, IToken>
             {
-                { NameToken.Length, new NumericToken(bytes.Count) }
+                { NameToken.Length, new NumericToken(bytes.Length) },
+                { NameToken.Length1, new NumericToken(fontFileBytes.Count) },
+                { NameToken.Filter, new ArrayToken(new []{ NameToken.FlateDecode }) }
             }), bytes);
 
             var fileRef = context.WriteObject(outputStream, embeddedFile);
@@ -118,6 +123,17 @@
             return result;
         }
 
+        private byte[] CompressBytes()
+        {
+            using (var memoryStream = new MemoryStream(fontFileBytes.ToArray()))
+            {
+                var parameters = new DictionaryToken(new Dictionary<NameToken, IToken>());
+                var flater = new FlateFilter(new DecodeParameterResolver(new NoOpLog()), new PngPredictor(), new NoOpLog());
+                var bytes = flater.Encode(memoryStream, parameters, 0);
+                return bytes;
+            }
+        }
+
         private static ArrayToken GetBoundingBox(PdfRectangle boundingBox, decimal scaling)
         {
             return new ArrayToken(new[]