add content order text extractor and example of use

src/UglyToad.PdfPig.DocumentLayoutAnalysis/TextExtractor/ContentOrderTextExtractor.cs

@@ -0,0 +1,124 @@
+namespace UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor
+{
+    using System;
+    using System.Text;
+    using Content;
+    using Util;
+
+    /// <summary>
+    /// Extracts text from a document based on the content order in the file.
+    /// </summary>
+    public static class ContentOrderTextExtractor
+    {
+        /// <summary>
+        /// Gets a human readable representation of the text from the page based on
+        /// the letter order of the original PDF document.
+        /// </summary>
+        /// <param name="page">A page from the document.</param>
+        /// <param name="addDoubleNewline">Whether to include a double new-line when the text is likely to be a new paragraph.</param>
+        public static string GetText(Page page, bool addDoubleNewline = false)
+        {
+            var sb = new StringBuilder();
+
+            var previous = default(Letter);
+            var hasJustAddedWhitespace = false;
+            for (var i = 0; i < page.Letters.Count; i++)
+            {
+                var letter = page.Letters[i];
+
+                if (string.IsNullOrEmpty(letter.Value))
+                {
+                    continue;
+                }
+
+                if (letter.Value == " " && !hasJustAddedWhitespace)
+                {
+                    if (previous != null && IsNewline(previous, letter, page, out _))
+                    {
+                        continue;
+                    }
+
+                    sb.Append(" ");
+                    previous = letter;
+                    hasJustAddedWhitespace = true;
+                    continue;
+                }
+
+                hasJustAddedWhitespace = false;
+
+                if (previous != null && letter.Value != " ")
+                {
+                    var nwPrevious = GetNonWhitespacePrevious(page, i);
+
+                    if (IsNewline(nwPrevious, letter, page, out var isDoubleNewline))
+                    {
+                        if (previous.Value == " ")
+                        {
+                            sb.Remove(sb.Length - 1, 1);
+                        }
+
+                        sb.AppendLine();
+                        if (addDoubleNewline && isDoubleNewline)
+                        {
+                            sb.AppendLine();
+                        }
+
+                        hasJustAddedWhitespace = true;
+                    }
+                    else if (previous.Value != " ")
+                    {
+                        var gap = letter.StartBaseLine.X - previous.EndBaseLine.X;
+                        
+                        if (WhitespaceSizeStatistics.IsProbablyWhitespace(gap, previous))
+                        {
+                            sb.Append(" ");
+                            hasJustAddedWhitespace = true;
+                        }
+                    }
+                }
+
+                sb.Append(letter.Value);
+                previous = letter;
+            }
+
+            return sb.ToString();
+        }
+
+        private static Letter GetNonWhitespacePrevious(Page page, int index)
+        {
+            for (var i = index - 1; i >= 0; i--)
+            {
+                var letter = page.Letters[i];
+                if (!string.IsNullOrWhiteSpace(letter.Value))
+                {
+                    return letter;
+                }
+            }
+
+            return null;
+        }
+
+        private static bool IsNewline(Letter previous, Letter letter, Page page, out bool isDoubleNewline)
+        {
+            isDoubleNewline = false;
+
+            if (previous == null)
+            {
+                return false;
+            }
+
+            var ptSizePrevious = (int)Math.Round(page.ExperimentalAccess.GetPointSize(previous));
+            var ptSize = (int)Math.Round(page.ExperimentalAccess.GetPointSize(letter));
+            var minPtSize = ptSize < ptSizePrevious ? ptSize : ptSizePrevious;
+
+            var gap = Math.Abs(previous.StartBaseLine.Y - letter.StartBaseLine.Y);
+
+            if (gap > minPtSize * 1.7 && previous.StartBaseLine.Y > letter.StartBaseLine.Y)
+            {
+                isDoubleNewline = true;
+            }
+
+            return gap > minPtSize * 0.9;
+        }
+    }
+}

src/UglyToad.PdfPig.Fonts/TrueType/Tables/GlyphDataTable.cs

@@ -15,7 +15,7 @@
     {
         private readonly IReadOnlyList<uint> glyphOffsets;
         private readonly PdfRectangle maxGlyphBounds;
-        private readonly TrueTypeDataBytes tableBytes;
+        private TrueTypeDataBytes tableBytes;
 
         /// <inheritdoc />
         public string Tag => TrueTypeHeaderTable.Glyf;
@@ -98,6 +98,11 @@
 
         private IReadOnlyList<IGlyphDescription> ReadGlyphs()
         {
+            if (tableBytes == null)
+            {
+                throw new InvalidOperationException("Bytes cache was discarded before lazy value evaluated.");
+            }
+
             var data = tableBytes;
 
             var offsets = glyphOffsets;
@@ -149,6 +154,8 @@
                 result[compositeLocation.Key] = ReadCompositeGlyph(data, compositeLocation.Value, compositeLocations, result, emptyGlyph);
             }
 
+            tableBytes = null;
+
             return result;
         }
 

src/UglyToad.PdfPig.Tests/Integration/LaTexTests.cs

@@ -1,4 +1,6 @@
-namespace UglyToad.PdfPig.Tests.Integration
+using UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor;
+
+namespace UglyToad.PdfPig.Tests.Integration
 {
     using System;
     using System.Collections.Generic;
@@ -159,6 +161,19 @@ used per estimate, we introduce a “complement class”
Naive Bayes is often us
                 Assert.NotNull(svg);
             }
         }
+
+        [Fact]
+        public void CanExtractContentOrderText()
+        {
+            using (var document = PdfDocument.Open(GetFilename()))
+            {
+                foreach (var page in document.GetPages())
+                {
+                    var text = ContentOrderTextExtractor.GetText(page);
+                    Assert.NotNull(text);
+                }
+            }
+        }
 
         private static IReadOnlyList<AssertablePositionData> GetPdfBoxPositionData()
         {

src/UglyToad.PdfPig/Util/WhitespaceSizeStatistics.cs

@@ -0,0 +1,20 @@
+namespace UglyToad.PdfPig.Util
+{
+    using Content;
+
+    /// <summary>
+    /// Measures of whitespace size based on point size.
+    /// </summary>
+    public static class WhitespaceSizeStatistics
+    {
+        /// <summary>
+        /// Get the average whitespace sized expected for a given letter.
+        /// </summary>
+        public static double GetExpectedWhitespaceSize(Letter letter) => letter.PointSize * 0.27;
+
+        /// <summary>
+        /// Check if the measured gap is probably big enough to be a whitespace character based on the letter.
+        /// </summary>
+        public static bool IsProbablyWhitespace(double gap, Letter letter) => gap > (GetExpectedWhitespaceSize(letter) - (letter.PointSize * 0.05));
+    }
+}