Handle invalid xml characters in text exporters and fix #655

src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/AltoXmlTextExporter.cs

@@ -16,11 +16,11 @@
     /// Alto 4.1 (XML) text exporter.
     /// <para>See https://github.com/altoxml/schema </para>
     /// </summary>
-    public class AltoXmlTextExporter : ITextExporter
+    public sealed class AltoXmlTextExporter : ITextExporter
     {
         private readonly IPageSegmenter pageSegmenter;
         private readonly IWordExtractor wordExtractor;
-
+        private readonly Func<string, string> invalidCharacterHandler;
         private readonly double scale;
         private readonly string indentChar;
 
@@ -33,6 +33,9 @@
         private int stringCount;
         private int glyphCount;
 
+        /// <inheritdoc/>
+        public InvalidCharStrategy InvalidCharStrategy { get; }
+
         /// <summary>
         /// Alto 4.1 (XML).
         /// <para>See https://github.com/altoxml/schema </para>
@@ -40,13 +43,50 @@
         /// <param name="wordExtractor">Extractor used to identify words in the document.</param>
         /// <param name="pageSegmenter">Segmenter used to split page into blocks.</param>
         /// <param name="scale">Scale multiplier to apply to output document, defaults to 1.</param>
-        /// <param name="indent">Character to use for indentation, defaults to tab.</param>
-        public AltoXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, double scale = 1, string indent = "\t")
+        /// <param name="indentChar">Character to use for indentation, defaults to tab.</param>
+        /// <param name="invalidCharacterHandler">How to handle invalid characters.</param>
+        public AltoXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter,
+                                   double scale, string indentChar,
+                                   Func<string, string> invalidCharacterHandler)
+            : this(wordExtractor, pageSegmenter, scale, indentChar,
+                  InvalidCharStrategy.Custom, invalidCharacterHandler)
+        { }
+
+        /// <summary>
+        /// Alto 4.1 (XML).
+        /// <para>See https://github.com/altoxml/schema </para>
+        /// </summary>
+        /// <param name="wordExtractor">Extractor used to identify words in the document.</param>
+        /// <param name="pageSegmenter">Segmenter used to split page into blocks.</param>
+        /// <param name="scale">Scale multiplier to apply to output document, defaults to 1.</param>
+        /// <param name="indentChar">Character to use for indentation, defaults to tab.</param>
+        /// <param name="invalidCharacterStrategy">How to handle invalid characters.</param>
+        public AltoXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter,
+                                   double scale = 1, string indentChar = "\t",
+                                   InvalidCharStrategy invalidCharacterStrategy = InvalidCharStrategy.DoNotCheck)
+             : this(wordExtractor, pageSegmenter, scale, indentChar,
+                  invalidCharacterStrategy, null)
+        { }
+
+        private AltoXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter,
+                           double scale, string indentChar,
+                           InvalidCharStrategy invalidCharacterStrategy,
+                           Func<string, string> invalidCharacterHandler)
         {
-            this.wordExtractor = wordExtractor ?? throw new ArgumentNullException(nameof(wordExtractor));
-            this.pageSegmenter = pageSegmenter ?? throw new ArgumentNullException(nameof(pageSegmenter));
+            this.wordExtractor = wordExtractor;
+            this.pageSegmenter = pageSegmenter;
             this.scale = scale;
-            indentChar = indent ?? string.Empty;
+            this.indentChar = indentChar ?? string.Empty;
+            InvalidCharStrategy = invalidCharacterStrategy;
+
+            if (invalidCharacterHandler is null)
+            {
+                this.invalidCharacterHandler = TextExporterHelper.GetXmlInvalidCharHandler(InvalidCharStrategy);
+            }
+            else
+            {
+                this.invalidCharacterHandler = invalidCharacterHandler;
+            }
         }
 
         /// <summary>
@@ -57,10 +97,7 @@
         public string Get(PdfDocument document, bool includePaths = false)
         {
             var altoDocument = CreateAltoDocument("unknown");
-            var altoPages = document.GetPages().Select(x => ToAltoPage(x, includePaths)).ToArray();
-
-            altoDocument.Layout.Pages = altoPages;
-
+            altoDocument.Layout.Pages = document.GetPages().Select(x => ToAltoPage(x, includePaths)).ToArray();
             return Serialize(altoDocument);
         }
 
@@ -128,8 +165,8 @@
                 {
                     Height = (float)Math.Round(page.Height * scale),    // TBD
                     Width = (float)Math.Round(page.Width * scale),      // TBD
-                    VerticalPosition = 0f,                                          // TBD
-                    HorizontalPosition = 0f,                                          // TBD
+                    VerticalPosition = 0f,                              // TBD
+                    HorizontalPosition = 0f,                            // TBD
                     ComposedBlocks = null,                              // TBD
                     GraphicalElements = null,                           // TBD
                     Illustrations = null,                               // TBD
@@ -141,9 +178,7 @@
             };
 
             var words = page.GetWords(wordExtractor);
-            var blocks = pageSegmenter.GetBlocks(words).Select(b => ToAltoTextBlock(b, page.Height)).ToArray();
-
-            altoPage.PrintSpace.TextBlock = blocks;
+            altoPage.PrintSpace.TextBlock = pageSegmenter.GetBlocks(words).Select(b => ToAltoTextBlock(b, page.Height)).ToArray();
 
             altoPage.PrintSpace.Illustrations = page.GetImages().Select(i => ToAltoIllustration(i, page.Height)).ToArray();
 
@@ -222,7 +257,6 @@
         {
             textLineCount++;
             var strings = textLine.Words
-                .Where(x => x.Text.All(XmlConvert.IsXmlChar))
                 .Select(w => ToAltoString(w, height)).ToArray();
 
             return new AltoDocument.AltoTextBlockTextLine
@@ -252,7 +286,7 @@
                 Width = (float)Math.Round(word.BoundingBox.Width * scale),
                 Glyph = glyphs,
                 Cc = string.Join("", glyphs.Select(g => 9f * (1f - g.Gc))), // from 0->1 to 9->0
-                Content = word.Text,
+                Content = invalidCharacterHandler(word.Text),
                 Language = null,
                 StyleRefs = null,
                 SubsContent = null,
@@ -272,7 +306,7 @@
                 Height = (float)Math.Round(letter.GlyphRectangle.Height * scale),
                 Width = (float)Math.Round(letter.GlyphRectangle.Width * scale),
                 Gc = 1.0f,
-                Content = letter.Value,
+                Content = invalidCharacterHandler(letter.Value),
                 Id = "P" + pageCount + "_ST" + stringCount.ToString("#00000") + "_G" + glyphCount.ToString("#00")
             };
         }
@@ -314,8 +348,8 @@
                 Processings = new[] { processing },
                 SourceImageInformation = new AltoDocument.AltoSourceImageInformation
                 {
-                    DocumentIdentifiers = new [] { documentIdentifier },
-                    FileIdentifiers = new [] { fileIdentifier },
+                    DocumentIdentifiers = new[] { documentIdentifier },
+                    FileIdentifiers = new[] { fileIdentifier },
                     FileName = fileName
                 }
             };
@@ -329,6 +363,7 @@
                 Encoding = System.Text.Encoding.UTF8,
                 Indent = true,
                 IndentChars = indentChar,
+                CheckCharacters = InvalidCharStrategy != InvalidCharStrategy.DoNotCheck,
             };
 
             using (var memoryStream = new System.IO.MemoryStream())
@@ -346,7 +381,12 @@
         {
             var serializer = new XmlSerializer(typeof(AltoDocument));
 
-            using (var reader = XmlReader.Create(xmlPath))
+            var settings = new XmlReaderSettings()
+            {
+                CheckCharacters = false
+            };
+
+            using (var reader = XmlReader.Create(xmlPath, settings))
             {
                 return (AltoDocument)serializer.Deserialize(reader);
             }

src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/HOcrTextExporter.cs

@@ -13,14 +13,14 @@
     /// hOCR v1.2 (HTML) text exporter.
     /// <para>See http://kba.cloud/hocr-spec/1.2/ </para>
     /// </summary>
-    public class HOcrTextExporter : ITextExporter
+    public sealed class HOcrTextExporter : ITextExporter
     {
         private const string XmlHeader = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\"\n\t\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n";
         private const string Hocrjs = "<script src='https://unpkg.com/hocrjs'></script>\n";
 
         private readonly IPageSegmenter pageSegmenter;
         private readonly IWordExtractor wordExtractor;
-
+        private readonly Func<string, string> invalidCharacterHandler;
         private readonly double scale;
         private readonly string indentChar;
 
@@ -32,16 +32,60 @@
         private int paraCount;
         private int imageCount;
 
+        /// <inheritdoc/>
+        public InvalidCharStrategy InvalidCharStrategy { get; }
+
         /// <summary>
         /// hOCR v1.2 (HTML)
         /// <para>See http://kba.cloud/hocr-spec/1.2/ </para>
         /// </summary>
-        public HOcrTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, double scale = 1.0, string indent = "\t")
+        /// <param name="wordExtractor">Extractor used to identify words in the document.</param>
+        /// <param name="pageSegmenter">Segmenter used to split page into blocks.</param>
+        /// <param name="scale">Scale multiplier to apply to output document, defaults to 1.</param>
+        /// <param name="indentChar">Character to use for indentation, defaults to tab.</param>
+        /// <param name="invalidCharacterHandler">How to handle invalid characters.</param>
+        public HOcrTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter,
+                                   double scale, string indentChar,
+                                   Func<string, string> invalidCharacterHandler)
+            : this(wordExtractor, pageSegmenter, scale, indentChar,
+                  InvalidCharStrategy.Custom, invalidCharacterHandler)
+        { }
+
+        /// <summary>
+        /// hOCR v1.2 (HTML)
+        /// <para>See http://kba.cloud/hocr-spec/1.2/ </para>
+        /// </summary>
+        /// <param name="wordExtractor">Extractor used to identify words in the document.</param>
+        /// <param name="pageSegmenter">Segmenter used to split page into blocks.</param>
+        /// <param name="scale">Scale multiplier to apply to output document, defaults to 1.</param>
+        /// <param name="indentChar">Character to use for indentation, defaults to tab.</param>
+        /// <param name="invalidCharacterStrategy">How to handle invalid characters.</param>
+        public HOcrTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter,
+                                   double scale = 1, string indentChar = "\t",
+                                   InvalidCharStrategy invalidCharacterStrategy = InvalidCharStrategy.DoNotCheck)
+             : this(wordExtractor, pageSegmenter, scale, indentChar,
+                  invalidCharacterStrategy, null)
+        { }
+
+        private HOcrTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter,
+                           double scale, string indentChar,
+                           InvalidCharStrategy invalidCharacterStrategy,
+                           Func<string, string> invalidCharacterHandler)
         {
             this.wordExtractor = wordExtractor;
             this.pageSegmenter = pageSegmenter;
             this.scale = scale;
-            indentChar = indent;
+            this.indentChar = indentChar ?? string.Empty;
+            InvalidCharStrategy = invalidCharacterStrategy;
+
+            if (invalidCharacterHandler is null)
+            {
+                this.invalidCharacterHandler = TextExporterHelper.GetXmlInvalidCharHandler(InvalidCharStrategy);
+            }
+            else
+            {
+                this.invalidCharacterHandler = invalidCharacterHandler;
+            }
         }
 
         /// <summary>
@@ -325,7 +369,7 @@
             }
             hocr += "'";
 
-            hocr += ">" + word.Text + "</span> ";
+            hocr += ">" + invalidCharacterHandler(word.Text) + "</span> ";
             return hocr;
         }
 

src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/InvalidCharStrategy.cs

@@ -0,0 +1,28 @@
+namespace UglyToad.PdfPig.DocumentLayoutAnalysis.Export
+{
+    /// <summary>
+    /// How to handle invalid characters.
+    /// </summary>
+    public enum InvalidCharStrategy : byte
+    {
+        /// <summary>
+        /// Custom strategy.
+        /// </summary>
+        Custom = 0,
+
+        /// <summary>
+        /// Do not check invalid character.
+        /// </summary>
+        DoNotCheck = 1,
+
+        /// <summary>
+        /// Remove invalid character.
+        /// </summary>
+        Remove = 2,
+
+        /// <summary>
+        /// Convert invalid character to hexadecimal representation.
+        /// </summary>
+        ConvertToHexadecimal = 3
+    }
+}

src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/PageXmlTextExporter.cs

@@ -3,38 +3,33 @@
     using Content;
     using Core;
     using DocumentLayoutAnalysis;
+    using Graphics;
     using Graphics.Colors;
     using PAGE;
+    using PageSegmenter;
+    using ReadingOrderDetector;
     using System;
     using System.Collections.Generic;
     using System.Linq;
     using System.Xml;
     using System.Xml.Serialization;
-    using PageSegmenter;
-    using ReadingOrderDetector;
-    using Graphics;
     using Util;
 
     /// <summary>
     /// PAGE-XML 2019-07-15 (XML) text exporter.
     /// <para>See https://github.com/PRImA-Research-Lab/PAGE-XML </para>
     /// </summary>
-    public class PageXmlTextExporter : ITextExporter
+    public sealed class PageXmlTextExporter : ITextExporter
     {
         private readonly IPageSegmenter pageSegmenter;
         private readonly IWordExtractor wordExtractor;
         private readonly IReadingOrderDetector readingOrderDetector;
-
+        private readonly Func<string, string> invalidCharacterHandler;
         private readonly double scale;
         private readonly string indentChar;
 
-        private int lineCount;
-        private int wordCount;
-        private int glyphCount;
-        private int regionCount;
-        private int groupOrderCount;
-
-        private List<PageXmlDocument.PageXmlRegionRefIndexed> orderedRegions;
+        /// <inheritdoc/>
+        public InvalidCharStrategy InvalidCharStrategy { get; }
 
         /// <summary>
         /// PAGE-XML 2019-07-15 (XML) text exporter.
@@ -42,20 +37,62 @@
         /// </summary>
         /// <param name="wordExtractor"></param>
         /// <param name="pageSegmenter"></param>
-		/// <param name="readingOrderDetector"></param>
+        /// <param name="readingOrderDetector"></param>
+        /// <param name="scale"></param>
+        /// <param name="indentChar">Indent character.</param>
+        /// <param name="invalidCharacterHandler">How to handle invalid characters.</param>
+        public PageXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter,
+                                   IReadingOrderDetector readingOrderDetector,
+                                   double scale, string indentChar,
+                                   Func<string, string> invalidCharacterHandler)
+            : this(wordExtractor, pageSegmenter, readingOrderDetector, scale, indentChar,
+                  InvalidCharStrategy.Custom, invalidCharacterHandler)
+        { }
+
+        /// <summary>
+        /// PAGE-XML 2019-07-15 (XML) text exporter.
+        /// <para>See https://github.com/PRImA-Research-Lab/PAGE-XML </para>
+        /// </summary>
+        /// <param name="wordExtractor"></param>
+        /// <param name="pageSegmenter"></param>
+        /// <param name="readingOrderDetector"></param>
         /// <param name="scale"></param>
         /// <param name="indent">Indent character.</param>
-        public PageXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, IReadingOrderDetector readingOrderDetector = null, double scale = 1.0, string indent = "\t")
+        /// <param name="invalidCharacterStrategy">How to handle invalid characters.</param>
+        public PageXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter,
+                                   IReadingOrderDetector readingOrderDetector = null,
+                                   double scale = 1.0, string indent = "\t",
+                                   InvalidCharStrategy invalidCharacterStrategy = InvalidCharStrategy.DoNotCheck)
+            : this(wordExtractor, pageSegmenter, readingOrderDetector, scale, indent,
+                  invalidCharacterStrategy, null)
+        { }
+
+        private PageXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter,
+                           IReadingOrderDetector readingOrderDetector,
+                           double scale, string indentChar,
+                           InvalidCharStrategy invalidCharacterStrategy,
+                           Func<string, string> invalidCharacterHandler)
         {
             this.wordExtractor = wordExtractor;
             this.pageSegmenter = pageSegmenter;
             this.readingOrderDetector = readingOrderDetector;
             this.scale = scale;
-            indentChar = indent;
+            this.indentChar = indentChar ?? string.Empty;
+            InvalidCharStrategy = invalidCharacterStrategy;
+
+            if (invalidCharacterHandler is null)
+            {
+                this.invalidCharacterHandler = TextExporterHelper.GetXmlInvalidCharHandler(InvalidCharStrategy);
+            }
+            else
+            {
+                this.invalidCharacterHandler = invalidCharacterHandler;
+            }
         }
 
         /// <summary>
         /// Get the PAGE-XML (XML) string of the pages layout.
+        /// <para>Not implemented, use <see cref="Get(Page)"/> instead.</para>
         /// </summary>
         /// <param name="document"></param>
         /// <param name="includePaths">Draw PdfPaths present in the page.</param>
@@ -80,26 +117,23 @@
         /// <param name="includePaths">Draw PdfPaths present in the page.</param>
         public string Get(Page page, bool includePaths)
         {
-            lineCount = 0;
-            wordCount = 0;
-            glyphCount = 0;
-            regionCount = 0;
-            groupOrderCount = 0;
-            orderedRegions = new List<PageXmlDocument.PageXmlRegionRefIndexed>();
+            PageXmlData data = new PageXmlData();
+
+            DateTime utcNow = DateTime.UtcNow;
 
             PageXmlDocument pageXmlDocument = new PageXmlDocument()
             {
                 Metadata = new PageXmlDocument.PageXmlMetadata()
                 {
-                    Created = DateTime.UtcNow,
-                    LastChange = DateTime.UtcNow,
+                    Created = utcNow,
+                    LastChange = utcNow,
                     Creator = "PdfPig",
                     Comments = pageSegmenter.GetType().Name + "|" + wordExtractor.GetType().Name,
                 },
                 PcGtsId = "pc-" + page.GetHashCode()
             };
 
-            pageXmlDocument.Page = ToPageXmlPage(page, includePaths);
+            pageXmlDocument.Page = ToPageXmlPage(page, data, includePaths);
 
             return Serialize(pageXmlDocument);
         }
@@ -151,17 +185,17 @@
         /// </summary>
         private string ToRgbEncoded(IColor color)
         {
-            var rgb = color.ToRGBValues();
-            int red = (int)Math.Round(255f * (float)rgb.r);
-            int green = 256 * (int)Math.Round(255f * (float)rgb.g);
-            int blue = 65536 * (int)Math.Round(255f * (float)rgb.b);
+            var (r, g, b) = color.ToRGBValues();
+            int red = Convert.ToByte(255.0 * r);
+            int green = 256 * Convert.ToByte(255.0 * g);
+            int blue = 65536 * Convert.ToByte(255.0 * b);
             int sum = red + green + blue;
 
             // as per below, red and blue order might be inverted... var colorWin = System.Drawing.Color.FromArgb(sum);
             return sum.ToString();
         }
 
-        private PageXmlDocument.PageXmlPage ToPageXmlPage(Page page, bool includePaths)
+        private PageXmlDocument.PageXmlPage ToPageXmlPage(Page page, PageXmlData data, bool includePaths)
         {
             var pageXmlPage = new PageXmlDocument.PageXmlPage
             {
@@ -182,16 +216,17 @@
                     blocks = readingOrderDetector.Get(blocks).ToList();
                 }
 
-                regions.AddRange(blocks.Select(b => ToPageXmlTextRegion(b, page.Width, page.Height)));
+                regions.AddRange(blocks.Select(b => ToPageXmlTextRegion(b, data, page.Width, page.Height)));
 
-                if (orderedRegions.Count > 0)
+                if (data.OrderedRegions.Count > 0)
                 {
+                    data.GroupOrdersCount++;
                     pageXmlPage.ReadingOrder = new PageXmlDocument.PageXmlReadingOrder()
                     {
                         Item = new PageXmlDocument.PageXmlOrderedGroup()
                         {
-                            Items = orderedRegions.ToArray(),
-                            Id = "g" + groupOrderCount++
+                            Items = data.OrderedRegions.ToArray(),
+                            Id = "g" + data.GroupOrdersCount
                         }
                     };
                 }
@@ -200,14 +235,14 @@
             var images = page.GetImages().ToList();
             if (images.Count > 0)
             {
-                regions.AddRange(images.Select(i => ToPageXmlImageRegion(i, page.Width, page.Height)));
+                regions.AddRange(images.Select(i => ToPageXmlImageRegion(i, data, page.Width, page.Height)));
             }
 
             if (includePaths)
             {
                 foreach (var path in page.ExperimentalAccess.Paths)
                 {
-                    var graphicalElement = ToPageXmlLineDrawingRegion(path, page.Width, page.Height);
+                    var graphicalElement = ToPageXmlLineDrawingRegion(path, data, page.Width, page.Height);
 
                     if (graphicalElement != null)
                     {
@@ -220,40 +255,40 @@
             return pageXmlPage;
         }
 
-        private PageXmlDocument.PageXmlLineDrawingRegion ToPageXmlLineDrawingRegion(PdfPath pdfPath, double pageWidth, double pageHeight)
+        private PageXmlDocument.PageXmlLineDrawingRegion ToPageXmlLineDrawingRegion(PdfPath pdfPath, PageXmlData data, double pageWidth, double pageHeight)
         {
             var bbox = pdfPath.GetBoundingRectangle();
             if (bbox.HasValue)
             {
-                regionCount++;
+                data.RegionsCount++;
                 return new PageXmlDocument.PageXmlLineDrawingRegion()
                 {
                     Coords = ToCoords(bbox.Value, pageWidth, pageHeight),
-                    Id = "r" + regionCount
+                    Id = "r" + data.RegionsCount
                 };
             }
             return null;
         }
 
-        private PageXmlDocument.PageXmlImageRegion ToPageXmlImageRegion(IPdfImage pdfImage, double pageWidth, double pageHeight)
+        private PageXmlDocument.PageXmlImageRegion ToPageXmlImageRegion(IPdfImage pdfImage, PageXmlData data, double pageWidth, double pageHeight)
         {
-            regionCount++;
+            data.RegionsCount++;
             var bbox = pdfImage.Bounds;
             return new PageXmlDocument.PageXmlImageRegion()
             {
                 Coords = ToCoords(bbox, pageWidth, pageHeight),
-                Id = "r" + regionCount
+                Id = "r" + data.RegionsCount
             };
         }
 
-        private PageXmlDocument.PageXmlTextRegion ToPageXmlTextRegion(TextBlock textBlock, double pageWidth, double pageHeight)
+        private PageXmlDocument.PageXmlTextRegion ToPageXmlTextRegion(TextBlock textBlock, PageXmlData data, double pageWidth, double pageHeight)
         {
-            regionCount++;
-            string regionId = "r" + regionCount;
+            data.RegionsCount++;
+            string regionId = "r" + data.RegionsCount;
 
             if (readingOrderDetector != null && textBlock.ReadingOrder > -1)
             {
-                orderedRegions.Add(new PageXmlDocument.PageXmlRegionRefIndexed()
+                data.OrderedRegions.Add(new PageXmlDocument.PageXmlRegionRefIndexed()
                 {
                     RegionRef = regionId,
                     Index = textBlock.ReadingOrder
@@ -264,40 +299,58 @@
             {
                 Coords = ToCoords(textBlock.BoundingBox, pageWidth, pageHeight),
                 Type = PageXmlDocument.PageXmlTextSimpleType.Paragraph,
-                TextLines = textBlock.TextLines.Select(l => ToPageXmlTextLine(l, pageWidth, pageHeight)).ToArray(),
-                TextEquivs = new[] { new PageXmlDocument.PageXmlTextEquiv() { Unicode = textBlock.Text } },
+                TextLines = textBlock.TextLines.Select(l => ToPageXmlTextLine(l, data, pageWidth, pageHeight)).ToArray(),
+                TextEquivs = new[]
+                {
+                    new PageXmlDocument.PageXmlTextEquiv()
+                    {
+                        Unicode = invalidCharacterHandler(textBlock.Text)
+                    }
+                },
                 Id = regionId
             };
         }
 
-        private PageXmlDocument.PageXmlTextLine ToPageXmlTextLine(TextLine textLine, double pageWidth, double pageHeight)
+        private PageXmlDocument.PageXmlTextLine ToPageXmlTextLine(TextLine textLine, PageXmlData data, double pageWidth, double pageHeight)
         {
-            lineCount++;
+            data.LinesCount++;
             return new PageXmlDocument.PageXmlTextLine()
             {
                 Coords = ToCoords(textLine.BoundingBox, pageWidth, pageHeight),
                 Production = PageXmlDocument.PageXmlProductionSimpleType.Printed,
-                Words = textLine.Words.Select(w => ToPageXmlWord(w, pageWidth, pageHeight)).ToArray(),
-                TextEquivs = new[] { new PageXmlDocument.PageXmlTextEquiv() { Unicode = textLine.Text } },
-                Id = "l" + lineCount
+                Words = textLine.Words.Select(w => ToPageXmlWord(w, data, pageWidth, pageHeight)).ToArray(),
+                TextEquivs = new[]
+                {
+                    new PageXmlDocument.PageXmlTextEquiv()
+                    {
+                        Unicode = invalidCharacterHandler(textLine.Text)
+                    }
+                },
+                Id = "l" + data.LinesCount
             };
         }
 
-        private PageXmlDocument.PageXmlWord ToPageXmlWord(Word word, double pageWidth, double pageHeight)
+        private PageXmlDocument.PageXmlWord ToPageXmlWord(Word word, PageXmlData data, double pageWidth, double pageHeight)
         {
-            wordCount++;
+            data.WordsCount++;
             return new PageXmlDocument.PageXmlWord()
             {
                 Coords = ToCoords(word.BoundingBox, pageWidth, pageHeight),
-                Glyphs = word.Letters.Select(l => ToPageXmlGlyph(l, pageWidth, pageHeight)).ToArray(),
-                TextEquivs = new[] { new PageXmlDocument.PageXmlTextEquiv() { Unicode = word.Text } },
-                Id = "w" + wordCount
+                Glyphs = word.Letters.Select(l => ToPageXmlGlyph(l, data, pageWidth, pageHeight)).ToArray(),
+                TextEquivs = new[]
+                {
+                    new PageXmlDocument.PageXmlTextEquiv()
+                    {
+                        Unicode = invalidCharacterHandler(word.Text)
+                    }
+                },
+                Id = "w" + data.WordsCount
             };
         }
 
-        private PageXmlDocument.PageXmlGlyph ToPageXmlGlyph(Letter letter, double pageWidth, double pageHeight)
+        private PageXmlDocument.PageXmlGlyph ToPageXmlGlyph(Letter letter, PageXmlData data, double pageWidth, double pageHeight)
         {
-            glyphCount++;
+            data.GlyphsCount++;
             return new PageXmlDocument.PageXmlGlyph()
             {
                 Coords = ToCoords(letter.GlyphRectangle, pageWidth, pageHeight),
@@ -309,8 +362,14 @@
                     FontFamily = letter.FontName,
                     TextColourRgb = ToRgbEncoded(letter.Color),
                 },
-                TextEquivs = new[] { new PageXmlDocument.PageXmlTextEquiv() { Unicode = letter.Value } },
-                Id = "c" + glyphCount
+                TextEquivs = new[]
+                {
+                    new PageXmlDocument.PageXmlTextEquiv()
+                    {
+                        Unicode = invalidCharacterHandler(letter.Value)
+                    }
+                },
+                Id = "c" + data.GlyphsCount
             };
         }
 
@@ -322,6 +381,7 @@
                 Encoding = System.Text.Encoding.UTF8,
                 Indent = true,
                 IndentChars = indentChar,
+                CheckCharacters = InvalidCharStrategy != InvalidCharStrategy.DoNotCheck,
             };
 
             using (var memoryStream = new System.IO.MemoryStream())
@@ -339,10 +399,34 @@
         {
             XmlSerializer serializer = new XmlSerializer(typeof(PageXmlDocument));
 
-            using (var reader = XmlReader.Create(xmlPath))
+            var settings = new XmlReaderSettings()
+            {
+                CheckCharacters = false
+            };
+
+            using (var reader = XmlReader.Create(xmlPath, settings))
             {
                 return (PageXmlDocument)serializer.Deserialize(reader);
             }
         }
+
+        /// <summary>
+        /// Class to keep track of a page data.
+        /// </summary>
+        private sealed class PageXmlData
+        {
+            public PageXmlData()
+            {
+                OrderedRegions = new List<PageXmlDocument.PageXmlRegionRefIndexed>();
+            }
+
+            public int LinesCount { get; set; }
+            public int WordsCount { get; set; }
+            public int GlyphsCount { get; set; }
+            public int RegionsCount { get; set; }
+            public int GroupOrdersCount { get; set; }
+
+            public List<PageXmlDocument.PageXmlRegionRefIndexed> OrderedRegions { get; }
+        }
     }
 }

src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/SvgTextExporter.cs

@@ -13,38 +13,72 @@
     /// <summary>
     /// Exports a page as an SVG.
     /// </summary>
-    public class SvgTextExporter : ITextExporter
+    public sealed class SvgTextExporter : ITextExporter
     {
-        private const int Rounding = 4;
+        private readonly Func<string, string> invalidCharacterHandler;
 
         private static readonly Dictionary<string, string> Fonts = new Dictionary<string, string>()
         {
             { "ArialMT", "Arial Rounded MT Bold" }
         };
 
+        /// <summary>
+        /// Used to round numbers.
+        /// </summary>
+        public int Rounding { get; } = 4;
+
+        /// <summary>
+        /// <inheritdoc/>
+        /// Not in use.
+        /// </summary>
+        public InvalidCharStrategy InvalidCharStrategy { get; }
+
+        /// <summary>
+        /// Svg text exporter.
+        /// </summary>
+        /// <param name="invalidCharacterHandler">How to handle invalid characters.</param>
+        public SvgTextExporter(Func<string, string> invalidCharacterHandler)
+            : this(InvalidCharStrategy.Custom, invalidCharacterHandler)
+        { }
+
+        /// <summary>
+        /// Svg text exporter.
+        /// </summary>
+        /// <param name="invalidCharacterStrategy">How to handle invalid characters.</param>
+        public SvgTextExporter(InvalidCharStrategy invalidCharacterStrategy = InvalidCharStrategy.DoNotCheck)
+            : this(invalidCharacterStrategy, null)
+        { }
+
+        private SvgTextExporter(InvalidCharStrategy invalidCharacterStrategy, Func<string, string> invalidCharacterHandler)
+        {
+            InvalidCharStrategy = invalidCharacterStrategy;
+
+            if (invalidCharacterHandler is null)
+            {
+                this.invalidCharacterHandler = TextExporterHelper.GetXmlInvalidCharHandler(InvalidCharStrategy);
+            }
+            else
+            {
+                this.invalidCharacterHandler = invalidCharacterHandler;
+            }
+        }
         /// <summary>
         /// Get the page contents as an SVG.
         /// </summary>
         public string Get(Page page)
         {
-            var builder = new StringBuilder($"<svg width='{page.Width}' height='{page.Height}'><g transform=\"scale(1, 1) translate(0, 0)\">");
+            var builder = new StringBuilder($"<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width='{Math.Round(page.Width, Rounding)}' height='{Math.Round(page.Height, Rounding)}'>\n<g transform=\"scale(1, 1) translate(0, 0)\">\n");
 
-            var paths = page.ExperimentalAccess.Paths;
-            foreach (var path in paths)
+            foreach (var path in page.ExperimentalAccess.Paths)
             {
-                if (path.IsClipping)
+                if (!path.IsClipping)
                 {
-                    //var svg = PathToSvg(path, page.Height);
-                    //svg = svg.Replace("stroke='black'", "stroke='yellow'");
-                    //builder.Append(svg);
-                }
-                else
-                {
-                    builder.Append(PathToSvg(path, page.Height));
+                    builder.AppendLine(PathToSvg(path, page.Height));
                 }
             }
 
             var doc = new XmlDocument();
+
             foreach (var letter in page.Letters)
             {
                 builder.Append(LetterToSvg(letter, page.Height, doc));
@@ -54,7 +88,7 @@
             return builder.ToString();
         }
 
-        private static string LetterToSvg(Letter l, double height, XmlDocument doc)
+        private string LetterToSvg(Letter l, double height, XmlDocument doc)
         {
             string fontFamily = GetFontFamily(l.FontName, out string style, out string weight);
             string rotation = "";
@@ -131,10 +165,10 @@
             return fontName;
         }
 
-        private static string XmlEscape(Letter letter, XmlDocument doc)
+        private string XmlEscape(Letter letter, XmlDocument doc)
         {
             XmlNode node = doc.CreateElement("root");
-            node.InnerText = letter.Value;
+            node.InnerText = invalidCharacterHandler(letter.Value);
             return node.InnerXml;
         }
 
@@ -146,7 +180,7 @@
             }
 
             var (r, g, b) = color.ToRGBValues();
-            return $"rgb({Math.Ceiling(r * 255)},{Math.Ceiling(g * 255)},{Math.Ceiling(b * 255)})";
+            return $"rgb({Convert.ToByte(r * 255)},{Convert.ToByte(g * 255)},{Convert.ToByte(b * 255)})";
         }
 
         private static string PathToSvg(PdfPath p, double height)
@@ -214,12 +248,11 @@
             }
 
             string fillColor = " fill='none'";
-            string fillRule = "";
+            const string fillRule = ""; // For further dev
 
             if (p.IsFilled)
             {
                 fillColor = $" fill='{ColorToSvg(p.FillColor)}'";
-                //if (p.FillingRule == FillingRule.EvenOdd) fillRule = " fill-rule='evenodd'";
             }
 
             var path = $"<path d='{glyph}'{fillColor}{fillRule}{strokeColor}{strokeWidth}{dashArray}{capStyle}{jointStyle}></path>";

src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/TextExporterHelper.cs

@@ -0,0 +1,71 @@
+namespace UglyToad.PdfPig.DocumentLayoutAnalysis.Export
+{
+    using System;
+    using System.Text;
+    using System.Xml;
+
+    internal static class TextExporterHelper
+    {
+        public static Func<string, string> GetXmlInvalidCharHandler(InvalidCharStrategy invalidCharacterStrategy)
+        {
+            switch (invalidCharacterStrategy)
+            {
+                case InvalidCharStrategy.DoNotCheck:
+                    return new Func<string, string>(s => s);
+
+                case InvalidCharStrategy.Remove:
+                    return new Func<string, string>(s =>
+                    {
+                        // https://stackoverflow.com/a/17735649
+                        if (string.IsNullOrEmpty(s))
+                        {
+                            return s;
+                        }
+
+                        int length = s.Length;
+                        StringBuilder stringBuilder = new StringBuilder(length);
+                        for (int i = 0; i < length; ++i)
+                        {
+                            if (XmlConvert.IsXmlChar(s[i]))
+                            {
+                                stringBuilder.Append(s[i]);
+                            }
+                        }
+
+                        return stringBuilder.ToString();
+                    });
+
+                case InvalidCharStrategy.ConvertToHexadecimal:
+                    return new Func<string, string>(s =>
+                    {
+                        // Adapted from https://stackoverflow.com/a/17735649
+                        if (string.IsNullOrEmpty(s))
+                        {
+                            return s;
+                        }
+
+                        int length = s.Length;
+                        StringBuilder stringBuilder = new StringBuilder(length);
+                        for (int i = 0; i < length; ++i)
+                        {
+                            if (XmlConvert.IsXmlChar(s[i]))
+                            {
+                                stringBuilder.Append(s[i]);
+                            }
+                            else
+                            {
+                                byte[] bytes = Encoding.UTF8.GetBytes(s[i].ToString());
+                                string hexString = BitConverter.ToString(bytes);
+                                stringBuilder.Append("0x").Append(hexString);
+                            }
+                        }
+
+                        return stringBuilder.ToString();
+                    });
+
+                default:
+                    throw new NotImplementedException("TODO");
+            }
+        }
+    }
+}

src/UglyToad.PdfPig.Tests/Integration/AltoXmlTextExporterTests.cs

@@ -0,0 +1,189 @@
+namespace UglyToad.PdfPig.Tests.Integration
+{
+    using System;
+    using System.IO;
+    using System.Linq;
+    using System.Text;
+    using System.Xml;
+    using UglyToad.PdfPig.DocumentLayoutAnalysis.Export;
+    using UglyToad.PdfPig.DocumentLayoutAnalysis.PageSegmenter;
+    using UglyToad.PdfPig.Util;
+    using Xunit;
+
+    public class AltoXmlTextExporterTests
+    {
+        [Fact]
+        public void Issue655NoCheckStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new AltoXmlTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance);
+
+            Assert.Equal(InvalidCharStrategy.DoNotCheck, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.nocheck.altoxml.xml", xml);
+
+                var pageXml = AltoXmlTextExporter.Deserialize("issue655.nocheck.altoxml.xml");
+
+                var textRegions = pageXml.Layout.Pages[0].PrintSpace.TextBlock;
+                Assert.Single(textRegions);
+
+                var textLines = textRegions.Single().TextLines;
+                Assert.Single(textLines);
+
+                var strings = textLines.Single().Strings;
+                Assert.Equal(2, strings.Length);
+
+                Assert.Equal("TM", strings[0].Content);
+                Assert.Equal("1\u00062345\u0006678\u0006ABC", strings[1].Content); // no check strategy, contains invalid xml chars
+            }
+        }
+
+        [Fact]
+        public void Issue655RemoveStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new AltoXmlTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance,
+                invalidCharacterStrategy: InvalidCharStrategy.Remove);
+
+            Assert.Equal(InvalidCharStrategy.Remove, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.remove.altoxml.xml", xml);
+
+                var pageXml = AltoXmlTextExporter.Deserialize("issue655.remove.altoxml.xml");
+
+                var textRegions = pageXml.Layout.Pages[0].PrintSpace.TextBlock;
+                Assert.Single(textRegions);
+
+                var textLines = textRegions.Single().TextLines;
+                Assert.Single(textLines);
+
+                var strings = textLines.Single().Strings;
+                Assert.Equal(2, strings.Length);
+
+                Assert.Equal("TM", strings[0].Content);
+                Assert.Equal("12345678ABC", strings[1].Content);
+            }
+        }
+
+        [Fact]
+        public void Issue655ConvertToHexadecimalStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new AltoXmlTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance,
+                invalidCharacterStrategy: InvalidCharStrategy.ConvertToHexadecimal);
+
+            Assert.Equal(InvalidCharStrategy.ConvertToHexadecimal, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.hex.altoxml.xml", xml);
+
+                var pageXml = AltoXmlTextExporter.Deserialize("issue655.hex.altoxml.xml");
+
+                var textRegions = pageXml.Layout.Pages[0].PrintSpace.TextBlock;
+                Assert.Single(textRegions);
+
+                var textLines = textRegions.Single().TextLines;
+                Assert.Single(textLines);
+
+                var strings = textLines.Single().Strings;
+                Assert.Equal(2, strings.Length);
+
+                Assert.Equal("TM", strings[0].Content);
+                Assert.Equal("10x0623450x066780x06ABC", strings[1].Content);
+            }
+        }
+
+        [Fact]
+        public void Issue655CustomStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new AltoXmlTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance, 1.0, "\t",
+                new Func<string, string>(s =>
+                {
+                    // Adapted from https://stackoverflow.com/a/17735649
+                    if (string.IsNullOrEmpty(s))
+                    {
+                        return s;
+                    }
+
+                    int length = s.Length;
+                    StringBuilder stringBuilder = new StringBuilder(length);
+                    for (int i = 0; i < length; ++i)
+                    {
+                        if (XmlConvert.IsXmlChar(s[i]))
+                        {
+                            stringBuilder.Append(s[i]);
+                        }
+                        else
+                        {
+                            stringBuilder.Append("!?");
+                        }
+                    }
+
+                    return stringBuilder.ToString();
+                }));
+
+            Assert.Equal(InvalidCharStrategy.Custom, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.custom.altoxml.xml", xml);
+
+                var pageXml = AltoXmlTextExporter.Deserialize("issue655.custom.altoxml.xml");
+
+                var textRegions = pageXml.Layout.Pages[0].PrintSpace.TextBlock;
+                Assert.Single(textRegions);
+
+                var textLines = textRegions.Single().TextLines;
+                Assert.Single(textLines);
+
+                var strings = textLines.Single().Strings;
+                Assert.Equal(2, strings.Length);
+
+                Assert.Equal("TM", strings[0].Content);
+                Assert.Equal("1!?2345!?678!?ABC", strings[1].Content);
+            }
+        }
+    }
+}

src/UglyToad.PdfPig.Tests/Integration/HOcrTextExporterTests.cs

@@ -0,0 +1,144 @@
+namespace UglyToad.PdfPig.Tests.Integration
+{
+    using System;
+    using System.IO;
+    using System.Text;
+    using System.Xml;
+    using UglyToad.PdfPig.DocumentLayoutAnalysis.Export;
+    using UglyToad.PdfPig.DocumentLayoutAnalysis.PageSegmenter;
+    using UglyToad.PdfPig.Util;
+    using Xunit;
+
+    public class HOcrTextExporterTests
+    {
+        [Fact]
+        public void Issue655NoCheckStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new HOcrTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance);
+
+            Assert.Equal(InvalidCharStrategy.DoNotCheck, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page, useHocrjs: true);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.nocheck.hocr.html", xml);
+
+                string rawText = File.ReadAllText("issue655.nocheck.hocr.html");
+                Assert.Contains("1\u00062345\u0006678\u0006ABC", rawText);
+            }
+        }
+
+        [Fact]
+        public void Issue655RemoveStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new HOcrTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance,
+                invalidCharacterStrategy: InvalidCharStrategy.Remove);
+
+            Assert.Equal(InvalidCharStrategy.Remove, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page, useHocrjs: true);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.remove.hocr.html", xml);
+
+                string rawText = File.ReadAllText("issue655.remove.hocr.html");
+                Assert.Contains("12345678ABC", rawText);
+            }
+        }
+
+        [Fact]
+        public void Issue655ConvertToHexadecimalStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new HOcrTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance,
+                invalidCharacterStrategy: InvalidCharStrategy.ConvertToHexadecimal);
+
+            Assert.Equal(InvalidCharStrategy.ConvertToHexadecimal, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page, useHocrjs: true);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.hex.hocr.html", xml);
+
+                string rawText = File.ReadAllText("issue655.hex.hocr.html");
+                Assert.Contains("10x0623450x066780x06ABC", rawText);
+            }
+        }
+
+        [Fact]
+        public void Issue655CustomStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new HOcrTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance, 1.0, "\t",
+                new Func<string, string>(s =>
+                {
+                    // Adapted from https://stackoverflow.com/a/17735649
+                    if (string.IsNullOrEmpty(s))
+                    {
+                        return s;
+                    }
+
+                    int length = s.Length;
+                    StringBuilder stringBuilder = new StringBuilder(length);
+                    for (int i = 0; i < length; ++i)
+                    {
+                        if (XmlConvert.IsXmlChar(s[i]))
+                        {
+                            stringBuilder.Append(s[i]);
+                        }
+                        else
+                        {
+                            stringBuilder.Append("!?");
+                        }
+                    }
+
+                    return stringBuilder.ToString();
+                }));
+
+            Assert.Equal(InvalidCharStrategy.Custom, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page, useHocrjs: true);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.custom.hocr.html", xml);
+
+                string rawText = File.ReadAllText("issue655.custom.hocr.html");
+                Assert.Contains("1!?2345!?678!?ABC", rawText);
+            }
+        }
+    }
+}

src/UglyToad.PdfPig.Tests/Integration/PageXmlTextExporterTests.cs

@@ -1,15 +1,17 @@
 namespace UglyToad.PdfPig.Tests.Integration
 {
-    using System;
-    using System.Collections.Generic;
-    using System.IO;
-    using System.Text;
-    using System.Text.RegularExpressions;
     using DocumentLayoutAnalysis.Export;
     using DocumentLayoutAnalysis.PageSegmenter;
     using DocumentLayoutAnalysis.ReadingOrderDetector;
     using PdfPig.Core;
     using PdfPig.Util;
+    using System;
+    using System.IO;
+    using System.Linq;
+    using System.Text;
+    using System.Text.RegularExpressions;
+    using System.Xml;
+    using UglyToad.PdfPig.DocumentLayoutAnalysis.Export.PAGE;
     using Xunit;
 
     public class PageXmlTextExporterTests
@@ -26,14 +28,11 @@
                 RecursiveXYCut.Instance,
                 UnsupervisedReadingOrderDetector.Instance);
 
-            string xml;
             using (var document = PdfDocument.Open(GetFilename()))
             {
                 var page = document.GetPage(1);
-                xml = pageXmlTextExporter.Get(page);
+                return pageXmlTextExporter.Get(page);
             }
-
-            return xml;
         }
 
         [Fact]
@@ -88,7 +87,173 @@
             Assert.Equal("1,199", PageXmlTextExporter.PointToString(topLeftPagePoint, pageWidth, pageHeight));
             Assert.Equal("1,1", PageXmlTextExporter.PointToString(bottomLeftPagePoint, pageWidth, pageHeight));
             Assert.Equal("99,1", PageXmlTextExporter.PointToString(bottomRightPagePoint, pageWidth, pageHeight));
-            Assert.Equal($"60,140", PageXmlTextExporter.PointToString(normalPoint, pageWidth, pageHeight));
+            Assert.Equal("60,140", PageXmlTextExporter.PointToString(normalPoint, pageWidth, pageHeight));
+        }
+
+        [Fact]
+        public void Issue655NoCheckStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new PageXmlTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance,
+                UnsupervisedReadingOrderDetector.Instance);
+
+            Assert.Equal(InvalidCharStrategy.DoNotCheck, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.nocheck.pagexml.xml", xml);
+
+                var pageXml = PageXmlTextExporter.Deserialize("issue655.nocheck.pagexml.xml");
+
+                var textRegions = pageXml.Page.Items.OfType<PageXmlDocument.PageXmlTextRegion>().ToArray();
+                Assert.Single(textRegions);
+
+                var textEquivs = textRegions.Single().TextEquivs;
+                Assert.Single(textEquivs);
+
+                string unicode = textEquivs.Single().Unicode;
+                Assert.Equal("TM 1\u00062345\u0006678\u0006ABC", unicode); // no check strategy, contains invalid xml chars
+            }
+        }
+
+        [Fact]
+        public void Issue655RemoveStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new PageXmlTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance,
+                UnsupervisedReadingOrderDetector.Instance,
+                invalidCharacterStrategy: InvalidCharStrategy.Remove);
+
+            Assert.Equal(InvalidCharStrategy.Remove, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.remove.pagexml.xml", xml);
+
+                var pageXml = PageXmlTextExporter.Deserialize("issue655.remove.pagexml.xml");
+
+                var textRegions = pageXml.Page.Items.OfType<PageXmlDocument.PageXmlTextRegion>().ToArray();
+                Assert.Single(textRegions);
+
+                var textEquivs = textRegions.Single().TextEquivs;
+                Assert.Single(textEquivs);
+
+                string unicode = textEquivs.Single().Unicode;
+                Assert.Equal("TM 12345678ABC", unicode);
+            }
+        }
+
+        [Fact]
+        public void Issue655ConvertToHexadecimalStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new PageXmlTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance,
+                UnsupervisedReadingOrderDetector.Instance,
+                invalidCharacterStrategy: InvalidCharStrategy.ConvertToHexadecimal);
+
+            Assert.Equal(InvalidCharStrategy.ConvertToHexadecimal, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.hex.pagexml.xml", xml);
+
+                var pageXml = PageXmlTextExporter.Deserialize("issue655.hex.pagexml.xml");
+
+                var textRegions = pageXml.Page.Items.OfType<PageXmlDocument.PageXmlTextRegion>().ToArray();
+                Assert.Single(textRegions);
+
+                var textEquivs = textRegions.Single().TextEquivs;
+                Assert.Single(textEquivs);
+
+                string unicode = textEquivs.Single().Unicode;
+                Assert.Equal("TM 10x0623450x066780x06ABC", unicode);
+            }
+        }
+
+        [Fact]
+        public void Issue655CustomStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new PageXmlTextExporter(
+                DefaultWordExtractor.Instance,
+                RecursiveXYCut.Instance,
+                UnsupervisedReadingOrderDetector.Instance, 1.0, "\t",
+                new Func<string, string>(s =>
+                {
+                    // Adapted from https://stackoverflow.com/a/17735649
+                    if (string.IsNullOrEmpty(s))
+                    {
+                        return s;
+                    }
+
+                    int length = s.Length;
+                    StringBuilder stringBuilder = new StringBuilder(length);
+                    for (int i = 0; i < length; ++i)
+                    {
+                        if (XmlConvert.IsXmlChar(s[i]))
+                        {
+                            stringBuilder.Append(s[i]);
+                        }
+                        else
+                        {
+                            stringBuilder.Append("!?");
+                        }
+                    }
+
+                    return stringBuilder.ToString();
+                }));
+
+            Assert.Equal(InvalidCharStrategy.Custom, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.custom.pagexml.xml", xml);
+
+                var pageXml = PageXmlTextExporter.Deserialize("issue655.custom.pagexml.xml");
+
+                var textRegions = pageXml.Page.Items.OfType<PageXmlDocument.PageXmlTextRegion>().ToArray();
+                Assert.Single(textRegions);
+
+                var textEquivs = textRegions.Single().TextEquivs;
+                Assert.Single(textEquivs);
+
+                string unicode = textEquivs.Single().Unicode;
+                Assert.Equal("TM 1!?2345!?678!?ABC", unicode);
+            }
         }
     }
 }

src/UglyToad.PdfPig.Tests/Integration/SvgTextExporterTests.cs

@@ -0,0 +1,153 @@
+namespace UglyToad.PdfPig.Tests.Integration
+{
+    using System;
+    using System.IO;
+    using System.Text;
+    using System.Xml;
+    using UglyToad.PdfPig.DocumentLayoutAnalysis.Export;
+    using Xunit;
+
+    public class SvgTextExporterTests
+    {
+        [Fact]
+        public void Doc68_1990_01_A()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("68-1990-01_A.pdf");
+
+            var pageXmlTextExporter = new SvgTextExporter();
+
+            Assert.Equal(InvalidCharStrategy.DoNotCheck, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(7);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("68-1990-01_A.7.svg", xml);
+            }
+        }
+
+        [Fact]
+        public void Issue655NoCheckStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new SvgTextExporter();
+
+            Assert.Equal(InvalidCharStrategy.DoNotCheck, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.nocheck.svg", xml);
+
+                string rawText = File.ReadAllText("issue655.nocheck.svg");
+                Assert.Contains(">&#x6;<", rawText);
+            }
+        }
+
+        [Fact]
+        public void Issue655RemoveStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new SvgTextExporter(InvalidCharStrategy.Remove);
+
+            Assert.Equal(InvalidCharStrategy.Remove, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.remove.svg", xml);
+
+                string rawText = File.ReadAllText("issue655.remove.svg");
+                Assert.DoesNotContain(">0x06<", rawText);
+            }
+        }
+
+        [Fact]
+        public void Issue655ConvertToHexadecimalStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new SvgTextExporter(InvalidCharStrategy.ConvertToHexadecimal);
+
+            Assert.Equal(InvalidCharStrategy.ConvertToHexadecimal, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.hex.svg", xml);
+
+                string rawText = File.ReadAllText("issue655.hex.svg");
+                Assert.Contains(">0x06<", rawText);
+            }
+        }
+
+        [Fact]
+        public void Issue655CustomStrategy()
+        {
+            var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf");
+
+            var pageXmlTextExporter = new SvgTextExporter(
+                new Func<string, string>(s =>
+                {
+                    // Adapted from https://stackoverflow.com/a/17735649
+                    if (string.IsNullOrEmpty(s))
+                    {
+                        return s;
+                    }
+
+                    int length = s.Length;
+                    StringBuilder stringBuilder = new StringBuilder(length);
+                    for (int i = 0; i < length; ++i)
+                    {
+                        if (XmlConvert.IsXmlChar(s[i]))
+                        {
+                            stringBuilder.Append(s[i]);
+                        }
+                        else
+                        {
+                            stringBuilder.Append("!?");
+                        }
+                    }
+
+                    return stringBuilder.ToString();
+                }));
+
+            Assert.Equal(InvalidCharStrategy.Custom, pageXmlTextExporter.InvalidCharStrategy);
+
+            using (var document = PdfDocument.Open(hex_0x0006))
+            {
+                var page = document.GetPage(1);
+
+                // Convert page to text
+                string xml = pageXmlTextExporter.Get(page);
+
+                // Save text to an xml file
+                File.WriteAllText("issue655.custom.svg", xml);
+
+                string rawText = File.ReadAllText("issue655.custom.svg");
+                Assert.Contains(">!?<", rawText);
+            }
+        }
+    }
+}