diff --git a/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/AltoXmlTextExporter.cs b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/AltoXmlTextExporter.cs index acbe5827..c259f833 100644 --- a/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/AltoXmlTextExporter.cs +++ b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/AltoXmlTextExporter.cs @@ -16,11 +16,11 @@ /// Alto 4.1 (XML) text exporter. /// See https://github.com/altoxml/schema /// - public class AltoXmlTextExporter : ITextExporter + public sealed class AltoXmlTextExporter : ITextExporter { private readonly IPageSegmenter pageSegmenter; private readonly IWordExtractor wordExtractor; - + private readonly Func invalidCharacterHandler; private readonly double scale; private readonly string indentChar; @@ -33,6 +33,9 @@ private int stringCount; private int glyphCount; + /// + public InvalidCharStrategy InvalidCharStrategy { get; } + /// /// Alto 4.1 (XML). /// See https://github.com/altoxml/schema @@ -40,13 +43,50 @@ /// Extractor used to identify words in the document. /// Segmenter used to split page into blocks. /// Scale multiplier to apply to output document, defaults to 1. - /// Character to use for indentation, defaults to tab. - public AltoXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, double scale = 1, string indent = "\t") + /// Character to use for indentation, defaults to tab. + /// How to handle invalid characters. + public AltoXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, + double scale, string indentChar, + Func invalidCharacterHandler) + : this(wordExtractor, pageSegmenter, scale, indentChar, + InvalidCharStrategy.Custom, invalidCharacterHandler) + { } + + /// + /// Alto 4.1 (XML). + /// See https://github.com/altoxml/schema + /// + /// Extractor used to identify words in the document. + /// Segmenter used to split page into blocks. + /// Scale multiplier to apply to output document, defaults to 1. + /// Character to use for indentation, defaults to tab. + /// How to handle invalid characters. + public AltoXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, + double scale = 1, string indentChar = "\t", + InvalidCharStrategy invalidCharacterStrategy = InvalidCharStrategy.DoNotCheck) + : this(wordExtractor, pageSegmenter, scale, indentChar, + invalidCharacterStrategy, null) + { } + + private AltoXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, + double scale, string indentChar, + InvalidCharStrategy invalidCharacterStrategy, + Func invalidCharacterHandler) { - this.wordExtractor = wordExtractor ?? throw new ArgumentNullException(nameof(wordExtractor)); - this.pageSegmenter = pageSegmenter ?? throw new ArgumentNullException(nameof(pageSegmenter)); + this.wordExtractor = wordExtractor; + this.pageSegmenter = pageSegmenter; this.scale = scale; - indentChar = indent ?? string.Empty; + this.indentChar = indentChar ?? string.Empty; + InvalidCharStrategy = invalidCharacterStrategy; + + if (invalidCharacterHandler is null) + { + this.invalidCharacterHandler = TextExporterHelper.GetXmlInvalidCharHandler(InvalidCharStrategy); + } + else + { + this.invalidCharacterHandler = invalidCharacterHandler; + } } /// @@ -57,10 +97,7 @@ public string Get(PdfDocument document, bool includePaths = false) { var altoDocument = CreateAltoDocument("unknown"); - var altoPages = document.GetPages().Select(x => ToAltoPage(x, includePaths)).ToArray(); - - altoDocument.Layout.Pages = altoPages; - + altoDocument.Layout.Pages = document.GetPages().Select(x => ToAltoPage(x, includePaths)).ToArray(); return Serialize(altoDocument); } @@ -128,8 +165,8 @@ { Height = (float)Math.Round(page.Height * scale), // TBD Width = (float)Math.Round(page.Width * scale), // TBD - VerticalPosition = 0f, // TBD - HorizontalPosition = 0f, // TBD + VerticalPosition = 0f, // TBD + HorizontalPosition = 0f, // TBD ComposedBlocks = null, // TBD GraphicalElements = null, // TBD Illustrations = null, // TBD @@ -141,9 +178,7 @@ }; var words = page.GetWords(wordExtractor); - var blocks = pageSegmenter.GetBlocks(words).Select(b => ToAltoTextBlock(b, page.Height)).ToArray(); - - altoPage.PrintSpace.TextBlock = blocks; + altoPage.PrintSpace.TextBlock = pageSegmenter.GetBlocks(words).Select(b => ToAltoTextBlock(b, page.Height)).ToArray(); altoPage.PrintSpace.Illustrations = page.GetImages().Select(i => ToAltoIllustration(i, page.Height)).ToArray(); @@ -222,7 +257,6 @@ { textLineCount++; var strings = textLine.Words - .Where(x => x.Text.All(XmlConvert.IsXmlChar)) .Select(w => ToAltoString(w, height)).ToArray(); return new AltoDocument.AltoTextBlockTextLine @@ -252,7 +286,7 @@ Width = (float)Math.Round(word.BoundingBox.Width * scale), Glyph = glyphs, Cc = string.Join("", glyphs.Select(g => 9f * (1f - g.Gc))), // from 0->1 to 9->0 - Content = word.Text, + Content = invalidCharacterHandler(word.Text), Language = null, StyleRefs = null, SubsContent = null, @@ -272,7 +306,7 @@ Height = (float)Math.Round(letter.GlyphRectangle.Height * scale), Width = (float)Math.Round(letter.GlyphRectangle.Width * scale), Gc = 1.0f, - Content = letter.Value, + Content = invalidCharacterHandler(letter.Value), Id = "P" + pageCount + "_ST" + stringCount.ToString("#00000") + "_G" + glyphCount.ToString("#00") }; } @@ -314,8 +348,8 @@ Processings = new[] { processing }, SourceImageInformation = new AltoDocument.AltoSourceImageInformation { - DocumentIdentifiers = new [] { documentIdentifier }, - FileIdentifiers = new [] { fileIdentifier }, + DocumentIdentifiers = new[] { documentIdentifier }, + FileIdentifiers = new[] { fileIdentifier }, FileName = fileName } }; @@ -329,6 +363,7 @@ Encoding = System.Text.Encoding.UTF8, Indent = true, IndentChars = indentChar, + CheckCharacters = InvalidCharStrategy != InvalidCharStrategy.DoNotCheck, }; using (var memoryStream = new System.IO.MemoryStream()) @@ -346,7 +381,12 @@ { var serializer = new XmlSerializer(typeof(AltoDocument)); - using (var reader = XmlReader.Create(xmlPath)) + var settings = new XmlReaderSettings() + { + CheckCharacters = false + }; + + using (var reader = XmlReader.Create(xmlPath, settings)) { return (AltoDocument)serializer.Deserialize(reader); } diff --git a/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/HOcrTextExporter.cs b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/HOcrTextExporter.cs index 40cc53dc..89be2858 100644 --- a/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/HOcrTextExporter.cs +++ b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/HOcrTextExporter.cs @@ -13,14 +13,14 @@ /// hOCR v1.2 (HTML) text exporter. /// See http://kba.cloud/hocr-spec/1.2/ /// - public class HOcrTextExporter : ITextExporter + public sealed class HOcrTextExporter : ITextExporter { private const string XmlHeader = "\n\n"; private const string Hocrjs = "\n"; private readonly IPageSegmenter pageSegmenter; private readonly IWordExtractor wordExtractor; - + private readonly Func invalidCharacterHandler; private readonly double scale; private readonly string indentChar; @@ -32,16 +32,60 @@ private int paraCount; private int imageCount; + /// + public InvalidCharStrategy InvalidCharStrategy { get; } + /// /// hOCR v1.2 (HTML) /// See http://kba.cloud/hocr-spec/1.2/ /// - public HOcrTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, double scale = 1.0, string indent = "\t") + /// Extractor used to identify words in the document. + /// Segmenter used to split page into blocks. + /// Scale multiplier to apply to output document, defaults to 1. + /// Character to use for indentation, defaults to tab. + /// How to handle invalid characters. + public HOcrTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, + double scale, string indentChar, + Func invalidCharacterHandler) + : this(wordExtractor, pageSegmenter, scale, indentChar, + InvalidCharStrategy.Custom, invalidCharacterHandler) + { } + + /// + /// hOCR v1.2 (HTML) + /// See http://kba.cloud/hocr-spec/1.2/ + /// + /// Extractor used to identify words in the document. + /// Segmenter used to split page into blocks. + /// Scale multiplier to apply to output document, defaults to 1. + /// Character to use for indentation, defaults to tab. + /// How to handle invalid characters. + public HOcrTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, + double scale = 1, string indentChar = "\t", + InvalidCharStrategy invalidCharacterStrategy = InvalidCharStrategy.DoNotCheck) + : this(wordExtractor, pageSegmenter, scale, indentChar, + invalidCharacterStrategy, null) + { } + + private HOcrTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, + double scale, string indentChar, + InvalidCharStrategy invalidCharacterStrategy, + Func invalidCharacterHandler) { this.wordExtractor = wordExtractor; this.pageSegmenter = pageSegmenter; this.scale = scale; - indentChar = indent; + this.indentChar = indentChar ?? string.Empty; + InvalidCharStrategy = invalidCharacterStrategy; + + if (invalidCharacterHandler is null) + { + this.invalidCharacterHandler = TextExporterHelper.GetXmlInvalidCharHandler(InvalidCharStrategy); + } + else + { + this.invalidCharacterHandler = invalidCharacterHandler; + } } /// @@ -325,7 +369,7 @@ } hocr += "'"; - hocr += ">" + word.Text + " "; + hocr += ">" + invalidCharacterHandler(word.Text) + " "; return hocr; } diff --git a/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/InvalidCharStrategy.cs b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/InvalidCharStrategy.cs new file mode 100644 index 00000000..a7db4c7d --- /dev/null +++ b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/InvalidCharStrategy.cs @@ -0,0 +1,28 @@ +namespace UglyToad.PdfPig.DocumentLayoutAnalysis.Export +{ + /// + /// How to handle invalid characters. + /// + public enum InvalidCharStrategy : byte + { + /// + /// Custom strategy. + /// + Custom = 0, + + /// + /// Do not check invalid character. + /// + DoNotCheck = 1, + + /// + /// Remove invalid character. + /// + Remove = 2, + + /// + /// Convert invalid character to hexadecimal representation. + /// + ConvertToHexadecimal = 3 + } +} diff --git a/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/PageXmlTextExporter.cs b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/PageXmlTextExporter.cs index 64a18810..47337bba 100644 --- a/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/PageXmlTextExporter.cs +++ b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/PageXmlTextExporter.cs @@ -3,38 +3,33 @@ using Content; using Core; using DocumentLayoutAnalysis; + using Graphics; using Graphics.Colors; using PAGE; + using PageSegmenter; + using ReadingOrderDetector; using System; using System.Collections.Generic; using System.Linq; using System.Xml; using System.Xml.Serialization; - using PageSegmenter; - using ReadingOrderDetector; - using Graphics; using Util; /// /// PAGE-XML 2019-07-15 (XML) text exporter. /// See https://github.com/PRImA-Research-Lab/PAGE-XML /// - public class PageXmlTextExporter : ITextExporter + public sealed class PageXmlTextExporter : ITextExporter { private readonly IPageSegmenter pageSegmenter; private readonly IWordExtractor wordExtractor; private readonly IReadingOrderDetector readingOrderDetector; - + private readonly Func invalidCharacterHandler; private readonly double scale; private readonly string indentChar; - private int lineCount; - private int wordCount; - private int glyphCount; - private int regionCount; - private int groupOrderCount; - - private List orderedRegions; + /// + public InvalidCharStrategy InvalidCharStrategy { get; } /// /// PAGE-XML 2019-07-15 (XML) text exporter. @@ -42,20 +37,62 @@ /// /// /// - /// + /// + /// + /// Indent character. + /// How to handle invalid characters. + public PageXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, + IReadingOrderDetector readingOrderDetector, + double scale, string indentChar, + Func invalidCharacterHandler) + : this(wordExtractor, pageSegmenter, readingOrderDetector, scale, indentChar, + InvalidCharStrategy.Custom, invalidCharacterHandler) + { } + + /// + /// PAGE-XML 2019-07-15 (XML) text exporter. + /// See https://github.com/PRImA-Research-Lab/PAGE-XML + /// + /// + /// + /// /// /// Indent character. - public PageXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, IReadingOrderDetector readingOrderDetector = null, double scale = 1.0, string indent = "\t") + /// How to handle invalid characters. + public PageXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, + IReadingOrderDetector readingOrderDetector = null, + double scale = 1.0, string indent = "\t", + InvalidCharStrategy invalidCharacterStrategy = InvalidCharStrategy.DoNotCheck) + : this(wordExtractor, pageSegmenter, readingOrderDetector, scale, indent, + invalidCharacterStrategy, null) + { } + + private PageXmlTextExporter(IWordExtractor wordExtractor, IPageSegmenter pageSegmenter, + IReadingOrderDetector readingOrderDetector, + double scale, string indentChar, + InvalidCharStrategy invalidCharacterStrategy, + Func invalidCharacterHandler) { this.wordExtractor = wordExtractor; this.pageSegmenter = pageSegmenter; this.readingOrderDetector = readingOrderDetector; this.scale = scale; - indentChar = indent; + this.indentChar = indentChar ?? string.Empty; + InvalidCharStrategy = invalidCharacterStrategy; + + if (invalidCharacterHandler is null) + { + this.invalidCharacterHandler = TextExporterHelper.GetXmlInvalidCharHandler(InvalidCharStrategy); + } + else + { + this.invalidCharacterHandler = invalidCharacterHandler; + } } /// /// Get the PAGE-XML (XML) string of the pages layout. + /// Not implemented, use instead. /// /// /// Draw PdfPaths present in the page. @@ -80,26 +117,23 @@ /// Draw PdfPaths present in the page. public string Get(Page page, bool includePaths) { - lineCount = 0; - wordCount = 0; - glyphCount = 0; - regionCount = 0; - groupOrderCount = 0; - orderedRegions = new List(); + PageXmlData data = new PageXmlData(); + + DateTime utcNow = DateTime.UtcNow; PageXmlDocument pageXmlDocument = new PageXmlDocument() { Metadata = new PageXmlDocument.PageXmlMetadata() { - Created = DateTime.UtcNow, - LastChange = DateTime.UtcNow, + Created = utcNow, + LastChange = utcNow, Creator = "PdfPig", Comments = pageSegmenter.GetType().Name + "|" + wordExtractor.GetType().Name, }, PcGtsId = "pc-" + page.GetHashCode() }; - pageXmlDocument.Page = ToPageXmlPage(page, includePaths); + pageXmlDocument.Page = ToPageXmlPage(page, data, includePaths); return Serialize(pageXmlDocument); } @@ -151,17 +185,17 @@ /// private string ToRgbEncoded(IColor color) { - var rgb = color.ToRGBValues(); - int red = (int)Math.Round(255f * (float)rgb.r); - int green = 256 * (int)Math.Round(255f * (float)rgb.g); - int blue = 65536 * (int)Math.Round(255f * (float)rgb.b); + var (r, g, b) = color.ToRGBValues(); + int red = Convert.ToByte(255.0 * r); + int green = 256 * Convert.ToByte(255.0 * g); + int blue = 65536 * Convert.ToByte(255.0 * b); int sum = red + green + blue; // as per below, red and blue order might be inverted... var colorWin = System.Drawing.Color.FromArgb(sum); return sum.ToString(); } - private PageXmlDocument.PageXmlPage ToPageXmlPage(Page page, bool includePaths) + private PageXmlDocument.PageXmlPage ToPageXmlPage(Page page, PageXmlData data, bool includePaths) { var pageXmlPage = new PageXmlDocument.PageXmlPage { @@ -182,16 +216,17 @@ blocks = readingOrderDetector.Get(blocks).ToList(); } - regions.AddRange(blocks.Select(b => ToPageXmlTextRegion(b, page.Width, page.Height))); + regions.AddRange(blocks.Select(b => ToPageXmlTextRegion(b, data, page.Width, page.Height))); - if (orderedRegions.Count > 0) + if (data.OrderedRegions.Count > 0) { + data.GroupOrdersCount++; pageXmlPage.ReadingOrder = new PageXmlDocument.PageXmlReadingOrder() { Item = new PageXmlDocument.PageXmlOrderedGroup() { - Items = orderedRegions.ToArray(), - Id = "g" + groupOrderCount++ + Items = data.OrderedRegions.ToArray(), + Id = "g" + data.GroupOrdersCount } }; } @@ -200,14 +235,14 @@ var images = page.GetImages().ToList(); if (images.Count > 0) { - regions.AddRange(images.Select(i => ToPageXmlImageRegion(i, page.Width, page.Height))); + regions.AddRange(images.Select(i => ToPageXmlImageRegion(i, data, page.Width, page.Height))); } if (includePaths) { foreach (var path in page.ExperimentalAccess.Paths) { - var graphicalElement = ToPageXmlLineDrawingRegion(path, page.Width, page.Height); + var graphicalElement = ToPageXmlLineDrawingRegion(path, data, page.Width, page.Height); if (graphicalElement != null) { @@ -220,40 +255,40 @@ return pageXmlPage; } - private PageXmlDocument.PageXmlLineDrawingRegion ToPageXmlLineDrawingRegion(PdfPath pdfPath, double pageWidth, double pageHeight) + private PageXmlDocument.PageXmlLineDrawingRegion ToPageXmlLineDrawingRegion(PdfPath pdfPath, PageXmlData data, double pageWidth, double pageHeight) { var bbox = pdfPath.GetBoundingRectangle(); if (bbox.HasValue) { - regionCount++; + data.RegionsCount++; return new PageXmlDocument.PageXmlLineDrawingRegion() { Coords = ToCoords(bbox.Value, pageWidth, pageHeight), - Id = "r" + regionCount + Id = "r" + data.RegionsCount }; } return null; } - private PageXmlDocument.PageXmlImageRegion ToPageXmlImageRegion(IPdfImage pdfImage, double pageWidth, double pageHeight) + private PageXmlDocument.PageXmlImageRegion ToPageXmlImageRegion(IPdfImage pdfImage, PageXmlData data, double pageWidth, double pageHeight) { - regionCount++; + data.RegionsCount++; var bbox = pdfImage.Bounds; return new PageXmlDocument.PageXmlImageRegion() { Coords = ToCoords(bbox, pageWidth, pageHeight), - Id = "r" + regionCount + Id = "r" + data.RegionsCount }; } - private PageXmlDocument.PageXmlTextRegion ToPageXmlTextRegion(TextBlock textBlock, double pageWidth, double pageHeight) + private PageXmlDocument.PageXmlTextRegion ToPageXmlTextRegion(TextBlock textBlock, PageXmlData data, double pageWidth, double pageHeight) { - regionCount++; - string regionId = "r" + regionCount; + data.RegionsCount++; + string regionId = "r" + data.RegionsCount; if (readingOrderDetector != null && textBlock.ReadingOrder > -1) { - orderedRegions.Add(new PageXmlDocument.PageXmlRegionRefIndexed() + data.OrderedRegions.Add(new PageXmlDocument.PageXmlRegionRefIndexed() { RegionRef = regionId, Index = textBlock.ReadingOrder @@ -264,40 +299,58 @@ { Coords = ToCoords(textBlock.BoundingBox, pageWidth, pageHeight), Type = PageXmlDocument.PageXmlTextSimpleType.Paragraph, - TextLines = textBlock.TextLines.Select(l => ToPageXmlTextLine(l, pageWidth, pageHeight)).ToArray(), - TextEquivs = new[] { new PageXmlDocument.PageXmlTextEquiv() { Unicode = textBlock.Text } }, + TextLines = textBlock.TextLines.Select(l => ToPageXmlTextLine(l, data, pageWidth, pageHeight)).ToArray(), + TextEquivs = new[] + { + new PageXmlDocument.PageXmlTextEquiv() + { + Unicode = invalidCharacterHandler(textBlock.Text) + } + }, Id = regionId }; } - private PageXmlDocument.PageXmlTextLine ToPageXmlTextLine(TextLine textLine, double pageWidth, double pageHeight) + private PageXmlDocument.PageXmlTextLine ToPageXmlTextLine(TextLine textLine, PageXmlData data, double pageWidth, double pageHeight) { - lineCount++; + data.LinesCount++; return new PageXmlDocument.PageXmlTextLine() { Coords = ToCoords(textLine.BoundingBox, pageWidth, pageHeight), Production = PageXmlDocument.PageXmlProductionSimpleType.Printed, - Words = textLine.Words.Select(w => ToPageXmlWord(w, pageWidth, pageHeight)).ToArray(), - TextEquivs = new[] { new PageXmlDocument.PageXmlTextEquiv() { Unicode = textLine.Text } }, - Id = "l" + lineCount + Words = textLine.Words.Select(w => ToPageXmlWord(w, data, pageWidth, pageHeight)).ToArray(), + TextEquivs = new[] + { + new PageXmlDocument.PageXmlTextEquiv() + { + Unicode = invalidCharacterHandler(textLine.Text) + } + }, + Id = "l" + data.LinesCount }; } - private PageXmlDocument.PageXmlWord ToPageXmlWord(Word word, double pageWidth, double pageHeight) + private PageXmlDocument.PageXmlWord ToPageXmlWord(Word word, PageXmlData data, double pageWidth, double pageHeight) { - wordCount++; + data.WordsCount++; return new PageXmlDocument.PageXmlWord() { Coords = ToCoords(word.BoundingBox, pageWidth, pageHeight), - Glyphs = word.Letters.Select(l => ToPageXmlGlyph(l, pageWidth, pageHeight)).ToArray(), - TextEquivs = new[] { new PageXmlDocument.PageXmlTextEquiv() { Unicode = word.Text } }, - Id = "w" + wordCount + Glyphs = word.Letters.Select(l => ToPageXmlGlyph(l, data, pageWidth, pageHeight)).ToArray(), + TextEquivs = new[] + { + new PageXmlDocument.PageXmlTextEquiv() + { + Unicode = invalidCharacterHandler(word.Text) + } + }, + Id = "w" + data.WordsCount }; } - private PageXmlDocument.PageXmlGlyph ToPageXmlGlyph(Letter letter, double pageWidth, double pageHeight) + private PageXmlDocument.PageXmlGlyph ToPageXmlGlyph(Letter letter, PageXmlData data, double pageWidth, double pageHeight) { - glyphCount++; + data.GlyphsCount++; return new PageXmlDocument.PageXmlGlyph() { Coords = ToCoords(letter.GlyphRectangle, pageWidth, pageHeight), @@ -309,8 +362,14 @@ FontFamily = letter.FontName, TextColourRgb = ToRgbEncoded(letter.Color), }, - TextEquivs = new[] { new PageXmlDocument.PageXmlTextEquiv() { Unicode = letter.Value } }, - Id = "c" + glyphCount + TextEquivs = new[] + { + new PageXmlDocument.PageXmlTextEquiv() + { + Unicode = invalidCharacterHandler(letter.Value) + } + }, + Id = "c" + data.GlyphsCount }; } @@ -322,6 +381,7 @@ Encoding = System.Text.Encoding.UTF8, Indent = true, IndentChars = indentChar, + CheckCharacters = InvalidCharStrategy != InvalidCharStrategy.DoNotCheck, }; using (var memoryStream = new System.IO.MemoryStream()) @@ -339,10 +399,34 @@ { XmlSerializer serializer = new XmlSerializer(typeof(PageXmlDocument)); - using (var reader = XmlReader.Create(xmlPath)) + var settings = new XmlReaderSettings() + { + CheckCharacters = false + }; + + using (var reader = XmlReader.Create(xmlPath, settings)) { return (PageXmlDocument)serializer.Deserialize(reader); } } + + /// + /// Class to keep track of a page data. + /// + private sealed class PageXmlData + { + public PageXmlData() + { + OrderedRegions = new List(); + } + + public int LinesCount { get; set; } + public int WordsCount { get; set; } + public int GlyphsCount { get; set; } + public int RegionsCount { get; set; } + public int GroupOrdersCount { get; set; } + + public List OrderedRegions { get; } + } } } diff --git a/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/SvgTextExporter.cs b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/SvgTextExporter.cs index 46151923..4137ac97 100644 --- a/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/SvgTextExporter.cs +++ b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/SvgTextExporter.cs @@ -13,38 +13,72 @@ /// /// Exports a page as an SVG. /// - public class SvgTextExporter : ITextExporter + public sealed class SvgTextExporter : ITextExporter { - private const int Rounding = 4; + private readonly Func invalidCharacterHandler; private static readonly Dictionary Fonts = new Dictionary() { { "ArialMT", "Arial Rounded MT Bold" } }; + /// + /// Used to round numbers. + /// + public int Rounding { get; } = 4; + + /// + /// + /// Not in use. + /// + public InvalidCharStrategy InvalidCharStrategy { get; } + + /// + /// Svg text exporter. + /// + /// How to handle invalid characters. + public SvgTextExporter(Func invalidCharacterHandler) + : this(InvalidCharStrategy.Custom, invalidCharacterHandler) + { } + + /// + /// Svg text exporter. + /// + /// How to handle invalid characters. + public SvgTextExporter(InvalidCharStrategy invalidCharacterStrategy = InvalidCharStrategy.DoNotCheck) + : this(invalidCharacterStrategy, null) + { } + + private SvgTextExporter(InvalidCharStrategy invalidCharacterStrategy, Func invalidCharacterHandler) + { + InvalidCharStrategy = invalidCharacterStrategy; + + if (invalidCharacterHandler is null) + { + this.invalidCharacterHandler = TextExporterHelper.GetXmlInvalidCharHandler(InvalidCharStrategy); + } + else + { + this.invalidCharacterHandler = invalidCharacterHandler; + } + } /// /// Get the page contents as an SVG. /// public string Get(Page page) { - var builder = new StringBuilder($""); + var builder = new StringBuilder($"\n\n"); - var paths = page.ExperimentalAccess.Paths; - foreach (var path in paths) + foreach (var path in page.ExperimentalAccess.Paths) { - if (path.IsClipping) + if (!path.IsClipping) { - //var svg = PathToSvg(path, page.Height); - //svg = svg.Replace("stroke='black'", "stroke='yellow'"); - //builder.Append(svg); - } - else - { - builder.Append(PathToSvg(path, page.Height)); + builder.AppendLine(PathToSvg(path, page.Height)); } } var doc = new XmlDocument(); + foreach (var letter in page.Letters) { builder.Append(LetterToSvg(letter, page.Height, doc)); @@ -54,7 +88,7 @@ return builder.ToString(); } - private static string LetterToSvg(Letter l, double height, XmlDocument doc) + private string LetterToSvg(Letter l, double height, XmlDocument doc) { string fontFamily = GetFontFamily(l.FontName, out string style, out string weight); string rotation = ""; @@ -131,10 +165,10 @@ return fontName; } - private static string XmlEscape(Letter letter, XmlDocument doc) + private string XmlEscape(Letter letter, XmlDocument doc) { XmlNode node = doc.CreateElement("root"); - node.InnerText = letter.Value; + node.InnerText = invalidCharacterHandler(letter.Value); return node.InnerXml; } @@ -146,7 +180,7 @@ } var (r, g, b) = color.ToRGBValues(); - return $"rgb({Math.Ceiling(r * 255)},{Math.Ceiling(g * 255)},{Math.Ceiling(b * 255)})"; + return $"rgb({Convert.ToByte(r * 255)},{Convert.ToByte(g * 255)},{Convert.ToByte(b * 255)})"; } private static string PathToSvg(PdfPath p, double height) @@ -214,12 +248,11 @@ } string fillColor = " fill='none'"; - string fillRule = ""; + const string fillRule = ""; // For further dev if (p.IsFilled) { fillColor = $" fill='{ColorToSvg(p.FillColor)}'"; - //if (p.FillingRule == FillingRule.EvenOdd) fillRule = " fill-rule='evenodd'"; } var path = $""; diff --git a/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/TextExporterHelper.cs b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/TextExporterHelper.cs new file mode 100644 index 00000000..2ce0f914 --- /dev/null +++ b/src/UglyToad.PdfPig.DocumentLayoutAnalysis/Export/TextExporterHelper.cs @@ -0,0 +1,71 @@ +namespace UglyToad.PdfPig.DocumentLayoutAnalysis.Export +{ + using System; + using System.Text; + using System.Xml; + + internal static class TextExporterHelper + { + public static Func GetXmlInvalidCharHandler(InvalidCharStrategy invalidCharacterStrategy) + { + switch (invalidCharacterStrategy) + { + case InvalidCharStrategy.DoNotCheck: + return new Func(s => s); + + case InvalidCharStrategy.Remove: + return new Func(s => + { + // https://stackoverflow.com/a/17735649 + if (string.IsNullOrEmpty(s)) + { + return s; + } + + int length = s.Length; + StringBuilder stringBuilder = new StringBuilder(length); + for (int i = 0; i < length; ++i) + { + if (XmlConvert.IsXmlChar(s[i])) + { + stringBuilder.Append(s[i]); + } + } + + return stringBuilder.ToString(); + }); + + case InvalidCharStrategy.ConvertToHexadecimal: + return new Func(s => + { + // Adapted from https://stackoverflow.com/a/17735649 + if (string.IsNullOrEmpty(s)) + { + return s; + } + + int length = s.Length; + StringBuilder stringBuilder = new StringBuilder(length); + for (int i = 0; i < length; ++i) + { + if (XmlConvert.IsXmlChar(s[i])) + { + stringBuilder.Append(s[i]); + } + else + { + byte[] bytes = Encoding.UTF8.GetBytes(s[i].ToString()); + string hexString = BitConverter.ToString(bytes); + stringBuilder.Append("0x").Append(hexString); + } + } + + return stringBuilder.ToString(); + }); + + default: + throw new NotImplementedException("TODO"); + } + } + } +} diff --git a/src/UglyToad.PdfPig.Tests/Integration/AltoXmlTextExporterTests.cs b/src/UglyToad.PdfPig.Tests/Integration/AltoXmlTextExporterTests.cs new file mode 100644 index 00000000..caa92a1d --- /dev/null +++ b/src/UglyToad.PdfPig.Tests/Integration/AltoXmlTextExporterTests.cs @@ -0,0 +1,189 @@ +namespace UglyToad.PdfPig.Tests.Integration +{ + using System; + using System.IO; + using System.Linq; + using System.Text; + using System.Xml; + using UglyToad.PdfPig.DocumentLayoutAnalysis.Export; + using UglyToad.PdfPig.DocumentLayoutAnalysis.PageSegmenter; + using UglyToad.PdfPig.Util; + using Xunit; + + public class AltoXmlTextExporterTests + { + [Fact] + public void Issue655NoCheckStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new AltoXmlTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance); + + Assert.Equal(InvalidCharStrategy.DoNotCheck, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.nocheck.altoxml.xml", xml); + + var pageXml = AltoXmlTextExporter.Deserialize("issue655.nocheck.altoxml.xml"); + + var textRegions = pageXml.Layout.Pages[0].PrintSpace.TextBlock; + Assert.Single(textRegions); + + var textLines = textRegions.Single().TextLines; + Assert.Single(textLines); + + var strings = textLines.Single().Strings; + Assert.Equal(2, strings.Length); + + Assert.Equal("TM", strings[0].Content); + Assert.Equal("1\u00062345\u0006678\u0006ABC", strings[1].Content); // no check strategy, contains invalid xml chars + } + } + + [Fact] + public void Issue655RemoveStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new AltoXmlTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance, + invalidCharacterStrategy: InvalidCharStrategy.Remove); + + Assert.Equal(InvalidCharStrategy.Remove, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.remove.altoxml.xml", xml); + + var pageXml = AltoXmlTextExporter.Deserialize("issue655.remove.altoxml.xml"); + + var textRegions = pageXml.Layout.Pages[0].PrintSpace.TextBlock; + Assert.Single(textRegions); + + var textLines = textRegions.Single().TextLines; + Assert.Single(textLines); + + var strings = textLines.Single().Strings; + Assert.Equal(2, strings.Length); + + Assert.Equal("TM", strings[0].Content); + Assert.Equal("12345678ABC", strings[1].Content); + } + } + + [Fact] + public void Issue655ConvertToHexadecimalStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new AltoXmlTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance, + invalidCharacterStrategy: InvalidCharStrategy.ConvertToHexadecimal); + + Assert.Equal(InvalidCharStrategy.ConvertToHexadecimal, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.hex.altoxml.xml", xml); + + var pageXml = AltoXmlTextExporter.Deserialize("issue655.hex.altoxml.xml"); + + var textRegions = pageXml.Layout.Pages[0].PrintSpace.TextBlock; + Assert.Single(textRegions); + + var textLines = textRegions.Single().TextLines; + Assert.Single(textLines); + + var strings = textLines.Single().Strings; + Assert.Equal(2, strings.Length); + + Assert.Equal("TM", strings[0].Content); + Assert.Equal("10x0623450x066780x06ABC", strings[1].Content); + } + } + + [Fact] + public void Issue655CustomStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new AltoXmlTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance, 1.0, "\t", + new Func(s => + { + // Adapted from https://stackoverflow.com/a/17735649 + if (string.IsNullOrEmpty(s)) + { + return s; + } + + int length = s.Length; + StringBuilder stringBuilder = new StringBuilder(length); + for (int i = 0; i < length; ++i) + { + if (XmlConvert.IsXmlChar(s[i])) + { + stringBuilder.Append(s[i]); + } + else + { + stringBuilder.Append("!?"); + } + } + + return stringBuilder.ToString(); + })); + + Assert.Equal(InvalidCharStrategy.Custom, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.custom.altoxml.xml", xml); + + var pageXml = AltoXmlTextExporter.Deserialize("issue655.custom.altoxml.xml"); + + var textRegions = pageXml.Layout.Pages[0].PrintSpace.TextBlock; + Assert.Single(textRegions); + + var textLines = textRegions.Single().TextLines; + Assert.Single(textLines); + + var strings = textLines.Single().Strings; + Assert.Equal(2, strings.Length); + + Assert.Equal("TM", strings[0].Content); + Assert.Equal("1!?2345!?678!?ABC", strings[1].Content); + } + } + } +} diff --git a/src/UglyToad.PdfPig.Tests/Integration/Documents/hex_0x0006.pdf b/src/UglyToad.PdfPig.Tests/Integration/Documents/hex_0x0006.pdf new file mode 100644 index 00000000..158b7a41 Binary files /dev/null and b/src/UglyToad.PdfPig.Tests/Integration/Documents/hex_0x0006.pdf differ diff --git a/src/UglyToad.PdfPig.Tests/Integration/HOcrTextExporterTests.cs b/src/UglyToad.PdfPig.Tests/Integration/HOcrTextExporterTests.cs new file mode 100644 index 00000000..e0646218 --- /dev/null +++ b/src/UglyToad.PdfPig.Tests/Integration/HOcrTextExporterTests.cs @@ -0,0 +1,144 @@ +namespace UglyToad.PdfPig.Tests.Integration +{ + using System; + using System.IO; + using System.Text; + using System.Xml; + using UglyToad.PdfPig.DocumentLayoutAnalysis.Export; + using UglyToad.PdfPig.DocumentLayoutAnalysis.PageSegmenter; + using UglyToad.PdfPig.Util; + using Xunit; + + public class HOcrTextExporterTests + { + [Fact] + public void Issue655NoCheckStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new HOcrTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance); + + Assert.Equal(InvalidCharStrategy.DoNotCheck, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page, useHocrjs: true); + + // Save text to an xml file + File.WriteAllText("issue655.nocheck.hocr.html", xml); + + string rawText = File.ReadAllText("issue655.nocheck.hocr.html"); + Assert.Contains("1\u00062345\u0006678\u0006ABC", rawText); + } + } + + [Fact] + public void Issue655RemoveStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new HOcrTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance, + invalidCharacterStrategy: InvalidCharStrategy.Remove); + + Assert.Equal(InvalidCharStrategy.Remove, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page, useHocrjs: true); + + // Save text to an xml file + File.WriteAllText("issue655.remove.hocr.html", xml); + + string rawText = File.ReadAllText("issue655.remove.hocr.html"); + Assert.Contains("12345678ABC", rawText); + } + } + + [Fact] + public void Issue655ConvertToHexadecimalStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new HOcrTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance, + invalidCharacterStrategy: InvalidCharStrategy.ConvertToHexadecimal); + + Assert.Equal(InvalidCharStrategy.ConvertToHexadecimal, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page, useHocrjs: true); + + // Save text to an xml file + File.WriteAllText("issue655.hex.hocr.html", xml); + + string rawText = File.ReadAllText("issue655.hex.hocr.html"); + Assert.Contains("10x0623450x066780x06ABC", rawText); + } + } + + [Fact] + public void Issue655CustomStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new HOcrTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance, 1.0, "\t", + new Func(s => + { + // Adapted from https://stackoverflow.com/a/17735649 + if (string.IsNullOrEmpty(s)) + { + return s; + } + + int length = s.Length; + StringBuilder stringBuilder = new StringBuilder(length); + for (int i = 0; i < length; ++i) + { + if (XmlConvert.IsXmlChar(s[i])) + { + stringBuilder.Append(s[i]); + } + else + { + stringBuilder.Append("!?"); + } + } + + return stringBuilder.ToString(); + })); + + Assert.Equal(InvalidCharStrategy.Custom, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page, useHocrjs: true); + + // Save text to an xml file + File.WriteAllText("issue655.custom.hocr.html", xml); + + string rawText = File.ReadAllText("issue655.custom.hocr.html"); + Assert.Contains("1!?2345!?678!?ABC", rawText); + } + } + } +} diff --git a/src/UglyToad.PdfPig.Tests/Integration/PageXmlTextExporterTests.cs b/src/UglyToad.PdfPig.Tests/Integration/PageXmlTextExporterTests.cs index 1860f24e..984ae1fb 100644 --- a/src/UglyToad.PdfPig.Tests/Integration/PageXmlTextExporterTests.cs +++ b/src/UglyToad.PdfPig.Tests/Integration/PageXmlTextExporterTests.cs @@ -1,15 +1,17 @@ namespace UglyToad.PdfPig.Tests.Integration { - using System; - using System.Collections.Generic; - using System.IO; - using System.Text; - using System.Text.RegularExpressions; using DocumentLayoutAnalysis.Export; using DocumentLayoutAnalysis.PageSegmenter; using DocumentLayoutAnalysis.ReadingOrderDetector; using PdfPig.Core; using PdfPig.Util; + using System; + using System.IO; + using System.Linq; + using System.Text; + using System.Text.RegularExpressions; + using System.Xml; + using UglyToad.PdfPig.DocumentLayoutAnalysis.Export.PAGE; using Xunit; public class PageXmlTextExporterTests @@ -26,14 +28,11 @@ RecursiveXYCut.Instance, UnsupervisedReadingOrderDetector.Instance); - string xml; using (var document = PdfDocument.Open(GetFilename())) { var page = document.GetPage(1); - xml = pageXmlTextExporter.Get(page); + return pageXmlTextExporter.Get(page); } - - return xml; } [Fact] @@ -88,7 +87,173 @@ Assert.Equal("1,199", PageXmlTextExporter.PointToString(topLeftPagePoint, pageWidth, pageHeight)); Assert.Equal("1,1", PageXmlTextExporter.PointToString(bottomLeftPagePoint, pageWidth, pageHeight)); Assert.Equal("99,1", PageXmlTextExporter.PointToString(bottomRightPagePoint, pageWidth, pageHeight)); - Assert.Equal($"60,140", PageXmlTextExporter.PointToString(normalPoint, pageWidth, pageHeight)); + Assert.Equal("60,140", PageXmlTextExporter.PointToString(normalPoint, pageWidth, pageHeight)); + } + + [Fact] + public void Issue655NoCheckStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new PageXmlTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance, + UnsupervisedReadingOrderDetector.Instance); + + Assert.Equal(InvalidCharStrategy.DoNotCheck, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.nocheck.pagexml.xml", xml); + + var pageXml = PageXmlTextExporter.Deserialize("issue655.nocheck.pagexml.xml"); + + var textRegions = pageXml.Page.Items.OfType().ToArray(); + Assert.Single(textRegions); + + var textEquivs = textRegions.Single().TextEquivs; + Assert.Single(textEquivs); + + string unicode = textEquivs.Single().Unicode; + Assert.Equal("TM 1\u00062345\u0006678\u0006ABC", unicode); // no check strategy, contains invalid xml chars + } + } + + [Fact] + public void Issue655RemoveStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new PageXmlTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance, + UnsupervisedReadingOrderDetector.Instance, + invalidCharacterStrategy: InvalidCharStrategy.Remove); + + Assert.Equal(InvalidCharStrategy.Remove, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.remove.pagexml.xml", xml); + + var pageXml = PageXmlTextExporter.Deserialize("issue655.remove.pagexml.xml"); + + var textRegions = pageXml.Page.Items.OfType().ToArray(); + Assert.Single(textRegions); + + var textEquivs = textRegions.Single().TextEquivs; + Assert.Single(textEquivs); + + string unicode = textEquivs.Single().Unicode; + Assert.Equal("TM 12345678ABC", unicode); + } + } + + [Fact] + public void Issue655ConvertToHexadecimalStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new PageXmlTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance, + UnsupervisedReadingOrderDetector.Instance, + invalidCharacterStrategy: InvalidCharStrategy.ConvertToHexadecimal); + + Assert.Equal(InvalidCharStrategy.ConvertToHexadecimal, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.hex.pagexml.xml", xml); + + var pageXml = PageXmlTextExporter.Deserialize("issue655.hex.pagexml.xml"); + + var textRegions = pageXml.Page.Items.OfType().ToArray(); + Assert.Single(textRegions); + + var textEquivs = textRegions.Single().TextEquivs; + Assert.Single(textEquivs); + + string unicode = textEquivs.Single().Unicode; + Assert.Equal("TM 10x0623450x066780x06ABC", unicode); + } + } + + [Fact] + public void Issue655CustomStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new PageXmlTextExporter( + DefaultWordExtractor.Instance, + RecursiveXYCut.Instance, + UnsupervisedReadingOrderDetector.Instance, 1.0, "\t", + new Func(s => + { + // Adapted from https://stackoverflow.com/a/17735649 + if (string.IsNullOrEmpty(s)) + { + return s; + } + + int length = s.Length; + StringBuilder stringBuilder = new StringBuilder(length); + for (int i = 0; i < length; ++i) + { + if (XmlConvert.IsXmlChar(s[i])) + { + stringBuilder.Append(s[i]); + } + else + { + stringBuilder.Append("!?"); + } + } + + return stringBuilder.ToString(); + })); + + Assert.Equal(InvalidCharStrategy.Custom, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.custom.pagexml.xml", xml); + + var pageXml = PageXmlTextExporter.Deserialize("issue655.custom.pagexml.xml"); + + var textRegions = pageXml.Page.Items.OfType().ToArray(); + Assert.Single(textRegions); + + var textEquivs = textRegions.Single().TextEquivs; + Assert.Single(textEquivs); + + string unicode = textEquivs.Single().Unicode; + Assert.Equal("TM 1!?2345!?678!?ABC", unicode); + } } } } diff --git a/src/UglyToad.PdfPig.Tests/Integration/SvgTextExporterTests.cs b/src/UglyToad.PdfPig.Tests/Integration/SvgTextExporterTests.cs new file mode 100644 index 00000000..5ec50a21 --- /dev/null +++ b/src/UglyToad.PdfPig.Tests/Integration/SvgTextExporterTests.cs @@ -0,0 +1,153 @@ +namespace UglyToad.PdfPig.Tests.Integration +{ + using System; + using System.IO; + using System.Text; + using System.Xml; + using UglyToad.PdfPig.DocumentLayoutAnalysis.Export; + using Xunit; + + public class SvgTextExporterTests + { + [Fact] + public void Doc68_1990_01_A() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("68-1990-01_A.pdf"); + + var pageXmlTextExporter = new SvgTextExporter(); + + Assert.Equal(InvalidCharStrategy.DoNotCheck, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(7); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("68-1990-01_A.7.svg", xml); + } + } + + [Fact] + public void Issue655NoCheckStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new SvgTextExporter(); + + Assert.Equal(InvalidCharStrategy.DoNotCheck, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.nocheck.svg", xml); + + string rawText = File.ReadAllText("issue655.nocheck.svg"); + Assert.Contains("><", rawText); + } + } + + [Fact] + public void Issue655RemoveStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new SvgTextExporter(InvalidCharStrategy.Remove); + + Assert.Equal(InvalidCharStrategy.Remove, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.remove.svg", xml); + + string rawText = File.ReadAllText("issue655.remove.svg"); + Assert.DoesNotContain(">0x06<", rawText); + } + } + + [Fact] + public void Issue655ConvertToHexadecimalStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new SvgTextExporter(InvalidCharStrategy.ConvertToHexadecimal); + + Assert.Equal(InvalidCharStrategy.ConvertToHexadecimal, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.hex.svg", xml); + + string rawText = File.ReadAllText("issue655.hex.svg"); + Assert.Contains(">0x06<", rawText); + } + } + + [Fact] + public void Issue655CustomStrategy() + { + var hex_0x0006 = IntegrationHelpers.GetDocumentPath("hex_0x0006.pdf"); + + var pageXmlTextExporter = new SvgTextExporter( + new Func(s => + { + // Adapted from https://stackoverflow.com/a/17735649 + if (string.IsNullOrEmpty(s)) + { + return s; + } + + int length = s.Length; + StringBuilder stringBuilder = new StringBuilder(length); + for (int i = 0; i < length; ++i) + { + if (XmlConvert.IsXmlChar(s[i])) + { + stringBuilder.Append(s[i]); + } + else + { + stringBuilder.Append("!?"); + } + } + + return stringBuilder.ToString(); + })); + + Assert.Equal(InvalidCharStrategy.Custom, pageXmlTextExporter.InvalidCharStrategy); + + using (var document = PdfDocument.Open(hex_0x0006)) + { + var page = document.GetPage(1); + + // Convert page to text + string xml = pageXmlTextExporter.Get(page); + + // Save text to an xml file + File.WriteAllText("issue655.custom.svg", xml); + + string rawText = File.ReadAllText("issue655.custom.svg"); + Assert.Contains(">!?<", rawText); + } + } + } +}