address bugs and add tests

when a stream contains two endstream declarations with
no data between them then the first declared endstream
should be obeyed

src/UglyToad.PdfPig/Tokenization/Scanner/PdfTokenScanner.cs

@@ -377,17 +377,11 @@
 
             long streamDataStart = inputBytes.CurrentOffset;
 
-            PossibleStreamEndLocation? possibleEndLocation = null;
-
-
+            // Negative indicates endobj.
+            bool isEndData = false;
+            Stack<EndLoc> endLocations = new Stack<EndLoc>();
             while (inputBytes.MoveNext())
             {
-                if (length.HasValue && read == length)
-                {
-                    // TODO: read ahead and check we're at the end...
-                    // break;
-                }
-
                 // We are reading 'end' (possibly).
                 if (commonPartPosition < endWordPart.Length && inputBytes.CurrentByte == endWordPart[commonPartPosition])
                 {
@@ -401,44 +395,54 @@
                         endObjPosition = 0;
                         endStreamPosition++;
 
-                        // We've finished reading 'endstream', add it to the end tokens we've seen.
-                        if (endStreamPosition == streamPart.Length && (!inputBytes.MoveNext() || ReadHelper.IsWhitespace(inputBytes.CurrentByte)))
+                        if (endStreamPosition == streamPart.Length)
                         {
-                            var token = new PossibleStreamEndLocation(inputBytes.CurrentOffset - OperatorToken.EndStream.Data.Length, OperatorToken.EndStream);
-
-                            possibleEndLocation = token;
-
-                            if (length.HasValue && read > length)
+                            // Token is at end of stream or is followed by whitespace
+                            if (!inputBytes.MoveNext() || ReadHelper.IsWhitespace(inputBytes.CurrentByte))
                             {
-                                break;
-                            }
+                                var location = inputBytes.CurrentOffset - EndstreamBytes.Length - 1;
+                                endLocations.Push(new EndLoc(true, location, !isEndData));
+                                isEndData = true;
 
-                            endStreamPosition = 0;
+                                if (length.HasValue && read > length)
+                                {
+                                    break;
+                                }
+
+                                endStreamPosition = 0;
+                                commonPartPosition = 0;
+                            }
+                            else
+                            {
+                                endStreamPosition = 0;
+                                commonPartPosition = 0;
+                                isEndData = false;
+                            }
                         }
                     }
                     else if (inputBytes.CurrentByte == objPart[endObjPosition])
                     {
                         // We are reading 'obj' after 'end'
-
                         endStreamPosition = 0;
                         endObjPosition++;
 
                         // We have finished reading 'endobj'.
                         if (endObjPosition == objPart.Length)
                         {
+                            var hasPreviousEndToken = endLocations.Count > 0;
                             // If we saw an 'endstream' or 'endobj' previously we've definitely hit the end now.
-                            if (possibleEndLocation != null)
+                            if (hasPreviousEndToken)
                             {
-                                var lastEndToken = possibleEndLocation.Value;
+                                var lastEndTokenLocation = endLocations.Peek();
 
-                                inputBytes.Seek(lastEndToken.Offset + lastEndToken.Type.Data.Length + 1);
+                                var correction = lastEndTokenLocation.IsEndStream ? EndstreamBytes.Length : "endobj".Length;
+                                inputBytes.Seek(lastEndTokenLocation.Offset + correction + 1);
 
                                 break;
                             }
 
-                            var token = new PossibleStreamEndLocation(inputBytes.CurrentOffset - OperatorToken.EndObject.Data.Length, OperatorToken.EndObject);
-
-                            possibleEndLocation = token;
+                            endLocations.Push(new EndLoc(false, inputBytes.CurrentOffset, !isEndData));
+                            isEndData = true;
 
                             if (read > length)
                             {
@@ -446,6 +450,10 @@
                             }
                         }
                     }
+                    else if (inputBytes.CurrentByte == endWordPart[0])
+                    {
+                        commonPartPosition = 1;
+                    }
                     else
                     {
                         // We were reading 'end' but then we had a character mismatch.
@@ -454,6 +462,7 @@
                         endStreamPosition = 0;
                         endObjPosition = 0;
                         commonPartPosition = 0;
+                        isEndData = false;
                     }
                 }
                 else
@@ -463,6 +472,11 @@
                     endStreamPosition = 0;
                     endObjPosition = 0;
                     commonPartPosition = (inputBytes.CurrentByte == endWordPart[0]) ? 1 : 0;
+
+                    if (commonPartPosition == 0 && !ReadHelper.IsWhitespace(inputBytes.CurrentByte))
+                    {
+                        isEndData = false;
+                    }
                 }
 
                 read++;
@@ -470,27 +484,36 @@
 
             long streamDataEnd = inputBytes.CurrentOffset + 1;
 
-            if (possibleEndLocation == null)
+            if (endLocations.Count == 0)
+            {
                 return false;
-
-            var lastEnd = possibleEndLocation;
-
-            var dataLength = lastEnd.Value.Offset - startDataOffset;
-
-            // 3 characters, 'e', '\n' and possibly '\r'
-            inputBytes.Seek(lastEnd.Value.Offset - 3);
-            inputBytes.MoveNext();
-
-            if (inputBytes.CurrentByte == '\r')
-            {
-                dataLength -= 3;
-            }
-            else
-            {
-                dataLength -= 2;
             }
 
-            Memory<byte> data = new byte[dataLength];
+            // Read until the first endstream or obj token indicator preceded by data.
+            EndLoc endLoc;
+            do
+            {
+                endLoc = endLocations.Pop();
+            } while (!endLoc.HasDataPreceding && endLocations.Count > 0);
+
+            var dataLength = endLoc.Offset - startDataOffset;
+
+            inputBytes.Seek(endLoc.Offset - 1);
+
+            var adjustment = 0;
+            bool isWhitespace;
+            do
+            {
+                inputBytes.MoveNext();
+                isWhitespace = ReadHelper.IsWhitespace(inputBytes.CurrentByte);
+                if (isWhitespace)
+                {
+                    adjustment++;
+                }
+                inputBytes.Seek(endLoc.Offset - 1 - adjustment);
+            } while (isWhitespace);
+
+            Memory<byte> data = new byte[dataLength - adjustment];
 
             inputBytes.Seek(streamDataStart);
             inputBytes.Read(data.Span);
@@ -902,5 +925,21 @@
             inputBytes?.Dispose();
             isDisposed = true;
         }
+
+        private record EndLoc
+        {
+            public bool IsEndStream { get; }
+
+            public long Offset { get; }
+
+            public bool HasDataPreceding { get; }
+
+            public EndLoc(bool isEndStream, long offset, bool hasDataPreceding)
+            {
+                IsEndStream = isEndStream;
+                Offset = offset;
+                HasDataPreceding = hasDataPreceding;
+            }
+        }
     }
 }

src/UglyToad.PdfPig/Tokenization/Scanner/PossibleStreamEndLocation.cs

@@ -1,35 +0,0 @@
-namespace UglyToad.PdfPig.Tokenization.Scanner
-{
-    using System;
-    using Tokens;
-
-    /// <summary>
-    /// Used internally by the <see cref="PdfTokenScanner"/> when reading streams to store any occurrences of 'endobj' or 'endstream' observed.
-    /// </summary>
-    internal readonly struct PossibleStreamEndLocation
-    {
-        /// <summary>
-        /// The offset at which the token started in the file.
-        /// </summary>
-        public long Offset { get; }
-
-        /// <summary>
-        /// The type, one of either <see cref="OperatorToken.EndObject"/> or <see cref="OperatorToken.EndStream"/>.
-        /// </summary>
-        public OperatorToken Type { get; }
-
-        /// <summary>
-        /// Create a new <see cref="PossibleStreamEndLocation"/>
-        /// </summary>
-        public PossibleStreamEndLocation(long offset, OperatorToken type)
-        {
-            Offset = offset;
-            Type = type ?? throw new ArgumentNullException(nameof(type));
-        }
-
-        public override string ToString()
-        {
-            return $"{Offset}: {Type}";
-        }
-    }
-}