fix test since document is now opened successfully but mediabox is broken

the file 10404 (ironically) contains not found references with number 43 0
for its info dictionary. changes brute-force code so that objects can be
entirely missing

.github/workflows/run_common_crawl_tests.yml

@@ -9,6 +9,11 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        pair: ["0000-0001", "0002-0003", "0004-0005", "0006-0007"]
+
     steps:
       - uses: actions/checkout@v2
 
@@ -19,42 +24,35 @@ jobs:
             8.0.x
             9.0.x
           
-      - name: Restore corpus cache 0000, 0001
+      - name: Restore corpus cache
         id: restore-corpus
         uses: actions/cache@v4
         with:
           path: corpus/
-          key: ${{ runner.os }}-pdf-corpus-0000-0001
+          key: ${{ runner.os }}-pdf-corpus-${{ matrix.pair }}
 
-      - name: Download corpus if cache missed 0000, 0001
+      - name: Download corpus if cache missed
         if: steps.restore-corpus.outputs.cache-hit != 'true'
         run: |
           mkdir -p corpus/zipfiles
           cd corpus/zipfiles
-          echo "Downloading 0000.zip"
-          wget -nv https://digitalcorpora.s3.amazonaws.com/corpora/files/CC-MAIN-2021-31-PDF-UNTRUNCATED/zipfiles/0000-0999/0000.zip -O 0000.zip
-          wget -nv https://digitalcorpora.s3.amazonaws.com/corpora/files/CC-MAIN-2021-31-PDF-UNTRUNCATED/zipfiles/0000-0999/0001.zip -O 0001.zip
+          for file in $(echo "${{ matrix.pair }}" | tr '-' ' '); do
+            echo "Downloading $file.zip"
+            wget -nv "https://digitalcorpora.s3.amazonaws.com/corpora/files/CC-MAIN-2021-31-PDF-UNTRUNCATED/zipfiles/0000-0999/$file.zip" -O "$file.zip"
+          done
           cd ..
           unzip 'zipfiles/*.zip' -d extracted
           # run: rm -f zipfiles/*.zip
 
       - name: Remove unwanted test files
         run: |
-          skip_files=(
-            "corpus/extracted/0000399.pdf"
-            "corpus/extracted/0000819.pdf"
-            "corpus/extracted/0000920.pdf"
-            "corpus/extracted/0000300.pdf"
-            "corpus/extracted/0001589.pdf"
-            "corpus/extracted/0001957.pdf"
-          )
-
-          for file in "${skip_files[@]}"; do
-            if [ -f "$file" ]; then
-              echo "Removing $file"
-              rm "$file"
+          while read f || [ -n "$f" ]; do
+            full="corpus/extracted/$f"
+            if [ -f "$full" ]; then
+              echo "Removing $full"
+              rm "$full"
             fi
-          done
+          done < tools/common-crawl-ignore.txt
 
       - name: Run tests against corpus
         run: dotnet run --project tools/UglyToad.PdfPig.ConsoleRunner/UglyToad.PdfPig.ConsoleRunner.csproj "corpus/extracted"

src/UglyToad.PdfPig.Tests/Integration/GithubIssuesTests.cs

@@ -102,8 +102,10 @@
         {
             var path = IntegrationHelpers.GetSpecificTestDocumentPath("Hang.pdf");
 
-            var ex = Assert.Throws<PdfDocumentFormatException>(() => PdfDocument.Open(path, new ParsingOptions() { UseLenientParsing = true }));
-            Assert.StartsWith("Could not locate object with reference:", ex.Message);
+            using var doc = PdfDocument.Open(path, new ParsingOptions { UseLenientParsing = true });
+
+            var ex = Assert.Throws<PdfDocumentFormatException>(() => doc.GetPage(1));
+            Assert.StartsWith("Could not find", ex.Message);
         }
 
         [Fact]

src/UglyToad.PdfPig/Content/ResourceStore.cs

@@ -21,8 +21,7 @@
 
         private readonly Dictionary<IndirectReference, IFont> loadedFonts = new Dictionary<IndirectReference, IFont>();
         private readonly Dictionary<NameToken, IFont> loadedDirectFonts = new Dictionary<NameToken, IFont>();
-        private readonly StackDictionary<NameToken, IndirectReference> currentFontState = new StackDictionary<NameToken, IndirectReference>();
-        private readonly StackDictionary<NameToken, IndirectReference> currentXObjectState = new StackDictionary<NameToken, IndirectReference>();
+        private readonly StackDictionary<NameToken, IndirectReference> currentResourceState = new StackDictionary<NameToken, IndirectReference>();
 
         private readonly Dictionary<NameToken, DictionaryToken> extendedGraphicsStates = new Dictionary<NameToken, DictionaryToken>();
 
@@ -54,8 +53,7 @@
             loadedNamedColorSpaceDetails.Clear();
 
             namedColorSpaces.Push();
-            currentFontState.Push();
-            currentXObjectState.Push();
+            currentResourceState.Push();
 
             if (resourceDictionary.TryGet(NameToken.Font, out var fontBase))
             {
@@ -80,7 +78,7 @@
                         throw new InvalidOperationException($"Expected the XObject dictionary value for key /{pair.Key} to be an indirect reference, instead got: {pair.Value}.");
                     }
 
-                    currentXObjectState[NameToken.Create(pair.Key)] = reference.Data;
+                    currentResourceState[NameToken.Create(pair.Key)] = reference.Data;
                 }
             }
 
@@ -187,8 +185,7 @@
         {
             lastLoadedFont = (null, null);
             loadedNamedColorSpaceDetails.Clear();
-            currentFontState.Pop();
-            currentXObjectState.Pop();
+            currentResourceState.Pop();
             namedColorSpaces.Pop();
         }
 
@@ -202,7 +199,7 @@
                 {
                     var reference = objectKey.Data;
 
-                    currentFontState[NameToken.Create(pair.Key)] = reference;
+                    currentResourceState[NameToken.Create(pair.Key)] = reference;
 
                     if (loadedFonts.ContainsKey(reference))
                     {
@@ -248,7 +245,7 @@
             }
 
             IFont? font;
-            if (currentFontState.TryGetValue(name, out var reference))
+            if (currentResourceState.TryGetValue(name, out var reference))
             {
                 loadedFonts.TryGetValue(reference, out font);
             }
@@ -338,7 +335,7 @@
         public bool TryGetXObject(NameToken name, [NotNullWhen(true)] out StreamToken? stream)
         {
             stream = null;
-            if (!currentXObjectState.TryGetValue(name, out var indirectReference))
+            if (!currentResourceState.TryGetValue(name, out var indirectReference))
             {
                 return false;
             }

src/UglyToad.PdfPig/Tokenization/Scanner/PdfTokenScanner.cs

@@ -770,7 +770,8 @@
 
             if (!MoveNext())
             {
-                return BruteForceFileToFindReference(reference);
+                TryBruteForceFileToFindReference(reference, out var bfObjectToken);
+                return bfObjectToken;
             }
 
             var found = (ObjectToken)CurrentToken!;
@@ -780,7 +781,9 @@
                 return found;
             }
 
-            return BruteForceFileToFindReference(reference);
+            TryBruteForceFileToFindReference(reference, out var bfToken);
+
+            return bfToken;
         }
 
         public void ReplaceToken(IndirectReference reference, IToken token)
@@ -790,8 +793,9 @@
             overwrittenTokens[reference] = new ObjectToken(0, reference, token);
         }
 
-        private ObjectToken BruteForceFileToFindReference(IndirectReference reference)
+        private bool TryBruteForceFileToFindReference(IndirectReference reference, [NotNullWhen(true)] out ObjectToken? result)
         {
+            result = null;
             try
             {
                 // Brute force read the entire file
@@ -806,10 +810,12 @@
 
                 if (!objectLocationProvider.TryGetCached(reference, out var objectToken))
                 {
-                    throw new PdfDocumentFormatException($"Could not locate object with reference: {reference} despite a full document search.");
+                    return false;
                 }
 
-                return objectToken;
+                result = objectToken;
+
+                return true;
             }
             finally
             {

tools/common-crawl-ignore.txt

@@ -0,0 +1,49 @@
+0000399.pdf
+0000819.pdf
+0000920.pdf
+0000300.pdf
+0001589.pdf
+0001957.pdf
+0002064.pdf
+0002187.pdf
+0002229.pdf
+0002244.pdf
+0002372.pdf
+0002554.pdf
+0002638.pdf
+0002883.pdf
+0002897.pdf
+0002966.pdf
+0003147.pdf
+0003269.pdf
+0003304.pdf
+0003892.pdf
+0003927.pdf
+0003983.pdf
+0004099.pdf
+0004537.pdf
+0004569.pdf
+0004791.pdf
+0004853.pdf
+0004889.pdf
+0004997.pdf
+0005482.pdf
+0005637.pdf
+0006036.pdf
+0006169.pdf
+0006207.pdf
+0006262.pdf
+0006339.pdf
+0006766.pdf
+0006844.pdf
+0007159.pdf
+0007559.pdf
+0008404.pdf
+0008443.pdf
+0008674.pdf
+0008978.pdf
+0009290.pdf
+0009309.pdf
+0009464.pdf
+0009706.pdf
+0009944.pdf