start adding support for undocumented revision 6 encryption

revision 6 was added in the pdf 2.0 specification which is document iso 32000-2:2017. because iso are rent-seeking they charge money to view this specification so it is effectively undocumented. this site details some of the algorithm https://web.archive.org/web/20180311160224/esec-lab.sogeti.com/posts/2011/09/14/the-undocumented-password-validation-algorithm-of-adobe-reader-x.html. the code in this change ports the pdfbox logic line by line. it doesn't implement the correct behaviour for owner password yet.

src/UglyToad.PdfPig/Encryption/EncryptionAlgorithmCode.cs

@@ -28,6 +28,10 @@
         /// <summary>
         ///  The security handler defines the use of encryption and decryption in the document with a key length of 256 bits.
         /// </summary>
-        SecurityHandlerInDocument256 = 5
+        SecurityHandlerInDocument256 = 5,
+        /// <summary>
+        /// Since ISO isn't fit for purpose they charge £200 to see the PDF 2 spec so it's not possible to know what the specification for this revision is. 
+        /// </summary>
+        UndocumentedDueToIso = 6
     }
 }

src/UglyToad.PdfPig/Encryption/EncryptionHandler.cs

@@ -182,12 +182,15 @@
             Array.Copy(encryptionDictionary.UserBytes, userPasswordHash, 32);
             Array.Copy(encryptionDictionary.UserBytes, 32, userValidationSalt, 0, 8);
 
+            byte[] result;
             if (encryptionDictionary.Revision == 6)
             {
-                throw new PdfDocumentEncryptedException($"Support for revision 6 encryption not implemented: {encryptionDictionary}.");
+                result = ComputeStupidIsoHash(truncatedPassword, userValidationSalt, null);
+            }
+            else
+            {
+                result = ComputeSha256Hash(truncatedPassword, userValidationSalt);
             }
-
-            var result = ComputeSha256Hash(truncatedPassword, userValidationSalt);
 
             return result.SequenceEqual(userPasswordHash);
         }
@@ -359,15 +362,15 @@
                         break;
                     }
                 case HexToken hexToken:
-                {
+                    {
-                    var data = hexToken.Bytes.ToArray();
+                        var data = hexToken.Bytes.ToArray();
 
-                    var decrypted = DecryptData(data, reference);
+                        var decrypted = DecryptData(data, reference);
 
-                    token = new HexToken(Hex.GetString(decrypted).ToCharArray());
+                        token = new HexToken(Hex.GetString(decrypted).ToCharArray());
 
-                    break;
+                        break;
-                }
+                    }
                 case DictionaryToken dictionary:
                     {
                         // PDFBOX-2936: avoid orphan /CF dictionaries found in US govt "I-" files
@@ -584,8 +587,16 @@
             var userKeySalt = new byte[8];
             Array.Copy(encryptionDictionary.UserBytes, 40, userKeySalt, 0, 8);
 
-            var intermediateKey = ComputeSha256Hash(password, userKeySalt);
+            byte[] intermediateKey;
-
+            if (encryptionDictionary.Revision == 6)
+            {
+                intermediateKey = ComputeStupidIsoHash(password, userKeySalt, null);
+            }
+            else
+            {
+                intermediateKey = ComputeSha256Hash(password, userKeySalt);
+            }
+            
             var iv = new byte[16];
 
             using (var rijndaelManaged = new RijndaelManaged { Key = intermediateKey, IV = iv, Mode = CipherMode.CBC, Padding = PaddingMode.None })
@@ -659,5 +670,117 @@
             Array.Copy(password, result, 127);
             return result;
         }
+
+        /// <summary>
+        /// This is the revision 6 algorithm the specification for which can't be viewed unless you pay the ISO a lot of money (which makes me extremely angry).
+        /// This is an attempt to port directly from PDFBox code.
+        /// </summary>
+        /// <param name="password">The truncated user or owner password.</param>
+        /// <param name="salt">The 8 byte user or owner key validation salt.</param>
+        /// <param name="vector">Used when hashing owner password in which case it is the 48 byte /U key.</param>
+        private static byte[] ComputeStupidIsoHash(byte[] password, byte[] salt, byte[] vector)
+        {
+            // There are some details here https://web.archive.org/web/20180311160224/esec-lab.sogeti.com/posts/2011/09/14/the-undocumented-password-validation-algorithm-of-adobe-reader-x.html
+            if (vector == null)
+            {
+                vector = EmptyArray<byte>.Instance;
+            }
+            else if (vector.Length > 0 && vector.Length < 48)
+            {
+                throw new PdfDocumentEncryptedException($"Vector for revision 6 owner password check (/U) is the wrong length, expected 48 bytes got {vector.Length} bytes.");
+            }
+            else if (vector.Length > 48)
+            {
+                var temp = new byte[48];
+                Array.Copy(vector, temp, temp.Length);
+                vector = temp;
+            }
+
+            password = TruncatePasswordTo127Bytes(password);
+
+            byte[] input;
+            using (var sha256 = SHA256.Create())
+            {
+                sha256.TransformBlock(password, 0, password.Length, null, 0);
+                sha256.TransformBlock(salt, 0, salt.Length, null, 0);
+                sha256.TransformBlock(vector, 0, vector.Length, null, 0);
+                sha256.TransformFinalBlock(EmptyArray<byte>.Instance, 0, 0);
+                input = sha256.Hash;
+            }
+
+            var key = new byte[16];
+            Array.Copy(input, key, 16);
+
+            var iv = new byte[16];
+            Array.Copy(input, 16, iv, 0, iv.Length);
+
+            byte[] x = null;
+            var i = 0;
+            while (i < 64 || i < x[x.Length - 1] + 32)
+            {
+                var roundResult = new byte[64 * (password.Length + input.Length + vector.Length)];
+
+                var position = 0;
+                for (var j = 0; j < 64; j++)
+                {
+                    Array.Copy(password, 0, roundResult, position, password.Length);
+                    position += password.Length;
+                    Array.Copy(input, 0, roundResult, position, input.Length);
+                    position += input.Length;
+                    if (vector.Length > 0)
+                    {
+                        Array.Copy(vector, 0, roundResult, position, vector.Length);
+                        position += vector.Length;
+                    }
+                }
+
+                using (var aes = Aes.Create())
+                {
+                    aes.Key = key;
+                    aes.IV = iv;
+                    aes.Mode = CipherMode.CBC;
+                    aes.Padding = PaddingMode.None;
+
+                    using (var decryptor = aes.CreateEncryptor())
+                    {
+                        x = decryptor.TransformFinalBlock(roundResult, 0, roundResult.Length);
+                    }
+                }
+
+                var sumOfFirstSixteenBytesOfX = x.Take(16).Sum(v => (long)v);
+                var mod3 = sumOfFirstSixteenBytesOfX % 3;
+
+                HashAlgorithm nextHash;
+                switch (mod3)
+                {
+                    case 0:
+                        nextHash = SHA256.Create();
+                        break;
+                    case 1:
+                        nextHash = SHA384.Create();
+                        break;
+                    case 2:
+                        nextHash = SHA512.Create();
+                        break;
+                    default:
+                        throw new PdfDocumentEncryptedException("Invalid remainder from summing first sixteen bytes of this round's hash.");
+                }
+
+                input = nextHash.ComputeHash(x);
+                Array.Copy(input, key, 16);
+                Array.Copy(input, 16, iv, 0, 16);
+
+                i++;
+            }
+
+            if (input.Length > 32)
+            {
+                var result = new byte[32];
+                Array.Copy(input, result, result.Length);
+                return result;
+            }
+
+            return input;
+        }
     }
 }