From a59d1fa97ef8f6f627515b308997b9e8f2042d61 Mon Sep 17 00:00:00 2001
From: Renaud Paquay <rpaquay@microsoft.com>
Date: Sun, 28 Nov 2010 13:28:03 -0800
Subject: [PATCH] Fix bug with ignoring whitespaces at end of expression

--HG--
branch : dev
---
 .../Scripting/ParserTests.cs                  |  8 ++
 .../Orchard.Scripting/Compiler/Tokenizer.cs   | 87 ++++++++++---------
 2 files changed, 52 insertions(+), 43 deletions(-)

diff --git a/src/Orchard.Tests.Modules/Scripting/ParserTests.cs b/src/Orchard.Tests.Modules/Scripting/ParserTests.cs
index 0397bec99..ff12f8c93 100644
--- a/src/Orchard.Tests.Modules/Scripting/ParserTests.cs
+++ b/src/Orchard.Tests.Modules/Scripting/ParserTests.cs
@@ -15,6 +15,14 @@ namespace Orchard.Tests.Modules.Scripting {
             });
         }
 
+        [Test]
+        public void ParserShouldIgnoreWhitespaces() {
+            var tree = new Parser("  true \n  ").Parse();
+            CheckTree(tree, new object[] {
+                "const", true,
+            });
+        }
+
         [Test]
         public void ParserShouldUnderstandBinaryExpressions() {
             var tree = new Parser("true+true").Parse();
diff --git a/src/Orchard.Web/Modules/Orchard.Scripting/Compiler/Tokenizer.cs b/src/Orchard.Web/Modules/Orchard.Scripting/Compiler/Tokenizer.cs
index 4ff5d3c09..a4107d831 100644
--- a/src/Orchard.Web/Modules/Orchard.Scripting/Compiler/Tokenizer.cs
+++ b/src/Orchard.Web/Modules/Orchard.Scripting/Compiler/Tokenizer.cs
@@ -14,52 +14,53 @@ namespace Orchard.Scripting.Compiler {
         }
 
         public Token NextToken() {
-            if (Eof())
-                return CreateToken(TokenKind.Eof);
+            while (true) {
+                if (Eof())
+                    return CreateToken(TokenKind.Eof);
 
-        LexAgain:
-            _startTokenIndex = _index;
-            char ch = Character();
-            switch (ch) {
-                case '(':
-                    NextCharacter();
-                    return CreateToken(TokenKind.OpenParen);
-                case ')':
-                    NextCharacter();
-                    return CreateToken(TokenKind.CloseParen);
-                case ',':
-                    NextCharacter();
-                    return CreateToken(TokenKind.Comma);
-                case '+':
-                    NextCharacter();
-                    return CreateToken(TokenKind.Plus);
-                case '-':
-                    NextCharacter();
-                    return CreateToken(TokenKind.Minus);
-                case '*':
-                    NextCharacter();
-                    return CreateToken(TokenKind.Mul);
-                case '/':
-                    NextCharacter();
-                    return CreateToken(TokenKind.Div);
-                case '"':
-                    return LexStringLiteral();
-                case '\'':
-                    return LexSingleQuotedStringLiteral();
-            }
+                _startTokenIndex = _index;
+                char ch = Character();
+                switch (ch) {
+                    case '(':
+                        NextCharacter();
+                        return CreateToken(TokenKind.OpenParen);
+                    case ')':
+                        NextCharacter();
+                        return CreateToken(TokenKind.CloseParen);
+                    case ',':
+                        NextCharacter();
+                        return CreateToken(TokenKind.Comma);
+                    case '+':
+                        NextCharacter();
+                        return CreateToken(TokenKind.Plus);
+                    case '-':
+                        NextCharacter();
+                        return CreateToken(TokenKind.Minus);
+                    case '*':
+                        NextCharacter();
+                        return CreateToken(TokenKind.Mul);
+                    case '/':
+                        NextCharacter();
+                        return CreateToken(TokenKind.Div);
+                    case '"':
+                        return LexStringLiteral();
+                    case '\'':
+                        return LexSingleQuotedStringLiteral();
+                }
 
-            if (IsDigitCharacter(ch)) {
-                return LexInteger();
-            }
-            else if (IsIdentifierCharacter(ch)) {
-                return LexIdentifierOrKeyword();
-            }
-            else if (IsWhitespaceCharacter(ch)) {
-                NextCharacter();
-                goto LexAgain;
-            }
+                if (IsDigitCharacter(ch)) {
+                    return LexInteger();
+                }
+                else if (IsIdentifierCharacter(ch)) {
+                    return LexIdentifierOrKeyword();
+                }
+                else if (IsWhitespaceCharacter(ch)) {
+                    NextCharacter();
+                    continue;
+                }
 
-            return CreateToken(TokenKind.Invalid, "Unrecognized character");
+                return CreateToken(TokenKind.Invalid, "Unrecognized character");
+            }
         }
 
         private Token LexIdentifierOrKeyword() {