Implement tokenization of relational operators

--HG--
branch : dev

src/Orchard.Tests.Modules/Scripting/TokenizerTests.cs

@@ -61,6 +61,12 @@ namespace Orchard.Tests.Modules.Scripting {
             CheckTokenSequence("1+2*3", TokenKind.Integer, TokenKind.Plus, TokenKind.Integer, TokenKind.Mul, TokenKind.Integer);
         }
 
+        [Test]
+        public void LexerShouldProcesSequenceOfTokens3() {
+            CheckTokenSequence("= == < <= > >= ! !=", TokenKind.Equal, TokenKind.EqualEqual, 
+                TokenKind.LessThan, TokenKind.LessThanEqual, 
+                TokenKind.GreaterThan, TokenKind.GreaterThanEqual, TokenKind.NotSign, TokenKind.NotEqual);
+        }
 
         private void CheckTokenSequence(string expression, params TokenKind[] tokenKinds) {
             var lexer = new Tokenizer(expression);

src/Orchard.Web/Modules/Orchard.Scripting/Compiler/Token.cs

@@ -7,7 +7,10 @@ namespace Orchard.Scripting.Compiler {
         public object Value { get; set; }
 
         public override string ToString() {
-            return String.Format("{0} ({1}) at position {2}", Kind, Value ?? "<noval>", Position);
+            if (Value == null)
+                return String.Format("Token {0} at position {1}", Kind, Position);
+            else
+                return String.Format("Token {0} ({1}) at position {2}", Kind, Value, Position);
         }
     }
 }

src/Orchard.Web/Modules/Orchard.Scripting/Compiler/TokenKind.cs

@@ -1,10 +1,12 @@
 namespace Orchard.Scripting.Compiler {
     public enum TokenKind {
+        Invalid,
         Eof,
         OpenParen,
         CloseParen,
         StringLiteral,
         SingleQuotedStringLiteral,
+        Identifier,
         Integer,
         Comma,
         Plus,
@@ -13,10 +15,16 @@
         Div,
         True,
         False,
-        Identifier,
         And,
         Or,
         Not,
-        Invalid
+        Equal,
+        EqualEqual,
+        NotSign,
+        NotEqual,
+        LessThan,
+        LessThanEqual,
+        GreaterThan,
+        GreaterThanEqual
     }
 }

src/Orchard.Web/Modules/Orchard.Scripting/Compiler/Tokenizer.cs

@@ -46,6 +46,14 @@ namespace Orchard.Scripting.Compiler {
                         return LexStringLiteral();
                     case '\'':
                         return LexSingleQuotedStringLiteral();
+                    case '!':
+                        return LexNotSign();
+                    case '=':
+                        return LexEqual();
+                    case '<':
+                        return LexLessThan();
+                    case '>':
+                        return LexGreaterThan();
                 }
 
                 if (IsDigitCharacter(ch)) {
@@ -63,6 +71,46 @@ namespace Orchard.Scripting.Compiler {
             }
         }
 
+        private Token LexNotSign() {
+            NextCharacter();
+            char ch = Character();
+            if (ch == '=') {
+                NextCharacter();
+                return CreateToken(TokenKind.NotEqual);
+            }
+            return CreateToken(TokenKind.NotSign);
+        }
+
+        private Token LexGreaterThan() {
+            NextCharacter();
+            char ch = Character();
+            if (ch == '=') {
+                NextCharacter();
+                return CreateToken(TokenKind.GreaterThanEqual);
+            }
+            return CreateToken(TokenKind.GreaterThan);
+        }
+
+        private Token LexLessThan() {
+            NextCharacter();
+            char ch = Character();
+            if (ch == '=') {
+                NextCharacter();
+                return CreateToken(TokenKind.LessThanEqual);
+            }
+            return CreateToken(TokenKind.LessThan);
+        }
+
+        private Token LexEqual() {
+            NextCharacter();
+            char ch = Character();
+            if (ch == '=') {
+                NextCharacter();
+                return CreateToken(TokenKind.EqualEqual);
+            }
+            return CreateToken(TokenKind.Equal);
+        }
+
         private Token LexIdentifierOrKeyword() {
             _stringBuilder.Clear();