Bunch of renames and class re-organization

--HG--
branch : dev
rename : src/Orchard.Tests.Modules/SimpleScriptingTests/ExpressionEvaluatorTests.cs => src/Orchard.Tests.Modules/SimpleScripting/EvaluatorTests.cs
rename : src/Orchard.Tests.Modules/SimpleScriptingTests/ExpressionParserTests.cs => src/Orchard.Tests.Modules/SimpleScripting/ParserTests.cs
rename : src/Orchard.Tests.Modules/SimpleScriptingTests/SimpleScriptingTests.cs => src/Orchard.Tests.Modules/SimpleScripting/SimpleScriptingTests.cs
rename : src/Orchard.Tests.Modules/SimpleScriptingTests/ExpressionTokenizerTests.cs => src/Orchard.Tests.Modules/SimpleScripting/TokenizerTests.cs
rename : src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/TokenKind.cs => src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/AbstractSyntaxTree.cs
rename : src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/TokenKind.cs => src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/IAstNodeWithToken.cs
rename : src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/ExpressionLexer.cs => src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/Lexer.cs
rename : src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/ExpressionParser.cs => src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/Parser.cs
rename : src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/TokenKind.cs => src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/TerminalKind.cs
rename : src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/ExpressionTokenizer.cs => src/Orchard.Web/Modules/Orchard.Widgets/SimpleScripting/Tokenizer.cs
This commit is contained in:
Renaud Paquay
2010-11-27 17:49:12 -08:00
parent 93835cba2a
commit 19504f2d40
19 changed files with 758 additions and 51 deletions

View File

@@ -135,10 +135,10 @@
</ItemGroup>
<ItemGroup>
<Compile Include="CodeGeneration\Commands\CodeGenerationCommandsTests.cs" />
<Compile Include="SimpleScriptingTests\ExpressionEvaluatorTests.cs" />
<Compile Include="SimpleScriptingTests\ExpressionParserTests.cs" />
<Compile Include="SimpleScriptingTests\ExpressionTokenizerTests.cs" />
<Compile Include="SimpleScriptingTests\SimpleScriptingTests.cs" />
<Compile Include="SimpleScripting\EvaluatorTests.cs" />
<Compile Include="SimpleScripting\ParserTests.cs" />
<Compile Include="SimpleScripting\TokenizerTests.cs" />
<Compile Include="SimpleScripting\SimpleScriptingTests.cs" />
<Compile Include="DatabaseEnabledTestsBase.cs" />
<Compile Include="Media\Extensions\LongExtensionsTests.cs" />
<EmbeddedResource Include="Packaging\HelloDriver.cs.txt" />

View File

@@ -0,0 +1,13 @@
using NUnit.Framework;
using Orchard.Widgets.SimpleScripting;
namespace Orchard.Tests.Modules.SimpleScriptingTests {
[TestFixture]
public class EvaluatorTests {
[Test]
public void EvaluateSimpleConstant() {
var tree = new Parser("1*2+3").Parse();
}
}
}

View File

@@ -5,10 +5,10 @@ using Orchard.Widgets.SimpleScripting;
namespace Orchard.Tests.Modules.SimpleScriptingTests {
[TestFixture]
public class ExpressionParserTests {
public class ParserTests {
[Test]
public void ParserShouldUnderstandConstantExpressions() {
var tree = new ExpressionParser("true").Parse();
var tree = new Parser("true").Parse();
CheckTree(tree, new object[] {
"const", true,
});
@@ -16,9 +16,9 @@ namespace Orchard.Tests.Modules.SimpleScriptingTests {
[Test]
public void ParserShouldUnderstandBinaryExpressions() {
var tree = new ExpressionParser("true+true").Parse();
var tree = new Parser("true+true").Parse();
CheckTree(tree, new object[] {
"binop", TokenKind.Plus,
"binop", TerminalKind.Plus,
"const", true,
"const", true,
});
@@ -26,11 +26,11 @@ namespace Orchard.Tests.Modules.SimpleScriptingTests {
[Test]
public void ParserShouldUnderstandOperatorPrecedence() {
var tree = new ExpressionParser("1+2*3").Parse();
var tree = new Parser("1+2*3").Parse();
CheckTree(tree, new object[] {
"binop", TokenKind.Plus,
"binop", TerminalKind.Plus,
"const", 1,
"binop", TokenKind.Mul,
"binop", TerminalKind.Mul,
"const", 2,
"const", 3,
});
@@ -38,10 +38,10 @@ namespace Orchard.Tests.Modules.SimpleScriptingTests {
[Test]
public void ParserShouldUnderstandOperatorPrecedence2() {
var tree = new ExpressionParser("1*2+3").Parse();
var tree = new Parser("1*2+3").Parse();
CheckTree(tree, new object[] {
"binop", TokenKind.Plus,
"binop", TokenKind.Mul,
"binop", TerminalKind.Plus,
"binop", TerminalKind.Mul,
"const", 1,
"const", 2,
"const", 3,
@@ -50,10 +50,10 @@ namespace Orchard.Tests.Modules.SimpleScriptingTests {
[Test]
public void ParserShouldUnderstandOperatorPrecedence3() {
var tree = new ExpressionParser("not true or true").Parse();
var tree = new Parser("not true or true").Parse();
CheckTree(tree, new object[] {
"binop", TokenKind.Or,
"unop", TokenKind.Not,
"binop", TerminalKind.Or,
"unop", TerminalKind.Not,
"const", true,
"const", true,
});
@@ -61,10 +61,10 @@ namespace Orchard.Tests.Modules.SimpleScriptingTests {
[Test]
public void ParserShouldUnderstandOperatorPrecedence4() {
var tree = new ExpressionParser("not (true or true)").Parse();
var tree = new Parser("not (true or true)").Parse();
CheckTree(tree, new object[] {
"unop", TokenKind.Not,
"binop", TokenKind.Or,
"unop", TerminalKind.Not,
"binop", TerminalKind.Or,
"const", true,
"const", true,
});
@@ -72,11 +72,11 @@ namespace Orchard.Tests.Modules.SimpleScriptingTests {
[Test]
public void ParserShouldUnderstandParenthesis() {
var tree = new ExpressionParser("1*(2+3)").Parse();
var tree = new Parser("1*(2+3)").Parse();
CheckTree(tree, new object[] {
"binop", TokenKind.Mul,
"binop", TerminalKind.Mul,
"const", 1,
"binop", TokenKind.Plus,
"binop", TerminalKind.Plus,
"const", 2,
"const", 3,
});
@@ -84,15 +84,15 @@ namespace Orchard.Tests.Modules.SimpleScriptingTests {
[Test]
public void ParserShouldUnderstandComplexExpressions() {
var tree = new ExpressionParser("not 1 * (2 / 4 * 6 + (3))").Parse();
var tree = new Parser("not 1 * (2 / 4 * 6 + (3))").Parse();
CheckTree(tree, new object[] {
"unop", TokenKind.Not,
"binop", TokenKind.Mul,
"unop", TerminalKind.Not,
"binop", TerminalKind.Mul,
"const", 1,
"binop", TokenKind.Plus,
"binop", TokenKind.Div,
"binop", TerminalKind.Plus,
"binop", TerminalKind.Div,
"const", 2,
"binop", TokenKind.Mul,
"binop", TerminalKind.Mul,
"const", 4,
"const", 6,
"const", 3,
@@ -101,15 +101,15 @@ namespace Orchard.Tests.Modules.SimpleScriptingTests {
[Test]
public void ParserShouldContainErrorExpressions() {
var tree = new ExpressionParser("1 + not 3").Parse();
var tree = new Parser("1 + not 3").Parse();
CheckTree(tree, new object[] {
"binop", TokenKind.Plus,
"binop", TerminalKind.Plus,
"const", 1,
"error",
});
}
private void CheckTree(ExpressionTree tree, object[] objects) {
private void CheckTree(AbstractSyntaxTree tree, object[] objects) {
Assert.That(tree, Is.Not.Null);
Assert.That(tree.Root, Is.Not.Null);
@@ -118,39 +118,39 @@ namespace Orchard.Tests.Modules.SimpleScriptingTests {
Assert.That(index, Is.EqualTo(objects.Length));
}
private void CheckExpression(ExpressionTree.Expression expression, int indent, object[] objects, ref int index) {
private void CheckExpression(AstNode astNode, int indent, object[] objects, ref int index) {
var exprName = (string)objects[index++];
Type type = null;
switch(exprName) {
case "const":
type = typeof(ExpressionTree.ConstantExpression);
type = typeof(ConstantAstNode);
break;
case "binop":
type = typeof(ExpressionTree.BinaryExpression);
type = typeof(BinaryAstNode);
break;
case "unop":
type = typeof(ExpressionTree.UnaryExpression);
type = typeof(UnaryAstNode);
break;
case "error":
type = typeof(ExpressionTree.ErrorExpression);
type = typeof(ErrorAstNode);
break;
}
Trace.WriteLine(string.Format("{0}: {1}{2} (Current: {3})", indent, new string(' ', indent * 2), type.Name, expression));
Trace.WriteLine(string.Format("{0}: {1}{2} (Current: {3})", indent, new string(' ', indent * 2), type.Name, astNode));
Assert.That(expression.GetType(), Is.EqualTo(type));
Assert.That(astNode.GetType(), Is.EqualTo(type));
if (exprName == "const") {
Assert.That((expression as ExpressionTree.ConstantExpression).Value, Is.EqualTo(objects[index++]));
Assert.That((astNode as ConstantAstNode).Value, Is.EqualTo(objects[index++]));
}
else if (exprName == "binop") {
Assert.That((expression as ExpressionTree.BinaryExpression).Operator.Kind, Is.EqualTo(objects[index++]));
Assert.That((astNode as BinaryAstNode).Operator.Kind, Is.EqualTo(objects[index++]));
}
else if (exprName == "unop") {
Assert.That((expression as ExpressionTree.UnaryExpression).Operator.Kind, Is.EqualTo(objects[index++]));
Assert.That((astNode as UnaryAstNode).Operator.Kind, Is.EqualTo(objects[index++]));
}
foreach(var child in expression.Children) {
foreach(var child in astNode.Children) {
CheckExpression(child, indent + 1, objects, ref index);
}
}

View File

@@ -0,0 +1,16 @@
using System.Linq;
using NUnit.Framework;
using Orchard.Tests.Stubs;
using Orchard.Widgets.Services;
using Orchard.Widgets.SimpleScripting;
namespace Orchard.Tests.Modules.SimpleScripting {
[TestFixture]
public class SimpleScriptingTests {
[Test]
public void EngineUnderstandsPrimitiveValues() {
//var engine = new ScriptingEngine(Enumerable.Empty<IRuleProvider>(), new StubCacheManager());
//Assert.That(engine.Matches("true"), Is.True);
}
}
}

View File

@@ -0,0 +1,76 @@
using NUnit.Framework;
using Orchard.Widgets.SimpleScripting;
namespace Orchard.Tests.Modules.SimpleScriptingTests {
[TestFixture]
public class TokenizerTests {
[Test]
public void LexerShouldProcessSingleQuotedStringLiteral() {
TestStringLiteral(@"'toto'", @"toto", TerminalKind.SingleQuotedStringLiteral);
TestStringLiteral(@"'to\'to'", @"to'to", TerminalKind.SingleQuotedStringLiteral);
TestStringLiteral(@"'to\\to'", @"to\to", TerminalKind.SingleQuotedStringLiteral);
TestStringLiteral(@"'to\ato'", @"to\ato", TerminalKind.SingleQuotedStringLiteral);
}
[Test]
public void LexerShouldProcessStringLiteral() {
TestStringLiteral(@"""toto""", @"toto", TerminalKind.StringLiteral);
TestStringLiteral(@"""to\'to""", @"to'to", TerminalKind.StringLiteral);
TestStringLiteral(@"""to\\to""", @"to\to", TerminalKind.StringLiteral);
TestStringLiteral(@"""to\ato""", @"toato", TerminalKind.StringLiteral);
}
private void TestStringLiteral(string value, string expected, TerminalKind expectedTerminalKind) {
var lexer = new Tokenizer(value);
var token1 = lexer.NextToken();
Assert.That(token1.Kind, Is.EqualTo(expectedTerminalKind));
Assert.That(token1.Value, Is.EqualTo(expected));
var token2 = lexer.NextToken();
Assert.That(token2.Kind, Is.EqualTo(TerminalKind.Eof));
}
[Test]
public void LexerShouldProcessReservedWords() {
TestReservedWord("true", true, TerminalKind.True);
TestReservedWord("false", false, TerminalKind.False);
TestReservedWord("not", null, TerminalKind.Not);
TestReservedWord("and", null, TerminalKind.And);
TestReservedWord("or", null, TerminalKind.Or);
}
private void TestReservedWord(string expression, object value, TerminalKind expectedTerminalKind) {
var lexer = new Tokenizer(expression);
var token1 = lexer.NextToken();
Assert.That(token1.Kind, Is.EqualTo(expectedTerminalKind));
Assert.That(token1.Value, Is.EqualTo(value));
var token2 = lexer.NextToken();
Assert.That(token2.Kind, Is.EqualTo(TerminalKind.Eof));
}
[Test]
public void LexerShouldProcesSequenceOfTokens() {
CheckTokenSequence("true false", TerminalKind.True, TerminalKind.False);
CheckTokenSequence("true toto false", TerminalKind.True, TerminalKind.Identifier, TerminalKind.False);
}
[Test]
public void LexerShouldProcesSequenceOfTokens2() {
CheckTokenSequence("1+2*3", TerminalKind.Integer, TerminalKind.Plus, TerminalKind.Integer, TerminalKind.Mul, TerminalKind.Integer);
}
private void CheckTokenSequence(string expression, params TerminalKind[] terminalKinds) {
var lexer = new Tokenizer(expression);
foreach (var kind in terminalKinds) {
var token = lexer.NextToken();
Assert.That(token.Kind, Is.EqualTo(kind));
}
var token2 = lexer.NextToken();
Assert.That(token2.Kind, Is.EqualTo(TerminalKind.Eof));
}
}
}

View File

@@ -75,12 +75,19 @@
<Compile Include="Services\RuleContext.cs" />
<Compile Include="Services\WidgetsService.cs" />
<Compile Include="Shapes.cs" />
<Compile Include="SimpleScripting\ExpressionLexer.cs" />
<Compile Include="SimpleScripting\ExpressionTokenizer.cs" />
<Compile Include="SimpleScripting\ExpressionParser.cs" />
<Compile Include="SimpleScripting\ExpressionTree.cs" />
<Compile Include="SimpleScripting\AstNode.cs" />
<Compile Include="SimpleScripting\BinaryAstNode.cs" />
<Compile Include="SimpleScripting\ConstantAstNode.cs" />
<Compile Include="SimpleScripting\ErrorAstNode.cs" />
<Compile Include="SimpleScripting\Lexer.cs" />
<Compile Include="SimpleScripting\Tokenizer.cs" />
<Compile Include="SimpleScripting\Parser.cs" />
<Compile Include="SimpleScripting\AbstractSyntaxTree.cs" />
<Compile Include="SimpleScripting\IAstNodeWithToken.cs" />
<Compile Include="SimpleScripting\ScriptingManager.cs" />
<Compile Include="SimpleScripting\TokenKind.cs" />
<Compile Include="SimpleScripting\Terminal.cs" />
<Compile Include="SimpleScripting\TerminalKind.cs" />
<Compile Include="SimpleScripting\UnaryAstNode.cs" />
<Compile Include="ViewModels\WidgetsIndexViewModel.cs" />
</ItemGroup>
<ItemGroup>

View File

@@ -0,0 +1,5 @@
namespace Orchard.Widgets.SimpleScripting {
public class AbstractSyntaxTree {
public AstNode Root { get; set; }
}
}

View File

@@ -0,0 +1,24 @@
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace Orchard.Widgets.SimpleScripting {
public class AstNode {
public virtual IEnumerable<AstNode> Children {
get {
return Enumerable.Empty<AstNode>();
}
}
public override string ToString() {
var sb = new StringBuilder();
sb.Append(this.GetType().Name);
var ewt = (this as IAstNodeWithToken);
if (ewt != null) {
sb.Append(" - ");
sb.Append(ewt.Terminal);
}
return sb.ToString();
}
}
}

View File

@@ -0,0 +1,30 @@
using System.Collections.Generic;
namespace Orchard.Widgets.SimpleScripting {
public class BinaryAstNode : AstNode, IAstNodeWithToken {
private readonly AstNode _left;
private readonly Terminal _terminal;
private readonly AstNode _right;
public BinaryAstNode(AstNode left, Terminal terminal, AstNode right) {
_left = left;
_terminal = terminal;
_right = right;
}
public Terminal Terminal {
get { return _terminal; }
}
public Terminal Operator {
get { return _terminal; }
}
public override IEnumerable<AstNode> Children {
get {
yield return _left;
yield return _right;
}
}
}
}

View File

@@ -0,0 +1,15 @@
namespace Orchard.Widgets.SimpleScripting {
public class ConstantAstNode : AstNode, IAstNodeWithToken {
private readonly Terminal _terminal;
public ConstantAstNode(Terminal terminal) {
_terminal = terminal;
}
public Terminal Terminal {
get { return _terminal; }
}
public object Value { get { return _terminal.Value; } }
}
}

View File

@@ -0,0 +1,25 @@
using System;
namespace Orchard.Widgets.SimpleScripting {
public class ErrorAstNode : AstNode, IAstNodeWithToken {
private readonly Terminal _terminal;
private readonly string _message;
public ErrorAstNode(Terminal terminal, string message) {
_terminal = terminal;
_message = message;
}
public Terminal Terminal {
get { return _terminal; }
}
public string Message {
get { return _message; }
}
public override string ToString() {
return String.Format("{0} - {1}", GetType().Name, Message);
}
}
}

View File

@@ -0,0 +1,5 @@
namespace Orchard.Widgets.SimpleScripting {
public interface IAstNodeWithToken {
Terminal Terminal { get; }
}
}

View File

@@ -0,0 +1,44 @@
using System.Collections.Generic;
namespace Orchard.Widgets.SimpleScripting {
public class Lexer {
private readonly Tokenizer _tokenizer;
private readonly List<Terminal> _tokens= new List<Terminal>();
private int _tokenIndex;
public Lexer(Tokenizer tokenizer) {
_tokenizer = tokenizer;
}
public Terminal Token() {
if (_tokenIndex == _tokens.Count) {
_tokens.Add(_tokenizer.NextToken());
}
return _tokens[_tokenIndex];
}
public void NextToken() {
_tokenIndex++;
}
public Marker Mark() {
return new Marker(_tokens.Count);
}
public void Mark(Marker marker) {
_tokenIndex = marker.TokenIndex;
}
public struct Marker {
private readonly int _tokenIndex;
public Marker(int tokenIndex) {
_tokenIndex = tokenIndex;
}
public int TokenIndex {
get { return _tokenIndex; }
}
}
}
}

View File

@@ -0,0 +1,170 @@
using System;
using System.Collections.Generic;
namespace Orchard.Widgets.SimpleScripting {
public class Parser {
private readonly string _expression;
private readonly Lexer _lexer;
private readonly List<string> _errors = new List<string>();
public Parser(string expression) {
_expression = expression;
_lexer = new Lexer(new Tokenizer(_expression));
}
public AbstractSyntaxTree Parse() {
var node = ParseExpression();
return new AbstractSyntaxTree { Root = node };
}
private AstNode ParseExpression() {
return ParseKeywordLogicalExpression();
}
private AstNode ParseKeywordLogicalExpression() {
return ParseKeywordOrExpression();
}
private AstNode ParseKeywordOrExpression() {
var expr = ParseKeywordAndExpression();
var token = IsMatch(TerminalKind.Or);
if (token != null)
{
var right = ParseKeywordOrExpression();
expr = new BinaryAstNode(expr, token, right);
}
return expr;
}
private AstNode ParseKeywordAndExpression() {
var expr = ParseKeywordNotExpression();
var token = IsMatch(TerminalKind.And);
if (token != null) {
var right = ParseKeywordAndExpression();
expr = new BinaryAstNode(expr, token, right);
}
return expr;
}
private AstNode ParseKeywordNotExpression() {
var token = IsMatch(TerminalKind.Not);
if (token != null) {
var expr = ParseKeywordNotExpression();
return new UnaryAstNode(expr, token);
}
return ParseRelationalExpression();
}
private AstNode ParseRelationalExpression() {
var expr = ParseAdditiveExpression();
//TODO
//var Terminal = IsMatch(TokenKind.Not);
//if (Terminal != null) {
return expr;
}
private AstNode ParseAdditiveExpression() {
var expr = ParseMultiplicativeExpression();
var token = IsMatch(TerminalKind.Plus, TerminalKind.Minus);
if (token != null) {
var right = ParseAdditiveExpression();
expr = new BinaryAstNode(expr, token, right);
}
return expr;
}
private AstNode ParseMultiplicativeExpression() {
var expr = ParseUnaryExpression();
var token = IsMatch(TerminalKind.Mul, TerminalKind.Div);
if (token != null) {
var right = ParseMultiplicativeExpression();
expr = new BinaryAstNode(expr, token, right);
}
return expr;
}
private AstNode ParseUnaryExpression() {
//TODO
return ParsePrimaryExpression();
}
private AstNode ParsePrimaryExpression() {
var token = _lexer.Token();
switch(_lexer.Token().Kind) {
case TerminalKind.True:
case TerminalKind.False:
case TerminalKind.SingleQuotedStringLiteral:
case TerminalKind.StringLiteral:
case TerminalKind.Integer:
return ProduceConstant(token);
case TerminalKind.OpenParen:
return ParseParenthesizedExpression();
default:
return ProduceError(token);
}
}
private AstNode ProduceConstant(Terminal terminal) {
_lexer.NextToken();
return new ConstantAstNode(terminal);
}
private AstNode ProduceError(Terminal terminal) {
_lexer.NextToken();
return new ErrorAstNode(terminal,
string.Format("Unexptected Terminal in primary expression ({0})", terminal));
}
private AstNode ParseParenthesizedExpression() {
Match(TerminalKind.OpenParen);
var expr = ParseExpression();
Match(TerminalKind.CloseParen);
return expr;
}
private void Match(TerminalKind kind) {
var token = _lexer.Token();
if (token.Kind == kind) {
_lexer.NextToken();
return;
}
AddError(token, string.Format("Expected Terminal {0}", kind));
}
private Terminal IsMatch(TerminalKind kind) {
var token = _lexer.Token();
if (token.Kind == kind) {
_lexer.NextToken();
return token;
}
return null;
}
private Terminal IsMatch(TerminalKind kind, TerminalKind kind2) {
var token = _lexer.Token();
if (token.Kind == kind || token.Kind == kind2) {
_lexer.NextToken();
return token;
}
return null;
}
private void AddError(Terminal terminal, string message) {
_errors.Add(message);
}
}
}

View File

@@ -28,11 +28,11 @@ namespace Orchard.Widgets.SimpleScripting {
return (bool)Convert.ChangeType(result, typeof (bool));
}
private ExpressionTree ParseExpression(string expression) {
return new ExpressionParser(expression).Parse();
private AbstractSyntaxTree ParseExpression(string expression) {
return new Parser(expression).Parse();
}
private object EvaluateExpression(ExpressionTree.Expression root) {
private object EvaluateExpression(AstNode root) {
throw new NotImplementedException();
}

View File

@@ -0,0 +1,13 @@
using System;
namespace Orchard.Widgets.SimpleScripting {
public class Terminal {
public TerminalKind Kind { get; set; }
public int Position { get; set; }
public object Value { get; set; }
public override string ToString() {
return String.Format("{0} ({1}) at position {2}", Kind, Value ?? "<noval>", Position);
}
}
}

View File

@@ -0,0 +1,21 @@
namespace Orchard.Widgets.SimpleScripting {
public enum TerminalKind {
Eof,
OpenParen,
CloseParen,
StringLiteral,
SingleQuotedStringLiteral,
Integer,
Plus,
Minus,
Mul,
Div,
True,
False,
Identifier,
And,
Or,
Not,
Invalid
}
}

View File

@@ -0,0 +1,215 @@
using System;
using System.Text;
namespace Orchard.Widgets.SimpleScripting {
public class Tokenizer {
private readonly string _expression;
private readonly StringBuilder _stringBuilder;
private int _index;
private int _startTokenIndex;
public Tokenizer(string expression) {
_expression = expression;
_stringBuilder = new StringBuilder();
}
public Terminal NextToken() {
if (Eof())
return CreateToken(TerminalKind.Eof);
LexAgain:
_startTokenIndex = _index;
char ch = Character();
switch (ch) {
case '(':
NextCharacter();
return CreateToken(TerminalKind.OpenParen);
case ')':
NextCharacter();
return CreateToken(TerminalKind.CloseParen);
case '+':
NextCharacter();
return CreateToken(TerminalKind.Plus);
case '-':
NextCharacter();
return CreateToken(TerminalKind.Minus);
case '*':
NextCharacter();
return CreateToken(TerminalKind.Mul);
case '/':
NextCharacter();
return CreateToken(TerminalKind.Div);
case '"':
return LexStringLiteral();
case '\'':
return LexSingleQuotedStringLiteral();
}
if (IsDigitCharacter(ch)) {
return LexInteger();
}
else if (IsIdentifierCharacter(ch)) {
return LexIdentifierOrKeyword();
}
else if (IsWhitespaceCharacter(ch)) {
NextCharacter();
goto LexAgain;
}
return CreateToken(TerminalKind.Invalid, "Unrecognized character");
}
private Terminal LexIdentifierOrKeyword() {
_stringBuilder.Clear();
_stringBuilder.Append(Character());
while (true) {
NextCharacter();
if (!Eof() && (IsIdentifierCharacter(Character()) || IsDigitCharacter(Character()))) {
_stringBuilder.Append(Character());
}
else {
return CreateIdentiferOrKeyword(_stringBuilder.ToString());
}
}
}
private Terminal LexInteger() {
_stringBuilder.Clear();
_stringBuilder.Append(Character());
while (true) {
NextCharacter();
if (!Eof() && IsDigitCharacter(Character())) {
_stringBuilder.Append(Character());
}
else {
return CreateToken(TerminalKind.Integer, Int32.Parse(_stringBuilder.ToString()));
}
}
}
private Terminal CreateIdentiferOrKeyword(string identifier) {
switch (identifier) {
case "true":
return CreateToken(TerminalKind.True, true);
case "false":
return CreateToken(TerminalKind.False, false);
case "or":
return CreateToken(TerminalKind.Or, null);
case "and":
return CreateToken(TerminalKind.And, null);
case "not":
return CreateToken(TerminalKind.Not, null);
default:
return CreateToken(TerminalKind.Identifier, identifier);
}
}
private bool IsWhitespaceCharacter(char character) {
return char.IsWhiteSpace(character);
}
private bool IsIdentifierCharacter(char ch) {
return
(ch >= 'a' && ch <= 'z') ||
(ch >= 'A' && ch <= 'Z') ||
(ch == '_');
}
private bool IsDigitCharacter(char ch) {
return ch >= '0' && ch <= '9';
}
private Terminal LexSingleQuotedStringLiteral() {
_stringBuilder.Clear();
while (true) {
NextCharacter();
if (Eof())
return CreateToken(TerminalKind.Invalid, "Unterminated string literal");
// Termination
if (Character() == '\'') {
NextCharacter();
return CreateToken(TerminalKind.SingleQuotedStringLiteral, _stringBuilder.ToString());
}
// backslash notation
else if (Character() == '\\') {
NextCharacter();
if (Eof())
return CreateToken(TerminalKind.Invalid, "Unterminated string literal");
if (Character() == '\\') {
_stringBuilder.Append('\\');
}
else if (Character() == '\'') {
_stringBuilder.Append('\'');
}
else {
_stringBuilder.Append('\\');
_stringBuilder.Append(Character());
}
}
// Regular character in string
else {
_stringBuilder.Append(Character());
}
}
}
private Terminal LexStringLiteral() {
_stringBuilder.Clear();
while (true) {
NextCharacter();
if (Eof())
return CreateToken(TerminalKind.Invalid, "Unterminated string literal");
// Termination
if (Character() == '"') {
NextCharacter();
return CreateToken(TerminalKind.StringLiteral, _stringBuilder.ToString());
}
// backslash notation
else if (Character() == '\\') {
NextCharacter();
if (Eof())
return CreateToken(TerminalKind.Invalid, "Unterminated string literal");
_stringBuilder.Append(Character());
}
// Regular character in string
else {
_stringBuilder.Append(Character());
}
}
}
private void NextCharacter() {
_index++;
}
private char Character() {
return _expression[_index];
}
private Terminal CreateToken(TerminalKind kind, object value = null) {
return new Terminal {
Kind = kind,
Position = _startTokenIndex,
Value = value
};
}
private bool Eof() {
return (_index >= _expression.Length);
}
}
}

View File

@@ -0,0 +1,28 @@
using System.Collections.Generic;
namespace Orchard.Widgets.SimpleScripting {
public class UnaryAstNode : AstNode, IAstNodeWithToken {
private readonly AstNode _expr;
private readonly Terminal _terminal;
public UnaryAstNode(AstNode expr, Terminal terminal) {
_expr = expr;
_terminal = terminal;
}
public Terminal Terminal {
get { return _terminal; }
}
public Terminal Operator {
get { return _terminal; }
}
public override IEnumerable<AstNode> Children {
get {
yield return _expr;
}
}
}
}