unit tests for tokenizing UTF16 encoded hex strings.

This commit is contained in:
Richard Webb
2019-06-23 01:19:43 +01:00
parent 0432f703c4
commit b5b862e63f

View File

@@ -37,6 +37,19 @@
Assert.Equal(expected, AssertHexToken(token).Data);
}
[Theory]
[InlineData("<FEFF004C0069006200720065004F0066006600690063006500200036002E0031>", "LibreOffice 6.1")]
[InlineData("<FEFF30533093306B3061306F4E16754C>", "こんにちは世界")]
public void HandlesUtf16Strings(string s, string expected)
{
var input = StringBytesTestConverter.Convert(s);
var result = tokenizer.TryTokenize(input.First, input.Bytes, out var token);
Assert.True(result);
Assert.Equal(expected, AssertHexToken(token).Data);
}
private static HexToken AssertHexToken(IToken token)
{
Assert.NotNull(token);