1 /*******************************************************************************
2 * Copyright (c) 2013, 2014 Etienne Bergeron
4 * All rights reserved. This program and the accompanying materials are
5 * made available under the terms of the Eclipse Public License v1.0 which
6 * accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * Etienne Bergeron - Initial API and implementation
11 *******************************************************************************/
13 package org
.eclipse
.tracecompass
.ctf
.parser
.tests
;
15 import static org
.junit
.Assert
.fail
;
17 import java
.util
.LinkedList
;
18 import java
.util
.List
;
20 import org
.antlr
.runtime
.ANTLRStringStream
;
21 import org
.antlr
.runtime
.CharStream
;
22 import org
.antlr
.runtime
.Token
;
23 import org
.eclipse
.tracecompass
.ctf
.parser
.CTFLexer
;
24 import org
.junit
.Ignore
;
25 import org
.junit
.Test
;
28 * This test validates the CTF-Lexer implementation.
30 * The test splits a string into tokens with the compiled lexer and
31 * validates the sequences of tokens produced by comparing their type
34 * @author Etienne Bergeron
36 public class CtfLexerTest
{
38 // ------------------------------------------------------------------------
40 // ------------------------------------------------------------------------
42 private final List
<Token
> tokens
= new LinkedList
<>();
44 // ------------------------------------------------------------------------
46 // ------------------------------------------------------------------------
48 private void tokenize(String content
) {
49 CharStream cs
= new ANTLRStringStream(content
);
50 CTFLexer lexer
= new CTFLexer(cs
);
54 Token token
= lexer
.nextToken();
55 if (token
== Token
.EOF_TOKEN
) {
62 private void checkToken(int type
, String content
) {
63 Token token
= tokens
.remove(0);
64 if (token
.getType() != type
) {
65 fail("Invalid type [value " + token
.getType()
66 + " but expect " + type
+ "]."
67 + " Fail to tokenize:" + content
);
68 } else if (token
.getText().compareTo(content
) != 0) {
69 fail("Invalid content [value " + token
.getText()
70 + " but expect " + content
+ "].");
74 private void checkSingle(int type
, String content
) {
76 checkToken(type
, content
);
79 // ------------------------------------------------------------------------
81 // ------------------------------------------------------------------------
84 * Validate the parsing of keywords
87 public void testKeywords() {
88 checkSingle(CTFLexer
.ALIGNTOK
, "align");
89 checkSingle(CTFLexer
.CONSTTOK
, "const");
90 checkSingle(CTFLexer
.CHARTOK
, "char");
91 checkSingle(CTFLexer
.DOUBLETOK
, "double");
92 checkSingle(CTFLexer
.ENUMTOK
, "enum");
93 checkSingle(CTFLexer
.EVENTTOK
, "event");
94 checkSingle(CTFLexer
.FLOATINGPOINTTOK
, "floating_point");
95 checkSingle(CTFLexer
.FLOATTOK
, "float");
96 checkSingle(CTFLexer
.INTEGERTOK
, "integer");
97 checkSingle(CTFLexer
.INTTOK
, "int");
98 checkSingle(CTFLexer
.LONGTOK
, "long");
99 checkSingle(CTFLexer
.SHORTTOK
, "short");
100 checkSingle(CTFLexer
.SIGNEDTOK
, "signed");
101 checkSingle(CTFLexer
.STREAMTOK
, "stream");
102 checkSingle(CTFLexer
.STRINGTOK
, "string");
103 checkSingle(CTFLexer
.STRUCTTOK
, "struct");
104 checkSingle(CTFLexer
.TRACETOK
, "trace");
105 checkSingle(CTFLexer
.TYPEALIASTOK
, "typealias");
106 checkSingle(CTFLexer
.TYPEDEFTOK
, "typedef");
107 checkSingle(CTFLexer
.UNSIGNEDTOK
, "unsigned");
108 checkSingle(CTFLexer
.VARIANTTOK
, "variant");
109 checkSingle(CTFLexer
.VOIDTOK
, "void");
110 checkSingle(CTFLexer
.BOOLTOK
, "_Bool");
111 checkSingle(CTFLexer
.COMPLEXTOK
, "_Complex");
112 checkSingle(CTFLexer
.IMAGINARYTOK
, "_Imaginary");
113 checkSingle(CTFLexer
.ENVTOK
, "env");
114 checkSingle(CTFLexer
.CLOCKTOK
, "clock");
115 checkSingle(CTFLexer
.CALLSITETOK
, "callsite");
116 checkSingle(CTFLexer
.NANNUMBERTOK
, "NaN");
117 checkSingle(CTFLexer
.INFINITYTOK
, "+inf");
118 checkSingle(CTFLexer
.NINFINITYTOK
, "-inf");
122 * Validate the parsing of symbols
125 public void testSymbols() {
126 tokenize(" , : ... ");
127 checkToken(CTFLexer
.WS
, " ");
128 checkToken(CTFLexer
.SEPARATOR
, ",");
129 checkToken(CTFLexer
.WS
, " ");
130 checkToken(CTFLexer
.COLON
, ":");
131 checkToken(CTFLexer
.WS
, " ");
132 checkToken(CTFLexer
.ELIPSES
, "...");
133 checkToken(CTFLexer
.WS
, " ");
135 tokenize(" = := = ");
136 checkToken(CTFLexer
.WS
, " ");
137 checkToken(CTFLexer
.ASSIGNMENT
, "=");
138 checkToken(CTFLexer
.WS
, " ");
139 checkToken(CTFLexer
.TYPE_ASSIGNMENT
, ":=");
140 checkToken(CTFLexer
.WS
, " ");
143 checkToken(CTFLexer
.WS
, " ");
144 checkToken(CTFLexer
.LT
, "<");
145 checkToken(CTFLexer
.LT
, "<");
146 checkToken(CTFLexer
.GT
, ">");
147 checkToken(CTFLexer
.GT
, ">");
148 checkToken(CTFLexer
.WS
, " ");
150 tokenize(" ({[]}) ");
151 checkToken(CTFLexer
.WS
, " ");
152 checkToken(CTFLexer
.LPAREN
, "(");
153 checkToken(CTFLexer
.LCURL
, "{");
154 checkToken(CTFLexer
.OPENBRAC
, "[");
155 checkToken(CTFLexer
.CLOSEBRAC
, "]");
156 checkToken(CTFLexer
.RCURL
, "}");
157 checkToken(CTFLexer
.RPAREN
, ")");
158 checkToken(CTFLexer
.WS
, " ");
161 checkToken(CTFLexer
.TERM
, ";");
162 checkToken(CTFLexer
.TERM
, ";");
165 checkToken(CTFLexer
.WS
, " ");
166 checkToken(CTFLexer
.SIGN
, "+");
167 checkToken(CTFLexer
.SIGN
, "+");
168 checkToken(CTFLexer
.WS
, " ");
169 checkToken(CTFLexer
.SIGN
, "-");
170 checkToken(CTFLexer
.SIGN
, "-");
171 checkToken(CTFLexer
.WS
, " ");
174 checkToken(CTFLexer
.ARROW
, "->");
175 checkToken(CTFLexer
.WS
, " ");
176 checkToken(CTFLexer
.DOT
, ".");
177 checkToken(CTFLexer
.POINTER
, "*");
178 checkToken(CTFLexer
.DOT
, ".");
182 * Validate the parsing of literals
185 public void testLiterals() {
186 tokenize("01 02 010");
187 checkToken(CTFLexer
.OCTAL_LITERAL
, "01");
188 checkToken(CTFLexer
.WS
, " ");
189 checkToken(CTFLexer
.OCTAL_LITERAL
, "02");
190 checkToken(CTFLexer
.WS
, " ");
191 checkToken(CTFLexer
.OCTAL_LITERAL
, "010");
193 tokenize("1 2 10 1024 ");
194 checkToken(CTFLexer
.DECIMAL_LITERAL
, "1");
195 checkToken(CTFLexer
.WS
, " ");
196 checkToken(CTFLexer
.DECIMAL_LITERAL
, "2");
197 checkToken(CTFLexer
.WS
, " ");
198 checkToken(CTFLexer
.DECIMAL_LITERAL
, "10");
199 checkToken(CTFLexer
.WS
, " ");
200 checkToken(CTFLexer
.DECIMAL_LITERAL
, "1024");
201 checkToken(CTFLexer
.WS
, " ");
203 tokenize("0x01 0x02 0x0F0");
204 checkToken(CTFLexer
.HEX_LITERAL
, "0x01");
205 checkToken(CTFLexer
.WS
, " ");
206 checkToken(CTFLexer
.HEX_LITERAL
, "0x02");
207 checkToken(CTFLexer
.WS
, " ");
208 checkToken(CTFLexer
.HEX_LITERAL
, "0x0F0");
212 * Validate the parsing of literals with hexa prefix
215 public void testLiteralPrefixes() {
216 checkSingle(CTFLexer
.HEX_LITERAL
, "0x1");
217 checkSingle(CTFLexer
.HEX_LITERAL
, "0X1");
221 * Validate the parsing of literals with type suffix
224 public void testLiteralSuffixes() {
225 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0l");
226 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0L");
227 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0ll");
228 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0LL");
229 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0ul");
230 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0uL");
231 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0ull");
232 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0uLL");
233 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0Ul");
234 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0UL");
235 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0Ull");
236 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0ULL");
240 * Validate the accepted characters in literals.
243 public void testLiteralDigits() {
244 checkSingle(CTFLexer
.OCTAL_LITERAL
, "001234567");
246 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "123456");
247 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "987654");
249 checkSingle(CTFLexer
.HEX_LITERAL
, "0x012345");
250 checkSingle(CTFLexer
.HEX_LITERAL
, "0x678990");
251 checkSingle(CTFLexer
.HEX_LITERAL
, "0xABCDEF");
252 checkSingle(CTFLexer
.HEX_LITERAL
, "0xabcdef");
256 * Validate zero literal to be the right token.
259 public void testLiteralZero() {
260 checkSingle(CTFLexer
.OCTAL_LITERAL
, "00");
261 checkSingle(CTFLexer
.DECIMAL_LITERAL
, "0");
262 checkSingle(CTFLexer
.HEX_LITERAL
, "0x0");
266 * Validate character literals
269 public void testCharLiteral() {
270 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'x'");
271 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\''");
272 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "' '");
273 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "L'1'");
277 * Validate escaped character literals
280 public void testEscapeCharLiteral() {
281 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\a'");
282 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\b'");
283 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\f'");
284 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\n'");
285 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\r'");
286 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\t'");
287 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\v'");
288 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\''");
289 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\\"'");
290 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\\\'");
292 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\001'");
293 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\01'");
294 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\1'");
296 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\x1A'");
297 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\x1a'");
298 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\xa'");
299 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\\x0'");
301 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\uABCD'");
302 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\u0123'");
303 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\u012345678'");
304 checkSingle(CTFLexer
.CHARACTER_LITERAL
, "'\uFEDCBA987'");
308 * Validate string literals
311 public void testStringLiteral() {
312 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\"");
313 checkSingle(CTFLexer
.STRING_LITERAL
, "\"x\"");
314 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\\"\"");
315 checkSingle(CTFLexer
.STRING_LITERAL
, "\" \"");
316 checkSingle(CTFLexer
.STRING_LITERAL
, "L\"1\"");
318 checkSingle(CTFLexer
.STRING_LITERAL
, "\"This is \\n a multiline\\r\\n\"");
319 checkSingle(CTFLexer
.STRING_LITERAL
, "L\"This is \\n a multiline\\r\\n\"");
323 * Validate string literals with escape sequence
326 public void testEscapeStringLiteral() {
327 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\a\"");
328 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\b\"");
329 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\f\"");
330 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\n\"");
331 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\r\"");
332 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\t\"");
333 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\v\"");
334 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\'\"");
335 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\\"\"");
336 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\\\\"");
338 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\001\"");
339 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\01\"");
340 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\1\"");
342 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\x1A\"");
343 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\x1a\"");
344 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\xa\"");
345 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\\x0\"");
347 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\uABCD\"");
348 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\u0123\"");
349 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\u012345678\"");
350 checkSingle(CTFLexer
.STRING_LITERAL
, "\"\uFEDCBA987\"");
354 * Validate spaces parsing
357 public void testWhitespaces() {
358 tokenize(" \r\t\n\u000C ");
359 checkToken(CTFLexer
.WS
, " ");
360 checkToken(CTFLexer
.WS
, " ");
361 checkToken(CTFLexer
.WS
, "\r");
362 checkToken(CTFLexer
.WS
, "\t");
363 checkToken(CTFLexer
.WS
, "\n");
364 checkToken(CTFLexer
.WS
, "\u000C");
365 checkToken(CTFLexer
.WS
, " ");
369 * Validate comments parsing
372 public void testComment() {
373 tokenize(" /* test */ ");
374 checkToken(CTFLexer
.WS
, " ");
375 checkToken(CTFLexer
.COMMENT
, "/* test */");
376 checkToken(CTFLexer
.WS
, " ");
380 * Validate complex nested comments parsing
383 public void testNestedComment() {
384 tokenize(" /* /* */ ");
385 checkToken(CTFLexer
.WS
, " ");
386 checkToken(CTFLexer
.COMMENT
, "/* /* */");
387 checkToken(CTFLexer
.WS
, " ");
389 tokenize(" /* /* * ** / */ ");
390 checkToken(CTFLexer
.WS
, " ");
391 checkToken(CTFLexer
.COMMENT
, "/* /* * ** / */");
392 checkToken(CTFLexer
.WS
, " ");
396 * Validate multi-lines comments
399 public void testMultiLineComment() {
400 tokenize(" /*\ntest\n*/ ");
401 checkToken(CTFLexer
.WS
, " ");
402 checkToken(CTFLexer
.COMMENT
, "/*\ntest\n*/");
403 checkToken(CTFLexer
.WS
, " ");
407 * Validate single line comments
410 public void testLineComment() {
411 tokenize(" // asdad\r\n ");
412 checkToken(CTFLexer
.WS
, " ");
413 checkToken(CTFLexer
.LINE_COMMENT
, "// asdad\r\n");
414 checkToken(CTFLexer
.WS
, " ");
418 * Validate incomplete comments parsing
420 @Ignore("Lexer must be fixed first")
422 public void testLineCommentWithEOF() {
424 checkToken(CTFLexer
.LINE_COMMENT
, "//");
428 * Validate parsing of mixed kind of comments
431 public void testMixedComment() {
432 tokenize(" // /*\n");
433 checkToken(CTFLexer
.WS
, " ");
434 checkToken(CTFLexer
.LINE_COMMENT
, "// /*\n");
436 tokenize(" /*\n//\n*/ ");
437 checkToken(CTFLexer
.WS
, " ");
438 checkToken(CTFLexer
.COMMENT
, "/*\n//\n*/");
439 checkToken(CTFLexer
.WS
, " ");
443 * Validate parsing identifiers
446 public void testIdentifier() {
447 tokenize("_ a a1 B ");
448 checkToken(CTFLexer
.IDENTIFIER
, "_");
449 checkToken(CTFLexer
.WS
, " ");
450 checkToken(CTFLexer
.IDENTIFIER
, "a");
451 checkToken(CTFLexer
.WS
, " ");
452 checkToken(CTFLexer
.IDENTIFIER
, "a1");
453 checkToken(CTFLexer
.WS
, " ");
454 checkToken(CTFLexer
.IDENTIFIER
, "B");
455 checkToken(CTFLexer
.WS
, " ");
459 * Validate accepted characters within an identifier
462 public void testIdentifierLetters() {
463 checkSingle(CTFLexer
.IDENTIFIER
, "ABCDEFGHI");
464 checkSingle(CTFLexer
.IDENTIFIER
, "JKLMNOPQR");
465 checkSingle(CTFLexer
.IDENTIFIER
, "STUVWXYZ");
466 checkSingle(CTFLexer
.IDENTIFIER
, "abcdefghi");
467 checkSingle(CTFLexer
.IDENTIFIER
, "jklmnopqr");
468 checkSingle(CTFLexer
.IDENTIFIER
, "stuvwxyz");
469 checkSingle(CTFLexer
.IDENTIFIER
, "_0123456789");
This page took 0.050396 seconds and 5 git commands to generate.