Make JetStream 2
[WebKit-https.git] / PerformanceTests / JetStream2 / RexBench / OfflineAssembler / parser.js
1 /*
2  * Copyright (C) 2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 "use strict";
27
28 const GlobalAnnotation = "global";
29 const LocalAnnotation = "local";
30
31 class SourceFile
32 {
33     constructor(fileName)
34     {
35         this._fileName = fileName;
36         let fileNumber = SourceFile.fileNames.indexOf(fileName);
37         if (fileNumber == -1) {
38             SourceFile.fileNames.push(fileName);
39             fileNumber = SourceFile.fileNames.length;
40         } else
41             fileNumber++; // File numbers are 1 based
42
43         this._fileNumber = fileNumber;
44     }
45
46     get name()
47     {
48         return this._fileName;
49     }
50
51     get fileNumber()
52     {
53         return this._fileNumber;
54     }
55 }
56
57 SourceFile.fileNames = [];
58
59 class CodeOrigin
60 {
61     constructor(sourceFile, lineNumber)
62     {
63         this._sourceFile = sourceFile;
64         this._lineNumber = lineNumber;
65     }
66
67     fileName()
68     {
69         return this._sourceFile.name;
70     }
71
72     debugDirective()
73     {
74         return emitWinAsm ? undefined : "\".loc " + this._sourceFile.fileNumber + " " + this._lineNumber + "\\n\"";
75     }
76
77     toString()
78     {
79         return this.fileName() + ":" + this._lineNumber;
80     }
81
82     get lineNumber()
83     {
84         return this._lineNumber;
85     }
86 }
87
88 class IncludeFile
89 {
90     constructor(moduleName, defaultDir)
91     {
92         this._fileName = moduleName + ".asm";
93     }
94
95     toString()
96     {
97         return fileName;
98     }
99
100     get fileName()
101     {
102         return this._fileName;
103     }
104 }
105
106 IncludeFile.includeDirs = [];
107
108
109 class Token
110 {
111     constructor(codeOrigin, string)
112     {
113         this._codeOrigin = codeOrigin;
114         this._string = string;
115     }
116
117     isEqualTo(other)
118     {
119         if (other instanceof Token)
120             return this.string === other.string;
121
122         return this.string === other;
123     }
124
125     isNotEqualTo(other)
126     {
127         return !this.isEqualTo(other);
128     }
129
130     get string()
131     {
132         return this._string;
133     }
134
135     get codeOrigin()
136     {
137         return this._codeOrigin;
138     }
139
140     toString()
141     {
142         return "" + this._string + "\" at " + this._codeOrigin;
143     }
144
145     parseError(comment)
146     {
147         if (!comment || !comment.length)
148             throw "Parse error: " + this;
149
150         throw "Parse error: " + this + ": " + comment;
151     }
152 }
153
154 class Annotation
155 {
156     constructor(codeOrigin, type, string)
157     {
158         this.codeOrigin = codeOrigin;
159         this.type = type;
160         this.string = string;
161     }
162
163     get codeOrigin()
164     {
165         return this.codeOrigin;
166     }
167
168     get type()
169     {
170         return this.type;
171     }
172
173     get string()
174     {
175         return this.string;
176     }
177 }
178
179
180 // The lexer. Takes a string and returns an array of tokens.
181
182 function lex(str, file)
183 {
184     function scanRegExp(source, regexp)
185     {
186         return source.match(regexp);
187     }
188
189     let result = [];
190     let lineNumber = 1;
191     let annotation = null;
192     let annotationType;
193     let whitespaceFound = false;
194
195     while (str.length)
196     {
197         let tokenMatch;
198         let annotation;
199         let annotationType;
200
201         if (tokenMatch = scanRegExp(str, /^#([^\n]*)/))
202             ; // comment, ignore
203         else if (tokenMatch = scanRegExp(str, /^\/\/\ ?([^\n]*)/)) {
204             // annotation
205             annotation = tokenMatch[0];
206             annotationType = whitespaceFound ? LocalAnnotation : GlobalAnnotation;
207         } else if (tokenMatch = scanRegExp(str, /^\n/)) {
208             /* We've found a '\n'.  Emit the last comment recorded if appropriate:
209              * We need to parse annotations regardless of whether the backend does
210              * anything with them or not. This is because the C++ backend may make
211              * use of this for its cloopDo debugging utility even if
212              * enableInstrAnnotations is not enabled.
213              */
214             if (annotation) {
215                 result.push(new Annotation(new CodeOrigin(file, lineNumber),
216                                            annotationType, annotation));
217                 annotation = null;
218             }
219             result.push(new Token(new CodeOrigin(file, lineNumber), tokenMatch[0]));
220             lineNumber++;
221         } else if (tokenMatch = scanRegExp(str, /^[a-zA-Z]([a-zA-Z0-9_.]*)/))
222             result.push(new Token(new CodeOrigin(file, lineNumber), tokenMatch[0]));
223         else if (tokenMatch = scanRegExp(str, /^\.([a-zA-Z0-9_]*)/))
224             result.push(new Token(new CodeOrigin(file, lineNumber), tokenMatch[0]));
225         else if (tokenMatch = scanRegExp(str, /^_([a-zA-Z0-9_]*)/))
226             result.push(new Token(new CodeOrigin(file, lineNumber), tokenMatch[0]));
227         else if (tokenMatch = scanRegExp(str, /^([ \t]+)/)) {
228             // whitespace, ignore
229             whitespaceFound = true;
230             str = str.slice(tokenMatch[0].length);
231             continue;
232         } else if (tokenMatch = scanRegExp(str, /^0x([0-9a-fA-F]+)/))
233             result.push(new Token(new CodeOrigin(file, lineNumber), Number.parseInt(tokenMatch[1], 16)));
234         else if (tokenMatch = scanRegExp(str, /^0([0-7]+)/))
235             result.push(new Token(new CodeOrigin(file, lineNumber), Number.parseInt(tokenMatch[1], 8)));
236         else if (tokenMatch = scanRegExp(str, /^([0-9]+)/))
237             result.push(new Token(new CodeOrigin(file, lineNumber), tokenMatch[0]));
238         else if (tokenMatch = scanRegExp(str,  /^::/))
239             result.push(new Token(new CodeOrigin(file, lineNumber), tokenMatch[0]));
240         else if (tokenMatch = scanRegExp(str, /^[:,\(\)\[\]=\+\-~\|&^*]/))
241             result.push(new Token(new CodeOrigin(file, lineNumber), tokenMatch[0]));
242         else if (tokenMatch = scanRegExp(str, /^\".*\"/))
243             result.push(new Token(new CodeOrigin(file, lineNumber), tokenMatch[0]));
244         else
245             throw "Lexer error at " + (new CodeOrigin(file, lineNumber)) + ", unexpected sequence " + str.slice(0, 20);
246
247         whitespaceFound = false;
248         str = str.slice(tokenMatch[0].length);
249     }
250
251     return result;
252 }
253
254 // Token identification.
255
256 function isRegister(token)
257 {
258     registerPattern.test(token.string);
259 }
260
261 function isInstruction(token)
262 {
263     return instructionSet.has(token.string);
264 }
265
266 function isKeyword(token)
267 {
268     return /^((true)|(false)|(if)|(then)|(else)|(elsif)|(end)|(and)|(or)|(not)|(global)|(macro)|(const)|(constexpr)|(sizeof)|(error)|(include))$/.test(token.string)
269         || isRegister(token)
270         || isInstruction(token);
271 }
272
273 function isIdentifier(token)
274 {
275     return /^[a-zA-Z]([a-zA-Z0-9_.]*)$/.test(token.string) && !isKeyword(token);
276 }
277
278 function isLabel(token)
279 {
280     let tokenString;
281     if (token instanceof Token)
282         tokenString = token.string;
283     else
284         tokenString = token;
285
286     return /^_([a-zA-Z0-9_]*)$/.test(tokenString);
287 }
288
289 function isLocalLabel(token)
290 {
291     let tokenString;
292     if (token instanceof Token)
293         tokenString = token.string;
294     else
295         tokenString = token;
296
297     return /^\.([a-zA-Z0-9_]*)$/.test(tokenString);
298 }
299
300 function isVariable(token)
301 {
302     return isIdentifier(token) || isRegister(token);
303 }
304
305 function isInteger(token)
306 {
307     return /^[0-9]/.test(token.string);
308 }
309
310 function isString(token)
311 {
312     return /^".*"/.test(token);
313 }
314
315
316 // The parser. Takes an array of tokens and returns an AST. Methods
317 // other than parse(tokens) are not for public consumption.
318
319 class Parser
320 {
321     constructor(data, fileName)
322     {
323         this.tokens = lex(data, fileName);
324         this.idx = 0;
325         this.annotation = null;
326     }
327
328     parseError(comment)
329     {
330         if (this.tokens[this.idx])
331             this.tokens[this.idx].parseError(comment);
332         else {
333             if (!comment.length)
334                 throw "Parse error at end of file";
335
336             throw "Parse error at end of file: " + comment;
337         }
338     }
339
340     consume(regexp)
341     {
342         if (regexp) {
343             if (!regexp.test(this.tokens[this.idx].string))
344                 this.parseError();
345         } else if (this.idx != this.tokens.length)
346             this.parseError();
347
348         this.idx++;
349     }
350
351     skipNewLine()
352     {
353         while (this.tokens[this.idx].isEqualTo("\n"))
354             this.idx++;
355     }
356
357     parsePredicateAtom()
358     {
359         if (this.tokens[this.idx].isEqualTo("not")) {
360             let codeOrigin = this.tokens[this.idx].codeOrigin;
361             this.idx++;
362             return new Not(codeOrigin, this.parsePredicateAtom());
363         }
364         if (this.tokens[this.idx].isEqualTo("(")) {
365             this.idx++;
366             skipNewLine();
367             let result = this.parsePredicate();
368             if (this.tokens[this.idx].isNotEqualTo(")"))
369                 parseError();
370             this.idx++;
371             return result;
372         }
373         if (this.tokens[this.idx].isEqualTo("true")) {
374             let result = True.instance();
375             this.idx++;
376             return result;
377         }
378         if (this.tokens[this.idx].isEqualTo("false")) {
379             let result = False.instance();
380             this.idx++;
381             return result;
382         }
383         if (isIdentifier(this.tokens[this.idx])) {
384             let result = Setting.forName(this.tokens[this.idx].codeOrigin, this.tokens[this.idx].string);
385             this.idx++;
386             return result;
387         }
388
389         this.parseError();
390     }
391
392     parsePredicateAnd()
393     {
394         let result = this.parsePredicateAtom();
395         while (this.tokens[this.idx].isEqualTo("and")) {
396             let codeOrigin = this.tokens[this.idx].codeOrigin;
397             this.idx++;
398             this.skipNewLine();
399             let right = this.parsePredicateAtom();
400             result = new And(codeOrigin, result, right);
401         }
402
403         return result;
404     }
405
406     parsePredicate()
407     {
408         // some examples of precedence:
409         // not a and b -> (not a) and b
410         // a and b or c -> (a and b) or c
411         // a or b and c -> a or (b and c)
412
413         let result = this.parsePredicateAnd();
414         while (this.tokens[this.idx].isEqualTo("or")) {
415             let codeOrigin = this.tokens[this.idx].codeOrigin;
416             this.idx++;
417             this.skipNewLine();
418             let right = this.parsePredicateAnd();
419             result = new Or(codeOrigin, result, right);
420         }
421
422         return result;
423     }
424
425     parseVariable()
426     {
427         let result;
428         if (isRegister(this.tokens[this.idx])) {
429             if (fprPattern.test(this.tokens[this.idx]))
430                 result = FPRegisterID.forName(this.tokens[this.idx].codeOrigin, this.tokens[this.idx].string);
431             else
432                 result = RegisterID.forName(this.tokens[this.idx].codeOrigin, this.tokens[this.idx].string);
433         } else if (isIdentifier(this.tokens[this.idx]))
434             result = Variable.forName(this.tokens[this.idx].codeOrigin, this.tokens[this.idx].string);
435         else
436             this.parseError();
437
438         this.idx++;
439         return result;
440     }
441
442     parseAddress(offset)
443     {
444         if (this.tokens[this.idx].isNotEqualTo("["))
445             this.parseError();
446
447         let codeOrigin = this.tokens[this.idx].codeOrigin;
448         let result;
449
450         // Three possibilities:
451         // []       -> AbsoluteAddress
452         // [a]      -> Address
453         // [a,b]    -> BaseIndex with scale = 1
454         // [a,b,c]  -> BaseIndex
455
456         this.idx++;
457         if (this.tokens[this.idx].isEqualTo("]")) {
458             this.idx++;
459             return new AbsoluteAddress(codeOrigin, offset);
460         }
461         let a = this.parseVariable();
462         if (this.tokens[this.idx].isEqualTo("]"))
463             result = new Address(codeOrigin, a, offset);
464         else
465         {
466             if (this.tokens[this.idx].isNotEqualTo(","))
467                 this.parseError();
468             this.idx++;
469             let b = this.parseVariable();
470             if (this.tokens[this.idx].isEqualTo("]"))
471                 result = new BaseIndex(codeOrigin, a, b, 1, offset);
472             else {
473                 if (this.tokens[this.idx].isNotEqualTo(","))
474                     this.parseError();
475                 this.idx++;
476                 if (!["1", "2", "4", "8"].includes(this.tokens[this.idx].string))
477                     this.parseError();
478                 let c = Number.parseInt(this.tokens[this.idx]);
479                 this.idx++;
480                 if (this.tokens[this.idx].isNotEqualTo("]"))
481                     this.parseError();
482                 result = new BaseIndex(codeOrigin, a, b, c, offset);
483             }
484         }
485         this.idx++;
486         return result;
487     }
488
489     parseColonColon()
490     {
491         this.skipNewLine();
492         let firstToken = this.tokens[this.idx];
493         let codeOrigin = this.tokens[this.idx].codeOrigin;
494         if (!isIdentifier(this.tokens[this.idx]))
495             this.parseError();
496         let names = [this.tokens[this.idx].string];
497         this.idx++;
498         while (this.tokens[this.idx].isEqualTo("::")) {
499             this.idx++;
500             if (!isIdentifier(this.tokens[this.idx]))
501                 this.parseError();
502             names.push(this.tokens[this.idx].string);
503             this.idx++;
504         }
505         if (!names.length)
506             firstToken.parseError();
507         return { codeOrigin: codeOrigin, names: names };
508     }
509
510     parseTextInParens()
511     {
512         this.skipNewLine();
513         let codeOrigin = this.tokens[this.idx].codeOrigin;
514         if (this.tokens[this.idx].isNotEqualTo("("))
515             throw "Missing \"(\" at " + codeOrigin;
516         this.idx++;
517         // need at least one item
518         if (this.tokens[this.idx].isEqualTo(")"))
519             throw "No items in list at " + codeOrigin;
520         let numEnclosedParens = 0;
521         let text = [];
522         while (this.tokens[this.idx].isNotEqualTo(")") || numEnclosedParens > 0) {
523             if (this.tokens[this.idx].isEqualTo("("))
524                 numEnclosedParens++;
525             else if (this.tokens[this.idx].isEqualTo(")"))
526                 numEnclosedParens--;
527
528             text.push(this.tokens[this.idx].string);
529             this.idx++;
530         }
531         this.idx++;
532         return { codeOrigin: codeOrigin, text: text };
533     }
534
535     parseExpressionAtom()
536     {
537         let result;
538         this.skipNewLine();
539         if (this.tokens[this.idx].isEqualTo("-")) {
540             this.idx++;
541             return new NegImmediate(this.tokens[this.idx - 1].codeOrigin, this.parseExpressionAtom());
542         }
543         if (this.tokens[this.idx].isEqualTo("~")) {
544             this.idx++;
545             return new BitnotImmediate(this.tokens[this.idx - 1].codeOrigin, this.parseExpressionAtom());
546         }
547         if (this.tokens[this.idx].isEqualTo("(")) {
548             this.idx++;
549             result = this.parseExpression();
550             if (this.tokens[this.idx].isNotEqualTo(")"))
551                 this.parseError();
552             this.idx++;
553             return result;
554         }
555         if (isInteger(this.tokens[this.idx])) {
556             result = new Immediate(this.tokens[this.idx].codeOrigin, Number.parseInt(this.tokens[this.idx]));
557             this.idx++;
558             return result;
559         }
560         if (isString(this.tokens[this.idx])) {
561             result = new StringLiteral(this.tokens[this.idx].codeOrigin, this.tokens[this.idx].string);
562             this.idx++;
563             return result;
564         }
565         if (isIdentifier(this.tokens[this.idx])) {
566             let {codeOrigin, names} = this.parseColonColon();
567             if (names.length > 1)
568                 return StructOffset.forField(codeOrigin, names.slice(0, -1).join('::'), names[names.length - 1]);
569
570             return Variable.forName(codeOrigin, names[0]);
571         }
572         if (isRegister(this.tokens[this.idx]))
573             return this.parseVariable();
574         if (this.tokens[this.idx].isEqualTo("sizeof")) {
575             this.idx++;
576             let {codeOrigin, names} = this.parseColonColon();
577             return Sizeof.forName(codeOrigin, names.join("::"));
578         }
579         if (this.tokens[this.idx].isEqualTo("constexpr")) {
580             this.idx++;
581             this.skipNewLine();
582             let codeOrigin;
583             let text;
584             let names;
585             if (this.tokens[this.idx].isEqualTo("(")) {
586                 ({ codeOrigin, text } = this.parseTextInParens());
587                 text = text.join("");
588             } else {
589                 ({ codeOrigin, names } = this.parseColonColon());
590                 text = names.join("::");
591             }
592             return ConstExpr.forName(codeOrigin, text);
593         }
594         if (isLabel(this.tokens[this.idx])) {
595             result = new LabelReference(this.tokens[this.idx].codeOrigin, Label.forName(this.tokens[this.idx].codeOrigin, this.tokens[this.idx].string));
596             this.idx++;
597             return result;
598         }
599         if (isLocalLabel(this.tokens[this.idx])) {
600             result = new LocalLabelReference(this.tokens[this.idx].codeOrigin, LocalLabel.forName(this.tokens[this.idx].codeOrigin, this.tokens[this.idx].string));
601             this.idx++;
602             return result;
603         }
604         this.parseError();
605     }
606
607     parseExpressionMul()
608     {
609         this.skipNewLine();
610         let result = this.parseExpressionAtom();
611         while (this.tokens[this.idx].isEqualTo("*")) {
612             if (this.tokens[this.idx].isEqualTo("*")) {
613                 this.idx++;
614                 result = new MulImmediates(this.tokens[this.idx - 1].codeOrigin, result, this.parseExpressionAtom());
615             } else
616                 throw "Invalid token " + this.tokens[this.idx] + " in multiply expression";
617         }
618         return result;
619     }
620
621     couldBeExpression()
622     {
623         return this.tokens[this.idx].isEqualTo("-") || this.tokens[this.idx].isEqualTo("~") || this.tokens[this.idx].isEqualTo("constexpr") || this.tokens[this.idx].isEqualTo("sizeof") || isInteger(this.tokens[this.idx]) || isString(this.tokens[this.idx]) || isVariable(this.tokens[this.idx]) || this.tokens[this.idx].isEqualTo("(");
624     }
625
626     parseExpressionAdd()
627     {
628         this.skipNewLine();
629         let result = this.parseExpressionMul();
630         while (this.tokens[this.idx].isEqualTo("+") || this.tokens[this.idx].isEqualTo("-")) {
631             if (this.tokens[this.idx].isEqualTo("+")) {
632                 this.idx++;
633                 result = new AddImmediates(this.tokens[this.idx - 1].codeOrigin, result, this.parseExpressionMul());
634             } else if (this.tokens[this.idx].isEqualTo("-")) {
635                 this.idx++;
636                 result = new SubImmediates(this.tokens[this.idx - 1].codeOrigin, result, this.parseExpressionMul());
637             } else
638                 throw "Invalid token " + this.tokens[this.idx] + " in addition expression";
639         }
640         return result;
641     }
642
643     parseExpressionAnd()
644     {
645         this.skipNewLine();
646         let result = this.parseExpressionAdd();
647         while (this.tokens[this.idx].isEqualTo("&")) {
648             this.idx++;
649             result = new AndImmediates(this.tokens[this.idx - 1].codeOrigin, result, this.parseExpressionAdd());
650         }
651         return result;
652     }
653
654     parseExpression()
655     {
656         this.skipNewLine();
657         let result = this.parseExpressionAnd();
658         while (this.tokens[this.idx].isEqualTo("|") || this.tokens[this.idx].isEqualTo("^")) {
659             if (this.tokens[this.idx].isEqualTo("|")) {
660                 this.idx++;
661                 result = new OrImmediates(this.tokens[this.idx - 1].codeOrigin, result, this.parseExpressionAnd());
662             } else if (this.tokens[this.idx].isEqualTo("^")) {
663                 this.idx++;
664                 result = new XorImmediates(this.tokens[this.idx - 1].codeOrigin, result, this.parseExpressionAnd());
665             } else
666                 throw "Invalid token " + this.tokens[this.idx] + " in expression";
667         }
668         return result;
669     }
670
671     parseOperand(comment)
672     {
673         this.skipNewLine();
674         if (this.couldBeExpression()) {
675             let expr = this.parseExpression();
676             if (this.tokens[this.idx].isEqualTo("["))
677                 return this.parseAddress(expr);
678
679             return expr;
680         }
681         if (this.tokens[this.idx].isEqualTo("["))
682             return this.parseAddress(new Immediate(this.tokens[this.idx].codeOrigin, 0));
683         if (isLabel(this.tokens[this.idx])) {
684             let result = new LabelReference(this.tokens[this.idx].codeOrigin, Label.forName(this.tokens[this.idx].codeOrigin, this.tokens[this.idx].string));
685             this.idx++;
686             return result;
687         }
688         if (isLocalLabel(this.tokens[this.idx])) {
689             let result = new LocalLabelReference(this.tokens[this.idx].codeOrigin, LocalLabel.forName(this.tokens[this.idx].codeOrigin, this.tokens[this.idx].string));
690             this.idx++;
691             return result;
692         }
693
694         this.parseError(comment);
695     }
696
697     parseMacroVariables()
698     {
699         this.skipNewLine();
700         this.consume(/^\($/);
701         let variables = [];
702         while (true) {
703             this.skipNewLine();
704             if (this.tokens[this.idx].isEqualTo(")")) {
705                 this.idx++;
706                 break
707             } else if (isIdentifier(this.tokens[this.idx])) {
708                  variables.push(Variable.forName(this.tokens[this.idx].codeOrigin, this.tokens[this.idx].string));
709                 this.idx++;
710                 this.skipNewLine();
711                 if (this.tokens[this.idx].isEqualTo(")")) {
712                     this.idx++;
713                     break;
714                 } else if (this.tokens[this.idx].isEqualTo(","))
715                     this.idx++;
716                 else
717                     this.parseError();
718             } else
719                 this.parseError();
720         }
721         return variables;
722     }
723
724     parseSequence(final, comment)
725     {
726         let firstCodeOrigin = this.tokens[this.idx].codeOrigin;
727         let list = [];
728         while (true) {
729             if ((this.idx == this.tokens.length && !final) || (final && final.test(this.tokens[this.idx].string)))
730                 break;
731             else if (this.tokens[this.idx] instanceof Annotation) {
732                 // This is the only place where we can encounter a global
733                 // annotation, and hence need to be able to distinguish between
734                 // them.
735                 // globalAnnotations are the ones that start from column 0. All
736                 // others are considered localAnnotations.  The only reason to
737                 // distinguish between them is so that we can format the output
738                 // nicely as one would expect.
739
740                 let codeOrigin = this.tokens[this.idx].codeOrigin;
741                 let annotationOpcode = (this.tokens[this.idx].type == GlobalAnnotation) ? "globalAnnotation" : "localAnnotation";
742                 list.push(new Instruction(codeOrigin, annotationOpcode, [], this.tokens[this.idx].string));
743                 this.annotation = null;
744                 this.idx += 2 // Consume the newline as well.
745             } else if (this.tokens[this.idx].isEqualTo("\n")) {
746                 // ignore
747                 this.idx++;
748             } else if (this.tokens[this.idx].isEqualTo("const")) {
749                 this.idx++;
750                 if (!isVariable(this.tokens[this.idx]))
751                     this.parseError();
752                 let variable = Variable.forName(this.tokens[this.idx].codeOrigin, this.tokens[this.idx].string);
753                 this.idx++;
754                 if (this.tokens[this.idx].isNotEqualTo("="))
755                     this.parseError();
756                 this.idx++;
757                 let value = this.parseOperand("while inside of const " + variable.name);
758                 list.push(new ConstDecl(this.tokens[this.idx].codeOrigin, variable, value));
759             } else if (this.tokens[this.idx].isEqualTo("error")) {
760                 list.push(new Error(this.tokens[this.idx].codeOrigin));
761                 this.idx++;
762             } else if (this.tokens[this.idx].isEqualTo("if")) {
763                 let codeOrigin = this.tokens[this.idx].codeOrigin;
764                 this.idx++;
765                 this.skipNewLine();
766                 let predicate = this.parsePredicate();
767                 this.consume(/^((then)|(\n))$/);
768                 this.skipNewLine();
769                 let ifThenElse = new IfThenElse(codeOrigin, predicate, this.parseSequence(/^((else)|(end)|(elsif))$/, "while inside of \"if " + predicate.dump() + "\""));
770                 list.push(ifThenElse);
771                 while (this.tokens[this.idx].isEqualTo("elsif")) {
772                     codeOrigin = this.tokens[this.idx].codeOrigin;
773                     this.idx++;
774                     this.skipNewLine();
775                     predicate = this.parsePredicate();
776                     this.consume(/^((then)|(\n))$/);
777                     this.skipNewLine();
778                     let elseCase = new IfThenElse(codeOrigin, predicate, this.parseSequence(/^((else)|(end)|(elsif))$/, "while inside of \"if " + predicate.dump() + "\""));
779                     ifThenElse.elseCase = elseCase;
780                     ifThenElse = elseCase;
781                 }
782                 if (this.tokens[this.idx].isEqualTo("else")) {
783                     this.idx++;
784                     ifThenElse.elseCase = this.parseSequence(/^end$/, "while inside of else case for \"if " + predicate.dump() + "\"");
785                     this.idx++;
786                 } else {
787                     if (this.tokens[this.idx].isNotEqualTo("end"))
788                         this.parseError();
789                     this.idx++;
790                 }
791             } else if (this.tokens[this.idx].isEqualTo("macro")) {
792                 let codeOrigin = this.tokens[this.idx].codeOrigin;
793                 this.idx++;
794                 this.skipNewLine();
795                 if (!isIdentifier(this.tokens[this.idx]))
796                     this.parseError();
797                 let name = this.tokens[this.idx].string;
798                 this.idx++;
799                 let variables = this.parseMacroVariables();
800                 let body = this.parseSequence(/^end$/, "while inside of macro " + name);
801                 this.idx++;
802                 list.push(new Macro(codeOrigin, name, variables, body));
803             } else if (this.tokens[this.idx].isEqualTo("global")) {
804                 let codeOrigin = this.tokens[this.idx].codeOrigin;
805                 this.idx++;
806                 this.skipNewLine();
807                 if (!isLabel(this.tokens[this.idx]))
808                     this.parseError();
809                 let name = this.tokens[this.idx].string;
810                 this.idx++;
811                 Label.setAsGlobal(codeOrigin, name);
812             } else if (isInstruction(this.tokens[this.idx])) {
813                 let codeOrigin = this.tokens[this.idx].codeOrigin;
814                 let name = this.tokens[this.idx].string;
815                 this.idx++;
816                 if ((!final && this.idx == this.tokens.size) || (final && final.test(this.tokens[this.idx]))) {
817                     // Zero operand instruction, and it's the last one.
818                     list.push(new Instruction(codeOrigin, name, [], this.annotation));
819                     this.annotation = null;
820                     break;
821                 } else if (this.tokens[this.idx] instanceof Annotation) {
822                     list.push(new Instruction(codeOrigin, name, [], this.tokens[this.idx].string));
823                     this.annotation = null;
824                     this.idx += 2 // Consume the newline as well.
825                 } else if (this.tokens[this.idx].isEqualTo("\n")) {
826                     // Zero operand instruction.
827                     list.push(new Instruction(codeOrigin, name, [], this.annotation));
828                     this.annotation = null;
829                     this.idx++;
830                 } else {
831                     // It's definitely an instruction, and it has at least one operand.
832                     let operands = [];
833                     let endOfSequence = false;
834                     while (true) {
835                         operands.push(this.parseOperand("while inside of instruction " + name));
836                         if ((!final && this.idx == this.tokens.size) || (final && final.test(this.tokens[this.idx].string))) {
837                             // The end of the instruction and of the sequence.
838                             endOfSequence = true;
839                             break;
840                         } else if (this.tokens[this.idx].isEqualTo(",")) {
841                             // Has another operand.
842                             this.idx++;
843                         } else if (this.tokens[this.idx] instanceof Annotation) {
844                             this.annotation = this.tokens[this.idx].string;
845                             this.idx += 2; // Consume the newline as well.
846                             break;
847                         } else if (this.tokens[this.idx].isEqualTo("\n")) {
848                             // The end of the instruction.
849                             this.idx++;
850                             break;
851                         } else
852                             this.parseError("Expected a comma, newline, or " + final + " after " + operands[operands.length - 1].dump());
853                     }
854                     list.push(new Instruction(codeOrigin, name, operands, this.annotation));
855                     this.annotation = null;
856                     if (endOfSequence)
857                         break;
858                 }
859             } else if (isIdentifier(this.tokens[this.idx])) {
860                 // Check for potential macro invocation:
861                 let codeOrigin = this.tokens[this.idx].codeOrigin;
862                 let name = this.tokens[this.idx].string;
863                 this.idx++;
864                 if (this.tokens[this.idx].isEqualTo("(")) {
865                     // Macro invocation.
866                     this.idx++;
867                     let operands = [];
868                     this.skipNewLine();
869                     if (this.tokens[this.idx].isEqualTo(")"))
870                         this.idx++;
871                     else {
872                         while (true) {
873                             this.skipNewLine();
874                             if (this.tokens[this.idx].isEqualTo("macro")) {
875                                 // It's a macro lambda!
876                                 let codeOriginInner = this.tokens[this.idx].codeOrigin;
877                                 this.idx++;
878                                 let variables = this.parseMacroVariables();
879                                 let body = this.parseSequence(/^end$/, "while inside of anonymous macro passed as argument to " + name);
880                                 this.idx++;
881                                 operands.push(new Macro(codeOriginInner, "", variables, body));
882                             } else
883                                 operands.push(this.parseOperand("while inside of macro call to " + name));
884
885                             this.skipNewLine();
886                             if (this.tokens[this.idx].isEqualTo(")")) {
887                                 this.idx++;
888                                 break
889                             } else if (this.tokens[this.idx].isEqualTo(","))
890                                 this.idx++;
891                             else
892                                 this.parseError("Unexpected " + this.tokens[this.idx].string + " while parsing invocation of macro " + name);
893
894                         }
895                     }
896                     // Check if there's a trailing annotation after the macro invoke:
897                     if (this.tokens[this.idx] instanceof Annotation) {
898                         this.annotation = this.tokens[this.idx].string;
899                         this.idx += 2 // Consume the newline as well.
900                     }
901                     list.push(new MacroCall(codeOrigin, name, operands, this.annotation));
902                     this.annotation = null;
903                 } else
904                     this.parseError("Expected \"(\" after " + name);
905             } else if (isLabel(this.tokens[this.idx]) || isLocalLabel(this.tokens[this.idx])) {
906                 let codeOrigin = this.tokens[this.idx].codeOrigin;
907                 let name = this.tokens[this.idx].string;
908                 this.idx++;
909                 if (this.tokens[this.idx].isNotEqualTo(":"))
910                     this.parseError();
911                 // It's a label.
912                 if (isLabel(name))
913                     list.push(Label.forName(codeOrigin, name, true));
914                 else
915                     list.push(LocalLabel.forName(codeOrigin, name));
916
917                 this.idx++;
918             } else if (this.tokens[this.idx].isEqualTo("include")) {
919                 this.idx++;
920                 if (!isIdentifier(this.tokens[this.idx]))
921                     this.parseError();
922                 let moduleName = this.tokens[this.idx].string;
923                 let fileName = new IncludeFile(moduleName, this.tokens[this.idx].codeOrigin.fileName.dirname).fileName;
924                 this.idx++;
925                 list.push(parse(fileName));
926             } else
927                 this.parseError("Expecting terminal " + final + " " + comment);
928         }
929         return new Sequence(firstCodeOrigin, list);
930     }
931
932     parseIncludes(final, comment)
933     {
934         let firstCodeOrigin = this.tokens[this.idx].codeOrigin;
935         let fileList = [];
936         fileList.push(this.tokens[this.idx].codeOrigin.fileName);
937         while (true) {
938             if ((this.idx == this.tokens.length && !final) || (final && final.test(this.tokens[this.idx])))
939                 break;
940             else if (this.tokens[this.idx].isEqualTo("include")) {
941                 this.idx++;
942                 if (!isIdentifier(this.tokens[this.idx]))
943                     this.parseError();
944                 let moduleName = this.tokens[this.idx].string;
945                 let fileName = new IncludeFile(moduleName, this.tokens[this.idx].codeOrigin.fileName.dirname).fileName;
946                 this.idx++;
947
948                 fileList.push(fileName);
949             } else
950                 this.idx++;
951         }
952
953         return fileList;
954     }
955 }
956
957 function parseData(data, fileName)
958 {
959     let parser = new Parser(data, new SourceFile(fileName));
960     return parser.parseSequence(null, "");
961 }
962
963 function parse(fileName)
964 {
965     return parseData(File.open(fileName).read(), fileName);
966 }