X-Git-Url: http://git.joshuawise.com/snipe.git/blobdiff_plain/1144856ba9d6018d9922c6ede7e97779a0fe6373..5c79bb689ab446551bc7ec4497e6c9b75582837e:/parse/l5.lex

diff --git a/parse/l5.lex b/parse/l5.lex
new file mode 100644
index 0000000..e10f8b7
--- /dev/null
+++ b/parse/l5.lex
@@ -0,0 +1,177 @@
+(* L5 Compiler
+ * Lexer
+ * Author: Kaustuv Chaudhuri <kaustuv+@cs.cmu.edu>
+ * Modified: Frank Pfenning <fp@cs.cmu.edu>
+ * Modified: Chris Lu <czl@andrew.cmu.edu>
+ * Modified: Joshua Wise <jwise@andrew.cmu.edu>
+ *)
+
+structure A = Ast
+structure S = Symbol
+
+type pos = int
+type svalue = Tokens.svalue
+type ('a,'b) token = ('a,'b) Tokens.token
+type lexresult = (svalue,pos) Tokens.token
+
+local
+  val commentLevel = ref 0
+  val commentPos = ref 0
+in
+  fun enterComment yypos =
+      ( commentLevel := !commentLevel + 1 ;
+	commentPos := yypos )
+    
+  fun exitComment () =
+      ( commentLevel := !commentLevel - 1 ;
+	!commentLevel = 0 )
+
+  fun number (yyt, yyp) =
+      let
+        val ext = ParseState.ext (yyp, yyp + size yyt)
+	val numOpt = Word32Signed.fromString yyt
+                     handle Overflow =>
+			    ( ErrorMsg.error ext
+                                ("integral constant `" ^ yyt ^ "' too large") ;
+			      NONE )
+      in
+	case numOpt
+	  of NONE => ( ErrorMsg.error ext
+                         ("cannot parse integral constant `" ^ yyt ^ "'");
+		       Tokens.INTNUM (Word32Signed.ZERO, yyp, yyp + size yyt) )
+	   | SOME n => Tokens.INTNUM (n,yyp,yyp + size yyt)
+      end
+  fun hexnumber (yyt, yyp) =
+      let
+        val t = String.extract (yyt, 2, NONE)
+        val ext = ParseState.ext (yyp, yyp + size yyt)
+	val numOpt = StringCvt.scanString (Word32.scan StringCvt.HEX) t
+                     handle Overflow =>
+			    ( ErrorMsg.error ext
+                                ("integral constant `" ^ yyt ^ "' too large") ;
+			      NONE )
+      in
+	case numOpt
+	  of NONE => ( ErrorMsg.error ext
+                         ("cannot parse integral constant `" ^ yyt ^ "'");
+		       Tokens.INTNUM (Word32Signed.ZERO, yyp, yyp + size yyt) )
+	   | SOME n => Tokens.INTNUM (n,yyp,yyp + size yyt)
+      end
+
+  fun eof () = 
+      ( if (!commentLevel > 0)
+          then (ErrorMsg.error (ParseState.ext (!commentPos,!commentPos)) "unterminated comment")
+          else ();
+	Tokens.EOF (0,0) )		(* bogus position information; unused *)
+
+end
+
+%%
+%header (functor L5LexFn(structure Tokens : L5_TOKENS));
+%full
+%s COMMENT COMMENT_LINE;
+
+id = [A-Za-z_][A-Za-z0-9_]*;
+decnum = [0-9][0-9]*;
+hexnum = 0x[0-9a-fA-F][0-9a-fA-F]*;
+
+ws = [\ \t\012];
+
+%%
+
+<INITIAL> {ws}+       => (lex ());
+<INITIAL> \n          => (ParseState.newline(yypos); lex());
+
+<INITIAL> "{"         => (Tokens.LBRACE (yypos, yypos + size yytext));
+<INITIAL> "}"         => (Tokens.RBRACE (yypos, yypos + size yytext));
+<INITIAL> "("         => (Tokens.LPAREN (yypos, yypos + size yytext));
+<INITIAL> ")"         => (Tokens.RPAREN (yypos, yypos + size yytext));
+
+<INITIAL> ";"         => (Tokens.SEMI (yypos, yypos + size yytext));
+
+<INITIAL> "="         => (Tokens.ASSIGN (yypos, yypos + size yytext));
+<INITIAL> "+="        => (Tokens.PLUSEQ (yypos, yypos + size yytext));
+<INITIAL> "-="        => (Tokens.MINUSEQ (yypos, yypos + size yytext));
+<INITIAL> "*="        => (Tokens.STAREQ (yypos, yypos + size yytext));
+<INITIAL> "/="        => (Tokens.SLASHEQ (yypos, yypos + size yytext));
+<INITIAL> "%="        => (Tokens.PERCENTEQ (yypos, yypos + size yytext));
+<INITIAL> "<<="       => (Tokens.LSHEQ (yypos, yypos + size yytext));
+<INITIAL> ">>="       => (Tokens.RSHEQ (yypos, yypos + size yytext));
+<INITIAL> "&="        => (Tokens.BITANDEQ (yypos, yypos + size yytext));
+<INITIAL> "^="        => (Tokens.BITXOREQ (yypos, yypos + size yytext));
+<INITIAL> "|="        => (Tokens.BITOREQ (yypos, yypos + size yytext));
+
+<INITIAL> "++"         => (Tokens.PLUSPLUS (yypos, yypos + size yytext));
+<INITIAL> "--"         => (Tokens.MINUSMINUS (yypos, yypos + size yytext));
+
+<INITIAL> "+"         => (Tokens.PLUS (yypos, yypos + size yytext));
+<INITIAL> "-"         => (Tokens.MINUS (yypos, yypos + size yytext));
+<INITIAL> "!"         => (Tokens.BANG (yypos, yypos + size yytext));
+<INITIAL> "*"         => (Tokens.STAR (yypos, yypos + size yytext));
+<INITIAL> "/"         => (Tokens.SLASH (yypos, yypos + size yytext));
+<INITIAL> "%"         => (Tokens.PERCENT (yypos, yypos + size yytext));
+<INITIAL> "<<"        => (Tokens.LSH (yypos, yypos + size yytext));
+<INITIAL> ">>"        => (Tokens.RSH (yypos, yypos + size yytext));
+<INITIAL> "||"        => (Tokens.LOGOR (yypos, yypos + size yytext));
+<INITIAL> "&&"        => (Tokens.LOGAND (yypos, yypos + size yytext));
+<INITIAL> "&"         => (Tokens.BITAND (yypos, yypos + size yytext));
+<INITIAL> "^"         => (Tokens.BITXOR (yypos, yypos + size yytext));
+<INITIAL> "|"         => (Tokens.BITOR (yypos, yypos + size yytext));
+<INITIAL> "~"         => (Tokens.BITNOT (yypos, yypos + size yytext));
+<INITIAL> "=="        => (Tokens.EQ (yypos, yypos + size yytext));
+<INITIAL> "!="        => (Tokens.NEQ (yypos, yypos + size yytext));
+<INITIAL> "<"         => (Tokens.LT (yypos, yypos + size yytext));
+<INITIAL> "<="        => (Tokens.LE (yypos, yypos + size yytext));
+<INITIAL> ">="        => (Tokens.GE (yypos, yypos + size yytext));
+<INITIAL> ">"         => (Tokens.GT (yypos, yypos + size yytext));
+
+<INITIAL> "?"         => (Tokens.QUESTION (yypos, yypos + size yytext));
+<INITIAL> ":"         => (Tokens.COLON (yypos, yypos + size yytext));
+<INITIAL> ","         => (Tokens.COMMA (yypos, yypos + size yytext));
+
+<INITIAL> "["         => (Tokens.LBRACKET (yypos, yypos + size yytext));
+<INITIAL> "]"         => (Tokens.RBRACKET (yypos, yypos + size yytext));
+<INITIAL> "->"        => (Tokens.ARROW (yypos, yypos + size yytext));
+<INITIAL> "."         => (Tokens.DOT (yypos, yypos + size yytext));
+
+<INITIAL> "return"    => (Tokens.RETURN (yypos, yypos + size yytext));
+<INITIAL> "if"        => (Tokens.IF (yypos, yypos + size yytext));
+<INITIAL> "while"     => (Tokens.WHILE (yypos, yypos + size yytext));
+<INITIAL> "for"       => (Tokens.FOR (yypos, yypos + size yytext));
+<INITIAL> "continue"  => (Tokens.CONTINUE (yypos, yypos + size yytext));
+<INITIAL> "break"     => (Tokens.BREAK (yypos, yypos + size yytext));
+<INITIAL> "else"      => (Tokens.ELSE (yypos, yypos + size yytext));
+<INITIAL> "var"       => (Tokens.VAR (yypos, yypos + size yytext));
+<INITIAL> "int"       => (Tokens.INT (yypos, yypos + size yytext));
+<INITIAL> "extern"    => (Tokens.EXTERN (yypos, yypos + size yytext));
+<INITIAL> "struct"    => (Tokens.STRUCT (yypos, yypos + size yytext));
+<INITIAL> "NULL"      => (Tokens.NULL (yypos, yypos + size yytext));
+<INITIAL> "new"       => (Tokens.NEW (yypos, yypos + size yytext));
+
+
+<INITIAL> {decnum}    => (number (yytext, yypos));
+<INITIAL> {hexnum}    => (hexnumber (yytext, yypos));
+
+<INITIAL> {id}        => (let
+                            val id = Symbol.symbol yytext
+                          in 
+                            Tokens.IDENT (id, yypos, yypos + size yytext)
+                          end);
+
+<INITIAL> "/*"        => (YYBEGIN COMMENT; enterComment yypos; lex());
+<INITIAL> "*/"        => (ErrorMsg.error (ParseState.ext (yypos, yypos)) "unbalanced comments";
+                          lex());
+
+<INITIAL> "//"        => (YYBEGIN COMMENT_LINE; lex());
+<INITIAL> "#"         => (YYBEGIN COMMENT_LINE; lex());
+<INITIAL> .           => (ErrorMsg.error (ParseState.ext (yypos,yypos))
+                              ("illegal character: \"" ^ yytext ^ "\"");
+                          lex ());
+
+<COMMENT> "/*"        => (enterComment yypos; lex());
+<COMMENT> "*/"        => (if exitComment () then YYBEGIN INITIAL else (); lex());
+<COMMENT> \n          => (ParseState.newline yypos; lex ());
+<COMMENT> .           => (lex());
+
+<COMMENT_LINE> \n     => (ParseState.newline yypos; YYBEGIN INITIAL; lex());
+<COMMENT_LINE> .      => (lex());