mirror/guile
mirror/guile
1
Fork 0
mirror of https://git.savannah.gnu.org/git/guile.git synced 2025-04-30 20:00:19 +02:00
Code Activity

Adapt ECMAScript parser and lexer to `(system base lalr)'.

Browse source 
* module/language/ecmascript/tokenize.scm: Use `make-lexical-token' and
  related procedures instead of pairs as tokens passed to the parser.
  Pass source location information in the form of `source-location'
  objects.

* module/language/ecmascript/parse.scm (read-ecmascript,
  read-ecmascript/1): Instantiate a new parser at each call.
  (parse-ecmascript): Rename to...
  (make-parser): ... this.  Change `->' to `:' in the grammar syntax.

* module/language/ecmascript/parse-lalr.scm: Remove.

* module/Makefile.am (ECMASCRIPT_LANG_SOURCES): Remove
  `language/ecmascript/parse-lalr.scm'.
This commit is contained in:
Ludovic Courtès 2010-03-20 00:08:36 +01:00
parent bd7131d3ad
commit 0ecd70a271
4 changed files with 271 additions and 1991 deletions
Download patch file Download diff file Expand all files Collapse all files

1
module/Makefile.am
View file

@ -117,7 +117,6 @@ VALUE_LANG_SOURCES = \
language/value/spec.scm language/value/spec.scm
ECMASCRIPT_LANG_SOURCES = \ ECMASCRIPT_LANG_SOURCES = \
language/ecmascript/parse-lalr.scm \
language/ecmascript/tokenize.scm \ language/ecmascript/tokenize.scm \
language/ecmascript/parse.scm \ language/ecmascript/parse.scm \
language/ecmascript/impl.scm \ language/ecmascript/impl.scm \

1731
module/language/ecmascript/parse-lalr.scm
View file

File diff suppressed because it is too large Load diff

470
module/language/ecmascript/parse.scm
View file

@ -1,6 +1,6 @@
;;; ECMAScript for Guile ;;; ECMAScript for Guile
;; Copyright (C) 2009 Free Software Foundation, Inc. ;; Copyright (C) 2009, 2010 Free Software Foundation, Inc.
;;;; This library is free software; you can redistribute it and/or ;;;; This library is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU Lesser General Public ;;;; modify it under the terms of the GNU Lesser General Public
@ -19,23 +19,29 @@
;;; Code: ;;; Code:
(define-module (language ecmascript parse) (define-module (language ecmascript parse)
#:use-module (language ecmascript parse-lalr) #:use-module (system base lalr)
#:use-module (language ecmascript tokenize) #:use-module (language ecmascript tokenize)
#:export (read-ecmascript read-ecmascript/1 parse-ecmascript)) #:export (read-ecmascript read-ecmascript/1 make-parser))
(define (syntax-error message . args) (define (syntax-error message . args)
(apply throw 'SyntaxError message args)) (apply throw 'SyntaxError message args))
(define (read-ecmascript port) (define (read-ecmascript port)
(parse-ecmascript (make-tokenizer port) syntax-error)) (let ((parse (make-parser)))
(parse (make-tokenizer port) syntax-error)))
(define (read-ecmascript/1 port) (define (read-ecmascript/1 port)
(parse-ecmascript (make-tokenizer/1 port) syntax-error)) (let ((parse (make-parser)))
(parse (make-tokenizer/1 port) syntax-error)))
(define *eof-object* (define *eof-object*
(call-with-input-string "" read-char)) (call-with-input-string "" read-char))
(define parse-ecmascript (define (make-parser)
;; Return a fresh ECMAScript parser. Parsers produced by `lalr-scm' are now
;; stateful (e.g., they won't invoke the tokenizer any more once it has
;; returned `*eoi*'), hence the need to instantiate new parsers.
(lalr-parser (lalr-parser
;; terminal (i.e. input) token types ;; terminal (i.e. input) token types
(lbrace rbrace lparen rparen lbracket rbracket dot semicolon comma < (lbrace rbrace lparen rparen lbracket rbracket dot semicolon comma <
@ -49,289 +55,289 @@
Identifier StringLiteral NumericLiteral RegexpLiteral) Identifier StringLiteral NumericLiteral RegexpLiteral)
(Program (SourceElements) -> $1 (Program (SourceElements) : $1
(*eoi*) -> *eof-object*) (*eoi*) : *eof-object*)
;; ;;
;; Verily, here we define statements. Expressions are defined ;; Verily, here we define statements. Expressions are defined
;; afterwards. ;; afterwards.
;; ;;
(SourceElement (Statement) -> $1 (SourceElement (Statement) : $1
(FunctionDeclaration) -> $1) (FunctionDeclaration) : $1)
(FunctionDeclaration (function Identifier lparen rparen lbrace FunctionBody rbrace) -> `(var (,$2 (lambda () ,$6))) (FunctionDeclaration (function Identifier lparen rparen lbrace FunctionBody rbrace) : `(var (,$2 (lambda () ,$6)))
(function Identifier lparen FormalParameterList rparen lbrace FunctionBody rbrace) -> `(var (,$2 (lambda ,$4 ,$7)))) (function Identifier lparen FormalParameterList rparen lbrace FunctionBody rbrace) : `(var (,$2 (lambda ,$4 ,$7))))
(FunctionExpression (function lparen rparen lbrace FunctionBody rbrace) -> `(lambda () ,$5) (FunctionExpression (function lparen rparen lbrace FunctionBody rbrace) : `(lambda () ,$5)
(function Identifier lparen rparen lbrace FunctionBody rbrace) -> `(lambda () ,$6) (function Identifier lparen rparen lbrace FunctionBody rbrace) : `(lambda () ,$6)
(function lparen FormalParameterList rparen lbrace FunctionBody rbrace) -> `(lambda ,$3 ,$6) (function lparen FormalParameterList rparen lbrace FunctionBody rbrace) : `(lambda ,$3 ,$6)
(function Identifier lparen FormalParameterList rparen lbrace FunctionBody rbrace) -> `(lambda ,$4 ,$7)) (function Identifier lparen FormalParameterList rparen lbrace FunctionBody rbrace) : `(lambda ,$4 ,$7))
(FormalParameterList (Identifier) -> `(,$1) (FormalParameterList (Identifier) : `(,$1)
(FormalParameterList comma Identifier) -> `(,@$1 ,$3)) (FormalParameterList comma Identifier) : `(,@$1 ,$3))
(SourceElements (SourceElement) -> $1 (SourceElements (SourceElement) : $1
(SourceElements SourceElement) -> (if (and (pair? $1) (eq? (car $1) 'begin)) (SourceElements SourceElement) : (if (and (pair? $1) (eq? (car $1) 'begin))
`(begin ,@(cdr $1) ,$2) `(begin ,@(cdr $1) ,$2)
`(begin ,$1 ,$2))) `(begin ,$1 ,$2)))
(FunctionBody (SourceElements) -> $1) (FunctionBody (SourceElements) : $1)
(Statement (Block) -> $1 (Statement (Block) : $1
(VariableStatement) -> $1 (VariableStatement) : $1
(EmptyStatement) -> $1 (EmptyStatement) : $1
(ExpressionStatement) -> $1 (ExpressionStatement) : $1
(IfStatement) -> $1 (IfStatement) : $1
(IterationStatement) -> $1 (IterationStatement) : $1
(ContinueStatement) -> $1 (ContinueStatement) : $1
(BreakStatement) -> $1 (BreakStatement) : $1
(ReturnStatement) -> $1 (ReturnStatement) : $1
(WithStatement) -> $1 (WithStatement) : $1
(LabelledStatement) -> $1 (LabelledStatement) : $1
(SwitchStatement) -> $1 (SwitchStatement) : $1
(ThrowStatement) -> $1 (ThrowStatement) : $1
(TryStatement) -> $1) (TryStatement) : $1)
(Block (lbrace StatementList rbrace) -> `(block ,$2)) (Block (lbrace StatementList rbrace) : `(block ,$2))
(StatementList (Statement) -> $1 (StatementList (Statement) : $1
(StatementList Statement) -> (if (and (pair? $1) (eq? (car $1) 'begin)) (StatementList Statement) : (if (and (pair? $1) (eq? (car $1) 'begin))
`(begin ,@(cdr $1) ,$2) `(begin ,@(cdr $1) ,$2)
`(begin ,$1 ,$2))) `(begin ,$1 ,$2)))
(VariableStatement (var VariableDeclarationList) -> `(var ,@$2)) (VariableStatement (var VariableDeclarationList) : `(var ,@$2))
(VariableDeclarationList (VariableDeclaration) -> `(,$1) (VariableDeclarationList (VariableDeclaration) : `(,$1)
(VariableDeclarationList comma VariableDeclaration) -> `(,@$1 ,$2)) (VariableDeclarationList comma VariableDeclaration) : `(,@$1 ,$2))
(VariableDeclarationListNoIn (VariableDeclarationNoIn) -> `(,$1) (VariableDeclarationListNoIn (VariableDeclarationNoIn) : `(,$1)
(VariableDeclarationListNoIn comma VariableDeclarationNoIn) -> `(,@$1 ,$2)) (VariableDeclarationListNoIn comma VariableDeclarationNoIn) : `(,@$1 ,$2))
(VariableDeclaration (Identifier) -> `(,$1) (VariableDeclaration (Identifier) : `(,$1)
(Identifier Initialiser) -> `(,$1 ,$2)) (Identifier Initialiser) : `(,$1 ,$2))
(VariableDeclarationNoIn (Identifier) -> `(,$1) (VariableDeclarationNoIn (Identifier) : `(,$1)
(Identifier Initialiser) -> `(,$1 ,$2)) (Identifier Initialiser) : `(,$1 ,$2))
(Initialiser (= AssignmentExpression) -> $2) (Initialiser (= AssignmentExpression) : $2)
(InitialiserNoIn (= AssignmentExpressionNoIn) -> $2) (InitialiserNoIn (= AssignmentExpressionNoIn) : $2)
(EmptyStatement (semicolon) -> '(begin)) (EmptyStatement (semicolon) : '(begin))
(ExpressionStatement (Expression semicolon) -> $1) (ExpressionStatement (Expression semicolon) : $1)
(IfStatement (if lparen Expression rparen Statement else Statement) -> `(if ,$3 ,$5 ,$7) (IfStatement (if lparen Expression rparen Statement else Statement) : `(if ,$3 ,$5 ,$7)
(if lparen Expression rparen Statement) -> `(if ,$3 ,$5)) (if lparen Expression rparen Statement) : `(if ,$3 ,$5))
(IterationStatement (do Statement while lparen Expression rparen semicolon) -> `(do ,$2 ,$5) (IterationStatement (do Statement while lparen Expression rparen semicolon) : `(do ,$2 ,$5)
(while lparen Expression rparen Statement) -> `(while ,$3 ,$5) (while lparen Expression rparen Statement) : `(while ,$3 ,$5)
(for lparen semicolon semicolon rparen Statement) -> `(for #f #f #f ,$6) (for lparen semicolon semicolon rparen Statement) : `(for #f #f #f ,$6)
(for lparen semicolon semicolon Expression rparen Statement) -> `(for #f #f ,$5 ,$7) (for lparen semicolon semicolon Expression rparen Statement) : `(for #f #f ,$5 ,$7)
(for lparen semicolon Expression semicolon rparen Statement) -> `(for #f ,$4 #f ,$7) (for lparen semicolon Expression semicolon rparen Statement) : `(for #f ,$4 #f ,$7)
(for lparen semicolon Expression semicolon Expression rparen Statement) -> `(for #f ,$4 ,$6 ,$8) (for lparen semicolon Expression semicolon Expression rparen Statement) : `(for #f ,$4 ,$6 ,$8)
(for lparen ExpressionNoIn semicolon semicolon rparen Statement) -> `(for ,$3 #f #f ,$7) (for lparen ExpressionNoIn semicolon semicolon rparen Statement) : `(for ,$3 #f #f ,$7)
(for lparen ExpressionNoIn semicolon semicolon Expression rparen Statement) -> `(for ,$3 #f ,$6 ,$8) (for lparen ExpressionNoIn semicolon semicolon Expression rparen Statement) : `(for ,$3 #f ,$6 ,$8)
(for lparen ExpressionNoIn semicolon Expression semicolon rparen Statement) -> `(for ,$3 ,$5 #f ,$8) (for lparen ExpressionNoIn semicolon Expression semicolon rparen Statement) : `(for ,$3 ,$5 #f ,$8)
(for lparen ExpressionNoIn semicolon Expression semicolon Expression rparen Statement) -> `(for ,$3 ,$5 ,$7 ,$9) (for lparen ExpressionNoIn semicolon Expression semicolon Expression rparen Statement) : `(for ,$3 ,$5 ,$7 ,$9)
(for lparen var VariableDeclarationListNoIn semicolon semicolon rparen Statement) -> `(for (var ,@$4) #f #f ,$8) (for lparen var VariableDeclarationListNoIn semicolon semicolon rparen Statement) : `(for (var ,@$4) #f #f ,$8)
(for lparen var VariableDeclarationListNoIn semicolon semicolon Expression rparen Statement) -> `(for (var ,@$4) #f ,$7 ,$9) (for lparen var VariableDeclarationListNoIn semicolon semicolon Expression rparen Statement) : `(for (var ,@$4) #f ,$7 ,$9)
(for lparen var VariableDeclarationListNoIn semicolon Expression semicolon rparen Statement) -> `(for (var ,@$4) ,$6 #f ,$9) (for lparen var VariableDeclarationListNoIn semicolon Expression semicolon rparen Statement) : `(for (var ,@$4) ,$6 #f ,$9)
(for lparen var VariableDeclarationListNoIn semicolon Expression semicolon Expression rparen Statement) -> `(for (var ,@$4) ,$6 ,$8 ,$10) (for lparen var VariableDeclarationListNoIn semicolon Expression semicolon Expression rparen Statement) : `(for (var ,@$4) ,$6 ,$8 ,$10)
(for lparen LeftHandSideExpression in Expression rparen Statement) -> `(for-in ,$3 ,$5 ,$7) (for lparen LeftHandSideExpression in Expression rparen Statement) : `(for-in ,$3 ,$5 ,$7)
(for lparen var VariableDeclarationNoIn in Expression rparen Statement) -> `(begin (var ,$4) (for-in (ref ,@$4) ,$6 ,$8))) (for lparen var VariableDeclarationNoIn in Expression rparen Statement) : `(begin (var ,$4) (for-in (ref ,@$4) ,$6 ,$8)))
(ContinueStatement (continue Identifier semicolon) -> `(continue ,$2) (ContinueStatement (continue Identifier semicolon) : `(continue ,$2)
(continue semicolon) -> `(continue)) (continue semicolon) : `(continue))
(BreakStatement (break Identifier semicolon) -> `(break ,$2) (BreakStatement (break Identifier semicolon) : `(break ,$2)
(break semicolon) -> `(break)) (break semicolon) : `(break))
(ReturnStatement (return Expression semicolon) -> `(return ,$2) (ReturnStatement (return Expression semicolon) : `(return ,$2)
(return semicolon) -> `(return)) (return semicolon) : `(return))
(WithStatement (with lparen Expression rparen Statement) -> `(with ,$3 ,$5)) (WithStatement (with lparen Expression rparen Statement) : `(with ,$3 ,$5))
(SwitchStatement (switch lparen Expression rparen CaseBlock) -> `(switch ,$3 ,@$5)) (SwitchStatement (switch lparen Expression rparen CaseBlock) : `(switch ,$3 ,@$5))
(CaseBlock (lbrace rbrace) -> '() (CaseBlock (lbrace rbrace) : '()
(lbrace CaseClauses rbrace) -> $2 (lbrace CaseClauses rbrace) : $2
(lbrace CaseClauses DefaultClause rbrace) -> `(,@$2 ,@$3) (lbrace CaseClauses DefaultClause rbrace) : `(,@$2 ,@$3)
(lbrace DefaultClause rbrace) -> `(,$2) (lbrace DefaultClause rbrace) : `(,$2)
(lbrace DefaultClause CaseClauses rbrace) -> `(,@$2 ,@$3)) (lbrace DefaultClause CaseClauses rbrace) : `(,@$2 ,@$3))
(CaseClauses (CaseClause) -> `(,$1) (CaseClauses (CaseClause) : `(,$1)
(CaseClauses CaseClause) -> `(,@$1 ,$2)) (CaseClauses CaseClause) : `(,@$1 ,$2))
(CaseClause (case Expression colon) -> `(case ,$2) (CaseClause (case Expression colon) : `(case ,$2)
(case Expression colon StatementList) -> `(case ,$2 ,$4)) (case Expression colon StatementList) : `(case ,$2 ,$4))
(DefaultClause (default colon) -> `(default) (DefaultClause (default colon) : `(default)
(default colon StatementList) -> `(default ,$3)) (default colon StatementList) : `(default ,$3))
(LabelledStatement (Identifier colon Statement) -> `(label ,$1 ,$3)) (LabelledStatement (Identifier colon Statement) : `(label ,$1 ,$3))
(ThrowStatement (throw Expression semicolon) -> `(throw ,$2)) (ThrowStatement (throw Expression semicolon) : `(throw ,$2))
(TryStatement (try Block Catch) -> `(try ,$2 ,$3 #f) (TryStatement (try Block Catch) : `(try ,$2 ,$3 #f)
(try Block Finally) -> `(try ,$2 #f ,$3) (try Block Finally) : `(try ,$2 #f ,$3)
(try Block Catch Finally) -> `(try ,$2 ,$3 ,$4)) (try Block Catch Finally) : `(try ,$2 ,$3 ,$4))
(Catch (catch lparen Identifier rparen Block) -> `(catch ,$3 ,$5)) (Catch (catch lparen Identifier rparen Block) : `(catch ,$3 ,$5))
(Finally (finally Block) -> `(finally ,$2)) (Finally (finally Block) : `(finally ,$2))
;; ;;
;; As promised, expressions. We build up to Expression bottom-up, so ;; As promised, expressions. We build up to Expression bottom-up, so
;; as to get operator precedence right. ;; as to get operator precedence right.
;; ;;
(PrimaryExpression (this) -> 'this (PrimaryExpression (this) : 'this
(null) -> 'null (null) : 'null
(true) -> 'true (true) : 'true
(false) -> 'false (false) : 'false
(Identifier) -> `(ref ,$1) (Identifier) : `(ref ,$1)
(StringLiteral) -> `(string ,$1) (StringLiteral) : `(string ,$1)
(RegexpLiteral) -> `(regexp ,$1) (RegexpLiteral) : `(regexp ,$1)
(NumericLiteral) -> `(number ,$1) (NumericLiteral) : `(number ,$1)
(ArrayLiteral) -> $1 (ArrayLiteral) : $1
(ObjectLiteral) -> $1 (ObjectLiteral) : $1
(lparen Expression rparen) -> $2) (lparen Expression rparen) : $2)
(ArrayLiteral (lbracket rbracket) -> '(array) (ArrayLiteral (lbracket rbracket) : '(array)
(lbracket Elision rbracket) -> '(array ,@$2) (lbracket Elision rbracket) : '(array ,@$2)
(lbracket ElementList rbracket) -> `(array ,@$2) (lbracket ElementList rbracket) : `(array ,@$2)
(lbracket ElementList comma rbracket) -> `(array ,@$2) (lbracket ElementList comma rbracket) : `(array ,@$2)
(lbracket ElementList comma Elision rbracket) -> `(array ,@$2)) (lbracket ElementList comma Elision rbracket) : `(array ,@$2))
(ElementList (AssignmentExpression) -> `(,$1) (ElementList (AssignmentExpression) : `(,$1)
(Elision AssignmentExpression) -> `(,@$1 ,$2) (Elision AssignmentExpression) : `(,@$1 ,$2)
(ElementList comma AssignmentExpression) -> `(,@$1 ,$3) (ElementList comma AssignmentExpression) : `(,@$1 ,$3)
(ElementList comma Elision AssignmentExpression) -> `(,@$1 ,@$3 ,$4)) (ElementList comma Elision AssignmentExpression) : `(,@$1 ,@$3 ,$4))
(Elision (comma) -> '((number 0)) (Elision (comma) : '((number 0))
(Elision comma) -> `(,@$1 (number 0))) (Elision comma) : `(,@$1 (number 0)))
(ObjectLiteral (lbrace rbrace) -> `(object) (ObjectLiteral (lbrace rbrace) : `(object)
(lbrace PropertyNameAndValueList rbrace) -> `(object ,@$2)) (lbrace PropertyNameAndValueList rbrace) : `(object ,@$2))
(PropertyNameAndValueList (PropertyName colon AssignmentExpression) -> `((,$1 ,$3)) (PropertyNameAndValueList (PropertyName colon AssignmentExpression) : `((,$1 ,$3))
(PropertyNameAndValueList comma PropertyName colon AssignmentExpression) -> `(,@$1 (,$3 ,$5))) (PropertyNameAndValueList comma PropertyName colon AssignmentExpression) : `(,@$1 (,$3 ,$5)))
(PropertyName (Identifier) -> $1 (PropertyName (Identifier) : $1
(StringLiteral) -> (string->symbol $1) (StringLiteral) : (string->symbol $1)
(NumericLiteral) -> $1) (NumericLiteral) : $1)
(MemberExpression (PrimaryExpression) -> $1 (MemberExpression (PrimaryExpression) : $1
(FunctionExpression) -> $1 (FunctionExpression) : $1
(MemberExpression lbracket Expression rbracket) -> `(aref ,$1 ,$3) (MemberExpression lbracket Expression rbracket) : `(aref ,$1 ,$3)
(MemberExpression dot Identifier) -> `(pref ,$1 ,$3) (MemberExpression dot Identifier) : `(pref ,$1 ,$3)
(new MemberExpression Arguments) -> `(new ,$2 ,$3)) (new MemberExpression Arguments) : `(new ,$2 ,$3))
(NewExpression (MemberExpression) -> $1 (NewExpression (MemberExpression) : $1
(new NewExpression) -> `(new ,$2 ())) (new NewExpression) : `(new ,$2 ()))
(CallExpression (MemberExpression Arguments) -> `(call ,$1 ,$2) (CallExpression (MemberExpression Arguments) : `(call ,$1 ,$2)
(CallExpression Arguments) -> `(call ,$1 ,$2) (CallExpression Arguments) : `(call ,$1 ,$2)
(CallExpression lbracket Expression rbracket) -> `(aref ,$1 ,$3) (CallExpression lbracket Expression rbracket) : `(aref ,$1 ,$3)
(CallExpression dot Identifier) -> `(pref ,$1 ,$3)) (CallExpression dot Identifier) : `(pref ,$1 ,$3))
(Arguments (lparen rparen) -> '() (Arguments (lparen rparen) : '()
(lparen ArgumentList rparen) -> $2) (lparen ArgumentList rparen) : $2)
(ArgumentList (AssignmentExpression) -> `(,$1) (ArgumentList (AssignmentExpression) : `(,$1)
(ArgumentList comma AssignmentExpression) -> `(,@$1 ,$3)) (ArgumentList comma AssignmentExpression) : `(,@$1 ,$3))
(LeftHandSideExpression (NewExpression) -> $1 (LeftHandSideExpression (NewExpression) : $1
(CallExpression) -> $1) (CallExpression) : $1)
(PostfixExpression (LeftHandSideExpression) -> $1 (PostfixExpression (LeftHandSideExpression) : $1
(LeftHandSideExpression ++) -> `(postinc ,$1) (LeftHandSideExpression ++) : `(postinc ,$1)
(LeftHandSideExpression --) -> `(postdec ,$1)) (LeftHandSideExpression --) : `(postdec ,$1))
(UnaryExpression (PostfixExpression) -> $1 (UnaryExpression (PostfixExpression) : $1
(delete UnaryExpression) -> `(delete ,$2) (delete UnaryExpression) : `(delete ,$2)
(void UnaryExpression) -> `(void ,$2) (void UnaryExpression) : `(void ,$2)
(typeof UnaryExpression) -> `(typeof ,$2) (typeof UnaryExpression) : `(typeof ,$2)
(++ UnaryExpression) -> `(preinc ,$2) (++ UnaryExpression) : `(preinc ,$2)
(-- UnaryExpression) -> `(predec ,$2) (-- UnaryExpression) : `(predec ,$2)
(+ UnaryExpression) -> `(+ ,$2) (+ UnaryExpression) : `(+ ,$2)
(- UnaryExpression) -> `(- ,$2) (- UnaryExpression) : `(- ,$2)
(~ UnaryExpression) -> `(~ ,$2) (~ UnaryExpression) : `(~ ,$2)
(! UnaryExpression) -> `(! ,$2)) (! UnaryExpression) : `(! ,$2))
(MultiplicativeExpression (UnaryExpression) -> $1 (MultiplicativeExpression (UnaryExpression) : $1
(MultiplicativeExpression * UnaryExpression) -> `(* ,$1 ,$3) (MultiplicativeExpression * UnaryExpression) : `(* ,$1 ,$3)
(MultiplicativeExpression / UnaryExpression) -> `(/ ,$1 ,$3) (MultiplicativeExpression / UnaryExpression) : `(/ ,$1 ,$3)
(MultiplicativeExpression % UnaryExpression) -> `(% ,$1 ,$3)) (MultiplicativeExpression % UnaryExpression) : `(% ,$1 ,$3))
(AdditiveExpression (MultiplicativeExpression) -> $1 (AdditiveExpression (MultiplicativeExpression) : $1
(AdditiveExpression + MultiplicativeExpression) -> `(+ ,$1 ,$3) (AdditiveExpression + MultiplicativeExpression) : `(+ ,$1 ,$3)
(AdditiveExpression - MultiplicativeExpression) -> `(- ,$1 ,$3)) (AdditiveExpression - MultiplicativeExpression) : `(- ,$1 ,$3))
(ShiftExpression (AdditiveExpression) -> $1 (ShiftExpression (AdditiveExpression) : $1
(ShiftExpression << MultiplicativeExpression) -> `(<< ,$1 ,$3) (ShiftExpression << MultiplicativeExpression) : `(<< ,$1 ,$3)
(ShiftExpression >> MultiplicativeExpression) -> `(>> ,$1 ,$3) (ShiftExpression >> MultiplicativeExpression) : `(>> ,$1 ,$3)
(ShiftExpression >>> MultiplicativeExpression) -> `(>>> ,$1 ,$3)) (ShiftExpression >>> MultiplicativeExpression) : `(>>> ,$1 ,$3))
(RelationalExpression (ShiftExpression) -> $1 (RelationalExpression (ShiftExpression) : $1
(RelationalExpression < ShiftExpression) -> `(< ,$1 ,$3) (RelationalExpression < ShiftExpression) : `(< ,$1 ,$3)
(RelationalExpression > ShiftExpression) -> `(> ,$1 ,$3) (RelationalExpression > ShiftExpression) : `(> ,$1 ,$3)
(RelationalExpression <= ShiftExpression) -> `(<= ,$1 ,$3) (RelationalExpression <= ShiftExpression) : `(<= ,$1 ,$3)
(RelationalExpression >= ShiftExpression) -> `(>= ,$1 ,$3) (RelationalExpression >= ShiftExpression) : `(>= ,$1 ,$3)
(RelationalExpression instanceof ShiftExpression) -> `(instanceof ,$1 ,$3) (RelationalExpression instanceof ShiftExpression) : `(instanceof ,$1 ,$3)
(RelationalExpression in ShiftExpression) -> `(in ,$1 ,$3)) (RelationalExpression in ShiftExpression) : `(in ,$1 ,$3))
(RelationalExpressionNoIn (ShiftExpression) -> $1 (RelationalExpressionNoIn (ShiftExpression) : $1
(RelationalExpressionNoIn < ShiftExpression) -> `(< ,$1 ,$3) (RelationalExpressionNoIn < ShiftExpression) : `(< ,$1 ,$3)
(RelationalExpressionNoIn > ShiftExpression) -> `(> ,$1 ,$3) (RelationalExpressionNoIn > ShiftExpression) : `(> ,$1 ,$3)
(RelationalExpressionNoIn <= ShiftExpression) -> `(<= ,$1 ,$3) (RelationalExpressionNoIn <= ShiftExpression) : `(<= ,$1 ,$3)
(RelationalExpressionNoIn >= ShiftExpression) -> `(>= ,$1 ,$3) (RelationalExpressionNoIn >= ShiftExpression) : `(>= ,$1 ,$3)
(RelationalExpressionNoIn instanceof ShiftExpression) -> `(instanceof ,$1 ,$3)) (RelationalExpressionNoIn instanceof ShiftExpression) : `(instanceof ,$1 ,$3))
(EqualityExpression (RelationalExpression) -> $1 (EqualityExpression (RelationalExpression) : $1
(EqualityExpression == RelationalExpression) -> `(== ,$1 ,$3) (EqualityExpression == RelationalExpression) : `(== ,$1 ,$3)
(EqualityExpression != RelationalExpression) -> `(!= ,$1 ,$3) (EqualityExpression != RelationalExpression) : `(!= ,$1 ,$3)
(EqualityExpression === RelationalExpression) -> `(=== ,$1 ,$3) (EqualityExpression === RelationalExpression) : `(=== ,$1 ,$3)
(EqualityExpression !== RelationalExpression) -> `(!== ,$1 ,$3)) (EqualityExpression !== RelationalExpression) : `(!== ,$1 ,$3))
(EqualityExpressionNoIn (RelationalExpressionNoIn) -> $1 (EqualityExpressionNoIn (RelationalExpressionNoIn) : $1
(EqualityExpressionNoIn == RelationalExpressionNoIn) -> `(== ,$1 ,$3) (EqualityExpressionNoIn == RelationalExpressionNoIn) : `(== ,$1 ,$3)
(EqualityExpressionNoIn != RelationalExpressionNoIn) -> `(!= ,$1 ,$3) (EqualityExpressionNoIn != RelationalExpressionNoIn) : `(!= ,$1 ,$3)
(EqualityExpressionNoIn === RelationalExpressionNoIn) -> `(=== ,$1 ,$3) (EqualityExpressionNoIn === RelationalExpressionNoIn) : `(=== ,$1 ,$3)
(EqualityExpressionNoIn !== RelationalExpressionNoIn) -> `(!== ,$1 ,$3)) (EqualityExpressionNoIn !== RelationalExpressionNoIn) : `(!== ,$1 ,$3))
(BitwiseANDExpression (EqualityExpression) -> $1 (BitwiseANDExpression (EqualityExpression) : $1
(BitwiseANDExpression & EqualityExpression) -> `(& ,$1 ,$3)) (BitwiseANDExpression & EqualityExpression) : `(& ,$1 ,$3))
(BitwiseANDExpressionNoIn (EqualityExpressionNoIn) -> $1 (BitwiseANDExpressionNoIn (EqualityExpressionNoIn) : $1
(BitwiseANDExpressionNoIn & EqualityExpressionNoIn) -> `(& ,$1 ,$3)) (BitwiseANDExpressionNoIn & EqualityExpressionNoIn) : `(& ,$1 ,$3))
(BitwiseXORExpression (BitwiseANDExpression) -> $1 (BitwiseXORExpression (BitwiseANDExpression) : $1
(BitwiseXORExpression ^ BitwiseANDExpression) -> `(^ ,$1 ,$3)) (BitwiseXORExpression ^ BitwiseANDExpression) : `(^ ,$1 ,$3))
(BitwiseXORExpressionNoIn (BitwiseANDExpressionNoIn) -> $1 (BitwiseXORExpressionNoIn (BitwiseANDExpressionNoIn) : $1
(BitwiseXORExpressionNoIn ^ BitwiseANDExpressionNoIn) -> `(^ ,$1 ,$3)) (BitwiseXORExpressionNoIn ^ BitwiseANDExpressionNoIn) : `(^ ,$1 ,$3))
(BitwiseORExpression (BitwiseXORExpression) -> $1 (BitwiseORExpression (BitwiseXORExpression) : $1
(BitwiseORExpression bor BitwiseXORExpression) -> `(bor ,$1 ,$3)) (BitwiseORExpression bor BitwiseXORExpression) : `(bor ,$1 ,$3))
(BitwiseORExpressionNoIn (BitwiseXORExpressionNoIn) -> $1 (BitwiseORExpressionNoIn (BitwiseXORExpressionNoIn) : $1
(BitwiseORExpressionNoIn bor BitwiseXORExpressionNoIn) -> `(bor ,$1 ,$3)) (BitwiseORExpressionNoIn bor BitwiseXORExpressionNoIn) : `(bor ,$1 ,$3))
(LogicalANDExpression (BitwiseORExpression) -> $1 (LogicalANDExpression (BitwiseORExpression) : $1
(LogicalANDExpression && BitwiseORExpression) -> `(and ,$1 ,$3)) (LogicalANDExpression && BitwiseORExpression) : `(and ,$1 ,$3))
(LogicalANDExpressionNoIn (BitwiseORExpressionNoIn) -> $1 (LogicalANDExpressionNoIn (BitwiseORExpressionNoIn) : $1
(LogicalANDExpressionNoIn && BitwiseORExpressionNoIn) -> `(and ,$1 ,$3)) (LogicalANDExpressionNoIn && BitwiseORExpressionNoIn) : `(and ,$1 ,$3))
(LogicalORExpression (LogicalANDExpression) -> $1 (LogicalORExpression (LogicalANDExpression) : $1
(LogicalORExpression or LogicalANDExpression) -> `(or ,$1 ,$3)) (LogicalORExpression or LogicalANDExpression) : `(or ,$1 ,$3))
(LogicalORExpressionNoIn (LogicalANDExpressionNoIn) -> $1 (LogicalORExpressionNoIn (LogicalANDExpressionNoIn) : $1
(LogicalORExpressionNoIn or LogicalANDExpressionNoIn) -> `(or ,$1 ,$3)) (LogicalORExpressionNoIn or LogicalANDExpressionNoIn) : `(or ,$1 ,$3))
(ConditionalExpression (LogicalORExpression) -> $1 (ConditionalExpression (LogicalORExpression) : $1
(LogicalORExpression ? AssignmentExpression colon AssignmentExpression) -> `(if ,$1 ,$3 ,$5)) (LogicalORExpression ? AssignmentExpression colon AssignmentExpression) : `(if ,$1 ,$3 ,$5))
(ConditionalExpressionNoIn (LogicalORExpressionNoIn) -> $1 (ConditionalExpressionNoIn (LogicalORExpressionNoIn) : $1
(LogicalORExpressionNoIn ? AssignmentExpressionNoIn colon AssignmentExpressionNoIn) -> `(if ,$1 ,$3 ,$5)) (LogicalORExpressionNoIn ? AssignmentExpressionNoIn colon AssignmentExpressionNoIn) : `(if ,$1 ,$3 ,$5))
(AssignmentExpression (ConditionalExpression) -> $1 (AssignmentExpression (ConditionalExpression) : $1
(LeftHandSideExpression AssignmentOperator AssignmentExpression) -> `(,$2 ,$1 ,$3)) (LeftHandSideExpression AssignmentOperator AssignmentExpression) : `(,$2 ,$1 ,$3))
(AssignmentExpressionNoIn (ConditionalExpressionNoIn) -> $1 (AssignmentExpressionNoIn (ConditionalExpressionNoIn) : $1
(LeftHandSideExpression AssignmentOperator AssignmentExpressionNoIn) -> `(,$2 ,$1 ,$3)) (LeftHandSideExpression AssignmentOperator AssignmentExpressionNoIn) : `(,$2 ,$1 ,$3))
(AssignmentOperator (=) -> '= (AssignmentOperator (=) : '=
(*=) -> '*= (*=) : '*=
(/=) -> '/= (/=) : '/=
(%=) -> '%= (%=) : '%=
(+=) -> '+= (+=) : '+=
(-=) -> '-= (-=) : '-=
(<<=) -> '<<= (<<=) : '<<=
(>>=) -> '>>= (>>=) : '>>=
(>>>=) -> '>>>= (>>>=) : '>>>=
(&=) -> '&= (&=) : '&=
(^=) -> '^= (^=) : '^=
(bor=) -> 'bor=) (bor=) : 'bor=)
(Expression (AssignmentExpression) -> $1 (Expression (AssignmentExpression) : $1
(Expression comma AssignmentExpression) -> `(begin ,$1 ,$3)) (Expression comma AssignmentExpression) : `(begin ,$1 ,$3))
(ExpressionNoIn (AssignmentExpressionNoIn) -> $1 (ExpressionNoIn (AssignmentExpressionNoIn) : $1
(ExpressionNoIn comma AssignmentExpressionNoIn) -> `(begin ,$1 ,$3)))) (ExpressionNoIn comma AssignmentExpressionNoIn) : `(begin ,$1 ,$3))))

58
module/language/ecmascript/tokenize.scm
View file

@ -1,6 +1,6 @@
;;; ECMAScript for Guile ;;; ECMAScript for Guile
;; Copyright (C) 2009 Free Software Foundation, Inc. ;; Copyright (C) 2009, 2010 Free Software Foundation, Inc.
;;;; This library is free software; you can redistribute it and/or ;;;; This library is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU Lesser General Public ;;;; modify it under the terms of the GNU Lesser General Public
@ -21,6 +21,7 @@
(define-module (language ecmascript tokenize) (define-module (language ecmascript tokenize)
#:use-module (ice-9 rdelim) #:use-module (ice-9 rdelim)
#:use-module ((srfi srfi-1) #:select (unfold-right)) #:use-module ((srfi srfi-1) #:select (unfold-right))
#:use-module (system base lalr)
#:export (next-token make-tokenizer make-tokenizer/1 tokenize tokenize/1)) #:export (next-token make-tokenizer make-tokenizer/1 tokenize tokenize/1))
(define (syntax-error message . args) (define (syntax-error message . args)
@ -75,8 +76,8 @@
(lp (read-char port)))))) (lp (read-char port))))))
(div? (div?
(case c1 (case c1
((#\=) (read-char port) `(/= . #f)) ((#\=) (read-char port) (make-lexical-token '/= #f #f))
(else `(/ . #f)))) (else (make-lexical-token '/ #f #f))))
(else (else
(read-regexp port))))) (read-regexp port)))))
@ -95,7 +96,9 @@
(char-numeric? c) (char-numeric? c)
(char=? c #\$) (char=? c #\$)
(char=? c #\_)))) (char=? c #\_))))
`(RegexpLiteral . (,(string-append head str) . ,(reverse flags))) (make-lexical-token 'RegexpLiteral #f
(cons (string-append head str)
(reverse flags)))
(begin (read-char port) (begin (read-char port)
(lp (peek-char port) (cons c flags)))))) (lp (peek-char port) (cons c flags))))))
((char=? terminator #\\) ((char=? terminator #\\)
@ -216,7 +219,7 @@
("import" . import) ("import" . import)
("public" . public))) ("public" . public)))
(define (read-identifier port) (define (read-identifier port loc)
(let lp ((c (peek-char port)) (chars '())) (let lp ((c (peek-char port)) (chars '()))
(if (or (eof-object? c) (if (or (eof-object? c)
(not (or (char-alphabetic? c) (not (or (char-alphabetic? c)
@ -225,10 +228,11 @@
(char=? c #\_)))) (char=? c #\_))))
(let ((word (list->string (reverse chars)))) (let ((word (list->string (reverse chars))))
(cond ((assoc-ref *keywords* word) (cond ((assoc-ref *keywords* word)
=> (lambda (x) `(,x . #f))) => (lambda (x) (make-lexical-token x loc #f)))
((assoc-ref *future-reserved-words* word) ((assoc-ref *future-reserved-words* word)
(syntax-error "word is reserved for the future, dude." word)) (syntax-error "word is reserved for the future, dude." word))
(else `(Identifier . ,(string->symbol word))))) (else (make-lexical-token 'Identifier loc
(string->symbol word)))))
(begin (read-char port) (begin (read-char port)
(lp (peek-char port) (cons c chars)))))) (lp (peek-char port) (cons c chars))))))
@ -368,7 +372,7 @@
(else (else
(lp (cons (list (string-ref (caar puncs) 0) #f) nodes) (lp (cons (list (string-ref (caar puncs) 0) #f) nodes)
puncs)))))) puncs))))))
(lambda (port) (lambda (port loc)
(let lp ((c (peek-char port)) (tree punc-tree) (candidate #f)) (let lp ((c (peek-char port)) (tree punc-tree) (candidate #f))
(cond (cond
((assv-ref tree c) ((assv-ref tree c)
@ -376,15 +380,17 @@
(read-char port) (read-char port)
(lp (peek-char port) (cdr node-tail) (car node-tail)))) (lp (peek-char port) (cdr node-tail) (car node-tail))))
(candidate (candidate
`(,candidate . #f)) (make-lexical-token candidate loc #f))
(else (else
(syntax-error "bad syntax: character not allowed" c))))))) (syntax-error "bad syntax: character not allowed" c)))))))
(define (next-token port div?) (define (next-token port div?)
(let ((c (peek-char port)) (let ((c (peek-char port))
(props `((filename . ,(port-filename port)) (loc (make-source-location (port-filename port)
(line . ,(port-line port)) (port-line port)
(column . ,(port-column port))))) (port-column port)
(false-if-exception (seek port 0 SEEK_CUR))
#f)))
(let ((tok (let ((tok
(case c (case c
((#\ht #\vt #\np #\space) ((#\ht #\vt #\np #\space)
@ -400,7 +406,7 @@
(read-slash port div?)) (read-slash port div?))
((#\" #\') ((#\" #\')
; string literal ; string literal
`(StringLiteral . ,(read-string port))) (make-lexical-token 'StringLiteral loc (read-string port)))
(else (else
(cond (cond
((eof-object? c) ((eof-object? c)
@ -409,15 +415,14 @@
(char=? c #\$) (char=? c #\$)
(char=? c #\_)) (char=? c #\_))
;; reserved word or identifier ;; reserved word or identifier
(read-identifier port)) (read-identifier port loc))
((char-numeric? c) ((char-numeric? c)
;; numeric -- also accept . FIXME, requires lookahead ;; numeric -- also accept . FIXME, requires lookahead
`(NumericLiteral . ,(read-numeric port))) (make-lexical-token 'NumericLiteral loc (read-numeric port)))
(else (else
;; punctuation ;; punctuation
(read-punctuation port))))))) (read-punctuation port loc)))))))
(if (pair? tok)
(set-source-properties! tok props))
tok))) tok)))
(define (make-tokenizer port) (define (make-tokenizer port)
@ -435,31 +440,32 @@
(if eoi? (if eoi?
'*eoi* '*eoi*
(let ((tok (next-token port div?))) (let ((tok (next-token port div?)))
(case (if (pair? tok) (car tok) tok) (case (if (lexical-token? tok) (lexical-token-category tok) tok)
((lparen) ((lparen)
(set! stack (cons 'lparen stack))) (set! stack (make-lexical-token 'lparen #f stack)))
((rparen) ((rparen)
(if (and (pair? stack) (eq? (car stack) 'lparen)) (if (and (pair? stack) (eq? (car stack) 'lparen))
(set! stack (cdr stack)) (set! stack (cdr stack))
(syntax-error "unexpected right parenthesis"))) (syntax-error "unexpected right parenthesis")))
((lbracket) ((lbracket)
(set! stack (cons 'lbracket stack))) (set! stack (make-lexical-token 'lbracket #f stack)))
((rbracket) ((rbracket)
(if (and (pair? stack) (eq? (car stack) 'lbracket)) (if (and (pair? stack) (eq? (car stack) 'lbracket))
(set! stack (cdr stack)) (set! stack (cdr stack))
(syntax-error "unexpected right bracket" stack))) (syntax-error "unexpected right bracket" stack)))
((lbrace) ((lbrace)
(set! stack (cons 'lbrace stack))) (set! stack (make-lexical-token 'lbrace #f stack)))
((rbrace) ((rbrace)
(if (and (pair? stack) (eq? (car stack) 'lbrace)) (if (and (pair? stack) (eq? (car stack) 'lbrace))
(set! stack (cdr stack)) (set! stack (cdr stack))
(syntax-error "unexpected right brace" stack))) (syntax-error "unexpected right brace" stack)))
((semicolon) ((semicolon)
(set! eoi? (null? stack)))) (set! eoi? (null? stack))))
(set! div? (and (pair? tok) (set! div? (and (lexical-token? tok)
(or (eq? (car tok) 'Identifier) (let ((cat (lexical-token-category tok)))
(eq? (car tok) 'NumericLiteral) (or (eq? cat 'Identifier)
(eq? (car tok) 'StringLiteral)))) (eq? cat 'NumericLiteral)
(eq? cat 'StringLiteral)))))
tok))))) tok)))))
(define (tokenize port) (define (tokenize port)