mirror/guile
mirror/guile
1
Fork 0
mirror of https://git.savannah.gnu.org/git/guile.git synced 2025-06-30 15:00:21 +02:00
Code Activity

ecmascript tokens have source info

Browse source 
* module/language/ecmascript/tokenize.scm: Attach source information to
  tokens. We have to enhance the lalr parser to actually let this
  information propagate through, though...
This commit is contained in:
Andy Wingo 2009-02-22 11:37:55 +01:00
parent 0b229e81a7
commit a3e34104db
1 changed files with 38 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

69
module/language/ecmascript/tokenize.scm
View file

@ -381,37 +381,44 @@
(syntax-error "bad syntax: character not allowed" c)))))))
(define (next-token port div?)
(let ((c (peek-char port)))
(case c
((#\ht #\vt #\np #\space)
; whitespace
(read-char port)
(next-token port div?))
((#\newline #\cr)
; line break
(read-char port)
(next-token port div?))
((#\/)
;; division, single comment, double comment, or regexp
(read-slash port div?))
((#\" #\')
; string literal
`(StringLiteral . ,(read-string port)))
(else
(cond
((eof-object? c)
'*eoi*)
((or (char-alphabetic? c)
(char=? c #\$)
(char=? c #\_))
;; reserved word or identifier
(read-identifier port))
((char-numeric? c)
;; numeric -- also accept . FIXME, requires lookahead
`(NumericLiteral . ,(read-numeric port)))
(else
;; punctuation
(read-punctuation port)))))))
(let ((c (peek-char port))
(props `((filename . ,(port-filename port))
(line . ,(port-line port))
(column . ,(port-column port)))))
(let ((tok
(case c
((#\ht #\vt #\np #\space)
; whitespace
(read-char port)
(next-token port div?))
((#\newline #\cr)
; line break
(read-char port)
(next-token port div?))
((#\/)
;; division, single comment, double comment, or regexp
(read-slash port div?))
((#\" #\')
; string literal
`(StringLiteral . ,(read-string port)))
(else
(cond
((eof-object? c)
'*eoi*)
((or (char-alphabetic? c)
(char=? c #\$)
(char=? c #\_))
;; reserved word or identifier
(read-identifier port))
((char-numeric? c)
;; numeric -- also accept . FIXME, requires lookahead
`(NumericLiteral . ,(read-numeric port)))
(else
;; punctuation
(read-punctuation port)))))))
(if (pair? tok)
(set-source-properties! tok props))
tok)))
(define (make-tokenizer port)
(let ((div? #f))