mirror of
https://git.savannah.gnu.org/git/guile.git
synced 2025-06-10 05:50:26 +02:00
ecmascript tokens have source info
* module/language/ecmascript/tokenize.scm: Attach source information to tokens. We have to enhance the lalr parser to actually let this information propagate through, though...
This commit is contained in:
parent
0b229e81a7
commit
a3e34104db
1 changed files with 38 additions and 31 deletions
|
@ -381,37 +381,44 @@
|
|||
(syntax-error "bad syntax: character not allowed" c)))))))
|
||||
|
||||
(define (next-token port div?)
|
||||
(let ((c (peek-char port)))
|
||||
(case c
|
||||
((#\ht #\vt #\np #\space)
|
||||
; whitespace
|
||||
(read-char port)
|
||||
(next-token port div?))
|
||||
((#\newline #\cr)
|
||||
; line break
|
||||
(read-char port)
|
||||
(next-token port div?))
|
||||
((#\/)
|
||||
;; division, single comment, double comment, or regexp
|
||||
(read-slash port div?))
|
||||
((#\" #\')
|
||||
; string literal
|
||||
`(StringLiteral . ,(read-string port)))
|
||||
(else
|
||||
(cond
|
||||
((eof-object? c)
|
||||
'*eoi*)
|
||||
((or (char-alphabetic? c)
|
||||
(char=? c #\$)
|
||||
(char=? c #\_))
|
||||
;; reserved word or identifier
|
||||
(read-identifier port))
|
||||
((char-numeric? c)
|
||||
;; numeric -- also accept . FIXME, requires lookahead
|
||||
`(NumericLiteral . ,(read-numeric port)))
|
||||
(else
|
||||
;; punctuation
|
||||
(read-punctuation port)))))))
|
||||
(let ((c (peek-char port))
|
||||
(props `((filename . ,(port-filename port))
|
||||
(line . ,(port-line port))
|
||||
(column . ,(port-column port)))))
|
||||
(let ((tok
|
||||
(case c
|
||||
((#\ht #\vt #\np #\space)
|
||||
; whitespace
|
||||
(read-char port)
|
||||
(next-token port div?))
|
||||
((#\newline #\cr)
|
||||
; line break
|
||||
(read-char port)
|
||||
(next-token port div?))
|
||||
((#\/)
|
||||
;; division, single comment, double comment, or regexp
|
||||
(read-slash port div?))
|
||||
((#\" #\')
|
||||
; string literal
|
||||
`(StringLiteral . ,(read-string port)))
|
||||
(else
|
||||
(cond
|
||||
((eof-object? c)
|
||||
'*eoi*)
|
||||
((or (char-alphabetic? c)
|
||||
(char=? c #\$)
|
||||
(char=? c #\_))
|
||||
;; reserved word or identifier
|
||||
(read-identifier port))
|
||||
((char-numeric? c)
|
||||
;; numeric -- also accept . FIXME, requires lookahead
|
||||
`(NumericLiteral . ,(read-numeric port)))
|
||||
(else
|
||||
;; punctuation
|
||||
(read-punctuation port)))))))
|
||||
(if (pair? tok)
|
||||
(set-source-properties! tok props))
|
||||
tok)))
|
||||
|
||||
(define (make-tokenizer port)
|
||||
(let ((div? #f))
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue