[as2api-dev] [CVS trunk] Use StringScanner to simplify the guts of the lexer
ever-so-slightly
David Holroyd
dave at badgers-in-foil.co.uk
Sat, 02 Jul 2005 23:18:49 +0000
<html>
<head>
<style><!--
body {background-color:#ffffff;}
.file {border:1px solid #eeeeee;margin-top:1em;margin-bottom:1em;}
.pathname {font-family:monospace; float:right;}
.fileheader {margin-bottom:.5em;}
.diff {margin:0;}
.tasklist {padding:4px;border:1px dashed #000000;margin-top:1em;}
.tasklist ul {margin-top:0;margin-bottom:0;}
tr.alt {background-color:#eeeeee}
#added {background-color:#ddffdd;}
#addedchars {background-color:#99ff99;font-weight:bolder;}
tr.alt #added {background-color:#ccf7cc;}
#removed {background-color:#ffdddd;}
#removedchars {background-color:#ff9999;font-weight:bolder;}
tr.alt #removed {background-color:#f7cccc;}
#info {color:#888888;}
#context {background-color:#eeeeee;}
td {padding-left:.3em;padding-right:.3em;}
tr.head {border-bottom-width:1px;border-bottom-style:solid;}
tr.head td {padding:0;padding-top:.2em;}
.task {background-color:#ffff00;}
.comment {padding:4px;border:1px dashed #000000;background-color:#ffffdd}
.error {color:red;}
hr {border-width:0px;height:2px;background:black;}
--></style>
</head>
<body>
<table cellspacing="0" cellpadding="0" border="0" rules="cols">
<tr class="head"><td colspan="4">Commit in <b><tt>trunk/as2api/parse</tt></b><span id="info"> on MAIN</span></td></tr>
<tr><td><tt><a href="#file1">lexer.rb</a></tt></td><td align="right" id="added">+10</td><td align="right" id="removed">-13</td><td nowrap="nowrap" align="center"><a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb?rev=178&content-type=text/vnd.viewcvs-markup">178</a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb.diff?r1=178&r2=179">-></a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb?rev=179&content-type=text/vnd.viewcvs-markup">179</a></td></tr>
</table>
<pre class="comment">
Use StringScanner to simplify the guts of the lexer ever-so-slightly
</pre>
<hr /><a name="file1" /><div class="file">
<span class="pathname"><a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk">trunk</a>/<a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api">as2api</a>/<a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse">parse</a></span><br />
<div class="fileheader"><big><b>lexer.rb</b></big> <small id="info"><a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb?rev=178&content-type=text/vnd.viewcvs-markup">178</a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb.diff?r1=178&r2=179">-></a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb?rev=179&content-type=text/vnd.viewcvs-markup">179</a></small></div>
<pre class="diff"><small id="info">--- trunk/as2api/parse/lexer.rb 2005-07-02 20:49:21 UTC (rev 178)
+++ trunk/as2api/parse/lexer.rb 2005-07-02 23:18:48 UTC (rev 179)
@@ -1,4 +1,6 @@
</small></pre><pre class="diff" id="context">
</pre><pre class="diff" id="added">+require 'strscan'
+
</pre><pre class="diff" id="context"> module ActionScript
module Parse
</pre><pre class="diff"><small id="info">@@ -245,7 +247,7 @@
</small></pre><pre class="diff" id="context"> private
def ASLexer.make_match(match)
</pre><pre class="diff" id="removed">- Regexp.new("\\A#{match}")
</pre><pre class="diff" id="added">+ Regexp.new(match)
</pre><pre class="diff" id="context"> end
h = "[0-9a-fA-F]"
</pre><pre class="diff"><small id="info">@@ -288,7 +290,6 @@
</small></pre><pre class="diff" id="context">
add_match(match) do |lex, match, io|
lex.emit(ActionScript::Parse.const_get(class_name).new(io.lineno))
</pre><pre class="diff" id="removed">- match.post_match
</pre><pre class="diff" id="context"> end
end
</pre><pre class="diff"><small id="info">@@ -300,12 +301,10 @@
</small></pre><pre class="diff" id="context"> # TODO: whitespace tokens don't span lines, which might not be the expected
# behaviour
lex.emit(WhitespaceToken.new(match[0], io.lineno))
</pre><pre class="diff" id="removed">- match.post_match
</pre><pre class="diff" id="context"> end
add_match(SINGLE_LINE_COMMENT) do |lex, match, io|
lex.emit(SingleLineCommentToken.new(match[1], io.lineno))
</pre><pre class="diff" id="removed">- match.post_match
</pre><pre class="diff" id="context"> end
add_match(OMULTI_LINE_COMMENT) do |lex, match, io|
</pre><pre class="diff"><small id="info">@@ -318,7 +317,7 @@
</small></pre><pre class="diff" id="context"> end
comment << $`
lex.emit(MultiLineCommentToken.new(comment, lineno))
</pre><pre class="diff" id="removed">- $'
</pre><pre class="diff" id="added">+ match.string = $'
</pre><pre class="diff" id="context"> end
Keywords.each do |keyword|
</pre><pre class="diff"><small id="info">@@ -335,7 +334,6 @@
</small></pre><pre class="diff" id="context">
add_match(ident) do |lex, match, io|
lex.emit(IdentifierToken.new(match[0], io.lineno))
</pre><pre class="diff" id="removed">- match.post_match
</pre><pre class="diff" id="context"> end
add_match(STRING_START1) do |lex, match, io|
</pre><pre class="diff"><small id="info">@@ -348,7 +346,7 @@
</small></pre><pre class="diff" id="context"> end
str << $1
lex.emit(StringToken.new(str, lineno))
</pre><pre class="diff" id="removed">- $'
</pre><pre class="diff" id="added">+ match.string = $'
</pre><pre class="diff" id="context"> end
add_match(STRING_START2) do |lex, match, io|
</pre><pre class="diff"><small id="info">@@ -361,12 +359,11 @@
</small></pre><pre class="diff" id="context"> end
str << $1
lex.emit(StringToken.new(str, lineno))
</pre><pre class="diff" id="removed">- $'
</pre><pre class="diff" id="added">+ match.string = $'
</pre><pre class="diff" id="context"> end
add_match(num) do |lex, match, io|
lex.emit(NumberToken.new(match[0], io.lineno))
</pre><pre class="diff" id="removed">- match.post_match
</pre><pre class="diff" id="context"> end
def check_fill
</pre><pre class="diff"><small id="info">@@ -376,14 +373,14 @@
</small></pre><pre class="diff" id="context"> end
def fill
</pre><pre class="diff" id="removed">- line = @io.readline
- while line.size>0
</pre><pre class="diff" id="added">+ line = StringScanner.new(@io.readline)
+ until line.eos?
</pre><pre class="diff" id="context"> matched = false
@@matches.each do |token_match|
re, action = token_match
</pre><pre class="diff" id="removed">- match = re.match(line)
</pre><pre class="diff" id="added">+ match = line.scan(re)
</pre><pre class="diff" id="context"> if match
</pre><pre class="diff" id="removed">- line = action.call(self, match, @io)
</pre><pre class="diff" id="added">+ action.call(self, line, @io)
</pre><pre class="diff" id="context"> matched = true
break
end
</pre></div>
<center><small><a href="http://www.badgers-in-foil.co.uk/projects/cvsspam/" title="commit -> email">CVSspam</a> 0.2.11</small></center>
</body></html>