[as2api-dev] [CVS trunk] Implement lexical actions with named callback methods,
rather than Procs.
David Holroyd
dave at badgers-in-foil.co.uk
Tue, 12 Jul 2005 17:11:49 +0000
<html>
<head>
<style><!--
body {background-color:#ffffff;}
.file {border:1px solid #eeeeee;margin-top:1em;margin-bottom:1em;}
.pathname {font-family:monospace; float:right;}
.fileheader {margin-bottom:.5em;}
.diff {margin:0;}
.tasklist {padding:4px;border:1px dashed #000000;margin-top:1em;}
.tasklist ul {margin-top:0;margin-bottom:0;}
tr.alt {background-color:#eeeeee}
#added {background-color:#ddffdd;}
#addedchars {background-color:#99ff99;font-weight:bolder;}
tr.alt #added {background-color:#ccf7cc;}
#removed {background-color:#ffdddd;}
#removedchars {background-color:#ff9999;font-weight:bolder;}
tr.alt #removed {background-color:#f7cccc;}
#info {color:#888888;}
#context {background-color:#eeeeee;}
td {padding-left:.3em;padding-right:.3em;}
tr.head {border-bottom-width:1px;border-bottom-style:solid;}
tr.head td {padding:0;padding-top:.2em;}
.task {background-color:#ffff00;}
.comment {padding:4px;border:1px dashed #000000;background-color:#ffffdd}
.error {color:red;}
hr {border-width:0px;height:2px;background:black;}
--></style>
</head>
<body>
<table cellspacing="0" cellpadding="0" border="0" rules="cols">
<tr class="head"><td colspan="4">Commit in <b><tt>trunk/as2api/parse</tt></b><span id="info"> on MAIN</span></td></tr>
<tr><td><tt><a href="#file1">lexer.rb</a></tt></td><td align="right" id="added">+33</td><td align="right" id="removed">-23</td><td nowrap="nowrap" align="center"><a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb?rev=191&content-type=text/vnd.viewcvs-markup">191</a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb.diff?r1=191&r2=192">-></a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb?rev=192&content-type=text/vnd.viewcvs-markup">192</a></td></tr>
</table>
<div class="tasklist"><ul>
<li><a href="#task1">TODO: whitespace tokens don't span lines, which might not be the expected</a></li>
</ul></div>
<pre class="comment">
Implement lexical actions with named callback methods, rather than Procs.
Part of ongoing refactoring to try and make Lexer infrastructure more
reusable.
</pre>
<hr /><a name="file1" /><div class="file">
<span class="pathname"><a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk">trunk</a>/<a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api">as2api</a>/<a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse">parse</a></span><br />
<div class="fileheader"><big><b>lexer.rb</b></big> <small id="info"><a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb?rev=191&content-type=text/vnd.viewcvs-markup">191</a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb.diff?r1=191&r2=192">-></a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/lexer.rb?rev=192&content-type=text/vnd.viewcvs-markup">192</a></small></div>
<pre class="diff"><small id="info">--- trunk/as2api/parse/lexer.rb 2005-07-11 22:18:56 UTC (rev 191)
+++ trunk/as2api/parse/lexer.rb 2005-07-12 17:11:48 UTC (rev 192)
@@ -274,8 +274,8 @@
</small></pre><pre class="diff" id="context"> w = "[ \t\r\n\f]*"
</pre><pre class="diff" id="removed">- def self.add_match(match)
- @@matches << [make_match(match), Proc.new]
</pre><pre class="diff" id="added">+ def self.add_match(match, lex_meth_sym, tok_class_sym)
+ @@matches << [make_match(match), lex_meth_sym, tok_class_sym]
</pre><pre class="diff" id="context"> end
def self.make_simple_token(name, value, match)
</pre><pre class="diff"><small id="info">@@ -288,26 +288,34 @@
</small></pre><pre class="diff" id="context"> EOE
ActionScript::Parse.const_set(class_name, the_class)
</pre><pre class="diff" id="removed">- add_match(match) do |match, io|
- ActionScript::Parse.const_get(class_name).new(io.lineno)
- end
</pre><pre class="diff" id="added">+ add_match(match, :lex_simple_token, class_name.to_sym)
</pre><pre class="diff" id="context"> end
</pre><pre class="diff" id="added">+ def lex_simple_token(class_sym, match, io)
+ ActionScript::Parse.const_get(class_sym).new(io.lineno)
+ end
+
</pre><pre class="diff" id="context"> def self.make_keyword_token(name)
make_simple_token(name.capitalize, name, "#{name}\\b")
end
</pre><pre class="diff" id="removed">- add_match(WHITESPACE) do |match, io|
- # TODO: whitespace tokens don't span lines, which might not be the expected
- # behaviour
- WhitespaceToken.new(match[0], io.lineno)
</pre><pre class="diff" id="added"><a name="task1" />+ # <span class="task">TODO</span>: whitespace tokens don't span lines, which might not be the expected
+ # behaviour
+ add_match(WHITESPACE, :lex_simplebody_token, :WhitespaceToken)
+
+ def lex_simplebody_token(class_sym, match, io)
+ ActionScript::Parse.const_get(class_sym).new(match[0], io.lineno)
</pre><pre class="diff" id="context"> end
</pre><pre class="diff" id="removed">- add_match(SINGLE_LINE_COMMENT) do |match, io|
</pre><pre class="diff" id="added">+ add_match(SINGLE_LINE_COMMENT, :lex_singlelinecoomment_token, :SingleLineCommentToken)
+
+ def lex_singlelinecoomment_token(class_sym, match, io)
</pre><pre class="diff" id="context"> SingleLineCommentToken.new(match[1], io.lineno)
end
</pre><pre class="diff" id="removed">- add_match(OMULTI_LINE_COMMENT) do |match, io|
</pre><pre class="diff" id="added">+ add_match(OMULTI_LINE_COMMENT, :lex_multilinecomment_token, :MultiLineCommentToken)
+
+ def lex_multilinecomment_token(class_sym, match, io)
</pre><pre class="diff" id="context"> lineno = io.lineno
line = match.post_match
comment = ''
</pre><pre class="diff"><small id="info">@@ -332,39 +340,41 @@
</small></pre><pre class="diff" id="context"> make_punctuation_token(*punct)
end
</pre><pre class="diff" id="removed">- add_match(ident) do |match, io|
- IdentifierToken.new(match[0], io.lineno)
- end
</pre><pre class="diff" id="added">+ add_match(ident, :lex_simplebody_token, :IdentifierToken)
</pre><pre class="diff" id="context">
</pre><pre class="diff" id="removed">- add_match(STRING_START1) do |match, io|
</pre><pre class="diff" id="added">+ add_match(STRING_START1, :lex_string1_token, :StringToken)
+
+ def lex_string1_token(class_sym, match, io)
</pre><pre class="diff" id="context"> lineno = io.lineno
line = match.post_match
str = ''
</pre><pre class="diff" id="removed">- until line =~ /<span id="removedchars">\A</span>#{STRING_END1}/o
</pre><pre class="diff" id="added">+ until line =~ /#{STRING_END1}/o
</pre><pre class="diff" id="context"> str << line
line = io.readline;
</pre><pre class="diff" id="added">+ raise "#{lineno}:unexpected EOF in string" if line.nil?
</pre><pre class="diff" id="context"> end
str << $1
match.string = $'
StringToken.new(str, lineno)
end
</pre><pre class="diff" id="removed">- add_match(STRING_START2) do |match, io|
</pre><pre class="diff" id="added">+ add_match(STRING_START2, :lex_string2_token, :StringToken)
+
+ def lex_string2_token(class_sym, match, io)
</pre><pre class="diff" id="context"> lineno = io.lineno
line = match.post_match
str = ''
</pre><pre class="diff" id="removed">- until line =~ /<span id="removedchars">\A</span>#{STRING_END2}/o
</pre><pre class="diff" id="added">+ until line =~ /#{STRING_END2}/o
</pre><pre class="diff" id="context"> str << line
line = io.readline;
</pre><pre class="diff" id="added">+ raise "#{lineno}:unexpected EOF in string" if line.nil?
</pre><pre class="diff" id="context"> end
str << $1
match.string = $'
StringToken.new(str, lineno)
end
</pre><pre class="diff" id="removed">- add_match(num) do |match, io|
- NumberToken.new(match[0], io.lineno)
- end
</pre><pre class="diff" id="added">+ add_match(num, :lex_simplebody_token, :NumberToken)
</pre><pre class="diff" id="context">
def check_fill
if @tokens.empty? && !@io.eof?
</pre><pre class="diff"><small id="info">@@ -379,9 +389,9 @@
</small></pre><pre class="diff" id="context"> until line.eos?
EOS
@@matches.each_with_index do |token_match, index|
</pre><pre class="diff" id="removed">- re, action = token_match
</pre><pre class="diff" id="added">+ re, lex_method, tok_class = token_match
</pre><pre class="diff" id="context"> text << "if line.scan(/#{re}/)\n"
</pre><pre class="diff" id="removed">- text << " emit(@@matches[#{index}][1].call(line, @io))\n"
</pre><pre class="diff" id="added">+ text << " emit(#{lex_method.to_s}(:#{tok_class.to_s}, line, @io))\n"
</pre><pre class="diff" id="context"> text << " next\n"
text << "end\n"
end
</pre></div>
<center><small><a href="http://www.badgers-in-foil.co.uk/projects/cvsspam/" title="commit -> email">CVSspam</a> 0.2.11</small></center>
</body></html>