[as2api-dev] [CVS trunk] Yet another doc comment parser. We are now on
rebuild-from-scratch number 3.
David Holroyd
dave at badgers-in-foil.co.uk
Fri, 15 Jul 2005 22:05:09 +0000
<html>
<head>
<style><!--
body {background-color:#ffffff;}
.file {border:1px solid #eeeeee;margin-top:1em;margin-bottom:1em;}
.pathname {font-family:monospace; float:right;}
.fileheader {margin-bottom:.5em;}
.diff {margin:0;}
.tasklist {padding:4px;border:1px dashed #000000;margin-top:1em;}
.tasklist ul {margin-top:0;margin-bottom:0;}
tr.alt {background-color:#eeeeee}
#added {background-color:#ddffdd;}
#addedchars {background-color:#99ff99;font-weight:bolder;}
tr.alt #added {background-color:#ccf7cc;}
#removed {background-color:#ffdddd;}
#removedchars {background-color:#ff9999;font-weight:bolder;}
tr.alt #removed {background-color:#f7cccc;}
#info {color:#888888;}
#context {background-color:#eeeeee;}
td {padding-left:.3em;padding-right:.3em;}
tr.head {border-bottom-width:1px;border-bottom-style:solid;}
tr.head td {padding:0;padding-top:.2em;}
.task {background-color:#ffff00;}
.comment {padding:4px;border:1px dashed #000000;background-color:#ffffdd}
.error {color:red;}
hr {border-width:0px;height:2px;background:black;}
--></style>
</head>
<body>
<table cellspacing="0" cellpadding="0" border="0" rules="cols">
<tr class="head"><td colspan="4">Commit in <b><tt>trunk/as2api</tt></b><span id="info"> on MAIN</span></td></tr>
<tr><td><tt>parse/<a href="#file1"><span id="added">doccomment_lexer.rb</span></a></tt></td><td align="right" id="added">+74</td><td></td><td nowrap="nowrap" align="right">added <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/doccomment_lexer.rb?rev=200&content-type=text/vnd.viewcvs-markup">200</a></td></tr>
<tr class="alt"><td><tt> /<a href="#file2"><span id="added">doccomment_parser.rb</span></a></tt></td><td align="right" id="added">+150</td><td></td><td nowrap="nowrap" align="right">added <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/doccomment_parser.rb?rev=200&content-type=text/vnd.viewcvs-markup">200</a></td></tr>
<tr><td><tt><a href="#file3">tc_doc_comment.rb</a></tt></td><td align="right" id="added">+12</td><td></td><td nowrap="nowrap" align="center"><a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/tc_doc_comment.rb?rev=199&content-type=text/vnd.viewcvs-markup">199</a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/tc_doc_comment.rb.diff?r1=199&r2=200">-></a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/tc_doc_comment.rb?rev=200&content-type=text/vnd.viewcvs-markup">200</a></td></tr>
<tr><td></td><td align="right" id="added">+236</td><td></td><td></td></tr>
</table>
<small id="info">2 added + 1 modified, total 3 files</small><br />
<pre class="comment">
Yet another doc comment parser. We are now on rebuild-from-scratch number 3.
This new code has the advantage that inline tags may now span lines and can
contain nested braces.
</pre>
<hr /><a name="file1" /><div class="file">
<span class="pathname" id="added"><a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk">trunk</a>/<a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api">as2api</a>/<a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse">parse</a></span><br />
<div class="fileheader" id="added"><big><b>doccomment_lexer.rb</b></big> <small id="info">added at <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/doccomment_lexer.rb?rev=200&content-type=text/vnd.viewcvs-markup">200</a></small></div>
<pre class="diff"><small id="info">--- trunk/as2api/parse/doccomment_lexer.rb 2005-07-14 12:22:50 UTC (rev 199)
+++ trunk/as2api/parse/doccomment_lexer.rb 2005-07-15 22:05:05 UTC (rev 200)
@@ -0,0 +1,74 @@
</small></pre><pre class="diff" id="added">+
+require 'parse/lexer'
+
+module ActionScript
+module Parse
+
+class DocWhitespaceToken < ASToken
+end
+
+class EndOfLineToken < ASToken
+end
+
+class StarsToken < ASToken
+end
+
+class ParaAtTagToken < ASToken
+ def to_s
+ "@#{@body}"
+ end
+end
+
+class InlineAtTagToken < ASToken
+ def to_s
+ "{@#{@body}"
+ end
+end
+
+class WordToken < ASToken
+end
+
+class DocCommentLexer < AbstractLexer
+ def lex_simple_token(class_sym, match, io)
+ ActionScript::Parse.const_get(class_sym).new(io.lineno)
+ end
+
+ def lex_simplebody_token(class_sym, match, io)
+ ActionScript::Parse.const_get(class_sym).new(match[0], io.lineno)
+ end
+
+ def lex_simplecapture_token(class_sym, match, io)
+ ActionScript::Parse.const_get(class_sym).new(match[1], io.lineno)
+ end
+end
+
+END_OF_LINE = "\r\n|\r|\n"
+DOC_WHITESPACE = "[ \t\f]"
+AT_INLINE_TAG = "\\{@([^ \t\r\n\f}{]+)"
+AT_PARA_TAG = "@([^ \t\r\n\f]+)"
+WHITESPACE_THEN_STARS = "[ \t]*\\*+"
+WORD = "[^ \t\f\n\r}{]+"
+
+def self.build_doc_lexer
+ builder = LexerBuilder.new
+
+ builder.add_match(WHITESPACE_THEN_STARS, :lex_simplebody_token, :StarsToken)
+ builder.add_match(DOC_WHITESPACE, :lex_simplebody_token, :DocWhitespaceToken)
+ builder.add_match(END_OF_LINE, :lex_simplebody_token, :EndOfLineToken)
+
+ builder.add_match(AT_INLINE_TAG, :lex_simplecapture_token, :InlineAtTagToken)
+ builder.add_match(AT_PARA_TAG, :lex_simplecapture_token, :ParaAtTagToken)
+
+ builder.make_punctuation_token(:LBrace, "{")
+ builder.make_punctuation_token(:RBrace, "}")
+
+ builder.add_match(WORD, :lex_simplebody_token, :WordToken)
+
+ builder.build_lexer(DocCommentLexer)
+end
+
+build_doc_lexer
+
+
+end
+end
</pre></div>
<hr /><a name="file2" /><div class="file">
<span class="pathname" id="added"><a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk">trunk</a>/<a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api">as2api</a>/<a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse">parse</a></span><br />
<div class="fileheader" id="added"><big><b>doccomment_parser.rb</b></big> <small id="info">added at <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/parse/doccomment_parser.rb?rev=200&content-type=text/vnd.viewcvs-markup">200</a></small></div>
<pre class="diff"><small id="info">--- trunk/as2api/parse/doccomment_parser.rb 2005-07-14 12:22:50 UTC (rev 199)
+++ trunk/as2api/parse/doccomment_parser.rb 2005-07-15 22:05:05 UTC (rev 200)
@@ -0,0 +1,150 @@
</small></pre><pre class="diff" id="added">+
+module ActionScript
+module Parse
+
+
+
+class DocCommentParser
+
+ def initialize(lexer)
+ @lex = lexer
+ @handler = nil
+ end
+
+ def handler=(handler)
+ @handler = handler
+ end
+
+ # this is the main parser entry-point
+ def parse_comment
+ @handler.comment_start(@lex.peek_next.lineno)
+ while @lex.peek_next
+ parse_line
+ end
+ @handler.comment_end
+ end
+
+ def parse_line
+ maybe_skip(StarsToken)
+ parse_whitespace
+ if lookahead?(ParaAtTagToken)
+ @handler.start_paragraph_tag(expect(ParaAtTagToken))
+ end
+ until lookahead?(EndOfLineToken) || eof?
+ if lookahead?(InlineAtTagToken)
+ parse_inline_tag
+ else
+ eat_text_token
+ end
+ end
+ unless eof?
+ eat_text_token_of_kind(EndOfLineToken)
+ end
+ end
+
+ def parse_whitespace
+ if lookahead?(DocWhitespaceToken)
+ eat_text_token_of_kind(DocWhitespaceToken)
+ end
+ end
+
+ def parse_inline_tag
+ tok = expect(InlineAtTagToken)
+ @handler.start_inline_tag(tok)
+ until lookahead?(RBraceToken)
+ err("end of input before closing brace for #{tok.inspect}") if eof?
+ if lookahead?(LBraceToken)
+ parse_brace_pair
+ elsif lookahead?(EndOfLineToken)
+ eat_text_token
+ maybe_skip(StarsToken)
+ else
+ eat_text_token
+ end
+ end
+ expect(RBraceToken)
+ @handler.end_inline_tag
+ end
+
+ def parse_brace_pair
+ eat_text_token_of_kind(LBraceToken)
+ until lookahead?(RBraceToken) || eof?
+ if lookahead?(LBraceToken)
+ parse_brace_pair
+ elsif lookahead?(EndOfLineToken)
+ eat_text_token
+ maybe_skip(StarsToken)
+ else
+ eat_text_token
+ end
+ end
+ eat_text_token_of_kind(RBraceToken)
+ end
+
+ # treats the text token, whatever kind it may be, as text without special
+ # meaning
+ def eat_text_token
+ @handler.text(@lex.get_next)
+ end
+
+ def eat_text_token_of_kind(kind)
+ @handler.text(expect(kind))
+ end
+
+ private
+ def expect(kind)
+ tok = @lex.get_next
+ unless tok.is_a?(kind)
+ err("Expected '#{kind}' but found '#{tok.inspect}'");
+ end
+ tok
+ end
+
+ def eof?
+ @lex.peek_next.nil?
+ end
+
+ def lookahead?(kind)
+ @lex.peek_next.is_a?(kind)
+ end
+
+ def lookaheads?(*kinds)
+ tnext = @lex.peek_next
+ kinds.each do |kind|
+ return true if tnext.is_a?(kind)
+ end
+ false
+ end
+
+ def speculate(kind)
+ if lookahead?(kind)
+ expect(kind)
+ yield
+ end
+ end
+
+ def maybe_skip(kind)
+ if lookahead?(kind)
+ expect(kind)
+ end
+ end
+
+ def err(msg)
+ raise msg
+ end
+
+end
+
+
+class DocCommentHandler
+ def comment_start(lineno); end
+ def comment_end; end
+ def text(text); end
+ def start_paragraph_tag(tag); end
+ def start_inline_tag(tag); end
+ def end_inline_tag; end
+end
+
+
+end # module Parse
+end # module ActionScript
</pre></div>
<hr /><a name="file3" /><div class="file">
<span class="pathname"><a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk">trunk</a>/<a
href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api">as2api</a></span><br />
<div class="fileheader"><big><b>tc_doc_comment.rb</b></big> <small id="info"><a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/tc_doc_comment.rb?rev=199&content-type=text/vnd.viewcvs-markup">199</a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/tc_doc_comment.rb.diff?r1=199&r2=200">-></a> <a href="http://svn.badgers-in-foil.co.uk/viewcvs.cgi/as2api/trunk/as2api/tc_doc_comment.rb?rev=200&content-type=text/vnd.viewcvs-markup">200</a></small></div>
<pre class="diff"><small id="info">--- trunk/as2api/tc_doc_comment.rb 2005-07-14 12:22:50 UTC (rev 199)
+++ trunk/as2api/tc_doc_comment.rb 2005-07-15 22:05:05 UTC (rev 200)
@@ -45,4 +45,16 @@
</small></pre><pre class="diff" id="context"> doc.parse(text)
assert("blat\nping", doc.describe_exception("foo.Bar"))
end
</pre><pre class="diff" id="added">+
+ def test_package_html
+ [
+ "<html><body>test</body></html>",
+ "<body>te<b>st</b></body>"
+ ].each do |text|
+ PackageHTML.process(text) do |element|
+ a = REXML::XPath.match(element, "descendant::text()").join
+ assert_equal("test", a)
+ end
+ end
+ end
</pre><pre class="diff" id="context"> end
</pre></div>
<center><small><a href="http://www.badgers-in-foil.co.uk/projects/cvsspam/" title="commit -> email">CVSspam</a> 0.2.11</small></center>
</body></html>