Calculate character number for each token

lib/rkelly/nodes/node.rb

@@ -5,7 +5,7 @@ class Node
       include RKelly::Visitors
       include Enumerable
 
-      attr_accessor :value, :comments, :line, :filename
+      attr_accessor :value, :comments, :line, :character, :filename
       def initialize(value)
         @value = value
         @comments = []

lib/rkelly/parser.rb

@@ -15,6 +15,7 @@ def #{im}(val, _values, result)
             }, _values, result)
           if token = val.find { |v| v.is_a?(Token) }
             r.line = token.line if r.respond_to?(:line)
+            r.character = token.character if r.respond_to?(:character)
             r.filename = @filename if r.respond_to?(:filename)
           end
           r

lib/rkelly/token.rb

@@ -1,6 +1,6 @@
 module RKelly
   class Token
-    attr_accessor :name, :value, :transformer, :line
+    attr_accessor :name, :value, :transformer, :line, :character
     def initialize(name, value, &transformer)
       @name         = name
       @value        = value

lib/rkelly/tokenizer.rb

@@ -97,6 +97,7 @@ def tokenize(string)
     def raw_tokens(string)
       tokens = []
       line_number = 1
+      char_number = 1
       accepting_regexp = true
       while string.length > 0
         longest_token = nil
@@ -116,7 +117,17 @@ def raw_tokens(string)
         end
 
         longest_token.line = line_number
-        line_number += longest_token.value.scan(/\n/).length
+        longest_token.character = char_number
+        if longest_token.value == "\n"
+          line_number += 1
+          char_number = 1
+        elsif longest_token.value.include?("\n")
+          line_number += longest_token.value.scan(/\n/).length
+          split_list = longest_token.value.split(/\n/)
+          char_number = split_list[-1].length + 1
+        else
+          char_number += longest_token.value.length
+        end
         string = string.slice(Range.new(longest_token.value.length, -1))
         tokens << longest_token
       end

test/test_char_number.rb

@@ -0,0 +1,23 @@
+require File.dirname(__FILE__) + "/helper"
+
+class CharNumberTest < NodeTestCase
+  def test_line_numbers
+    parser = RKelly::Parser.new
+    ast = parser.parse(<<-eojs)
+      /**
+       * This is an awesome test comment.
+       */
+      function aaron() {
+        var x = 10;
+        return 1 + 1;
+      }
+    eojs
+    func = ast.pointcut(FunctionDeclNode).matches.first
+    assert func
+    assert_equal(7, func.character)
+
+    return_node = ast.pointcut(ReturnNode).matches.first
+    assert return_node
+    assert_equal(9, return_node.character)
+  end
+end