Reverts r3014 (CodeRay back to 0.7.6).

git-svn-id: svn+ssh://rubyforge.org/var/svn/redmine/trunk@3079 e93f8b46-1217-0410-a6f0-8f06a7374b81

vendor/plugins/coderay-0.7.6.227/lib/coderay/encoders/tokens.rb

@@ -0,0 +1,44 @@
+module CodeRay
+module Encoders
+
+  # The Tokens encoder converts the tokens to a simple
+  # readable format. It doesn't use colors and is mainly
+  # intended for console output.
+  #
+  # The tokens are converted with Tokens.write_token.
+  #
+  # The format is:
+  #
+  #   <token-kind> \t <escaped token-text> \n
+  #
+  # Example:
+  #
+  #   require 'coderay'
+  #   puts CodeRay.scan("puts 3 + 4", :ruby).tokens
+  #
+  # prints:
+  #
+  #   ident   puts
+  #   space
+  #   integer 3
+  #   space
+  #   operator        +
+  #   space
+  #   integer 4
+  #
+  class Tokens < Encoder
+
+    include Streamable
+    register_for :tokens
+
+    FILE_EXTENSION = 'tok'
+
+  protected
+    def token text, kind
+      @out << CodeRay::Tokens.write_token(text, kind)
+    end
+
+  end
+
+end
+end