tokens.rb

require 'operators'
require 'set'

module Tokens

  Keywords = Set[
    :begin, :break, :case, :class, :def, :do, :else, :end, :ensure, :if,
    :module, :require, :require_relative, :rescue, :then, :unless, :when, :elsif,
    :protected, :next, :lambda, :stabby_lambda, :while, :until, :for, :in, :alias
  ]

  # Methods can end with one of these.
  # e.g.: empty?, flatten!, foo=
  MethodEndings = Set["?", "=", "!"]

  ALPHA_LOW  = ?a .. ?z
  ALPHA_HIGH = ?A .. ?Z
  DIGITS = ?0 .. ?9

  class CharSet
    def initialize *c
      @chars = Set[*c]
    end

    def << c
      @chars << c
    end

    def === other
      @chars.member?(other)
    end

    def member? other
      @chars.member?(other)
    end

    def to_a
      @chars.to_a
    end
  end

  ALPHA = CharSet.new(*ALPHA_LOW.to_a, *ALPHA_HIGH.to_a)
  ALNUM = CharSet.new(*ALPHA.to_a, *DIGITS.to_a)
end

require 'atom'
require 'sym'
require 'quoted'


module Tokens
  # Match a (optionally specific) keyword
  class Keyword
    def self.expect(s,match)
      a = Atom.expect(s)
      return a if (a == match)
      s.unget(a.to_s) if a
      return nil
    end
  end
end

module Tokens
  # A methodname can be an atom followed by one of the method endings
  # defined in MethodEndings (see top).
  class Methodname

    def self.expect(s)
      # FIXME: This is horribly inefficient.
      name = nil
      OPER_METHOD.each do |op|
        return name.to_sym if name = s.expect(op)
      end

      pre_name = s.expect(Atom)
      if pre_name
        suff_name = MethodEndings.select{ |me| s.expect(me) }.first

        pre_name = pre_name ? pre_name.to_s : nil
        suff_name = suff_name ? suff_name.to_s : nil
        # methodname is prefix + suffix
        return (pre_name.to_s + suff_name.to_s).to_sym
      end

      return nil
    end
  end
end

module Tokens
  class Int
    def self.expect(s, allow_negative = true)
      tmp = ""
      # Only consume leading '-' if allowed (i.e., after an operator, not after ')')
      tmp << s.get if allow_negative && s.peek == ?-
      c = s.peek
      if (c == nil) || (?0 .. ?9).member?(c) == false
        return nil
      end

      # Check for hex (0x) or binary (0b) prefix
      radix = 10
      if s.peek == ?0
        tmp << s.get
        c = s.peek
        if c == ?x || c == ?X
          # Hexadecimal
          radix = 16
          tmp << s.get
          while (c = s.peek) && ((c == ?_) || (?0 .. ?9).member?(c) || (?a .. ?f).member?(c) || (?A .. ?F).member?(c))
            tmp << s.get
          end
        elsif c == ?b || c == ?B
          # Binary
          radix = 2
          tmp << s.get
          while (c = s.peek) && ((c == ?_) || c == ?0 || c == ?1)
            tmp << s.get
          end
        else
          # Octal number starting with 0
          radix = 8
          while (c = s.peek) && ((c == ?_) || (?0 .. ?7).member?(c))
            tmp << s.get
          end
        end
      else
        # Regular decimal number
        while (c = s.peek) && ((c == ?_) || (?0 .. ?9).member?(c))
          tmp << s.get
        end
      end
      return nil if tmp == ""

      # Parse the number based on radix
      num = 0
      i = 0
      len = tmp.length
      neg = false
      if tmp[0] == ?-
        neg = true
        i += 1
      end

      # Skip 0x or 0b prefix
      if radix == 16 && i < len && tmp[i] == ?0 && (tmp[i+1] == ?x || tmp[i+1] == ?X)
        i += 2
      elsif radix == 2 && i < len && tmp[i] == ?0 && (tmp[i+1] == ?b || tmp[i+1] == ?B)
        i += 2
      end

      while i < len
        s = tmp[i]
        i = i + 1

        # Skip underscores in numbers (they're legal separators)
        if s == ?_
          # Continue to next iteration
        else
          digit_value = nil
          if radix == 10
            if (?0..?9).member?(s)
              digit_value = s.ord - ?0.ord
            else
              break
            end
          elsif radix == 16
            if (?0..?9).member?(s)
              digit_value = s.ord - ?0.ord
            elsif (?a..?f).member?(s)
              digit_value = s.ord - ?a.ord + 10
            elsif (?A..?F).member?(s)
              digit_value = s.ord - ?A.ord + 10
            else
              break
            end
          elsif radix == 2
            if s == ?0
              digit_value = 0
            elsif s == ?1
              digit_value = 1
            else
              break
            end
          elsif radix == 8
            if (?0..?7).member?(s)
              digit_value = s.ord - ?0.ord
            else
              break
            end
          end

          if digit_value
            num = num * radix + digit_value
          end
        end
      end
      if neg
        num = num * (-1)
      end

      return num
    end
  end

  class Number
    def self.expect(s, allow_negative = true)
      i = Int.expect(s, allow_negative)
      return nil if i.nil?

      # IMPORTANT: Check for float/rational literals FIRST before converting
      # large integers to heap integers. If we have "4294967295.0", we want
      # to handle it as a Float, not try to convert 4294967295 to heap integer
      # and then fail when trying to append ".0" to the AST node.

      # Check for float FIRST
      if s.peek == ?.
        s.get  # consume '.'
        if !(?0..?9).member?(s.peek)
          s.unget(".")
          # Not a float, fall through to check for large integer
        else
          # It's a float - parse fractional part and return Float
          # Fractional part is never negative
          f = Int.expect(s, false)
          # Convert to string and let MRI parse as float
          # This works for any size number since MRI handles it
          num = "#{i}.#{f}"

          # Check for scientific notation exponent: e+19, e-10, E5, etc.
          if s.peek == ?e || s.peek == ?E
            s.get  # consume 'e' or 'E'
            num << "e"

            # Optional sign
            if s.peek == ?+ || s.peek == ?-
              num << s.get
            end

            # Exponent digits
            exp = Int.expect(s, false)
            if exp
              num << exp.to_s
            end
          end

          return num.to_f
        end
      end

      # Check for Rational literal: <number>r or <number>/<number>r
      if s.peek == ?r
        # Simple rational: 5r = Rational(5, 1)
        s.get  # consume 'r'
        return [:call, :Rational, [i, 1]]
      elsif s.peek == ?/
        # Could be rational literal: 6/5r
        s.get  # consume '/'

        # Try to parse denominator (never negative in rational literals)
        denom = Int.expect(s, false)
        if denom && s.peek == ?r
          # It's a rational literal!
          s.get  # consume 'r'
          return [:call, :Rational, [i, denom]]
        else
          # Not a rational literal, unget and continue
          if denom
            denom_str = denom.to_s
            s.unget(denom_str)
          end
          s.unget("/")
          # Fall through to check for large integer
        end
      end

      # Now check if integer exceeds fixnum range (-2^30 to 2^30-1)
      # If so, create a heap integer via Integer.__from_literal
      # IMPORTANT: Use fixnum arithmetic to avoid bootstrap issues
      # (literals that exceed fixnum range would trigger this very code!)
      half_max = 536870911  # 2^29 - 1 (fits in fixnum)
      max_fixnum = half_max * 2 + 1  # 2^30 - 1
      min_fixnum = -max_fixnum - 1   # -2^30

      if i > max_fixnum || i < min_fixnum
        # Extract sign and magnitude
        sign = i < 0 ? -1 : 1
        magnitude = i.abs

        # Split into 30-bit limbs (least significant first)
        # Compute limb_base as 2^30 using fixnum arithmetic
        # Use 2^29 * 2 to avoid exceeding fixnum range
        limb_base = 536870912 * 2  # (2^29) * 2 = 2^30
        limbs = []
        while magnitude > 0
          limbs << (magnitude % limb_base)
          magnitude = magnitude / limb_base
        end
        limbs << 0 if limbs.empty?

        # Generate AST: Integer.__from_literal([limbs...], sign)
        # This is a class method call on Integer
        return [:callm, :Integer, :__from_literal, [[:array, *limbs], sign]]
      end

      # Regular fixnum - return as-is
      return i
    end
  end
end

module Tokens
  class Tokenizer
    attr_accessor :keywords
    attr_accessor :newline_before_current
    attr_reader :at_newline

    def scanner
      @s
    end

    def last_ws_consumed_newline
      @s.last_ws_consumed_newline
    end

    def initialize(scanner,parser)
      @s = scanner
      @parser = parser
      @keywords = Keywords.dup
      @first = true
      @lastop = false
      @newline_before_current = false
      @at_newline = false
    end

    def each
      @first = true
      # Reset @lastop to allow consuming leading whitespace/newlines in new parse
      @lastop = true
      while t = get and t[0]
        # FIXME: Fails without parentheses
        yield(*t)
        @first = false
      end
    end

    def unget(token)
      # a bit of a hack. Breaks down if we ever unget more than one token from the tokenizer.
      # token is an array [text, operator, type], so use token[0] not token.to_s
      token_text = token.is_a?(Array) ? token[0].to_s : token.to_s
      s = Scanner::ScannerString.new(token_text)
      # Always attach position - this ensures consistent behavior
      s.position = @lastpos
      @s.unget(s)
    end

    def get_quoted_exp(unget = false)
      @s.unget("%") if unget
      @s.expect(Quoted) { @parser.parse_defexp } #, nil]
    end

    def get_raw(prev_lastop = false)
      # FIXME: Workaround for a bug where "first" is not
      # identified as a variable if first introduced inside
      # the case block. Placing this here until the bug
      # is fixed.
      first = nil

      case @s.peek
      when ?`,?",?'
        return [get_quoted_exp, nil]
      when DIGITS
        return [Number.expect(@s, prev_lastop), nil]
      when ALPHA, ?@, ?$, ?:, ?_
        if @s.peek == ?:
          @s.get
          if @s.peek == ?:
            @s.get
            return ["::", Operators["::"]]
          end
          @s.unget(":")
        end
        buf = @s.expect(Atom)
        return [@s.get, Operators[":"]] if !buf
        # Don't check symbols (whose to_s starts with :) against operators
        # This prevents empty symbol :"" (which has to_s ":") from matching ternary operator
        s = buf.to_s
        return [buf, Operators[s]] if s[0] != ?: && Operators.member?(s)
        if @keywords.member?(buf)
          return [buf,nil, :keyword]
        end
        return [buf, nil]
      when ?%
        # Handle percent literals or modulo operator
        # Note: %s(...) for s-expressions is handled by SEXParser before tokenization
        #
        # Heuristic to distinguish percent literals from modulo:
        # 1. After an operator or at statement start: always percent literal
        # 2. Followed by letter + non-alnum (like %Q{): always percent literal
        # 3. Followed by non-alnum that's not space (like %{): percent literal if after operator/start
        # 4. Otherwise: modulo operator
        #
        # This handles both:
        # - Standard cases: x = %Q{...} (at statement start)
        # - Argument cases: eval %Q{...} (after identifier, but followed by Q{)

        # Separate handling for %Q and %{ specifically
        # These need to work even after an identifier (like `eval %{...}`)
        percent_start_pos = @s.position  # Save position before consuming
        pct_char = @s.get  # consume '%'

        # Check for %{ (plain percent literal with brace delimiter)
        if @s.peek == ?{
          delim = "{"
          closing = "}"
          @s.get  # consume opening delimiter

          # Parse content until closing delimiter, handling interpolation
          ret = nil
          buf = ""
          depth = 1

          while depth > 0
            c = @s.peek
            raise CompilerError.new("Unterminated percent literal", percent_start_pos) if c == nil

            @s.get
            if c == "\\"
              # Handle escape sequences
              next_char = @s.peek
              if next_char
                @s.get
                buf << "\\" << next_char
              else
                buf << "\\"
              end
            elsif c == "{"
              depth += 1
              buf << c
            elsif c == closing
              depth -= 1
              buf << c if depth > 0
            elsif c == "#"
              # Check for interpolation #{ (only if not the closing delimiter)
              if @s.peek == "{"
                # Use Quoted.handle_interpolation helper
                result = Tokens::Quoted.handle_interpolation(@s, ret, buf) { @parser.parse_defexp }
                if result
                  ret = result
                  buf = ""
                end
              else
                # Not interpolation, add literal "#" to buffer
                buf << "#"
              end
            else
              buf << c
            end
          end

          # Return interpolated string or plain string
          if ret
            ret << buf if buf != ""
            return [ret, nil]
          else
            return [buf, nil]
          end
        elsif @s.peek == ?Q
          type = @s.get  # consume 'Q'

          # Check if we have a delimiter
          delim = @s.peek
          is_delimiter = delim && !ALNUM.member?(delim)
          if is_delimiter
            # Determine closing delimiter
            closing = case delim
            when "{" then "}"
            when "(" then ")"
            when "[" then "]"
            when "<" then ">"
            else delim
            end

            @s.get  # consume opening delimiter

            # Parse content until closing delimiter, handling interpolation
            ret = nil
            buf = ""
            depth = 1
            paired = (delim == "{" || delim == "(" || delim == "[" || delim == "<")

            while depth > 0
              c = @s.peek
              raise CompilerError.new("Unterminated percent literal", percent_start_pos) if c == nil

              @s.get
              if c == "\\"
                # Handle escape sequences
                next_char = @s.peek
                if next_char
                  @s.get
                  buf << "\\" << next_char
                else
                  buf << "\\"
                end
              elsif paired && c == delim
                depth += 1
                buf << c
              elsif c == closing
                depth -= 1
                buf << c if depth > 0
              elsif c == "#"
                # Check for interpolation #{ (only if not the closing delimiter)
                if @s.peek == "{"
                  # Use Quoted.handle_interpolation helper
                  result = Tokens::Quoted.handle_interpolation(@s, ret, buf) { @parser.parse_defexp }
                  if result
                    ret = result
                    buf = ""
                  end
                else
                  # Not interpolation, add literal "#" to buffer
                  buf << "#"
                end
              else
                buf << c
              end
            end

            # Return interpolated string or plain string
            if ret
              ret << buf if buf != ""
              return [ret, nil]
            else
              return [buf, nil]
            end
          else
            # Not a valid delimiter - treat as modulo
            @s.unget  # put back Q
            @s.unget  # put back %
            return read_token  # retry
          end
        elsif @s.peek == ?s
          # %s{} is a symbol literal
          # Note: %s() is hijacked for s-expressions, so only handle %s{}
          @s.get  # consume 's'
          if @s.peek == ?{
            @s.get  # consume '{'
            buf = ""
            while @s.peek && @s.peek != ?}
              buf << @s.get
            end
            @s.get if @s.peek == ?}  # consume '}'
            return [":#{buf}".to_sym, nil]
          else
            # Not %s{} - unget and let other code handle it
            @s.unget  # put back s
            @s.unget  # put back %
            return read_token  # retry
          end
        elsif @first || prev_lastop
          # '%' already consumed above
          # percent_start_pos already set before consuming '%'

          # Check for type character
          type = nil
          if @s.peek && ALPHA.member?(@s.peek)
            type = @s.get
          end

          # Check if we have a delimiter
          # Percent literals can use any non-alphanumeric character as delimiter
          delim = @s.peek
          is_delimiter = delim && !ALNUM.member?(delim)
          if is_delimiter
            # Determine closing delimiter
            closing = case delim
            when "{" then "}"
            when "(" then ")"
            when "[" then "]"
            when "<" then ">"
            else delim
            end

            @s.get  # consume opening delimiter

            # Parse content until closing delimiter
            # For %Q, %W, %I, %x, %r: handle interpolation
            # For %q, %w, %i: no interpolation
            ret = nil
            buf = ""
            depth = 1
            paired = (delim == "{" || delim == "(" || delim == "[" || delim == "<")
            needs_interpolation = (type == ?Q || type == nil || type == ?W || type == ?I || type == ?x || type == ?r)

            while depth > 0
              c = @s.peek
              raise CompilerError.new("Unterminated percent literal", percent_start_pos) if c == nil

              @s.get

              if c.ord == 92 && delim != "\\"  # backslash (but not if backslash is the delimiter)
                # Escape sequence - consume next character literally
                buf << c.chr
                next_c = @s.get
                buf << next_c.chr if next_c
              elsif paired && c == delim
                depth += 1
                buf << c.chr
              elsif c == closing
                depth -= 1
                buf << c.chr if depth > 0
              elsif needs_interpolation && c == "#"
                # Check for interpolation #{ (only for types that support it, and only if not the delimiter)
                if @s.peek == "{"
                  # Use Quoted.handle_interpolation helper
                  result = Tokens::Quoted.handle_interpolation(@s, ret, buf) { @parser.parse_defexp }
                  if result
                    ret = result
                    buf = ""
                  end
                else
                  # Not interpolation, add literal "#" to buffer
                  buf << "#"
                end
              else
                buf << c.chr
              end
            end

            # Finalize content
            content = if ret
              ret << buf if buf != ""
              ret
            else
              buf
            end

            # Return based on type
            case type
            when ?Q, nil
              # %Q{} or %{} - double-quoted string (with interpolation)
              return [content, nil]
            when ?q
              # %q{} - single-quoted string
              return [content, nil]
            when ?w
              # %w{} - array of words (no interpolation)
              return [[:array, *content.split], nil]
            when ?W
              # %W{} - array of words (with interpolation)
              # content might be a string or [:concat, ...] array
              if content.is_a?(Array)
                # Interpolated - need to split at runtime
                # For now, just return the interpolated string wrapped in a split call
                # TODO: This is a simplification - proper implementation would split on whitespace
                return [[:call, content, :split], nil]
              else
                # No interpolation - split at compile time
                return [[:array, *content.split], nil]
              end
            when ?i, ?I
              # %i{} - array of symbols (no interpolation)
              # %I{} - array of symbols (with interpolation)
              # Must prefix with : so transform.rb recognizes them as symbols
              if content.is_a?(Array)
                # Interpolated - call helper to split and convert to symbols at runtime
                return [[:callm, content, :__percent_I], nil]
              else
                # No interpolation - split at compile time
                symbols = content.split.map { |word| (":#{word}").to_sym }
                return [[:array, *symbols], nil]
              end
            when ?x
              # %x{} - command execution (same as backticks)
              # For now, only support literal strings without interpolation
              # Build the AST node directly: [:call, :system, [string]]
              # TODO: Support interpolation like backticks do
              return [[:call, :system, [content]], nil]
            when ?r
              # %r{} - regexp literal
              # For now, convert to Regexp.new(string) call without interpolation or modifiers
              # TODO: Support interpolation and modifiers (i, m, x, o)
              return [[:callm, :Regexp, :new, content], nil]
            else
              # Unknown type - treat as modulo
              @s.unget(type.chr) if type
              @s.unget("%")
            end
          else
            # No delimiter - treat as modulo
            @s.unget(type.chr) if type
            @s.unget("%")
          end
        else
          # Not %Q and not (@first || prev_lastop) - treat as modulo
          # '%' already consumed above, so don't consume again
        end

        # Modulo operator or %= assignment
        # '%' already consumed above
        if @s.peek == ?=
          @s.get
          return ["%=", Operators["%="]]
        end
        return ["%", Operators["%"]]
      when ?/
        if @first || prev_lastop
          # Parse regexp literal with interpolation support
          @s.get  # consume '/'
          ret = nil
          buf = ""
          while true
            c = @s.peek
            if c == nil
              raise "Unterminated regexp"
            end
            @s.get
            if c == ?/
              # End of regexp - capture modifiers
              # Regexp options: i=1, x=2, m=4, fixedencoding=16, noencoding=32
              options = 0
              while @s.peek && (@s.peek == ?i || @s.peek == ?m || @s.peek == ?x || @s.peek == ?o || @s.peek == ?e || @s.peek == ?n || @s.peek == ?s || @s.peek == ?u)
                mod = @s.get
                case mod
                when ?i then options |= 1   # IGNORECASE
                when ?x then options |= 2   # EXTENDED
                when ?m then options |= 4   # MULTILINE
                when ?u, ?e, ?s then options |= 16  # FIXEDENCODING
                when ?n then options |= 32  # NOENCODING
                # 'o' is ignored (once-only evaluation - not relevant for us)
                end
              end
              # Finalize pattern
              if ret
                ret << buf if buf != ""
                pattern = ret
              else
                pattern = buf
              end
              return [[:callm, :Regexp, :new, [pattern, options]], nil]
            elsif c == ?\\
              # Escape sequence
              buf << c.chr
              next_c = @s.get
              buf << next_c.chr if next_c
            elsif c == ?#
              # Check for interpolation #{
              if @s.peek == ?{
                # Use Quoted.handle_interpolation helper
                result = Tokens::Quoted.handle_interpolation(@s, ret, buf) { @parser.parse_defexp }
                if result
                  ret = result
                  buf = ""
                else
                  buf << c.chr
                end
              else
                buf << c.chr
              end
            else
              buf << c.chr
            end
          end
        else
          # Division operator or /= assignment
          @s.get
          if @s.peek == ?=
            @s.get
            return ["/=", Operators["/="]]
          end
          return ["/", Operators["/"]]
        end
      when ?-
        @s.get
        # Only parse as negative number if last token was an operator (not after ')' etc.)
        # Special case: Don't create negative literal if followed by ** to fix precedence
        # e.g., -2**12 should parse as -(2**12) not (-2)**12
        if prev_lastop && DIGITS.member?(@s.peek)
          # Look ahead: consume the number and check what follows
          num_str = ""
          while DIGITS.member?(@s.peek)
            num_str << @s.get
          end
          # Skip whitespace and check for **
          @s.nolfws
          followed_by_power = (@s.peek == ?* && (@s.get; @s.peek == ?*))

          # Only create negative literal if NOT followed by **
          if !followed_by_power
            # Unget the number and use Number.expect to properly handle large integers
            @s.unget(num_str)
            @s.unget("-")
            return [Number.expect(@s, true), nil]
          end

          # Followed by **, so unget the number and first * (second * is still in scanner)
          # Then return "-" operator by falling through
          @s.unget("*")
          @s.unget(num_str)
          # Fall through to return "-" operator (already consumed at line 404)
        end
        @lastop = true
        if @s.peek == ?>
          @s.get
          return [:stabby_lambda, nil, :atom]
        end
        if @s.peek == ?=
          @s.get
          return ["-=",Operators["-="]]
        end
        return ["-",Operators["-"]]
      when nil
        return [nil, nil]
      else
        # Special cases - two/three character operators, and character constants

        first = @s.get

        # Special case: Handle anonymous splat (* = value) or (*)
        # If we see * in prefix position followed by = or ), treat as identifier :_
        if first == "*" && (@first || prev_lastop)
          # Peek ahead through whitespace to check for = or )
          ws_chars = ""
          while @s.peek && [" ", "\t", "\r", "\n"].member?(@s.peek)
            ws_chars << @s.get
          end

          next_char = @s.peek
          if next_char == ?=
            # Check it's not **= or *= (compound assignment)
            equals = @s.get
            following = @s.peek
            if following && following != ?= && following != ?*
              # This is * followed by = (not *= or **=)
              # Unget everything and return :_ as identifier
              @s.unget(equals) if equals
              @s.unget(ws_chars.reverse) if ws_chars && !ws_chars.empty?
              return [:_, nil]
            end
            # It's *= or **= or something else, unget and continue
            @s.unget(equals) if equals
          elsif next_char == ?)
            # This is (*) - anonymous splat in parentheses
            # Return :_ as identifier
            @s.unget(ws_chars.reverse) if ws_chars && !ws_chars.empty?
            return [:_, nil]
          end
          # Unget whitespace and continue normal operator handling
          @s.unget(ws_chars.reverse) if ws_chars && !ws_chars.empty?
        end

        if second = @s.get
          if first == "?" and !([32, 10, 9, 13].member?(second[0].chr.ord))
            # FIXME: This changed in Ruby 1.9 to return a string, which is just plain idiotic.
            if second == "\\"
              third = @s.get

              # Handle common escape sequences
              case third
              when "e"
                return [27, nil]  # ESC
              when "t"
                return [9, nil]   # TAB
              when "n"
                return [10, nil]  # LF
              when "r"
                return [13, nil]  # CR
              when "M"
                # Meta escape: \M-x or \M-\C-x or \M-\c-x
                if @s.peek == "-"
                  @s.get  # consume the dash
                  # Check if it's \M-\C- or \M-\c-
                  if @s.peek == "\\"
                    @s.get  # consume backslash
                    ctrl_char = @s.get
                    if ctrl_char == "C" || ctrl_char == "c"
                      if @s.peek == "-"
                        @s.get  # consume dash
                        ch = @s.get
                        # Meta-Control: set both bit 7 and apply control
                        return [((ch[0].chr.ord & 0x1f) | 0x80), nil]
                      end
                    end
                  else
                    # Just \M-x: set bit 7
                    ch = @s.get
                    return [(ch[0].chr.ord | 0x80), nil]
                  end
                end
              when "C", "c"
                # Control escape: \C-x or \c-x
                if @s.peek == "-"
                  @s.get  # consume the dash
                  ch = @s.get
                  # Control: mask to lower 5 bits
                  return [(ch[0].chr.ord & 0x1f), nil]
                end
              else
                # For other escapes, return the character itself
                return [third[0].chr.ord, nil]
              end
            else
              return [second[0].chr.ord,nil]
            end
          end

          buf = first + second

          # Check for heredoc: << or <<- or <<~
          # Always check for heredoc pattern - will fall back to << operator if not heredoc
          if buf == "<<"
            # Peek ahead to see if this is a heredoc
            squiggly = false
            dash = false
            if @s.peek == ?~
              squiggly = true
              @s.get
            elsif @s.peek == ?-
              dash = true
              @s.get
            end

            # Check if next character starts an identifier or quoted heredoc marker
            if @s.peek && (ALPHA.member?(@s.peek) || @s.peek == ?_ || @s.peek == ?' || @s.peek == ?")
              # This is a heredoc!
              # Read the heredoc marker
              marker = ""
              quoted = false
              quote_char = nil

              if @s.peek == ?' || @s.peek == ?"
                quote_char = @s.get
                quoted = true
                while @s.peek && @s.peek != quote_char
                  marker << @s.get.chr
                end
                @s.get if @s.peek == quote_char  # consume closing quote
              else
                # Unquoted identifier
                while @s.peek && (ALPHA.member?(@s.peek) || DIGITS.member?(@s.peek) || @s.peek == ?_)
                  marker << @s.get.chr
                end
              end

              # Save the rest of the line (there might be more code after the heredoc marker)
              rest_of_line = ""
              while @s.peek && @s.peek != ?\n
                rest_of_line << @s.get.chr
              end
              @s.get if @s.peek == ?\n  # consume newline

              # Read heredoc body directly from the scanner using shared
              # escape/interpolation handling from Quoted.
              interpolate = (quote_char != ?')

              if interpolate
                result = Quoted.expect_heredoc(@s, marker, squiggly) { @parser.parse_defexp }
              else
                # Single-quoted heredoc: no escape processing, read raw body
                result = Quoted.expect_heredoc_squoted(@s, marker, squiggly)
              end

              # Put back the rest of the line so it can be parsed
              @s.unget(rest_of_line) if rest_of_line && !rest_of_line.empty?

              return [result, nil]
            else
              # Not a heredoc, put back the ~ or - if we consumed it
              @s.unget("~") if squiggly
              @s.unget("-") if dash
              # Fall through to normal << operator handling
            end
          end

          if third = @s.get
            buf2 = buf + third
            op = Operators[buf2]
            if op
              op_val = op.is_a?(Hash) ? op[:infix_or_postfix] : op
              @lastop = true if op_val && (op_val.type == :infix || op_val.type == :lp || op_val.type == :prefix)
              return [buf2, op]
            end
            @s.unget(third)
          end
          op = Operators[buf]
          if op
            op_val = op.is_a?(Hash) ? op[:infix_or_postfix] : op
            @lastop = true if op_val && (op_val.type == :infix || op_val.type == :lp || op_val.type == :prefix)
            return [buf, op]
          end
          @s.unget(second)
        end
        op = Operators[first]
        if op
          op_val = op.is_a?(Hash) ? op[:infix_or_postfix] : op
          @lastop = true if op_val && (op_val.type == :infix || op_val.type == :lp || op_val.type == :prefix)
          return [first, op]
        end
        @s.unget(first)
        return [nil, nil]
      end
    end

    def ws
      @s.ws
    end

    def nolfws
      @s.nolfws
    end

    attr_reader :lasttoken

    def get
      @lasttoken = @curtoken

      if @last.is_a?(Array) && @last[1].is_a?(Oper) && @last[1].sym == :callm
        @lastop = false
        @lastpos = @s.position
        res = Methodname.expect(@s)
        # Support .() syntax for lambda/proc calls - insert :call as method name
        if !res && @s.peek == ?(
          res = :call
        end
        res = [res,nil] if res
      else
        # Check if there's a newline before consuming any whitespace
        # This must be done BEFORE calling ws/nolfws as they consume the newline
        @newline_before_current = (@s.peek && @s.peek.ord == 10)

        # FIXME: This rule should likely cover more
        # cases; may want additional flags
        if @lastop && @last && @last[0] != :return
          @s.ws
        else
          @s.nolfws
        end

        # Parser bug fix: Track newlines for @lastop to fix negative number parsing
        # This fixes: 4.ceildiv(-3) followed by -4.ceildiv(3) on next line
        # Without this, -4 is parsed as binary subtraction instead of unary minus
        # Store boolean directly to avoid local variable issues in self-hosted compiler
        @at_newline = @s.peek ? (@s.peek.ord == 10) : false

        prev_lastop = @lastop
        @lastop = false
        @lastpos = @s.position
        res = get_raw(prev_lastop)
      end
      # The is_a? weeds out hashes, which we assume don't contain :rp operators
      # Save old @last to check what token preceded the newline
      old_last = @last
      @last = res
      # FIXME: res can be nil in some contexts, causing crashes later
      # Parser bug fix: Only set @lastop = true after newline if previous token was an operator
      # Stabby lambda method calls (-> { x }.a) should NOT skip newlines
      # But operators like ) should skip newlines to allow: 4.ceildiv(-3) \n -4.ceildiv(3)
      if @at_newline && old_last && old_last[1] && old_last[1].is_a?(Oper)
        # Previous token was an operator, so newline can be skipped
        @lastop = true
      else
        # Normal logic: only set @lastop for non-rp operators
        @lastop = res && res[1] && (!res[1].is_a?(Oper) || res[1].type != :rp)
      end
      @curtoken = res
      return res
    end

  end
end