tokenizer.js

var tokens = [];
var source = '';
var regexQueue = [];
var debugFlag = false;

function logSrc () {
  console.log(source);
}

function logTok () {
  console.log('Tokens: ' + tokens);
}

exports.__defineSetter__('debug', function (value) {
  if (typeof value === 'boolean') debugFlag = value;
});

exports.rule = function (tokenType, re) {
  regexQueue.push(function () {
    var ret = false;
    var result = re.exec(source);
    if (result) {
      var value = result[0]
      if (debugFlag) console.log(tokenType + ' token: ' + value);
      var token = {
        value,
        tokenType
      }
      tokens.push(token);
      source = source.substring(value.length);
      ret = true;
    }
    return ret;
  });
}

exports.tokenize = function (src) {
  tokens = [];
  source = '';
  source = src;
  if (debugFlag) {
    console.log('-- Starting tokenizer --');
    logSrc();
    console.log('--                    --');
  }

  while (source) {
    var foundToken = regexQueue.some(function (element, index, array) {
      return element();
    });
    if (!foundToken) {
      console.error('Did not find any tokens on this pass:');
      logSrc();
      process.exit(1);
    }
  }

  if (debugFlag) {
    console.log('-- Tokenizing complete --');
    console.log(tokens);
    console.log('--                     --');
  }
  return tokens;
}