NodeBASIC Lexer

Current Token Output

[ { type: 'identifier', match: 'The' },
  { type: 'identifier', match: 'quick' },
  { type: 'identifier', match: 'brown' },
  { type: 'identifier', match: 'dog' },
  { type: 'identifier', match: 'jumped' },
  { type: 'identifier', match: 'over' },
  { type: 'identifier', match: 'the' },
  { type: 'identifier', match: 'lazy' },
  { type: 'identifier', match: 'rabbit' },
  { type: 'operator', match: '=' },
  { type: 'identifier', match: '1' },
  { type: 'eol', match: ';' },
  { type: 'identifier', match: 'message' },
  { type: 'operator', match: '=' },
  { type: 'string',
    match: 'Hello, I am you from another universe!' },
  { type: 'eol', match: ';' } ]

lexer.js

//NodeBASIC
//By ShadowCX11

parser = {
	"tokenTypes": (function(){
		var tokens   = {},
		ws                   = '\\s';
		tokens["string"]     = new RegExp('"([^"]*)\\"'),
		tokens["identifier"] = new RegExp('(\\w+[\\w|0-9]*)'),
		tokens["number"]     = new RegExp('([0-9]+(?:\\.(?:[0-9]+))?)'),
		tokens["operator"]   = new RegExp('([\\+|\\-|\\*|\\/|\\=])'),
		tokens["eol"]        = new RegExp('([\r\n|\r|\n|;])'),
		tokens["whitespace"] = new RegExp("(" + ws + "+)");
		return tokens;
	})(),
	"tokenize": function(string){
		var tokens = this.tokens = [],
			index = 0,
			scp = {},
			str = string;
		while(true){
			scp["token"] = null,
			str = str.substr(index);
			if(str === ""){
				break;
			}
			for(var type in this.tokenTypes){
				//console.log(type)
				var regexp = this.tokenTypes[type],
					test   = regexp.test(str);
				if(test){
					var d = regexp.exec(str),
					possibleToken = {
						"type": type,
						"match": d[1],
						"index": d.index
					};
					if(type !== "eol"){console.log('"' + d[1] + '"', '=', type);};
					thisIndex = d.index;
					scpIndex = scp["token"] !== null ? scp["token"].index : 0;
					if((thisIndex <= scpIndex || scp["token"] === null) && type !== "whitespace"){
						scp.token = possibleToken;
						index = thisIndex + d[1].length;
					};
				};
			}
			if(scp["token"] === null){
				console.log(tokens)
				console.log(possibleToken)
				throw new Error("SyntaxError in NodeBASIC (" + str + ")");
			};
			var token = scp.token;
			delete token.index;
			tokens.push(token);
		};
		return this.tokens = tokens;
	}
};
console.log(parser.tokenize('The quick brown dog jumped over the lazy rabbit = 1; message = "Hello, I am you from another universe!";'));