update grammar a bit

[jwcc.git] / lib / tokenizer / tokenizer.lua

function chartonumber(char)
	local chartable = { ["0"] = 0, ["1"] = 1, ["2"] = 2, ["3"] = 3, ["4"] = 4, ["5"] = 5,
			    ["6"] = 6, ["7"] = 7, ["8"] = 8, ["9"] = 9 }
	return chartable[char]
end

function iswhitespace(char)
	local whitespace = { [" "] = true, ["\r"] = true, ["\n"] = true, ["\t"] = true, [""] = true}
	return whitespace[char]
end

function isidentifierchar(char)
	if chartonumber(char) then
		return true
	end
	if (char:byte(1) >= ("A"):byte(1) and char:byte(1) <= ("Z"):byte(1)) or
	   (char:byte(1) >= ("a"):byte(1) and char:byte(1) <= ("z"):byte(1)) or
	   char == "_" then
		return true
	end
	return false
end

function readToken(input)
	local token = {}
	local keywords = {"if", "int", "(", ")", "{", "}", ";", ",", "+", "-", "*", "/", "while", "<", ">", "==", "=", "return"}
	
	-- strip off whitespace from the input
	while iswhitespace(input:sub(1,1)) and input:len() > 0 do
		input = input:sub(2)
	end
	
	if input:len() == 0 then
		return "", nil
	end
	
	for i,keyword in pairs(keywords) do
		if input:sub(1,keyword:len()) == keyword then
			input = input:sub(keyword:len() + 1)
			token.type = keyword
			return input,token
		end
	end
	
	-- okay, let's try to tokenize a number
	if chartonumber(input:sub(1,1)) then
		token.type = "number"
		token.value = 0
		while chartonumber(input:sub(1,1)) do
			token.value = token.value*10 + chartonumber(input:sub(1,1))
			input = input:sub(2)
		end
		if not iswhitespace(input:sub(1,1))
		   and input:sub(1,1) ~= ")"
		   and input:sub(1,1) ~= "}"
		   and input:sub(1,1) ~= ";"
		   and input:sub(1,1) ~= "+"
		   and input:sub(1,1) ~= ","
		   and input:sub(1,1) ~= "-" then
			error("expected one of whitespace, ), }, ;, +, - after number; got "..input:sub(1,1))
		end
		return input,token
	end
	
	-- ok, let's try to tokenize an identifier now.
	if isidentifierchar(input:sub(1,1)) then
		token.type = "identifier"
		token.value = ""
		while isidentifierchar(input:sub(1,1)) do
			token.value = token.value .. input:sub(1,1)
			input = input:sub(2)
		end
		return input,token
	end
	
	error("invalid character to start token: "..input:sub(1,1).." ("..input:byte(1)..")")
end

function tokenize(input)
	local tokenlist = {}
	while input:len() > 0 do
		local token
		input,token = readToken(input)
		table.insert(tokenlist, token)
	end
	return tokenlist
end