CoreHelpers.tl

@@ -53,7 +53,7 @@ end
 
 
 function CoreHelpers.parseToken(state: Environment): string
-	local chr = ""
+	local chr = state.activeInputStream:readCurrentCharacter()
 	local token = ""
 	while(not CoreHelpers.isWhitespace(chr)) do
 		token = token..chr

InputStream.tl

@@ -3,6 +3,7 @@ local type InputStream = record
 	offset: number
 	refill: function(): string
 
+	new: function(InputStream): InputStream
 	new: function(InputStream, function(): string): InputStream
 
 	setRefill: function(InputStream, function(): string)
@@ -18,36 +19,28 @@ function InputStream:_manageBuffer()
 		self.str = self.refill()
 		self.offset = 1
 	end
-	if self.offset > #self.str then
+	if self.offset > #self.str + 1 then
 		self.str = self.refill()
 		self.offset = 1
 	end
 end
 
-local function initial_stream(): string
-	return "DUMMY"
-end
 
 
 -- constructors
 function InputStream:new(): InputStream
-	local mt = setmetatable(
+	return setmetatable(
 		{
 			string = "",
-			offset = 1,
-			refill = initial_stream
+			offset = 1
 		} as InputStream,
 		istream_mt)
-	mt:setRefill(initial_stream)
-	return mt
 end
 function InputStream:new(refill: function(): string): InputStream
-	refill = refill or initial_stream
 	local mt = setmetatable(
 		{
 			string = "",
-			offset = 1,
-			refill = function(): string return "DUMMY" end
+			offset = 1
 		} as InputStream,
 		istream_mt)
 	mt:setRefill(refill)
@@ -60,7 +53,6 @@ end
 -- setters/getters
 function InputStream:setRefill(func: function(): string)
 	self.refill = func
-	self:_manageBuffer()
 end
 
 -- 
@@ -70,10 +62,9 @@ function InputStream:readCurrentCharacter(): string
 end
 
 function InputStream:advanceOffset(): string
-	self:_manageBuffer()
-	local current_char = self.str:sub(self.offset, self.offset)
 	self.offset = self.offset + 1
-	return current_char
+	self:_manageBuffer()
+	return self.str:sub(self.offset, self.offset)
 end
 
 return InputStream