@ -9,12 +9,14 @@ class Lexer {
public function __construct(string $sourceCode, ?string $fileName = null) {
public function __construct(string $sourceCode, ?string $fileName = null) {
$this->fileName = $fileName;
$this->fileName = $fileName;
$unifiedSource = str_replace(["\r\n", "\r"], "\n", $sourceCode);
$unifiedSource = str_replace(["\r\n", "\r"], "\n", $sourceCode);
$unifiedSource = str_replace(" ", "\t", $unifiedSource);
$this->source = explode("\n", trim($unifiedSource, "\n"));
$this->source = explode("\n", trim($unifiedSource, "\n"));
}
}
public function tokenize() : array {
public function tokenize() : array {
$tokens = [];
$tokens = [];
$row = 1;
$row = 1;
$col = 1;
foreach ($this->source as $line) {
foreach ($this->source as $line) {
if (strlen($line) < 1 ) {
if (strlen($line) < 1 ) {
@ -129,7 +131,6 @@ class Lexer {
$row++;
$row++;
}
}
$clearBuffer();
array_push($tokens, new Token(TokenType::EOF, "\0", [$col, $row, $this->fileName]));
array_push($tokens, new Token(TokenType::EOF, "\0", [$col, $row, $this->fileName]));
return $tokens;
return $tokens;