From 5274edb5484f75266304d21840f8e854f2a1d3c5 Mon Sep 17 00:00:00 2001
From: mryouse
Date: Thu, 30 Jun 2022 03:08:39 +0000
Subject: refactor: types are not natively lexed

---
 neb/lexer.py | 28 +++++++---------------------
 1 file changed, 7 insertions(+), 21 deletions(-)

(limited to 'neb/lexer.py')

diff --git a/neb/lexer.py b/neb/lexer.py
index 22a9710..6962b5e 100644
--- a/neb/lexer.py
+++ b/neb/lexer.py
@@ -23,7 +23,7 @@ keywords = {
 
 
 WHITESPACE = [" ", "\n", "\t"]
-SEPARATORS = WHITESPACE + [")"]
+SEPARATORS = WHITESPACE + [")", "]"]
 DIGITS = list("0123456789")
 
 def lex(data):
@@ -49,6 +49,12 @@ def lex(data):
             tokens.append(Token(TokenType.OPEN_PAREN, "(", None, line))
         elif char == ")":
             tokens.append(Token(TokenType.CLOSE_PAREN, ")", None, line))
+        elif char == "[":
+            tokens.append(Token(TokenType.OPEN_BRACKET, "[", None, line))
+        elif char == "]":
+            tokens.append(Token(TokenType.CLOSE_BRACKET, "]", None, line))
+        elif char == ":":
+            tokens.append(Token(TokenType.COLON, ":", None, line))
         # numbers
         elif char in DIGITS or char == ".":
             tok, length = get_number(data[current:], line)
@@ -65,11 +71,6 @@ def lex(data):
             tok, length = get_bool(data[current+1:], line)
             tokens.append(tok)
             current += length
-        #types
-        elif char == ":":
-            tok, length = get_type(data[current:], line) # include :
-            tokens.append(tok)
-            current += length
         # symbols
         else:
             tok, length = get_symbol(data[current:], line)
@@ -153,18 +154,3 @@ def get_symbol(data, line):
         if counter >= len(data):
             break
     return Token(TokenType.SYMBOL, value, None, line), counter - 1
-
-def get_type(data, line):
-    counter = 0
-    value = ""
-    while data[counter] not in SEPARATORS:
-        value += data[counter]
-        counter += 1
-        if counter >= len(data):
-            break
-    if value in types:
-        typ = types[value]
-    else:
-        typ = TokenType.USER_TYPE
-    return Token(typ, value, None, line), counter - 1
-
-- 
cgit v1.2.3