refactor: types are not natively lexed

3 files changed, 15 insertions, 24 deletions

diff --git a/neb/lexer.py b/neb/lexer.py
index 22a9710..6962b5e 100644
--- a/neb/lexer.py
+++ b/neb/lexer.py
@@ -23,7 +23,7 @@ keywords = {
 
 
 WHITESPACE = [" ", "\n", "\t"]
-SEPARATORS = WHITESPACE + [")"]
+SEPARATORS = WHITESPACE + [")", "]"]
 DIGITS = list("0123456789")
 
 def lex(data):
@@ -49,6 +49,12 @@ def lex(data):
             tokens.append(Token(TokenType.OPEN_PAREN, "(", None, line))
         elif char == ")":
             tokens.append(Token(TokenType.CLOSE_PAREN, ")", None, line))
+        elif char == "[":
+            tokens.append(Token(TokenType.OPEN_BRACKET, "[", None, line))
+        elif char == "]":
+            tokens.append(Token(TokenType.CLOSE_BRACKET, "]", None, line))
+        elif char == ":":
+            tokens.append(Token(TokenType.COLON, ":", None, line))
         # numbers
         elif char in DIGITS or char == ".":
             tok, length = get_number(data[current:], line)
@@ -65,11 +71,6 @@ def lex(data):
             tok, length = get_bool(data[current+1:], line)
             tokens.append(tok)
             current += length
-        #types
-        elif char == ":":
-            tok, length = get_type(data[current:], line) # include :
-            tokens.append(tok)
-            current += length
         # symbols
         else:
             tok, length = get_symbol(data[current:], line)
@@ -153,18 +154,3 @@ def get_symbol(data, line):
         if counter >= len(data):
             break
     return Token(TokenType.SYMBOL, value, None, line), counter - 1
-
-def get_type(data, line):
-    counter = 0
-    value = ""
-    while data[counter] not in SEPARATORS:
-        value += data[counter]
-        counter += 1
-        if counter >= len(data):
-            break
-    if value in types:
-        typ = types[value]
-    else:
-        typ = TokenType.USER_TYPE
-    return Token(typ, value, None, line), counter - 1
-
diff --git a/neb/parser.py b/neb/parser.py
index 08ab0be..bbaf7a8 100644
--- a/neb/parser.py
+++ b/neb/parser.py
@@ -17,7 +17,7 @@ def parseExpression(token, prev, tokens):
         elif token.type_ in (TokenType.STRING, TokenType.TRUE, TokenType.FALSE, TokenType.INT, TokenType.FLOAT):
             expr, inc = parseLiteral(token, prev, tokens[idx+1:])
             args.append(expr)
-        elif token.type_ in (TokenType.INT_TYPE, TokenType.FLOAT_TYPE, TokenType.STRING_TYPE, TokenType.ANY_TYPE, TokenType.LIST_TYPE, TokenType.NUMBER_TYPE, TokenType.BOOL_TYPE, TokenType.LITERAL_TYPE, TokenType.USER_TYPE):
+        elif token.type_ == TokenType.COLON:
             expr, inc = parseType(token, prev, tokens[idx+1:])
             args.append(expr)
         else:
@@ -44,7 +44,9 @@ def parseLiteral(token, prev, tokens):
         return Literal(token.value), 1
 
 def parseType(token, prev, tokens):
-    return Type(token.text), 1
+    # if the next token is a symbol, combine for a type
+    if len(tokens) > 0 and tokens[0].type_ == TokenType.SYMBOL:
+        return Type(f":{tokens[0].text}"), 2
 
 def parse(tokens):
     idx = 0
@@ -61,7 +63,7 @@ def parse(tokens):
         elif token.type_ in (TokenType.FALSE, TokenType.TRUE, TokenType.STRING, TokenType.INT, TokenType.FLOAT):
             lit, counter = parseLiteral(token, prev, tokens[idx+1:])
             exprs.append(lit)
-        elif token.type_ in (TokenType.INT_TYPE, TokenType.FLOAT_TYPE, TokenType.STRING_TYPE, TokenType.ANY_TYPE, TokenType.LIST_TYPE, TokenType.NUMBER_TYPE, TokenType.BOOL_TYPE, TokenType.LITERAL_TYPE, TokenType.USER_TYPE):
+        elif token.type_ == TokenType.COLON:
             typ, counter = parseType(token, prev, tokens[idx+1:])
             exprs.append(typ)
         else:
diff --git a/neb/structs.py b/neb/structs.py
index 5298e61..b8effd0 100644
--- a/neb/structs.py
+++ b/neb/structs.py
@@ -11,6 +11,8 @@ class TokenType(Enum):
 
     OPEN_PAREN = auto()
     CLOSE_PAREN = auto()
+    OPEN_BRACKET = auto()
+    CLOSE_BRACKET = auto()
 
     EOF = auto()
 
@@ -43,6 +45,7 @@ class TokenType(Enum):
     USER_TYPE = auto()
 
     MANY = auto()
+    COLON = auto()
 
 @dataclass
 class Token: