Conditional Statements: Code
evaluator.py
import builtins
import io
import re
import sys
from contextlib import redirect_stdout
from unittest.mock import patch
import parser
import tokenizer
def global_environment(environment):
# The runner uses one dictionary. Tests can supply nested environments;
# the two input/output variables still belong to the root dictionary.
while "$PARENT" in environment:
environment = environment["$PARENT"]
return environment
def evaluate(ast, environment):
"""Return (value, status); None means evaluation completed normally.
A special status is propagated before any remaining computation or effect.
Only the structure responsible for that status may consume it. For now,
exit travels through the program to the runner, as in the old interpreter.
The status check after each child evaluation is repeated on purpose: evaluate
a child, and if its status is not None, return that pair before doing more.
"""
if ast["tag"] == "assert":
# An exit during the condition takes precedence over assertion handling.
value, status = evaluate(ast["expression"], environment)
if status is not None:
return value, status
if type(value) is not bool:
raise TypeError("assert requires a boolean expression")
if value:
return None, None
message = "Assertion failed: " + ast["explanation"]
# Report here, then request ordinary program exit. No exception payload
# needs to travel through the tree because Vertex has no catch mechanism.
print(message, file=sys.stderr)
return 1, "exit"
if ast["tag"] == "exit":
value = 0
if ast["expression"] is not None:
value, status = evaluate(ast["expression"], environment)
if status is not None:
return value, status
if type(value) is not int:
raise TypeError("exit code must be an integer")
return value, "exit"
if ast["tag"] == "boolean":
return ast["value"], None
if ast["tag"] == "not":
# Do not let Python coerce numbers or strings to truth values for Vertex.
value, status = evaluate(ast["operand"], environment)
if status is not None:
return value, status
if type(value) is not bool:
raise TypeError("not requires a boolean operand")
return not value, None
if ast["tag"] in ("and", "or"):
# Evaluate and validate the left operand before considering the right.
# An eager evaluation of both children would consume input even when
# that input cannot affect the result.
left, status = evaluate(ast["left"], environment)
if status is not None:
return left, status
if type(left) is not bool:
raise TypeError(f"{ast['tag']} requires boolean operands")
# Do not evaluate the right side when the left already decides the result.
if ast["tag"] == "and" and not left:
return False, None
if ast["tag"] == "or" and left:
return True, None
right, status = evaluate(ast["right"], environment)
if status is not None:
return right, status
if type(right) is not bool:
raise TypeError(f"{ast['tag']} requires boolean operands")
# Only true-and-right and false-or-right reach here. In both cases the
# validated right boolean is also the result of the whole expression.
return right, None
if ast["tag"] == "number":
return ast["value"], None
# ===== CHAPTER 3: strings are values =====
if ast["tag"] == "string":
return ast["value"], None
if ast["tag"] == "identifier":
identifier = ast["value"]
env = environment
while True:
if identifier in env:
return env[identifier], None
if "$PARENT" in env:
env = env["$PARENT"]
continue
raise ValueError(f"Unknown identifier: {identifier}")
# ===== CHAPTER 3: dedicated input expression =====
if ast["tag"] == "input":
prompt_ast = ast["prompt"]
prompt = None
if prompt_ast is not None:
prompt, status = evaluate(prompt_ast, environment)
if status is not None:
return prompt, status
if prompt_ast is not None and not isinstance(prompt, str):
raise TypeError("input prompt must be a string")
supplied = global_environment(environment).get("__input", "")
if not isinstance(supplied, str):
raise TypeError("__input must be a string")
if supplied != "":
# Clear before returning so a later input expression cannot reuse it.
global_environment(environment)["__input"] = ""
return supplied, None
if prompt_ast is None:
return builtins.input(), None
return builtins.input(prompt), None
if ast["tag"] == "type_query":
# Evaluate exactly once: type(input()) still consumes input. Report
# Vertex types, not Python's distinction between int and float.
value, status = evaluate(ast["expression"], environment)
if status is not None:
return value, status
if type(value) is bool:
return "boolean", None
if type(value) in (int, float):
return "number", None
if type(value) is str:
return "string", None
raise TypeError("type argument must be a Vertex value")
if ast["tag"] == "string_conversion":
value, status = evaluate(ast["expression"], environment)
if status is not None:
return value, status
if type(value) is bool:
if value:
return "true", None
return "false", None
if type(value) in (int, float, str):
return str(value), None
raise TypeError("string argument must be a number, string, or boolean")
if ast["tag"] == "number_conversion":
value, status = evaluate(ast["expression"], environment)
if status is not None:
return value, status
# Explicit conversion permits booleans as 1/0 without making them
# implicit arithmetic operands. Existing numbers retain their type.
if type(value) is bool:
if value:
return 1, None
return 0, None
if type(value) in (int, float):
return value, None
if type(value) is not str:
raise TypeError("number argument must be a number, string, or boolean")
text = value.strip()
if not re.fullmatch(r"[+-]?(?:\d*\.\d+|\d+\.\d*|\d+)", text):
raise ValueError(f"Invalid number: {value!r}")
if "." in text:
return float(text), None
return int(text), None
if ast["tag"] == "boolean_conversion":
value, status = evaluate(ast["expression"], environment)
if status is not None:
return value, status
if type(value) is bool:
return value, None
if type(value) in (int, float):
return value != 0, None
if type(value) is str:
# Do not use Python's bool(text): even "false" would become true.
text = value.strip().lower()
if text in ("true", "false"):
return text == "true", None
raise ValueError(f"Invalid boolean: {value!r}")
raise TypeError("boolean argument must be a number, string, or boolean")
if ast["tag"] == "assign":
value, status = evaluate(ast["expression"], environment)
if status is not None:
return value, status
target = ast["target"]
if target["tag"] != "identifier":
raise ValueError("Assignment requires an identifier destination")
name = target["value"]
destination = environment
if name in ("__input", "__output"):
destination = global_environment(environment)
destination[name] = value
return None, None
if ast["tag"] == "unary-":
value, status = evaluate(ast["operand"], environment)
if status is not None:
return value, status
if type(value) is bool:
raise TypeError("Unary minus requires a number")
return -value, None
if ast["tag"] in ("==", "!=", "<", "<=", ">", ">="):
# Comparisons evaluate both operands, left to right. Unlike logical
# operators they have no short-circuit case.
left, status = evaluate(ast["left"], environment)
if status is not None:
return left, status
right, status = evaluate(ast["right"], environment)
if status is not None:
return right, status
numeric = type(left) in (int, float) and type(right) in (int, float)
if ast["tag"] in ("==", "!="):
# Integers and floats share numeric equality. Other types must match:
# true is not 1, and the string "1" is not the number 1.
equal = (numeric or type(left) is type(right)) and left == right
if ast["tag"] == "==":
return equal, None
return not equal, None
strings = type(left) is str and type(right) is str
if not (numeric or strings):
raise TypeError("Ordering comparisons require two numbers or two strings")
# Python string ordering is lexicographic by Unicode code point:
# the first unequal character decides; a matching shorter prefix wins.
if ast["tag"] == "<":
return left < right, None
if ast["tag"] == "<=":
return left <= right, None
if ast["tag"] == ">":
return left > right, None
return left >= right, None
# Python's operators deliberately provide the Chapter 3 behavior:
# string + string concatenates, string * integer repeats, and integer *
# string repeats in the opposite order. Numeric behavior is unchanged.
if ast["tag"] in ("+", "-", "*", "/"):
left, status = evaluate(ast["left"], environment)
if status is not None:
return left, status
right, status = evaluate(ast["right"], environment)
if status is not None:
return right, status
# Python treats bool as an integer; Vertex keeps truth values distinct.
if type(left) is bool or type(right) is bool:
raise TypeError("Arithmetic does not accept boolean operands")
if ast["tag"] == "+":
return left + right, None
if ast["tag"] == "-":
return left - right, None
if ast["tag"] == "*":
return left * right, None
return left / right, None
if ast["tag"] == "print":
result, status = evaluate(ast["expression"], environment)
if status is not None:
return result, status
# Use Vertex's lowercase boolean spellings instead of Python's True/False.
# The terminal and __output must receive the same text.
if type(result) is bool:
if result:
text = "true"
else:
text = "false"
else:
text = str(result)
print(text)
global_environment(environment)["__output"] = text
return None, None
if ast["tag"] == "statement_list":
for statement in ast["statements"]:
value, status = evaluate(statement, environment)
if status is not None:
return value, status
return None, None
# ===== CHAPTER 7: conditionals =====
if ast["tag"] == "if":
condition, status = evaluate(ast["condition"], environment)
if status is not None:
return condition, status
if type(condition) is not bool:
raise TypeError("if condition must be a boolean expression")
# The branch not taken is never evaluated: it cannot read input,
# exit, or fail an assertion. Both branches share this environment;
# a block introduces no scope of its own.
if condition:
return evaluate(ast["then"], environment)
if ast["else"] is not None:
return evaluate(ast["else"], environment)
return None, None
if ast["tag"] == "program":
globals_ = global_environment(environment)
globals_.setdefault("__input", "")
globals_.setdefault("__output", "")
value, status = evaluate(ast["statements"], environment)
if status is not None:
return value, status
return None, None
raise ValueError(f"Unknown AST node: {ast}")
def evaluate_source(source, environment=None):
"""Return ((value, status), environment), retaining both result components."""
if environment is None:
environment = {}
ast = parser.parse(tokenizer.tokenize(source))
result = evaluate(ast, environment)
return result, environment
def test_evaluate_numbers():
print("test evaluate numbers")
ast, rest = parser.parse_expression(tokenizer.tokenize("3*(4+5)"))
assert rest[0]["tag"] is None
assert evaluate(ast, {}) == (27, None)
ast, rest = parser.parse_expression(tokenizer.tokenize("-1.5+2"))
assert rest[0]["tag"] is None
assert evaluate(ast, {}) == (0.5, None)
def test_evaluate_strings():
# ===== CHAPTER 3 TESTS =====
print("test evaluate strings")
cases = [
('"dog" + "cat"', "dogcat"),
('"dog" * 2', "dogdog"),
('2 * "dog"', "dogdog"),
('("ha" * 3) + "!"', "hahaha!"),
]
for source, expected in cases:
ast, rest = parser.parse_expression(tokenizer.tokenize(source))
assert rest[0]["tag"] is None
assert evaluate(ast, {}) == (expected, None)
def test_evaluate_environments():
print("test evaluate environments")
result, environment = evaluate_source('name="Ada";greeting="Hello, "+name')
assert result == (None, None)
assert environment == {"name": "Ada", "greeting": "Hello, Ada", "__input": "", "__output": ""}
def test_evaluate_input():
# ===== CHAPTER 3 TESTS =====
print("test evaluate input")
ast, rest = parser.parse_expression(tokenizer.tokenize("input()"))
assert rest[0]["tag"] is None
with patch("builtins.input", return_value="one line") as fake_input:
assert evaluate(ast, {}) == ("one line", None)
fake_input.assert_called_once_with()
ast, rest = parser.parse_expression(
tokenizer.tokenize('input("Your " + "name? ")')
)
assert rest[0]["tag"] is None
with patch("builtins.input", return_value="Ada") as fake_input:
assert evaluate(ast, {}) == ("Ada", None)
fake_input.assert_called_once_with("Your name? ")
ast, rest = parser.parse_expression(tokenizer.tokenize("input(42)"))
assert rest[0]["tag"] is None
with patch("builtins.input") as fake_input:
try:
evaluate(ast, {})
except TypeError as error:
assert str(error) == "input prompt must be a string"
else:
raise Exception("Expected TypeError for non-string input prompt")
fake_input.assert_not_called()
def test_evaluate_print():
# ===== CHAPTER 3 TEST: parser accepts only print(expression) =====
print("test evaluate print")
output = io.StringIO()
with redirect_stdout(output):
result, environment = evaluate_source('x="ha"*3;print(x)')
assert result == (None, None)
assert environment == {"x": "hahaha", "__input": "", "__output": "hahaha"}
assert output.getvalue() == "hahaha\n"
def test_evaluate_chapter_3_program():
# ===== CHAPTER 3 INTEGRATION TEST =====
print("test evaluate Chapter 3 program")
source = """
name = input("What is your name? ");
greeting = "Hello, " + name + "!";
print(greeting);
line = input();
print("You entered: " + line)
"""
output = io.StringIO()
with patch("builtins.input", side_effect=["Ada", "testing"] ) as fake_input:
with redirect_stdout(output):
result, environment = evaluate_source(source)
assert result == (None, None)
assert environment == {
"name": "Ada",
"greeting": "Hello, Ada!",
"line": "testing",
"__input": "",
"__output": "You entered: testing",
}
assert output.getvalue() == "Hello, Ada!\nYou entered: testing\n"
assert fake_input.call_count == 2
assert fake_input.call_args_list[0].args == ("What is your name? ",)
assert fake_input.call_args_list[1].args == ()
def test_evaluate_if():
# ===== CHAPTER 7 TEST =====
print("test evaluate if")
result, environment = evaluate_source('if (true) { x=1 }')
assert result == (None, None)
assert environment["x"] == 1
result, environment = evaluate_source('if (false) { x=1 }')
assert result == (None, None)
assert "x" not in environment
result, environment = evaluate_source('if (false) { x=1 } else { x=2 }')
assert environment["x"] == 2
result, environment = evaluate_source('x=1; if (true) { }; x=2')
assert result == (None, None)
assert environment["x"] == 2
if __name__ == "__main__":
test_evaluate_numbers()
test_evaluate_strings()
test_evaluate_environments()
test_evaluate_input()
test_evaluate_print()
test_evaluate_chapter_3_program()
test_evaluate_if()
print("done.")
example.v
print("Chapter 7: Conditional Statements");
// A single branch: only runs when the condition is true. An if statement
// still needs the usual ";" separator before the next statement, the same
// as any other statement.
if (true) {
print("single branch runs");
};
if (false) {
print("single branch is skipped");
};
// Two branches: exactly one of them runs.
grade = 72;
if (grade >= 90) {
letter = "A";
} else {
letter = "B or lower";
};
print(letter);
// This grammar has no "else if" shortcut. A chain is written as an else
// block containing its own nested if, with its own braces.
if (grade >= 90) {
letter = "A";
} else {
if (grade >= 80) {
letter = "B";
} else {
if (grade >= 70) {
letter = "C";
} else {
letter = "D or lower";
};
};
};
print(letter);
// A block can hold several statements.
if (true) {
a = 1;
b = 2;
print(a + b);
};
// The branch not taken is never evaluated: no input read, no exit, no
// assertion failure, even though the code is right there in the source.
if (false) {
stuck = exit(9) + number(input());
} else {
print("unselected branch never touched exit or input");
};
// A status still escapes outward through nested conditionals, and it still
// stops whatever would have run after it, the same way it did in Chapter 6.
if (true) {
if (true) {
assert grade > 100, "grade should exceed 100";
};
};
print("This statement must not run");
parser.py
# parser.py
from tokenizer import tokenize
# EBNF
#
# program ::= statement_list
# statement_list ::= { ";" } statement { ";" { ";" } statement } { ";" }
# statement ::= assignment_statement | print_statement | exit_expression
# | assert_statement | if_statement
# assert_statement ::= "assert" expression "," <string>
# assignment_statement ::= <identifier> "=" expression
#
# ===== CHAPTER 3: print now requires parentheses =====
# print_statement ::= "print" "(" expression ")"
#
# ===== CHAPTER 7: conditionals and statement blocks =====
# if_statement ::= "if" "(" expression ")" block [ "else" block ]
# block ::= "{" [ statement_list ] { ";" } "}"
# A bare statement is never a valid if/else body. An else that itself
# branches must write its own braces around a nested if_statement: this
# grammar has no "else if" shortcut.
#
# expression ::= logic_or
# logic_or ::= logic_and { "or" logic_and }
# logic_and ::= logic_not { "and" logic_not }
# logic_not ::= "not" logic_not | comparison
# comparison ::= arithmetic_expression [ compare_op arithmetic_expression ]
# compare_op ::= "==" | "!=" | "<" | "<=" | ">" | ">="
# arithmetic_expression ::= term { ("+" | "-") term }
# term ::= unary { ("*" | "/") unary }
# unary ::= "-" unary | factor
#
# ===== CHAPTER 3: strings and input are expression forms =====
# factor ::= <number> | <string> | <identifier> | "true" | "false" | input_expression
# | number_expression | string_expression | boolean_expression
# | type_expression | exit_expression | "(" expression ")"
# exit_expression ::= "exit" "(" [ expression ] ")"
# input_expression ::= "input" "(" [ expression ] ")"
# number_expression ::= "number" "(" expression ")"
# string_expression ::= "string" "(" expression ")"
# boolean_expression ::= "boolean" "(" expression ")"
# type_expression ::= "type" "(" expression ")"
#
# input has function-shaped syntax, but this chapter does not implement
# general function calls, parameters, or function values.
def require(tokens, tag, message):
if tokens[0]["tag"] != tag:
raise SyntaxError(f"{message}, got {tokens[0]}")
return tokens[1:]
def parse_input_expression(tokens):
# ===== CHAPTER 3 =====
# input_expression ::= "input" "(" [ expression ] ")"
tokens = require(tokens, "input", "Expected 'input'")
tokens = require(tokens, "(", "Expected '(' after 'input'")
if tokens[0]["tag"] == ")":
return {"tag": "input", "prompt": None}, tokens[1:]
prompt, tokens = parse_expression(tokens)
tokens = require(tokens, ")", "Expected ')' after input prompt")
return {"tag": "input", "prompt": prompt}, tokens
def parse_factor(tokens):
token = tokens[0]
if token["tag"] == "exit":
return parse_exit_expression(tokens)
# Both keywords become the same kind of literal node, with different values.
if token["tag"] in ("true", "false"):
return {"tag": "boolean", "value": token["tag"] == "true"}, tokens[1:]
if token["tag"] == "number":
return {"tag": "number", "value": token["value"]}, tokens[1:]
# ===== CHAPTER 3: string expression =====
if token["tag"] == "string":
return {"tag": "string", "value": token["value"]}, tokens[1:]
if token["tag"] == "identifier":
return {"tag": "identifier", "value": token["value"]}, tokens[1:]
# ===== CHAPTER 3: dedicated input expression =====
if token["tag"] == "input":
return parse_input_expression(tokens)
# All four forms take one expression. The AST retains the operation tag;
# checking the argument's runtime type belongs to the evaluator.
operations = {"number_conversion": "number", "string_conversion": "string",
"boolean_conversion": "boolean", "type_query": "type"}
if token["tag"] in operations:
name = operations[token["tag"]]
tokens = require(tokens[1:], "(", f"Expected '(' after '{name}'")
expression, tokens = parse_expression(tokens)
tokens = require(tokens, ")", f"Expected ')' after {name} argument")
return {"tag": token["tag"], "expression": expression}, tokens
if token["tag"] == "(":
# Parentheses restart at the lowest-precedence rule, allowing a complete
# logical expression wherever a factor is expected.
node, tokens = parse_expression(tokens[1:])
tokens = require(tokens, ")", "Expected ')'")
return node, tokens
raise SyntaxError(f"Expected factor, got {token}")
def parse_unary(tokens):
"""unary ::= "-" unary | factor"""
if tokens[0]["tag"] == "-":
operand, tokens = parse_unary(tokens[1:])
return {"tag": "unary-", "operand": operand}, tokens
return parse_factor(tokens)
def parse_term(tokens):
"""term ::= unary { ("*" | "/") unary }"""
left, tokens = parse_unary(tokens)
while tokens[0]["tag"] in ["*", "/"]:
operator = tokens[0]["tag"]
right, tokens = parse_unary(tokens[1:])
left = {"tag": operator, "left": left, "right": right}
return left, tokens
def parse_arithmetic_expression(tokens):
"""arithmetic_expression ::= term { ("+" | "-") term }"""
left, tokens = parse_term(tokens)
while tokens[0]["tag"] in ["+", "-"]:
operator = tokens[0]["tag"]
right, tokens = parse_term(tokens[1:])
left = {"tag": operator, "left": left, "right": right}
return left, tokens
def parse_comparison(tokens):
"""comparison ::= arithmetic_expression [ compare_op arithmetic_expression ]"""
# Arithmetic binds more tightly than comparisons: 1 + 2 < 4 compares 3 to 4.
left, tokens = parse_arithmetic_expression(tokens)
# One optional operator, not a loop: unparenthesized comparison chains are
# deliberately outside this grammar. The unused token will cause an error.
if tokens[0]["tag"] in ("==", "!=", "<", "<=", ">", ">="):
operator = tokens[0]["tag"]
right, tokens = parse_arithmetic_expression(tokens[1:])
return {"tag": operator, "left": left, "right": right}, tokens
return left, tokens
def parse_logic_not(tokens):
# Recursive negation accepts "not not x" and "!!x". Falling through to
# comparison makes "not x == y" mean "not (x == y)".
if tokens[0]["tag"] == "not":
operand, tokens = parse_logic_not(tokens[1:])
return {"tag": "not", "operand": operand}, tokens
return parse_comparison(tokens)
def parse_logic_and(tokens):
# Each operand comes from the next tighter precedence level. The loop
# constructs a left-associated tree without evaluating either operand.
left, tokens = parse_logic_not(tokens)
while tokens[0]["tag"] == "and":
right, tokens = parse_logic_not(tokens[1:])
left = {"tag": "and", "left": left, "right": right}
return left, tokens
def parse_logic_or(tokens):
# Parsing an entire conjunction first makes "and" bind more tightly than "or".
left, tokens = parse_logic_and(tokens)
while tokens[0]["tag"] == "or":
right, tokens = parse_logic_and(tokens[1:])
left = {"tag": "or", "left": left, "right": right}
return left, tokens
def parse_expression(tokens):
# Every expression context enters through the lowest-precedence rule.
# Operator aliases were normalized by the tokenizer, not by these helpers.
return parse_logic_or(tokens)
def parse_print_statement(tokens):
# ===== CHAPTER 3: parentheses are required =====
# print_statement ::= "print" "(" expression ")"
tokens = require(tokens, "print", "Expected 'print'")
tokens = require(tokens, "(", "Expected '(' after 'print'")
expression, tokens = parse_expression(tokens)
tokens = require(tokens, ")", "Expected ')' after print argument")
return {"tag": "print", "expression": expression}, tokens
def parse_assignment_statement(tokens):
# assignment_statement ::= <identifier> "=" expression
if tokens[0]["tag"] != "identifier":
raise SyntaxError(f"Expected identifier, got {tokens[0]}")
# The target names a destination. It must not read an existing binding;
# assigning a name for the first time is valid.
identifier = {"tag": "identifier", "value": tokens[0]["value"]}
tokens = require(tokens[1:], "=", "Expected '=' for assignment")
expression, tokens = parse_expression(tokens)
return {
"tag": "assign",
"target": identifier,
"expression": expression,
}, tokens
def parse_statement(tokens):
if tokens[0]["tag"] == "assert":
return parse_assert_statement(tokens)
if tokens[0]["tag"] == "exit":
return parse_exit_expression(tokens)
if tokens[0]["tag"] == "print":
return parse_print_statement(tokens)
# ===== CHAPTER 7 =====
if tokens[0]["tag"] == "if":
return parse_if_statement(tokens)
if tokens[0]["tag"] == "identifier":
return parse_assignment_statement(tokens)
raise SyntaxError(f"Expected statement, got {tokens[0]}")
def parse_block(tokens):
# ===== CHAPTER 7 =====
# block ::= "{" [ statement_list ] { ";" } "}"
# A block wraps a statement_list in braces, so running it does not differ
# from running the top-level program's statement_list, including
# accepting a trailing semicolon before the closing brace. Unlike the
# top-level program, the statement_list itself is optional: "{}" is an
# empty block, not an error. Extra semicolons are harmless even when
# there are no statements. The top-level program is unchanged.
tokens = require(tokens, "{", "Expected '{' to start a block")
while tokens[0]["tag"] == ";":
tokens = tokens[1:]
if tokens[0]["tag"] == "}":
return {"tag": "statement_list", "statements": []}, tokens[1:]
statements, tokens = parse_statement_list(tokens)
tokens = require(tokens, "}", "Expected '}' to close a block")
return statements, tokens
def parse_if_statement(tokens):
# ===== CHAPTER 7 =====
# if_statement ::= "if" "(" expression ")" block [ "else" block ]
# Both branches require braces, so there is no dangling-else ambiguity to
# resolve: this function checks for "else" immediately after parsing its
# own then-block, before returning. An "else" always belongs to whichever
# "if" statement's then-block just closed.
tokens = require(tokens, "if", "Expected 'if'")
tokens = require(tokens, "(", "Expected '(' after 'if'")
condition, tokens = parse_expression(tokens)
tokens = require(tokens, ")", "Expected ')' after if condition")
then_block, tokens = parse_block(tokens)
else_block = None
if tokens[0]["tag"] == "else":
else_block, tokens = parse_block(tokens[1:])
return {"tag": "if", "condition": condition, "then": then_block,
"else": else_block}, tokens
def parse_assert_statement(tokens):
# The required explanation is a string literal, not another computation.
tokens = require(tokens, "assert", "Expected 'assert'")
expression, tokens = parse_expression(tokens)
tokens = require(tokens, ",", "Expected ',' and explanation string after assertion")
if tokens[0]["tag"] != "string":
raise SyntaxError("Expected explanation string after ','")
explanation = tokens[0]["value"]
tokens = tokens[1:]
return {"tag": "assert", "expression": expression,
"explanation": explanation}, tokens
def parse_exit_expression(tokens):
# A dedicated expression, also allowed as a standalone statement.
# Nesting it in an operand makes status propagation observable.
tokens = require(tokens, "exit", "Expected 'exit'")
tokens = require(tokens, "(", "Expected '(' after 'exit'")
if tokens[0]["tag"] == ")":
return {"tag": "exit", "expression": None}, tokens[1:]
expression, tokens = parse_expression(tokens)
tokens = require(tokens, ")", "Expected ')' after exit argument")
return {"tag": "exit", "expression": expression}, tokens
def parse_statement_list(tokens):
# statement_list ::= { ";" } statement { ";" { ";" } statement } { ";" }
statements = []
while tokens[0]["tag"] == ";":
tokens = tokens[1:]
statement, tokens = parse_statement(tokens)
statements.append(statement)
while tokens[0]["tag"] == ";":
while tokens[0]["tag"] == ";":
tokens = tokens[1:]
# ===== CHAPTER 7 =====
# A trailing semicolon is allowed at the end of any statement_list,
# not only at the true end of input: "}" ends a block the same way
# None ends the program.
if tokens[0]["tag"] in (None, "}"):
break
statement, tokens = parse_statement(tokens)
statements.append(statement)
return {"tag": "statement_list", "statements": statements}, tokens
def parse_program(tokens):
statements, tokens = parse_statement_list(tokens)
return {"tag": "program", "statements": statements}, tokens
def parse(tokens):
ast, tokens = parse_program(tokens)
# A valid prefix is not enough: reject missing separators and extra operators.
if tokens[0]["tag"] is not None:
raise SyntaxError(f"Unexpected token: {tokens[0]}")
return ast
def expect_syntax_error(source, text):
try:
parse(tokenize(source))
except SyntaxError as error:
assert text in str(error), str(error)
else:
raise Exception(f"Expected SyntaxError for {source!r}")
def test_parse_factor():
print("test parse_factor()")
ast, rest = parse_factor(tokenize("3"))
assert ast == {"tag": "number", "value": 3}
assert rest[0]["tag"] is None
ast, rest = parse_factor(tokenize("(3+4)"))
assert ast == {
"tag": "+",
"left": {"tag": "number", "value": 3},
"right": {"tag": "number", "value": 4},
}
assert rest[0]["tag"] is None
def test_parse_strings():
# ===== CHAPTER 3 TESTS =====
print("test parse strings")
ast, rest = parse_expression(tokenize('"dog" * 2 + "!"'))
assert ast == {
"tag": "+",
"left": {
"tag": "*",
"left": {"tag": "string", "value": "dog"},
"right": {"tag": "number", "value": 2},
},
"right": {"tag": "string", "value": "!"},
}
assert rest[0]["tag"] is None
def test_parse_input_expression():
# ===== CHAPTER 3 TESTS =====
print("test parse_input_expression()")
ast, rest = parse_input_expression(tokenize("input()"))
assert ast == {"tag": "input", "prompt": None}
assert rest[0]["tag"] is None
ast, rest = parse_input_expression(tokenize('input("Name? ")'))
assert ast == {
"tag": "input",
"prompt": {"tag": "string", "value": "Name? "},
}
assert rest[0]["tag"] is None
ast, rest = parse_expression(tokenize('input("Name? ") + "!"'))
assert ast["tag"] == "+"
assert ast["left"]["tag"] == "input"
assert rest[0]["tag"] is None
expect_syntax_error("x=input", "Expected '('")
expect_syntax_error('x=input("Name? "', "Expected ')'")
def test_parse_unary():
print("test parse_unary()")
ast, rest = parse_unary(tokenize("--3"))
assert ast == {
"tag": "unary-",
"operand": {"tag": "unary-", "operand": {"tag": "number", "value": 3}},
}
assert rest[0]["tag"] is None
def test_parse_term_and_expression():
print("test parse term and expression")
ast, rest = parse_expression(tokenize("3*4+5-6"))
assert ast == {
"tag": "-",
"left": {
"tag": "+",
"left": {
"tag": "*",
"left": {"tag": "number", "value": 3},
"right": {"tag": "number", "value": 4},
},
"right": {"tag": "number", "value": 5},
},
"right": {"tag": "number", "value": 6},
}
assert rest[0]["tag"] is None
def test_parse_print_statement():
# ===== CHAPTER 3 TESTS: only the parenthesized form is valid =====
print("test parse_print_statement()")
ast, rest = parse_print_statement(tokenize('print("hello")'))
assert ast == {
"tag": "print",
"expression": {"tag": "string", "value": "hello"},
}
assert rest[0]["tag"] is None
ast, rest = parse_print_statement(tokenize("print(1+1*3)"))
assert ast["tag"] == "print"
assert rest[0]["tag"] is None
expect_syntax_error("print 1", "Expected '('")
expect_syntax_error("print(1", "Expected ')'")
def test_parse_assignment_statement():
print("test parse_assignment_statement()")
ast, rest = parse_assignment_statement(tokenize('greeting="Hello"'))
assert ast == {
"tag": "assign",
"target": {"tag": "identifier", "value": "greeting"},
"expression": {"tag": "string", "value": "Hello"},
}
assert rest[0]["tag"] is None
def test_parse_block():
# ===== CHAPTER 7 TESTS =====
print("test parse_block()")
ast, rest = parse_block(tokenize("{ x=1 }"))
assert ast == {"tag": "statement_list",
"statements": [{"tag": "assign",
"target": {"tag": "identifier", "value": "x"},
"expression": {"tag": "number", "value": 1}}]}
assert rest[0]["tag"] is None
ast, rest = parse_block(tokenize("{ x=1; y=2 }"))
assert [statement["tag"] for statement in ast["statements"]] == ["assign", "assign"]
assert rest[0]["tag"] is None
# A trailing semicolon before the closing brace is allowed, the same as
# one before the end of the whole program.
ast, rest = parse_block(tokenize("{ x=1; y=2; }"))
assert [statement["tag"] for statement in ast["statements"]] == ["assign", "assign"]
assert rest[0]["tag"] is None
# Nesting: a block may contain another brace-delimited block via "if".
ast, rest = parse_block(tokenize("{ if (true) { x=1 } }"))
assert ast["statements"][0]["tag"] == "if"
assert rest[0]["tag"] is None
try:
parse_block(tokenize("x=1"))
except SyntaxError as error:
assert "Expected '{'" in str(error)
else:
raise Exception("Expected SyntaxError for a block missing '{'")
# A block's statement_list is optional: "{}" is empty, not an error.
ast, rest = parse_block(tokenize("{ }"))
assert ast == {"tag": "statement_list", "statements": []}
assert rest[0]["tag"] is None
for source in ("{ ; }", "{ ;;; }", "{ ; // comment\n ; }"):
ast, rest = parse_block(tokenize(source))
assert ast == {"tag": "statement_list", "statements": []}
assert rest[0]["tag"] is None
expect_syntax_error("if (true) { ;", "Expected statement")
expect_syntax_error("if (true) { x=1", "Expected '}'")
def test_parse_if_statement():
# ===== CHAPTER 7 TESTS =====
print("test parse_if_statement()")
ast, rest = parse_if_statement(tokenize("if (x) { y=1 }"))
assert ast == {
"tag": "if",
"condition": {"tag": "identifier", "value": "x"},
"then": {"tag": "statement_list",
"statements": [{"tag": "assign",
"target": {"tag": "identifier", "value": "y"},
"expression": {"tag": "number", "value": 1}}]},
"else": None,
}
assert rest[0]["tag"] is None
ast, rest = parse_if_statement(tokenize("if (x) { y=1 } else { y=2 }"))
assert ast["then"]["statements"][0]["expression"]["value"] == 1
assert ast["else"]["statements"][0]["expression"]["value"] == 2
assert rest[0]["tag"] is None
# An "else if" chain is written as an else-block containing a nested if.
ast, rest = parse_if_statement(tokenize("if (x) { y=1 } else { if (z) { y=2 } }"))
assert ast["else"]["statements"][0]["tag"] == "if"
assert rest[0]["tag"] is None
# A branch's statement_list is optional: an empty then or else is legal.
ast, rest = parse_if_statement(tokenize("if (x) { }"))
assert ast["then"] == {"tag": "statement_list", "statements": []}
assert ast["else"] is None
assert rest[0]["tag"] is None
ast, rest = parse_if_statement(tokenize("if (x) { } else { }"))
assert ast["then"] == {"tag": "statement_list", "statements": []}
assert ast["else"] == {"tag": "statement_list", "statements": []}
assert rest[0]["tag"] is None
# Bodies must be braced; a bare statement is never a valid branch.
expect_syntax_error("if (x) y=1", "Expected '{'")
expect_syntax_error("if (x) { y=1 } else y=2", "Expected '{'")
expect_syntax_error("if x { y=1 }", "Expected '('")
expect_syntax_error("if (x { y=1 }", "Expected ')'")
# An if statement needs the usual ";" before the next statement, the
# same as any other statement; "else" must directly follow the
# preceding "}" with no semicolon in between.
ast = parse(tokenize("if (true) { x=1 }; y=2"))
assert [statement["tag"] for statement in ast["statements"]["statements"]] == ["if", "assign"]
expect_syntax_error("if (true) { x=1 } y=2", "Unexpected token")
expect_syntax_error("if (true) { x=1 }; else { x=2 }", "Expected statement")
def test_parse_statement_list():
print("test parse_statement_list()")
source = ';;;name=input("Name? ");greeting="Hello, "+name;print(greeting);;;'
ast, rest = parse_statement_list(tokenize(source))
assert [statement["tag"] for statement in ast["statements"]] == [
"assign",
"assign",
"print",
]
assert rest[0]["tag"] is None
for source in ["", ";;;"]:
expect_syntax_error(source, "Expected statement")
def test_parse_program():
print("test parse program")
ast = parse(tokenize('x="ha"*3;print(x)'))
assert ast["tag"] == "program"
assert len(ast["statements"]["statements"]) == 2
# Adjacent statements still require a semicolon.
expect_syntax_error('x="hello" print(x)', "Unexpected token")
if __name__ == "__main__":
test_parse_factor()
test_parse_strings()
test_parse_input_expression()
test_parse_unary()
test_parse_term_and_expression()
test_parse_print_statement()
test_parse_assignment_statement()
test_parse_block()
test_parse_if_statement()
test_parse_statement_list()
test_parse_program()
print("done.")
run-tests.sh
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")"
python3 tokenizer.py
python3 parser.py
python3 evaluator.py
python3 -m unittest discover -v
stderr_capture="$(mktemp)"
actual_output="$(./vertex example.v < /dev/null 2>"$stderr_capture")" && actual_status=0 || actual_status=$?
actual_stderr="$(cat "$stderr_capture")"
rm -f "$stderr_capture"
expected_output='Chapter 7: Conditional Statements
single branch runs
B or lower
C
3
unselected branch never touched exit or input'
expected_status=1
expected_stderr='Assertion failed: grade should exceed 100'
failed=0
if [[ "$actual_output" != "$expected_output" ]]; then
echo "example.v stdout did not match" >&2
diff -u <(printf '%s\n' "$expected_output") <(printf '%s\n' "$actual_output")
failed=1
fi
if [[ "$actual_status" != "$expected_status" ]]; then
echo "example.v exit code was $actual_status, expected $expected_status" >&2
failed=1
fi
if [[ "$actual_stderr" != "$expected_stderr" ]]; then
echo "example.v stderr did not match" >&2
diff -u <(printf '%s\n' "$expected_stderr") <(printf '%s\n' "$actual_stderr")
failed=1
fi
if [[ "$failed" -ne 0 ]]; then
exit 1
fi
echo "All Chapter 7 tests passed."
runner.py
import sys
from evaluator import evaluate
from parser import parse
from tokenizer import tokenize
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: python runner.py <program>")
sys.exit(1)
program = sys.argv[1]
if program.endswith((".t", ".v")):
with open(program, "r") as source_file:
program = source_file.read().strip()
tokens = tokenize(program)
ast = parse(tokens)
value, status = evaluate(ast, {})
if status == "exit":
# Keep process termination outside the recursive evaluator.
sys.exit(value)
if status is not None:
raise ValueError(f"Unhandled evaluation status: {status}")
test_assertions.py
import contextlib
import io
from pathlib import Path
import subprocess
import sys
import unittest
from unittest.mock import patch
from evaluator import evaluate_source
from parser import parse
from tokenizer import tokenize
class AssertionTests(unittest.TestCase):
def test_success_continues(self):
result, env = evaluate_source('assert 1<2,"Expected increasing values";assert true,"ok";done=3')
self.assertEqual(result, (None, None))
self.assertEqual(env['done'], 3)
def test_failure_reports_once_and_prevents_later_effects(self):
for statement, message in [('assert 1==2,"wrong value"',
'Assertion failed: wrong value'),
('assert false,""', 'Assertion failed: '),
('assert false,"line\\nnext"',
'Assertion failed: line\nnext')]:
with self.subTest(statement=statement):
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err), \
patch('builtins.input') as read:
result, env = evaluate_source(
'before=1;' + statement + ';print(input());after=2')
self.assertEqual(result, (1, 'exit'))
self.assertEqual(env['before'], 1)
self.assertNotIn('after', env)
self.assertEqual(env['__output'], '')
self.assertEqual(out.getvalue(), '')
self.assertEqual(err.getvalue(), message + '\n')
read.assert_not_called()
def test_condition_runs_once_and_propagates_exit(self):
with patch('builtins.input', return_value='yes') as read:
self.assertEqual(evaluate_source('assert input()=="yes","Expected yes"')[0], (None, None))
read.assert_called_once_with()
err = io.StringIO()
with contextlib.redirect_stderr(err):
self.assertEqual(evaluate_source('assert exit(7),"unused"')[0], (7, 'exit'))
self.assertEqual(err.getvalue(), '')
self.assertEqual(evaluate_source('assert true or exit(7),"Expected true"')[0], (None, None))
def test_requires_boolean(self):
for expression in ('0', '1', '"true"'):
with self.subTest(expression=expression), self.assertRaises(TypeError):
evaluate_source('assert ' + expression + ',"Requires boolean"')
def test_syntax_and_keyword_boundaries(self):
self.assertEqual([t['tag'] for t in tokenize('assert assertion assert_value')],
['assert', 'identifier', 'identifier', None])
for source in ('assert', 'assert true', 'assert false;',
'assert true;done=1', 'assert true,', 'assert true,42',
'assert true,message', 'assert true,"a","b"',
'assert=true', 'x=assert true', 'print(assert true)',
'assert true,"a"+"b"', 'assert false,"failure";x=1+'):
with self.subTest(source=source), self.assertRaises(SyntaxError):
parse(tokenize(source))
def test_runner_handles_assertion_exit_without_duplicate_diagnostic(self):
runner = Path(__file__).with_name('runner.py')
for source, code, message in [('assert true,"Expected true"', 0, ''),
('assert false,"stopped";print(9)', 1, 'Assertion failed: stopped\n'),
('assert false,"bad value"', 1,
'Assertion failed: bad value\n')]:
result = subprocess.run([sys.executable, str(runner), source],
capture_output=True, text=True,
stdin=subprocess.DEVNULL)
self.assertEqual(result.returncode, code)
self.assertEqual(result.stdout, '')
self.assertEqual(result.stderr, message)
if __name__ == '__main__':
unittest.main()
test_assignment_targets.py
"""Assignment destinations and print effects in the released interpreter."""
import contextlib
import io
import unittest
from evaluator import evaluate
from parser import parse_assignment_statement, parse_print_statement
from tokenizer import tokenize
class AssignmentTests(unittest.TestCase):
def test_target_is_identifier_and_creates_binding(self):
tree, _ = parse_assignment_statement(tokenize("x = 2"))
self.assertEqual(tree["target"], {"tag": "identifier", "value": "x"})
environment = {}
self.assertEqual(evaluate(tree, environment), (None, None))
self.assertEqual(environment["x"], 2)
def test_reassignment_reads_old_value_only_on_right(self):
tree, _ = parse_assignment_statement(tokenize("x = x + 3"))
environment = {"x": 2}
self.assertEqual(evaluate(tree, environment), (None, None))
self.assertEqual(environment["x"], 5)
def test_non_identifier_targets_are_rejected(self):
for source in ['"x" = 2', "2 = 3", "(x) = 2"]:
with self.subTest(source=source):
with self.assertRaises((SyntaxError, AssertionError)):
parse_assignment_statement(tokenize(source))
def test_string_node_is_not_an_assignment_destination(self):
tree = {"tag": "assign", "target": {"tag": "string", "value": "x"},
"expression": {"tag": "number", "value": 2}}
environment = {}
with self.assertRaises(ValueError):
evaluate(tree, environment)
self.assertEqual(environment, {})
def test_print_returns_none_and_writes_output(self):
tree, _ = parse_print_statement(tokenize("print(3)"))
output = io.StringIO()
with contextlib.redirect_stdout(output):
result = evaluate(tree, {})
self.assertEqual(result, (None, None))
self.assertEqual(output.getvalue(), "3\n")
if __name__ == "__main__":
unittest.main()
test_booleans_and_comparisons.py
import io
import unittest
from contextlib import redirect_stdout
from unittest.mock import patch
from evaluator import evaluate_source
from parser import parse
from tokenizer import tokenize
class BooleanComparisonTests(unittest.TestCase):
def value(self, expression):
return evaluate_source("result=" + expression)[1]["result"]
def test_token_order_and_keyword_boundaries(self):
tokens = tokenize("== != <= >= < > = true false true_value falsehood")
self.assertEqual([t["tag"] for t in tokens],
["==", "!=", "<=", ">=", "<", ">", "=", "true",
"false", "identifier", "identifier", None])
for source in ("true=1", "false=2"):
with self.subTest(source=source), self.assertRaises(SyntaxError):
parse(tokenize(source))
def test_literals_and_assignment_nodes(self):
self.assertIs(self.value("true"), True)
self.assertIs(self.value("false"), False)
assignment = parse(tokenize("ready=true"))["statements"]["statements"][0]
self.assertEqual(assignment["target"], {"tag": "identifier", "value": "ready"})
self.assertEqual(assignment["expression"], {"tag": "boolean", "value": True})
def test_comparison_values_and_types(self):
cases = [("1==1.0", True), ("1==2", False), ("1!=2", True),
("1!=1", False), ("1<2", True), ("2<1", False),
("2<=2", True), ("3<=2", False), ("3>2", True),
("2>3", False), ("2>=2", True), ("2>=3", False),
('"dog"=="dog"', True), ('"dog"!="cat"', True),
('"dog"=="cat"', False), ("true==false", False),
("true!=false", True), ("true==1", False),
("false==0", False), ('"1"==1', False),
("-1.5<.5", True), ("1+2*3==7", True),
('"ha"*2=="haha"', True), ("(1<2)==true", True)]
for expression, expected in cases:
with self.subTest(expression=expression):
self.assertIs(self.value(expression), expected)
def test_precedence_tree(self):
node = parse(tokenize("x=1+2*3<8"))["statements"]["statements"][0]["expression"]
self.assertEqual(node["tag"], "<")
self.assertEqual(node["left"]["tag"], "+")
self.assertEqual(node["left"]["right"]["tag"], "*")
def test_print_capture(self):
output = io.StringIO()
with redirect_stdout(output):
_, env = evaluate_source("print(true); saved=__output; print(2<1)")
self.assertEqual(output.getvalue(), "true\nfalse\n")
self.assertEqual(env["saved"], "true")
self.assertEqual(env["__output"], "false")
def test_input_and_evaluation_order(self):
with patch("builtins.input", side_effect=["2", "3"]) as read:
self.assertIs(self.value("number(input()) < number(input())"), True)
self.assertEqual(read.call_count, 2)
with patch("builtins.input") as read:
_, env = evaluate_source('__input="2";result=number(input())==2')
read.assert_not_called()
self.assertIs(env["result"], True)
self.assertEqual(env["__input"], "")
def test_type_errors(self):
for expression in ('"a"<2', '1<"2"', "true<false", "true+1",
"1-true", '"ha"*true', "false/2", "-true"):
with self.subTest(expression=expression), self.assertRaises(TypeError):
self.value(expression)
def test_invalid_syntax(self):
for source in ("x=1<2<3", "x=1==2==3", "x=1<", "x=<2",
"x=true and", "x=not", "print true"):
with self.subTest(source=source), self.assertRaises(SyntaxError):
parse(tokenize(source))
if __name__ == "__main__":
unittest.main()
test_chapter_2_compatibility.py
import io
import unittest
from contextlib import redirect_stdout
from evaluator import evaluate
from parser import parse
from tokenizer import tokenize
class Chapter2CompatibilityTests(unittest.TestCase):
def test_numeric_programs(self):
cases = [
("x=2; print(x); y=x+3*4; print(y)", "2\n14\n"),
(";;;x=-1.5+.5*5.;;;print(x);;;", "1.0\n"),
("x=--3; x=x+1; print(-(x+2))", "-6\n"),
("x=20/2/2; print(x); print(10-3-2)", "5.0\n5\n"),
("// start\nx=8; // assignment\nprint(x/2)// end", "4.0\n"),
]
for source, expected in cases:
with self.subTest(source=source):
output = io.StringIO()
with redirect_stdout(output):
self.assertEqual(evaluate(parse(tokenize(source)), {}), (None, None))
self.assertEqual(output.getvalue(), expected)
def test_comment_tokens_and_positions(self):
for ending in ("\n", "\r\n", "\r"):
tokens = tokenize("8// ignored @ ; /" + ending + "/2")
self.assertEqual([t["tag"] for t in tokens], ["number", "/", "number", None])
for source in ("//", "// at end", "8 // trailing"):
tokens = tokenize(source)
if source.startswith("8"):
expected_tags = ["number", None]
else:
expected_tags = [None]
self.assertEqual([t["tag"] for t in tokens], expected_tags)
self.assertEqual(tokens[-1]["column"], len(source) + 1)
tokens = tokenize("// first\n 8// second\r\n /2")
self.assertEqual((tokens[0]["line"], tokens[0]["column"]), (2, 3))
self.assertEqual((tokens[1]["line"], tokens[1]["column"]), (3, 2))
self.assertEqual([t["tag"] for t in tokenize("/ /")], ["/", "/", None])
def test_comments_and_strings(self):
tokens = tokenize('"https://example.org" // "unterminated \\q\n"//"')
self.assertEqual([t["tag"] for t in tokens], ["string", "string", None])
self.assertEqual(tokens[0]["value"], "https://example.org")
self.assertEqual(tokens[1]["value"], "//")
def test_rejected_programs(self):
for source in ("", ";;;", "// only a comment", "x=2 print(x)",
"x=2 // ;\nprint(x)", "3=2", "print 2"):
with self.subTest(source=source), self.assertRaises(SyntaxError):
parse(tokenize(source))
def test_evaluation_errors(self):
for source, error in (("print(missing)", ValueError),
("x=3/0", ZeroDivisionError)):
with self.subTest(source=source), self.assertRaises(error):
evaluate(parse(tokenize(source)), {})
if __name__ == "__main__":
unittest.main()
test_conditionals.py
from pathlib import Path
import subprocess
import sys
import unittest
from unittest.mock import patch
import evaluator
from parser import parse, parse_expression
from tokenizer import tokenize
class ConditionalTests(unittest.TestCase):
def run_source(self, source, environment=None):
if environment is None:
environment = {}
result, environment = evaluator.evaluate_source(source, environment)
return result, environment
def test_single_branch(self):
result, env = self.run_source('if (true) { x=1 }')
self.assertEqual(result, (None, None))
self.assertEqual(env['x'], 1)
result, env = self.run_source('if (false) { x=1 }')
self.assertEqual(result, (None, None))
self.assertNotIn('x', env)
def test_two_branch(self):
for condition, expected in [('true', 1), ('false', 2)]:
with self.subTest(condition=condition):
_, env = self.run_source(f'if ({condition}) {{ x=1 }} else {{ x=2 }}')
self.assertEqual(env['x'], expected)
def test_condition_requires_boolean(self):
with self.assertRaises(TypeError):
self.run_source('if (1) { x=1 }')
def test_empty_block_is_legal(self):
# A block's statement_list is optional: "{}" is empty, not an error.
result, env = self.run_source('x=1; if (true) { }; x=2')
self.assertEqual(result, (None, None))
self.assertEqual(env['x'], 2)
result, env = self.run_source('if (false) { } else { x=1 }')
self.assertEqual(env['x'], 1)
result, env = self.run_source('if (true) { } else { x=1 }')
self.assertNotIn('x', env)
def test_semicolon_only_blocks(self):
for body in (';', ';;;', '; // comment\n ;'):
with self.subTest(body=body):
result, env = self.run_source(
f'x=1; if (true) {{ {body} }}; '
f'if (false) {{ x=99 }} else {{ {body} }}; x=2')
self.assertEqual(result, (None, None))
self.assertEqual(env['x'], 2)
result, env = self.run_source('if (true) { ;;; x=1;;; y=2;;; }')
self.assertEqual(result, (None, None))
self.assertEqual((env['x'], env['y']), (1, 2))
def test_exit_in_condition_skips_both_branches(self):
# An exit found while evaluating the condition propagates immediately:
# it is never checked against the boolean requirement, and neither
# branch runs.
result, env = self.run_source('if (exit(7)) { x=1 } else { x=2 }')
self.assertEqual(result, (7, 'exit'))
self.assertNotIn('x', env)
def test_condition_evaluated_exactly_once(self):
with patch('builtins.input', return_value='true') as fake_input:
_, env = self.run_source('if (boolean(input())) { x=1 } else { x=2 }')
fake_input.assert_called_once_with()
self.assertEqual(env['x'], 1)
def test_multi_statement_block(self):
_, env = self.run_source('if (true) { x=1; y=2; z=x+y }')
self.assertEqual((env['x'], env['y'], env['z']), (1, 2, 3))
def test_nested_conditionals(self):
source = '''
if (true) {
if (false) { x=1 } else { x=2 }
} else {
x=3
}
'''
_, env = self.run_source(source)
self.assertEqual(env['x'], 2)
def test_else_if_chain_via_nested_braces(self):
# This grammar has no "else if" shortcut: a chain is an else-block
# containing its own if_statement, written out with its own braces.
source = '''
grade = 85;
if (grade >= 90) {
letter="A"
} else {
if (grade >= 80) {
letter="B"
} else {
letter="C"
}
}
'''
_, env = self.run_source(source)
self.assertEqual(env['letter'], 'B')
def test_unselected_branch_is_never_evaluated(self):
# The branch not taken must not read input, exit, or fail an assertion.
with patch('builtins.input') as read:
result, env = self.run_source(
'if (false) { x=number(input()) } else { x=1 }')
read.assert_not_called()
self.assertEqual(env['x'], 1)
result, env = self.run_source('if (false) { exit(9) } else { x=1 }')
self.assertEqual(result, (None, None))
self.assertEqual(env['x'], 1)
result, env = self.run_source(
'if (true) { x=1 } else { assert false, "should not run" }')
self.assertEqual(result, (None, None))
self.assertEqual(env['x'], 1)
def test_status_escapes_nested_blocks_and_stops_later_statements(self):
result, env = self.run_source(
'x=1; if (true) { if (true) { exit(7) } }; x=2')
self.assertEqual(result, (7, 'exit'))
self.assertEqual(env['x'], 1)
result, env = self.run_source(
'x=1; if (true) { exit(7); x=99 }; x=2')
self.assertEqual(result, (7, 'exit'))
self.assertEqual(env['x'], 1)
def test_if_is_a_statement_not_an_expression(self):
for source in ('x = if (true) { 1 }', 'print(if (true) { 1 })'):
with self.subTest(source=source), self.assertRaises(SyntaxError):
parse(tokenize(source))
def test_unbraced_bodies_are_rejected(self):
for source in ('if (true) x=1', 'if (true) { x=1 } else x=2'):
with self.subTest(source=source):
with self.assertRaises(SyntaxError) as raised:
parse(tokenize(source))
self.assertIn("Expected '{'", str(raised.exception))
def test_braced_single_statement_and_nested_blocks_parse(self):
parse(tokenize('if (true) { x=1 }'))
parse(tokenize('if (true) { x=1 } else { x=2 }'))
parse(tokenize('if (true) { if (false) { x=1 } else { x=2 } }'))
parse(tokenize('if (true) { }'))
parse(tokenize('if (true) { } else { }'))
def test_runner_handles_if(self):
runner = Path(__file__).with_name('runner.py')
cases = [
('if (true) { print(1) } else { print(2) }', 0, '1\n'),
('if (false) { print(1) } else { print(2) }', 0, '2\n'),
('if (true) { exit(3) }; print(9)', 3, ''),
]
for source, code, output in cases:
result = subprocess.run([sys.executable, str(runner), source],
capture_output=True, text=True,
stdin=subprocess.DEVNULL)
self.assertEqual(result.returncode, code)
self.assertEqual(result.stdout, output)
self.assertEqual(result.stderr, '')
if __name__ == '__main__':
unittest.main()
test_conversions_and_type.py
import unittest
from unittest.mock import patch
from evaluator import evaluate_source
from parser import parse
from tokenizer import tokenize
class ConversionAndTypeTests(unittest.TestCase):
def value(self, expression):
return evaluate_source("result=" + expression)[1]["result"]
def test_conversion_matrix(self):
cases = [('number(2)', 2), ('number(2.5)', 2.5),
('number(" .5 ")', .5), ('number(true)', 1), ('number(false)', 0),
('string(2)', '2'), ('string(2.5)', '2.5'), ('string("abc")', 'abc'),
('string(true)', 'true'), ('string(false)', 'false'),
('boolean(true)', True), ('boolean(false)', False),
('boolean(0)', False), ('boolean(-0.0)', False),
('boolean(-2)', True), ('boolean(.5)', True),
('boolean(" TrUe ")', True), ('boolean(" FALSE ")', False)]
for source, expected in cases:
with self.subTest(source=source):
result = self.value(source)
self.assertEqual(result, expected)
self.assertIs(type(result), type(expected))
def test_invalid_text(self):
for source in ('boolean("")', 'boolean("yes")', 'boolean("0")',
'boolean("1")', 'number("true")', 'number("2+3")'):
with self.subTest(source=source), self.assertRaises(ValueError):
self.value(source)
def test_type_names(self):
for expression, expected in [('42', 'number'), ('2.5', 'number'),
('"42"', 'string'), ('true', 'boolean'),
('1<2', 'boolean'), ('type(false)', 'string')]:
with self.subTest(expression=expression):
self.assertEqual(self.value(f'type({expression})'), expected)
def test_argument_evaluated_once(self):
for name, supplied, expected in [('type', 'false', 'string'),
('boolean', 'false', False),
('string', 'text', 'text'),
('number', '42', 42)]:
with self.subTest(name=name), patch('builtins.input', return_value=supplied) as read:
self.assertEqual(self.value(f'{name}(input())'), expected)
read.assert_called_once_with()
_, env = evaluate_source('__input="true"; result=type(input())')
self.assertEqual(env['result'], 'string')
self.assertEqual(env['__input'], '')
with self.assertRaises(ValueError):
self.value('type(missing)')
def test_keyword_boundaries_and_syntax(self):
self.assertEqual([t['tag'] for t in tokenize('boolean type boolean_value typewriter')],
['boolean_conversion', 'type_query', 'identifier', 'identifier', None])
for name in ('number', 'string', 'boolean', 'type'):
for source in (f'x={name}()', f'x={name}(1,2)', f'x={name}(1', f'{name}=1'):
with self.subTest(source=source), self.assertRaises(SyntaxError):
parse(tokenize(source))
def test_explicit_conversion_does_not_change_implicit_rules(self):
self.assertEqual(self.value('number(true)+2'), 3)
self.assertIs(self.value('boolean(1) and true'), True)
self.assertIs(self.value('true==1'), False)
for source in ('true+2', '1 and true'):
with self.subTest(source=source), self.assertRaises(TypeError):
self.value(source)
if __name__ == '__main__':
unittest.main()
test_evaluation_status.py
import contextlib
import io
from pathlib import Path
import subprocess
import sys
import unittest
from unittest.mock import patch
import evaluator
from parser import parse, parse_expression
from tokenizer import tokenize
class EvaluationStatusTests(unittest.TestCase):
def expression(self, source, environment=None):
node, rest = parse_expression(tokenize(source))
self.assertIsNone(rest[0]["tag"])
if environment is None:
environment = {}
return evaluator.evaluate(node, environment)
def test_normal_values_and_statuses(self):
for source, value in [('42', 42), ('true', True), ('"hello"', 'hello'),
('number("2")+3', 5), ('type(false)', 'boolean'),
('false or true', True), ('1<2', True)]:
with self.subTest(source=source):
self.assertEqual(self.expression(source), (value, None))
self.assertEqual(evaluator.evaluate_source('x=2')[0], (None, None))
def test_exit_defaults_and_nested_exit(self):
self.assertEqual(self.expression('exit()'), (0, 'exit'))
self.assertEqual(self.expression('exit(7)'), (7, 'exit'))
self.assertEqual(self.expression('exit(exit(9))'), (9, 'exit'))
for source in ('exit(true)', 'exit(2.5)', 'exit("7")'):
with self.subTest(source=source), self.assertRaises(TypeError):
self.expression(source)
def test_unary_and_conversion_propagation(self):
for source in ('-exit(7)', 'not exit(7)', 'number(exit(7))',
'string(exit(7))', 'boolean(exit(7))', 'type(exit(7))'):
with self.subTest(source=source):
self.assertEqual(self.expression(source), (7, 'exit'))
def test_binary_propagation_before_later_work(self):
for operator in ('+', '-', '*', '/', '==', '!=', '<', '<=', '>', '>=',
'and', 'or'):
with self.subTest(operator=operator), patch('builtins.input') as read:
self.assertEqual(self.expression(f'exit(7) {operator} input()'),
(7, 'exit'))
read.assert_not_called()
# Needed right operands propagate before applying the operation or
# checking the exit value against the operation's operand types.
for source in ('1+exit(7)', '1-exit(7)', '1*exit(7)', '1/exit(7)',
'1==exit(7)', '1!=exit(7)', '1<exit(7)', '1<=exit(7)',
'1>exit(7)', '1>=exit(7)', 'true and exit(7)',
'false or exit(7)'):
with self.subTest(source=source):
self.assertEqual(self.expression(source), (7, 'exit'))
def test_short_circuit_still_skips_exit(self):
self.assertEqual(self.expression('false && exit(7)'), (False, None))
self.assertEqual(self.expression('true || exit(7)'), (True, None))
def test_input_prompt_exits_before_reading(self):
env = {'__input': 'saved'}
with patch('builtins.input') as read:
self.assertEqual(self.expression('input(exit(7))', env), (7, 'exit'))
read.assert_not_called()
self.assertEqual(env['__input'], 'saved')
def test_assignment_print_and_statement_list_stop(self):
env = {'x': 3, '__output': 'previous', '__input': 'saved'}
output = io.StringIO()
with contextlib.redirect_stdout(output), patch('builtins.input') as read:
result, _ = evaluator.evaluate_source(
'x=exit(7);print(input());later=2', env)
self.assertEqual(result, (7, 'exit'))
self.assertEqual(env, {'x': 3, '__output': 'previous', '__input': 'saved'})
self.assertEqual(output.getvalue(), '')
read.assert_not_called()
with contextlib.redirect_stdout(output):
result, _ = evaluator.evaluate_source('print(exit(8));later=2', env)
self.assertEqual(result, (8, 'exit'))
self.assertEqual(output.getvalue(), '')
self.assertEqual(env['__output'], 'previous')
self.assertNotIn('later', env)
def test_propagation_does_not_undo_earlier_effects(self):
env = {'__input': 'first'}
self.assertEqual(self.expression('input()+exit(7)', env), (7, 'exit'))
self.assertEqual(env['__input'], '')
def test_generic_status_is_not_limited_to_exit(self):
# A synthetic child stands in for future break/continue/return nodes.
# Ordinary parents must carry unfamiliar statuses without consuming them.
original = evaluator.evaluate
for status in ('break', 'continue', 'return'):
for source in ('x=marker+missing', 'x=type(marker)', 'print(marker)'):
def child(node, env):
if node.get('tag') == 'identifier' and node.get('value') == 'marker':
return 17, status
return original(node, env)
with self.subTest(status=status, source=source):
env = {}
with patch.object(evaluator, 'evaluate', side_effect=child):
self.assertEqual(original(parse(tokenize(source)), env),
(17, status))
self.assertNotIn('x', env)
self.assertEqual(env['__output'], '')
def test_exit_keyword_and_syntax(self):
self.assertEqual([t['tag'] for t in tokenize('exit exiting exit_code')],
['exit', 'identifier', 'identifier', None])
for source in ('exit=1', 'exit', 'exit(1,2)', 'exit(', 'exit(1);x=1+'):
with self.subTest(source=source), self.assertRaises(SyntaxError):
parse(tokenize(source))
def test_runner_handles_exit(self):
runner = Path(__file__).with_name('runner.py')
for source, code, output in [('print(1);exit(7);print(2)', 7, '1\n'),
('exit();print(2)', 0, ''),
('print(3)', 0, '3\n')]:
result = subprocess.run([sys.executable, str(runner), source],
capture_output=True, text=True,
stdin=subprocess.DEVNULL)
self.assertEqual(result.returncode, code)
self.assertEqual(result.stdout, output)
self.assertEqual(result.stderr, '')
if __name__ == '__main__':
unittest.main()
test_io_and_number.py
import io
import unittest
from contextlib import redirect_stdout
from unittest.mock import patch
from evaluator import evaluate_source
from parser import parse
from tokenizer import tokenize
class InputOutputAndNumberTests(unittest.TestCase):
def test_globals_exist_before_first_use(self):
_, env = evaluate_source('saved_input=__input;saved_output=__output')
self.assertEqual(env, {
'__input': '', '__output': '', 'saved_input': '', 'saved_output': '',
})
def test_print_captures_latest_text(self):
output = io.StringIO()
with redirect_stdout(output):
_, env = evaluate_source('print(42);first=__output;print("a\\nb")')
self.assertEqual(output.getvalue(), '42\na\nb\n')
self.assertEqual(env['first'], '42')
self.assertEqual(env['__output'], 'a\nb')
def test_supplied_input_is_consumed_once(self):
output = io.StringIO()
with patch('builtins.input', return_value='keyboard') as read:
with redirect_stdout(output):
_, env = evaluate_source(
'__input="Ada";a=input("Name? ");cleared=__input;'
'b=input("Next? ");__input="Grace";c=input()'
)
self.assertEqual([env[key] for key in ('a', 'b', 'c')], ['Ada', 'keyboard', 'Grace'])
self.assertEqual(env['cleared'], '')
self.assertEqual(env['__input'], '')
read.assert_called_once_with('Next? ')
self.assertEqual(output.getvalue(), '')
def test_external_input_and_global_assignment(self):
root = {'__input': '21', '__output': 'previous'}
local = {'$PARENT': root}
with patch('builtins.input') as read, redirect_stdout(io.StringIO()):
_, env = evaluate_source(
'previous=__output;print(number(input())*2);__input="next"', local
)
read.assert_not_called()
self.assertEqual(env['previous'], 'previous')
self.assertEqual(root['__output'], '42')
self.assertEqual(root['__input'], 'next')
self.assertNotIn('__output', local)
self.assertNotIn('__input', local)
def test_prompt_and_supplied_input_types(self):
for source in ('__input="ready";x=input(42)', '__input=42;x=input()'):
with self.subTest(source=source), patch('builtins.input') as read:
with self.assertRaises(TypeError):
evaluate_source(source)
read.assert_not_called()
def test_number_values_and_types(self):
for text, expected in [('42', 42), ('-3', -3), ('+3', 3),
(' 2.5 ', 2.5), ('.5', .5), ('5.', 5.0)]:
with self.subTest(text=text):
_, env = evaluate_source('value=number(text)', {'text': text})
self.assertEqual(env['value'], expected)
self.assertIs(type(env['value']), type(expected))
_, env = evaluate_source('value="ha"*number("3");numbered=number("1"+"2")')
self.assertEqual(env['value'], 'hahaha')
self.assertEqual(env['numbered'], 12)
def test_invalid_number_arguments(self):
for text in ('', ' ', 'abc', '1.2.3', '2+3', '1e3', 'nan', 'inf'):
with self.subTest(text=text), self.assertRaises(ValueError):
evaluate_source('x=number(text)', {'text': text})
self.assertEqual(evaluate_source('x=number(42)')[1]['x'], 42)
def test_number_syntax(self):
for source in ('x=number', 'x=number()', 'x=number("1"', 'x=number("1","2")'):
with self.subTest(source=source), self.assertRaises(SyntaxError):
parse(tokenize(source))
tokens = tokenize('number numbered number_value')
self.assertEqual([t['tag'] for t in tokens],
['number_conversion', 'identifier', 'identifier', None])
def test_string_conversion(self):
for expression, expected in [('42', '42'), ('-3', '-3'), ('2.5', '2.5'),
('5.', '5.0'), ('1+2*3', '7')]:
with self.subTest(expression=expression):
_, env = evaluate_source(f'value=string({expression})')
self.assertEqual(env['value'], expected)
_, env = evaluate_source('value="Answer: "+string(number("21")*2)')
self.assertEqual(env['value'], 'Answer: 42')
for source in ('x=string("42")', 'x=string(input())'):
with self.subTest(source=source):
self.assertEqual(evaluate_source(source, {'__input': '42'})[1]['x'], '42')
def test_string_syntax(self):
for source in ('x=string', 'x=string()', 'x=string(1', 'x=string(1,2)'):
with self.subTest(source=source), self.assertRaises(SyntaxError):
parse(tokenize(source))
tokens = tokenize('string stringed string_value')
self.assertEqual([t['tag'] for t in tokens],
['string_conversion', 'identifier', 'identifier', None])
if __name__ == '__main__':
unittest.main()
test_logical_operators.py
import unittest
from unittest.mock import patch
from evaluator import evaluate_source
from parser import parse
from tokenizer import tokenize
class LogicalOperatorTests(unittest.TestCase):
def value(self, source):
return evaluate_source("result=" + source)[1]["result"]
def test_normalized_tokens(self):
tags = [t["tag"] for t in tokenize("and && or || not ! != android origin notable")]
self.assertEqual(tags, ["and", "and", "or", "or", "not", "not", "!=",
"identifier", "identifier", "identifier", None])
for source in ("and=1", "or=1", "not=1"):
with self.subTest(source=source), self.assertRaises(SyntaxError):
parse(tokenize(source))
def test_same_tree_for_both_spellings(self):
words = parse(tokenize("x=not false and true or false"))
symbols = parse(tokenize("x=!false && true || false"))
self.assertEqual(words, symbols)
node = words["statements"]["statements"][0]["expression"]
self.assertEqual(node["tag"], "or")
self.assertEqual(node["left"]["tag"], "and")
self.assertEqual(node["left"]["left"]["tag"], "not")
def test_truth_tables(self):
for left in (False, True):
for right in (False, True):
for operator in ("and", "&&", "or", "||"):
if operator in ("and", "&&"):
expected = left and right
else:
expected = left or right
source = f"{str(left).lower()} {operator} {str(right).lower()}"
with self.subTest(source=source):
self.assertIs(self.value(source), expected)
for operator in ("not ", "!"):
self.assertIs(self.value(operator + str(left).lower()), not left)
def test_precedence_and_repeated_negation(self):
for source, expected in [("true or false and false", True),
("(true or false) and false", False),
("not 1+2*3==7", False),
("!1!=2", False), ("!!true", True),
("not !false", False),
("1<2 && 3>=3 or false", True)]:
with self.subTest(source=source):
self.assertIs(self.value(source), expected)
def test_short_circuit_skips_errors_and_input(self):
# A skipped child is never evaluated, including its runtime type checks.
for source, expected in [("false and missing", False),
("true or 1/0==0", True),
("false && input()", False),
("true || input()", True)]:
with self.subTest(source=source), patch("builtins.input") as read:
self.assertIs(self.value(source), expected)
read.assert_not_called()
_, env = evaluate_source('__input="ready";x=false and input()=="ready"')
self.assertEqual(env["__input"], "ready")
def test_needed_right_operand_runs_once(self):
for source in ('true and input()=="yes"', 'false || input()=="yes"'):
with self.subTest(source=source), patch("builtins.input", return_value="yes") as read:
self.assertIs(self.value(source), True)
read.assert_called_once_with()
def test_boolean_operands_required(self):
for source in ("not 1", '!"text"', "1 and true", "0 or false",
"true and 2", 'false || "text"'):
with self.subTest(source=source), self.assertRaises(TypeError):
self.value(source)
def test_incomplete_operators(self):
for source in ("x=true &&", "x=false or", "x=!", "x=and true",
"x=true & false", "x=true | false"):
with self.subTest(source=source), self.assertRaises(SyntaxError):
parse(tokenize(source))
if __name__ == "__main__":
unittest.main()
test_string_ordering.py
import operator
import unittest
from unittest.mock import patch
from evaluator import evaluate_source
class StringOrderingTests(unittest.TestCase):
def value(self, expression):
return evaluate_source("result=" + expression)[1]["result"]
def test_all_operators_and_boundaries(self):
pairs = [("ant", "bee"), ("bee", "ant"), ("ant", "ant"),
("", ""), ("", "a"), ("a", ""), ("a", "ant"),
("ant", "a"), ("Z", "a"), ("10", "2"),
("z", "\u00e9")]
for symbol, compare in [("<", operator.lt), ("<=", operator.le),
(">", operator.gt), (">=", operator.ge)]:
for left, right in pairs:
expression = f'"{left}" {symbol} "{right}"'
with self.subTest(expression=expression):
self.assertIs(self.value(expression), compare(left, right))
def test_incompatible_types_remain_errors(self):
for symbol in ("<", "<=", ">", ">="):
for left, right in [('"1"', "1"), ("1", '"1"'),
("true", '"a"'), ('"a"', "false"),
("false", "true")]:
with self.subTest(symbol=symbol, left=left, right=right):
with self.assertRaises(TypeError):
self.value(f"{left} {symbol} {right}")
def test_exit_propagates_before_comparison_or_storage(self):
with patch("builtins.input") as read:
result, env = evaluate_source('result="old";result=exit(7)<input()')
read.assert_not_called()
self.assertEqual(result, (7, "exit"))
self.assertEqual(env["result"], "old")
result, env = evaluate_source('__input="ant";result="old";result=input()<exit(7)')
self.assertEqual(result, (7, "exit"))
self.assertEqual(env["result"], "old")
self.assertEqual(env["__input"], "")
def test_operands_evaluated_once_left_to_right(self):
with patch("builtins.input", side_effect=["ant", "bee"]) as read:
self.assertIs(self.value("input() < input()"), True)
self.assertEqual(read.call_count, 2)
if __name__ == "__main__":
unittest.main()
tokenizer.py
import re
# At each source position, the first matching pattern wins.
# Longer operators and reserved words therefore precede their shorter matches.
patterns = [
(r"\s+", "whitespace"),
(r"//[^\r\n]*", "comment"), # Before division; leave the line ending for whitespace.
# ===== CHAPTER 3: string literals =====
# Strings use double quotes and stay on one source line.
(r'"(?:\\[^\r\n]|[^"\\\r\n])*"', "string"),
(r"\d*\.\d+|\d+\.\d*|\d+", "number"),
(r"==", "=="),
# Inequality must win over the single-character spelling of "not".
(r"!=", "!="),
# Normalize both spellings here, so the parser only needs word-form tags.
# Word boundaries keep names such as "android" and "notable" intact.
(r"and\b|&&", "and"),
(r"or\b|\|\|", "or"),
(r"not\b|!", "not"),
(r"<=", "<="),
(r">=", ">="),
(r"<", "<"),
(r">", ">"),
(r"\+", "+"),
(r"\-", "-"),
(r"\/", "/"),
(r"\*", "*"),
(r"\(", "("),
(r"\)", ")"),
# ===== CHAPTER 7: statement blocks =====
(r"\{", "{"),
(r"\}", "}"),
(r"\=", "="),
(r"\;", ";"),
(r",", ","),
(r"assert\b", "assert"),
(r"print\b", "print"),
(r"exit\b", "exit"),
(r"true\b", "true"),
(r"false\b", "false"),
# ===== CHAPTER 7: conditional keywords =====
(r"if\b", "if"),
(r"else\b", "else"),
# ===== CHAPTER 3: input expression =====
# input is a dedicated expression form for now, not a general function.
(r"input\b", "input"),
(r"number\b", "number_conversion"),
(r"string\b", "string_conversion"),
(r"boolean\b", "boolean_conversion"),
(r"type\b", "type_query"),
(r"[a-zA-Z_][\w]*", "identifier"),
(r".", "error"),
]
patterns = [(re.compile(pattern), tag) for pattern, tag in patterns]
def decode_string(text, line, column):
# The tokenizer has already matched the surrounding quotes. Decode only
# Vertex's supported escapes, not the full Python string-literal language.
escapes = {"n": "\n", "t": "\t", "r": "\r", '"': '"', "\\": "\\"}
result = ""
position = 1
while position < len(text) - 1:
character = text[position]
if character == "\\":
position += 1
character = text[position]
if character not in escapes:
raise SyntaxError(
f"Unknown escape \\{character} at line {line}, "
f"column {column + position - 1}"
)
character = escapes[character]
# Append the decoded character once; do not interpret it again.
result += character
position += 1
return result
def tokenize(characters):
"Tokenize a string using the patterns above"
tokens = []
position = 0
line = 1
column = 1
while position < len(characters):
match = None
current_tag = None
for pattern, tag in patterns:
# Match at this position, never search past an unrecognized character.
match = pattern.match(characters, position)
if match:
current_tag = tag
break
assert match is not None
value = match.group(0)
if current_tag == "error":
raise SyntaxError(f"Unexpected character: {value!r}")
# Comments and whitespace advance through the source but produce no token.
# A string is matched as a whole, so "//" inside it is not a comment.
if current_tag not in ("whitespace", "comment"):
token = {"tag": current_tag, "line": line, "column": column}
if current_tag == "number":
if "." in value:
token["value"] = float(value)
else:
token["value"] = int(value)
elif current_tag == "string":
token["value"] = decode_string(value, line, column)
elif current_tag == "identifier":
token["value"] = value
tokens.append(token)
# Count source characters, not decoded string characters. An escaped
# newline in a string does not move the following token to another line.
for character in value:
if character == "\n":
line += 1
column = 1
else:
column += 1
position = match.end()
# An explicit end marker lets parser helpers inspect the next token safely.
tokens.append({"tag": None, "line": line, "column": column})
return tokens
def test_digits():
print("test tokenize digits")
tokens = tokenize("123")
assert tokens[0]["tag"] == "number"
assert tokens[0]["value"] == 123
assert tokens[1]["tag"] is None
def test_floats():
print("test tokenize floats")
for text, expected in [("1.5", 1.5), (".5", 0.5), ("5.", 5.0)]:
tokens = tokenize(text)
assert tokens[0]["tag"] == "number"
assert tokens[0]["value"] == expected
assert tokens[1]["tag"] is None
def test_strings():
# ===== CHAPTER 3 TESTS =====
print("test tokenize strings")
tokens = tokenize(r'"hello" "line\nfeed" "say \"hello\""')
assert [token["tag"] for token in tokens] == [
"string",
"string",
"string",
None,
]
assert tokens[0]["value"] == "hello"
assert tokens[1]["value"] == "line\nfeed"
assert tokens[2]["value"] == 'say "hello"'
def test_string_escapes():
print("test string escapes")
for source, expected in [
('""', ""),
(r'"\n\t\r\"\\"', '\n\t\r"\\'),
(r'"\\n"', "\\n"),
(r'"ends\\"', "ends\\"),
('"plain text"', "plain text"),
]:
assert tokenize(source)[0]["value"] == expected
for escape in [r"\q", r"\x41", r"\u0041", r"\101", r"\b", r"\'"]:
try:
tokenize('\n "' + escape + '"')
except SyntaxError as error:
assert "Unknown escape" in str(error)
assert "line 2, column 4" in str(error)
else:
raise AssertionError(f"Accepted unsupported escape: {escape}")
for source in ['"a\nb"', '"a\rb"', '"a\r\nb"', '"a\\\nb"', '"a\\\rb"', '"ends\\"']:
try:
tokenize(source)
except SyntaxError:
pass
else:
raise AssertionError(f"Accepted malformed string: {source!r}")
tokens = tokenize(r'"\n" next')
assert tokens[1]["line"] == 1
assert tokens[1]["column"] == 6
def test_operators():
print("test tokenize operators")
tokens = tokenize("+ - * / ( ) = ;")
tags = [token["tag"] for token in tokens]
assert tags == ["+", "-", "*", "/", "(", ")", "=", ";", None]
def test_keywords():
print("test tokenize keywords")
tokens = tokenize("print input printer input_value")
tags = [token["tag"] for token in tokens]
assert tags == ["print", "input", "identifier", "identifier", None]
assert tokens[2]["value"] == "printer"
assert tokens[3]["value"] == "input_value"
def test_blocks_and_conditionals():
# ===== CHAPTER 7 TEST =====
print("test tokenize blocks and conditionals")
tokens = tokenize("if (x) { y } else { z }")
tags = [token["tag"] for token in tokens]
assert tags == ["if", "(", "identifier", ")", "{", "identifier", "}",
"else", "{", "identifier", "}", None]
tokens = tokenize("ifx elsewhere")
tags = [token["tag"] for token in tokens]
assert tags == ["identifier", "identifier", None]
assert tokens[0]["value"] == "ifx"
assert tokens[1]["value"] == "elsewhere"
def test_identifiers():
print("test tokenize identifiers")
tokens = tokenize("foo bar baz")
tags = [token["tag"] for token in tokens]
assert tags == ["identifier", "identifier", "identifier", None]
assert tokens[0]["value"] == "foo"
assert tokens[2]["value"] == "baz"
def test_expressions():
print("test tokenize expressions")
tokens = tokenize('"dog"*2+"!"')
assert [token["tag"] for token in tokens] == [
"string",
"*",
"number",
"+",
"string",
None,
]
def test_whitespace():
print("test tokenize whitespace")
tokens = tokenize("1 +\t2 \n* 3")
assert [token["tag"] for token in tokens] == [
"number",
"+",
"number",
"*",
"number",
None,
]
def test_error():
print("test tokenize error")
try:
tokenize("1@@@")
except SyntaxError as error:
assert str(error) == "Unexpected character: '@'"
else:
raise Exception("Expected SyntaxError")
def test_unterminated_string():
# ===== CHAPTER 3 TEST =====
print("test unterminated string")
try:
tokenize('"never closed')
except SyntaxError as error:
assert str(error) == "Unexpected character: '\"'"
else:
raise Exception("Expected SyntaxError")
if __name__ == "__main__":
test_digits()
test_floats()
test_strings()
test_string_escapes()
test_operators()
test_keywords()
test_blocks_and_conditionals()
test_expressions()
test_identifiers()
test_whitespace()
test_error()
test_unterminated_string()
print("done.")
vertex
#!/usr/bin/env bash
exec python3 "$(dirname "$0")/runner.py" "$@"
The files are available in the course repository.