diff --git a/.github/workflows/dexpr-testing.yml b/.github/workflows/dexpr-testing.yml new file mode 100644 index 000000000000..1a3a1ed6e064 --- /dev/null +++ b/.github/workflows/dexpr-testing.yml @@ -0,0 +1,79 @@ +name: dexpr + +# dexpr is plain C++ with no Kokkos, MPI, netCDF or EKAT dependency, so unlike +# the rest of E3SM it builds and tests on a stock GitHub runner in about a +# minute. That is why this workflow does not need the self-hosted ghci-snl +# machines the eamxx workflows run on, and checks out no submodules. + +on: + # Runs on PRs against master, but only if they touch the parser + pull_request: + branches: [ master ] + types: [opened, synchronize, ready_for_review, reopened] + paths: + - 'share/dexpr/**' + - '.github/workflows/dexpr-testing.yml' + + # Also guard master itself, so a bad merge is caught immediately + push: + branches: [ master ] + paths: + - 'share/dexpr/**' + - '.github/workflows/dexpr-testing.yml' + + # Manual run for debug purposes only + workflow_dispatch: + +concurrency: + # Two runs are in the same group if they are testing the same git ref + # - if trigger=pull_request, the ref is refs/pull//merge + # - for other triggers, the ref is the branch tested + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: ${{ matrix.compiler }} / ${{ matrix.build_type }} + runs-on: ubuntu-latest + + strategy: + # One compiler failing should not hide what the others would have said + fail-fast: false + matrix: + compiler: [gcc, clang] + build_type: [Debug, Release] + + steps: + - name: Check out the repository + uses: actions/checkout@v7 + with: + persist-credentials: false + show-progress: false + + - name: Show action trigger + uses: ./.github/actions/show-workflow-trigger + + - name: Select the compiler + run: | + if [ "${{ matrix.compiler }}" = "gcc" ]; then + echo "CC=gcc" >> "$GITHUB_ENV" + echo "CXX=g++" >> "$GITHUB_ENV" + else + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" + fi + + - name: Configure + run: | + cmake -S share/dexpr -B build \ + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ + -DDEXPR_WERROR=ON + + - name: Build + run: cmake --build build --parallel + + - name: Test + run: ctest --test-dir build --output-on-failure --parallel + + - name: Smoke test the tool + run: ./build/dexpr functions diff --git a/share/README b/share/README index 71ef125c9408..73bb5087d194 100644 --- a/share/README +++ b/share/README @@ -30,6 +30,10 @@ streams - code for managing "streams" of data files. test - unit tests for some of the share code +dexpr - lexer and parser for the diagnostics expression language; a + standalone C++20 library with no dependencies, built and tested on + its own rather than as part of csm_share. See dexpr/README.md. + This code was originaly part of CIME in CIME/src/share. Brought in to E3SM from CIME hash b95a28b417b9b27 from May 4, 2021 diff --git a/share/dexpr/.gitignore b/share/dexpr/.gitignore new file mode 100644 index 000000000000..8c0aadfed7fd --- /dev/null +++ b/share/dexpr/.gitignore @@ -0,0 +1,2 @@ +**/build/** +**/.cache/** diff --git a/share/dexpr/CMakeLists.txt b/share/dexpr/CMakeLists.txt new file mode 100644 index 000000000000..cbdd34afd2fa --- /dev/null +++ b/share/dexpr/CMakeLists.txt @@ -0,0 +1,57 @@ +cmake_minimum_required(VERSION 3.20) + +project(dexpr + VERSION 0.1.0 + DESCRIPTION "Lexer and parser for the diagnostics expression language" + LANGUAGES CXX) + +# This library is deliberately standalone: it has no Kokkos, MPI, netCDF or +# EKAT dependency, and is not wired into the CIME/csm_share build. That is +# what lets it be configured and tested anywhere cmake and a C++20 compiler +# exist, including a stock GitHub runner. +option(DEXPR_ENABLE_TESTS "Build the unit tests" ON) +option(DEXPR_ENABLE_TOOL "Build the command line tool" ON) + +add_library(dexpr + src/ast_print.cpp + src/lexer.cpp + src/parser.cpp + src/precedences.cpp + src/tokens.cpp +) + +target_compile_features(dexpr PUBLIC cxx_std_20) + +target_include_directories(dexpr + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) + +option(DEXPR_WERROR "Treat compiler warnings as errors" OFF) + +# Warnings stay on: with no external headers in the include path there is +# nothing here that anyone has to fight, so the library should stay clean. +# CI turns DEXPR_WERROR on to keep it that way. The flags are set here rather +# than per target so tests/ inherits exactly the same set. +set(DEXPR_WARNINGS) +if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang") + set(DEXPR_WARNINGS -Wall -Wextra -Wpedantic) + if (DEXPR_WERROR) + list(APPEND DEXPR_WARNINGS -Werror) + endif() +endif() + +target_compile_options(dexpr PRIVATE ${DEXPR_WARNINGS}) + +if (DEXPR_ENABLE_TOOL) + # The target is dexpr_cli because the library already owns the name dexpr; + # the binary it produces is still just `dexpr`. + add_executable(dexpr_cli tools/dexpr.cpp) + set_target_properties(dexpr_cli PROPERTIES OUTPUT_NAME dexpr) + target_link_libraries(dexpr_cli PRIVATE dexpr) +endif() + +if (DEXPR_ENABLE_TESTS) + include(CTest) + if (BUILD_TESTING) + add_subdirectory(tests) + endif() +endif() diff --git a/share/dexpr/LICENSE b/share/dexpr/LICENSE new file mode 100644 index 000000000000..bc0c4da12972 --- /dev/null +++ b/share/dexpr/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Peter Schwartz + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/share/dexpr/README.md b/share/dexpr/README.md new file mode 100644 index 000000000000..a085a0512237 --- /dev/null +++ b/share/dexpr/README.md @@ -0,0 +1,141 @@ +# dexpr + +A lexer and parser for the diagnostics expression language: the small DSL used +to describe derived diagnostics as expressions over model fields, rather than +as hand-written code per diagnostic. + +Given a string like + +```text +x*y.derivative(dx=dy,['col']).where(x>0) +``` + +which will be grouped as + +```text +(x*y.derivative((dx=dy), ['col']).where((x>0))) +``` + +`dexpr` produces an abstract syntax tree. It does not evaluate anything, and it +knows nothing about fields, grids or timesteps -- turning an AST into an actual +diagnostic is the caller's job. the Parser::parse function returns a pointer to the +root node of the AST. + +Example Usage: + +```c++ +parser::Parser parser{Lexer{string_input}}; +const auto expr = parser.parse(); // expr is a std::unique_ptr +auto string_representation = ast::to_string(*expr); //to_string implmented as a Vistor +``` + +## Layout + +| Path | Contents | +| --- | --- | +| `include/dexpr/` | Public headers | +| `src/` | Lexer, parser, AST printing, token and precedence tables | +| `tests/` | Catch2 unit tests | +| `tools/` | `dexpr`, a small command line helper | + +The grammar is handled by a hand-written lexer feeding a Pratt (top-down operator precedence) parser. +AST nodes are represented by a `std::variant`, visited through +`Expression::visit`, which serves as a generic wrapper around `std::visit`. +This design means the nodes should be stable and transformations acting on the AST can be easily added. + +The basic set of callable functions lives in one place, `supported_functions.hpp`. +Nothing else in the library is diagnostics-specific. + +### Parser terminology + +The expression parser uses a hand-written +[Pratt parser](https://tdop.github.io/), +also known as *top-down operator-precedence parsing*. Pratt parsing associates +parsing behavior with tokens and uses operator precedence to determine how an +expression is grouped. + +Some terminology used throughout the implementation: + +- **Prefix expression** — an expression beginning with an operator + and consumes an expression to its right, e.g. `-x` or `!x`. + Literals and identifers are considered Prefix expressions for parser-function dispatch + but are stored in the AST as a more specific node. + +- **Infix expression** — an expression that contains operator located **in**-between a left- and right-hand + expression, e.g. `x + y` or `x < y`. + +- **Precedence / binding power** — determines how tightly an operator binds + relative to surrounding operators. For example, multiplication has higher + precedence than addition, so `x + y * z` is parsed as `x + (y * z)`. + Exponentiation is right-associative: `x ** y ** z` is parsed as + `x ** (y ** z)`. This is represented by conditionally modifying its Precedence to + reflect right-binding power. + +- **Prefix parse function** — parses an expression beginning with a token that can begin an expression, + such as an identifier, literal, unary operator, or opening parenthesis. + +- **Infix parse function** — extends an expression based on the `Precedence` of the following token. + It receives the expression to its left and parses the required expression(s) to its right. + Example tokens include: aritmetic operators, logical operators, opening parenthesis, etc... + +## Building + +`dexpr` is deliberately standalone. It has no Kokkos, MPI, netCDF or EKAT +dependency, it is not part of the CIME or `csm_share` build, and it requires +only CMake 3.20 and a C++20 compiler: + +```shell +./run_tests.sh +``` + +which is shorthand for + +```shell +cmake -S . -B build -DDEXPR_ENABLE_TESTS=ON -DDEXPR_ENABLE_TOOL=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON +cmake --build build --parallel +ctest --test-dir build --output-on-failure --parallel +``` + +Catch2 v3 is used for the tests. An installed copy is used if there is one; +otherwise CMake fetches a pinned version. + +### Compiler requirement + +C++20, and in practice GCC 11 or newer: numeric literals are parsed and +printed with floating-point ``, which is the binding constraint. + +### Options + +| Option | Default | Effect | +| --- | --- | --- | +| `DEXPR_ENABLE_TESTS` | `ON` | Build the unit tests | +| `DEXPR_ENABLE_TOOL` | `ON` | Build the `dexpr` command line tool | +| `DEXPR_WERROR` | `OFF` | Treat compiler warnings as errors; CI sets this | + +## Testing + +Every `TEST_CASE` is registered with ctest individually, so a failure names +itself. The same commands run in CI, across gcc and clang in both Debug and +Release, on every pull request touching this directory. + +## Planned work + +Not implemented here, recorded so it is not rediscovered: + +- **Component-supplied functions.** The callable set is fixed in + `supported_functions.hpp` and nothing consults it, so `nope(x)` parses and is + never rejected. The plan is a registry a component fills in at init -- each + function's name, its parameters in positional order, and whether the call is + written free (`where(...)`) or as a method (`T_mid.interp(...)`) -- plus a + pass over the AST that checks calls against it. That pass stays out of the + parser on purpose, so `foo(a, b=c)` parses the same whether or not `foo` + exists. +- **Operator syntax is fixed.** A registry would let a component add functions + but not operators. A new operator means editing the token enum, the lexer, + the precedence table and the parser's dispatch tables. + +## Provenance + +Vendored from [peterdschwartz/e3sm_diags_parser](https://github.com/peterdschwartz/e3sm_diags_parser) +at `d680d50`, MIT licensed; see `LICENSE`. The upstream name, and its `edp` +abbreviation, were dropped in favour of `dexpr` once the code moved here. diff --git a/share/dexpr/include/dexpr/ast.hpp b/share/dexpr/include/dexpr/ast.hpp new file mode 100644 index 000000000000..224469c88778 --- /dev/null +++ b/share/dexpr/include/dexpr/ast.hpp @@ -0,0 +1,104 @@ +/** + * @file ast.hpp + * @brief Defines the AST nodes generated by the parser. + * + * AST node types are represented by the ExpressionVariant std::variant. + * Expression provides a visit() method that wraps std::visit, allowing + * callers to visit the underlying node while keeping the variant private. + * + * Walking the AST: + * - Parser::parse() returns an Expression representing the root of the parsed + * expression. Define a visitor and invoke it with expr.visit(visitor). + */ + +#ifndef DEXPR_AST_HPP +#define DEXPR_AST_HPP + +#include +#include +#include +#include +#include +#include +#include + +namespace dexpr::ast { +struct Expression; + +// Do not allow nodes to have multiple owners. +using ExprPtr = std::unique_ptr; + +struct Identifier { + std::string value; +}; + +struct UnaryExpression { + TokenTypes op; + ExprPtr right; +}; + +struct BinaryExpression { + ExprPtr left; + TokenTypes op; + ExprPtr right; +}; + +struct FuncExpression { + ExprPtr function; + std::vector args; +}; + +struct ArrayExpression { + std::vector elements; +}; + +struct StringLiteral { + std::string value; +}; + +// Literals are stored at full precision regardless of how the consumer will +// eventually use them. Narrowing here would lose information before the +// expression ever meets a field: 273.15 is 273.14999389648438 as a float, and +// a legal threshold like 1e40 would not survive at all. +struct FloatLiteral { + double value; +}; + +struct IntegerLiteral { + int value; +}; + +using ExpressionVariant = + std::variant; + +template +concept ExpressionNode = std::constructible_from; + +struct Expression { + template + explicit Expression(T&& value) : node_(std::forward(value)) {} + + // This member function will be used for visitors so that + // the node variant can remain private + // NOTE: decltype(auto) allows visitors to return references + template decltype(auto) visit(Visitor&& visitor) const { + return std::visit(std::forward(visitor), node_); + } + +private: + ExpressionVariant node_; +}; + +template + requires std::constructible_from +ExprPtr make_expression(Args&&... args) { + return std::make_unique(Node{std::forward(args)...}); +} + +// Functions +std::string to_string(const Expression& expr); + +} // namespace dexpr::ast + +#endif diff --git a/share/dexpr/include/dexpr/lexer.hpp b/share/dexpr/include/dexpr/lexer.hpp new file mode 100644 index 000000000000..0bcb5e422b3b --- /dev/null +++ b/share/dexpr/include/dexpr/lexer.hpp @@ -0,0 +1,54 @@ +/** + * @file lexer.hpp + * @brief Defines the Lexer which turns a user supplied string into `tokens` + * + * The Lexer accepts a string input. + * At the request of the Parser, + * The input is scanned character by character + * and tokenized based on the tokens defined in 'tokens.hpp'. + * + */ +#ifndef DEXPR_LEXER_HPP +#define DEXPR_LEXER_HPP + +#include +#include + +namespace dexpr { + +class Lexer { + +public: + explicit Lexer(std::string input); + ~Lexer() = default; + + Token next_token(); + +private: + std::string input_; + int position_; + int read_position_; + char current_char_; + // Position of current_char_, 1-based, maintained by read_char(). + int line_; + int column_; + + // functions + // Scans without position; next_token() stamps it. + Token scan_token(); + void skip_whitespace(); + + std::string read_identifier(); + // seen_dot is true when the caller already consumed a leading '.', so that + // a second one is not folded into the same literal. + std::string read_number(bool seen_dot = false); + bool read_to_delim(char ch, std::string& out); + + char peek_char() const; + void read_char(); + Token make_token(TokenTypes kind) const; +}; + +} // namespace dexpr + +#endif diff --git a/share/dexpr/include/dexpr/parser.hpp b/share/dexpr/include/dexpr/parser.hpp new file mode 100644 index 000000000000..909b38ca3133 --- /dev/null +++ b/share/dexpr/include/dexpr/parser.hpp @@ -0,0 +1,94 @@ +/** + * @file parser.hpp + * @brief Defines the Parser for diagnostic expressions. + * + * The Parser receives tokens from the lexer and contains the rules + * for generating the Abstract Syntax Tree (AST). A malformed expression will + * generate a ParserError. + * + * The grammar rules for order-of-operations are encoded in precedences.hpp +*/ +#ifndef DEXPR_PARSER_HPP +#define DEXPR_PARSER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace dexpr::parser { + +class ParserError : public std::runtime_error { +public: + explicit ParserError(const std::vector& errors) + : std::runtime_error(join_msgs(errors)) {} + +private: + static std::string join_msgs(const std::vector& errors) { + std::string result = "Parser errors:\n"; + + for (const auto& error : errors) { + result += " - " + error + '\n'; + } + + return result; + } +}; + +class Parser { + +public: + explicit Parser(Lexer lexer); + + ast::ExprPtr parse(); + bool has_errors(); + +private: + Lexer lexer_; + Token cur_token_; + Token peek_token_; + std::vector errors_; + + // Prefix parsing functions must return ExprPtr and take no arugments (other + // than this) + using PrefixFn = ast::ExprPtr (Parser::*)(); + // Infix parsing function must return ExprPtr and take an ExprPtr as an + // argument + using InfixFn = ast::ExprPtr (Parser::*)(ast::ExprPtr); + + PrefixFn get_prefix_parse_fn(TokenTypes tok_type); + InfixFn get_infix_parse_fn(TokenTypes tok_type); + + // Functions + void add_error(std::string msg); + void next_token(); + + bool peek_token_is(TokenTypes expected_type); + + bool expect_peek_and_advance(TokenTypes expected_type); + Precedence peek_precedence(); + + ast::ExprPtr parse_expression(Precedence prec); + + // prefix member functions: + ast::ExprPtr parse_identifier(); + ast::ExprPtr parse_integer_literal(); + ast::ExprPtr parse_string_literal(); + ast::ExprPtr parse_float_literal(); + ast::ExprPtr parse_prefix_expression(); + ast::ExprPtr parse_grouped_expression(); + ast::ExprPtr parse_array_expression(); + + // infix member functions: + ast::ExprPtr parse_infix_expression(ast::ExprPtr left_expr); + ast::ExprPtr parse_function_expression(ast::ExprPtr expr); + std::vector parse_list_of_expressions(TokenTypes end_token); +}; + +} // namespace dexpr::parser + +#endif diff --git a/share/dexpr/include/dexpr/precedences.hpp b/share/dexpr/include/dexpr/precedences.hpp new file mode 100644 index 000000000000..12e562306e79 --- /dev/null +++ b/share/dexpr/include/dexpr/precedences.hpp @@ -0,0 +1,51 @@ +/** + * @file precedences.hpp + * @brief Defines operator precedence for diagnostic expressions. + * + * Precedence determines how tightly operators bind to their operands during + * expression parsing. Higher precedence values bind more tightly than lower + * precedence values. + * + * Each operator TokenType is mapped to a Precedence level by + * token_precedence(). The Pratt parser uses these levels to determine the + * grouping of expressions without requiring explicit parentheses. + * + * For example, given: + * @code + * x = -w * y + * @endcode + * + * with: + * @code + * Precedence::Equal < Precedence::Product < Precedence::Prefix + * @endcode + * + * the expression is parsed as: + * @code + * x = ((-w) * y) + * @endcode + */ +#ifndef DEXPR_PRECEDENCES_HPP +#define DEXPR_PRECEDENCES_HPP + +#include +namespace dexpr::parser { + +enum class Precedence { + Lowest, + Assignment, + Logical, + Equalitative, + Comparison, + Additive, + Multiplicative, + Prefix, + Exponent, + Call, +}; + +Precedence token_precedence(TokenTypes type); +Precedence right_binding_precedence(TokenTypes type); +} // namespace dexpr::parser + +#endif diff --git a/share/dexpr/include/dexpr/supported_functions.hpp b/share/dexpr/include/dexpr/supported_functions.hpp new file mode 100644 index 000000000000..6855845edec2 --- /dev/null +++ b/share/dexpr/include/dexpr/supported_functions.hpp @@ -0,0 +1,74 @@ +#ifndef DEXPR_SUPPORTED_FUNCTIONS_HPP +#define DEXPR_SUPPORTED_FUNCTIONS_HPP + +#include +#include +#include +#include +#include +namespace dexpr { +struct SupportedFunction { + std::string_view name; + std::string_view desc; + std::span arguments; + + std::string to_string() const { + std::string str_{name}; + str_ += "("; + if (!arguments.empty()) { + for (auto val : arguments) { + str_ += std::string(val) + ","; + } + str_.pop_back(); // removes trailing comma + } + str_ += ")"; + str_ += "\n--- " + std::string(desc); + + return str_; + } +}; +inline std::ostream& operator<<(std::ostream& os, + const SupportedFunction& function) { + return os << function.to_string() << '\n'; +} + +inline constexpr std::array where_args{ + std::string_view{""}, +}; + +inline constexpr std::array sum_args{ + std::string_view{"dims=[..]"}, +}; + +inline constexpr std::array derivative_args{ + std::string_view{"dx"}, + std::string_view{"dims=[..]"}, +}; + +inline constexpr std::array tend_args{}; + +inline constexpr std::array supported{ + SupportedFunction{ + .name = "where", + .desc = "applies condition to operand", + .arguments = where_args, + }, + SupportedFunction{ + .name = "sum", + .desc = "sums operand over designated indices (int or name)", + .arguments = sum_args, + }, + SupportedFunction{ + .name = "derivative", + .desc = "takes derivative w.r.t. `dx` over designated dimension", + .arguments = derivative_args, + }, + SupportedFunction{ + .name = "tend", + .desc = "calculates the tendency of a variable over time", + .arguments = tend_args, + }, +}; +} // namespace dexpr + +#endif diff --git a/share/dexpr/include/dexpr/tokens.hpp b/share/dexpr/include/dexpr/tokens.hpp new file mode 100644 index 000000000000..dbfa2aeedda4 --- /dev/null +++ b/share/dexpr/include/dexpr/tokens.hpp @@ -0,0 +1,85 @@ +/** + * @file tokens.hpp + * @brief Defines the all the tokens that may be produced by the lexer and + * consumed by the Parser + */ +#ifndef DEXPR_TOKENS_HPP +#define DEXPR_TOKENS_HPP + +#include +#include +#include +#include +namespace dexpr { + +enum class TokenTypes { + EndofFile, + Illegal, + + Identifier, + Integer, + Float, + String, + + // Operators + Assign, + Plus, + Minus, + Asterisk, + Bang, + Slash, + Exp, + Equal, + GreaterThan, + GreaterEqual, + LessThan, + LessEq, + NotEqual, + Or, + And, + Dot, + + // DELIMITERS + Comma, + LeftParen, + RightParen, + Colon, + ArrayLeftBracket, + ArrayRightBracket, +}; + +std::string_view to_string(TokenTypes type); + +struct Token { + TokenTypes type; + std::string literal; + // Start of the token in the input, 1-based. + int line = 1; + int column = 1; +}; + +// "line 1, column 7" +std::string position_of(const Token& tok); +std::string to_string(const Token& tok); + +// constexpr rather than a namespace-scope const container: the latter has +// internal linkage, so it would be built once per translation unit. +struct Keyword { + std::string_view name; // spelling, always lower case; matching folds first + TokenTypes type; + std::string_view literal; // what the resulting token carries +}; + +inline constexpr std::array keywords{{ + {"or", TokenTypes::Or, "or"}, + {"and", TokenTypes::And, "and"}, + {"not", TokenTypes::Bang, "!"}, +}}; +Token identifier_lookup(const Token& tok); + +std::string binary_op_to_string(const TokenTypes type); +std::string unary_op_to_string(const TokenTypes type); + +} // namespace dexpr + +#endif diff --git a/share/dexpr/run_tests.sh b/share/dexpr/run_tests.sh new file mode 100755 index 000000000000..8c4d2813f625 --- /dev/null +++ b/share/dexpr/run_tests.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +set -euo pipefail + +# Configure if needed; this is a no-op once build/ has a CMakeCache.txt. +cmake -S . -B build -DDEXPR_ENABLE_TESTS=ON -DDEXPR_ENABLE_TOOL=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + +cmake --build build --parallel +ctest --test-dir build --output-on-failure --parallel diff --git a/share/dexpr/src/ast_print.cpp b/share/dexpr/src/ast_print.cpp new file mode 100644 index 000000000000..e2312169ba47 --- /dev/null +++ b/share/dexpr/src/ast_print.cpp @@ -0,0 +1,102 @@ +#include +#include +#include +#include +#include +#include +#include + +/** + * @file ast_print.cpp + * @brief Implementation of Visitors for printing AST nodes + */ +namespace dexpr::ast { + +namespace { + +struct ToStringVisitor { + + std::string operator()(const Identifier& expr) const; + std::string operator()(const UnaryExpression& expr) const; + std::string operator()(const BinaryExpression& expr) const; + std::string operator()(const FuncExpression& expr) const; + std::string operator()(const ArrayExpression& expr) const; + std::string operator()(const StringLiteral& expr) const; + std::string operator()(const FloatLiteral& expr) const; + std::string operator()(const IntegerLiteral& expr) const; +}; + +std::string expr_list_to_string(std::span vals) { + + std::string result; + bool first = true; + + std::ranges::for_each(vals, [&](const ExprPtr& val) { + if (!first) { + result += ", "; + } + first = false; + result += to_string(*val); + }); + return result; +} + +std::string ToStringVisitor::operator()(const Identifier& expr) const { + return expr.value; +}; + +std::string ToStringVisitor::operator()(const UnaryExpression& expr) const { + return "(" + unary_op_to_string(expr.op) + to_string(*expr.right) + ")"; +}; + +std::string ToStringVisitor::operator()(const BinaryExpression& expr) const { + // '.' binds at Call, the tightest level, so wrapping it adds noise without + // disambiguating anything... + if (expr.op == TokenTypes::Dot) { + return to_string(*expr.left) + "." + to_string(*expr.right); + } + return "(" + to_string(*expr.left) + binary_op_to_string(expr.op) + + to_string(*expr.right) + ")"; +}; +std::string ToStringVisitor::operator()(const FuncExpression& expr) const { + return to_string(*expr.function) + "(" + expr_list_to_string(expr.args) + ")"; +}; + +std::string ToStringVisitor::operator()(const ArrayExpression& expr) const { + return "[" + expr_list_to_string(expr.elements) + "]"; +}; + +std::string ToStringVisitor::operator()(const StringLiteral& expr) const { + const char quote = + expr.value.find('\'') == std::string::npos ? '\'' : '"'; + return std::string(1, quote) + expr.value + quote; +}; +std::string ToStringVisitor::operator()(const IntegerLiteral& expr) const { + return std::to_string(expr.value); +}; +std::string ToStringVisitor::operator()(const FloatLiteral& expr) const { + // Shortest round-trip form; 24 chars is the worst case. + std::array buffer{}; + const auto [end, ec] = + std::to_chars(buffer.data(), buffer.data() + buffer.size(), expr.value); + if (ec != std::errc{}) { + return ""; + } + + std::string result(buffer.data(), end); + + // Otherwise "1500" would lex back as an integer, not the float it came from. + if (std::isfinite(expr.value) && + result.find_first_of(".eE") == std::string::npos) { + result += ".0"; + } + return result; +}; + +} // namespace + +std::string to_string(const Expression& expr) { + return expr.visit(ToStringVisitor{}); +} + +} // namespace dexpr::ast diff --git a/share/dexpr/src/lexer.cpp b/share/dexpr/src/lexer.cpp new file mode 100644 index 000000000000..5153fb90c519 --- /dev/null +++ b/share/dexpr/src/lexer.cpp @@ -0,0 +1,258 @@ +#include +#include +#include +#include +#include + +namespace { + +bool is_numeric(const char ch) { + return std::isdigit(static_cast(ch)); +} + +bool is_identifier_char(const char ch) { + return std::isalpha(static_cast(ch)) || ch == '_' || + is_numeric(ch); +} + +} // namespace + +namespace dexpr { + +Lexer::Lexer(std::string input) + : input_{std::move(input)}, position_{0}, read_position_{0}, + current_char_{'\0'}, line_{1}, column_{0} { + read_char(); +} + +void Lexer::read_char() { + // column_ starts at 0 so the constructor's priming read lands the first + // character on column 1. + if (current_char_ == '\n') { + line_ += 1; + column_ = 1; + } else { + column_ += 1; + } + + if (read_position_ >= static_cast(input_.length())) { + current_char_ = '\0'; + } else { + current_char_ = input_.at(read_position_); + } + position_ = read_position_; + read_position_ += 1; +} + +char Lexer::peek_char() const { + if (read_position_ >= static_cast(input_.length())) { + return '\0'; + } else { + return input_[read_position_]; + } +} + +void Lexer::skip_whitespace() { + while (std::isspace(static_cast(current_char_))) { + read_char(); + } +} + +Token Lexer::make_token(TokenTypes kind) const { + return {kind, std::string(1, current_char_)}; +} + +bool Lexer::read_to_delim(char ch, std::string& out) { + const auto start_pos = position_ + 1; + while (peek_char() != ch && peek_char() != '\0') { + read_char(); + } + + const bool closed = peek_char() == ch; + read_char(); + + auto count = position_ - start_pos; + out = input_.substr(start_pos, count); + return closed; +} + +std::string Lexer::read_number(bool seen_dot) { + const auto start_pos = position_; + + // At most one decimal point belongs to a single number: "1.2.3" is two + // numbers, not one malformed one + while (is_numeric(current_char_) || (current_char_ == '.' && !seen_dot)) { + if (current_char_ == '.') { + seen_dot = true; + } + read_char(); + } + + // Only consume an exponent if a digit actually follows it + if (current_char_ == 'e' || current_char_ == 'E') { + const auto sign_offset = (peek_char() == '+' || peek_char() == '-') ? 1 : 0; + const auto digit_pos = read_position_ + sign_offset; + + if (digit_pos < static_cast(input_.length()) && + is_numeric(input_[digit_pos])) { + read_char(); // 'e' + if (current_char_ == '+' || current_char_ == '-') { + read_char(); // sign + } + while (is_numeric(current_char_)) { + read_char(); + } + } + } + + return input_.substr(start_pos, position_ - start_pos); +} + +std::string Lexer::read_identifier() { + auto start_pos = position_; + while (is_identifier_char(current_char_)) { + read_char(); + } + auto length = position_ - start_pos; + return input_.substr(start_pos, length); +} + +Token Lexer::next_token() { + skip_whitespace(); + + // Captured before scanning: the position is where the token starts. + const auto line = line_; + const auto column = column_; + + auto tok = scan_token(); + tok.line = line; + tok.column = column; + return tok; +} + +Token Lexer::scan_token() { + + // Value-initialized: type would otherwise be indeterminate. + Token tok{}; + + switch (current_char_) { + case '=': + if (peek_char() == '=') { + tok = {TokenTypes::Equal, "=="}; + read_char(); + } else { + tok = make_token(TokenTypes::Assign); + } + break; + case '(': + tok = make_token(TokenTypes::LeftParen); + break; + case ')': + tok = make_token(TokenTypes::RightParen); + break; + case ',': + tok = make_token(TokenTypes::Comma); + break; + case '+': + tok = make_token(TokenTypes::Plus); + break; + case '-': + tok = make_token(TokenTypes::Minus); + break; + case '[': + tok = make_token(TokenTypes::ArrayLeftBracket); + break; + case ']': + tok = make_token(TokenTypes::ArrayRightBracket); + break; + case '/': + tok = make_token(TokenTypes::Slash); + break; + case '*': + if (peek_char() == '*') { + read_char(); + tok = {TokenTypes::Exp, "**"}; + } else { + tok = make_token(TokenTypes::Asterisk); + } + break; + case '<': + if (peek_char() == '=') { + read_char(); + tok = {TokenTypes::LessEq, "<="}; + } else { + tok = make_token(TokenTypes::LessThan); + } + break; + + case '>': + if (peek_char() == '=') { + read_char(); + tok = {TokenTypes::GreaterEqual, ">="}; + } else { + tok = make_token(TokenTypes::GreaterThan); + } + break; + case '!': + if (peek_char() == '=') { + read_char(); + tok = {TokenTypes::NotEqual, "!="}; + } else { + // '!' is an alias for the `not` keyword + tok = make_token(TokenTypes::Bang); + } + break; + case '\0': + return {TokenTypes::EndofFile, ""}; + case ':': + tok = make_token(TokenTypes::Colon); + break; + case '"': + case '\'': { + std::string literal; + const bool closed = read_to_delim(current_char_, literal); + // An unterminated literal is reported as Illegal rather than silently + // accepted as a well-formed string. + tok = {closed ? TokenTypes::String : TokenTypes::Illegal, literal}; + break; + } + case '.': { + if (is_numeric(peek_char())) { + // The '.' that got us here is part of the literal, so read_number() must + // not fold in a second one: ".5.3" is two numbers, exactly as "1.2.3" is. + read_char(); + auto number = read_number(/*seen_dot=*/true); + number.insert(0, "0."); + return {TokenTypes::Float, number}; + } else { + tok = make_token(TokenTypes::Dot); + break; + } + } + default: { + + if (is_numeric(current_char_)) { + auto number = read_number(); + + // 'E' counts as much as 'e'; missing it classified "1E5" as an integer, + // and integer parsing then stopped at the 'E' and silently returned 1. + if (number.find_first_of(".eE") != std::string::npos) { + return {TokenTypes::Float, number}; + } else { + return {TokenTypes::Integer, number}; + } + } else if (is_identifier_char(current_char_)) { + return identifier_lookup({TokenTypes::Identifier, read_identifier()}); + } else { + auto illegal = make_token(TokenTypes::Illegal); + read_char(); + return illegal; + } + + } // default + } // end switch + read_char(); + return tok; +} + +} // namespace dexpr diff --git a/share/dexpr/src/parser.cpp b/share/dexpr/src/parser.cpp new file mode 100644 index 000000000000..5a4a06c7ff96 --- /dev/null +++ b/share/dexpr/src/parser.cpp @@ -0,0 +1,291 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace dexpr::parser { + +namespace { + +// std::from_chars rather than std::sto*: it is locale-independent (std::stof +// reads "1.5" as 1 wherever ',' is the decimal separator), it does not throw, +// and it reports where it stopped. Requiring it to stop at the end of the +// literal means a malformed one is rejected outright instead of being read as +// its leading prefix, which is exactly how "0.5.3" used to become 0.5. +template +std::optional parse_number(const std::string& literal) { + T value{}; + const auto* const first = literal.data(); + const auto* const last = first + literal.size(); + + const auto [stopped_at, ec] = std::from_chars(first, last, value); + if (ec != std::errc{} || stopped_at != last) { + return std::nullopt; + } + return value; +} + +} // namespace + +bool Parser::peek_token_is(TokenTypes expected_type) { + return peek_token_.type == expected_type; +}; + +void Parser::add_error(std::string msg) { errors_.push_back(std::move(msg)); } + +bool Parser::expect_peek_and_advance(TokenTypes expected_type) { + if (peek_token_is(expected_type)) { + next_token(); + return true; + } else { + add_error("Expected " + std::string(to_string(expected_type)) + ", got " + + to_string(peek_token_) + " at " + position_of(peek_token_)); + return false; + } +} + +Precedence Parser::peek_precedence() { + return token_precedence(peek_token_.type); +} + +bool Parser::has_errors() { return !errors_.empty(); } + +void Parser::next_token() { + cur_token_ = peek_token_; + peek_token_ = lexer_.next_token(); + if (peek_token_is(TokenTypes::Illegal)) { + add_error("Illegal token " + to_string(peek_token_) + " at " + + position_of(peek_token_)); + } +} + +// prefix_parse_fns_{{ +// }}, +// infix_parse_fns_{{ +// }} + +Parser::PrefixFn Parser::get_prefix_parse_fn(TokenTypes tok_type) { + switch (tok_type) { + case TokenTypes::Identifier: + return &Parser::parse_identifier; + case TokenTypes::Integer: + return &Parser::parse_integer_literal; + case TokenTypes::Float: + return &Parser::parse_float_literal; + case TokenTypes::String: + return &Parser::parse_string_literal; + case TokenTypes::Minus: + case TokenTypes::Bang: + return &Parser::parse_prefix_expression; + case TokenTypes::ArrayLeftBracket: + return &Parser::parse_array_expression; + case TokenTypes::LeftParen: + return &Parser::parse_grouped_expression; + default: + return nullptr; + } +} + +Parser::InfixFn Parser::get_infix_parse_fn(TokenTypes tok_type) { + switch (tok_type) { + case TokenTypes::Plus: + case TokenTypes::Minus: + case TokenTypes::Asterisk: + case TokenTypes::Exp: + case TokenTypes::Assign: + case TokenTypes::Slash: + case TokenTypes::Equal: + case TokenTypes::NotEqual: + case TokenTypes::GreaterThan: + case TokenTypes::GreaterEqual: + case TokenTypes::LessThan: + case TokenTypes::LessEq: + case TokenTypes::Or: + case TokenTypes::And: + case TokenTypes::Dot: + return &Parser::parse_infix_expression; + case TokenTypes::LeftParen: + return &Parser::parse_function_expression; + default: + return nullptr; + } +} + +/** + * @brief Parses an expression using Pratt/operator-precedence parsing. + * + * Parsing begins by dispatching on the current token to construct the initial + * left-hand expression using the appropriate prefix parse function. The parser + * then examines subsequent tokens and extends that expression with infix + * operations if the next operator has **higher** precedence. + * + * The Precedence prec argument sets the minimum binding precedence for + * this invocation. Parsing stops when the next token has equal or lower + * precedence, allowing the calling parse function to retain ownership of that + * operator and thereby determine the grouping of the resulting AST. + * + * For example, when parsing: + * @code + * x + y * z + * @endcode + * + * the multiplication operator has higher precedence than addition, so the + * right-hand side of the addition is parsed as: + * @code + * y * z + * @endcode + * + * producing the grouping: + * @code + * x + (y * z) + * @endcode + * + * @param prec Minimum precedence required for subsequent operators to bind to + * the expression currently being parsed. + * @return Pointer to the root expression node of the parsed AST subtree. + * @throws ParserError If the current token cannot begin an expression. + */ +ast::ExprPtr Parser::parse_expression(Precedence prec) { + const auto prefix_fn = get_prefix_parse_fn(cur_token_.type); + if (prefix_fn == nullptr) { + add_error("Unexpected Prefix Token " + to_string(cur_token_) + " at " + + position_of(cur_token_)); + throw ParserError(errors_); + } + auto left_expr = (this->*prefix_fn)(); + + while (!peek_token_is(TokenTypes::EndofFile) && prec < peek_precedence()) { + const auto infix_fn = get_infix_parse_fn(peek_token_.type); + if (infix_fn == nullptr) { + return left_expr; + } + next_token(); + left_expr = (this->*infix_fn)(std::move(left_expr)); + } + return left_expr; +} + +ast::ExprPtr Parser::parse_identifier() { + return ast::make_expression(cur_token_.literal); +} +ast::ExprPtr Parser::parse_string_literal() { + return ast::make_expression(cur_token_.literal); +} + +ast::ExprPtr Parser::parse_integer_literal() { + if (const auto value = parse_number(cur_token_.literal)) { + return ast::make_expression(*value); + } + add_error("Integer literal out of range: " + cur_token_.literal + " at " + + position_of(cur_token_)); + throw ParserError(errors_); +} + +ast::ExprPtr Parser::parse_float_literal() { + if (const auto value = parse_number(cur_token_.literal)) { + return ast::make_expression(*value); + } + add_error("Float literal out of range: " + cur_token_.literal + " at " + + position_of(cur_token_)); + throw ParserError(errors_); +} +ast::ExprPtr Parser::parse_prefix_expression() { + auto op = cur_token_.type; + next_token(); + auto right_expr = parse_expression(Precedence::Prefix); + return ast::make_expression(op, std::move(right_expr)); +} + +ast::ExprPtr Parser::parse_grouped_expression() { + next_token(); + auto expr = parse_expression(Precedence::Lowest); + if (!expect_peek_and_advance(TokenTypes::RightParen)) { + // Throwing rather than returning null: parse() would catch this at the end + // anyway, but until then the null travels through the tree builders, and + // every visitor dereferences its children unguarded. + throw ParserError(errors_); + } + return expr; +} + +ast::ExprPtr Parser::parse_infix_expression(ast::ExprPtr left_expr) { + const auto op = cur_token_.type; + const auto prec = right_binding_precedence(op); + next_token(); + + auto right_expr = parse_expression(prec); + + return ast::make_expression(std::move(left_expr), op, + std::move(right_expr)); +} + +std::vector +Parser::parse_list_of_expressions(TokenTypes end_token) { + // Should this consume the end-token or not? + + std::vector expressions; + if (peek_token_is(end_token)) { + next_token(); + return expressions; + } + next_token(); + + expressions.push_back(parse_expression(Precedence::Lowest)); + // should this be an input arg as well ...? + while (peek_token_is(TokenTypes::Comma)) { + next_token(); + next_token(); // Comma should be consumed + expressions.push_back(parse_expression(Precedence::Lowest)); + } + + if (!expect_peek_and_advance(end_token)) { + throw ParserError(errors_); + } + return expressions; +} + +ast::ExprPtr Parser::parse_function_expression(ast::ExprPtr func) { + auto args = parse_list_of_expressions(TokenTypes::RightParen); + return ast::make_expression(std::move(func), + std::move(args)); +} + +ast::ExprPtr Parser::parse_array_expression() { + return ast::make_expression( + parse_list_of_expressions(TokenTypes::ArrayRightBracket)); +} + +Parser::Parser(Lexer lexer) : lexer_{std::move(lexer)} { + next_token(); + next_token(); +} + +/** + * @brief Public entry point to parse an expression. + * + * Parses the string given to the Lexer used to construct the parser. + * + * @return Pointer to root of the AST. + * @throws ParserError + * @see Parser::parse_expression + * @warning Assumes input has only **one** expression + */ +ast::ExprPtr Parser::parse() { + auto expr = parse_expression(Precedence::Lowest); + + if (!peek_token_is(TokenTypes::EndofFile)) { + add_error("Unexpected trailing input " + to_string(peek_token_) + " at " + + position_of(peek_token_)); + } + + if (has_errors()) { + throw ParserError(errors_); + } + return expr; +} + +} // namespace dexpr::parser diff --git a/share/dexpr/src/precedences.cpp b/share/dexpr/src/precedences.cpp new file mode 100644 index 000000000000..19824d16007d --- /dev/null +++ b/share/dexpr/src/precedences.cpp @@ -0,0 +1,71 @@ +#include +#include +#include + +namespace dexpr::parser { + +// Lowest, +// Equal, +// LessGreater, +// Sum, +// Product, +// Prefix, +// Bounds, +// Call, + +Precedence token_precedence(TokenTypes type) { + switch (type) { + + case TokenTypes::Assign: + return Precedence::Assignment; + + case TokenTypes::And: + case TokenTypes::Or: + return Precedence::Logical; + + case TokenTypes::Equal: + case TokenTypes::NotEqual: + return Precedence::Equalitative; + + case TokenTypes::GreaterThan: + case TokenTypes::GreaterEqual: + case TokenTypes::LessThan: + case TokenTypes::LessEq: + return Precedence::Comparison; + + case TokenTypes::Plus: + case TokenTypes::Minus: + return Precedence::Additive; + + case TokenTypes::Slash: + case TokenTypes::Asterisk: + return Precedence::Multiplicative; + + case TokenTypes::Bang: + return Precedence::Prefix; + + case TokenTypes::Exp: + return Precedence::Exponent; + + case TokenTypes::Dot: + case TokenTypes::LeftParen: + return Precedence::Call; + + default: + return Precedence::Lowest; + } +} + +Precedence right_binding_precedence(TokenTypes type) { + const auto prec = token_precedence(type); + + switch (type) { + // '**' is the only right-associative operator: 2**3**2 is 2**(3**2). + case TokenTypes::Exp: + return static_cast(static_cast(prec) - 1); + default: + return prec; + } +} + +} // namespace dexpr::parser diff --git a/share/dexpr/src/tokens.cpp b/share/dexpr/src/tokens.cpp new file mode 100644 index 000000000000..26a00c5fef3d --- /dev/null +++ b/share/dexpr/src/tokens.cpp @@ -0,0 +1,154 @@ +#include +#include +#include +#include +#include + +namespace dexpr { + +std::string_view to_string(TokenTypes type) { + switch (type) { + + case TokenTypes::EndofFile: + return "EndofFile"; + case TokenTypes::Illegal: + return "Illegal"; + + case TokenTypes::Identifier: + return "Identifier"; + case TokenTypes::Integer: + return "Integer"; + case TokenTypes::Float: + return "Float"; + case TokenTypes::String: + return "String"; + // Operators + case TokenTypes::Assign: + return "Assign"; + case TokenTypes::Plus: + return "Plus"; + case TokenTypes::Minus: + return "Minus"; + case TokenTypes::Asterisk: + return "Asterisk"; + case TokenTypes::Bang: + return "Bang"; + case TokenTypes::Slash: + return "Slash"; + case TokenTypes::Exp: + return "Exp"; + case TokenTypes::Equal: + return "Equal"; + case TokenTypes::GreaterThan: + return "GreaterThan"; + case TokenTypes::GreaterEqual: + return "GreaterEqual"; + case TokenTypes::LessThan: + return "LessThan"; + case TokenTypes::LessEq: + return "LessEq"; + case TokenTypes::NotEqual: + return "NotEqual"; + case TokenTypes::Or: + return "Or"; + case TokenTypes::And: + return "And"; + case TokenTypes::Dot: + return "Dot"; + + // DELIMITERS + case TokenTypes::Comma: + return "Comma"; + case TokenTypes::LeftParen: + return "LeftParen"; + case TokenTypes::RightParen: + return "RightParen"; + case TokenTypes::Colon: + return "Colon"; + case TokenTypes::ArrayLeftBracket: + return "ArrayLeftBracket"; + case TokenTypes::ArrayRightBracket: + return "ArrayRightBracket"; + } + // No default above: -Wswitch then flags a newly added token type. + return "UNKNOWN"; +} + +std::string position_of(const Token& tok) { + return "line " + std::to_string(tok.line) + ", column " + + std::to_string(tok.column); +} + +std::string to_string(const Token& tok) { + return "{Type: " + std::string(to_string(tok.type)) + + ", Literal: " + tok.literal + "}"; +} +Token identifier_lookup(const Token& tok) { + // This function checks to see if an identifier is a keyword + // keywords are case-insensitive; identifiers are case-sensitive + // identifier case intact. + std::string folded = tok.literal; + std::transform( + folded.begin(), folded.end(), folded.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + + for (const auto& keyword : keywords) { + if (keyword.name == folded) { + return {keyword.type, std::string(keyword.literal)}; + } + } + return tok; +} + +std::string binary_op_to_string(const TokenTypes type) { + + switch (type) { + case TokenTypes::Plus: + return "+"; + case TokenTypes::Minus: + return "-"; + case TokenTypes::Asterisk: + return "*"; + case TokenTypes::Slash: + return "/"; + case TokenTypes::Equal: + return "=="; + case TokenTypes::NotEqual: + return "!="; + case TokenTypes::LessThan: + return "<"; + case TokenTypes::LessEq: + return "<="; + case TokenTypes::GreaterThan: + return ">"; + case TokenTypes::GreaterEqual: + return ">="; + // Word operators need surrounding whitespace to stay lexable when printed + case TokenTypes::And: + return " and "; + case TokenTypes::Or: + return " or "; + case TokenTypes::Assign: + return "="; + case TokenTypes::Exp: + return "**"; + case TokenTypes::Dot: + return "."; + default: + throw std::invalid_argument{"Invalid Binary Operator" + std::string(to_string(type))}; + } +} + +std::string unary_op_to_string(const TokenTypes type) { + switch (type) { + case TokenTypes::Plus: + return "+"; + case TokenTypes::Minus: + return "-"; + case TokenTypes::Bang: + return "!"; + default: + throw std::invalid_argument{"Invalid Unary Operator"}; + } +} +} diff --git a/share/dexpr/tests/CMakeLists.txt b/share/dexpr/tests/CMakeLists.txt new file mode 100644 index 000000000000..16c39776d2e8 --- /dev/null +++ b/share/dexpr/tests/CMakeLists.txt @@ -0,0 +1,36 @@ +# Prefer a Catch2 that is already installed; fall back to fetching one, so that +# a bare `cmake -S . -B build` still works on a machine that has never seen it. +find_package(Catch2 3 QUIET) + +if (NOT Catch2_FOUND) + include(FetchContent) + + FetchContent_Declare( + Catch2 + GIT_REPOSITORY https://github.com/catchorg/Catch2.git + GIT_TAG v3.8.1 + ) + + FetchContent_MakeAvailable(Catch2) + + # Needed for include(Catch) below when Catch2 comes in as a subproject. + list(APPEND CMAKE_MODULE_PATH ${Catch2_SOURCE_DIR}/extras) +endif() + +add_executable(dexpr_tests + test_lexer.cpp + test_parser.cpp +) + +target_link_libraries(dexpr_tests + PRIVATE + dexpr + Catch2::Catch2WithMain +) + +target_compile_options(dexpr_tests PRIVATE ${DEXPR_WARNINGS}) + +include(Catch) + +# Registers each TEST_CASE as its own ctest test, so a failure names itself. +catch_discover_tests(dexpr_tests) diff --git a/share/dexpr/tests/test_lexer.cpp b/share/dexpr/tests/test_lexer.cpp new file mode 100644 index 000000000000..6a83f2471df7 --- /dev/null +++ b/share/dexpr/tests/test_lexer.cpp @@ -0,0 +1,419 @@ +#include + +#include +#include + +namespace dexpr +{ + +namespace +{ // anonymous + +// safety net inside tests +constexpr std::size_t k_max_token = 1024; + +std::vector +lex_all(std::string input) +{ + Lexer lexer{input}; + std::vector tokens; + while (tokens.size() < k_max_token) { + auto tok = lexer.next_token(); + tokens.push_back(tok); + if (tok.type == TokenTypes::EndofFile) { + break; + } + } + return tokens; +} + +void +check_tokens(const std::string &input, const std::vector &expected) +{ + const auto actual = lex_all(input); + + INFO("Input: " << input); + REQUIRE(actual.size() < k_max_token); + REQUIRE(actual.size() == expected.size()); + + for (std::size_t i = 0; i < expected.size(); ++i) { + INFO("Token #" << i << "\n Expected: " << to_string(expected[i]) + << "\n Received: " << to_string(actual[i])); + REQUIRE(expected[i].type == actual[i].type); // unwind + CHECK(expected[i].literal == actual[i].literal); // keep going + } +} + +const Token k_eof{TokenTypes::EndofFile, ""}; + +} // namespace + +TEST_CASE("lexer: empty and whitespace-only input", "[lexer]") +{ + check_tokens("", {k_eof}); + check_tokens(" \t ", {k_eof}); +} + +// TODO: for now, newlines are simply ignored as whitespace; tacky but workable +TEST_CASE("lexer: newline is like whitespace", "[lexer]") +{ + check_tokens("x\ny _w my_z\n", { + {TokenTypes::Identifier, "x"}, + {TokenTypes::Identifier, "y"}, + {TokenTypes::Identifier, "_w"}, + {TokenTypes::Identifier, "my_z"}, + k_eof, + }); +} + +TEST_CASE("lexer: single-character operators and delimiters", "[lexer]") +{ + check_tokens("+-*/(),:[].", { + {TokenTypes::Plus, "+"}, + {TokenTypes::Minus, "-"}, + {TokenTypes::Asterisk, "*"}, + {TokenTypes::Slash, "/"}, + {TokenTypes::LeftParen, "("}, + {TokenTypes::RightParen, ")"}, + {TokenTypes::Comma, ","}, + {TokenTypes::Colon, ":"}, + {TokenTypes::ArrayLeftBracket, "["}, + {TokenTypes::ArrayRightBracket, "]"}, + {TokenTypes::Dot, "."}, + k_eof, + }); +} + +TEST_CASE("lexer: multi-character operators", "[lexer]") +{ + check_tokens("== <= >= ** < > =", { + {TokenTypes::Equal, "=="}, + {TokenTypes::LessEq, "<="}, + {TokenTypes::GreaterEqual, ">="}, + {TokenTypes::Exp, "**"}, + {TokenTypes::LessThan, "<"}, + {TokenTypes::GreaterThan, ">"}, + {TokenTypes::Assign, "="}, + k_eof, + }); +} + +TEST_CASE("lexer: integer literals", "[lexer]") +{ + check_tokens("0 5 42 0 55", { + {TokenTypes::Integer, "0"}, + {TokenTypes::Integer, "5"}, + {TokenTypes::Integer, "42"}, + {TokenTypes::Integer, "0"}, + {TokenTypes::Integer, "55"}, + k_eof, + }); +} + +TEST_CASE("lexer: decimal float literals", "[lexer]") +{ + check_tokens("1.", {{TokenTypes::Float, "1."}, k_eof}); + check_tokens("1.5", {{TokenTypes::Float, "1.5"}, k_eof}); + check_tokens("0.25", {{TokenTypes::Float, "0.25"}, k_eof}); + check_tokens(".025", {{TokenTypes::Float, "0.025"}, k_eof}); +} + +TEST_CASE("lexer: exponent-form float literals", "[lexer]") +{ + check_tokens("1.0e-4", {{TokenTypes::Float, "1.0e-4"}, k_eof}); + check_tokens("2.5e+3", {{TokenTypes::Float, "2.5e+3"}, k_eof}); + check_tokens("1.5e3", {{TokenTypes::Float, "1.5e3"}, k_eof}); + check_tokens("1e5", {{TokenTypes::Float, "1e5"}, k_eof}); + check_tokens("1.e5", {{TokenTypes::Float, "1.e5"}, k_eof}); + // Either case marks an exponent, and the literal keeps whichever was + // written -- the lexer no longer rewrites the input. std::stof accepts both. + check_tokens("1.0E-4", {{TokenTypes::Float, "1.0E-4"}, k_eof}); +} + +TEST_CASE("lexer: a number followed by an operator", "[lexer]") +{ + check_tokens("1.0e-4+2", { + {TokenTypes::Float, "1.0e-4"}, + {TokenTypes::Plus, "+"}, + {TokenTypes::Integer, "2"}, + k_eof, + }); +} + +TEST_CASE("lexer: truncated exponent does not throw", "[lexer]") +{ + check_tokens("1.0e", { + {TokenTypes::Float, "1.0"}, + {TokenTypes::Identifier, "e"}, + k_eof, + }); + check_tokens("1.0e+", { + {TokenTypes::Float, "1.0"}, + {TokenTypes::Identifier, "e"}, + {TokenTypes::Plus, "+"}, + k_eof, + }); +} + +TEST_CASE("lexer: a second decimal point ends the number", "[lexer]") +{ + check_tokens("1.2.3", { + {TokenTypes::Float, "1.2"}, + {TokenTypes::Float, "0.3"}, + k_eof, + }); +} + +TEST_CASE("lexer: identifiers may contain digits", "[lexer]") +{ + check_tokens("bc_a1 so4_a2 O3 num_a1", { + {TokenTypes::Identifier, "bc_a1"}, + {TokenTypes::Identifier, "so4_a2"}, + {TokenTypes::Identifier, "O3"}, + {TokenTypes::Identifier, "num_a1"}, + k_eof, + }); + check_tokens("_a1", {{TokenTypes::Identifier, "_a1"}, k_eof}); + check_tokens("dst_a3_at_lev_10", {{TokenTypes::Identifier, "dst_a3_at_lev_10"}, k_eof}); +} + +TEST_CASE("lexer: an identifier cannot start with a digit", "[lexer]") +{ + check_tokens("2x", { + {TokenTypes::Integer, "2"}, + {TokenTypes::Identifier, "x"}, + k_eof, + }); + check_tokens("500hPa", { + {TokenTypes::Integer, "500"}, + {TokenTypes::Identifier, "hPa"}, + k_eof, + }); +} + +TEST_CASE("lexer: digits do not break member access", "[lexer]") +{ + check_tokens("bc_a1.mean", { + {TokenTypes::Identifier, "bc_a1"}, + {TokenTypes::Dot, "."}, + {TokenTypes::Identifier, "mean"}, + k_eof, + }); +} + +TEST_CASE("lexer: a keyword with a digit appended is an identifier", "[lexer]") +{ + check_tokens("and2 or1 not3", { + {TokenTypes::Identifier, "and2"}, + {TokenTypes::Identifier, "or1"}, + {TokenTypes::Identifier, "not3"}, + k_eof, + }); +} + +TEST_CASE("lexer: keywords are case-insensitive, normalized but not identifiers", "[lexer]") +{ + check_tokens("AND or Not NOT android nothing", { + {TokenTypes::And, "and"}, + {TokenTypes::Or, "or"}, + {TokenTypes::Bang, "!"}, + {TokenTypes::Bang, "!"}, + {TokenTypes::Identifier, "android"}, + {TokenTypes::Identifier, "nothing"}, + k_eof, + }); +} + +TEST_CASE("lexer: string literals in both quote styles", "[lexer]") +{ + check_tokens("'col'", {{TokenTypes::String, "col"}, k_eof}); + check_tokens("\"col\"", {{TokenTypes::String, "col"}, k_eof}); +} + +TEST_CASE("lexer: string literals preserve case", "[lexer]") +{ + check_tokens("'MyVar'", {{TokenTypes::String, "MyVar"}, k_eof}); + check_tokens("\"SHOC_tke\"", {{TokenTypes::String, "SHOC_tke"}, k_eof}); + check_tokens("MyField T_mid", { + {TokenTypes::Identifier, "MyField"}, + {TokenTypes::Identifier, "T_mid"}, + k_eof, + }); +} + +TEST_CASE("lexer: unterminated string literal is illegal", "[lexer]") +{ + check_tokens("'abc", {{TokenTypes::Illegal, "abc"}, k_eof}); + check_tokens("\"abc", {{TokenTypes::Illegal, "abc"}, k_eof}); + check_tokens("'", {{TokenTypes::Illegal, ""}, k_eof}); + check_tokens("'ok' + 'bad", { + {TokenTypes::String, "ok"}, + {TokenTypes::Plus, "+"}, + {TokenTypes::Illegal, "bad"}, + k_eof, + }); +} + +// The Illegal branch in next_token() returned early, skipping the read_char() +// at the end of the function -- so the scanner stayed parked on the offending +// character and re-emitted it forever. Any consumer draining to EndofFile +// (lex_all included) hung rather than failing. +TEST_CASE("lexer: illegal characters advance the scanner", "[lexer]") +{ + check_tokens("a@b", { + {TokenTypes::Identifier, "a"}, + {TokenTypes::Illegal, "@"}, + {TokenTypes::Identifier, "b"}, + k_eof, + }); + check_tokens("@", {{TokenTypes::Illegal, "@"}, k_eof}); + check_tokens("@#@", { + {TokenTypes::Illegal, "@"}, + {TokenTypes::Illegal, "#"}, + {TokenTypes::Illegal, "@"}, + k_eof, + }); +} + +TEST_CASE("lexer: not-equal operator", "[lexer]") +{ + check_tokens("3 != 4", { + {TokenTypes::Integer, "3"}, + {TokenTypes::NotEqual, "!="}, + {TokenTypes::Integer, "4"}, + k_eof, + }); + check_tokens("x!=y", { + {TokenTypes::Identifier, "x"}, + {TokenTypes::NotEqual, "!="}, + {TokenTypes::Identifier, "y"}, + k_eof, + }); +} + +TEST_CASE("lexer: bang is an alias for not", "[lexer]") +{ + check_tokens("!x", { + {TokenTypes::Bang, "!"}, + {TokenTypes::Identifier, "x"}, + k_eof, + }); + check_tokens("not x", { + {TokenTypes::Bang, "!"}, + {TokenTypes::Identifier, "x"}, + k_eof, + }); +} + +TEST_CASE("lexer: bang alias does not swallow not-equal", "[lexer]") +{ + check_tokens("x != y", { + {TokenTypes::Identifier, "x"}, + {TokenTypes::NotEqual, "!="}, + {TokenTypes::Identifier, "y"}, + k_eof, + }); + check_tokens("a ! = b", { + {TokenTypes::Identifier, "a"}, + {TokenTypes::Bang, "!"}, + {TokenTypes::Assign, "="}, + {TokenTypes::Identifier, "b"}, + k_eof, + }); + check_tokens("! x", { + {TokenTypes::Bang, "!"}, + {TokenTypes::Identifier, "x"}, + k_eof, + }); +} + +// A malformed literal must not be quietly repaired into a valid-looking one: +// ".5.3" used to lex as a single Float("0.5.3"), which the parser then read as +// 0.5 and silently dropped the rest. +TEST_CASE("lexer: a second decimal point starts a new number", "[lexer]") +{ + check_tokens(".5.3", { + {TokenTypes::Float, "0.5"}, + {TokenTypes::Float, "0.3"}, + k_eof, + }); + // The leading-dot form must agree with the ordinary form + check_tokens("1.2.3", { + {TokenTypes::Float, "1.2"}, + {TokenTypes::Float, "0.3"}, + k_eof, + }); + check_tokens(".5", {{TokenTypes::Float, "0.5"}, k_eof}); +} + +// "1E5" used to lex as Integer("1E5"); integer parsing then stopped at the 'E' +// and the expression silently evaluated to 1. +TEST_CASE("lexer: exponent marker is case-insensitive", "[lexer]") +{ + check_tokens("1E5", {{TokenTypes::Float, "1E5"}, k_eof}); + check_tokens("1e5", {{TokenTypes::Float, "1e5"}, k_eof}); + check_tokens("1E+5", {{TokenTypes::Float, "1E+5"}, k_eof}); + check_tokens("1E-5", {{TokenTypes::Float, "1E-5"}, k_eof}); + check_tokens("2.5E3", {{TokenTypes::Float, "2.5E3"}, k_eof}); + // Still an identifier when no digit follows, not a broken number + check_tokens("1E", { + {TokenTypes::Integer, "1"}, + {TokenTypes::Identifier, "E"}, + k_eof, + }); +} + +TEST_CASE("lexer: tokens carry their source position", "[lexer]") +{ + // 1234567 + const auto tokens = lex_all("x + 12"); + REQUIRE(tokens.size() == 4); + CHECK(tokens[0].line == 1); + CHECK(tokens[0].column == 1); // x + CHECK(tokens[1].column == 3); // + + CHECK(tokens[2].column == 5); // 12 + CHECK(tokens[3].column == 7); // end of input +} + +TEST_CASE("lexer: position survives multi-character tokens", "[lexer]") +{ + // 123456789 + const auto tokens = lex_all("ab <= 'c'"); + REQUIRE(tokens.size() == 4); + CHECK(tokens[0].column == 1); // ab + CHECK(tokens[1].column == 4); // <= + CHECK(tokens[2].column == 7); // 'c' +} + +TEST_CASE("lexer: newlines advance the line and reset the column", "[lexer]") +{ + const auto tokens = lex_all("a +\n b\nc"); + REQUIRE(tokens.size() == 5); + CHECK(tokens[0].line == 1); + CHECK(tokens[0].column == 1); // a + CHECK(tokens[1].line == 1); + CHECK(tokens[1].column == 3); // + + CHECK(tokens[2].line == 2); + CHECK(tokens[2].column == 3); // b, past two spaces + CHECK(tokens[3].line == 3); + CHECK(tokens[3].column == 1); // c +} + +TEST_CASE("some lexer token stream", "[lexer]") +{ + check_tokens(" not x <= 1.0e-4 and y + 5=1", { + {TokenTypes::Bang, "!"}, + {TokenTypes::Identifier, "x"}, + {TokenTypes::LessEq, "<="}, + {TokenTypes::Float, "1.0e-4"}, + {TokenTypes::And, "and"}, + {TokenTypes::Identifier, "y"}, + {TokenTypes::Plus, "+"}, + {TokenTypes::Integer, "5"}, + {TokenTypes::Assign, "="}, + {TokenTypes::Integer, "1"}, + {TokenTypes::EndofFile, ""}, + }); +} + +} // namespace dexpr diff --git a/share/dexpr/tests/test_list_supported_functions.cpp b/share/dexpr/tests/test_list_supported_functions.cpp new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/share/dexpr/tests/test_parser.cpp b/share/dexpr/tests/test_parser.cpp new file mode 100644 index 000000000000..b62cfa3adf0d --- /dev/null +++ b/share/dexpr/tests/test_parser.cpp @@ -0,0 +1,315 @@ +#include "catch2/catch_message.hpp" +#include +#include +#include +#include +#include +#include + +namespace dexpr { + +namespace { // anonymous + +std::string parse_to_string(const std::string& input) { + parser::Parser parser{Lexer{input}}; + const auto expr = parser.parse(); + REQUIRE(expr != nullptr); + return ast::to_string(*expr); +} + +void check_parse(const std::string& input, const std::string& expected) { + INFO("Input: " << input); + CHECK(parse_to_string(input) == expected); +} + +// Returns the message of the ParserError raised by `input`, or "" if it parsed. +std::string parse_error(const std::string& input) { + try { + parse_to_string(input); + } catch (const parser::ParserError& e) { + return e.what(); + } + return ""; +} + +void check_rejected(const std::string& input, const std::string& substring) { + INFO("Input: " << input); + const auto msg = parse_error(input); + CHECK_FALSE(msg.empty()); + CHECK(msg.find(substring) != std::string::npos); +} + +} // namespace + +TEST_CASE("parser: bare literals and identifiers", "[parser]") { + check_parse("x", "x"); + check_parse("42", "42"); + check_parse("'col'", "'col'"); +} + +TEST_CASE("parser: field names containing digits", "[parser]") { + check_parse("bc_a1", "bc_a1"); + check_parse("O3", "O3"); + check_parse("bc_a1 + so4_a2", "(bc_a1+so4_a2)"); + check_parse("O3.mean(dim='lev')", "O3.mean((dim='lev'))"); + check_parse("bc_a1.interp(plev=500, units='hPa')", + "bc_a1.interp((plev=500), (units='hPa'))"); +} + +TEST_CASE("parser: simple infix expressions", "[parser]") { + check_parse("x + 1", "(x+1)"); + check_parse("x * y", "(x*y)"); + check_parse("x = y", "(x=y)"); +} + +TEST_CASE("parser: multiplication binds tighter than addition", "[parser]") { + check_parse("1 + 2 * 3", "(1+(2*3))"); + check_parse("1 * 2 + 3", "((1*2)+3)"); +} + +TEST_CASE("parser: comparison binds tighter than equality", "[parser]") { + check_parse("a < b == c", "((a= y", "(x>=y)"); + check_parse("x <= y", "(x<=y)"); +} + +TEST_CASE("parser: division and strict comparison print correctly", + "[parser]") { + check_parse("x / y", "(x/y)"); + check_parse("x < y", "(x y", "(x>y)"); +} + +TEST_CASE("parser: equality operators print back", "[parser]") { + check_parse("x == y", "(x==y)"); + check_parse("x != y", "(x!=y)"); + // The printed form has to lex again as the same operator, not as '!' + CHECK(parse_to_string("(x!=y)") == "(x!=y)"); +} + +TEST_CASE("parser: arithmetic binds tighter than comparison", "[parser]") { + check_parse("a + b < c", "((a+b) y", "((-x)>y)"); + check_parse("not x and y", "((!x) and y)"); +} + +TEST_CASE("parser: unary applies to grouped and called operands", "[parser]") { + check_parse("-(1+2)", "(-(1+2))"); + check_parse("not (x > 0)", "(!(x>0))"); + check_parse("-f(x)", "(-f(x))"); + check_parse("f(-x)", "f((-x))"); + check_parse("[-1, -2]", "[(-1), (-2)]"); +} + +TEST_CASE("parser: call expressions", "[parser]") { + check_parse("f()", "f()"); + check_parse("f('a', 2, x)", "f('a', 2, x)"); + check_parse("a.b(1).c", "a.b(1).c"); +} + +TEST_CASE("parser: member access and method calls", "[parser]") { + check_parse("x.foo", "x.foo"); + check_parse("x.sum()", "x.sum()"); + check_parse("x.where(y>0)", "x.where((y>0))"); + // Grouping looser than '.' is still shown. + check_parse("(a+b).c", "(a+b).c"); + check_parse("f(x).y", "f(x).y"); + check_parse("f(x.y)", "f(x.y)"); +} + +TEST_CASE("parser: float literals keep their fractional part", "[parser]") { + check_parse("1.5", "1.5"); + check_parse("1.5 + 2.5", "(1.5+2.5)"); + check_parse("0.1", "0.1"); + check_parse("2.5e-3", "0.0025"); + check_parse("3.14159265", "3.14159265"); // no longer rounded to float + check_parse("1", "1"); +} + +TEST_CASE("parser: floats print in a form that lexes back as a float", + "[parser]") { + check_parse("1.5e3", "1500.0"); + check_parse("100.0", "100.0"); + check_parse("0.0", "0.0"); + check_parse("1e30", "1e+30"); + check_parse("1.0e-9", "1e-09"); + CHECK(parse_to_string("1500.0") == "1500.0"); + CHECK(parse_to_string("1e+30") == "1e+30"); + CHECK(parse_to_string("1e-09") == "1e-09"); +} + +// Printing must produce something that parses back to the same value. +TEST_CASE("parser: printing a float is the inverse of parsing it", "[parser]") { + for (const auto* input : { + "1.5", + "0.1", + "0.0025", + "273.15", + "3.14159265", + "1500.0", + "0.0", + "1e+30", + "1e-09", + "1e+40", + "1e-40", + // Values that only survive at full double precision + "1.0000000000000002", + "2.2250738585072014e-308", + "1.7976931348623157e+308", + }) { + INFO("Input: " << input); + const auto once = parse_to_string(input); + const auto twice = parse_to_string(once); + CHECK(once == twice); + } +} + +TEST_CASE("parser: array literals", "[parser]") { + check_parse("[]", "[]"); + check_parse("[1]", "[1]"); + check_parse("[1, 2, 3]", "[1, 2, 3]"); + check_parse("[[1,2],[3]]", "[[1, 2], [3]]"); +} + +TEST_CASE("parser: binary operators are left-associative", "[parser]") { + check_parse("a.b.c", "a.b.c"); + check_parse("a and b and c", "((a and b) and c)"); +} + +TEST_CASE("parser: exponentiation is right-associative", "[parser]") { + check_parse("2 ** 3 ** 2", "(2**(3**2))"); + check_parse("2 ** -x", "(2**(-x))"); +} + +TEST_CASE("parser: exponentiation outranks unary and arithmetic", "[parser]") { + check_parse("-x ** 2", "(-(x**2))"); + check_parse("-x ** -y", "(-(x**(-y)))"); + check_parse("x ** 2 * 3", "((x**2)*3)"); + check_parse("x * y ** 2", "(x*(y**2))"); + check_parse("a ** b + c", "((a**b)+c)"); +} + +TEST_CASE("parser: attribute access outranks unary", "[parser]") { + check_parse("-x.y", "(-x.y)"); + check_parse("-a.b.c", "(-a.b.c)"); + check_parse("not x.y", "(!x.y)"); +} + +TEST_CASE("parser: unbalanced delimiters are reported", "[parser]") { + CHECK_THROWS_AS(parse_to_string("(1 + 2"), parser::ParserError); + CHECK_THROWS_AS(parse_to_string("f(x"), parser::ParserError); + CHECK_THROWS_AS(parse_to_string("[1,2"), parser::ParserError); + // The expected/actual pair is readable rather than run together. + check_rejected("(1 + 2", "Expected RightParen, got"); +} + +TEST_CASE("parser: unparseable input throws", "[parser]") { + CHECK_THROWS_AS(parse_to_string("@"), parser::ParserError); + check_rejected("@", "Illegal token"); + check_rejected(".x", "Unexpected Prefix Token"); + check_rejected("", "Unexpected Prefix Token"); +} + +TEST_CASE("parser: input must be consumed in full", "[parser]") { + // Without this check a typo parses as its leading fragment and the rest is + // silently dropped -- "x y" would quietly evaluate as "x". + check_rejected("x y", "Unexpected trailing input"); + check_rejected("1 2", "Unexpected trailing input"); + check_rejected("x, y", "Unexpected trailing input"); + check_rejected("f(x) g(y)", "Unexpected trailing input"); + // Colon has a precedence but no infix handler, so it ends the expression. + check_rejected("x:y", "Unexpected trailing input"); +} + +TEST_CASE("parser: illegal tokens are reported, not printed", "[parser]") { + // These used to write to std::cout and let the parse succeed on the prefix. + check_rejected("x % y", "Illegal token"); + check_rejected("x; y", "Illegal token"); +} + +TEST_CASE("parser: out-of-range literals are parser errors", "[parser]") { + // Previously escaped as a bare std::out_of_range from stoi/stof. + CHECK_THROWS_AS(parse_to_string("2147483648"), parser::ParserError); + check_rejected("2147483648", "Integer literal out of range"); + check_rejected("1e400", "Float literal out of range"); + // The largest int still parses. + check_parse("2147483647", "2147483647"); +} + +// Literals are held as double, so the range and precision of a threshold +// survive parsing. As float, 1e40 overflowed and 273.15 came back as +// 273.14999389648438. +TEST_CASE("parser: literals keep double range and precision", "[parser]") { + check_parse("1e40", "1e+40"); + check_parse("1e-40", "1e-40"); + check_parse("273.15", "273.15"); + check_parse("0.1", "0.1"); + // 17 significant digits round-trip; a float would have flattened these two + // onto the same value. + check_parse("1.0000000000000002", "1.0000000000000002"); + check_parse("2.2250738585072014e-308", "2.2250738585072014e-308"); +} + +TEST_CASE("parser: errors say where they happened", "[parser]") { + // 1234567 + check_rejected("x + + y", "line 1, column 5"); + check_rejected("(1 + 2", "line 1, column 7"); + check_rejected("x y", "line 1, column 3"); + check_rejected("a @ b", "line 1, column 3"); + // Position is reported for the offending token, not for the start of input + check_rejected("foo(a, b))", "line 1, column 10"); +} + +TEST_CASE("parser: check string literals") { + check_parse(R"(name == "hello")", R"((name=='hello'))"); + check_parse(R"(name == 'hello')", R"((name=='hello'))"); + check_parse(R"(name == "It's me")", R"((name=="It's me"))"); + check_rejected(R"(name == "say "hello"")","Unexpected trailing input"); +} + +TEST_CASE("some parsed expression", "[parser]") { + check_parse("x*y.derivative(dx=dy,['col']).where(x>0)", + "(x*y.derivative((dx=dy), ['col']).where((x>0)))"); + check_parse("x.where(condition=a < b and c != d)", + "x.where((condition=((a +#include + +#include + +namespace { + +void print_functions() { + std::cout << "Supported functions\n\n"; + + for (const auto& function : dexpr::supported) { + std::cout << " " << function << '\n'; + } +} + +void print_help() { + std::cout << +R"(Usage: + dexpr functions + dexpr help +)"; +} + +} // namespace + +int main(int argc, char* argv[]) { + if (argc == 1) { + print_help(); + return 0; + } + + std::string_view command{argv[1]}; + + if (command == "functions") { + print_functions(); + return 0; + } + + if (command == "help") { + print_help(); + return 0; + } + + std::cerr << "Unknown command: " << command << '\n'; + return 1; +}