Extensible SQL Lexer and Parser in MoonBit
Dependencies
// Parse a simple SELECT statement
let sql = "SELECT name, age FROM users WHERE age > 18"
let statements = parse_sql(sql)
let pretty_output = statements.stmts[0] |> pretty_print
println(pretty_output)
// Parse complex queries with CTEs
let complex_sql = """
WITH sales AS (
SELECT user_id, SUM(amount) as total
FROM orders
WHERE date > '2023-01-01'
GROUP BY user_id
)
SELECT u.name, s.total
FROM users u
JOIN sales s ON u.id = s.user_id
ORDER BY s.total DESC
LIMIT 10
"""
let result = parse_sql(complex_sql)
// Parse transaction control statements
let tcl_sql = "BEGIN TRANSACTION; INSERT INTO users VALUES ('Alice'); COMMIT;"
let tcl_statements = parse_sql(tcl_sql)
// Parse data control statements
let dcl_sql = "GRANT SELECT, INSERT ON users TO alice WITH GRANT OPTION"
let dcl_result = parse_sql(dcl_sql)
// Dialect-specific parsing
let mysql_sql = "SHOW TABLES LIKE 'user%'"
let mysql_result = parse_sql(dialect=MySQL::{}, mysql_sql)moon testpub trait Dialect {
supports_string_literal_backslash_escape(Self) -> Bool
supports_boolean_literals(Self) -> Bool
supports_filter_during_aggregation(Self) -> Bool
supports_within_after_array_aggregation(Self) -> Bool
requires_column_types_in_create_table(Self) -> Bool
supports_if_not_exists(Self) -> Bool
supports_double_quoted_identifiers(Self) -> Bool
supports_array_syntax(Self) -> Bool
supports_named_parameters(Self) -> Bool
parse_expr(Self, tokens : ArrayView[Token]) -> (Expr, ArrayView[Token])? raise ParserError
parse_statement(Self, parser : Parser, tokens : ArrayView[Token]) -> (Statement, ArrayView[Token])? raise ParserError
read_keyword(Self, word : String) -> Keyword?
}pub(all) struct ANSI {
}impl Pretty for AccessExprimpl Eq for AccessExprimpl Show for AccessExprpub(all) enum AlterIndexOperation {
RenameTo(String)
SetTablespace(String)
Reset(Array[String])
Set(Array[IndexParameter])
}impl Pretty for AlterIndexOperationimpl Eq for AlterIndexOperationimpl Show for AlterIndexOperationimpl Pretty for AlterIndexStmtimpl Eq for AlterIndexStmtimpl Show for AlterIndexStmtpub(all) enum AlterTableOperation {
DropColumn(String, Bool)
}impl Pretty for AlterTableOperationimpl Eq for AlterTableOperationimpl Show for AlterTableOperationpub(all) struct AlterTableStmt {
table_name : ObjectName
if_exists : Bool
operation : AlterTableOperation
}impl Pretty for AlterTableStmtimpl Eq for AlterTableStmtimpl Show for AlterTableStmtimpl Pretty for Assignmentimpl Eq for Assignmentimpl Show for Assignmentpub(all) struct BeginStmt {
work : Bool
transaction : Bool
}pub(all) struct BigQuery {
}pub(all) enum BinaryOperator {
Eq
Neq
Lt
Gt
LtEq
GtEq
Spaceship
Plus
Minus
Mul
Div
IntegerDiv
Mod
And
Or
JsonExtract
JsonExtractText
JsonExtractPath
JsonExtractPathText
JsonContains
JsonContainedIn
}impl Pretty for BinaryOperatorimpl Eq for BinaryOperatorimpl Show for BinaryOperatorpub(all) struct ClickHouse {
}impl Dialect for ClickHousefn parse_expr(_self : ClickHouse, _tokens : ArrayView[Token]) -> (Expr, ArrayView[Token])? raise ParserErrorfn parse_statement(_self : ClickHouse, _parser : Parser, _tokens : ArrayView[Token]) -> (Statement, ArrayView[Token])? raise ParserErrorimpl Pretty for ColumnDefOptionimpl Eq for ColumnDefOptionimpl Show for ColumnDefOptionpub(all) struct CommitStmt {
work : Bool
transaction : Bool
}impl Pretty for CommitStmtimpl Eq for CommitStmtimpl Show for CommitStmtimpl Pretty for ConflictActionimpl Eq for ConflictActionimpl Show for ConflictActionimpl Pretty for ConflictTargetimpl Eq for ConflictTargetimpl Show for ConflictTargetpub(all) enum CopyDirection {
To
From
}impl Pretty for CopyDirectionimpl Eq for CopyDirectionimpl Show for CopyDirectionimpl Pretty for CopyForceQuoteimpl Eq for CopyForceQuoteimpl Show for CopyForceQuotepub(all) enum CopyFormat {
Csv
Text
Binary
}impl Pretty for CopyFormatimpl Eq for CopyFormatimpl Show for CopyFormatpub(all) enum CopyOption {
Format(CopyFormat)
Delimiter(String)
Null(String)
Header(Bool?)
Quote(String)
Escape(String)
ForceQuote(CopyForceQuote)
ForceNotNull(Array[String])
ForceNull(Array[String])
Encoding(String)
}impl Pretty for CopyOptionimpl Eq for CopyOptionimpl Show for CopyOptionimpl Pretty for CopySourceimpl Eq for CopySourceimpl Show for CopySourcepub(all) struct CopyStmt {
source : CopySource
direction : CopyDirection
target : CopyTarget
format_options : Array[CopyOption]
}pub(all) enum CopyTarget {
File(String)
Stdin
Stdout
Program(String)
}impl Pretty for CopyTargetimpl Eq for CopyTargetimpl Show for CopyTargetpub(all) struct CreateDatabaseStmt {
name : String
if_not_exists : Bool
character_set : String?
collate : String?
}impl Pretty for CreateDatabaseStmtimpl Eq for CreateDatabaseStmtimpl Show for CreateDatabaseStmtpub(all) struct CreateFunctionStmt {
name : String
parameters : Array[FunctionParameter]
return_type : DataType?
language : String?
body : String?
deterministic : Bool
if_not_exists : Bool
}impl Pretty for CreateFunctionStmtimpl Eq for CreateFunctionStmtimpl Show for CreateFunctionStmtpub(all) struct CreateIndexStmt {
unique : Bool
concurrently : Bool
if_not_exists : Bool
name : String
table_name : ObjectName
index_method : IndexMethod?
columns : Array[IndexColumn]
where_clause : Expr?
}impl Pretty for CreateIndexStmtimpl Eq for CreateIndexStmtimpl Show for CreateIndexStmtpub(all) struct CreateProcedureStmt {
name : String
parameters : Array[FunctionParameter]
language : String?
body : String?
if_not_exists : Bool
}impl Pretty for CreateProcedureStmtimpl Eq for CreateProcedureStmtimpl Show for CreateProcedureStmtpub(all) struct CreateSchemaStmt {
name : String
if_not_exists : Bool
authorization : String?
}impl Pretty for CreateSchemaStmtimpl Eq for CreateSchemaStmtimpl Show for CreateSchemaStmtpub(all) struct CreateSequenceStmt {
name : String
if_not_exists : Bool
temporary : Bool
increment : Int?
minvalue : SequenceLimit?
maxvalue : SequenceLimit?
start_with : Int?
cache : Int?
cycle : Bool?
owned_by : SequenceOwnedBy?
}impl Pretty for CreateSequenceStmtimpl Eq for CreateSequenceStmtimpl Show for CreateSequenceStmtpub(all) enum CreateTableDefinition {
Columns(Array[ColumnDef], Array[TableConstraint])
AsQuery(QueryStmt)
}impl Pretty for CreateTableDefinitionimpl Eq for CreateTableDefinitionimpl Show for CreateTableDefinitionpub(all) struct CreateTableStmt {
name : String
if_not_exists : Bool
definition : CreateTableDefinition
}impl Pretty for CreateTableStmtimpl Eq for CreateTableStmtimpl Show for CreateTableStmttype CreateViewStmtimpl Pretty for CreateViewStmtimpl Eq for CreateViewStmtimpl Show for CreateViewStmtpub(all) enum DataType {
Integer
Smallint
Bigint
Float(Int?)
Real
Double
Char(Int)
Varchar(Int)
Text
Boolean
Timestamp
Blob
}pub(all) enum DatetimeUnit {
Year
Month
Day
Hour
Minute
Second
}impl Pretty for DatetimeUnitimpl Eq for DatetimeUnitimpl Show for DatetimeUnitpub(all) struct DeallocateStmt {
name : String
prepare : Bool
}impl Pretty for DeallocateStmtimpl Eq for DeallocateStmtimpl Show for DeallocateStmtimpl Pretty for DeleteStmtimpl Eq for DeleteStmtimpl Show for DeleteStmtpub(all) struct DropIndexStmt {
name : String
if_exists : Bool
concurrently : Bool
table_name : ObjectName?
}impl Pretty for DropIndexStmtimpl Eq for DropIndexStmtimpl Show for DropIndexStmtimpl Pretty for DropTableStmtimpl Eq for DropTableStmtimpl Show for DropTableStmtpub(all) struct DropViewStmt {
name : String
}impl Pretty for DropViewStmtimpl Eq for DropViewStmtimpl Show for DropViewStmtpub(all) struct DuckDB {
}pub(all) enum DuplicateTreatment {
Distinct
All
}impl Pretty for DuplicateTreatmentimpl Eq for DuplicateTreatmentimpl Show for DuplicateTreatmentimpl Pretty for ExecuteStmtimpl Eq for ExecuteStmtimpl Show for ExecuteStmtpub(all) enum Expr {
Identifier(String)
CompoundIdentifier(Array[String])
Literal(Literal)
BinaryOperation(Expr, BinaryOperator, Expr)
UnaryOperation(UnaryOperator, Expr)
FunctionCall(String, DuplicateTreatment?, Array[Expr], Expr?)
Wildcard
Datetime(String)
Interval(String, IntervalQualifier)
Like(Bool, Expr, Expr)
ILike(Bool, Expr, Expr)
SubQuery(QueryStmt)
Exists(Bool, QueryStmt)
Between(Bool, Expr, Expr, Expr)
Extract(PrimaryDatetimeField, Expr)
Case(CaseExpr)
InList(Bool, Expr, Array[Expr])
InSubQuery(Bool, Expr, QueryStmt)
Substring(Expr, Expr?, Expr?)
Placeholder(String)
Array(ArrayExpr)
CompoundFieldAccess(Expr, Array[AccessExpr])
WindowFunction(String, DuplicateTreatment?, Array[Expr], WindowSpec)
}impl Pretty for FunctionParameterimpl Eq for FunctionParameterimpl Show for FunctionParameterpub(all) struct Generic {
}pub(all) struct GrantStmt {
privileges : Array[Privilege]
objects : Array[ObjectName]
grantees : Array[String]
with_grant_option : Bool
}impl Pretty for IndexColumnimpl Eq for IndexColumnimpl Show for IndexColumnpub(all) enum IndexMethod {
Btree
Hash
Gin
Gist
Spgist
Brin
}impl Pretty for IndexMethodimpl Eq for IndexMethodimpl Show for IndexMethodpub(all) struct IndexParameter {
name : String
value : String
}impl Pretty for IndexParameterimpl Eq for IndexParameterimpl Show for IndexParameterimpl Pretty for InsertSourceimpl Eq for InsertSourceimpl Show for InsertSourcepub(all) struct InsertStmt {
table_name : ObjectName
columns : Array[String]
source : InsertSource
or : SqliteOnConflict?
on : OnInsert?
}impl Pretty for InsertStmtimpl Eq for InsertStmtimpl Show for InsertStmtpub(all) enum IntervalQualifier {
Single(PrimaryDatetimeField)
Range(PrimaryDatetimeField, PrimaryDatetimeField)
}impl Pretty for IntervalQualifierimpl Eq for IntervalQualifierimpl Show for IntervalQualifierimpl Pretty for JoinConstraintimpl Eq for JoinConstraintimpl Show for JoinConstraintpub(all) enum JoinOperator {
Join(JoinConstraint)
Left(JoinConstraint)
LeftOuter(JoinConstraint)
Right(JoinConstraint)
RightOuter(JoinConstraint)
Full(JoinConstraint)
FullOuter(JoinConstraint)
Inner(JoinConstraint)
Cross
}impl Pretty for JoinOperatorimpl Eq for JoinOperatorimpl Show for JoinOperatorpub(all) enum Keyword {
Select
From
Where
As
Group
Order
By
Asc
Desc
Nulls
First
Last
Year
Month
Day
Hour
Minute
Second
Date
Interval
To
Like
Not
ILike
Exists
Between
And
Or
Extract
Filter
Case
When
Matched
Target
Then
Else
Having
End
In
Join
Left
Right
Full
Outer
Inner
Cross
On
Using
Limit
Offset
Create
Table
Integer
Int
Smallint
Bigint
Real
Double
Precision
Char
Character
Collate
Authorization
Varchar
Varing
Text
Time
Boolean
Float
Timestamp
Blob
Null
Default
Unique
View
Drop
Distinct
All
Substring
For
Primary
Key
Foreign
References
Check
Union
Intersect
Except
Top
Insert
Into
Values
Merge
If
Delete
Update
Set
Replace
Rollback
Abort
Fail
Ignore
Truncate
Alter
Column
Show
Tables
Columns
Status
Databases
Database
Div
Lock
Unlock
Listen
Notify
Schemas
Schema
Variables
Processlist
Grants
Functions
Function
Extended
Global
Session
Procedure
Returns
Language
Deterministic
Event
Trigger
Duplicate
Conflict
Do
Nothing
Constraint
Array
With
Begin
Start
Transaction
Commit
Savepoint
Release
Work
Grant
Revoke
Privileges
Usage
Use
Execute
Prepare
Deallocate
Immediate
Connect
Temporary
Temp
Option
Restrict
Cascade
Index
Copy
Format
Stdin
Stdout
Program
Header
ForceQuote
ForceNotNull
ForceNull
Encoding
Load
Data
Local
Infile
Fields
Lines
Terminated
Enclosed
Escaped
Starting
Optionally
Rename
Tablespace
Reset
Btree
Hash
Gin
Gist
Spgist
Brin
Concurrently
Over
Window
Partition
Rows
Range
Preceding
Following
Current
Unbounded
Row
Out
InOut
Sequence
Increment
Minvalue
Maxvalue
Cache
Cycle
Owned
No
}pub(all) enum Literal {
Integer(Int)
Double(Double)
String(String)
Boolean(Bool)
Null
}pub(all) enum LoadDataDuplicateHandling {
Replace
Ignore
}impl Pretty for LoadDataDuplicateHandlingimpl Eq for LoadDataDuplicateHandlingimpl Show for LoadDataDuplicateHandlingpub(all) struct LoadDataFieldsOptions {
terminated_by : String?
enclosed_by : String?
optionally_enclosed : Bool
escaped_by : String?
}impl Pretty for LoadDataFieldsOptionsimpl Eq for LoadDataFieldsOptionsimpl Show for LoadDataFieldsOptionspub(all) struct LoadDataLinesOptions {
starting_by : String?
terminated_by : String?
}impl Pretty for LoadDataLinesOptionsimpl Eq for LoadDataLinesOptionsimpl Show for LoadDataLinesOptionspub(all) struct LoadDataStmt {
is_local : Bool
filename : String
duplicate_handling : LoadDataDuplicateHandling?
table_name : ObjectName
character_set : String?
fields_options : LoadDataFieldsOptions?
lines_options : LoadDataLinesOptions?
ignore_lines : Int?
columns : Array[String]?
set_assignments : Array[Assignment]?
}impl Pretty for LoadDataStmtimpl Eq for LoadDataStmtimpl Show for LoadDataStmtimpl Pretty for MergeActionimpl Eq for MergeActionimpl Show for MergeActionpub(all) enum MergeMatchType {
Matched
NotMatched
}impl Pretty for MergeMatchTypeimpl Eq for MergeMatchTypeimpl Show for MergeMatchTypeimpl Pretty for MergeSourceimpl Eq for MergeSourceimpl Show for MergeSourcepub(all) struct MergeStmt {
target_table : ObjectName
target_alias : String?
source : MergeSource
source_alias : String?
join_condition : Expr
when_clauses : Array[MergeWhenClause]
}pub(all) struct MergeWhenClause {
match_type : MergeMatchType
condition : Expr?
action : MergeAction
}impl Pretty for MergeWhenClauseimpl Eq for MergeWhenClauseimpl Show for MergeWhenClausepub(all) struct MySQL {
}impl Pretty for ObjectNameimpl Eq for ObjectNameimpl Show for ObjectNamepub(all) struct OnConflictClause {
conflict_target : ConflictTarget?
conflict_action : ConflictAction
}impl Pretty for OnConflictClauseimpl Eq for OnConflictClauseimpl Show for OnConflictClauseimpl Pretty for OrderByExprimpl Eq for OrderByExprimpl Show for OrderByExprpub(all) enum ParameterMode {
In
Out
InOut
}impl Pretty for ParameterModeimpl Eq for ParameterModeimpl Show for ParameterModepub(all) struct Postgres {
}pub(all) enum Precedence {
PlusMinus
MulDivMod
Eq
Like
And
Or
Between
UnaryNot
JsonOperator
}impl Pretty for PrepareStmtimpl Eq for PrepareStmtimpl Show for PrepareStmttype PrimaryDatetimeFieldimpl Pretty for PrimaryDatetimeFieldimpl Eq for PrimaryDatetimeFieldimpl Show for PrimaryDatetimeFieldimpl Pretty for Projectionimpl Eq for Projectionimpl Show for Projectionpub(all) struct Redshift {
}pub(all) struct ReleaseSavepointStmt {
savepoint_keyword : Bool
name : String
}impl Pretty for ReleaseSavepointStmtimpl Eq for ReleaseSavepointStmtimpl Show for ReleaseSavepointStmtpub(all) enum RevokeOption {
Restrict
Cascade
}impl Pretty for RevokeOptionimpl Eq for RevokeOptionimpl Show for RevokeOptionpub(all) struct RevokeStmt {
grant_option_for : Bool
privileges : Array[Privilege]
objects : Array[ObjectName]
grantees : Array[String]
cascade : RevokeOption?
}impl Pretty for RevokeStmtimpl Eq for RevokeStmtimpl Show for RevokeStmtpub(all) struct RollbackStmt {
work : Bool
transaction : Bool
savepoint : String?
}impl Pretty for RollbackStmtimpl Eq for RollbackStmtimpl Show for RollbackStmtpub(all) struct SQLite {
}pub(all) struct SavepointStmt {
name : String
}impl Pretty for SavepointStmtimpl Eq for SavepointStmtimpl Show for SavepointStmttype SelectStmtimpl Pretty for SelectStmtimpl Eq for SelectStmtimpl Show for SelectStmtpub(all) enum SequenceLimit {
Value(Int)
NoLimit
}impl Pretty for SequenceLimitimpl Eq for SequenceLimitimpl Show for SequenceLimitimpl Pretty for SequenceOwnedByimpl Eq for SequenceOwnedByimpl Show for SequenceOwnedByimpl Pretty for SetAssignmentimpl Eq for SetAssignmentimpl Show for SetAssignmenttype SetExprtype SetOperatorimpl Pretty for SetOperatorimpl Eq for SetOperatorimpl Show for SetOperatorpub(all) enum SetScope {
Global
Session
UserVar
Local
}impl Pretty for ShowFilterimpl Eq for ShowFilterimpl Show for ShowFilterpub(all) struct ShowStmt {
show_type : ShowType
object : ObjectName?
filter : ShowFilter?
extended : Bool
full : Bool
global_scope : Bool
}pub(all) enum ShowType {
Tables
Columns
Status
Databases
Schemas
Variables
Processlist
Grants
Functions
CreateTable
CreateView
CreateFunction
CreateProcedure
CreateEvent
CreateTrigger
}pub(all) struct Snowflake {
}pub(all) enum SqliteOnConflict {
Rollback
Abort
Fail
Ignore
Replace
}impl Pretty for SqliteOnConflictimpl Eq for SqliteOnConflictimpl Show for SqliteOnConflictpub(all) enum Statement {
Query(QueryStmt)
CreateTable(CreateTableStmt)
CreateView(CreateViewStmt)
CreateIndex(CreateIndexStmt)
CreateDatabase(CreateDatabaseStmt)
CreateSchema(CreateSchemaStmt)
CreateFunction(CreateFunctionStmt)
CreateProcedure(CreateProcedureStmt)
CreateSequence(CreateSequenceStmt)
DropView(DropViewStmt)
DropTable(DropTableStmt)
DropIndex(DropIndexStmt)
Insert(InsertStmt)
Delete(DeleteStmt)
Update(UpdateStmt)
Merge(MergeStmt)
Truncate(TruncateStmt)
AlterTable(AlterTableStmt)
AlterIndex(AlterIndexStmt)
Show(ShowStmt)
Set(SetStmt)
Use(UseStmt)
Copy(CopyStmt)
LoadData(LoadDataStmt)
Prepare(PrepareStmt)
Execute(ExecuteStmt)
Deallocate(DeallocateStmt)
LockTables(Array[ObjectName])
UnlockTables
Listen(String)
Notify(String, String?)
Begin(BeginStmt)
Commit(CommitStmt)
Rollback(RollbackStmt)
Savepoint(SavepointStmt)
ReleaseSavepoint(ReleaseSavepointStmt)
Grant(GrantStmt)
Revoke(RevokeStmt)
}impl Pretty for Statementsimpl Eq for Statementsimpl Show for Statementsimpl Pretty for TableAliasimpl Eq for TableAliasimpl Show for TableAliaspub(all) enum TableConstraint {
Unique(Array[OrderByExpr])
PrimaryKey(Array[OrderByExpr])
ForeignKey(Array[OrderByExpr], ObjectName, Array[String])
Check(Expr)
}impl Pretty for TableConstraintimpl Eq for TableConstraintimpl Show for TableConstraintimpl Pretty for TableFactorimpl Eq for TableFactorimpl Show for TableFactorpub(all) enum Token {
Keyword(Keyword)
Identifier(String)
Number(String)
StringLiteral(String)
Boolean(Bool)
Comma
Semicolon
Colon
Eq
DoubleEq
Neq
Lt
Gt
LtEq
GtEq
Spaceship
Plus
Minus
Mul
Div
Mod
LBracket
RBracket
LBrace
RBrace
LParen
RParen
Period
Placeholder(String)
JsonExtract
JsonExtractText
JsonExtractPath
JsonExtractPathText
JsonContains
JsonContainedIn
Unknown(Char)
Eof
}impl Pretty for TruncateStmtimpl Eq for TruncateStmtimpl Show for TruncateStmtpub(all) enum UnaryOperator {
Plus
Minus
Not
}impl Pretty for UnaryOperatorimpl Eq for UnaryOperatorimpl Show for UnaryOperatorpub(all) struct UpdateStmt {
table_name : ObjectName
assignments : Array[Assignment]
where_clause : Expr?
}impl Pretty for UpdateStmtimpl Eq for UpdateStmtimpl Show for UpdateStmtpub(all) struct ViewColumnDef {
name : String
}impl Pretty for ViewColumnDefimpl Eq for ViewColumnDefimpl Show for ViewColumnDefimpl Pretty for WindowFrameBoundimpl Eq for WindowFrameBoundimpl Show for WindowFrameBoundpub(all) struct WindowFrameClause {
frame_units : WindowFrameUnits
frame_start : WindowFrameBound
frame_end : WindowFrameBound?
}impl Pretty for WindowFrameClauseimpl Eq for WindowFrameClauseimpl Show for WindowFrameClausepub(all) enum WindowFrameUnits {
Rows
Range
}impl Eq for WindowFrameUnitsimpl Show for WindowFrameUnitspub(all) struct WindowSpec {
partition_by : Array[Expr]
order_by : Array[OrderByExpr]
frame_clause : WindowFrameClause?
}impl Pretty for WindowSpecimpl Eq for WindowSpecimpl Show for WindowSpecExtensible SQL Lexer and Parser in MoonBit
Dependencies