Incremental, chunk-invariant parsing of streaming JSON tool-call arguments.
///|
test {
let parser = Parser::new()
// 逐块喂入已经提取出来的参数片段字节
let first = parser.feed(b"{\"city\":\"上")
inspect(first.length(), content="3")
let second = parser.feed(b"海\",\"count\":12}")
inspect(second.length(), content="5")
// 只有正常结束才可能得到可交付的文档
let (_, result) = parser.finish()
match result {
FinishResult::Completed(doc) =>
inspect(doc.to_string(), content="{\"city\":\"上海\",\"count\":12}")
_ => fail("应当完成")
}
}///|
test {
let limits = Limits::new(max_depth=3)
let parser = Parser::new(limits~)
let failed = try {
let _ = parser.feed(b"[[[[1]]]]")
false
} catch {
ParseError::LimitExceeded(..) => true
_ => false
} noraise {
_ => false
}
inspect(failed, content="true")
}///|
test {
let path = Path::root().child_key("items").child_index(0).child_key("name")
inspect(path.to_string(), content="$.items[0].name")
}///|
test {
let parser = Parser::new()
let preview = @preview.Preview::new()
for event in parser.feed(b"{\"city\":\"上") {
preview.apply(event)
}
// 字符串正在增长,不是最终值
inspect(preview.to_preview_string(), content="{\"city\":\"上…,…}")
inspect(
preview.lookup(Path::root().child_key("city"))
is @preview.NodeState::IncompleteString(..),
content="true",
)
}///|
test {
let session = @session.Session::new()
let first = @session.CallKey::new(response="resp-1", choice=0, index=0)
let second = @session.CallKey::new(response="resp-1", choice=0, index=1)
session.open(first, id=Some("call_a"), name=Some("create_ticket"))
session.open(second, id=Some("call_b"), name=Some("send_mail"))
// 片段交错到达
let _ = session.feed(first, b"{\"title\":\"登")
let _ = session.feed(second, b"{\"to\":\"a@b.c\"}")
let _ = session.feed(first, b"录失败\"}")
let (_, first_result) = session.finish(first)
let (_, second_result) = session.finish(second)
inspect(
match first_result {
FinishResult::Completed(doc) => doc.to_string()
_ => "?"
},
content="{\"title\":\"登录失败\"}",
)
inspect(
match second_result {
FinishResult::Completed(doc) => doc.to_string()
_ => "?"
},
content="{\"to\":\"a@b.c\"}",
)
}pub(all) suberror ParseError {
InvalidSyntax(pos~ : Int, expected~ : String, found~ : String)
InvalidUtf8(pos~ : Int, kind~ : Utf8ErrorKind)
InvalidEscape(pos~ : Int, detail~ : String)
InvalidNumber(pos~ : Int, lexeme~ : String)
InvalidUnicodeEscape(pos~ : Int, detail~ : String)
DuplicateKey(pos~ : Int, key~ : String, path~ : Path)
TrailingContent(pos~ : Int)
LimitExceeded(kind~ : LimitKind, pos~ : Int, allowed~ : Int, actual~ : Int)
AlreadyFinished(op~ : String)
InvalidLimits(detail~ : String)
} derive(Eq, ToJson, Debug)pub(all) enum Event {
ObjectBegin(path~ : Path, span~ : Span)
ObjectEnd(path~ : Path, span~ : Span)
ArrayBegin(path~ : Path, span~ : Span)
ArrayEnd(path~ : Path, span~ : Span)
KeyComplete(path~ : Path, name~ : String, span~ : Span)
StringDelta(path~ : Path, text~ : String, span~ : Span)
ValueComplete(path~ : Path, value~ : LeafValue, span~ : Span)
} derive(Eq, ToJson, Debug)pub(all) enum FinishResult {
Completed(CompletedDocument)
Incomplete(pos~ : Int, expected~ : String)
Aborted(reason~ : EndReason)
} derive(Eq, ToJson, Debug)pub(all) enum LeafValue {
Null
Bool(Bool)
Number(NumberLiteral)
String(String)
} derive(Eq, ToJson, Debug)fn Limits::new(max_depth? : Int, max_token_bytes? : Int, max_nodes? : Int, max_total_bytes? : Int) -> Limits raise ParseErrorfn Parser::finish(self : Parser, reason? : EndReason) -> (Array[Event], FinishResult) raise ParseErrortest {
let parser = Parser::new()
let _ = parser.feed(b"[1")
let (events, result) = parser.finish()
// 末尾数字只有在 finish 时才能被确认,收尾事件一并交付。
inspect(events.length(), content="1")
inspect(result is FinishResult::Incomplete(..), content="true")
}test {
let parser = Parser::new()
let events = parser.feed(b"{\"city\":")
inspect(events.length(), content="2")
}Install
Download zipIncremental, chunk-invariant parsing of streaming JSON tool-call arguments.