The streaming contract of byte-stream codecs: step through input chunks into bounded output slices, with bounded memory.
moon add gaato/codec_bytes///|
/// Decodes `chunks` with any decoder, handing each slice of output to `sink`.
fn[D : @codec_bytes.Decode] decode_all(
decoder : D,
chunks : Array[Bytes],
sink : (BytesView) -> Unit,
) -> Unit raise {
let out = FixedArray::make(64 * 1024, b'\x00')
for i, chunk in chunks {
let mut input = chunk[:]
for ;; {
let status = decoder.step(
input,
out.mut_view(),
end=i == chunks.length() - 1,
)
input = input[decoder.last_consumed():]
sink(Bytes::from_array(out[:decoder.last_produced()]))
match status {
Done => return
NeedMoreInput => break
NeedMoreOutput => ()
}
}
}
}///|
/// A decoder of the identity format: output is input.
struct Identity {
mut taken : Int32
mut written : Int32
}
///|
impl @codec_bytes.Decode for Identity with fn step(self, input, output, end~) {
let n = input.length().min(output.length())
for i in 0..<n {
output[i] = input[i]
}
self.taken = n
self.written = n
if n < input.length() {
NeedMoreOutput
} else if end {
Done
} else {
NeedMoreInput
}
}
///|
impl @codec_bytes.Decode for Identity with fn last_consumed(self) {
self.taken
}
///|
impl @codec_bytes.Decode for Identity with fn last_produced(self) {
self.written
}
///|
impl @codec_bytes.Decode for Identity with fn reset(self) {
self.taken = 0
self.written = 0
}
///|
impl @codec_bytes.Decode for Identity with fn memory_usage(_self) {
0
}
///|
test "drive a decoder through the contract" {
let decoder : Identity = { taken: 0, written: 0, }
let decoded = Buffer()
decode_all(decoder, [b"hel", b"lo"], bytes => {
decoded.write_bytes(bytes.to_owned())
})
assert_eq(decoded.to_bytes(), b"hello")
}pub(open) trait Decode {
fn step(Self, BytesView, MutArrayView[Byte], end~ : Bool) -> Status raise CodecError
fn last_consumed(Self) -> Int
fn last_produced(Self) -> Int
fn reset(Self) -> Unit
fn memory_usage(Self) -> Int
}pub(open) trait Encode {
fn step(Self, BytesView, MutArrayView[Byte], action~ : EncodeAction) -> Status raise CodecError
fn last_consumed(Self) -> Int
fn last_produced(Self) -> Int
fn reset(Self) -> Unit
fn memory_usage(Self) -> Int
}Install
Download zipThe streaming contract of byte-stream codecs: step through input chunks into bounded output slices, with bounded memory.