#unicode

#Features

• Unicode Normalization Forms from UAX #15: NFD, NFC, NFKD, and NFKC
• Punycode encoding and decoding from RFC 3492
• IDNA processing from UTS #46, including mapping, validation, Bidi checks, joiner checks, and DNS length checks
• Unicode Bidirectional Algorithm support from UAX #9
• General_Category, XID_Start, and XID_Continue lookup plus simple/full case mapping from the Unicode Character Database

#Installation

moon add tonyfettes/ucd
moon add tonyfettes/normalization
moon add tonyfettes/punycode
moon add tonyfettes/bidi
moon add tonyfettes/idna

import {
  "tonyfettes/ucd"
  "tonyfettes/normalization"
  "tonyfettes/punycode"
  "tonyfettes/idna"
  "tonyfettes/bidi"
}

#Usage

#Normalization

let composed = @normalization.nfc("e\u{0301}") // "é"
let decomposed = @normalization.nfd("é") // "e" + combining acute
let compatible = @normalization.nfkc("fi") // "fi"

let already_nfc = @normalization.is_normalized(
  composed,
  @normalization.NFC,
)

let normalized = @normalization.normalize(
  "text",
  @normalization.NFKC,
)

#Punycode

let encoded = try! @punycode.encode("münchen") // "mnchen-3ya"
let decoded = try! @punycode.decode("mnchen-3ya") // "münchen"

let chinese = try! @punycode.encode("中文") // "fiq228c"

#IDNA

let ascii = try! @idna.to_ascii("münchen.de")
// "xn--mnchen-3ya.de"

let unicode = try! @idna.to_unicode("xn--mnchen-3ya.de")
// "münchen.de"

let checked = try! @idna.to_ascii(
  "example.com",
  use_std3_ascii_rules=true,
  check_hyphens=true,
  check_bidi=true,
  check_joiners=true,
  verify_dns_length=true,
)

try @idna.to_ascii("example..com") catch {
  err => println("invalid domain: \{err}")
} noraise {
  ascii => println(ascii)
}

#Bidi

let direction = @bidi.detect_direction("Hello World") // LTR
let needs_bidi = @bidi.requires_bidi("Hello \u{05E9}\u{05DC}\u{05D5}\u{05DD}")

let paragraph = @bidi.process("abc\u{05D0}\u{05D1}")
let visual = @bidi.reorder_string(paragraph)
let order = @bidi.reorder(paragraph)

let forced = @bidi.process(
  "\u{05D0}\u{05D1}\u{05D2}",
  direction=@bidi.Direction::LTR,
)

#Case And Category Data

let category = @ucd.general_category('A') // Lu
let group = category.group() // L
let can_start = @ucd.is_xid_start('A') // true
let can_continue = @ucd.is_xid_continue('0') // true

let simple = @ucd.to_simple_uppercase('a') // 'A'
let full = @ucd.to_uppercase('\u{00DF}') // "SS"
let lower = @ucd.to_lowercase('\u{0130}') // "i" + combining dot above

#Public Packages

#tonyfettes/ucd

#tonyfettes/normalization

#tonyfettes/punycode

#tonyfettes/idna

• use_std3_ascii_rules? : Bool
• check_hyphens? : Bool
• check_bidi? : Bool
• check_joiners? : Bool
• verify_dns_length? : Bool

#tonyfettes/bidi

#Development

moon check
moon test
moon test -p tonyfettes/normalization
moon fmt
moon info
moon build

moon run --target native tools/gen data
moon run --target native tools/gen tests

# Or regenerate everything in one pass:
moon run --target native tools/gen all

moon -C bidi package --list

#Releasing

moon run --target native tools/release -- prepare --version 0.4.0

moon run --target native tools/release -- publish \
  --version 0.4.0 \
  --execute

moon run --target native tools/release -- publish \
  --version 0.4.0 \
  --from normalization \
  --execute

#Standards

• UAX #9: Unicode Bidirectional Algorithm
• UAX #15: Unicode Normalization Forms
• UAX #44: Unicode Character Database
• RFC 3492: Punycode
• UTS #46: Unicode IDNA Compatibility Processing
• RFC 5891: Internationalized Domain Names in Applications
• RFC 5892: The Unicode Code Points and IDNA
• RFC 5893: Right-to-Left Scripts for IDNA

#License