#mbglot

#At a glance

///|
test {
  let d = @mbglot.detect(
    "Olá, gostaria de mudar o endereço de entrega do meu pedido antes do envio.",
  ).unwrap()
  assert_eq(d.lang, @mbglot.Por) // franc-min alone says Spanish
  assert_eq(d.lang.bcp47(), "pt")
  assert_true(d.reliable)
}

#Status

• detect(text, options?) -> Detection? and rank(text, options?) -> Array[(Lang, Double)], with Options for allow / deny lists and a minimum length, and a reliable flag. In Mode::Franc they give the same rankings and scores as franc-min 6.2.0 (checked by a golden test, see below); the default Mode::Enhanced removes markup first and settles closely related languages by their letters and common words (see Related languages).
• Detector::new(options?): the same detect, rank and segments for a stream of texts, with the options worked out once (see Speed).
• Lang: the 76 languages of franc-min 6.2.0, with code() (ISO 639-3), name() (English), native_name(), iso639_1(), bcp47(), speakers(), Lang::from_code, Lang::from_bcp47 and Lang::all() (see Languages).
• Script: the 20 writing systems franc-min distinguishes, with name() and Script::all().
• detect_script(text) -> Script?: the writing system used by most characters, with franc-min's Unicode ranges, sample limit (first 2048 UTF-16 code units) and tie order. Returns None for text with no letters of a supported script (empty, digits, punctuation, emoji).
• Trigram extraction matching trigram-utils 2.0.1 (internal): runs of ASCII punctuation, digits and whitespace become one space, the text is trimmed and lowercased like JavaScript's toLowerCase (Unicode 17.0.0 tables, including İ and final sigma), padded with spaces and cut into trigrams of UTF-16 code units.
• Trigram profiles (internal): the 66 profiles of franc-min 6.2.0, 300 trigrams each, generated into profiles.mbt by tools/gen_profiles.mjs. CI regenerates them from npm and fails if they differ from the committed file.
• segments(text, options?) -> Array[Segment]: splits a text that mixes languages into consecutive segments with their own detection. This is mbglot's own procedure on top of franc's scores (franc has no segmentation); see below.
• Profile::train and Classifier: profiles trained from sample text, saved as plain text and compared with franc's method, alone or with the built-in ones (see Custom profiles).
• cn-nnzy/mbglot/full: franc 6.2.0 with all its languages (see All of franc's languages).
• Accumulator: the language of a stream fed in chunks, without re-reading earlier chunks (see Streams).
• explain(text, options?): the steps that decided a detection and the trigrams that weigh for and against it (see Why this language).
• chinese_variant(text): simplified or traditional Chinese, and kana and Hangul shares that settle Chinese, Japanese and Korean (see Chinese, Japanese and Korean).
• strip_markup(text): removes URLs, e-mail addresses, mentions, hashtags, HTML and code before detection, with a map back to the original offsets (see Markup). On by default.
• An evaluation package, cn-nnzy/mbglot/eval: accuracy, confusion counts and per-language precision, recall and F1 for any predictor over labelled samples (see Measuring accuracy), and a public benchmark on 5,541 Tatoeba sentences in 72 languages and 2,720 FLORES-200 sentences in 68 (see Benchmark).
• A command-line tool in cmd/main with JSON output and a mode for files of one text per line, a ticket-routing example in examples/tickets and a browser demo in demo.

#Usage

moon add cn-nnzy/mbglot

import {
  "cn-nnzy/mbglot" @mbglot,
}

#Command line

$ moon run cmd/main -- "Hola, mi pedido llegó incompleto y falta una de las piezas."
language:   Spanish (spa)
script:     Latin
confidence: 0.506 (reliable)
runner-up:  Italian (ita), margin 0.226
top 5:
  1. spa  Spanish                  1.000
  2. ita  Italian                  0.774
  3. por  Portuguese               0.702
  4. lin  Lingala                  0.689
  5. fra  French                   0.687

$ moon run cmd/main -- --segments "Здравствуйте, заказ не пришёл. Hi, could you check the tracking number for order 4471 please? 谢谢，麻烦尽快处理一下这个问题。"
[0, 31)      rus  Russian                  reliable     Здравствуйте, заказ не пришёл.
[31, 94)     eng  English                  reliable     Hi, could you check the tracking numb...
[94, 110)    cmn  Mandarin Chinese         reliable     谢谢，麻烦尽快处理一下这个问题。

$ moon run cmd/main -- --file messages.txt
1	eng	0.680	reliable
2	-	-	-
3	fra	0.263	reliable
4	bos	0.040	not reliable

$ moon run cmd/main -- --json --top 2 "Olá, gostaria de mudar o endereço de entrega do meu pedido antes do envio."
{"detection":{"lang":"por","bcp47":"pt","name":"Portuguese","script":"Latin","confidence":0.428,"margin":0.068,"reliable":true,"runner_up":"spa"},"top":[{"lang":"por","score":1},{"lang":"spa","score":0.932}]}

#Example: routing support tickets

$ moon run examples/tickets
#1001  support-en  (eng English)
#1002  support-es  (spa Spanish)
#1003  support-fr  (fra French)
#1004  support-de  (deu German)
#1005  support-pt  (por Portuguese)
#1006  support-ru  (rus Russian)
#1007  support-zh  (cmn Mandarin Chinese)
#1008  support-ja  (jpn Japanese)
#1009  support-ar  (arb Standard Arabic)
#1010  triage  (tur Turkish, not reliable)
#1011  triage  (no language)
#1012  triage  (por Portuguese, not reliable)

#Browser demo

node tools/build_demo.mjs _site
python -m http.server -d _site

#Languages

• 61 languages told apart by trigram profiles within the Latin, Cyrillic, Arabic and Devanagari scripts (66 profiles, since Bosnian, Serbian, Northern Uzbek, North Azerbaijani and Malay each have two scripts).
• 15 languages identified by a script used only by them: Mandarin Chinese (Han), Japanese (Kana), Korean (Hangul), Bengali, Telugu, Tamil, Gujarati, Kannada, Malayalam, Burmese, Panjabi (Gurmukhi), Amharic (Ethiopic), Thai, Sinhala and Modern Greek. Javanese script also maps to Javanese, which has a Latin profile as well.

///|
test {
  assert_eq(@mbglot.Lang::from_bcp47("pt-BR"), Some(@mbglot.Por))
  assert_eq(@mbglot.Lang::from_bcp47("zh-Hant-TW"), Some(@mbglot.Cmn))
  assert_eq(@mbglot.Lang::from_bcp47("zh-yue"), None)
  assert_eq(@mbglot.Cmn.bcp47(), "zh")
  assert_eq(@mbglot.Ceb.bcp47(), "ceb")
  assert_eq(@mbglot.Rus.native_name(), "русский")
}

#Detecting a language

///|
test {
  let text = "Hola, mi pedido llegó incompleto y falta una de las piezas."
  let detection = @mbglot.detect(text).unwrap()
  assert_eq(detection.lang, @mbglot.Spa)
  assert_eq(detection.lang.name(), "Spanish")
  assert_eq(detection.script, @mbglot.Latin)
  assert_eq(@mbglot.detect("12:30 👍"), None)
}

• in Mode::Franc, the margin is at least reliable_min_margin (0.15) and the text has at least reliable_min_length (30) UTF-16 code units;
• in Mode::Enhanced, the confidence is at least reliable_min_confidence (0.18), or short_reliable_min_confidence (0.4) for texts under 30 code units, and the runner-up is not a related language that the text gives no reason to rule out (Indonesian against Malay with only shared words, for example).

#Options

///|
test {
  let options = @mbglot.Options::new(allow=[@mbglot.Eng, @mbglot.Por])
  let ranked = @mbglot.rank("Hola, mi pedido llegó incompleto.", options~)
  assert_eq(ranked.map(r => r.0), [@mbglot.Por, @mbglot.Eng])
}

///|
test {
  let detector = @mbglot.Detector::new(
    options=@mbglot.Options::new(allow=[Eng, Fra, Deu]),
  )
  let langs = [
    "Where is my parcel?", "Où est mon colis ?", "Wo ist mein Paket?",
  ].map(text => detector.detect(text).map(d => d.lang))
  assert_eq(langs, [Some(Eng), Some(Fra), Some(Deu)])
}

#Related languages

• Spanish and Portuguese (ñ, ¿ against ã, õ, ç, lh, nh; articles and pronouns);
• Russian, Ukrainian, Belarusian, Bulgarian, Serbian, Kazakh and Bosnian in Cyrillic (ы, э, ё; ї, є, ґ; ў; ђ, ј, љ, њ, ћ, џ; ә, қ, ң, ө, ұ, ү; and common words such as Bulgarian е, са, ще, това);
• Croatian, Serbian and Bosnian in Latin script (words such as tko, ko, što, šta, ekavian and ijekavian forms);
• Indonesian and Malay, Rundi and Kinyarwanda, Tagalog and Cebuano (common words, and cy, jy, by against vy);
• Hausa and Nigerian Fulfulde (ƙ against ŋ, ɲ, and common words such as kuma, wanda against je, nde), Javanese and Sundanese (dh, th against eu, and words such as sing, lan against anu, jeung);
• Standard Arabic, Persian, Urdu, Saraiki, Sorani Kurdish and Pashto in Arabic script (letters such as ة, ڕ, ێ, ە, ټ, ښ, ٹ, ے, ٻ, ڄ, and common words).

#Short texts

///|
test {
  // franc's trigrams alone prefer Komi for these three Russian words.
  let franc = @mbglot.Options::new(mode=Franc)
  assert_eq(
    @mbglot.detect("это мой дом", options=franc).map(d => d.lang),
    Some(@mbglot.Koi),
  )
  assert_eq(
    @mbglot.detect("это мой дом").map(d => d.lang),
    Some(@mbglot.Rus),
  )
}

///|
test {
  let text = "Здравствуйте! После оплаты подписка так и не активировалась, а деньги списались."
  assert_eq(@mbglot.detect(text).map(d => d.lang), Some(@mbglot.Rus))
  let franc = @mbglot.Options::new(mode=Franc)
  assert_eq(
    @mbglot.detect(text, options=franc).map(d => d.lang),
    Some(@mbglot.Bul),
  )
}

#Chinese, Japanese and Korean

///|
test {
  let text = "我们公司的产品在“美味の店”有售，欢迎大家来购买。"
  assert_eq(@mbglot.detect(text).map(d => d.lang), Some(@mbglot.Cmn))
  let franc = @mbglot.Options::new(mode=Franc)
  assert_eq(
    @mbglot.detect(text, options=franc).map(d => d.lang),
    Some(@mbglot.Jpn),
  )
  assert_eq(@mbglot.chinese_variant(text), Some(Simplified))
  assert_eq(
    @mbglot.chinese_variant("請幫忙查一下物流資訊，謝謝。"),
    Some(Traditional),
  )
}

#Markup

///|
test {
  let text = "Il mio ordine non è arrivato " +
    "https://www.example.com/the/order/status/page/for/customers/with/questions"
  assert_eq(@mbglot.detect(text).map(d => d.lang), Some(@mbglot.Ita))
  let raw = @mbglot.Options::new(strip_markup=false)
  assert_eq(
    @mbglot.detect(text, options=raw).map(d => d.lang),
    Some(@mbglot.Eng),
  )
}

///|
test {
  let s = @mbglot.strip_markup("Merci @dana <b>!</b>")
  assert_eq(s.text(), "Merci    ! ")
  assert_eq(s.removed().map(r => r.kind), [Mention, HtmlTag, HtmlTag])
  assert_eq(s.original_offset(6), 6) // the space for @dana
  assert_eq(s.original_offset(9), 15) // the "!"
}

#Why this language

• steps: franc's choice by trigram distance (Trigrams), then each pass of Mode::Enhanced that changed the language at the top: the shares of kana, Han and Hangul (ScriptShare), common words in a short text (CommonWords), letters and words that set a related language apart (RelatedLanguages, with only the clues that the language franc chose lacks) and a near tie broken by number of speakers (NearTie);
• trigrams: every trigram of the text that weighs differently for the detected language and the runner-up, with its count, its rank in both profiles and the difference it makes to franc's distance, strongest first.

///|
test {
  let why = @mbglot.explain(
    "Olá, gostaria de mudar o endereço de entrega do meu pedido antes do envio.",
  )
  assert_eq(why.steps, [Trigrams(Spa), RelatedLanguages(Por, ["ç", "o", "do"])])
  assert_eq(why.trigrams[0].trigram, " do")
}

$ moon run cmd/main -- --explain --top 0 "Olá, gostaria de mudar o endereço de entrega do meu pedido antes do envio."
language:   Portuguese (por)
script:     Latin
confidence: 0.428 (reliable)
runner-up:  Spanish (spa), margin 0.068
why:
  trigram statistics rank Spanish (spa) first
  letters and words that set Portuguese (por) apart from a related language: ç, o, do
trigrams for por over spa: " do" +233, "ido" +103, "o e" +101, "do " +87, "ntr" +51
trigrams for spa over por: "ere" -290, "der" -286, " en" -269, "ant" -145, "ari" -86

#Mixed-language text

///|
test {
  let text = "Hello, my order has not arrived yet and I need it for Friday. " +
    "Hola, mi pedido no ha llegado todavía y lo necesito para el viernes."
  let parts = @mbglot.segments(text)
  assert_eq(parts.map(s => s.detection.unwrap().lang), [
    @mbglot.Eng,
    @mbglot.Spa,
  ])
  assert_eq(parts[1].start, 62)
}

#Custom profiles

///|
test {
  let esperanto = @mbglot.Profile::train(
    "epo",
    "Saluton, mia nomo estas Petro kaj mi loĝas en granda urbo. " +
    "Mi tre ŝatas legi librojn kaj promeni en la parko kun mia hundo. " +
    "Hodiaŭ la vetero estas bela, do ni iros al la maro. " +
    "Ĉu vi volas veni kun ni? Ni manĝos fiŝojn kaj trinkos kafon.",
  )
  let profiles = [@mbglot.Eng, @mbglot.Spa, @mbglot.Ita].map(lang => {
    @mbglot.Profile::of_lang(lang).unwrap()
  })
  profiles.push(esperanto)
  let classifier = @mbglot.Classifier::new(profiles)
  assert_eq(
    classifier.classify("Ĉu vi scias, kie estas la stacidomo?"),
    Some("epo"),
  )
  // A profile saves to plain text and reads back.
  let saved = esperanto.save()
  assert_eq(@mbglot.Profile::parse(saved).trigrams(), esperanto.trigrams())
}

#Streams

///|
test {
  let chat = @mbglot.Accumulator::new()
  chat.feed("ok")
  assert_eq(chat.detect(), None)
  chat.feed("Hola, ¿cómo estás?")
  chat.feed("Muy bien, gracias. Tengo una pregunta sobre mi pedido.")
  assert_eq(chat.detect().map(d => d.lang), Some(@mbglot.Spa))
}

#All of franc's languages

///|
test {
  assert_eq(
    @full.franc("Bon dia, voldria saber quan arribarà la meva comanda."),
    "cat",
  )
  assert_eq(
    @full.franc(
      "שלום, ההזמנה שלי עדיין לא הגיעה. אפשר לבדוק?",
    ),
    "heb",
  )
  assert_eq(@full.franc_all("hi"), [("und", 1.0)])
  assert_eq(@full.languages().length(), 178)
}

#Script detection

///|
test {
  assert_eq(
    @mbglot.detect_script("Привет, мир"),
    Some(@mbglot.Cyrillic),
  )
  assert_eq(@mbglot.detect_script("12:30 👍"), None)
}

#Measuring accuracy

///|
test {
  let samples : Array[@eval.Sample] = [
    {
      lang: Eng,
      text: "Thanks for the quick reply, the replacement arrived today.",
    },
    {
      lang: Deu,
      text: "Vielen Dank, die Ersatzlieferung ist heute angekommen.",
    },
  ]
  let report = @eval.evaluate(samples, text => {
    match @mbglot.detect(text) {
      Some(d) => Some(d.lang)
      None => None
    }
  })
  assert_eq(report.total(), 2)
  assert_eq(report.count(Deu, Some(Deu)), 1)
}

#Benchmark

#A second corpus: FLORES-200

#Speed

#Development

moon update                # once: fetches the registry index for moonbitlang/x
moon fmt
moon info
moon check --target all
moon test --target wasm
moon test --target wasm-gc
moon test --target js      # needs Node.js
moon test --target native  # needs a C compiler

#franc parity

npm install --prefix "$TMP/franc" --no-save --ignore-scripts franc-min@6.2.0
curl -fsSL -o "$TMP/fixtures.js" https://raw.githubusercontent.com/wooorm/franc/3f9f0b51a96c5df32a407dad865a3011ac0fa2d1/test/fixtures.js
node tools/gen_golden.mjs "$TMP/franc/node_modules/franc-min" "$TMP/fixtures.js" golden_wbtest.mbt
moon fmt

#License