pub struct AccessRecord {
  verb : String
  url : String
  status : Int
  agent : String
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct Coverage {
  total : Int
  allowed : Int
  blocked : Int
  unmatched : Int
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct CrawlPolicy {
  agent : String
  crawl_delay : Int?
  sitemaps : Array[String]
  hosts : Array[String]
  group_agents : Array[String]
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct CrawlProfile {
  name : String
  agent : String
  seed_urls : Array[String]
  max_urls : Int
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct Decision {
  allowed : Bool
  agent : String
  path : String
  matched_kind : String
  matched_pattern : String
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct Directive {
  line : Int
  key : String
  value : String
  raw : String
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct FetchSlot {
  url : String
  earliest_tick : Int
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct FrontierItem {
  url : String
  path : String
  allowed : Bool
  reason : String
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct Group {
  agents : Array[String]
  rules : Array[Rule]
  crawl_delay : Int?
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct PathExpectation {
  path : String
  allowed : Bool
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct Robots {
  groups : Array[Group]
  sitemaps : Array[String]
  hosts : Array[String]
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct RobotsStats {
  groups : Int
  agents : Int
  allows : Int
  disallows : Int
  crawl_delays : Int
  sitemaps : Int
  hosts : Int
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct Rule {
  kind : String
  pattern : String
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct Scenario {
  name : String
  robots : String
  agent : String
  expectations : Array[PathExpectation]
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

pub struct SitemapEntry {
  loc : String
  lastmod : String
  changefreq : String
  priority : String
} derive(Eq, 
@moonbitlang/core/debug.Debug
Debug)

fn access_record(verb : String, url : String, status : Int, agent : String) -> AccessRecord

fn access_summary(records : Array[AccessRecord]) -> String

fn agent_family(agent : String) -> String

fn agent_matches_token(agent : String, token : String) -> Bool

fn agent_report(agent : String, tokens : Array[String]) -> String

fn agent_score_against(agent : String, token : String) -> Int

fn agent_tokens(agent : String) -> Array[String]

fn allow_all(agent : String) -> Robots

fn allow_public_private(private_prefix : String, public_prefix : String) -> Robots

fn allow_rule(pattern : String) -> Rule

fn allowed(text : String, agent : String, path : String) -> Bool

fn allowed_paths(text : String, agent : String, paths : Array[String]) -> Array[String]

fn append_host(robots : Robots, host : String) -> Robots

fn append_sitemap(robots : Robots, sitemap : String) -> Robots

fn ascii_space(ch : Char) -> Bool

fn attach_path(base_url : String, path : String) -> String

fn audit(text : String) -> Array[String]

fn audit_report(text : String) -> String

fn auto_coverage_report(text : String, agent : String) -> String

fn best_agent_token(agent : String, tokens : Array[String]) -> String

fn best_group(robots : Robots, agent : String) -> Group?

fn block_all(agent : String) -> Robots

fn block_paths(agent : String, paths : Array[String]) -> Robots

fn blocked_report(text : String, agent : String, urls : Array[String]) -> String

fn bool_word(value : Bool) -> String

fn built_in_scenarios() -> Array[Scenario]

fn built_in_scenarios_report() -> String

fn can_fetch_url(text : String, agent : String, url : String) -> Bool

fn canonical_match_path(path : String) -> String

fn classify_path(path : String) -> String

fn clean_path(path : String) -> String

fn common_path_prefix(paths : Array[String]) -> String

fn common_two_path_prefix(left : String, right : String) -> String

fn compact(text : String) -> String

fn compare_agents(text : String, left_agent : String, right_agent : String, paths : Array[String]) -> String

fn compare_versions(old_text : String, new_text : String, agent : String, paths : Array[String]) -> String

fn compliance_report(text : String) -> String

fn contains_rule(rules : Array[Rule], target : Rule) -> Bool

fn contains_string(values : Array[String], value : String) -> Bool

fn contains_string_case_insensitive(values : Array[String], value : String) -> Bool

fn count_status(records : Array[AccessRecord], status : Int) -> Int

fn coverage(text : String, agent : String, paths : Array[String]) -> Coverage

fn coverage_line(c : Coverage) -> String

fn coverage_report(text : String, agent : String, paths : Array[String]) -> String

fn crawl_delay_or(text : String, agent : String, fallback : Int) -> Int

fn csv_escape(text : String) -> String

fn csv_row(values : Array[String]) -> String

fn decide(robots : Robots, agent : String, path : String) -> Decision

fn decide_group(group : Group, agent : String, path : String) -> Decision

fn decide_url(robots : Robots, agent : String, url : String) -> Decision

fn decision_line(decision : Decision) -> String

fn decision_matrix(text : String, agents : Array[String], paths : Array[String]) -> String

fn decisions_csv(text : String, agent : String, paths : Array[String]) -> String

fn decode_percent_path(path : String) -> String

fn dedupe_urls(urls : Array[String]) -> Array[String]

fn delay_text(delay : Int?) -> String

fn diagnostics_report(text : String, agent : String, paths : Array[String]) -> String

fn directive_count(text : String, key : String) -> Int

fn directive_report(text : String) -> String

fn directive_suggestion_report(text : String) -> String

fn directive_values(text : String, key : String) -> Array[String]

fn directives(text : String) -> Array[Directive]

fn directives_for_key(text : String, key : String) -> Array[Directive]

fn disallow_rule(pattern : String) -> Rule

fn disallowed_paths(text : String, agent : String, paths : Array[String]) -> Array[String]

fn due_urls(now : Int, slots : Array[FetchSlot]) -> Array[String]

fn edit_distance(left : String, right : String) -> Int

fn empty_robots() -> Robots

fn encode_path_spaces(path : String) -> String

fn expect_path(path : String, allowed : Bool) -> PathExpectation

fn explain(text : String, agent : String, paths : Array[String]) -> String

fn explain_urls(text : String, agent : String, urls : Array[String]) -> String

fn fenced(text : String) -> String

fn filter_urls(text : String, agent : String, urls : Array[String]) -> Array[String]

fn first_directive_value(text : String, key : String) -> String?

fn frontier(text : String, agent : String, urls : Array[String]) -> Array[FrontierItem]

fn frontier_allowed(items : Array[FrontierItem]) -> Array[String]

fn frontier_blocked(items : Array[FrontierItem]) -> Array[String]

fn frontier_csv(items : Array[FrontierItem]) -> String

fn frontier_report(items : Array[FrontierItem]) -> String

fn group_add_allow(group : Group, pattern : String) -> Group

fn group_add_disallow(group : Group, pattern : String) -> Group

fn group_agent_score(group : Group, agent : String) -> Int

fn group_for_agents(agents : Array[String]) -> Group

fn group_matches_agent(group : Group, agent : String) -> Bool

fn group_rule_count(group : Group, kind : String) -> Int

fn group_set_delay(group : Group, delay : Int) -> Group

fn group_summary(group : Group) -> String

fn group_urls_by_host(urls : Array[String]) -> Array[String]

fn group_with(agents : Array[String], rules : Array[Rule]) -> Group

fn has_global_block(robots : Robots) -> Bool

fn has_percent_escape(text : String) -> Bool

fn health_report(text : String) -> String

fn hex_value(ch : Char) -> Int?

fn host_matches_robots(robots : Robots, url : String) -> Bool

fn invalid_percent_offsets(text : String) -> Array[Int]

fn inventory_report(text : String) -> String

fn invert_group(group : Group) -> Group

fn is_allow(rule : Rule) -> Bool

fn is_clean(text : String) -> Bool

fn is_disallow(rule : Rule) -> Bool

fn is_due(now : Int, slot : FetchSlot) -> Bool

fn join_ints(values : Array[Int], sep : String) -> String

fn join_lines(lines : Array[String]) -> String

fn join_with(values : Array[String], sep : String) -> String

fn known_directive(key : String) -> Bool

fn largest_group(robots : Robots) -> Group?

fn largest_group_summary(text : String) -> String

fn lint(text : String) -> Array[String]

fn lint_report(text : String) -> String

fn lower_ascii(text : String) -> String

fn markdown_decision_table(robots : Robots, agent : String, paths : Array[String]) -> String

fn markdown_full(text : String, agent : String, paths : Array[String]) -> String

fn markdown_hosts(robots : Robots) -> String

fn markdown_inventory(text : String) -> String

fn markdown_report(text : String, agent : String, paths : Array[String]) -> String

fn markdown_sitemaps(robots : Robots) -> String

fn merge_groups_by_agent(robots : Robots) -> Robots

fn min_int(a : Int, b : Int) -> Int

fn missing_common_blocks(text : String, agent : String) -> Array[String]

fn new_group(agent : String) -> Group

fn next_allowed_tick(text : String, agent : String, previous_tick : Int) -> Int

fn next_batch(text : String, agent : String, urls : Array[String], limit : Int) -> Array[String]

fn normalize_agent(agent : String) -> String

fn normalize_path(path : String) -> String

fn normalize_robots(robots : Robots) -> Robots

fn origin(url : String) -> String?

fn origin_or(url : String, fallback : String) -> String

fn parse(text : String) -> Robots

fn parse_errors(text : String) -> Array[String]

fn parse_non_negative_int(text : String) -> Int?

fn parse_report(text : String) -> String

fn path_basename(path : String) -> String

fn path_depth(path : String) -> Int

fn path_extension(path : String) -> String

fn path_has_extension(path : String, ext : String) -> Bool

fn path_parent(path : String) -> String

fn path_score(path : String) -> Int

fn pattern_matches(pattern : String, path : String) -> Bool

fn pending_urls(now : Int, slots : Array[FetchSlot]) -> Array[String]

fn percent_report(text : String) -> String

fn policy_for(text : String, agent : String) -> CrawlPolicy

fn policy_from(robots : Robots, agent : String) -> CrawlPolicy

fn policy_summary(policy : CrawlPolicy) -> String

fn profile(name : String, agent : String, seed_urls : Array[String], max_urls : Int) -> CrawlProfile

fn profile_markdown(text : String, p : CrawlProfile) -> String

fn profile_plan(text : String, p : CrawlProfile) -> Array[String]

fn profile_report(text : String, p : CrawlProfile) -> String

fn protects_sensitive_defaults(text : String, agent : String) -> Bool

fn rank_agents(agent : String, tokens : Array[String]) -> Array[String]

fn recipe_allow_all(agent : String) -> String

fn recipe_api_docs(sitemap : String) -> String

fn recipe_block_all(agent : String) -> String

fn recipe_blog(sitemap : String) -> String

fn recipe_catalog() -> String

fn recipe_for_kind(kind : String, sitemap : String) -> String