bhcli-simple

A simplified version of bhcli
git clone https://git.dasho.dev/bhcli-simple.git
Log | Files | Refs | README

harm.rs (2849B)

      1 use regex::Regex;
      2 
      3 /// The type of harmful content detected.
      4 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
      5 pub enum Reason {
      6    RacialSlur,
      7    CsabTalk,
      8    CsabRequest,
      9 }
     10 
     11 impl Reason {
     12    pub fn description(&self) -> &'static str {
     13        match self {
     14            Reason::RacialSlur => "using a racial slur (sorry if this is false)",
     15            Reason::CsabTalk => "referencing child sexual abuse material (sorry if this is false)",
     16            Reason::CsabRequest => "requesting child sexual abuse material (sorry if this is false)",
     17        }
     18    }
     19 }
     20 
     21 /// Result of scoring a message.
     22 pub struct ScoreResult {
     23    pub score: u32,
     24    pub reason: Option<Reason>,
     25 }
     26 
     27 /// Return a severity score between 0 and 100 based on harmful content and
     28 /// provide a reason when content is detected.
     29 pub fn score_message(message: &str) -> ScoreResult {
     30    let msg = message.to_lowercase();
     31    let collapsed: String = msg.chars().filter(|c| c.is_alphanumeric()).collect();
     32    let normalized: String = collapsed
     33        .chars()
     34        .map(|c| match c {
     35            '0' => 'o',
     36            '1' => 'i',
     37            '3' => 'e',
     38            '4' => 'a',
     39            '5' => 's',
     40            '7' => 't',
     41            _ => c,
     42        })
     43        .collect();
     44 
     45    let mut score = 0u32;
     46    let mut reason = None;
     47 
     48    // Detect uses of racial slurs (N-word and common variants)
     49    let nword_re = Regex::new(r"nigg(?:er|a)").unwrap();
     50    if nword_re.is_match(&msg) || normalized.contains("nigger") {
     51        let directed_re = Regex::new(r"(?:you|u|@\S+).{0,20}?nigg(?:er|a)").unwrap();
     52        if directed_re.is_match(&msg) {
     53            score = score.max(70);
     54        } else {
     55            score = score.max(40);
     56        }
     57        reason.get_or_insert(Reason::RacialSlur);
     58    }
     59 
     60    // Detect CSAM related talk (various obfuscations)
     61    let csam_terms = ["csam", "childporn", "pedo", "chees pizza", "childsex", "childsexualabuse", "cp"];
     62    if csam_terms.iter().any(|t| msg.contains(t) || normalized.contains(t)) {
     63        let request_re = Regex::new(
     64            r"\b(send|share|looking|where|has|download|anyone|link|give|provide)\b",
     65        )
     66        .unwrap();
     67        if request_re.is_match(&msg) {
     68            score = score.max(90);
     69            reason = Some(Reason::CsabRequest);
     70        } else {
     71            score = score.max(50);
     72            reason.get_or_insert(Reason::CsabTalk);
     73        }
     74    }
     75 
     76    if score > 100 {
     77        score = 100;
     78    }
     79 
     80    ScoreResult { score, reason }
     81 }
     82 
     83 /// Determine which action should be taken based on the score.
     84 pub fn action_from_score(score: u32) -> Option<Action> {
     85    match score {
     86        0..=39 => None,
     87        40..=92 => Some(Action::Warn),
     88        93..=99 => Some(Action::Kick),
     89        _ => Some(Action::Ban),
     90    }
     91 }
     92 
     93 #[derive(Debug, PartialEq, Eq)]
     94 pub enum Action {
     95    Warn,
     96    Kick,
     97    Ban,
     98 }