bhcli

A TUI for chatting on LE PHP Chats
git clone https://git.dasho.dev/bhcli.git
Log | Files | Refs | README

harm.rs (2962B)

      1 use regex::Regex;
      2 
      3 /// The type of harmful content detected.
      4 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
      5 pub enum Reason {
      6    RacialSlur,
      7    CsabTalk,
      8    CsabRequest,
      9 }
     10 
     11 impl Reason {
     12    pub fn description(&self) -> &'static str {
     13        match self {
     14            Reason::RacialSlur => "using a racial slur (sorry if this is false)",
     15            Reason::CsabTalk => "referencing child sexual abuse material (sorry if this is false)",
     16            Reason::CsabRequest => {
     17                "requesting child sexual abuse material (sorry if this is false)"
     18            }
     19        }
     20    }
     21 }
     22 
     23 /// Result of scoring a message.
     24 pub struct ScoreResult {
     25    pub score: u32,
     26    pub reason: Option<Reason>,
     27 }
     28 
     29 /// Return a severity score between 0 and 100 based on harmful content and
     30 /// provide a reason when content is detected.
     31 pub fn score_message(message: &str) -> ScoreResult {
     32    let msg = message.to_lowercase();
     33    let collapsed: String = msg.chars().filter(|c| c.is_alphanumeric()).collect();
     34    let normalized: String = collapsed
     35        .chars()
     36        .map(|c| match c {
     37            '0' => 'o',
     38            '1' => 'i',
     39            '3' => 'e',
     40            '4' => 'a',
     41            '5' => 's',
     42            '7' => 't',
     43            _ => c,
     44        })
     45        .collect();
     46 
     47    let mut score = 0u32;
     48    let mut reason = None;
     49 
     50    // Detect uses of racial slurs (N-word and common variants)
     51    let nword_re = Regex::new(r"nigg(?:er|a)").unwrap();
     52    if nword_re.is_match(&msg) || normalized.contains("nigger") {
     53        let directed_re = Regex::new(r"(?:you|u|@\S+).{0,20}?nigg(?:er|a)").unwrap();
     54        if directed_re.is_match(&msg) {
     55            score = score.max(70);
     56        } else {
     57            score = score.max(40);
     58        }
     59        reason.get_or_insert(Reason::RacialSlur);
     60    }
     61 
     62    // Detect CSAM related talk (various obfuscations)
     63    let csam_terms = [
     64        "csam",
     65        "childporn",
     66        "pedo",
     67        "chees pizza",
     68        "childsex",
     69        "childsexualabuse",
     70        "cp",
     71    ];
     72    if csam_terms
     73        .iter()
     74        .any(|t| msg.contains(t) || normalized.contains(t))
     75    {
     76        let request_re =
     77            Regex::new(r"\b(send|share|looking|where|has|download|anyone|link|give|provide)\b")
     78                .unwrap();
     79        if request_re.is_match(&msg) {
     80            score = score.max(90);
     81            reason = Some(Reason::CsabRequest);
     82        } else {
     83            score = score.max(50);
     84            reason.get_or_insert(Reason::CsabTalk);
     85        }
     86    }
     87 
     88    if score > 100 {
     89        score = 100;
     90    }
     91 
     92    ScoreResult { score, reason }
     93 }
     94 
     95 /// Determine which action should be taken based on the score.
     96 pub fn action_from_score(score: u32) -> Option<Action> {
     97    match score {
     98        0..=39 => None,
     99        40..=92 => Some(Action::Warn),
    100        93..=99 => Some(Action::Kick),
    101        _ => Some(Action::Ban),
    102    }
    103 }
    104 
    105 #[derive(Debug, PartialEq, Eq)]
    106 pub enum Action {
    107    Warn,
    108    Kick,
    109    Ban,
    110 }