Search and tokenize Portuguese text with RSLP

interface SuffixRule {
  suffix: string; // The suffix to be removed
  minStemLength: number; // Minimum length of the word to apply this rule
  replacement: string; // Optional replacement suffix
  exceptions: string[]; // List of exceptions where the rule should not be applied
}

"ao", 3, "", ["pão", "mão", "cão"]

type SuffixRule = {
  suffix: string;
  minStemLength: number;
  replacement: string;
  exceptions: string[];
};

export type Step = {
    name: string;
    minWordLength: number;
    isWholeWordMatch?: boolean;
    conditions: string[];
    rules: SuffixRule[];
}

"Plural", 3, 1, {"s"},

export async function parseRSLPRules(ruleLines: string[]): Step[] {
    const steps: Step[] = [];
    let currentStep: Step | undefined;

      for (const line of ruleLines) { // Iterate through each line in the file
        const trimmedLine = line.trim();

        if (trimmedLine.length === 0) continue; // Skip empty lines
        if (trimmedLine.startsWith('#')) continue; // Skip comments

        if (!currentStep) {
            currentStep = parseStep(trimmedLine); // Start a new step

            continue;
        }

        if (trimmedLine.endsWith(';')) { // End of current step
            const finalRule = parseRule(trimmedLine.slice(0, -1));

            currentStep.rules.push(finalRule)
            steps.push(currentStep);
            currentStep = undefined;
            continue;
        }

        const rule = parseRule(trimmedLine); // Parse the rule

        if (!currentStep) {
            console.error("Rule found without a current step:", trimmedLine);
            throw new Error("Rule found without a current step");
        }

        currentStep.rules.push(rule); // Add the rule to the current step
    }

    return steps;
}

function parseStep(line: string): Step {
    const stepMatch = line.match(/{[s]*"([^"]+)"[s]*,[s]*(d+)[s]*,[s]*(d+)[s]*,[s]*(.+)/); // Regex to match the step format

    if (!stepMatch) throw new Error(`Invalid step format: ${line}`);

    const name = stepMatch[1]?.replace(/"/g, ''); // The name of the step
    if (!name) {
        console.error("Invalid step name:", stepMatch[1]);
        throw new Error("Invalid step name");
    }

    const minWordLength = parseInt(stepMatch[2] || 'error', 10); // Minimum length of the word to apply any rule in this step
    if (isNaN(minWordLength)) {
        console.error("Invalid minimum word length:", stepMatch[2]);
        throw new Error("Invalid minimum word length");
    }

    const isWholeWordMatch = (stepMatch[3] || '0') === '1'; // Convert to boolean

    const conditionsStr = stepMatch[4]?.trim() || '';
    const conditionsRegex = /"([^"]+)"/g;
    const conditions: string[] = []; // List to hold the extracted conditions

    let match;
    while ((match = conditionsRegex.exec(conditionsStr)) !== null) { // Extract conditions from the matched group
        if (match[1]) {
            conditions.push(match[1].trim());
        }
    }

    return {
        name,
        minWordLength,
        isWholeWordMatch,
        conditions,
        rules: []
    };
}

function parseRule(line: string): SuffixRule {
    const ruleMatch = line.match(/{[s]*"([^"]*)"[s]*,[s]*(d+)(?:[s]*,[s]*"([^"]*)")?(?:[s]*,[s]*{([^}]*)})?[s]*}/); // Regex to match the rule format

    if (!ruleMatch) {
        throw new Error(`Invalid rule format: ${line}`);
    }

    const suffix = ruleMatch[1]?.replace(/"/g, ''); // The suffix to be removed
    if (suffix === undefined) {
        console.error("Invalid rule suffix:", ruleMatch[1]);
        throw new Error("Invalid rule suffix");
    }

    const minStemLength = parseInt(ruleMatch[2] || 'error', 10); // Minimum length of the word to apply this rule
    if (isNaN(minStemLength) || minStemLength < 0) {
        console.error("Invalid minimum stem length:", ruleMatch[2]);
        throw new Error("Invalid minimum stem length");
    }

    const replacement = ruleMatch[3] !== undefined ? ruleMatch[3].replace(/"/g, '') : ''; // Optional replacement suffix
    
    const exceptionStr = ruleMatch[4]?.trim() || ''; 
    const exceptionRegex = /"([^"]+)"/g;

    const exceptions: string[] = [];
    let match;

    while ((match = exceptionRegex.exec(exceptionStr)) !== null) { // Extract exceptions from the matched group
        if (match[1]) {
            exceptions.push(match[1].trim());
        }
    }

    return {
        suffix,
        minStemLength,
        replacement,
        exceptions
    };
}

class RSLPStemmer {
    private steps: Step[];

    constructor(steps: Step[]) {
        this.steps = steps;
    }

    /**
     * Stems a word according to the RSLP algorithm
     * @param word The word to stem
     * @returns The stemmed word
     */
    public stem(word: string): string {
        if (!word) return word;
        
        let stemmed = word.toLowerCase();
        stemmed = this.stripSpecialCharacters(stemmed);

        for (const step of this.steps) {
            if (stemmed.length < step.minWordLength) {
                continue;
            }

            if (step.conditions.length > 0) {
                const matchesCondition = step.conditions.some(condition => 
                    stemmed.endsWith(condition));
                if (!matchesCondition) {
                    continue;
                }
            }

            for (const rule of step.rules) {
                if (this.canApplyRule(stemmed, rule, step.isWholeWordMatch)) {
                    stemmed = this.applyRule(stemmed, rule);
                    break;
                }
            }
        }
        
        return stemmed;
    }

    /**
     * Strips special characters from a word (keeping only alphanumeric characters, including those with accents)
     */
    private stripSpecialCharacters(word: string): string {
        return word.replace(/[^a-zA-ZÀ-ÖØ-öø-ÿ0-9]/g, '').trim();
    }
    
    /**
     * Checks if a rule can be applied to a word
     */
    private canApplyRule(word: string, rule: SuffixRule, isWholeWordMatch: boolean = false): boolean {
        if (!word.endsWith(rule.suffix)) {
            return false;
        }
        
        if (rule.exceptions.length > 0) {
            if (isWholeWordMatch) {
                if (rule.exceptions.includes(word)) {
                    return false;
                }
            } else {
                for (const exception of rule.exceptions) {
                    if (word.endsWith(exception)) {
                        return false;
                    }
                }
            }
        }
        
        const stemLength = word.length - rule.suffix.length;
        if (stemLength < rule.minStemLength) {
            return false;
        }
        
        return true;
    }
    
    /**
     * Applies a rule to a word
     */
    private applyRule(word: string, rule: SuffixRule): string {
        const stem = word.slice(0, word.length - rule.suffix.length);
        return stem + rule.replacement;
    }
}

const steps = await Bun.file('path/to/portuguese.rslp').text().then(text => {
    const lines = text.split('\n');

    return parseRSLPRules(lines);
});

const stemmer = new RSLPStemmer(steps);

const words = ['corações', 'amigas', 'meninas', 'cão', 'pães', 'correndo', 'felizmente'];
const expected = ['coraç', 'amig', 'menin', 'cão', 'pães', 'corr', 'feliz'];

words.forEach((word, index) => {
    const stemmed = stemmer.stem(word);
    console.log(`Word: ${word} => Stemmed: ${stemmed} | Expected: ${expected[index]}`);
});