Java RAG系统文档预处理性能瓶颈实录：从String.replaceAll高频调用到Unicode字符集陷阱

// 业务代码中的典型"防御性"清洗
public String cleanText(String raw) {
    return raw.replaceAll("\\s+", " ")           // 收缩空白
              .replaceAll("[\\p{Punct}&&[^.,]]", "") // 移除标点（坑点）
              .replaceAll("[\\u200B\\uFEFF]", "");    // 移除零宽字符
}

// java.lang.String.replaceAll
public String replaceAll(String regex, String replacement) {
    return Pattern.compile(regex).matcher(this).replaceAll(replacement);
}

private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
private static final Pattern PUNCT_PATTERN = Pattern.compile("[\\p{Punct}&&[^.,]]");
private static final Pattern ZERO_WIDTH_PATTERN = Pattern.compile("[\\u200B\\uFEFF]");

public String cleanText(String raw) {
    return ZERO_WIDTH_PATTERN.matcher(
               PUNCT_PATTERN.matcher(
                   WHITESPACE_PATTERN.matcher(raw).replaceAll(" ")
               ).replaceAll("")
           ).replaceAll("");
}

public class FastCharFilter {
    private static final boolean[] IS_PUNCT = new boolean[Character.MAX_CODE_POINT];
    private static final boolean[] IS_ZERO_WIDTH = new boolean[Character.MAX_CODE_POINT];
    
    static {
        // 预计算：只针对ASCII常用标点做精确映射，放弃Unicode全量属性
        for (int cp = 0; cp < 0x7F; cp++) {
            if (cp == '!' || cp == '"' || cp == '#' || cp == '$' || cp == '%' || cp == '&' || cp == '\'' ||
                cp == '(' || cp == ')' || cp == '*' || cp == '+' || cp == ',' || cp == '-' || cp == '.' ||
                cp == '/' || cp == ':' || cp == ';' || cp == '<' || cp == '=' || cp == '>' || cp == '?' ||
                cp == '@' || cp == '[' || cp == '\\' || cp == ']' || cp == '^' || cp == '_' || cp == '`' ||
                cp == '{' || cp == '|' || cp == '}' || cp == '~') {
                IS_PUNCT[cp] = true;
            }
        }
        // 零宽字符精确匹配
        IS_ZERO_WIDTH[0x200B] = true; // ZERO WIDTH SPACE
        IS_ZERO_WIDTH[0xFEFF] = true; // BOM
        IS_ZERO_WIDTH[0x200C] = true; // ZWNJ
        IS_ZERO_WIDTH[0x200D] = true; // ZWJ
    }

    public static String filter(CharSequence input) {
        StringBuilder sb = new StringBuilder(input.length());
        boolean lastWasSpace = false;
        for (int i = 0; i < input.length(); ) {
            int cp = input.codePointAt(i);
            i += Character.charCount(cp);
            
            // 1. 过滤零宽字符
            if (cp < IS_ZERO_WIDTH.length && IS_ZERO_WIDTH[cp]) {
                continue;
            }
            
            // 2. 空白处理（仅处理ASCII空格和\t\n\r）
            if (cp == ' ' || cp == '\t' || cp == '\n' || cp == '\r') {
                if (!lastWasSpace) {
                    sb.append(' ');
                    lastWasSpace = true;
                }
                continue;
            }
            lastWasSpace = false;
            
            // 3. 标点过滤（仅处理ASCII）
            if (cp < IS_PUNCT.length && IS_PUNCT[cp]) {
                continue;
            }
            
            // 4. 追加有效字符（处理增补平面）
            sb.appendCodePoint(cp);
        }
        return sb.toString();
    }
}