Python 10个自动处理txt文本的方法，提升效率

import chardet

def smart_read_txt(file_path):
    with open(file_path, 'rb') as f:
        raw_data = f.read()
        encoding = chardet.detect(raw_data)['encoding']
    
    with open(file_path, 'r', encoding=encoding) as f:
        return f.read()

# 示例调用
content = smart_read_txt('unknown_encoding.txt')
print(content[:500])  # 打印前500字符

def multi_filter_txt(file_path, exclude_keywords=None):
    if exclude_keywords is None:
        exclude_keywords = []
    
    with open(file_path, 'r', encoding='utf-8') as f:
        lines = f.readlines()
    
    filtered = []
    for line in lines:
        line_stripped = line.strip()
        if not line_stripped:  # 跳过空行
            continue
        if line_stripped.startswith('//'):  # 跳过注释
            continue
        if any(keyword in line for keyword in exclude_keywords):
            continue
        filtered.append(line)
    
    return filtered

# 示例调用
clean_lines = multi_filter_txt('server.log', ['DEBUG', 'TEST'])
for line in clean_lines:
    print(line.strip())

from difflib import SequenceMatcher

def text_similarity(file1, file2):
    with open(file1, 'r', encoding='utf-8') as f1, \
         open(file2, 'r', encoding='utf-8') as f2:
        text1 = f1.read()
        text2 = f2.read()
    
    similarity = SequenceMatcher(None, text1, text2).ratio()
    return round(similarity * 100, 2)

# 示例调用
similarity_score = text_similarity('doc_v1.txt', 'doc_v2.txt')
print(f"文本相似度: {similarity_score}%")

def reorganize_paragraphs(file_path, max_line_length=80):
    with open(file_path, 'r', encoding='utf-8') as f:
        content = f.read()
    
    paragraphs = content.split('\n\n')
    reformed_paragraphs = []
    
    for para in paragraphs:
        words = para.split()
        current_line = []
        reformed_para = []
        
        for word in words:
            if len(' '.join(current_line + [word])) <= max_line_length:
                current_line.append(word)
            else:
                reformed_para.append(' '.join(current_line))
                current_line = [word]
        
        if current_line:
            reformed_para.append(' '.join(current_line))
        
        reformed_paragraphs.append('\n'.join(reformed_para))
    
    return '\n\n'.join(reformed_paragraphs)

# 示例调用
new_content = reorganize_paragraphs('messy_text.txt')
print(new_content)

import os
import glob

def batch_rename_and_update(folder_path, old_str, new_str):
    txt_files = glob.glob(os.path.join(folder_path, '*.txt'))
    
    for file_path in txt_files:
        # 更新文件内容
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()
        
        new_content = content.replace(old_str, new_str)
        
        with open(file_path, 'w', encoding='utf-8') as f:
            f.write(new_content)
        
        # 更新文件名
        dir_name = os.path.dirname(file_path)
        base_name = os.path.basename(file_path)
        new_base_name = base_name.replace(old_str, new_str)
        new_file_path = os.path.join(dir_name, new_base_name)
        
        os.rename(file_path, new_file_path)
        print(f"已处理: {base_name} -> {new_base_name}")

# 示例调用
batch_rename_and_update('./documents', 'old_project', 'new_project')

def highlight_keywords(file_path, keywords, highlight_tag='**'):
    with open(file_path, 'r', encoding='utf-8') as f:
        content = f.read()
    
    for keyword in keywords:
        content = content.replace(keyword, f'{highlight_tag}{keyword}{highlight_tag}')
    
    return content

# 示例调用
highlighted = highlight_keywords('report.txt', ['重要', '紧急', '注意'], '==')
print(highlighted)

import nltk
from collections import defaultdict

def generate_summary(file_path, num_sentences=3):
    with open(file_path, 'r', encoding='utf-8') as f:
        text = f.read()
    
    sentences = nltk.sent_tokenize(text)
    word_freq = defaultdict(int)
    
    for sentence in sentences:
        words = nltk.word_tokenize(sentence.lower())
        for word in words:
            if word.isalpha():
                word_freq[word] += 1
    
    sentence_scores = defaultdict(int)
    for i, sentence in enumerate(sentences):
        words = nltk.word_tokenize(sentence.lower())
        for word in words:
            if word in word_freq:
                sentence_scores[i] += word_freq[word]
        sentence_scores[i] /= len(words)  # 平均词频
    
    top_sentences = sorted(sentence_scores, key=sentence_scores.get, reverse=True)[:num_sentences]
    summary = ' '.join([sentences[i] for i in sorted(top_sentences)])
    
    return summary

# 示例调用（需要先安装nltk和下载punkt）
summary = generate_summary('long_document.txt')
print("文档摘要:", summary)

from textblob import TextBlob

def sentiment_analysis(file_path):
    with open(file_path, 'r', encoding='utf-8') as f:
        text = f.read()
    
    blob = TextBlob(text)
    sentiment = blob.sentiment
    
    if sentiment.polarity > 0.1:
        emotion = "积极"
    elif sentiment.polarity < -0.1:
        emotion = "消极"
    else:
        emotion = "中性"
    
    return {
        'polarity': round(sentiment.polarity, 3),
        'subjectivity': round(sentiment.subjectivity, 3),
        'emotion': emotion
    }

# 示例调用
result = sentiment_analysis('customer_feedback.txt')
print(f"情感极性: {result['polarity']}")
print(f"主观程度: {result['subjectivity']}")
print(f"情感分类: {result['emotion']}")

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB

def text_category_predictor(file_path, training_data):
    """
    training_data格式: [('文本内容', '类别'), ...]
    """
    texts, labels = zip(*training_data)
    
    vectorizer = TfidfVectorizer()
    X_train = vectorizer.fit_transform(texts)
    
    classifier = MultinomialNB()
    classifier.fit(X_train, labels)
    
    with open(file_path, 'r', encoding='utf-8') as f:
        new_text = f.read()
    
    X_new = vectorizer.transform([new_text])
    prediction = classifier.predict(X_new)
    
    return prediction

# 示例调用
training_examples = [
    ('技术支持相关问题', '技术'),
    ('销售和价格咨询', '销售'),
    ('产品功能问题', '产品'),
    ('投诉和建议', '客服')
]

category = text_category_predictor('new_ticket.txt', training_examples)
print(f"预测类别: {category}")

from datetime import datetime

def auto_generate_report(data_dict, template_file, output_file):
    with open(template_file, 'r', encoding='utf-8') as f:
        template = f.read()
    
    # 添加时间戳
    data_dict['generate_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    
    # 替换模板变量
    report_content = template
    for key, value in data_dict.items():
        report_content = report_content.replace(f'{{{{{key}}}}}', str(value))
    
    with open(output_file, 'w', encoding='utf-8') as f:
        f.write(report_content)
    
    return output_file

# 示例调用
report_data = {
    'project_name': '数据分析项目',
    'author': '张三',
    'completion_rate': '85%',
    'key_findings': '发现用户活跃度提升明显',
    'next_steps': '继续优化算法模型'
}

output_path = auto_generate_report(report_data, 'report_template.txt', 'final_report.txt')
print(f"报告已生成: {output_path}")