Python程序：批量抓取博客园文章并转换为Markdown格式

import os
import re
import requests
from bs4 import BeautifulSoup
import html2text
from urllib.parse import urlparse
import time

class CnblogsToMarkdown:
    def __init__(self, blog_url, output_dir='blog_posts'):
        """
        初始化博客园文章抓取和转换工具
        
        参数:
            blog_url: 博客园个人主页URL，如' https://www.cnblogs.com/yourname/'
            output_dir: 输出目录，默认为'blog_posts'
        """
        self.blog_url = blog_url
        self.output_dir = output_dir
        self.image_dir = os.path.join(output_dir, 'images')
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        }
        self.h = html2text.HTML2Text()
        self.h.body_width = 0  # 不换行
        self.h.ignore_links = False
        self.h.ignore_images = False
        
        # 创建输出目录
        os.makedirs(self.output_dir, exist_ok=True)
        os.makedirs(self.image_dir, exist_ok=True)

    def get_all_post_links(self, max_pages=10):
        """
        获取博客园所有文章的链接
        
        参数:
            max_pages: 最大爬取页数，默认为10
        返回:
            文章链接列表
        """
        post_links = []
        
        for page in range(1, max_pages + 1):
            page_url = f"{self.blog_url}?page={page}"
            try:
                response = requests.get(page_url, headers=self.headers)
                response.encoding = 'utf-8'
                soup = BeautifulSoup(response.text, 'lxml')
                
                # 查找文章链接
                post_items = soup.select('div.postTitle a')
                if not post_items:
                    break  # 没有更多文章了
                
                for item in post_items:
                    post_links.append(item['href'])
                
                print(f"已获取第 {page} 页的文章链接，共 {len(post_items)} 篇")
                
            except Exception as e:
                print(f"获取第 {page} 页文章链接失败: {str(e)}")
                continue
        
        return post_links

    def download_image(self, image_url):
        """
        下载图片到本地并返回本地路径
        
        参数:
            image_url: 图片URL
        返回:
            本地图片路径
        """
        try:
            # 解析图片文件名
            parsed_url = urlparse(image_url)
            image_name = os.path.basename(parsed_url.path)
            
            # 如果图片名无效，使用时间戳作为文件名
            if not image_name or '.' not in image_name:
                image_name = f"{int(time.time())}.jpg"
            
            local_path = os.path.join(self.image_dir, image_name)
            
            # 下载图片
            response = requests.get(image_url, headers=self.headers, stream=True)
            if response.status_code == 200:
                with open(local_path, 'wb') as f:
                    for chunk in response.iter_content(1024):
                        f.write(chunk)
                return local_path
        except Exception as e:
            print(f"下载图片失败: {image_url}, 错误: {str(e)}")
            return None

    def convert_to_markdown(self, post_url):
        """
        将单篇文章转换为Markdown格式并保存到本地
        
        参数:
            post_url: 文章URL
        返回:
            保存的文件路径
        """
        try:
            # 获取文章内容
            response = requests.get(post_url, headers=self.headers)
            response.encoding = 'utf-8'
            soup = BeautifulSoup(response.text, 'lxml')
            
            # 获取文章标题
            title = soup.find('h1', class_='postTitle').text.strip()
            # 清理标题中的非法文件名字符
            title = re.sub(r'[\\/*?:"<>|]', '', title)
            
            # 获取文章内容
            content_div = soup.find('div', class_='blogpost-body')
            if not content_div:
                content_div = soup.find('div', id='cnblogs_post_body')
            
            # 下载文章中的图片并替换链接
            for img in content_div.find_all('img'):
                img_url = img.get('src')
                if img_url and img_url.startswith('http'):
                    local_path = self.download_image(img_url)
                    if local_path:
                        # 使用相对路径替换图片链接
                        rel_path = os.path.relpath(local_path, self.output_dir)
                        img['src'] = rel_path.replace('\\', '/')  # 统一使用斜杠
            
            # 转换为Markdown
            html_content = str(content_div)
            markdown_content = self.h.handle(html_content)
            
            # 添加Front Matter (YAML格式的元数据)
            front_matter = f"""---
title: "{title}"
date: {time.strftime('%Y-%m-%d %H:%M:%S')}
original_url: "{post_url}"
---

"""
            markdown_content = front_matter + markdown_content
            
            # 保存Markdown文件
            filename = f"{title}.md"
            filepath = os.path.join(self.output_dir, filename)
            
            with open(filepath, 'w', encoding='utf-8') as f:
                f.write(markdown_content)
            
            print(f"已保存: {filename}")
            return filepath
            
        except Exception as e:
            print(f"处理文章 {post_url} 失败: {str(e)}")
            return None

    def run(self, max_pages=10):
        """
        运行整个抓取和转换流程
        
        参数:
            max_pages: 最大爬取页数，默认为10
        """
        print("开始获取博客园文章链接...")
        post_links = self.get_all_post_links(max_pages)
        
        print(f"\n共找到 {len(post_links)} 篇文章，开始转换...")
        for i, link in enumerate(post_links, 1):
            print(f"\n处理第 {i}/{len(post_links)} 篇: {link}")
            self.convert_to_markdown(link)
        
        print("\n所有文章处理完成！")


if __name__ == '__main__':
    # 使用示例
    your_blog_url = 'https://www.cnblogs.com/yourname/'  # 替换为你的博客园主页URL
    converter = CnblogsToMarkdown(your_blog_url)
    converter.run(max_pages=5)  # 抓取前5页的文章

pip install requests beautifulsoup4 html2text

python cnblogs_to_markdown.py