https://zh.annas-archive.org/torrents

如题，解压前 26G ，解压后 458G

让 GPT 写了个脚本转到 sqlite ，看看会有多大。

脚本如下：

# -*- coding: utf-8 -*-
import sqlite3
import json
def create_table_from_jsonl(file_path, db_path):
连接到 SQLite 数据库
conn = sqlite3.connect(db_path)
cursor = conn.cursor()

# 读取第一行以推断表结构
with open(file_path, 'r', encoding='utf-8') as file:
    first_line = file.readline()
    first_record = json.loads(first_line)
    columns = list(first_record.keys())

    # 创建具有推断列的表
    columns_definition = ", ".join([f"{col} TEXT" for col in columns])
    cursor.execute(f"CREATE TABLE IF NOT EXISTS data ({columns_definition});")
    conn.commit()

write_count = 0  # 写入计数器

# 插入其余数据
with open(file_path, 'r', encoding='utf-8') as file:
    for line in file:
        record = json.loads(line)
        placeholders = ", ".join(["?"] * len(columns))
        values = [json.dumps(record.get(col, None)) if isinstance(record.get(col, None), (dict, list)) else record.get(col, None) for col in columns]

        # 只在值数量和列数量一致时执行插入操作
        if len(values) == len(columns):
            cursor.execute(f"INSERT INTO data ({', '.join(columns)}) VALUES ({placeholders});", values)
            write_count += 1
            print(f"这是第 {write_count} 次写入")

        else:
            print(f"Skipping record due to mismatched column count: {record}")

        # 定期提交以保存更改并释放内存
        if write_count % 1000 == 0:
            conn.commit()

# 最终提交并关闭连接
conn.commit()
conn.close()
上传的 JSONL 文件路径
file_path = '/mnt/data/sample.jsonl'
SQLite 数据库文件路径
db_path = '/mnt/data/sample.db'
创建表并插入数据
create_table_from_jsonl(file_path, db_path)
print("数据已成功写入 SQLite 数据库。")

ps V 站这个 markdown 没有语法高亮的么……