import re, json, sys , os
import pandas as pd


# 非AI项目 资源清单 数据库  每条数据转为json字符串
def knowledge_json(file, sheet_names:list, save_file):
    length_record = []
    save_jsonl_data = []
    for sheet_name in sheet_names:
        data = pd.read_excel(file, sheet_name=sheet_name, keep_default_na=False)
        data = data.to_dict(orient='records')

        for line in data:
            if "序号" in line:
                line.pop("序号")

            try:
                line = json.dumps(line, ensure_ascii=False)
                line = re.sub("\n", "", line)
                line = re.sub("\t", "", line)

                length_record.append(len(line))
                if len(line) > 3000:
                    print(line)
                save_jsonl_data.append(line)
            except:
                print(line)

    res =  "\n".join(save_jsonl_data)

    with open(save_file, 'w', encoding='utf-8') as f:
        f.write(res)

    print(f"max length: {max(length_record)}")



if __name__ == "__main__":
    file = r"D:\0_shu_chuan_work\work\AI_planner\data\非AI工具.xlsx"
    save_file = r"D:\0_shu_chuan_work\work\AI_planner\data\伴学工具0.txt"  # max length: 157
    sheet_names = ['伴学工具20250214']    # max length: 157
    sheet_names = ['第三方自有资源评级详表']  # max length: 429
    sheet_names = ['品牌资源（爱奇艺+慕课）']  # max length: 333
    sheet_names = ['小睿资讯服务']          # max length: 2916
    sheet_names = ['测评库资源']       #  max length: 259
    knowledge_json(file, sheet_names, save_file)


    print("finished.")

