打开anaconda prompt
conda activate pytorch
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple transformers datasets tokenizers
pycharm
找到pytorch下的python.exe
#将模型下载到本地调用 from transformers import AutoModelForCausalLM,AutoTokenizer#将模型和分词工具下载到本地,并指定保存路径 model_name = "uer/gpt2-chinese-cluecorpussmall" cache_dir = "model/uer/gpt2-chinese-cluecorpussmall"#下载模型 AutoModelForCausalLM.from_pretrained(model_name, cache_dir=cache_dir) #下载分词工具 AutoTokenizer.from_pretrained(model_name,cache_dir=cache_dir)print(f"模型分词器已下载到:{cache_dir}")
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline# 设置具体包含config.json的目录 model_dir = r"D:\pycharm\sth\demo\transFormers_test\model\uer\gpt2-chinese-cluecorpussmall\models--uer--gpt2-chinese-cluecorpussmall\snapshots\c2c0249d8a2731f269414cc3b22dff021f8e07a3"# 将模型和分词工具下载到本地,并指定保存路径 model = AutoModelForCausalLM.from_pretrained(model_dir) tokenizer = AutoTokenizer.from_pretrained(model_dir)# 使用模型和分词器创建生成文本的pipeline generator = pipeline("text-generation", model=model, tokenizer=tokenizer,device="cuda")# 生成内容 # output = generator("你好,我是一款语言模型,",max_length=50,num_return_sequences=1)output = generator("你好,我是一款语言模型,",num_return_sequences=1, # 设置返回多少个独立的生成序列max_length=50,truncation=True,# 生成文本以适应文本最大长度temperature=0.7,# 控制文本生成的随机性,值越高,生成多样性越好top_k=50, # 限制模型在每一步生成时仅从概率最高的k个词中随机选择下一个词top_p=0.9, # 进一步限制模型生成时的词汇选择范围,选择一组概率累计达到p的词汇,模型只会从这个概率集合中采样clean_up_tokenization_spaces=True # 设置生成文本分词时的空格是否保留 ) print(output)
from datasets import load_dataset, load_from_disk# # 在线加载 dataset = load_dataset("lansinuote/ChnSentiCorp", cache_dir="E:/DeepLearning/data") # # 保存为可本地加载的格式 save_path = "E:/DeepLearning/data/my_chn_senti_corp" # dataset.save_to_disk(save_path) # 从本地加载 loaded_dataset = load_from_disk(save_path) print(loaded_dataset)test_data = dataset["test"] print(test_data) for data in test_data:print(data)