GPT-2模型
模型來源：從下述網站上下載config.json, merges.txt, tokenizer.json, vocal.json以及pytorch_model.bingpt2 at main (huggingfac
模型來源：
從下述網站上下載config.json, merges.txt, tokenizer.json, vocal.json以及pytorch_model.bin
gpt2 at main (huggingface.co)文件結構：
代碼示例（finetune.py）：
from torch.utils.data import TensorDatasetnfrom torch.utils.data import DataLoadernfrom tqdm import trangennfrom transformers import GPT2Tokenizer, GPT2LMHeadModelnimport randomnimport torchnnnmodel_path = './models/gpt-2/'ndata_path = './data/'ntokenizer = GPT2Tokenizer.from_pretrained(model_path) n ndef select_top_k(predictions, k=10):n '''n 執行top-k選擇，避免出現單詞循環現象的出現n @params prediction: GPT-2模型對下一個詞的預測向量n @params k : 默認 k=10n '''n predicted_index = random.choice(n predictions[0, -1, :].sort(descending=True)[1][:10]).item()n return predicted_indexnndef preprocess_data(path):n '''n 數據預處理n @params path: 數據的路徑n return : List[int] 分成固定長度的"token索引片段"n '''n with open(path,'r') as f:n dataset = f.read()nn # print(len(dataset))nn indexed_text = tokenizer.encode(dataset)n del(dataset)nn dataset_cut = []n for i in range(len(indexed_text)//512):n # 將文本分段成 長度為512的片段n dataset_cut.append(indexed_text[i*512:(i+1)*512])n del(indexed_text)n return dataset_cutnndef do_train(epoch:int,optimizer:torch.optim.Optimizer,model,train_loader:DataLoader):n '''n 進行模型訓練n @params epoch: 訓練的循環次數n @params optimizer: 參數優化器n @params model: 使用的預訓練模型(GPT-2)n @params train_loader: 訓練數據集n '''n model.train()n for epoch in trange(epoch, desc="Epoch"):n total_loss = 0n for batch_idx ,(data,target) in enumerate(train_loader):n optimizer.zero_grad()n # print(batch_idx ,(data,target))n loss = model(data,labels=target).lossn total_loss += lossn loss.backward()n optimizer.step()n print('average loss:',total_loss/len(train_loader))nndef do_test(text,length,model):n '''n 模型根據前文內容預測后續句子n @params text: 前文n @params length: 后續句子的長度n @params model: 使用的預訓練模型(GPT-2)n '''n indexed_tokens = tokenizer.encode(text)n tokens_tensor = torch.tensor([indexed_tokens])n n model.eval()n total_predicted_text = textn n # 使訓練后的模型進行 500 次預測n for _ in range(length):n tokens_tensor = tokens_tensor.to('cuda')n n with torch.no_grad():n outputs = model(tokens_tensor)n predictions = outputs[0]n n predicted_index = select_top_k(predictions, k=10)n n predicted_text = tokenizer.decode(indexed_tokens + [predicted_index])n total_predicted_text += tokenizer.decode(predicted_index)n if '<|endoftext|>' in total_predicted_text:n # 如果出現文本結束標志，就結束文本生成n breakn n indexed_tokens += [predicted_index]n n if len(indexed_tokens) > 1023:n # 模型最長輸入長度為1024，如果長度過長則截斷n indexed_tokens = indexed_tokens[-1023:]n n tokens_tensor = torch.tensor([indexed_tokens])n n print(total_predicted_text)nndef main():n DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')nn ## load datan dataset_cut = preprocess_data(data_path+'romeo_and_juliet.txt')n dataset_tensor = torch.tensor(dataset_cut).to(DEVICE)nn train_set = TensorDataset(dataset_tensor,dataset_tensor) # 標簽與樣本數據相同n train_loader = DataLoader(dataset = train_set,batch_size = 2, shuffle=False)nn ## load modeln model = GPT2LMHeadModel.from_pretrained(model_path+'pytorch_model.bin',config=model_path+'config.json')n model.to(DEVICE)nn epoch = 30n optimizer = torch.optim.Adam(model.parameters(),lr=1e-5)nn ## model trainingn do_train(epoch,optimizer,model,train_loader)nn ## do a testn do_test(text="At least, the whisper goes so.",length=500,model=model)n nn nif __name__ == "__main__":n main()
