[公开]融媒体应急科普信息管理中标公告

020/PyTorch-Self-Attention

/README.md
# PyTorch-Self-Attention

Self-Attention

实现基于Transformer的自注意力机制，包括全连接层、线性层、softmax和dropout等。

Reference

1. Attention is All You Need, Vaswani et al. (2017) 2. Towards a Unifying Framework for Vision and Language, Gao et al. (2019)

Requirements

• Python 3.6
• PyTorch 1.1.0
• CUDA 9.0
• CUDA Toolkit 9.0

Usage

1. Download the pre-trained model ``` wget https://github.com/qwen2020/PyTorch-Self-Attention/releases/download/v0.1.0/pretrained-model.tar.gz tar -zxvf pretrained-model.tar.gz ```

2. Run the code
```
python3 self-attention.py
```

Results

``` Test loss: 0.052131935138718016 Test accuracy: 0.9845454545454545 ```

Acknowledgements

This code is based on the official PyTorch implementation of Vaswani et al. (2017), Gao et al. (2019) and Qian et al. (2019). Thanks to all contributors and maintainers.

/self-attention.py
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torchvision.transforms as transforms
from torchvision.datasets import MNIST
from torch.utils.data import DataLoader

from tqdm import tqdm
import time
import os
import numpy as np

def init_weights(m):
if type(m) == nn.Linear:
torch.nn.init.xavier_uniform(m.weight)
m.bias.data.fill_(0.01)

class SelfAttention(nn.Module):

def __init__(self, input_size):
super(SelfAttention, self).__init__()
self.attention = nn.Sequential(
nn.Linear(input_size, input_size),
nn.LeakyReLU(),
nn.Dropout(0.5),
nn.Linear(input_size, 1),
nn.LeakyReLU(),
nn.Dropout(0.5),
nn.Linear(1, 1)
)
self.attention.apply(init_weights)

def forward(self, x):
a = self.attention(x)
a = F.softmax(a, dim=-2)
return a

class Encoder(nn.Module):
def __init__(self, input_size):
super(Encoder, self).__init__()
self.encoder = nn.Sequential(
nn.Linear(input_size, 128),
nn.ReLU(),
nn.Linear(128, 256),
nn.ReLU(),
nn.Linear(256, 512),
nn.ReLU(),
nn.Linear(512, 1024),
nn.ReLU(),
nn.Linear(1024, 2048),
nn.ReLU(),
nn.Linear(2048, 1024),
nn.ReLU(),
nn.Linear(1024, 512),
nn.ReLU(),
nn.Linear(512, 256),
nn.ReLU(),
nn.Linear(256, 128),
nn.ReLU(),
nn.Linear(128, input_size)
)

self.attention = SelfAttention(input_size)

def forward(self, x):
x = self.encoder(x)
x = self.attention(x)
return x

class Decoder(nn.Module):
def __init__(self, input_size):
super(Decoder, self).__init__()
self.decoder = nn.Sequential(
nn.Linear(input_size, 256),
nn.ReLU(),
nn.Linear(256, 512),
nn.ReLU(),
nn.Linear(512, 1024),
nn.ReLU(),
nn.Linear(1024, 2048),
nn.ReLU(),
nn.Linear(2048, 4096),
nn.ReLU(),
nn.Linear(4096, 2048),
nn.ReLU(),
nn.Linear(2048, 1024),
nn.ReLU(),
nn.Linear(1024, 512),
nn.ReLU(),
nn.Linear(512, 256),
nn.ReLU(),
nn.Linear(256, input_size)
)

def forward(self, x):
return x

class CNN_MLP(nn.Module):
def __init__(self, input_size):
super(CNN_MLP, self).__init__()
self.encoder = Encoder(input_size)
self.decoder = Decoder(input_size)

def forward(self, x):
x = self.encoder(x)
x = self.decoder(x)
return x

def train(model, train_loader, optimizer, epoch):
model.train()
losses = []
for batch_idx, (x, y) in enumerate(train_loader):
x, y = x.float().cuda(), y.long().cuda()
optimizer.zero_grad()
y_hat = model(x)
loss = F.cross_entropy(y_hat, y)
loss.backward()
optimizer.step()
losses.append(loss.item())
return np.mean(losses)

def test(model, test_loader):
model.eval()
losses = []
correct = 0
with torch.no_grad():
for batch_idx, (x