支招 | 使用Pytorch进行文本分类

• 2019 年 10 月 4 日
• 筆記

01. BILSTM+ATTENTION

网络结构

代码实现

Attention计算

1. 将BILSTM网络输出的结果（shape：[batch_size, time_step, hidden_dims * num_directions(=2)]）拆成两个大小为[batch_size, time_step, hidden_dims]的Tensor；
2. 将第一步拆出的两个Tensor进行相加运算得到h（shape：[batch_size, time_step, hidden_dims]）；
3. 将h作为输入，通过self.attention_layer得到attention的计算向量atten_w（shape：[batch_size, time_step, hidden_dims]）；
4. 将第二步的h进行tanh()激活，得到m（shape：[batch_size, time_step, hidden_dims]），留待后续进行残差计算；
5. 将atten_w的2、3维度进行调换，并与m进行矩阵的乘法运算，得到atten_context（shape：[batch_size, time_step, time_step]）;
6. 将atten_context最后一维进行softmax计算，得到当前时刻相对于所有时刻的权重：softmax_w（shape：[batch_size, time_step, time_step]）；
7. 将h的2、3维度进行调换，并与softmax_w进行矩阵运算，得到基于权重的context（shape：[batch_size, hidden_dims, time_step]）；
8. 将h的2、3维度进行调换，并与context进行求和运算，得到context_with_attn（shape：[batch_size, hidden_dims, time_step]）；
9. 将context_with_attn的最后一维进行求和，得到result（shape：[batch_size, hidden_dims]）；
10. 计算结束，返回result；

模型效果

1层BILSTM在训练集准确率：99.8%，测试集准确率：96.5%；

2层BILSTM在训练集准确率：99.9%，测试集准确率：97.3%；

调参

dropout的值要在 0.1 以下（经验之谈，笔者在实践中发现，dropout取0.1时比dropout取0.3时在测试集准确率能提高0.5%）。

02. Transformer

前言

文本分类不是生成式的任务，因此只使用Transformer的编码部分（Encoder）进行特征提取。如果不熟悉Transformer模型的原理请移步：https://blog.csdn.net/dendi_hust/article/details/98759771

网络结构

代码实现

• 自注意力模型：

class TextSlfAttnNet(nn.Module):      ''' 自注意力模型 '''        def __init__(self,                   config: TextSlfAttnConfig,                   char_size=5000,                   pinyin_size=5000):          super(TextSlfAttnNet, self).__init__()          # 字向量          self.char_embedding = nn.Embedding(char_size, config.embedding_size)          # 拼音向量          self.pinyin_embedding = nn.Embedding(pinyin_size, config.embedding_size)          # 位置向量          self.pos_embedding = nn.Embedding.from_pretrained(              get_sinusoid_encoding_table(config.max_sen_len, config.embedding_size, padding_idx=0),              freeze=True)            self.layer_stack = nn.ModuleList([              EncoderLayer(config.embedding_size, config.hidden_dims, config.n_heads, config.k_dims, config.v_dims, dropout=config.keep_dropout)              for _ in range(config.hidden_layers)          ])            self.fc_out = nn.Sequential(              nn.Dropout(config.keep_dropout),              nn.Linear(config.embedding_size, config.hidden_dims),              nn.ReLU(inplace=True),              nn.Dropout(config.keep_dropout),              nn.Linear(config.hidden_dims, config.num_classes),            )          def forward(self, char_id, pinyin_id, pos_id):          char_inputs = self.char_embedding(char_id)          pinyin_iputs = self.pinyin_embedding(pinyin_id)            sen_inputs = torch.cat((char_inputs, pinyin_iputs), dim=1)          # sentence_length = sen_inputs.size()[1]          # pos_id = torch.LongTensor(np.array([i for i in range(sentence_length)]))          pos_inputs = self.pos_embedding(pos_id)          # batch_size * sen_len * embedding_size          inputs = sen_inputs + pos_inputs            for layer in self.layer_stack:              inputs, _ = layer(inputs)            enc_outs = inputs.permute(0, 2, 1)          enc_outs = torch.sum(enc_outs, dim=-1)          return self.fc_out(enc_outs)

• 编码层（EncoderLayer）

  class EncoderLayer(nn.Module):      '''编码层'''        def __init__(self, d_model, d_inner, n_head, d_k, d_v, dropout=0.1):          '''            :param d_model: 模型输入维度          :param d_inner: 前馈神经网络隐层维度          :param n_head:  多头注意力          :param d_k:     键向量          :param d_v:     值向量          :param dropout:          '''          super(EncoderLayer, self).__init__()          self.slf_attn = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)          self.pos_ffn = PositionwiseFeedForward(d_model, d_inner, dropout=dropout)        def forward(self, enc_input, non_pad_mask=None, slf_attn_mask=None):          '''            :param enc_input:          :param non_pad_mask:          :param slf_attn_mask:          :return:          '''          enc_output, enc_slf_attn = self.slf_attn(enc_input, enc_input, enc_input, mask=slf_attn_mask)          if non_pad_mask is not None:              enc_output *= non_pad_mask            enc_output = self.pos_ffn(enc_output)          if non_pad_mask is not None:              enc_output *= non_pad_mask          return enc_output, enc_slf_attn

• 多头注意力（`MultiHeadAttention`）

class MultiHeadAttention(nn.Module):      '''          “多头”注意力模型      '''        def __init__(self, n_head, d_model, d_k, d_v, dropout=0.1):          '''            :param n_head: “头”数          :param d_model: 输入维度          :param d_k: 键向量维度          :param d_v: 值向量维度          :param dropout:          '''          super(MultiHeadAttention, self).__init__()            self.n_head = n_head          self.d_k = d_k          self.d_v = d_v          # 产生 查询向量q，键向量k， 值向量v          self.w_qs = nn.Linear(d_model, n_head * d_k)          self.w_ks = nn.Linear(d_model, n_head * d_k)          self.w_vs = nn.Linear(d_model, n_head * d_v)            nn.init.normal_(self.w_qs.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_k)))          nn.init.normal_(self.w_ks.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_k)))          nn.init.normal_(self.w_vs.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_v)))            self.attention = ScaledDotProductAttention(temperature=np.power(d_k, 0.5))          self.layer_normal = nn.LayerNorm(d_model)            self.fc = nn.Linear(n_head * d_v, d_model)          nn.init.xavier_normal_(self.fc.weight)            self.dropout = nn.Dropout(dropout)          def forward(self, q, k, v, mask=None):          '''          计算多头注意力          :param q: 用于产生  查询向量          :param k: 用于产生  键向量          :param v:  用于产生 值向量          :param mask:          :return:          '''          d_k, d_v, n_head = self.d_k, self.d_v, self.n_head            sz_b, len_q, _ = q.size()          sz_b, len_k, _ = k.size()          sz_b, len_v, _ = v.size()            residual = q            q = self.w_qs(q).view(sz_b, len_q, n_head, d_k)          k = self.w_ks(k).view(sz_b, len_k, n_head, d_k)          v = self.w_vs(v).view(sz_b, len_v, n_head, d_v)            # (n*b) x lq x dk          q = q.permute(2, 0, 1, 3).contiguous().view(-1, len_q, d_k)          # (n*b) x lk x dk          k = k.permute(2, 0, 1, 3).contiguous().view(-1, len_k, d_k)          # (n*b) x lv x dv          v = v.permute(2, 0, 1, 3).contiguous().view(-1, len_v, d_v)            # mask = mask.repeat(n_head, 1, 1)  # (n*b) x .. x ..          #          output, attn = self.attention(q, k, v, mask=None)          # (n_heads * batch_size) * lq * dv          output = output.view(n_head, sz_b, len_q, d_v)          # batch_size * len_q * (n_heads * dv)          output = output.permute(1, 2, 0, 3).contiguous().view(sz_b, len_q, -1)          output = self.dropout(self.fc(output))          output = self.layer_normal(output + residual)          return output, attn

• 前馈神经网络（`PositionwiseFeedForward`）

class PositionwiseFeedForward(nn.Module):      '''          前馈神经网络      '''        def __init__(self, d_in, d_hid, dropout=0.1):          '''            :param d_in:    输入维度          :param d_hid:   隐藏层维度          :param dropout:          '''          super(PositionwiseFeedForward, self).__init__()          self.w_1 = nn.Conv1d(d_in, d_hid, 1)          self.w_2 = nn.Conv1d(d_hid, d_in, 1)          self.layer_normal = nn.LayerNorm(d_in)          self.dropout = nn.Dropout(dropout)        def forward(self, x):          residual = x          output = x.transpose(1, 2)          output = self.w_2(F.relu(self.w_1(output)))          output = output.transpose(1, 2)          output = self.dropout(output)          output = self.layer_normal(output + residual)          return output

• 位置函数

def get_sinusoid_encoding_table(n_position, d_hid, padding_idx=None):      '''      计算位置向量      :param n_position:      位置的最大值      :param d_hid:           位置向量的维度，和字向量维度相同（要相加求和）      :param padding_idx:      :return:      '''        def cal_angle(position, hid_idx):          return position / np.power(10000, 2 * (hid_idx // 2) / d_hid)        def get_posi_angle_vec(position):          return [cal_angle(position, hid_j) for hid_j in range(d_hid)]        sinusoid_table = np.array([get_posi_angle_vec(pos_i) for pos_i in range(n_position)])        sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2])  # dim 2i      sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2])  # dim 2i+1        if padding_idx is not None:          # zero vector for padding dimension          sinusoid_table[padding_idx] = 0.        return torch.FloatTensor(sinusoid_table)

实践经验

在分类任务中，与`BILSTM+ATTENTION`相比：

1. 模型比`LSTM`大很多，同样的任务`LSTM`模型6M左右，`Transformer`模型55M；

2. 收敛速度比较慢；

3. 超参比较多，不易调参，但同时也意味着弹性比较大；

4. 效果和`BILSTM`模型差不多。

03. TextCNN

网络结构

代码实现

class TextCNN(nn.Module):      def __init__(self,                   config:TCNNConfig,                   char_size = 5000, pinyin_size=5000):          super(TextCNN, self).__init__()          self.learning_rate = config.learning_rate          self.keep_dropout = config.keep_dropout          self.sequence_length = config.sequence_length          self.char_embedding_size = config.char_embedding_size          self.pinyin_embedding_size = config.pinyin_embedding_size          self.filter_list = config.filter_list          self.out_channels = config.out_channels          self.l2_reg_lambda = config.l2_reg_lambda          self.model_dir = config.model_dir          self.data_save_frequency = config.data_save_frequency          self.model_save_frequency = config.model_save_frequency          self.char_size = char_size          self.pinyin_size = pinyin_size          self.embedding_size = self.char_embedding_size          self.total_filters_size = self.out_channels * len(self.filter_list)          self.build_model()      def build_model(self):  # 初始化字向量          self.char_embeddings = nn.Embedding(self.char_size, self.char_embedding_size)  # 字向量参与更新          self.char_embeddings.weight.requires_grad = True  # 初始化拼音向量          self.pinyin_embeddings = nn.Embedding(self.pinyin_size, self.pinyin_embedding_size)          self.pinyin_embeddings.weight.requires_grad = True          self.conv_list = nn.ModuleList()              conv_list = [nn.Sequential(              nn.Conv1d(self.embedding_size, self.out_channels, filter_size),              nn.BatchNorm1d(self.out_channels),              nn.ReLU(inplace=True)  ) for filter_size in self.filter_list]  # 卷积列表          self.conv_lists_layer = nn.ModuleList(conv_list)                      self.output_layer = nn.Sequential(              nn.Dropout(self.keep_dropout),              nn.Linear(self.total_filters_size, self.total_filters_size),              nn.ReLU(inplace=True),              nn.Linear(self.total_filters_size, 2)  )      def forward(self, char_id, pinyin_id):  # char_id = torch.from_numpy(np.array(input[0])).long()  # pinyin_id = torch.from_numpy(np.array(input[1])).long()          pooled_outputs = []          sen_char = self.char_embeddings(char_id)          sen_pinyin = self.pinyin_embeddings(pinyin_id)          sen_embed = torch.cat((sen_char, sen_pinyin), dim=1)  # 转换成 (N C SEN_LEN) 的形式          sen_embed = sen_embed.permute(0, 2, 1)  for conv in self.conv_lists_layer:  # print(sen_embed.shape)              conv_output = conv(sen_embed)              max_polling_output = torch.max(conv_output, dim=2)              pooled_outputs.append(max_polling_output[0])              total_pool = torch.cat(pooled_outputs, 1)          flatten_pool = total_pool.view(-1, self.total_filters_size)          fc_output = self.output_layer(flatten_pool)  return fc_output