From c3d104a69b854fa127dda60628a41b570de36561 Mon Sep 17 00:00:00 2001
From: prabhuomkar <prabhuomkar@pm.me>
Date: Sat, 26 Sep 2020 11:57:01 +0530
Subject: [PATCH] Initialized transformers, planning for october release

---
 tutorials/popular/transformers/CMakeLists.txt | 29 +++++++++
 .../include/positional_encoding.h             | 17 +++++
 .../transformers/include/transformer.h        | 30 +++++++++
 tutorials/popular/transformers/src/main.cpp   | 62 +++++++++++++++++++
 .../transformers/src/positional_encoding.cpp  | 29 +++++++++
 .../popular/transformers/src/transformer.cpp  | 56 +++++++++++++++++
 6 files changed, 223 insertions(+)
 create mode 100644 tutorials/popular/transformers/CMakeLists.txt
 create mode 100644 tutorials/popular/transformers/include/positional_encoding.h
 create mode 100644 tutorials/popular/transformers/include/transformer.h
 create mode 100644 tutorials/popular/transformers/src/main.cpp
 create mode 100644 tutorials/popular/transformers/src/positional_encoding.cpp
 create mode 100644 tutorials/popular/transformers/src/transformer.cpp

diff --git a/tutorials/popular/transformers/CMakeLists.txt b/tutorials/popular/transformers/CMakeLists.txt
new file mode 100644
index 0000000..5f04d96
--- /dev/null
+++ b/tutorials/popular/transformers/CMakeLists.txt
@@ -0,0 +1,29 @@
+cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
+
+project(transformers VERSION 1.0.0 LANGUAGES CXX)
+
+set(EXECUTABLE_NAME transformers)
+
+add_executable(${EXECUTABLE_NAME})
+target_sources(${EXECUTABLE_NAME} PRIVATE src/main.cpp
+                                          src/transformer.cpp
+                                          src/positional_encoding.cpp
+                                          include/transformer.h
+                                          include/positional_encoding.h
+)
+
+target_include_directories(${EXECUTABLE_NAME} PRIVATE include)
+
+target_link_libraries(${EXECUTABLE_NAME} ${TORCH_LIBRARIES})
+
+set_target_properties(${EXECUTABLE_NAME} PROPERTIES
+  CXX_STANDARD 14
+  CXX_STANDARD_REQUIRED YES
+)
+
+if(MSVC)
+    set_target_properties(${EXECUTABLE_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR})
+    set_target_properties(${EXECUTABLE_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR})
+    include(copy_torch_dlls)
+    copy_torch_dlls(${EXECUTABLE_NAME})
+endif(MSVC)
diff --git a/tutorials/popular/transformers/include/positional_encoding.h b/tutorials/popular/transformers/include/positional_encoding.h
new file mode 100644
index 0000000..79c9751
--- /dev/null
+++ b/tutorials/popular/transformers/include/positional_encoding.h
@@ -0,0 +1,17 @@
+// Copyright 2020-present pytorch-cpp Authors
+#pragma once
+
+#include <torch/torch.h>
+
+class PositionalEncodingImpl : public torch::nn::Module {
+ public:
+    PositionalEncodingImpl(int64_t d_model,
+                           double dropout = 0.1,
+                           int64_t max_len = 5000);
+    torch::Tensor forward(torch::Tensor x);
+
+    torch::nn::Dropout drpout;
+    torch::Tensor pe;
+};
+
+TORCH_MODULE(PositionalEncoding);
diff --git a/tutorials/popular/transformers/include/transformer.h b/tutorials/popular/transformers/include/transformer.h
new file mode 100644
index 0000000..f4be06e
--- /dev/null
+++ b/tutorials/popular/transformers/include/transformer.h
@@ -0,0 +1,30 @@
+// Copyright 2020-present pytorch-cpp Authors
+#pragma once
+
+#include <torch/torch.h>
+#include "positional_encoding.h"
+
+class TransformerImpl : public torch::nn::Module {
+ public:
+    TransformerImpl(int64_t ntoken,
+                    int64_t ninp,
+                    int64_t nhead,
+                    int64_t nhid,
+                    int64_t nlayers,
+                    double dropout = 0.5);
+    torch::Tensor forward(torch::Tensor src);
+
+    torch::Tensor src_mask;
+    PositionalEncoding pos_encoder;
+    torch::nn::TransformerEncoderLayer encoder_layers;
+    torch::nn::TransformerEncoder transformer_encoder;
+    torch::nn::Embedding encoder;
+    int64_t _ninp;
+    torch::nn::Linear decoder;
+
+ private:
+    torch::Tensor _generate_square_subsequent_mask(int sz);
+    void init_weights();
+};
+
+TORCH_MODULE(Transformer);
diff --git a/tutorials/popular/transformers/src/main.cpp b/tutorials/popular/transformers/src/main.cpp
new file mode 100644
index 0000000..571cb05
--- /dev/null
+++ b/tutorials/popular/transformers/src/main.cpp
@@ -0,0 +1,62 @@
+// Copyright 2020-present pytorch-cpp Authors
+// Original: https://pytorch.org/tutorials/beginner/blitz/neural_networks_tutorial.html
+#include <torch/torch.h>
+#include <iostream>
+#include <iomanip>
+#include "nnet.h"
+
+int main() {
+    std::cout << "Deep Learning with PyTorch: A 60 Minute Blitz\n\n";
+    std::cout << "Neural Networks\n\n";
+
+    std::cout << "Define the network\n\n";
+    Net net = Net();
+    net->to(torch::kCPU);
+    std::cout << net << "\n\n";
+
+    // The learnable parameters of a model are returned by net.parameters():
+    auto params = net->parameters();
+    std::cout << params.size() << '\n';
+    std::cout << params.at(0).sizes() << "\n\n";  // conv1's .weight
+
+    // Let’s try a random 32x32 input:
+    auto input = torch::randn({1, 1, 32, 32});
+    auto out = net->forward(input);
+    std::cout << out << "\n\n";
+
+    // Zero the gradient buffers of all parameters and backprops with random gradients:
+    net->zero_grad();
+    out.backward(torch::randn({1, 10}));
+
+    std::cout << "Loss Function\n\n";
+
+    auto output = net->forward(input);
+    auto target = torch::randn(10);  // a dummy target, for example
+    target = target.view({1, -1});  // make it the same shape as output
+    torch::nn::MSELoss criterion;
+    auto loss = criterion(output, target);
+    std::cout << loss << "\n\n";
+
+    // For illustration, let us follow a few steps backward:
+    std::cout << "loss.grad_fn:\n" << loss.grad_fn() << '\n';  // MSELoss
+
+    std::cout << "Backprop\n\n";
+
+    // Now we shall call loss.backward(), and have a look at conv1’s bias gradients before and after the backward:
+    net->zero_grad();  // zeroes the gradient buffers of all parameters
+    std::cout << "conv1.bias.grad before backward:\n" << net->conv1->bias.grad() << '\n';
+    loss.backward();
+    std::cout << "conv1.bias.grad after backward:\n" << net->conv1->bias.grad() << "\n\n";
+
+    std::cout << "Update the weights\n\n";
+
+    // create your optimizer
+    auto learning_rate = 0.01;
+    auto optimizer = torch::optim::SGD(net->parameters(), torch::optim::SGDOptions(learning_rate));
+    // in your training loop:
+    optimizer.zero_grad();   // zero the gradient buffers
+    output = net->forward(input);
+    loss = criterion(output, target);
+    loss.backward();
+    optimizer.step();    // Does the update
+}
diff --git a/tutorials/popular/transformers/src/positional_encoding.cpp b/tutorials/popular/transformers/src/positional_encoding.cpp
new file mode 100644
index 0000000..2032eee
--- /dev/null
+++ b/tutorials/popular/transformers/src/positional_encoding.cpp
@@ -0,0 +1,29 @@
+// Copyright 2020-present pytorch-cpp Authors
+#include "positional_encoding.h"
+#include <torch/torch.h>
+#include <cmath>
+
+using torch::indexing::Slice;
+using torch::indexing::None;
+
+PositionalEncodingImpl::PositionalEncodingImpl(int64_t d_model,
+                                               double dropout,
+                                               int64_t max_len)
+        : drpout(dropout) {
+    pe = torch::zeros({max_len, d_model});
+    auto position = torch::arange(0, max_len,
+        torch::TensorOptions(torch::kFloat32)).unsqueeze_(1);
+    auto div_term = torch::exp(torch::arange(0, d_model, 2,
+        torch::TensorOptions(torch::kFloat32)) * (-std::log(10000.0) / d_model));
+    pe.index({Slice(), Slice({0, None, 2})}) = torch::sin(position * div_term);
+    pe.index({Slice(), Slice({1, None, 2})}) = torch::cos(position * div_term);
+    pe = pe.unsqueeze_(0).transpose_(0, 1);
+
+    register_buffer("pe", pe);
+    register_module("drpout", drpout);
+}
+
+torch::Tensor PositionalEncodingImpl::forward(torch::Tensor x) {
+    auto out = x + pe.index({Slice(None, x.size(0)), Slice()});
+    return drpout->forward(out);
+}
diff --git a/tutorials/popular/transformers/src/transformer.cpp b/tutorials/popular/transformers/src/transformer.cpp
new file mode 100644
index 0000000..60a231c
--- /dev/null
+++ b/tutorials/popular/transformers/src/transformer.cpp
@@ -0,0 +1,56 @@
+// Copyright 2020-present pytorch-cpp Authors
+#include <torch/torch.h>
+#include "transformer.h"
+
+TransformerImpl::TransformerImpl(int64_t ntoken,
+                                 int64_t ninp,
+                                 int64_t nhead,
+                                 int64_t nhid,
+                                 int64_t nlayers,
+                                 double dropout)
+    :
+        pos_encoder(ninp, dropout),
+        encoder_layers(ninp, nhead, nhid, dropout),
+        transformer_encoder(encoder_layers, nlayers),
+        encoder(ntoken, ninp),
+        decoder(ninp, ntoken) {
+    _ninp = ninp;
+    // TODO(omkar, markus): init src_mask
+
+    init_weights();
+
+    register_module("pos_encoder", pos_encoder);
+    register_module("encoder_layers", encoder_layers);
+    register_module("transformer_encoder", transformer_encoder);
+    register_module("encoder", encoder);
+    register_module("decoder", decoder);
+}
+
+torch::Tensor TransformerImpl::_generate_square_subsequent_mask(int sz) {
+    auto mask = (torch::triu(torch::ones({sz, sz})) == 1).transpose_(0, 1);
+    // TODO(omkar, markus): convert to c++
+    // mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
+    return mask;
+}
+
+void TransformerImpl::init_weights() {
+    double initrange = 0.1;
+    // TODO(omkar, markus): convert to c++
+    // self.encoder.weight.data.uniform_(-initrange, initrange)
+    // self.decoder.bias.data.zero_()
+    // self.decoder.weight.data.uniform_(-initrange, initrange)
+}
+
+torch::Tensor TransformerImpl::forward(torch::Tensor src) {
+    // TODO(omkar, markus): convert to c++
+    // if self.src_mask is None or self.src_mask.size(0) != src.size(0):
+    //     device = src.device
+    //     mask = self._generate_square_subsequent_mask(src.size(0)).to(device)
+    //     self.src_mask = mask
+
+    auto out = encoder->forward(src) * sqrt(ninp);
+    out = pos_encoder->forward(out);
+    out = transformer_encoder->forward(out, src_mask);
+    out = decoder->forward(out);
+    return out;
+}
