nn-cpp/layers.cpp at main · at-src/nn-cpp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
#include <vector>
#include <memory>
#include "activations.cpp"
#include "utils.cpp"

class Layer {
public:
    virtual ~Layer() = default;
    virtual std::vector<double> forward(const std::vector<double>& input) = 0;
    virtual std::vector<double> backward(const std::vector<double>& error, double lr) = 0;
};

class DenseLayer : public Layer {
private:
    std::vector<std::vector<double>> weights;
    std::vector<double> bias;
    std::vector<double> last_input;

public:
    DenseLayer(int in_size, int out_size) {
        weights = init_weights(out_size, in_size);
        bias = init_bias(out_size);
    }

    std::vector<double> forward(const std::vector<double>& input) override {
        last_input = input;
        std::vector<double> output;
        output.reserve(weights.size());

        for (const auto& w_row : weights) {
            double sum = dot_product(w_row, input);
            output.push_back(sum);
        }

        output = vector_add(output, bias);
        return output;
    }

    std::vector<double> backward(const std::vector<double>& error, double lr) override {
        std::vector<double> input_grad(last_input.size(), 0.0);
        auto weights_t = matrix_transpose(weights);

        for (size_t i = 0; i < weights_t.size(); i++) {
            input_grad[i] = dot_product(weights_t[i], error);
        }

        for (size_t i = 0; i < weights.size(); i++) {
            for (size_t j = 0; j < weights[i].size(); j++) {
                weights[i][j] -= lr * error[i] * last_input[j];
            }
            bias[i] -= lr * error[i];
        }

        return input_grad;
    }
};

class ActivationLayer : public Layer {
protected:
    std::vector<double> last_input;

public:
    ActivationLayer() = default;
    virtual ~ActivationLayer() = default;
};

class SigmoidLayer : public ActivationLayer {
public:
    std::vector<double> forward(const std::vector<double>& input) override {
        last_input = input;
        return apply_sigmoid(input);
    }

    std::vector<double> backward(const std::vector<double>& error, double lr) override {
        (void)lr;
        auto deriv = apply_sigmoid_derivative(last_input);
        return vector_multiply(error, deriv);
    }
};

class ReLULayer : public ActivationLayer {
public:
    std::vector<double> forward(const std::vector<double>& input) override {
        last_input = input;
        return apply_relu(input);
    }

    std::vector<double> backward(const std::vector<double>& error, double lr) override {
        (void)lr;
        auto deriv = apply_relu_derivative(last_input);
        return vector_multiply(error, deriv);
    }
};

class TanhLayer : public ActivationLayer {
public:
    std::vector<double> forward(const std::vector<double>& input) override {
        last_input = input;
        return apply_tanh(input);
    }

    std::vector<double> backward(const std::vector<double>& error, double lr) override {
        (void)lr;
        auto deriv = apply_tanh_derivative(last_input);
        return vector_multiply(error, deriv);
    }
};

class LeakyReLULayer : public ActivationLayer {
private:
    double alpha = 0.01;

public:
    LeakyReLULayer(double a = 0.01) : alpha(a) {}

    std::vector<double> forward(const std::vector<double>& input) override {
        last_input = input;
        return apply_leaky_relu(input, alpha);
    }

    std::vector<double> backward(const std::vector<double>& error, double lr) override {
        (void)lr;
        auto deriv = apply_leaky_relu_derivative(last_input, alpha);
        return vector_multiply(error, deriv);
    }
};

class LinearLayer : public ActivationLayer {
public:
    std::vector<double> forward(const std::vector<double>& input) override {
        last_input = input;
        return input;
    }

    std::vector<double> backward(const std::vector<double>& error, double lr) override {
        (void)lr;
        return error;
    }
};