From dc538733da8d49e7240d00fb05517053076fe261 Mon Sep 17 00:00:00 2001
From: 3gg <3gg@shellblade.net>
Date: Sat, 16 Dec 2023 11:06:03 -0800
Subject: Define vector outer product (nnMatrixMulOuter), which removes the
need to transpose layer inputs during training.
---
src/lib/include/neuralnet/matrix.h | 13 ++++++++++---
src/lib/src/matrix.c | 26 ++++++++++++++++++++++++++
src/lib/src/train.c | 28 +++-------------------------
3 files changed, 39 insertions(+), 28 deletions(-)
(limited to 'src')
diff --git a/src/lib/include/neuralnet/matrix.h b/src/lib/include/neuralnet/matrix.h
index f80b985..4cb0d25 100644
--- a/src/lib/include/neuralnet/matrix.h
+++ b/src/lib/include/neuralnet/matrix.h
@@ -56,13 +56,20 @@ void nnMatrixInitConstant(nnMatrix*, R value);
/// Multiply two matrices.
void nnMatrixMul(const nnMatrix* left, const nnMatrix* right, nnMatrix* out);
-/// Multiply two matrices, row variant.
+/// Multiply two matrices, row-by-row variant.
///
-/// This function multiples two matrices row-by-row instead of row-by-column.
-/// nnMatrixMul(A, B, O) == nnMatrixMulRows(A, B^T, O).
+/// This function multiples two matrices row-by-row instead of row-by-column,
+/// which is equivalent to regular multiplication after transposing the right
+/// hand matrix.
+///
+/// nnMatrixMul(A, B, O) == nnMatrixMulRows(A, B^T, O).
void nnMatrixMulRows(
const nnMatrix* left, const nnMatrix* right, nnMatrix* out);
+/// Compute the outer product of two vectors.
+void nnMatrixMulOuter(
+ const nnMatrix* left, const nnMatrix* right, nnMatrix* out);
+
/// Matrix multiply-add.
///
/// out = left + (right * scale)
diff --git a/src/lib/src/matrix.c b/src/lib/src/matrix.c
index d5c3fcc..29511eb 100644
--- a/src/lib/src/matrix.c
+++ b/src/lib/src/matrix.c
@@ -189,6 +189,32 @@ void nnMatrixMulRows(
}
}
+void nnMatrixMulOuter(
+ const nnMatrix* left, const nnMatrix* right, nnMatrix* out) {
+ assert(left != 0);
+ assert(right != 0);
+ assert(out != 0);
+ assert(out != left);
+ assert(out != right);
+ assert((left->rows == 1) || (left->cols == 1)); // Vector.
+ assert((right->rows == 1) || (right->cols == 1)); // Vector.
+ const int N = left->rows * left->cols;
+ const int M = right->rows * right->cols;
+ assert((out->rows == N) && (out->cols == M));
+
+ const R* left_value = left->values;
+ R* out_value = out->values;
+
+ for (int i = 0; i < N; ++i) {
+ const R* right_value = right->values;
+
+ for (int j = 0; j < M; ++j) {
+ *out_value++ = *left_value * *right_value++;
+ }
+ left_value++;
+ }
+}
+
void nnMatrixMulAdd(
const nnMatrix* left, const nnMatrix* right, R scale, nnMatrix* out) {
assert(left);
diff --git a/src/lib/src/train.c b/src/lib/src/train.c
index ccff553..fe9f598 100644
--- a/src/lib/src/train.c
+++ b/src/lib/src/train.c
@@ -153,14 +153,9 @@ void nnTrain(
nnGradientElements* gradient_elems =
calloc(net->num_layers, sizeof(nnGradientElements));
- // Allocate the output transpose vectors for weight delta calculation.
- // This is one column vector per layer.
- nnMatrix* outputs_T = calloc(net->num_layers, sizeof(nnMatrix));
-
assert(errors != 0);
assert(weight_deltas != 0);
assert(gradient_elems);
- assert(outputs_T);
for (int l = 0; l < net->num_layers; ++l) {
const int layer_input_size = nnLayerInputSize(net, l);
@@ -169,7 +164,6 @@ void nnTrain(
errors[l] = nnMatrixMake(1, layer_output_size);
weight_deltas[l] = nnMatrixMake(layer_input_size, layer_output_size);
- outputs_T[l] = nnMatrixMake(layer_output_size, 1);
// Allocate the gradient elements and vectors for weight delta calculation.
nnGradientElements* elems = &gradient_elems[l];
@@ -199,9 +193,6 @@ void nnTrain(
// the outputs.
const nnMatrix* const training_outputs = query->network_outputs;
- // A vector to store the training input transposed.
- nnMatrix training_inputs_T = nnMatrixMake(inputs->cols, 1);
-
// If debug mode is requested, we will show progress every Nth iteration.
const int progress_frame =
(params->max_iterations < PROGRESS_THRESHOLD)
@@ -223,10 +214,6 @@ void nnTrain(
const nnMatrix training_targets =
nnMatrixBorrowRows((nnMatrix*)targets, sample, 1);
- // Will need the input transposed for backpropagation.
- // Assuming one training input per iteration for now.
- nnMatrixTranspose(&training_inputs, &training_inputs_T);
-
// Forward pass.
nnQuery(net, query, &training_inputs);
@@ -240,14 +227,11 @@ void nnTrain(
nnMatrixSub(
training_outputs, &training_targets, &errors[net->num_layers - 1]);
- // Update outputs_T, which we need during weight updates.
- for (int l = 0; l < net->num_layers; ++l) {
- nnMatrixTranspose(&query->layer_outputs[l], &outputs_T[l]);
- }
-
// Update weights and biases for each internal layer, back-propagating
// errors along the way.
for (int l = net->num_layers - 1; l >= 0; --l) {
+ const nnMatrix* layer_input =
+ (l == 0) ? &training_inputs : &query->layer_outputs[l - 1];
const nnMatrix* layer_output = &query->layer_outputs[l];
nnGradientElements* elems = &gradient_elems[l];
nnMatrix* gradient = &elems->gradient;
@@ -310,10 +294,7 @@ void nnTrain(
nnMatrix* layer_biases = &linear->biases;
// Outer product to compute the weight deltas.
- // This layer's input is the previous layer's output.
- const nnMatrix* input_T =
- (l == 0) ? &training_inputs_T : &outputs_T[l - 1];
- nnMatrixMul(input_T, gradient, &weight_deltas[l]);
+ nnMatrixMulOuter(layer_input, gradient, &weight_deltas[l]);
// Update weights.
nnMatrixScale(&weight_deltas[l], params->learning_rate);
@@ -360,7 +341,6 @@ void nnTrain(
// Clean up.
for (int l = 0; l < net->num_layers; ++l) {
nnMatrixDel(&errors[l]);
- nnMatrixDel(&outputs_T[l]);
nnMatrixDel(&weight_deltas[l]);
nnGradientElements* elems = &gradient_elems[l];
@@ -378,9 +358,7 @@ void nnTrain(
break;
}
}
- nnMatrixDel(&training_inputs_T);
free(errors);
- free(outputs_T);
free(weight_deltas);
free(gradient_elems);
}
--
cgit v1.2.3