From 4c8e5c893954e0db44d8cd4d8bfce9657f09e5be Mon Sep 17 00:00:00 2001 From: "J. Neugebauer" Date: Mon, 18 Jul 2022 11:06:08 +0200 Subject: [PATCH 1/6] USing Colt library as optional dependency --- build.gradle | 2 + .../java/schule/ngb/zm/ml/DoubleMatrix.java | 147 ++++++++++++++ src/main/java/schule/ngb/zm/ml/Matrix.java | 88 +++----- .../java/schule/ngb/zm/ml/MatrixFactory.java | 192 ++++++++++++++++++ .../java/schule/ngb/zm/ml/NeuralNetwork.java | 22 +- .../java/schule/ngb/zm/ml/NeuronLayer.java | 76 +++---- ...{MatrixTest.java => DoubleMatrixTest.java} | 19 +- .../schule/ngb/zm/ml/NeuralNetworkTest.java | 21 +- 8 files changed, 439 insertions(+), 128 deletions(-) create mode 100644 src/main/java/schule/ngb/zm/ml/DoubleMatrix.java create mode 100644 src/main/java/schule/ngb/zm/ml/MatrixFactory.java rename src/test/java/schule/ngb/zm/ml/{MatrixTest.java => DoubleMatrixTest.java} (69%) diff --git a/build.gradle b/build.gradle index 9a02a81..0fafb27 100644 --- a/build.gradle +++ b/build.gradle @@ -28,6 +28,8 @@ dependencies { runtimeOnly 'com.googlecode.soundlibs:tritonus-share:0.3.7.4' runtimeOnly 'com.googlecode.soundlibs:mp3spi:1.9.5.4' + api 'colt:colt:1.2.0' + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' } diff --git a/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java b/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java new file mode 100644 index 0000000..a4936d3 --- /dev/null +++ b/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java @@ -0,0 +1,147 @@ +package schule.ngb.zm.ml; + +import schule.ngb.zm.Constants; + +import java.util.Arrays; +import java.util.function.DoubleUnaryOperator; + +// TODO: Move Math into Matrix class +// TODO: Implement support for optional sci libs +public class DoubleMatrix implements Matrix { + + private int columns, rows; + + double[][] coefficients; + + public DoubleMatrix( int rows, int cols ) { + this.rows = rows; + this.columns = cols; + coefficients = new double[rows][cols]; + } + + public DoubleMatrix( double[][] coefficients ) { + this.rows = coefficients.length; + this.columns = coefficients[0].length; + this.coefficients = coefficients; + } + + public int columns() { + return columns; + } + + public int rows() { + return rows; + } + + public double[][] getCoefficients() { + return coefficients; + } + + public double get( int row, int col ) { + return coefficients[row][col]; + } + + public Matrix set( int row, int col, double value ) { + coefficients[row][col] = value; + return this; + } + + public Matrix initializeRandom() { + coefficients = MLMath.matrixApply(coefficients, (d) -> Constants.randomGaussian()); + return this; + } + + public Matrix initializeRandom( double lower, double upper ) { + coefficients = MLMath.matrixApply(coefficients, (d) -> ((upper-lower) * (Constants.randomGaussian()+1) * .5) + lower); + return this; + } + + public Matrix initializeOne() { + coefficients = MLMath.matrixApply(coefficients, (d) -> 1.0); + return this; + } + + public Matrix initializeZero() { + coefficients = MLMath.matrixApply(coefficients, (d) -> 0.0); + return this; + } + + @Override + public String toString() { + //return Arrays.deepToString(coefficients); + StringBuilder sb = new StringBuilder(); + sb.append('['); + sb.append('\n'); + for( int i = 0; i < coefficients.length; i++ ) { + sb.append('\t'); + sb.append(Arrays.toString(coefficients[i])); + sb.append('\n'); + } + sb.append(']'); + + return sb.toString(); + } + + @Override + public Matrix transpose() { + coefficients = MLMath.matrixTranspose(coefficients); + return this; + } + + @Override + public Matrix multiply( Matrix B ) { + coefficients = MLMath.matrixMultiply(coefficients, B.getCoefficients()); + return this; + } + + @Override + public Matrix multiplyAddBias( Matrix B, Matrix C ) { + double[] biases = Arrays.stream(C.getCoefficients()).mapToDouble((arr) -> arr[0]).toArray(); + coefficients = MLMath.biasAdd( + MLMath.matrixMultiply(coefficients, B.getCoefficients()), + biases + ); + return this; + } + + @Override + public Matrix multiplyLeft( Matrix B ) { + coefficients = MLMath.matrixMultiply(B.getCoefficients(), coefficients); + return this; + } + + @Override + public Matrix add( Matrix B ) { + coefficients = MLMath.matrixAdd(coefficients, B.getCoefficients()); + return this; + } + + @Override + public Matrix sub( Matrix B ) { + coefficients = MLMath.matrixSub(coefficients, B.getCoefficients()); + return this; + } + + @Override + public Matrix scale( double scalar ) { + return this; + } + + @Override + public Matrix scale( Matrix S ) { + coefficients = MLMath.matrixScale(coefficients, S.getCoefficients()); + return this; + } + + @Override + public Matrix apply( DoubleUnaryOperator op ) { + this.coefficients = MLMath.matrixApply(coefficients, op); + return this; + } + + @Override + public Matrix duplicate() { + return new DoubleMatrix(MLMath.copyMatrix(coefficients)); + } + +} diff --git a/src/main/java/schule/ngb/zm/ml/Matrix.java b/src/main/java/schule/ngb/zm/ml/Matrix.java index 734fe7b..f778d39 100644 --- a/src/main/java/schule/ngb/zm/ml/Matrix.java +++ b/src/main/java/schule/ngb/zm/ml/Matrix.java @@ -1,82 +1,48 @@ package schule.ngb.zm.ml; -import schule.ngb.zm.Constants; +import java.util.function.DoubleUnaryOperator; -import java.util.Arrays; +public interface Matrix { -// TODO: Move Math into Matrix class -// TODO: Implement support for optional sci libs -public class Matrix { + int columns(); - private int columns, rows; + int rows(); - double[][] coefficients; + double[][] getCoefficients(); - public Matrix( int rows, int cols ) { - this.rows = rows; - this.columns = cols; - coefficients = new double[rows][cols]; - } + double get( int row, int col ); - public Matrix( double[][] coefficients ) { - this.coefficients = coefficients; - this.rows = coefficients.length; - this.columns = coefficients[0].length; - } + Matrix set( int row, int col, double value ); - public int getColumns() { - return columns; - } + Matrix initializeRandom(); - public int getRows() { - return rows; - } + Matrix initializeRandom( double lower, double upper ); - public double[][] getCoefficients() { - return coefficients; - } + Matrix initializeOne(); - public double get( int row, int col ) { - return coefficients[row][col]; - } + Matrix initializeZero(); - public void initializeRandom() { - coefficients = MLMath.matrixApply(coefficients, (d) -> Constants.randomGaussian()); - } + Matrix transpose(); - public void initializeRandom( double lower, double upper ) { - coefficients = MLMath.matrixApply(coefficients, (d) -> ((upper-lower) * (Constants.randomGaussian()+1) * .5) + lower); - } + Matrix multiply( Matrix B ); - public void initializeIdentity() { - initializeZero(); - for( int i = 0; i < Math.min(rows, columns); i++ ) { - this.coefficients[i][i] = 1.0; - } - } + Matrix multiplyAddBias( Matrix B, Matrix C ); - public void initializeOne() { - coefficients = MLMath.matrixApply(coefficients, (d) -> 1.0); - } + Matrix multiplyLeft( Matrix B ); - public void initializeZero() { - coefficients = MLMath.matrixApply(coefficients, (d) -> 0.0); - } + Matrix add( Matrix B ); - @Override - public String toString() { - //return Arrays.deepToString(coefficients); - StringBuilder sb = new StringBuilder(); - sb.append('['); - sb.append('\n'); - for( int i = 0; i < coefficients.length; i++ ) { - sb.append('\t'); - sb.append(Arrays.toString(coefficients[i])); - sb.append('\n'); - } - sb.append(']'); + Matrix sub( Matrix B ); - return sb.toString(); - } + + Matrix scale( double scalar ); + + Matrix scale( Matrix S ); + + Matrix apply( DoubleUnaryOperator op ); + + Matrix duplicate(); + + String toString(); } diff --git a/src/main/java/schule/ngb/zm/ml/MatrixFactory.java b/src/main/java/schule/ngb/zm/ml/MatrixFactory.java new file mode 100644 index 0000000..040e6cf --- /dev/null +++ b/src/main/java/schule/ngb/zm/ml/MatrixFactory.java @@ -0,0 +1,192 @@ +package schule.ngb.zm.ml; + +import cern.colt.matrix.DoubleMatrix2D; +import cern.colt.matrix.impl.DenseDoubleMatrix2D; +import schule.ngb.zm.Constants; +import schule.ngb.zm.util.Log; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.function.DoubleUnaryOperator; + +public class MatrixFactory { + + public static void main( String[] args ) { + System.out.println( + MatrixFactory.create(new double[][]{ {1.0, 0.0}, {0.0, 1.0} }).toString() + ); + } + + public static final Matrix create( int rows, int cols ) { + try { + return getMatrixType().getDeclaredConstructor(int.class, int.class).newInstance(rows, cols); + } catch( Exception ex ) { + LOG.error(ex, "Could not initialize matrix implementation for class <%s>. Using internal implementation.", matrixType); + } + return new DoubleMatrix(rows, cols); + } + + public static final Matrix create( double[][] values ) { + try { + return getMatrixType().getDeclaredConstructor(double[][].class).newInstance((Object)values); + } catch( Exception ex ) { + LOG.error(ex, "Could not initialize matrix implementation for class <%s>. Using internal implementation.", matrixType); + } + return new DoubleMatrix(values); + } + + private static Class matrixType = null; + + private static final Class getMatrixType() { + if( matrixType == null ) { + try { + Class clazz = Class.forName("cern.colt.matrix.impl.DenseDoubleMatrix2D", false, MatrixFactory.class.getClassLoader()); + matrixType = ColtMatrix.class; + LOG.info("Colt library found. Using as matrix implementation."); + } catch( ClassNotFoundException e ) { + LOG.info("Colt library not found. Falling back on internal implementation."); + matrixType = DoubleMatrix.class; + } + } + return matrixType; + } + + private static final Log LOG = Log.getLogger(MatrixFactory.class); + + static class ColtMatrix implements Matrix { + + cern.colt.matrix.DoubleMatrix2D matrix; + + public ColtMatrix( double[][] doubles ) { + matrix = new cern.colt.matrix.impl.DenseDoubleMatrix2D(doubles); + } + + public ColtMatrix( int rows, int cols ) { + matrix = new cern.colt.matrix.impl.DenseDoubleMatrix2D(rows, cols); + } + + @Override + public int columns() { + return matrix.columns(); + } + + @Override + public int rows() { + return matrix.rows(); + } + + @Override + public double get( int row, int col ) { + return matrix.get(row, col); + } + + @Override + public Matrix set( int row, int col, double value ) { + matrix.set(row, col, value); + return this; + } + + @Override + public double[][] getCoefficients() { + return this.matrix.toArray(); + } + + @Override + public Matrix initializeRandom() { + matrix.assign((d) -> Constants.randomGaussian()); + return this; + } + + @Override + public Matrix initializeRandom( double lower, double upper ) { + matrix.assign((d) -> ((upper-lower) * (Constants.randomGaussian()+1) * .5) + lower); + return this; + } + + @Override + public Matrix initializeOne() { + this.matrix.assign(1.0); + return this; + } + + @Override + public Matrix initializeZero() { + this.matrix.assign(0.0); + return this; + } + + @Override + public Matrix apply( DoubleUnaryOperator op ) { + this.matrix.assign((d) -> op.applyAsDouble(d)); + return this; + } + + @Override + public Matrix transpose() { + this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.transpose(this.matrix); + return this; + } + + @Override + public Matrix multiply( Matrix B ) { + ColtMatrix CB = (ColtMatrix)B; + this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(matrix, CB.matrix); + return this; + } + + @Override + public Matrix multiplyLeft( Matrix B ) { + ColtMatrix CB = (ColtMatrix)B; + this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(CB.matrix, matrix); + return this; + } + + @Override + public Matrix multiplyAddBias( Matrix B, Matrix C ) { + ColtMatrix CB = (ColtMatrix)B; + this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(matrix, CB.matrix); + // TODO: add bias + return this; + } + + @Override + public Matrix add( Matrix B ) { + ColtMatrix CB = (ColtMatrix)B; + matrix.assign(CB.matrix, (d1,d2) -> d1+d2); + return this; + } + + @Override + public Matrix sub( Matrix B ) { + ColtMatrix CB = (ColtMatrix)B; + matrix.assign(CB.matrix, (d1,d2) -> d1-d2); + return this; + } + + @Override + public Matrix scale( double scalar ) { + this.matrix.assign((d) -> d*scalar); + return this; + } + + @Override + public Matrix scale( Matrix S ) { + this.matrix.forEachNonZero((r, c, d) -> d * S.get(r, c)); + return this; + } + + @Override + public Matrix duplicate() { + ColtMatrix newMatrix = new ColtMatrix(matrix.rows(), matrix.columns()); + newMatrix.matrix.assign(this.matrix); + return newMatrix; + } + + @Override + public String toString() { + return matrix.toString(); + } + + } + +} diff --git a/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java b/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java index b6464e5..c559463 100644 --- a/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java +++ b/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java @@ -23,13 +23,13 @@ public class NeuralNetwork { for( int i = 0; i < layer.getInputCount(); i++ ) { for( int j = 0; j < layer.getNeuronCount(); j++ ) { - out.print(layer.weights.coefficients[i][j]); + //out.print(layer.weights.coefficients[i][j]); out.print(' '); } out.println(); } for( int j = 0; j < layer.getNeuronCount(); j++ ) { - out.print(layer.biases[j]); + //out.print(layer.biases[j]); out.print(' '); } out.println(); @@ -56,13 +56,13 @@ public class NeuralNetwork { for( int i = 0; i < inputs; i++ ) { split = in.readLine().split(" "); for( int j = 0; j < neurons; j++ ) { - layer.weights.coefficients[i][j] = Double.parseDouble(split[j]); + //layer.weights.coefficients[i][j] = Double.parseDouble(split[j]); } } // Load Biases split = in.readLine().split(" "); for( int j = 0; j < neurons; j++ ) { - layer.biases[j] = Double.parseDouble(split[j]); + //layer.biases[j] = Double.parseDouble(split[j]); } layers.add(layer); @@ -107,7 +107,7 @@ public class NeuralNetwork { private NeuronLayer[] layers; - private double[][] output; + private Matrix output; private double learningRate = 0.1; @@ -162,17 +162,25 @@ public class NeuralNetwork { this.learningRate = pLearningRate; } - public double[][] getOutput() { + public Matrix getOutput() { return output; } - public double[][] predict( double[][] inputs ) { + public Matrix predict( double[][] inputs ) { //this.output = layers[1].apply(layers[0].apply(inputs)); + return predict(MatrixFactory.create(inputs)); + } + + public Matrix predict( Matrix inputs ) { this.output = layers[0].apply(inputs); return this.output; } public void learn( double[][] expected ) { + learn(MatrixFactory.create(expected)); + } + + public void learn( Matrix expected ) { layers[layers.length - 1].backprop(expected, learningRate); } diff --git a/src/main/java/schule/ngb/zm/ml/NeuronLayer.java b/src/main/java/schule/ngb/zm/ml/NeuronLayer.java index 9767fad..037d777 100644 --- a/src/main/java/schule/ngb/zm/ml/NeuronLayer.java +++ b/src/main/java/schule/ngb/zm/ml/NeuronLayer.java @@ -4,9 +4,9 @@ import java.util.Arrays; import java.util.function.DoubleUnaryOperator; import java.util.function.Function; -public class NeuronLayer implements Function { +public class NeuronLayer implements Function { - public static NeuronLayer fromArray( double[][] weights ) { + /*public static NeuronLayer fromArray( double[][] weights ) { NeuronLayer layer = new NeuronLayer(weights[0].length, weights.length); for( int i = 0; i < weights[0].length; i++ ) { for( int j = 0; j < weights.length; j++ ) { @@ -27,24 +27,26 @@ public class NeuronLayer implements Function { layer.biases[j] = biases[j]; } return layer; - } - + }*/ + Matrix weights; - double[] biases; + Matrix biases; NeuronLayer previous, next; DoubleUnaryOperator activationFunction, activationFunctionDerivative; - double[][] lastOutput, lastInput; + Matrix lastOutput, lastInput; public NeuronLayer( int neurons, int inputs ) { - weights = new Matrix(inputs, neurons); - weights.initializeRandom(-1, 1); + weights = MatrixFactory + .create(inputs, neurons) + .initializeRandom(-1, 1); - biases = new double[neurons]; - Arrays.fill(biases, 0.0); // TODO: Random? + biases = MatrixFactory + .create(neurons, 1) + .initializeZero(); activationFunction = MLMath::sigmoid; activationFunctionDerivative = MLMath::sigmoidDerivative; @@ -89,41 +91,42 @@ public class NeuronLayer implements Function { return weights; } + public Matrix getBiases() { + return biases; + } + public int getNeuronCount() { - return weights.coefficients[0].length; + return weights.columns(); } public int getInputCount() { - return weights.coefficients.length; + return weights.rows(); } - public double[][] getLastOutput() { + public Matrix getLastOutput() { return lastOutput; } - public void setWeights( double[][] newWeights ) { - weights.coefficients = MLMath.copyMatrix(newWeights); + public void setWeights( Matrix newWeights ) { + weights = newWeights.duplicate(); } - public void adjustWeights( double[][] adjustment ) { - weights.coefficients = MLMath.matrixAdd(weights.coefficients, adjustment); + public void adjustWeights( Matrix adjustment ) { + weights.add(adjustment); } @Override public String toString() { - return weights.toString() + "\n" + Arrays.toString(biases); + return weights.toString() + "\n" + biases.toString(); } @Override - public double[][] apply( double[][] inputs ) { + public Matrix apply( Matrix inputs ) { lastInput = inputs; - lastOutput = MLMath.matrixApply( - MLMath.biasAdd( - MLMath.matrixMultiply(inputs, weights.coefficients), - biases - ), - activationFunction - ); + lastOutput = inputs + .multiplyAddBias(weights, biases) + .apply(activationFunction); + if( next != null ) { return next.apply(lastOutput); } else { @@ -132,35 +135,34 @@ public class NeuronLayer implements Function { } @Override - public Function compose( Function before ) { + public Function compose( Function before ) { return ( in ) -> apply(before.apply(in)); } @Override - public Function andThen( Function after ) { + public Function andThen( Function after ) { return ( in ) -> after.apply(apply(in)); } - public void backprop( double[][] expected, double learningRate ) { - double[][] error, delta, adjustment; + public void backprop( Matrix expected, double learningRate ) { + Matrix error, delta, adjustment; if( next == null ) { - error = MLMath.matrixSub(expected, this.lastOutput); + error = expected.duplicate().sub(lastOutput); } else { - error = MLMath.matrixMultiply(expected, MLMath.matrixTranspose(next.weights.coefficients)); + error = expected.duplicate().multiply(next.weights.transpose()); } - delta = MLMath.matrixScale(error, MLMath.matrixApply(this.lastOutput, this.activationFunctionDerivative)); + error.scale(lastOutput.duplicate().apply(this.activationFunctionDerivative)); // Hier schon leraningRate anwenden? // See https://towardsdatascience.com/understanding-and-implementing-neural-networks-in-java-from-scratch-61421bb6352c //delta = MLMath.matrixApply(delta, ( x ) -> learningRate * x); if( previous != null ) { - previous.backprop(delta, learningRate); + previous.backprop(error, learningRate); } - biases = MLMath.biasAdjust(biases, MLMath.matrixApply(delta, ( x ) -> learningRate * x)); + // biases = MLMath.biasAdjust(biases, MLMath.matrixApply(delta, ( x ) -> learningRate * x)); - adjustment = MLMath.matrixMultiply(MLMath.matrixTranspose(lastInput), delta); - adjustment = MLMath.matrixApply(adjustment, ( x ) -> learningRate * x); + adjustment = lastInput.duplicate().transpose().multiply(error).apply((d) -> learningRate*d); this.adjustWeights(adjustment); } diff --git a/src/test/java/schule/ngb/zm/ml/MatrixTest.java b/src/test/java/schule/ngb/zm/ml/DoubleMatrixTest.java similarity index 69% rename from src/test/java/schule/ngb/zm/ml/MatrixTest.java rename to src/test/java/schule/ngb/zm/ml/DoubleMatrixTest.java index 6277e62..1ff70a8 100644 --- a/src/test/java/schule/ngb/zm/ml/MatrixTest.java +++ b/src/test/java/schule/ngb/zm/ml/DoubleMatrixTest.java @@ -6,22 +6,11 @@ import java.util.Arrays; import static org.junit.jupiter.api.Assertions.*; -class MatrixTest { - - @Test - void initializeIdentity() { - Matrix m = new Matrix(4, 4); - m.initializeIdentity(); - - assertArrayEquals(new double[]{1.0, 0.0, 0.0, 0.0}, m.coefficients[0]); - assertArrayEquals(new double[]{0.0, 1.0, 0.0, 0.0}, m.coefficients[1]); - assertArrayEquals(new double[]{0.0, 0.0, 1.0, 0.0}, m.coefficients[2]); - assertArrayEquals(new double[]{0.0, 0.0, 0.0, 1.0}, m.coefficients[3]); - } +class DoubleMatrixTest { @Test void initializeOne() { - Matrix m = new Matrix(4, 4); + DoubleMatrix m = new DoubleMatrix(4, 4); m.initializeOne(); double[] ones = new double[]{1.0, 1.0, 1.0, 1.0}; @@ -33,7 +22,7 @@ class MatrixTest { @Test void initializeZero() { - Matrix m = new Matrix(4, 4); + DoubleMatrix m = new DoubleMatrix(4, 4); m.initializeZero(); double[] zeros = new double[]{0.0, 0.0, 0.0, 0.0}; @@ -45,7 +34,7 @@ class MatrixTest { @Test void initializeRandom() { - Matrix m = new Matrix(4, 4); + DoubleMatrix m = new DoubleMatrix(4, 4); m.initializeRandom(-1, 1); assertTrue(Arrays.stream(m.coefficients[0]).allMatch((d) -> -1.0 <= d && d < 1.0)); diff --git a/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java b/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java index f81fe4d..af85c88 100644 --- a/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java +++ b/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java @@ -3,6 +3,7 @@ package schule.ngb.zm.ml; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import schule.ngb.zm.util.Log; +import schule.ngb.zm.util.Timer; import java.io.File; import java.util.ArrayList; @@ -18,7 +19,7 @@ class NeuralNetworkTest { Log.enableGlobalDebugging(); } - @Test + /*@Test void readWrite() { // XOR Dataset NeuralNetwork net = new NeuralNetwork(2, 4, 1); @@ -53,7 +54,7 @@ class NeuralNetworkTest { } assertArrayEquals(net.predict(inputs), net2.predict(inputs)); - } + }*/ @Test void learnXor() { @@ -78,14 +79,14 @@ class NeuralNetworkTest { } // calculate predictions - double[][] predictions = net.predict(inputs); + Matrix predictions = net.predict(inputs); for( int i = 0; i < 4; i++ ) { - int parsed_pred = predictions[i][0] < 0.5 ? 0 : 1; + int parsed_pred = predictions.get(i, 0) < 0.5 ? 0 : 1; System.out.printf( "{%.0f, %.0f} = %.4f (%d) -> %s\n", inputs[i][0], inputs[i][1], - predictions[i][0], + predictions.get(i, 0), parsed_pred, parsed_pred == outputs[i][0] ? "correct" : "miss" ); @@ -112,9 +113,13 @@ class NeuralNetworkTest { outputs[i][0] = trainingData.get(i).result; } + Timer timer = new Timer(); + System.out.println("Training the neural net to learn "+OPERATION+"..."); + timer.start(); net.train(inputs, outputs, TRAINING_CYCLES); - System.out.println(" finished training"); + timer.stop(); + System.out.println(" finished training (" + timer.getMillis() + "ms)"); for( int i = 1; i <= net.getLayerCount(); i++ ) { System.out.println("Layer " +i + " weights"); @@ -136,9 +141,9 @@ class NeuralNetworkTest { System.out.printf( "Prediction on data (%.2f, %.2f) was %.4f, expected %.2f (of by %.4f)\n", data.a, data.b, - net.getOutput()[0][0], + net.getOutput().get(0, 0), data.result, - net.getOutput()[0][0] - data.result + net.getOutput().get(0, 0) - data.result ); } From b79f26f51ecd295a92178c24134c6862cd57eb38 Mon Sep 17 00:00:00 2001 From: "J. Neugebauer" Date: Tue, 19 Jul 2022 09:14:00 +0200 Subject: [PATCH 2/6] Matric interface umbenannt --- build.gradle | 2 +- .../java/schule/ngb/zm/ml/DoubleMatrix.java | 48 ++++++++--------- src/main/java/schule/ngb/zm/ml/MLMatrix.java | 52 +++++++++++++++++++ src/main/java/schule/ngb/zm/ml/Matrix.java | 48 ----------------- .../java/schule/ngb/zm/ml/MatrixFactory.java | 44 ++++++++-------- .../java/schule/ngb/zm/ml/NeuralNetwork.java | 10 ++-- .../java/schule/ngb/zm/ml/NeuronLayer.java | 29 +++++------ .../schule/ngb/zm/ml/NeuralNetworkTest.java | 5 +- 8 files changed, 118 insertions(+), 120 deletions(-) create mode 100644 src/main/java/schule/ngb/zm/ml/MLMatrix.java delete mode 100644 src/main/java/schule/ngb/zm/ml/Matrix.java diff --git a/build.gradle b/build.gradle index 0fafb27..465207f 100644 --- a/build.gradle +++ b/build.gradle @@ -28,7 +28,7 @@ dependencies { runtimeOnly 'com.googlecode.soundlibs:tritonus-share:0.3.7.4' runtimeOnly 'com.googlecode.soundlibs:mp3spi:1.9.5.4' - api 'colt:colt:1.2.0' + compileOnlyApi 'colt:colt:1.2.0' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' diff --git a/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java b/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java index a4936d3..04391f4 100644 --- a/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java +++ b/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java @@ -7,7 +7,7 @@ import java.util.function.DoubleUnaryOperator; // TODO: Move Math into Matrix class // TODO: Implement support for optional sci libs -public class DoubleMatrix implements Matrix { +public class DoubleMatrix implements MLMatrix { private int columns, rows; @@ -33,7 +33,7 @@ public class DoubleMatrix implements Matrix { return rows; } - public double[][] getCoefficients() { + public double[][] coefficients() { return coefficients; } @@ -41,27 +41,27 @@ public class DoubleMatrix implements Matrix { return coefficients[row][col]; } - public Matrix set( int row, int col, double value ) { + public MLMatrix set( int row, int col, double value ) { coefficients[row][col] = value; return this; } - public Matrix initializeRandom() { + public MLMatrix initializeRandom() { coefficients = MLMath.matrixApply(coefficients, (d) -> Constants.randomGaussian()); return this; } - public Matrix initializeRandom( double lower, double upper ) { + public MLMatrix initializeRandom( double lower, double upper ) { coefficients = MLMath.matrixApply(coefficients, (d) -> ((upper-lower) * (Constants.randomGaussian()+1) * .5) + lower); return this; } - public Matrix initializeOne() { + public MLMatrix initializeOne() { coefficients = MLMath.matrixApply(coefficients, (d) -> 1.0); return this; } - public Matrix initializeZero() { + public MLMatrix initializeZero() { coefficients = MLMath.matrixApply(coefficients, (d) -> 0.0); return this; } @@ -83,64 +83,64 @@ public class DoubleMatrix implements Matrix { } @Override - public Matrix transpose() { + public MLMatrix transpose() { coefficients = MLMath.matrixTranspose(coefficients); return this; } @Override - public Matrix multiply( Matrix B ) { - coefficients = MLMath.matrixMultiply(coefficients, B.getCoefficients()); + public MLMatrix multiply( MLMatrix B ) { + coefficients = MLMath.matrixMultiply(coefficients, B.coefficients()); return this; } @Override - public Matrix multiplyAddBias( Matrix B, Matrix C ) { - double[] biases = Arrays.stream(C.getCoefficients()).mapToDouble((arr) -> arr[0]).toArray(); + public MLMatrix multiplyAddBias( MLMatrix B, MLMatrix C ) { + double[] biases = Arrays.stream(C.coefficients()).mapToDouble(( arr) -> arr[0]).toArray(); coefficients = MLMath.biasAdd( - MLMath.matrixMultiply(coefficients, B.getCoefficients()), + MLMath.matrixMultiply(coefficients, B.coefficients()), biases ); return this; } @Override - public Matrix multiplyLeft( Matrix B ) { - coefficients = MLMath.matrixMultiply(B.getCoefficients(), coefficients); + public MLMatrix multiplyLeft( MLMatrix B ) { + coefficients = MLMath.matrixMultiply(B.coefficients(), coefficients); return this; } @Override - public Matrix add( Matrix B ) { - coefficients = MLMath.matrixAdd(coefficients, B.getCoefficients()); + public MLMatrix add( MLMatrix B ) { + coefficients = MLMath.matrixAdd(coefficients, B.coefficients()); return this; } @Override - public Matrix sub( Matrix B ) { - coefficients = MLMath.matrixSub(coefficients, B.getCoefficients()); + public MLMatrix sub( MLMatrix B ) { + coefficients = MLMath.matrixSub(coefficients, B.coefficients()); return this; } @Override - public Matrix scale( double scalar ) { + public MLMatrix scale( double scalar ) { return this; } @Override - public Matrix scale( Matrix S ) { - coefficients = MLMath.matrixScale(coefficients, S.getCoefficients()); + public MLMatrix scale( MLMatrix S ) { + coefficients = MLMath.matrixScale(coefficients, S.coefficients()); return this; } @Override - public Matrix apply( DoubleUnaryOperator op ) { + public MLMatrix apply( DoubleUnaryOperator op ) { this.coefficients = MLMath.matrixApply(coefficients, op); return this; } @Override - public Matrix duplicate() { + public MLMatrix duplicate() { return new DoubleMatrix(MLMath.copyMatrix(coefficients)); } diff --git a/src/main/java/schule/ngb/zm/ml/MLMatrix.java b/src/main/java/schule/ngb/zm/ml/MLMatrix.java new file mode 100644 index 0000000..0c2af15 --- /dev/null +++ b/src/main/java/schule/ngb/zm/ml/MLMatrix.java @@ -0,0 +1,52 @@ +package schule.ngb.zm.ml; + +import java.util.function.DoubleUnaryOperator; + +public interface MLMatrix { + + int columns(); + + int rows(); + + double[][] coefficients(); + + double get( int row, int col ); + + MLMatrix set( int row, int col, double value ); + + MLMatrix initializeRandom(); + + MLMatrix initializeRandom( double lower, double upper ); + + MLMatrix initializeOne(); + + MLMatrix initializeZero(); + + /** + * Erzeugt die transponierte Matrix zu dieser. + * @return + */ + MLMatrix transpose(); + + MLMatrix multiply( MLMatrix B ); + + MLMatrix multiplyAddBias( MLMatrix B, MLMatrix C ); + + MLMatrix multiplyLeft( MLMatrix B ); + + MLMatrix add( MLMatrix B ); + + MLMatrix sub( MLMatrix B ); + + + MLMatrix scale( double scalar ); + + MLMatrix scale( MLMatrix S ); + + MLMatrix apply( DoubleUnaryOperator op ); + + MLMatrix duplicate(); + + String toString(); + +} diff --git a/src/main/java/schule/ngb/zm/ml/Matrix.java b/src/main/java/schule/ngb/zm/ml/Matrix.java deleted file mode 100644 index f778d39..0000000 --- a/src/main/java/schule/ngb/zm/ml/Matrix.java +++ /dev/null @@ -1,48 +0,0 @@ -package schule.ngb.zm.ml; - -import java.util.function.DoubleUnaryOperator; - -public interface Matrix { - - int columns(); - - int rows(); - - double[][] getCoefficients(); - - double get( int row, int col ); - - Matrix set( int row, int col, double value ); - - Matrix initializeRandom(); - - Matrix initializeRandom( double lower, double upper ); - - Matrix initializeOne(); - - Matrix initializeZero(); - - Matrix transpose(); - - Matrix multiply( Matrix B ); - - Matrix multiplyAddBias( Matrix B, Matrix C ); - - Matrix multiplyLeft( Matrix B ); - - Matrix add( Matrix B ); - - Matrix sub( Matrix B ); - - - Matrix scale( double scalar ); - - Matrix scale( Matrix S ); - - Matrix apply( DoubleUnaryOperator op ); - - Matrix duplicate(); - - String toString(); - -} diff --git a/src/main/java/schule/ngb/zm/ml/MatrixFactory.java b/src/main/java/schule/ngb/zm/ml/MatrixFactory.java index 040e6cf..b0a8901 100644 --- a/src/main/java/schule/ngb/zm/ml/MatrixFactory.java +++ b/src/main/java/schule/ngb/zm/ml/MatrixFactory.java @@ -5,8 +5,6 @@ import cern.colt.matrix.impl.DenseDoubleMatrix2D; import schule.ngb.zm.Constants; import schule.ngb.zm.util.Log; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.util.function.DoubleUnaryOperator; public class MatrixFactory { @@ -17,7 +15,7 @@ public class MatrixFactory { ); } - public static final Matrix create( int rows, int cols ) { + public static final MLMatrix create( int rows, int cols ) { try { return getMatrixType().getDeclaredConstructor(int.class, int.class).newInstance(rows, cols); } catch( Exception ex ) { @@ -26,7 +24,7 @@ public class MatrixFactory { return new DoubleMatrix(rows, cols); } - public static final Matrix create( double[][] values ) { + public static final MLMatrix create( double[][] values ) { try { return getMatrixType().getDeclaredConstructor(double[][].class).newInstance((Object)values); } catch( Exception ex ) { @@ -35,9 +33,9 @@ public class MatrixFactory { return new DoubleMatrix(values); } - private static Class matrixType = null; + private static Class matrixType = null; - private static final Class getMatrixType() { + private static final Class getMatrixType() { if( matrixType == null ) { try { Class clazz = Class.forName("cern.colt.matrix.impl.DenseDoubleMatrix2D", false, MatrixFactory.class.getClassLoader()); @@ -53,7 +51,7 @@ public class MatrixFactory { private static final Log LOG = Log.getLogger(MatrixFactory.class); - static class ColtMatrix implements Matrix { + static class ColtMatrix implements MLMatrix { cern.colt.matrix.DoubleMatrix2D matrix; @@ -81,68 +79,68 @@ public class MatrixFactory { } @Override - public Matrix set( int row, int col, double value ) { + public MLMatrix set( int row, int col, double value ) { matrix.set(row, col, value); return this; } @Override - public double[][] getCoefficients() { + public double[][] coefficients() { return this.matrix.toArray(); } @Override - public Matrix initializeRandom() { + public MLMatrix initializeRandom() { matrix.assign((d) -> Constants.randomGaussian()); return this; } @Override - public Matrix initializeRandom( double lower, double upper ) { + public MLMatrix initializeRandom( double lower, double upper ) { matrix.assign((d) -> ((upper-lower) * (Constants.randomGaussian()+1) * .5) + lower); return this; } @Override - public Matrix initializeOne() { + public MLMatrix initializeOne() { this.matrix.assign(1.0); return this; } @Override - public Matrix initializeZero() { + public MLMatrix initializeZero() { this.matrix.assign(0.0); return this; } @Override - public Matrix apply( DoubleUnaryOperator op ) { + public MLMatrix apply( DoubleUnaryOperator op ) { this.matrix.assign((d) -> op.applyAsDouble(d)); return this; } @Override - public Matrix transpose() { + public MLMatrix transpose() { this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.transpose(this.matrix); return this; } @Override - public Matrix multiply( Matrix B ) { + public MLMatrix multiply( MLMatrix B ) { ColtMatrix CB = (ColtMatrix)B; this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(matrix, CB.matrix); return this; } @Override - public Matrix multiplyLeft( Matrix B ) { + public MLMatrix multiplyLeft( MLMatrix B ) { ColtMatrix CB = (ColtMatrix)B; this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(CB.matrix, matrix); return this; } @Override - public Matrix multiplyAddBias( Matrix B, Matrix C ) { + public MLMatrix multiplyAddBias( MLMatrix B, MLMatrix C ) { ColtMatrix CB = (ColtMatrix)B; this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(matrix, CB.matrix); // TODO: add bias @@ -150,33 +148,33 @@ public class MatrixFactory { } @Override - public Matrix add( Matrix B ) { + public MLMatrix add( MLMatrix B ) { ColtMatrix CB = (ColtMatrix)B; matrix.assign(CB.matrix, (d1,d2) -> d1+d2); return this; } @Override - public Matrix sub( Matrix B ) { + public MLMatrix sub( MLMatrix B ) { ColtMatrix CB = (ColtMatrix)B; matrix.assign(CB.matrix, (d1,d2) -> d1-d2); return this; } @Override - public Matrix scale( double scalar ) { + public MLMatrix scale( double scalar ) { this.matrix.assign((d) -> d*scalar); return this; } @Override - public Matrix scale( Matrix S ) { + public MLMatrix scale( MLMatrix S ) { this.matrix.forEachNonZero((r, c, d) -> d * S.get(r, c)); return this; } @Override - public Matrix duplicate() { + public MLMatrix duplicate() { ColtMatrix newMatrix = new ColtMatrix(matrix.rows(), matrix.columns()); newMatrix.matrix.assign(this.matrix); return newMatrix; diff --git a/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java b/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java index c559463..2eae11b 100644 --- a/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java +++ b/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java @@ -107,7 +107,7 @@ public class NeuralNetwork { private NeuronLayer[] layers; - private Matrix output; + private MLMatrix output; private double learningRate = 0.1; @@ -162,16 +162,16 @@ public class NeuralNetwork { this.learningRate = pLearningRate; } - public Matrix getOutput() { + public MLMatrix getOutput() { return output; } - public Matrix predict( double[][] inputs ) { + public MLMatrix predict( double[][] inputs ) { //this.output = layers[1].apply(layers[0].apply(inputs)); return predict(MatrixFactory.create(inputs)); } - public Matrix predict( Matrix inputs ) { + public MLMatrix predict( MLMatrix inputs ) { this.output = layers[0].apply(inputs); return this.output; } @@ -180,7 +180,7 @@ public class NeuralNetwork { learn(MatrixFactory.create(expected)); } - public void learn( Matrix expected ) { + public void learn( MLMatrix expected ) { layers[layers.length - 1].backprop(expected, learningRate); } diff --git a/src/main/java/schule/ngb/zm/ml/NeuronLayer.java b/src/main/java/schule/ngb/zm/ml/NeuronLayer.java index 037d777..8846029 100644 --- a/src/main/java/schule/ngb/zm/ml/NeuronLayer.java +++ b/src/main/java/schule/ngb/zm/ml/NeuronLayer.java @@ -1,10 +1,9 @@ package schule.ngb.zm.ml; -import java.util.Arrays; import java.util.function.DoubleUnaryOperator; import java.util.function.Function; -public class NeuronLayer implements Function { +public class NeuronLayer implements Function { /*public static NeuronLayer fromArray( double[][] weights ) { NeuronLayer layer = new NeuronLayer(weights[0].length, weights.length); @@ -29,15 +28,15 @@ public class NeuronLayer implements Function { return layer; }*/ - Matrix weights; + MLMatrix weights; - Matrix biases; + MLMatrix biases; NeuronLayer previous, next; DoubleUnaryOperator activationFunction, activationFunctionDerivative; - Matrix lastOutput, lastInput; + MLMatrix lastOutput, lastInput; public NeuronLayer( int neurons, int inputs ) { weights = MatrixFactory @@ -87,11 +86,11 @@ public class NeuronLayer implements Function { } } - public Matrix getWeights() { + public MLMatrix getWeights() { return weights; } - public Matrix getBiases() { + public MLMatrix getBiases() { return biases; } @@ -103,15 +102,15 @@ public class NeuronLayer implements Function { return weights.rows(); } - public Matrix getLastOutput() { + public MLMatrix getLastOutput() { return lastOutput; } - public void setWeights( Matrix newWeights ) { + public void setWeights( MLMatrix newWeights ) { weights = newWeights.duplicate(); } - public void adjustWeights( Matrix adjustment ) { + public void adjustWeights( MLMatrix adjustment ) { weights.add(adjustment); } @@ -121,7 +120,7 @@ public class NeuronLayer implements Function { } @Override - public Matrix apply( Matrix inputs ) { + public MLMatrix apply( MLMatrix inputs ) { lastInput = inputs; lastOutput = inputs .multiplyAddBias(weights, biases) @@ -135,17 +134,17 @@ public class NeuronLayer implements Function { } @Override - public Function compose( Function before ) { + public Function compose( Function before ) { return ( in ) -> apply(before.apply(in)); } @Override - public Function andThen( Function after ) { + public Function andThen( Function after ) { return ( in ) -> after.apply(apply(in)); } - public void backprop( Matrix expected, double learningRate ) { - Matrix error, delta, adjustment; + public void backprop( MLMatrix expected, double learningRate ) { + MLMatrix error, delta, adjustment; if( next == null ) { error = expected.duplicate().sub(lastOutput); } else { diff --git a/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java b/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java index af85c88..35ff30c 100644 --- a/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java +++ b/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java @@ -5,13 +5,10 @@ import org.junit.jupiter.api.Test; import schule.ngb.zm.util.Log; import schule.ngb.zm.util.Timer; -import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Random; -import static org.junit.jupiter.api.Assertions.*; - class NeuralNetworkTest { @BeforeAll @@ -79,7 +76,7 @@ class NeuralNetworkTest { } // calculate predictions - Matrix predictions = net.predict(inputs); + MLMatrix predictions = net.predict(inputs); for( int i = 0; i < 4; i++ ) { int parsed_pred = predictions.get(i, 0) < 0.5 ? 0 : 1; From bf261b5e9b869ae9a4e519d835ab4e36048f2267 Mon Sep 17 00:00:00 2001 From: "J. Neugebauer" Date: Tue, 19 Jul 2022 20:05:37 +0200 Subject: [PATCH 3/6] =?UTF-8?q?Colt=20als=20optionale=20Abh=C3=A4ngigkeit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DAs Anlernen des NN geht um den Faktor 20 schneller, wenn Colt benutzt wird. --- build.gradle | 7 +- src/main/java/schule/ngb/zm/Constants.java | 3 +- .../java/schule/ngb/zm/ml/DoubleMatrix.java | 190 +++++---- src/main/java/schule/ngb/zm/ml/MLMatrix.java | 110 ++++- .../java/schule/ngb/zm/ml/MatrixFactory.java | 137 ++++--- .../java/schule/ngb/zm/ml/NeuronLayer.java | 32 +- .../schule/ngb/zm/ml/DoubleMatrixTest.java | 46 --- .../java/schule/ngb/zm/ml/MLMatrixTest.java | 375 ++++++++++++++++++ .../schule/ngb/zm/ml/NeuralNetworkTest.java | 40 +- 9 files changed, 735 insertions(+), 205 deletions(-) delete mode 100644 src/test/java/schule/ngb/zm/ml/DoubleMatrixTest.java create mode 100644 src/test/java/schule/ngb/zm/ml/MLMatrixTest.java diff --git a/build.gradle b/build.gradle index 465207f..746e57e 100644 --- a/build.gradle +++ b/build.gradle @@ -28,10 +28,13 @@ dependencies { runtimeOnly 'com.googlecode.soundlibs:tritonus-share:0.3.7.4' runtimeOnly 'com.googlecode.soundlibs:mp3spi:1.9.5.4' - compileOnlyApi 'colt:colt:1.2.0' + //compileOnlyApi 'colt:colt:1.2.0' + api 'colt:colt:1.2.0' + //api 'net.sourceforge.parallelcolt:parallelcolt:0.10.1' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' - testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' + testImplementation 'org.junit.jupiter:junit-jupiter-params:5.8.1' + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' } test { diff --git a/src/main/java/schule/ngb/zm/Constants.java b/src/main/java/schule/ngb/zm/Constants.java index f29544f..a6baee1 100644 --- a/src/main/java/schule/ngb/zm/Constants.java +++ b/src/main/java/schule/ngb/zm/Constants.java @@ -1269,7 +1269,8 @@ public class Constants { } /** - * Erzeugt eine Pseudozufallszahl nach einer Gaussverteilung. + * Erzeugt eine Pseudozufallszahl zwischen -1 und 1 nach einer + * Normalverteilung mit Mittelwert 0 und Standardabweichung 1. * * @return Eine Zufallszahl. * @see Random#nextGaussian() diff --git a/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java b/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java index 04391f4..758a929 100644 --- a/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java +++ b/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java @@ -4,10 +4,11 @@ import schule.ngb.zm.Constants; import java.util.Arrays; import java.util.function.DoubleUnaryOperator; +import java.util.stream.IntStream; // TODO: Move Math into Matrix class // TODO: Implement support for optional sci libs -public class DoubleMatrix implements MLMatrix { +public final class DoubleMatrix implements MLMatrix { private int columns, rows; @@ -22,7 +23,9 @@ public class DoubleMatrix implements MLMatrix { public DoubleMatrix( double[][] coefficients ) { this.rows = coefficients.length; this.columns = coefficients[0].length; - this.coefficients = coefficients; + this.coefficients = Arrays.stream(coefficients) + .map(double[]::clone) + .toArray(double[][]::new); } public int columns() { @@ -47,22 +50,133 @@ public class DoubleMatrix implements MLMatrix { } public MLMatrix initializeRandom() { - coefficients = MLMath.matrixApply(coefficients, (d) -> Constants.randomGaussian()); - return this; + return initializeRandom(-1.0, 1.0); } public MLMatrix initializeRandom( double lower, double upper ) { - coefficients = MLMath.matrixApply(coefficients, (d) -> ((upper-lower) * (Constants.randomGaussian()+1) * .5) + lower); + applyInPlace((d) -> ((upper-lower) * Constants.random()) + lower); return this; } public MLMatrix initializeOne() { - coefficients = MLMath.matrixApply(coefficients, (d) -> 1.0); + applyInPlace((d) -> 1.0); return this; } public MLMatrix initializeZero() { - coefficients = MLMath.matrixApply(coefficients, (d) -> 0.0); + applyInPlace((d) -> 0.0); + return this; + } + + @Override + public MLMatrix duplicate() { + return new DoubleMatrix(coefficients); + } + + @Override + public MLMatrix multiplyTransposed( MLMatrix B ) { + return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( + ( i ) -> IntStream.range(0, B.rows()).mapToDouble( + ( j ) -> IntStream.range(0, columns).mapToDouble( + (k) -> coefficients[i][k]*B.get(j,k) + ).sum() + ).toArray() + ).toArray(double[][]::new)); + } + + @Override + public MLMatrix multiplyAddBias( final MLMatrix B, final MLMatrix C ) { + return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( + ( i ) -> IntStream.range(0, B.columns()).mapToDouble( + ( j ) -> IntStream.range(0, columns).mapToDouble( + (k) -> coefficients[i][k]*B.get(k,j) + ).sum() + C.get(0, j) + ).toArray() + ).toArray(double[][]::new)); + } + + @Override + public MLMatrix transposedMultiplyAndScale( final MLMatrix B, final double scalar ) { + return new DoubleMatrix(IntStream.range(0, columns).parallel().mapToObj( + ( i ) -> IntStream.range(0, B.columns()).mapToDouble( + ( j ) -> IntStream.range(0, rows).mapToDouble( + (k) -> coefficients[k][i]*B.get(k,j)*scalar + ).sum() + ).toArray() + ).toArray(double[][]::new)); + } + + @Override + public MLMatrix add( MLMatrix B ) { + return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( + ( i ) -> IntStream.range(0, columns).mapToDouble( + ( j ) -> coefficients[i][j] + B.get(i, j) + ).toArray() + ).toArray(double[][]::new)); + } + + @Override + public MLMatrix addInPlace( MLMatrix B ) { + coefficients = IntStream.range(0, rows).parallel().mapToObj( + ( i ) -> IntStream.range(0, columns).mapToDouble( + ( j ) -> coefficients[i][j] + B.get(i, j) + ).toArray() + ).toArray(double[][]::new); + return this; + } + + @Override + public MLMatrix sub( MLMatrix B ) { + return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( + ( i ) -> IntStream.range(0, columns).mapToDouble( + ( j ) -> coefficients[i][j] - B.get(i, j) + ).toArray() + ).toArray(double[][]::new)); + } + + @Override + public MLMatrix colSums() { + double[][] sums = new double[1][columns]; + for( int c = 0; c < columns; c++ ) { + for( int r = 0; r < rows; r++ ) { + sums[0][c] += coefficients[r][c]; + } + } + return new DoubleMatrix(sums); + } + + @Override + public MLMatrix scaleInPlace( final double scalar ) { + coefficients = Arrays.stream(coefficients).parallel().map( + ( arr ) -> Arrays.stream(arr).map( + (d) -> d * scalar + ).toArray() + ).toArray(double[][]::new); + return this; + } + + @Override + public MLMatrix scaleInPlace( final MLMatrix S ) { + coefficients = IntStream.range(0, coefficients.length).parallel().mapToObj( + ( i ) -> IntStream.range(0, coefficients[i].length).mapToDouble( + ( j ) -> coefficients[i][j] * S.get(i, j) + ).toArray() + ).toArray(double[][]::new); + return this; + } + + @Override + public MLMatrix apply( DoubleUnaryOperator op ) { + return new DoubleMatrix(Arrays.stream(coefficients).parallel().map( + ( arr ) -> Arrays.stream(arr).map(op).toArray() + ).toArray(double[][]::new)); + } + + @Override + public MLMatrix applyInPlace( DoubleUnaryOperator op ) { + this.coefficients = Arrays.stream(coefficients).parallel().map( + ( arr ) -> Arrays.stream(arr).map(op).toArray() + ).toArray(double[][]::new); return this; } @@ -82,66 +196,4 @@ public class DoubleMatrix implements MLMatrix { return sb.toString(); } - @Override - public MLMatrix transpose() { - coefficients = MLMath.matrixTranspose(coefficients); - return this; - } - - @Override - public MLMatrix multiply( MLMatrix B ) { - coefficients = MLMath.matrixMultiply(coefficients, B.coefficients()); - return this; - } - - @Override - public MLMatrix multiplyAddBias( MLMatrix B, MLMatrix C ) { - double[] biases = Arrays.stream(C.coefficients()).mapToDouble(( arr) -> arr[0]).toArray(); - coefficients = MLMath.biasAdd( - MLMath.matrixMultiply(coefficients, B.coefficients()), - biases - ); - return this; - } - - @Override - public MLMatrix multiplyLeft( MLMatrix B ) { - coefficients = MLMath.matrixMultiply(B.coefficients(), coefficients); - return this; - } - - @Override - public MLMatrix add( MLMatrix B ) { - coefficients = MLMath.matrixAdd(coefficients, B.coefficients()); - return this; - } - - @Override - public MLMatrix sub( MLMatrix B ) { - coefficients = MLMath.matrixSub(coefficients, B.coefficients()); - return this; - } - - @Override - public MLMatrix scale( double scalar ) { - return this; - } - - @Override - public MLMatrix scale( MLMatrix S ) { - coefficients = MLMath.matrixScale(coefficients, S.coefficients()); - return this; - } - - @Override - public MLMatrix apply( DoubleUnaryOperator op ) { - this.coefficients = MLMath.matrixApply(coefficients, op); - return this; - } - - @Override - public MLMatrix duplicate() { - return new DoubleMatrix(MLMath.copyMatrix(coefficients)); - } - } diff --git a/src/main/java/schule/ngb/zm/ml/MLMatrix.java b/src/main/java/schule/ngb/zm/ml/MLMatrix.java index 0c2af15..bab792c 100644 --- a/src/main/java/schule/ngb/zm/ml/MLMatrix.java +++ b/src/main/java/schule/ngb/zm/ml/MLMatrix.java @@ -22,29 +22,123 @@ public interface MLMatrix { MLMatrix initializeZero(); + //MLMatrix transpose(); + + //MLMatrix multiply( MLMatrix B ); + /** - * Erzeugt die transponierte Matrix zu dieser. + * Erzeugt eine neue Matrix C mit dem Ergebnis der Matrixoperation + * 
+	 * C = A.B + V
+	 *
+ * wobei A dieses Matrixobjekt ist und {@code .} für die + * Matrixmultiplikation steht. + * + * @param B + * @param V * @return */ - MLMatrix transpose(); + MLMatrix multiplyAddBias( MLMatrix B, MLMatrix V ); - MLMatrix multiply( MLMatrix B ); + /** + * Erzeugt eine neue Matrix C mit dem Ergebnis der Matrixoperation + * 
+	 * C = A.t(B)
+	 *
+ * wobei A dieses Matrixobjekt ist und {@code t(B)} für die + * Transposition der Matrix B> steht. + * + * @param B + * @return + */ + MLMatrix multiplyTransposed( MLMatrix B ); - MLMatrix multiplyAddBias( MLMatrix B, MLMatrix C ); - - MLMatrix multiplyLeft( MLMatrix B ); + MLMatrix transposedMultiplyAndScale( MLMatrix B, double scalar ); + /** + * Erzeugt eine neue Matrix C mit dem Ergebnis der + * komponentenweisen Matrix-Addition + * 
+	 * C = A+B
+	 *
+ * wobei A dieses Matrixobjekt ist. Für ein Element + * C_ij in C gilt + * 
+	 * C_ij = A_ij + B_ij
+	 *
+ * + * @param B Die zweite Matrix. + * @return Ein neues Matrixobjekt mit dem Ergebnis. + */ MLMatrix add( MLMatrix B ); + /** + * Setzt dies Matrix auf das Ergebnis der + * komponentenweisen Matrix-Addition + * 
+	 * A = A+B
+	 *
+ * wobei A dieses Matrixobjekt ist. Für ein Element + * A_ij in A gilt + * 
+	 * A_ij = A_ij + B_ij
+	 *
+ * + * @param B Die zweite Matrix. + * @return Diese Matrix selbst (method chaining). + */ + MLMatrix addInPlace( MLMatrix B ); + + /** + * Erzeugt eine neue Matrix C mit dem Ergebnis der + * komponentenweisen Matrix-Subtraktion + * 
+	 * C = A-B
+	 *
+ * wobei A dieses Matrixobjekt ist. Für ein Element + * C_ij in C gilt + * 
+	 * C_ij = A_ij - B_ij
+	 *
+ * + * @param B + * @return + */ MLMatrix sub( MLMatrix B ); + MLMatrix scaleInPlace( double scalar ); - MLMatrix scale( double scalar ); + MLMatrix scaleInPlace( MLMatrix S ); - MLMatrix scale( MLMatrix S ); + /** + * Berechnet eine neue Matrix mit nur einer Zeile, die die Spaltensummen + * dieser Matrix enthalten. + * @return + */ + MLMatrix colSums(); + /** + * Endet die gegebene Funktion auf jeden Wert der Matrix an. + * + * @param op + * @return + */ MLMatrix apply( DoubleUnaryOperator op ); + /** + * Endet die gegebene Funktion auf jeden Wert der Matrix an. + * + * @param op + * @return + */ + MLMatrix applyInPlace( DoubleUnaryOperator op ); + + /** + * Erzeugt eine neue Matrix mit denselben Dimenstionen und Koeffizienten wie + * diese Matrix. + * + * @return + */ MLMatrix duplicate(); String toString(); diff --git a/src/main/java/schule/ngb/zm/ml/MatrixFactory.java b/src/main/java/schule/ngb/zm/ml/MatrixFactory.java index b0a8901..6a4d6d4 100644 --- a/src/main/java/schule/ngb/zm/ml/MatrixFactory.java +++ b/src/main/java/schule/ngb/zm/ml/MatrixFactory.java @@ -1,7 +1,6 @@ package schule.ngb.zm.ml; -import cern.colt.matrix.DoubleMatrix2D; -import cern.colt.matrix.impl.DenseDoubleMatrix2D; +import cern.colt.matrix.DoubleFactory2D; import schule.ngb.zm.Constants; import schule.ngb.zm.util.Log; @@ -11,7 +10,7 @@ public class MatrixFactory { public static void main( String[] args ) { System.out.println( - MatrixFactory.create(new double[][]{ {1.0, 0.0}, {0.0, 1.0} }).toString() + MatrixFactory.create(new double[][]{{1.0, 0.0}, {0.0, 1.0}}).toString() ); } @@ -26,14 +25,14 @@ public class MatrixFactory { public static final MLMatrix create( double[][] values ) { try { - return getMatrixType().getDeclaredConstructor(double[][].class).newInstance((Object)values); + return getMatrixType().getDeclaredConstructor(double[][].class).newInstance((Object) values); } catch( Exception ex ) { LOG.error(ex, "Could not initialize matrix implementation for class <%s>. Using internal implementation.", matrixType); } return new DoubleMatrix(values); } - private static Class matrixType = null; + static Class matrixType = null; private static final Class getMatrixType() { if( matrixType == null ) { @@ -63,6 +62,10 @@ public class MatrixFactory { matrix = new cern.colt.matrix.impl.DenseDoubleMatrix2D(rows, cols); } + public ColtMatrix( ColtMatrix matrix ) { + this.matrix = matrix.matrix.copy(); + } + @Override public int columns() { return matrix.columns(); @@ -91,13 +94,12 @@ public class MatrixFactory { @Override public MLMatrix initializeRandom() { - matrix.assign((d) -> Constants.randomGaussian()); - return this; + return initializeRandom(-1.0, 1.0); } @Override public MLMatrix initializeRandom( double lower, double upper ) { - matrix.assign((d) -> ((upper-lower) * (Constants.randomGaussian()+1) * .5) + lower); + matrix.assign(( d ) -> ((upper - lower) * Constants.random()) + lower); return this; } @@ -114,72 +116,97 @@ public class MatrixFactory { } @Override - public MLMatrix apply( DoubleUnaryOperator op ) { - this.matrix.assign((d) -> op.applyAsDouble(d)); - return this; + public MLMatrix duplicate() { + ColtMatrix newMatrix = new ColtMatrix(matrix.rows(), matrix.columns()); + newMatrix.matrix.assign(this.matrix); + return newMatrix; } @Override - public MLMatrix transpose() { - this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.transpose(this.matrix); - return this; - } - - @Override - public MLMatrix multiply( MLMatrix B ) { - ColtMatrix CB = (ColtMatrix)B; - this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(matrix, CB.matrix); - return this; - } - - @Override - public MLMatrix multiplyLeft( MLMatrix B ) { - ColtMatrix CB = (ColtMatrix)B; - this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(CB.matrix, matrix); - return this; + public MLMatrix multiplyTransposed( MLMatrix B ) { + ColtMatrix CB = (ColtMatrix) B; + ColtMatrix newMatrix = new ColtMatrix(0, 0); + newMatrix.matrix = matrix.zMult(CB.matrix, null, 1.0, 0.0, false, true); + return newMatrix; } @Override public MLMatrix multiplyAddBias( MLMatrix B, MLMatrix C ) { - ColtMatrix CB = (ColtMatrix)B; - this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(matrix, CB.matrix); - // TODO: add bias - return this; + ColtMatrix CB = (ColtMatrix) B; + ColtMatrix newMatrix = new ColtMatrix(0, 0); + newMatrix.matrix = DoubleFactory2D.dense.repeat(((ColtMatrix) C).matrix, rows(), 1); + matrix.zMult(CB.matrix, newMatrix.matrix, 1.0, 1.0, false, false); + return newMatrix; + } + + @Override + public MLMatrix transposedMultiplyAndScale( final MLMatrix B, final double scalar ) { + ColtMatrix CB = (ColtMatrix) B; + ColtMatrix newMatrix = new ColtMatrix(0, 0); + newMatrix.matrix = matrix.zMult(CB.matrix, null, scalar, 0.0, true, false); + return newMatrix; } @Override public MLMatrix add( MLMatrix B ) { - ColtMatrix CB = (ColtMatrix)B; - matrix.assign(CB.matrix, (d1,d2) -> d1+d2); + ColtMatrix CB = (ColtMatrix) B; + ColtMatrix newMatrix = new ColtMatrix(this); + newMatrix.matrix.assign(CB.matrix, ( d1, d2 ) -> d1 + d2); + return newMatrix; + } + + @Override + public MLMatrix addInPlace( MLMatrix B ) { + ColtMatrix CB = (ColtMatrix) B; + matrix.assign(CB.matrix, ( d1, d2 ) -> d1 + d2); return this; } @Override public MLMatrix sub( MLMatrix B ) { - ColtMatrix CB = (ColtMatrix)B; - matrix.assign(CB.matrix, (d1,d2) -> d1-d2); - return this; - } - - @Override - public MLMatrix scale( double scalar ) { - this.matrix.assign((d) -> d*scalar); - return this; - } - - @Override - public MLMatrix scale( MLMatrix S ) { - this.matrix.forEachNonZero((r, c, d) -> d * S.get(r, c)); - return this; - } - - @Override - public MLMatrix duplicate() { - ColtMatrix newMatrix = new ColtMatrix(matrix.rows(), matrix.columns()); - newMatrix.matrix.assign(this.matrix); + ColtMatrix CB = (ColtMatrix) B; + ColtMatrix newMatrix = new ColtMatrix(this); + newMatrix.matrix.assign(CB.matrix, ( d1, d2 ) -> d1 - d2); return newMatrix; } + @Override + public MLMatrix colSums() { + double[][] sums = new double[1][matrix.columns()]; + for( int c = 0; c < matrix.columns(); c++ ) { + for( int r = 0; r < matrix.rows(); r++ ) { + sums[0][c] += matrix.getQuick(r, c); + } + } + return new ColtMatrix(sums); + } + + @Override + public MLMatrix scaleInPlace( double scalar ) { + this.matrix.assign(( d ) -> d * scalar); + return this; + } + + @Override + public MLMatrix scaleInPlace( MLMatrix S ) { + this.matrix.forEachNonZero(( r, c, d ) -> d * S.get(r, c)); + return this; + } + + @Override + public MLMatrix apply( DoubleUnaryOperator op ) { + ColtMatrix newMatrix = new ColtMatrix(matrix.rows(), matrix.columns()); + newMatrix.matrix.assign(matrix); + newMatrix.matrix.assign(( d ) -> op.applyAsDouble(d)); + return newMatrix; + } + + @Override + public MLMatrix applyInPlace( DoubleUnaryOperator op ) { + this.matrix.assign(( d ) -> op.applyAsDouble(d)); + return this; + } + @Override public String toString() { return matrix.toString(); diff --git a/src/main/java/schule/ngb/zm/ml/NeuronLayer.java b/src/main/java/schule/ngb/zm/ml/NeuronLayer.java index 8846029..26a76eb 100644 --- a/src/main/java/schule/ngb/zm/ml/NeuronLayer.java +++ b/src/main/java/schule/ngb/zm/ml/NeuronLayer.java @@ -41,10 +41,10 @@ public class NeuronLayer implements Function { public NeuronLayer( int neurons, int inputs ) { weights = MatrixFactory .create(inputs, neurons) - .initializeRandom(-1, 1); + .initializeRandom(); biases = MatrixFactory - .create(neurons, 1) + .create(1, neurons) .initializeZero(); activationFunction = MLMath::sigmoid; @@ -110,10 +110,6 @@ public class NeuronLayer implements Function { weights = newWeights.duplicate(); } - public void adjustWeights( MLMatrix adjustment ) { - weights.add(adjustment); - } - @Override public String toString() { return weights.toString() + "\n" + biases.toString(); @@ -121,10 +117,10 @@ public class NeuronLayer implements Function { @Override public MLMatrix apply( MLMatrix inputs ) { - lastInput = inputs; + lastInput = inputs.duplicate(); lastOutput = inputs .multiplyAddBias(weights, biases) - .apply(activationFunction); + .applyInPlace(activationFunction); if( next != null ) { return next.apply(lastOutput); @@ -144,14 +140,16 @@ public class NeuronLayer implements Function { } public void backprop( MLMatrix expected, double learningRate ) { - MLMatrix error, delta, adjustment; + MLMatrix error, adjustment; if( next == null ) { - error = expected.duplicate().sub(lastOutput); + error = expected.sub(lastOutput); } else { - error = expected.duplicate().multiply(next.weights.transpose()); + error = expected.multiplyTransposed(next.weights); } - error.scale(lastOutput.duplicate().apply(this.activationFunctionDerivative)); + error.scaleInPlace( + lastOutput.apply(this.activationFunctionDerivative) + ); // Hier schon leraningRate anwenden? // See https://towardsdatascience.com/understanding-and-implementing-neural-networks-in-java-from-scratch-61421bb6352c //delta = MLMath.matrixApply(delta, ( x ) -> learningRate * x); @@ -159,10 +157,14 @@ public class NeuronLayer implements Function { previous.backprop(error, learningRate); } - // biases = MLMath.biasAdjust(biases, MLMath.matrixApply(delta, ( x ) -> learningRate * x)); + biases.addInPlace( + error.colSums().scaleInPlace( + -learningRate / (double) error.rows() + ) + ); - adjustment = lastInput.duplicate().transpose().multiply(error).apply((d) -> learningRate*d); - this.adjustWeights(adjustment); + adjustment = lastInput.transposedMultiplyAndScale(error, learningRate); + weights.addInPlace(adjustment); } } diff --git a/src/test/java/schule/ngb/zm/ml/DoubleMatrixTest.java b/src/test/java/schule/ngb/zm/ml/DoubleMatrixTest.java deleted file mode 100644 index 1ff70a8..0000000 --- a/src/test/java/schule/ngb/zm/ml/DoubleMatrixTest.java +++ /dev/null @@ -1,46 +0,0 @@ -package schule.ngb.zm.ml; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; - -import static org.junit.jupiter.api.Assertions.*; - -class DoubleMatrixTest { - - @Test - void initializeOne() { - DoubleMatrix m = new DoubleMatrix(4, 4); - m.initializeOne(); - - double[] ones = new double[]{1.0, 1.0, 1.0, 1.0}; - assertArrayEquals(ones, m.coefficients[0]); - assertArrayEquals(ones, m.coefficients[1]); - assertArrayEquals(ones, m.coefficients[2]); - assertArrayEquals(ones, m.coefficients[3]); - } - - @Test - void initializeZero() { - DoubleMatrix m = new DoubleMatrix(4, 4); - m.initializeZero(); - - double[] zeros = new double[]{0.0, 0.0, 0.0, 0.0}; - assertArrayEquals(zeros, m.coefficients[0]); - assertArrayEquals(zeros, m.coefficients[1]); - assertArrayEquals(zeros, m.coefficients[2]); - assertArrayEquals(zeros, m.coefficients[3]); - } - - @Test - void initializeRandom() { - DoubleMatrix m = new DoubleMatrix(4, 4); - m.initializeRandom(-1, 1); - - assertTrue(Arrays.stream(m.coefficients[0]).allMatch((d) -> -1.0 <= d && d < 1.0)); - assertTrue(Arrays.stream(m.coefficients[1]).allMatch((d) -> -1.0 <= d && d < 1.0)); - assertTrue(Arrays.stream(m.coefficients[2]).allMatch((d) -> -1.0 <= d && d < 1.0)); - assertTrue(Arrays.stream(m.coefficients[3]).allMatch((d) -> -1.0 <= d && d < 1.0)); - } - -} diff --git a/src/test/java/schule/ngb/zm/ml/MLMatrixTest.java b/src/test/java/schule/ngb/zm/ml/MLMatrixTest.java new file mode 100644 index 0000000..be311f8 --- /dev/null +++ b/src/test/java/schule/ngb/zm/ml/MLMatrixTest.java @@ -0,0 +1,375 @@ +package schule.ngb.zm.ml; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.*; + +class MLMatrixTest { + + private TestInfo info; + + @BeforeEach + void saveTestInfo( TestInfo info ) { + this.info = info; + } + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void initializeOne( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix m = MatrixFactory.create(4, 4); + m.initializeOne(); + + assertEquals(mType, m.getClass()); + + for( int i = 0; i < m.rows(); i++ ) { + for( int j = 0; j < m.columns(); j++ ) { + assertEquals(1.0, m.get(i, j)); + } + } + } + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void initializeZero( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix m = MatrixFactory.create(4, 4); + m.initializeZero(); + + assertEquals(mType, m.getClass()); + + for( int i = 0; i < m.rows(); i++ ) { + for( int j = 0; j < m.columns(); j++ ) { + assertEquals(0.0, m.get(i, j)); + } + } + } + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void initializeRandom( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix m = MatrixFactory.create(4, 4); + m.initializeRandom(); + + assertEquals(mType, m.getClass()); + + for( int i = 0; i < m.rows(); i++ ) { + for( int j = 0; j < m.columns(); j++ ) { + double d = m.get(i, j); + assertTrue(-1.0 <= d && d < 1.0); + } + } + } + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void multiplyTransposed( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix A = MatrixFactory.create(new double[][]{ + {1.0, 2.0, 3.0, 4.0}, + {1.0, 2.0, 3.0, 4.0}, + {1.0, 2.0, 3.0, 4.0} + }); + MLMatrix B = MatrixFactory.create(new double[][]{ + {1, 3, 5, 7}, + {2, 4, 6, 8} + }); + + MLMatrix C = A.multiplyTransposed(B); + + assertEquals(mType, A.getClass()); + assertEquals(mType, B.getClass()); + assertEquals(mType, C.getClass()); + + assertEquals(3, C.rows()); + assertEquals(2, C.columns()); + for( int i = 0; i < C.rows(); i++ ) { + assertEquals(50.0, C.get(i, 0)); + assertEquals(60.0, C.get(i, 1)); + } + } + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void multiplyAddBias( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix A = MatrixFactory.create(new double[][]{ + {1.0, 2.0, 3.0, 4.0}, + {1.0, 2.0, 3.0, 4.0}, + {1.0, 2.0, 3.0, 4.0} + }); + MLMatrix B = MatrixFactory.create(new double[][]{ + {1.0, 2.0}, + {3.0, 4.0}, + {5.0, 6.0}, + {7.0, 8.0} + }); + MLMatrix V = MatrixFactory.create(new double[][]{ + {1000.0, 2000.0} + }); + + MLMatrix C = A.multiplyAddBias(B, V); + + assertEquals(mType, A.getClass()); + assertEquals(mType, B.getClass()); + assertEquals(mType, C.getClass()); + assertEquals(mType, V.getClass()); + + assertEquals(3, C.rows()); + assertEquals(2, C.columns()); + for( int i = 0; i < C.rows(); i++ ) { + assertEquals(1050.0, C.get(i, 0)); + assertEquals(2060.0, C.get(i, 1)); + } + } + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void transposedMultiplyAndScale( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix A = MatrixFactory.create(new double[][]{ + {1, 1, 1}, + {2, 2, 2}, + {3, 3, 3}, + {4, 4, 4} + }); + MLMatrix B = MatrixFactory.create(new double[][]{ + {1.0, 2.0}, + {3.0, 4.0}, + {5.0, 6.0}, + {7.0, 8.0} + }); + + MLMatrix C = A.transposedMultiplyAndScale(B, 2.0); + + assertEquals(mType, A.getClass()); + assertEquals(mType, B.getClass()); + assertEquals(mType, C.getClass()); + + assertEquals(3, C.rows()); + assertEquals(2, C.columns()); + for( int i = 0; i < C.rows(); i++ ) { + assertEquals(100.0, C.get(i, 0)); + assertEquals(120.0, C.get(i, 1)); + } + } + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void apply( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix M = MatrixFactory.create(new double[][]{ + {1, 1, 1}, + {2, 2, 2}, + {3, 3, 3}, + {4, 4, 4} + }); + + MLMatrix R = M.apply(( d ) -> d * d); + + assertEquals(mType, M.getClass()); + assertEquals(mType, R.getClass()); + assertNotSame(M, R); + + for( int i = 0; i < M.rows(); i++ ) { + for( int j = 0; j < M.columns(); j++ ) { + assertEquals( + (i + 1) * (i + 1), R.get(i, j), + msg("(%d,%d)", "apply", i, j) + ); + } + } + + MLMatrix M2 = M.applyInPlace(( d ) -> d * d * d); + assertSame(M, M2); + for( int i = 0; i < M.rows(); i++ ) { + for( int j = 0; j < M.columns(); j++ ) { + assertEquals( + (i + 1) * (i + 1) * (i + 1), M.get(i, j), + msg("(%d,%d)", "applyInPlace", i, j) + ); + } + } + } + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void add( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix M = MatrixFactory.create(new double[][]{ + {1, 1, 1}, + {2, 2, 2}, + {3, 3, 3}, + {4, 4, 4} + }); + + MLMatrix R = M.add(M); + + assertEquals(mType, M.getClass()); + assertEquals(mType, R.getClass()); + assertNotSame(M, R); + + for( int i = 0; i < M.rows(); i++ ) { + for( int j = 0; j < M.columns(); j++ ) { + assertEquals( + (i + 1) + (i + 1), R.get(i, j), + msg("(%d,%d)", "add", i, j) + ); + } + } + + MLMatrix M2 = M.addInPlace(R); + assertSame(M, M2); + for( int i = 0; i < M.rows(); i++ ) { + for( int j = 0; j < M.columns(); j++ ) { + assertEquals( + (i + 1) + (i + 1) + (i + 1), M.get(i, j), + msg("(%d,%d)", "addInPlace", i, j) + ); + } + } + } + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void sub( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix M = MatrixFactory.create(new double[][]{ + {1, 1, 1}, + {2, 2, 2}, + {3, 3, 3}, + {4, 4, 4} + }); + + MLMatrix R = M.sub(M); + + assertEquals(mType, M.getClass()); + assertEquals(mType, R.getClass()); + assertNotSame(M, R); + + for( int i = 0; i < M.rows(); i++ ) { + for( int j = 0; j < M.columns(); j++ ) { + assertEquals( + 0.0, R.get(i, j), + msg("(%d,%d)", "sub", i, j) + ); + } + } + } + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void colSums( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix M = MatrixFactory.create(new double[][]{ + {1, 2, 3}, + {1, 2, 3}, + {1, 2, 3}, + {1, 2, 3} + }); + + MLMatrix R = M.colSums(); + + assertEquals(mType, M.getClass()); + assertEquals(mType, R.getClass()); + assertNotSame(M, R); + + assertEquals(1, R.rows()); + assertEquals(3, R.columns()); + for( int j = 0; j < M.columns(); j++ ) { + assertEquals( + (j+1)*4, R.get(0, j), + msg("(%d,%d)", "colSums", 0, j) + ); + } + } + + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void duplicate( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix M = MatrixFactory.create(new double[][]{ + {1, 2, 3}, + {1, 2, 3}, + {1, 2, 3}, + {1, 2, 3} + }); + + MLMatrix R = M.duplicate(); + + assertEquals(mType, M.getClass()); + assertEquals(mType, R.getClass()); + assertNotSame(M, R); + + for( int i = 0; i < M.rows(); i++ ) { + for( int j = 0; j < M.columns(); j++ ) { + assertEquals( + M.get(i, j), R.get(i, j), + msg("(%d,%d)", "duplicate", i, j) + ); + } + } + } + + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void scale( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix M = MatrixFactory.create(new double[][]{ + {1, 1, 1}, + {2, 2, 2}, + {3, 3, 3}, + {4, 4, 4} + }); + + MLMatrix M2 = M.scaleInPlace(2.0); + + assertEquals(mType, M.getClass()); + assertEquals(mType, M2.getClass()); + assertSame(M, M2); + + for( int i = 0; i < M.rows(); i++ ) { + for( int j = 0; j < M.columns(); j++ ) { + assertEquals( + (i+1)*2.0, M2.get(i, j), + msg("(%d,%d)", "scaleInPlace", i, j) + ); + } + } + + MLMatrix M3 = M.scaleInPlace(M); + assertSame(M, M3); + for( int i = 0; i < M.rows(); i++ ) { + for( int j = 0; j < M.columns(); j++ ) { + assertEquals( + ((i+1)*2.0)*((i+1)*2.0), M.get(i, j), + msg("(%d,%d)", "addInPlace", i, j) + ); + } + } + } + + private String msg( String msg, String methodName, Object... args ) { + String testName = this.info.getTestMethod().get().getName(); + String className = MatrixFactory.matrixType.getSimpleName(); + return String.format("[" + testName + "(" + className + ") " + methodName + "()] " + msg, args); + } + +} diff --git a/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java b/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java index 35ff30c..f523899 100644 --- a/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java +++ b/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java @@ -2,6 +2,7 @@ package schule.ngb.zm.ml; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import schule.ngb.zm.Constants; import schule.ngb.zm.util.Log; import schule.ngb.zm.util.Timer; @@ -16,6 +17,13 @@ class NeuralNetworkTest { Log.enableGlobalDebugging(); } + @BeforeAll + static void setupMatrixLibrary() { + Constants.setSeed(1001); + MatrixFactory.matrixType = MatrixFactory.ColtMatrix.class; + //MatrixFactory.matrixType = DoubleMatrix.class; + } + /*@Test void readWrite() { // XOR Dataset @@ -107,7 +115,7 @@ class NeuralNetworkTest { for( int i = 0; i < trainingData.size(); i++ ) { inputs[i][0] = trainingData.get(i).a; inputs[i][1] = trainingData.get(i).b; - outputs[i][0] = trainingData.get(i).result; + outputs[i][0] = trainingData.get(i).getResult(); } Timer timer = new Timer(); @@ -139,8 +147,8 @@ class NeuralNetworkTest { "Prediction on data (%.2f, %.2f) was %.4f, expected %.2f (of by %.4f)\n", data.a, data.b, net.getOutput().get(0, 0), - data.result, - net.getOutput().get(0, 0) - data.result + data.getResult(), + net.getOutput().get(0, 0) - data.getResult() ); } @@ -183,7 +191,6 @@ class NeuralNetworkTest { double a; double b; - double result; CalcType type; TestData( double a, double b ) { @@ -191,6 +198,8 @@ class NeuralNetworkTest { this.b = b; } + abstract double getResult(); + } private static final class AddData extends TestData { @@ -199,7 +208,9 @@ class NeuralNetworkTest { public AddData( double a, double b ) { super(a, b); - result = a + b; + } + double getResult() { + return a+b; } } @@ -210,7 +221,9 @@ class NeuralNetworkTest { public SubData( double a, double b ) { super(a, b); - result = a - b; + } + double getResult() { + return a-b; } } @@ -221,7 +234,9 @@ class NeuralNetworkTest { public MulData( double a, double b ) { super(a, b); - result = a * b; + } + double getResult() { + return a*b; } } @@ -235,7 +250,9 @@ class NeuralNetworkTest { if( b == 0.0 ) { b = .1; } - result = a / b; + } + double getResult() { + return a/b; } } @@ -246,7 +263,12 @@ class NeuralNetworkTest { public ModData( double b, double a ) { super(b, a); - result = a % b; + if( b == 0.0 ) { + b = .1; + } + } + double getResult() { + return a%b; } } From b6b4ffe6a537b6a27829eb00cd9fb1271434440c Mon Sep 17 00:00:00 2001 From: "J. Neugebauer" Date: Tue, 19 Jul 2022 22:52:23 +0200 Subject: [PATCH 4/6] =?UTF-8?q?Weitere=20Tests=20eingef=C3=BCgt=20und=20ve?= =?UTF-8?q?rbessert?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/schule/ngb/zm/ml/MLMatrixTest.java | 17 +++++++++++++++++ .../schule/ngb/zm/ml/NeuralNetworkTest.java | 9 ++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/test/java/schule/ngb/zm/ml/MLMatrixTest.java b/src/test/java/schule/ngb/zm/ml/MLMatrixTest.java index be311f8..2c153f6 100644 --- a/src/test/java/schule/ngb/zm/ml/MLMatrixTest.java +++ b/src/test/java/schule/ngb/zm/ml/MLMatrixTest.java @@ -16,6 +16,23 @@ class MLMatrixTest { this.info = info; } + @ParameterizedTest + @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) + void get( Class mType ) { + MatrixFactory.matrixType = mType; + + MLMatrix M = MatrixFactory.create(new double[][]{ + {1, 2, 3}, + {4, 5, 6} + }); + + assertEquals(mType, M.getClass()); + + assertEquals(1.0, M.get(0,0)); + assertEquals(4.0, M.get(1,0)); + assertEquals(6.0, M.get(1,2)); + } + @ParameterizedTest @ValueSource( classes = {DoubleMatrix.class, MatrixFactory.ColtMatrix.class} ) void initializeOne( Class mType ) { diff --git a/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java b/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java index f523899..3dca84e 100644 --- a/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java +++ b/src/test/java/schule/ngb/zm/ml/NeuralNetworkTest.java @@ -20,8 +20,8 @@ class NeuralNetworkTest { @BeforeAll static void setupMatrixLibrary() { Constants.setSeed(1001); - MatrixFactory.matrixType = MatrixFactory.ColtMatrix.class; - //MatrixFactory.matrixType = DoubleMatrix.class; + //MatrixFactory.matrixType = MatrixFactory.ColtMatrix.class; + MatrixFactory.matrixType = DoubleMatrix.class; } /*@Test @@ -153,12 +153,11 @@ class NeuralNetworkTest { } private List createTrainingSet( int trainingSetSize, CalcType operation ) { - Random random = new Random(); List tuples = new ArrayList<>(); for( int i = 0; i < trainingSetSize; i++ ) { - double s1 = random.nextDouble() * 0.5; - double s2 = random.nextDouble() * 0.5; + double s1 = Constants.random() * 0.5; + double s2 = Constants.random() * 0.5; switch( operation ) { case ADD: From d3997561fcc9e40676432e16e66a4403573dc9b3 Mon Sep 17 00:00:00 2001 From: "J. Neugebauer" Date: Tue, 19 Jul 2022 22:53:46 +0200 Subject: [PATCH 5/6] Streams durch Schleifen ersetzt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Der Overhead durch die parallelen Streams war zu hoch. Jedenfalls bei den relativ kleinen Matrizen im Test. Bei größeren Matrizen könnte die Parallelität einen Vorteil bringen. Ggf. sollte dies getesett werden und abhängig von der Größe die bestte Methode gewählt werden. --- .../java/schule/ngb/zm/ml/DoubleMatrix.java | 202 ++++++++++++------ 1 file changed, 137 insertions(+), 65 deletions(-) diff --git a/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java b/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java index 758a929..4a4008c 100644 --- a/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java +++ b/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java @@ -2,9 +2,7 @@ package schule.ngb.zm.ml; import schule.ngb.zm.Constants; -import java.util.Arrays; import java.util.function.DoubleUnaryOperator; -import java.util.stream.IntStream; // TODO: Move Math into Matrix class // TODO: Implement support for optional sci libs @@ -12,20 +10,33 @@ public final class DoubleMatrix implements MLMatrix { private int columns, rows; - double[][] coefficients; + double[] coefficients; - public DoubleMatrix( int rows, int cols ) { + public DoubleMatrix( int rows, int cols ) { this.rows = rows; this.columns = cols; - coefficients = new double[rows][cols]; + coefficients = new double[rows * cols]; } public DoubleMatrix( double[][] coefficients ) { this.rows = coefficients.length; this.columns = coefficients[0].length; - this.coefficients = Arrays.stream(coefficients) - .map(double[]::clone) - .toArray(double[][]::new); + this.coefficients = new double[rows * columns]; + for( int j = 0; j < columns; j++ ) { + for( int i = 0; i < rows; i++ ) { + this.coefficients[idx(i, j)] = coefficients[i][j]; + } + } + } + + public DoubleMatrix( DoubleMatrix other ) { + this.rows = other.rows(); + this.columns = other.columns(); + this.coefficients = new double[rows * columns]; + System.arraycopy( + other.coefficients, 0, + this.coefficients, 0, + rows * columns); } public int columns() { @@ -37,15 +48,19 @@ public final class DoubleMatrix implements MLMatrix { } public double[][] coefficients() { - return coefficients; + return new double[rows][columns]; + } + + int idx( int r, int c ) { + return c * rows + r; } public double get( int row, int col ) { - return coefficients[row][col]; + return coefficients[idx(row, col)]; } public MLMatrix set( int row, int col, double value ) { - coefficients[row][col] = value; + coefficients[idx(row, col)] = value; return this; } @@ -54,145 +69,202 @@ public final class DoubleMatrix implements MLMatrix { } public MLMatrix initializeRandom( double lower, double upper ) { - applyInPlace((d) -> ((upper-lower) * Constants.random()) + lower); + applyInPlace(( d ) -> ((upper - lower) * Constants.random()) + lower); return this; } public MLMatrix initializeOne() { - applyInPlace((d) -> 1.0); + applyInPlace(( d ) -> 1.0); return this; } public MLMatrix initializeZero() { - applyInPlace((d) -> 0.0); + applyInPlace(( d ) -> 0.0); return this; } @Override public MLMatrix duplicate() { - return new DoubleMatrix(coefficients); + return new DoubleMatrix(this); } @Override public MLMatrix multiplyTransposed( MLMatrix B ) { - return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( + /*return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( ( i ) -> IntStream.range(0, B.rows()).mapToDouble( - ( j ) -> IntStream.range(0, columns).mapToDouble( - (k) -> coefficients[i][k]*B.get(j,k) + ( j ) -> IntStream.range(0, columns).mapToDouble( + ( k ) -> get(i, k) * B.get(j, k) ).sum() ).toArray() - ).toArray(double[][]::new)); + ).toArray(double[][]::new));*/ + DoubleMatrix result = new DoubleMatrix(rows, B.rows()); + for( int i = 0; i < rows; i++ ) { + for( int j = 0; j < B.rows(); j++ ) { + result.coefficients[result.idx(i, j)] = 0.0; + for( int k = 0; k < columns; k++ ) { + result.coefficients[result.idx(i, j)] += get(i, k) * B.get(j, k); + } + } + } + return result; } @Override public MLMatrix multiplyAddBias( final MLMatrix B, final MLMatrix C ) { - return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( + /*return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( ( i ) -> IntStream.range(0, B.columns()).mapToDouble( - ( j ) -> IntStream.range(0, columns).mapToDouble( - (k) -> coefficients[i][k]*B.get(k,j) + ( j ) -> IntStream.range(0, columns).mapToDouble( + ( k ) -> get(i, k) * B.get(k, j) ).sum() + C.get(0, j) ).toArray() - ).toArray(double[][]::new)); + ).toArray(double[][]::new));*/ + DoubleMatrix result = new DoubleMatrix(rows, B.columns()); + for( int i = 0; i < rows; i++ ) { + for( int j = 0; j < B.columns(); j++ ) { + result.coefficients[result.idx(i, j)] = 0.0; + for( int k = 0; k < columns; k++ ) { + result.coefficients[result.idx(i, j)] += get(i, k) * B.get(k, j); + } + result.coefficients[result.idx(i, j)] += C.get(0, j); + } + } + return result; } @Override public MLMatrix transposedMultiplyAndScale( final MLMatrix B, final double scalar ) { - return new DoubleMatrix(IntStream.range(0, columns).parallel().mapToObj( + /*return new DoubleMatrix(IntStream.range(0, columns).parallel().mapToObj( ( i ) -> IntStream.range(0, B.columns()).mapToDouble( ( j ) -> IntStream.range(0, rows).mapToDouble( - (k) -> coefficients[k][i]*B.get(k,j)*scalar + ( k ) -> get(k, i) * B.get(k, j) * scalar ).sum() ).toArray() - ).toArray(double[][]::new)); + ).toArray(double[][]::new));*/ + DoubleMatrix result = new DoubleMatrix(columns, B.columns()); + for( int i = 0; i < columns; i++ ) { + for( int j = 0; j < B.columns(); j++ ) { + result.coefficients[result.idx(i, j)] = 0.0; + for( int k = 0; k < rows; k++ ) { + result.coefficients[result.idx(i, j)] += get(k, i) * B.get(k, j); + } + result.coefficients[result.idx(i, j)] *= scalar; + } + } + return result; } @Override public MLMatrix add( MLMatrix B ) { - return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( + /*return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( ( i ) -> IntStream.range(0, columns).mapToDouble( - ( j ) -> coefficients[i][j] + B.get(i, j) + ( j ) -> get(i, j) + B.get(i, j) ).toArray() - ).toArray(double[][]::new)); + ).toArray(double[][]::new));*/ + DoubleMatrix sum = new DoubleMatrix(rows, columns); + for( int j = 0; j < columns; j++ ) { + for( int i = 0; i < rows; i++ ) { + sum.coefficients[idx(i, j)] = coefficients[idx(i, j)] + B.get(i, j); + } + } + return sum; } @Override public MLMatrix addInPlace( MLMatrix B ) { - coefficients = IntStream.range(0, rows).parallel().mapToObj( - ( i ) -> IntStream.range(0, columns).mapToDouble( - ( j ) -> coefficients[i][j] + B.get(i, j) - ).toArray() - ).toArray(double[][]::new); + for( int j = 0; j < columns; j++ ) { + for( int i = 0; i < rows; i++ ) { + coefficients[idx(i, j)] += B.get(i, j); + } + } return this; } @Override public MLMatrix sub( MLMatrix B ) { - return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( + /*return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( ( i ) -> IntStream.range(0, columns).mapToDouble( - ( j ) -> coefficients[i][j] - B.get(i, j) + ( j ) -> get(i, j) - B.get(i, j) ).toArray() - ).toArray(double[][]::new)); + ).toArray(double[][]::new));*/ + DoubleMatrix diff = new DoubleMatrix(rows, columns); + for( int j = 0; j < columns; j++ ) { + for( int i = 0; i < rows; i++ ) { + diff.coefficients[idx(i, j)] = coefficients[idx(i, j)] - B.get(i, j); + } + } + return diff; } @Override public MLMatrix colSums() { - double[][] sums = new double[1][columns]; - for( int c = 0; c < columns; c++ ) { - for( int r = 0; r < rows; r++ ) { - sums[0][c] += coefficients[r][c]; + /*DoubleMatrix colSums = new DoubleMatrix(1, columns); + colSums.coefficients = IntStream.range(0, columns).parallel().mapToDouble( + ( j ) -> IntStream.range(0, rows).mapToDouble( + ( i ) -> get(i, j) + ).sum() + ).toArray(); + return colSums;*/ + DoubleMatrix colSums = new DoubleMatrix(1, columns); + for( int j = 0; j < columns; j++ ) { + colSums.coefficients[j] = 0.0; + for( int i = 0; i < rows; i++ ) { + colSums.coefficients[j] += coefficients[idx(i, j)]; } } - return new DoubleMatrix(sums); + return colSums; } @Override public MLMatrix scaleInPlace( final double scalar ) { - coefficients = Arrays.stream(coefficients).parallel().map( - ( arr ) -> Arrays.stream(arr).map( - (d) -> d * scalar - ).toArray() - ).toArray(double[][]::new); + for( int i = 0; i < coefficients.length; i++ ) { + coefficients[i] *= scalar; + } return this; } @Override public MLMatrix scaleInPlace( final MLMatrix S ) { - coefficients = IntStream.range(0, coefficients.length).parallel().mapToObj( - ( i ) -> IntStream.range(0, coefficients[i].length).mapToDouble( - ( j ) -> coefficients[i][j] * S.get(i, j) - ).toArray() - ).toArray(double[][]::new); + for( int j = 0; j < columns; j++ ) { + for( int i = 0; i < rows; i++ ) { + coefficients[idx(i, j)] *= S.get(i, j); + } + } return this; } @Override public MLMatrix apply( DoubleUnaryOperator op ) { - return new DoubleMatrix(Arrays.stream(coefficients).parallel().map( - ( arr ) -> Arrays.stream(arr).map(op).toArray() - ).toArray(double[][]::new)); + DoubleMatrix result = new DoubleMatrix(rows, columns); + for( int i = 0; i < coefficients.length; i++ ) { + result.coefficients[i] = op.applyAsDouble(coefficients[i]); + } + return result; } @Override public MLMatrix applyInPlace( DoubleUnaryOperator op ) { - this.coefficients = Arrays.stream(coefficients).parallel().map( - ( arr ) -> Arrays.stream(arr).map(op).toArray() - ).toArray(double[][]::new); + for( int i = 0; i < coefficients.length; i++ ) { + coefficients[i] = op.applyAsDouble(coefficients[i]); + } return this; } @Override public String toString() { - //return Arrays.deepToString(coefficients); StringBuilder sb = new StringBuilder(); - sb.append('['); + sb.append(rows); + sb.append(" x "); + sb.append(columns); + sb.append(" Matrix"); sb.append('\n'); - for( int i = 0; i < coefficients.length; i++ ) { - sb.append('\t'); - sb.append(Arrays.toString(coefficients[i])); + for( int i = 0; i < rows; i++ ) { + for( int j = 0; j < columns; j++ ) { + sb.append(get(i, j)); + if( j < columns - 1 ) + sb.append(' '); + } sb.append('\n'); } - sb.append(']'); - return sb.toString(); } From 16477463d4556737dae6e122f23ab3366d5f710a Mon Sep 17 00:00:00 2001 From: "J. Neugebauer" Date: Wed, 20 Jul 2022 17:09:09 +0200 Subject: [PATCH 6/6] java doc und refactorings --- .../java/schule/ngb/zm/ml/DoubleMatrix.java | 146 ++++++++- src/main/java/schule/ngb/zm/ml/MLMatrix.java | 284 ++++++++++++++---- .../java/schule/ngb/zm/ml/MatrixFactory.java | 51 +++- .../java/schule/ngb/zm/ml/NeuralNetwork.java | 62 ++-- .../java/schule/ngb/zm/ml/NeuronLayer.java | 41 ++- .../java/schule/ngb/zm/ml/MLMatrixTest.java | 34 +++ 6 files changed, 510 insertions(+), 108 deletions(-) diff --git a/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java b/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java index 4a4008c..2144217 100644 --- a/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java +++ b/src/main/java/schule/ngb/zm/ml/DoubleMatrix.java @@ -4,12 +4,60 @@ import schule.ngb.zm.Constants; import java.util.function.DoubleUnaryOperator; -// TODO: Move Math into Matrix class -// TODO: Implement support for optional sci libs +/** + * Eine einfache Implementierung der {@link MLMatrix} zur Verwendung in + * {@link NeuralNetwork}s. + *

+ * Diese Klasse stellt die interne Implementierung der Matrixoperationen dar, + * die zur Berechnung der Gewichte in einem {@link NeuronLayer} notwendig sind. + *

+ * Die Klasse ist nur minimal optimiert und sollte nur für kleine Netze + * verwendet werden. Für größere Netze sollte auf eine der optionalen + * Bibliotheken wie + * Colt zurückgegriffen werden. + */ public final class DoubleMatrix implements MLMatrix { - private int columns, rows; + /** + * Anzahl Zeilen der Matrix. + */ + private int rows; + /** + * Anzahl Spalten der Matrix. + */ + private int columns; + + /** + * Die Koeffizienten der Matrix. + *

+ * Um den Overhead bei Speicher und Zugriffszeiten von zweidimensionalen + * Arrays zu vermeiden wird ein eindimensionales Array verwendet und die + * Indizes mit Spaltenpriorität berechnet. Der Index i des Koeffizienten + * {@code r,c} in Zeile {@code r} und Spalte {@code c} wird bestimmt durch + * 

+	 * i = c * rows + r
+	 *
+ *

+ * Die Werte einer Spalte liegen also hintereinander im Array. Dies sollte + * einen leichten Vorteil bei der {@link #colSums() Spaltensummen} geben. + * Generell sollte eine Iteration über die Matrix der Form + * 


+	 * for( int j = 0; j < columns; j++ ) {
+	 *     for( int i = 0; i < rows; i++ ) {
+	 *         // ...
+	 *     }
+	 * }
+	 *
+ * etwas schneller sein als + * 

+	 * for( int i = 0; i < rows; i++ ) {
+	 *     for( int j = 0; j < columns; j++ ) {
+	 *         // ...
+	 *     }
+	 * }
+	 *
+ */ double[] coefficients; public DoubleMatrix( int rows, int cols ) { @@ -29,6 +77,11 @@ public final class DoubleMatrix implements MLMatrix { } } + /** + * Initialisiert diese Matrix als Kopie der angegebenen Matrix. + * + * @param other Die zu kopierende Matrix. + */ public DoubleMatrix( DoubleMatrix other ) { this.rows = other.rows(); this.columns = other.columns(); @@ -39,55 +92,100 @@ public final class DoubleMatrix implements MLMatrix { rows * columns); } + /** + * {@inheritDoc} + */ + @Override public int columns() { return columns; } + /** + * {@inheritDoc} + */ + @Override public int rows() { return rows; } - public double[][] coefficients() { - return new double[rows][columns]; - } - + /** + * {@inheritDoc} + */ int idx( int r, int c ) { return c * rows + r; } + /** + * {@inheritDoc} + */ + @Override public double get( int row, int col ) { - return coefficients[idx(row, col)]; + try { + return coefficients[idx(row, col)]; + } catch( ArrayIndexOutOfBoundsException ex ) { + throw new IllegalArgumentException("No element at row=" + row + ", column=" + col, ex); + } } + /** + * {@inheritDoc} + */ + @Override public MLMatrix set( int row, int col, double value ) { - coefficients[idx(row, col)] = value; + try { + coefficients[idx(row, col)] = value; + } catch( ArrayIndexOutOfBoundsException ex ) { + throw new IllegalArgumentException("No element at row=" + row + ", column=" + col, ex); + } return this; } + /** + * {@inheritDoc} + */ + @Override public MLMatrix initializeRandom() { return initializeRandom(-1.0, 1.0); } + /** + * {@inheritDoc} + */ + @Override public MLMatrix initializeRandom( double lower, double upper ) { applyInPlace(( d ) -> ((upper - lower) * Constants.random()) + lower); return this; } + /** + * {@inheritDoc} + */ + @Override public MLMatrix initializeOne() { applyInPlace(( d ) -> 1.0); return this; } + /** + * {@inheritDoc} + */ + @Override public MLMatrix initializeZero() { applyInPlace(( d ) -> 0.0); return this; } + /** + * {@inheritDoc} + */ @Override public MLMatrix duplicate() { return new DoubleMatrix(this); } + /** + * {@inheritDoc} + */ @Override public MLMatrix multiplyTransposed( MLMatrix B ) { /*return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( @@ -109,6 +207,9 @@ public final class DoubleMatrix implements MLMatrix { return result; } + /** + * {@inheritDoc} + */ @Override public MLMatrix multiplyAddBias( final MLMatrix B, final MLMatrix C ) { /*return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( @@ -131,6 +232,9 @@ public final class DoubleMatrix implements MLMatrix { return result; } + /** + * {@inheritDoc} + */ @Override public MLMatrix transposedMultiplyAndScale( final MLMatrix B, final double scalar ) { /*return new DoubleMatrix(IntStream.range(0, columns).parallel().mapToObj( @@ -153,6 +257,9 @@ public final class DoubleMatrix implements MLMatrix { return result; } + /** + * {@inheritDoc} + */ @Override public MLMatrix add( MLMatrix B ) { /*return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( @@ -169,6 +276,9 @@ public final class DoubleMatrix implements MLMatrix { return sum; } + /** + * {@inheritDoc} + */ @Override public MLMatrix addInPlace( MLMatrix B ) { for( int j = 0; j < columns; j++ ) { @@ -179,6 +289,9 @@ public final class DoubleMatrix implements MLMatrix { return this; } + /** + * {@inheritDoc} + */ @Override public MLMatrix sub( MLMatrix B ) { /*return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj( @@ -195,6 +308,9 @@ public final class DoubleMatrix implements MLMatrix { return diff; } + /** + * {@inheritDoc} + */ @Override public MLMatrix colSums() { /*DoubleMatrix colSums = new DoubleMatrix(1, columns); @@ -214,6 +330,9 @@ public final class DoubleMatrix implements MLMatrix { return colSums; } + /** + * {@inheritDoc} + */ @Override public MLMatrix scaleInPlace( final double scalar ) { for( int i = 0; i < coefficients.length; i++ ) { @@ -222,6 +341,9 @@ public final class DoubleMatrix implements MLMatrix { return this; } + /** + * {@inheritDoc} + */ @Override public MLMatrix scaleInPlace( final MLMatrix S ) { for( int j = 0; j < columns; j++ ) { @@ -232,6 +354,9 @@ public final class DoubleMatrix implements MLMatrix { return this; } + /** + * {@inheritDoc} + */ @Override public MLMatrix apply( DoubleUnaryOperator op ) { DoubleMatrix result = new DoubleMatrix(rows, columns); @@ -241,6 +366,9 @@ public final class DoubleMatrix implements MLMatrix { return result; } + /** + * {@inheritDoc} + */ @Override public MLMatrix applyInPlace( DoubleUnaryOperator op ) { for( int i = 0; i < coefficients.length; i++ ) { diff --git a/src/main/java/schule/ngb/zm/ml/MLMatrix.java b/src/main/java/schule/ngb/zm/ml/MLMatrix.java index bab792c..8de5e76 100644 --- a/src/main/java/schule/ngb/zm/ml/MLMatrix.java +++ b/src/main/java/schule/ngb/zm/ml/MLMatrix.java @@ -2,142 +2,312 @@ package schule.ngb.zm.ml; import java.util.function.DoubleUnaryOperator; +/** + * Interface für Matrizen, die in {@link NeuralNetwork} Klassen verwendet + * werden. + *

+ * Eine implementierende Klasse muss generell zwei Konstruktoren bereitstellen: + *

    + *
  1. {@code MLMatrix(int rows, int columns)} erstellt eine Matrix mit den + * angegebenen Dimensionen und setzt alle Koeffizienten auf 0. + *
  2. {@code MLMatrix(double[][] coefficients} erstellt eine Matrix mit der + * durch das Array gegebenen Dimensionen und setzt die Werte auf die + * jeweiligen Werte des Arrays. + *
+ *

+ * Das Interface ist nicht dazu gedacht eine allgemeine Umsetzung für + * Matrizen-Algebra abzubilden, sondern soll gezielt die im Neuralen Netzwerk + * verwendeten Algorithmen umsetzen. Einerseits würde eine ganz allgemeine + * Matrizen-Klasse nicht im Rahmen der Zeichenmaschine liegen und auf der + * anderen Seite bietet eine Konzentration auf die verwendeten Algorithmen mehr + * Spielraum zur Optimierung. + *

+ * Intern wird das Interface von {@link DoubleMatrix} implementiert. Die Klasse + * ist eine weitestgehend naive Implementierung der Algorithmen mit kleineren + * Optimierungen. Die Verwendung eines generalisierten Interfaces erlaubt aber + * zukünftig die optionale Integration spezialisierterer Algebra-Bibliotheken + * wie + * Colt, um auch große + * Netze effizient berechnen zu können. + */ public interface MLMatrix { + /** + * Die Anzahl der Spalten der Matrix. + * + * @return Spaltenzahl. + */ int columns(); + /** + * Die Anzahl der Zeilen der Matrix. + * + * @return Zeilenzahl. + */ int rows(); - double[][] coefficients(); + /** + * Gibt den Wert an der angegebenen Stelle der Matrix zurück. + * + * @param row Die Spaltennummer zwischen 0 und {@code rows()-1}. + * @param col Die Zeilennummer zwischen 0 und {@code columns()-1} + * @return Den Koeffizienten in der Zeile {@code row} und der Spalte + * {@code col}. + * @throws IllegalArgumentException Falls {@code row >= rows()} oder + * {@code col >= columns()}. + */ + double get( int row, int col ) throws IllegalArgumentException; - double get( int row, int col ); - - MLMatrix set( int row, int col, double value ); + /** + * Setzt den Wert an der angegebenen Stelle der Matrix. + * + * @param row Die Spaltennummer zwischen 0 und {@code rows()-1}. + * @param col Die Zeilennummer zwischen 0 und {@code columns()-1} + * @param value Der neue Wert. + * @return Diese Matrix selbst (method chaining). + * @throws IllegalArgumentException Falls {@code row >= rows()} oder + * {@code col >= columns()}. + */ + MLMatrix set( int row, int col, double value ) throws IllegalArgumentException; + /** + * Setzt jeden Wert in der Matrix auf eine Zufallszahl zwischen -1 und 1. + *

+ * Nach Möglichkeit sollte der + * {@link schule.ngb.zm.Constants#random(int, int) Zufallsgenerator der + * Zeichenmaschine} verwendet werden. + * + * @return Diese Matrix selbst (method chaining). + */ MLMatrix initializeRandom(); + /** + * Setzt jeden Wert in der Matrix auf eine Zufallszahl innerhalb der + * angegebenen Grenzen. + *

+ * Nach Möglichkeit sollte der + * {@link schule.ngb.zm.Constants#random(int, int) Zufallsgenerator der + * Zeichenmaschine} verwendet werden. + * + * @param lower Untere Grenze der Zufallszahlen. + * @param upper Obere Grenze der Zufallszahlen. + * @return Diese Matrix selbst (method chaining). + */ MLMatrix initializeRandom( double lower, double upper ); + /** + * Setzt alle Werte der Matrix auf 1. + * + * @return Diese Matrix selbst (method chaining). + */ MLMatrix initializeOne(); + /** + * Setzt alle Werte der Matrix auf 0. + * + * @return Diese Matrix selbst (method chaining). + */ MLMatrix initializeZero(); - //MLMatrix transpose(); - - //MLMatrix multiply( MLMatrix B ); + /** + * Erzeugt eine neue Matrix {@code C} mit dem Ergebnis der Matrixoperation + * 

+	 * C = this . B + V'
+	 *
+ * wobei {@code this} dieses Matrixobjekt ist und {@code .} für die + * Matrixmultiplikation steht. {@vode V'} ist die Matrix {@code V} + * {@code rows()}-mal untereinander wiederholt. + *

+ * Wenn diese Matrix die Dimension r x c hat, dann muss die Matrix {@code B} + * die Dimension c x m haben und {@code V} eine 1 x m Matrix sein. Die + * Matrix {@code V'} hat also die Dimension r x m, ebenso wie das Ergebnis + * der Operation. + * + * @param B Eine {@code columns()} x m Matrix mit der Multipliziert wird. + * @param V Eine 1 x {@code B.columns()} Matrix mit den Bias-Werten. + * @return Eine {@code rows()} x m Matrix. + * @throws IllegalArgumentException Falls die Dimensionen der Matrizen nicht + * zur Operation passen. Also + * {@code this.columns() != B.rows()} oder + * {@code B.columns() != V.columns()} oder + * {@code V.rows() != 1}. + */ + MLMatrix multiplyAddBias( MLMatrix B, MLMatrix V ) throws IllegalArgumentException; /** - * Erzeugt eine neue Matrix C mit dem Ergebnis der Matrixoperation + * Erzeugt eine neue Matrix {@code C} mit dem Ergebnis der Matrixoperation * 

-	 * C = A.B + V
+	 * C = this . t(B)
 	 *
- * wobei A dieses Matrixobjekt ist und {@code .} für die + * wobei {@code this} dieses Matrixobjekt ist, {@code t(B)} die + * Transposition der Matrix {@code B} ist und {@code .} für die * Matrixmultiplikation steht. + *

+ * Wenn diese Matrix die Dimension r x c hat, dann muss die Matrix {@code B} + * die Dimension m x c haben und das Ergebnis ist eine r x m Matrix. * - * @param B - * @param V - * @return + * @param B Eine m x {@code columns()} Matrix. + * @return Eine {@code rows()} x m Matrix. + * @throws IllegalArgumentException Falls die Dimensionen der Matrizen nicht + * zur Operation passen. Also + * {@code this.columns() != B.columns()}. */ - MLMatrix multiplyAddBias( MLMatrix B, MLMatrix V ); + MLMatrix multiplyTransposed( MLMatrix B ) throws IllegalArgumentException; /** - * Erzeugt eine neue Matrix C mit dem Ergebnis der Matrixoperation + * Erzeugt eine neue Matrix {@code C} mit dem Ergebnis der Matrixoperation * 

-	 * C = A.t(B)
+	 * C = t(this) . B * scalar
 	 *
- * wobei A dieses Matrixobjekt ist und {@code t(B)} für die - * Transposition der Matrix B> steht. + * wobei {@code this} dieses Matrixobjekt ist, {@code t(this)} die + * Transposition dieser Matrix ist und {@code .} für die + * Matrixmultiplikation steht. {@code *} bezeichnet die + * Skalarmultiplikation, bei der jeder Wert der Matrix mit {@code scalar} + * multipliziert wird. + *

+ * Wenn diese Matrix die Dimension r x c hat, dann muss die Matrix {@code B} + * die Dimension r x m haben und das Ergebnis ist eine c x m Matrix. * - * @param B - * @return + * @param B Eine m x {@code columns()} Matrix. + * @return Eine {@code rows()} x m Matrix. + * @throws IllegalArgumentException Falls die Dimensionen der Matrizen nicht + * zur Operation passen. Also + * {@code this.rows() != B.rows()}. */ - MLMatrix multiplyTransposed( MLMatrix B ); - - MLMatrix transposedMultiplyAndScale( MLMatrix B, double scalar ); + MLMatrix transposedMultiplyAndScale( MLMatrix B, double scalar ) throws IllegalArgumentException; /** - * Erzeugt eine neue Matrix C mit dem Ergebnis der - * komponentenweisen Matrix-Addition + * Erzeugt eine neue Matrix {@code C} mit dem Ergebnis der komponentenweisen + * Matrix-Addition * 

-	 * C = A+B
+	 * C = this + B
 	 *
- * wobei A dieses Matrixobjekt ist. Für ein Element - * C_ij in C gilt + * wobei {@code this} dieses Matrixobjekt ist. Für ein Element {@code C_ij} + * in {@code C} gilt * 
 	 * C_ij = A_ij + B_ij
 	 *
+ *

+ * Die Matrix {@code B} muss dieselbe Dimension wie diese Matrix haben. * - * @param B Die zweite Matrix. - * @return Ein neues Matrixobjekt mit dem Ergebnis. + * @param B Eine {@code rows()} x {@code columns()} Matrix. + * @return Eine {@code rows()} x {@code columns()} Matrix. + * @throws IllegalArgumentException Falls die Dimensionen der Matrizen nicht + * zur Operation passen. Also + * {@code this.rows() != B.rows()} oder + * {@code this.columns() != B.columns()}. */ - MLMatrix add( MLMatrix B ); + MLMatrix add( MLMatrix B ) throws IllegalArgumentException; /** - * Setzt dies Matrix auf das Ergebnis der - * komponentenweisen Matrix-Addition + * Setzt diese Matrix auf das Ergebnis der komponentenweisen + * Matrix-Addition * 

-	 * A = A+B
+	 * A' = A + B
 	 *
- * wobei A dieses Matrixobjekt ist. Für ein Element - * A_ij in A gilt + * wobei {@code A} dieses Matrixobjekt ist und {@code A'} diese Matrix nach + * der Operation. Für ein Element {@code A'_ij} in {@code A'} gilt * 
-	 * A_ij = A_ij + B_ij
+	 * A'_ij = A_ij + B_ij
 	 *
+ *

+ * Die Matrix {@code B} muss dieselbe Dimension wie diese Matrix haben. * - * @param B Die zweite Matrix. - * @return Diese Matrix selbst (method chaining). + * @param B Eine {@code rows()} x {@code columns()} Matrix. + * @return Eine {@code rows()} x {@code columns()} Matrix. + * @throws IllegalArgumentException Falls die Dimensionen der Matrizen nicht + * zur Operation passen. Also + * {@code this.rows() != B.rows()} oder + * {@code this.columns() != B.columns()}. */ - MLMatrix addInPlace( MLMatrix B ); + MLMatrix addInPlace( MLMatrix B ) throws IllegalArgumentException; /** - * Erzeugt eine neue Matrix C mit dem Ergebnis der - * komponentenweisen Matrix-Subtraktion + * Erzeugt eine neue Matrix {@code C} mit dem Ergebnis der komponentenweisen + * Matrix-Subtraktion * 

-	 * C = A-B
+	 * C = A - B
 	 *
- * wobei A dieses Matrixobjekt ist. Für ein Element - * C_ij in C gilt + * wobei {@code A} dieses Matrixobjekt ist. Für ein Element {@code C_ij} in + * {@code C} gilt * 
 	 * C_ij = A_ij - B_ij
 	 *
+ *

+ * Die Matrix {@code B} muss dieselbe Dimension wie diese Matrix haben. * - * @param B - * @return + * @param B Eine {@code rows()} x {@code columns()} Matrix. + * @return Eine {@code rows()} x {@code columns()} Matrix. + * @throws IllegalArgumentException Falls die Dimensionen der Matrizen nicht + * zur Operation passen. Also + * {@code this.rows() != B.rows()} oder + * {@code this.columns() != B.columns()}. */ - MLMatrix sub( MLMatrix B ); + MLMatrix sub( MLMatrix B ) throws IllegalArgumentException; + /** + * Multipliziert jeden Wert dieser Matrix mit dem angegebenen Skalar. + *

+ * Ist {@code A} dieses Matrixobjekt und {@code A'} diese Matrix nach der + * Operation, dann gilt für ein Element {@code A'_ij} in {@code A'} + * 

+	 * A'_ij = A_ij * scalar
+	 *
+ * + * @param scalar Ein Skalar. + * @return Diese Matrix selbst (method chaining) + */ MLMatrix scaleInPlace( double scalar ); - MLMatrix scaleInPlace( MLMatrix S ); + /** + * Multipliziert jeden Wert dieser Matrix mit dem entsprechenden Wert in der + * Matrix {@code S}. + *

+ * Ist {@code A} dieses Matrixobjekt und {@code A'} diese Matrix nach der + * Operation, dann gilt für ein Element {@code A'_ij} in {@code A'} + * 

+	 * A'_ij = A_ij * S_ij
+	 *
+ * + * @param S Eine {@code rows()} x {@code columns()} Matrix. + * @return Diese Matrix selbst (method chaining) + * @throws IllegalArgumentException Falls die Dimensionen der Matrizen nicht + * zur Operation passen. Also + * {@code this.rows() != B.rows()} oder + * {@code this.columns() != B.columns()}. + */ + MLMatrix scaleInPlace( MLMatrix S ) throws IllegalArgumentException; /** * Berechnet eine neue Matrix mit nur einer Zeile, die die Spaltensummen * dieser Matrix enthalten. - * @return + * + * @return Eine 1 x {@code columns()} Matrix. */ MLMatrix colSums(); /** - * Endet die gegebene Funktion auf jeden Wert der Matrix an. + * Erzeugt eine neue Matrix, deren Werte gleich den Werten dieser Matrix + * nach der Anwendung der angegebenen Funktion sind. * - * @param op - * @return + * @param op Eine Operation {@code (double) -> double}. + * @return Eine {@code rows()} x {@code columns()} Matrix. */ MLMatrix apply( DoubleUnaryOperator op ); /** * Endet die gegebene Funktion auf jeden Wert der Matrix an. * - * @param op - * @return + * @param op Eine Operation {@code (double) -> double}. + * @return Diese Matrix selbst (method chaining) */ MLMatrix applyInPlace( DoubleUnaryOperator op ); /** - * Erzeugt eine neue Matrix mit denselben Dimenstionen und Koeffizienten wie + * Erzeugt eine neue Matrix mit denselben Dimensionen und Koeffizienten wie * diese Matrix. * - * @return + * @return Eine Kopie dieser Matrix. */ MLMatrix duplicate(); diff --git a/src/main/java/schule/ngb/zm/ml/MatrixFactory.java b/src/main/java/schule/ngb/zm/ml/MatrixFactory.java index 6a4d6d4..889380a 100644 --- a/src/main/java/schule/ngb/zm/ml/MatrixFactory.java +++ b/src/main/java/schule/ngb/zm/ml/MatrixFactory.java @@ -6,14 +6,27 @@ import schule.ngb.zm.util.Log; import java.util.function.DoubleUnaryOperator; +/** + * Zentrale Klasse zur Erstellung neuer Matrizen. Generell sollten neue Matrizen + * nicht direkt erstellt werden, sondern durch den Aufruf von + * {@link #create(int, int)} oder {@link #create(double[][])}. Die Fabrik + * ermittelt automatisch die beste verfügbare Implementierung und initialisiert + * eine entsprechende Implementierung von {@link MLMatrix}. + *

+ * Derzeit werden die optionale Bibliothek Colt und die interne + * Implementierung {@link DoubleMatrix} unterstützt. + */ public class MatrixFactory { - public static void main( String[] args ) { - System.out.println( - MatrixFactory.create(new double[][]{{1.0, 0.0}, {0.0, 1.0}}).toString() - ); - } - + /** + * Erstellt eine neue Matrix mit den angegebenen Dimensionen und + * initialisiert alle Werte mit 0. + * + * @param rows Anzahl der Zeilen. + * @param cols Anzahl der Spalten. + * @return Eine {@code rows} x {@code cols} Matrix. + */ public static final MLMatrix create( int rows, int cols ) { try { return getMatrixType().getDeclaredConstructor(int.class, int.class).newInstance(rows, cols); @@ -23,6 +36,14 @@ public class MatrixFactory { return new DoubleMatrix(rows, cols); } + /** + * Erstellt eine neue Matrix mit den Dimensionen des angegebenen Arrays und + * initialisiert die Werte mit den entsprechenden Werten des Arrays. + * + * @param values Die Werte der Matrix. + * @return Eine {@code values.length} x {@code values[0].length} Matrix mit + * den Werten des Arrays. + */ public static final MLMatrix create( double[][] values ) { try { return getMatrixType().getDeclaredConstructor(double[][].class).newInstance((Object) values); @@ -32,8 +53,17 @@ public class MatrixFactory { return new DoubleMatrix(values); } + /** + * Die verwendete {@link MLMatrix} Implementierung, aus der Matrizen erzeugt + * werden. + */ static Class matrixType = null; + /** + * Ermittelt die beste verfügbare Implementierung von {@link MLMatrix}. + * + * @return Die verwendete {@link MLMatrix} Implementierung. + */ private static final Class getMatrixType() { if( matrixType == null ) { try { @@ -50,6 +80,10 @@ public class MatrixFactory { private static final Log LOG = Log.getLogger(MatrixFactory.class); + /** + * Interner Wrapper der DoubleMatrix2D Klasse aus der Colt Bibliothek, um + * das {@link MLMatrix} Interface zu implementieren. + */ static class ColtMatrix implements MLMatrix { cern.colt.matrix.DoubleMatrix2D matrix; @@ -87,11 +121,6 @@ public class MatrixFactory { return this; } - @Override - public double[][] coefficients() { - return this.matrix.toArray(); - } - @Override public MLMatrix initializeRandom() { return initializeRandom(-1.0, 1.0); diff --git a/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java b/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java index 2eae11b..b1cd4d6 100644 --- a/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java +++ b/src/main/java/schule/ngb/zm/ml/NeuralNetwork.java @@ -15,7 +15,7 @@ public class NeuralNetwork { Writer writer = ResourceStreamProvider.getWriter(source); PrintWriter out = new PrintWriter(writer) ) { - for( NeuronLayer layer: network.layers ) { + for( NeuronLayer layer : network.layers ) { out.print(layer.getNeuronCount()); out.print(' '); out.print(layer.getInputCount()); @@ -23,20 +23,44 @@ public class NeuralNetwork { for( int i = 0; i < layer.getInputCount(); i++ ) { for( int j = 0; j < layer.getNeuronCount(); j++ ) { - //out.print(layer.weights.coefficients[i][j]); + out.print(layer.weights.get(i, j)); out.print(' '); } out.println(); } for( int j = 0; j < layer.getNeuronCount(); j++ ) { - //out.print(layer.biases[j]); + out.print(layer.biases.get(0, j)); out.print(' '); } out.println(); } out.flush(); } catch( IOException ex ) { - LOG.warn(ex, ""); + LOG.error(ex, ""); + } + } + + public static void saveToDataFile( String source, NeuralNetwork network ) { + try( + OutputStream stream = ResourceStreamProvider.getOutputStream(source); + DataOutputStream out = new DataOutputStream(stream) + ) { + for( NeuronLayer layer : network.layers ) { + out.writeInt(layer.getNeuronCount()); + out.writeInt(layer.getInputCount()); + + for( int i = 0; i < layer.getInputCount(); i++ ) { + for( int j = 0; j < layer.getNeuronCount(); j++ ) { + out.writeDouble(layer.weights.get(i, j)); + } + } + for( int j = 0; j < layer.getNeuronCount(); j++ ) { + out.writeDouble(layer.biases.get(0, j)); + } + } + out.flush(); + } catch( IOException ex ) { + LOG.error(ex, ""); } } @@ -56,13 +80,13 @@ public class NeuralNetwork { for( int i = 0; i < inputs; i++ ) { split = in.readLine().split(" "); for( int j = 0; j < neurons; j++ ) { - //layer.weights.coefficients[i][j] = Double.parseDouble(split[j]); + layer.weights.set(i, j, Double.parseDouble(split[j])); } } // Load Biases split = in.readLine().split(" "); for( int j = 0; j < neurons; j++ ) { - //layer.biases[j] = Double.parseDouble(split[j]); + layer.biases.set(0, j, Double.parseDouble(split[j])); } layers.add(layer); @@ -70,29 +94,30 @@ public class NeuralNetwork { return new NeuralNetwork(layers); } catch( IOException | NoSuchElementException ex ) { - LOG.warn(ex, "Could not load neural network from source <%s>", source); + LOG.error(ex, "Could not load neural network from source <%s>", source); } return null; } - /*public static NeuralNetwork loadFromFile( String source ) { + public static NeuralNetwork loadFromDataFile( String source ) { try( InputStream stream = ResourceStreamProvider.getInputStream(source); - Scanner in = new Scanner(stream) + DataInputStream in = new DataInputStream(stream) ) { List layers = new LinkedList<>(); - while( in.hasNext() ) { - int neurons = in.nextInt(); - int inputs = in.nextInt(); + while( in.available() > 0 ) { + int neurons = in.readInt(); + int inputs = in.readInt(); NeuronLayer layer = new NeuronLayer(neurons, inputs); for( int i = 0; i < inputs; i++ ) { for( int j = 0; j < neurons; j++ ) { - layer.weights.coefficients[i][j] = in.nextDouble(); + layer.weights.set(i, j, in.readDouble()); } } + // Load Biases for( int j = 0; j < neurons; j++ ) { - layer.biases[j] = in.nextDouble(); + layer.biases.set(0, j, in.readDouble()); } layers.add(layer); @@ -100,10 +125,10 @@ public class NeuralNetwork { return new NeuralNetwork(layers); } catch( IOException | NoSuchElementException ex ) { - LOG.warn(ex, "Could not load neural network from source <%s>", source); + LOG.error(ex, "Could not load neural network from source <%s>", source); } return null; - }*/ + } private NeuronLayer[] layers; @@ -128,7 +153,7 @@ public class NeuralNetwork { for( int i = 0; i < layers.size(); i++ ) { this.layers[i] = layers.get(i); if( i > 0 ) { - this.layers[i-1].setNextLayer(this.layers[i]); + this.layers[i - 1].setNextLayer(this.layers[i]); } } } @@ -138,7 +163,7 @@ public class NeuralNetwork { for( int i = 0; i < layers.length; i++ ) { this.layers[i] = layers[i]; if( i > 0 ) { - this.layers[i-1].setNextLayer(this.layers[i]); + this.layers[i - 1].setNextLayer(this.layers[i]); } } } @@ -146,6 +171,7 @@ public class NeuralNetwork { public int getLayerCount() { return layers.length; } + public NeuronLayer[] getLayers() { return layers; } diff --git a/src/main/java/schule/ngb/zm/ml/NeuronLayer.java b/src/main/java/schule/ngb/zm/ml/NeuronLayer.java index 26a76eb..4ce061d 100644 --- a/src/main/java/schule/ngb/zm/ml/NeuronLayer.java +++ b/src/main/java/schule/ngb/zm/ml/NeuronLayer.java @@ -3,30 +3,45 @@ package schule.ngb.zm.ml; import java.util.function.DoubleUnaryOperator; import java.util.function.Function; +/** + * Implementierung einer Neuronenebene in einem Neuonalen Netz. + *

+ * Eine Ebene besteht aus einer Anzahl an Neuronen die jeweils eine + * Anzahl Eingänge haben. Die Eingänge erhalten als Signal die Ausgabe + * der vorherigen Ebene und berechnen die Ausgabe des jeweiligen Neurons. + */ public class NeuronLayer implements Function { - /*public static NeuronLayer fromArray( double[][] weights ) { - NeuronLayer layer = new NeuronLayer(weights[0].length, weights.length); + public static NeuronLayer fromArray( double[][] weights, boolean transpose ) { + NeuronLayer layer; + if( transpose ) { + layer = new NeuronLayer(weights.length, weights[0].length); + } else { + layer = new NeuronLayer(weights[0].length, weights.length); + } + for( int i = 0; i < weights[0].length; i++ ) { for( int j = 0; j < weights.length; j++ ) { - layer.weights.coefficients[i][j] = weights[i][j]; + if( transpose ) { + layer.weights.set(j, i, weights[i][j]); + } else { + layer.weights.set(i, j, weights[i][j]); + } } } + return layer; } - public static NeuronLayer fromArray( double[][] weights, double[] biases ) { - NeuronLayer layer = new NeuronLayer(weights[0].length, weights.length); - for( int i = 0; i < weights[0].length; i++ ) { - for( int j = 0; j < weights.length; j++ ) { - layer.weights.coefficients[i][j] = weights[i][j]; - } - } + public static NeuronLayer fromArray( double[][] weights, double[] biases, boolean transpose ) { + NeuronLayer layer = fromArray(weights, transpose); + for( int j = 0; j < biases.length; j++ ) { - layer.biases[j] = biases[j]; + layer.biases.set(0, j, biases[j]); } + return layer; - }*/ + } MLMatrix weights; @@ -112,7 +127,7 @@ public class NeuronLayer implements Function { @Override public String toString() { - return weights.toString() + "\n" + biases.toString(); + return "weights:\n" + weights.toString() + "\nbiases:\n" + biases.toString(); } @Override diff --git a/src/test/java/schule/ngb/zm/ml/MLMatrixTest.java b/src/test/java/schule/ngb/zm/ml/MLMatrixTest.java index 2c153f6..1e0a1da 100644 --- a/src/test/java/schule/ngb/zm/ml/MLMatrixTest.java +++ b/src/test/java/schule/ngb/zm/ml/MLMatrixTest.java @@ -1,9 +1,11 @@ package schule.ngb.zm.ml; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import schule.ngb.zm.util.Timer; import static org.junit.jupiter.api.Assertions.*; @@ -389,4 +391,36 @@ class MLMatrixTest { return String.format("[" + testName + "(" + className + ") " + methodName + "()] " + msg, args); } + //@ParameterizedTest + //@ValueSource( classes = {MatrixFactory.ColtMatrix.class, DoubleMatrix.class} ) + void speed( Class mType ) { + MatrixFactory.matrixType = mType; + + int N = 10; + int rows = 1000; + int cols = 1000; + + Timer timer = new Timer(); + + MLMatrix M = MatrixFactory.create(rows, cols); + timer.start(); + for( int i = 0; i < N; i++ ) { + M.initializeRandom(); + } + timer.stop(); + System.err.println(msg("%d iterations: %d ms", "initializeRandom", N, timer.getMillis())); + + timer.reset(); + + MLMatrix B = MatrixFactory.create(rows*2, M.columns()); + B.initializeRandom(); + + timer.start(); + for( int i = 0; i < N; i++ ) { + M.multiplyTransposed(B); + } + timer.stop(); + System.err.println(msg("%d iterations: %d ms", "multiplyTransposed", N, timer.getMillis())); + } + }