Colt als optionale Abhängigkeit

DAs Anlernen des NN geht um den Faktor 20 schneller, wenn Colt benutzt wird.

src/main/java/schule/ngb/zm/Constants.java

@@ -1269,7 +1269,8 @@ public class Constants {
 	}
 
 	/**
-	 * Erzeugt eine Pseudozufallszahl nach einer Gaussverteilung.
+	 * Erzeugt eine Pseudozufallszahl zwischen -1 und 1 nach einer
+	 * Normalverteilung mit Mittelwert 0 und Standardabweichung 1.
 	 *
 	 * @return Eine Zufallszahl.
 	 * @see Random#nextGaussian()

src/main/java/schule/ngb/zm/ml/DoubleMatrix.java

@@ -4,10 +4,11 @@ import schule.ngb.zm.Constants;
 
 import java.util.Arrays;
 import java.util.function.DoubleUnaryOperator;
+import java.util.stream.IntStream;
 
 // TODO: Move Math into Matrix class
 // TODO: Implement support for optional sci libs
-public class DoubleMatrix implements MLMatrix {
+public final class DoubleMatrix implements MLMatrix {
 
 	private int columns, rows;
 
@@ -22,7 +23,9 @@ public class DoubleMatrix implements MLMatrix {
 	public DoubleMatrix( double[][] coefficients ) {
 		this.rows = coefficients.length;
 		this.columns = coefficients[0].length;
-		this.coefficients = coefficients;
+		this.coefficients = Arrays.stream(coefficients)
+				.map(double[]::clone)
+				.toArray(double[][]::new);
 	}
 
 	public int columns() {
@@ -47,22 +50,133 @@ public class DoubleMatrix implements MLMatrix {
 	}
 
 	public MLMatrix initializeRandom() {
-		coefficients = MLMath.matrixApply(coefficients, (d) -> Constants.randomGaussian());
-		return this;
+		return initializeRandom(-1.0, 1.0);
 	}
 
 	public MLMatrix initializeRandom( double lower, double upper ) {
-		coefficients = MLMath.matrixApply(coefficients, (d) -> ((upper-lower) * (Constants.randomGaussian()+1) * .5) + lower);
+		applyInPlace((d) -> ((upper-lower) * Constants.random()) + lower);
 		return this;
 	}
 
 	public MLMatrix initializeOne() {
-		coefficients = MLMath.matrixApply(coefficients, (d) -> 1.0);
+		applyInPlace((d) -> 1.0);
 		return this;
 	}
 
 	public MLMatrix initializeZero() {
-		coefficients = MLMath.matrixApply(coefficients, (d) -> 0.0);
+		applyInPlace((d) -> 0.0);
+		return this;
+	}
+
+	@Override
+	public MLMatrix duplicate() {
+		return new DoubleMatrix(coefficients);
+	}
+
+	@Override
+	public MLMatrix multiplyTransposed( MLMatrix B ) {
+		return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj(
+			( i ) -> IntStream.range(0, B.rows()).mapToDouble(
+				( j ) ->  IntStream.range(0, columns).mapToDouble(
+					(k) -> coefficients[i][k]*B.get(j,k)
+				).sum()
+			).toArray()
+		).toArray(double[][]::new));
+	}
+
+	@Override
+	public MLMatrix multiplyAddBias( final MLMatrix B, final MLMatrix C ) {
+		return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj(
+			( i ) -> IntStream.range(0, B.columns()).mapToDouble(
+				( j ) ->  IntStream.range(0, columns).mapToDouble(
+					(k) -> coefficients[i][k]*B.get(k,j)
+				).sum() + C.get(0, j)
+			).toArray()
+		).toArray(double[][]::new));
+	}
+
+	@Override
+	public MLMatrix transposedMultiplyAndScale( final MLMatrix B, final double scalar ) {
+		return new DoubleMatrix(IntStream.range(0, columns).parallel().mapToObj(
+			( i ) -> IntStream.range(0, B.columns()).mapToDouble(
+				( j ) -> IntStream.range(0, rows).mapToDouble(
+					(k) -> coefficients[k][i]*B.get(k,j)*scalar
+				).sum()
+			).toArray()
+		).toArray(double[][]::new));
+	}
+
+	@Override
+	public MLMatrix add( MLMatrix B ) {
+		return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj(
+			( i ) -> IntStream.range(0, columns).mapToDouble(
+				( j ) -> coefficients[i][j] + B.get(i, j)
+			).toArray()
+		).toArray(double[][]::new));
+	}
+
+	@Override
+	public MLMatrix addInPlace( MLMatrix B ) {
+		coefficients = IntStream.range(0, rows).parallel().mapToObj(
+			( i ) -> IntStream.range(0, columns).mapToDouble(
+				( j ) -> coefficients[i][j] + B.get(i, j)
+			).toArray()
+		).toArray(double[][]::new);
+		return this;
+	}
+
+	@Override
+	public MLMatrix sub( MLMatrix B ) {
+		return new DoubleMatrix(IntStream.range(0, rows).parallel().mapToObj(
+			( i ) -> IntStream.range(0, columns).mapToDouble(
+				( j ) -> coefficients[i][j] - B.get(i, j)
+			).toArray()
+		).toArray(double[][]::new));
+	}
+
+	@Override
+	public MLMatrix colSums() {
+		double[][] sums = new double[1][columns];
+		for( int c = 0; c < columns; c++ ) {
+			for( int r = 0; r < rows; r++ ) {
+				sums[0][c] += coefficients[r][c];
+			}
+		}
+		return new DoubleMatrix(sums);
+	}
+
+	@Override
+	public MLMatrix scaleInPlace( final double scalar ) {
+		coefficients = Arrays.stream(coefficients).parallel().map(
+			( arr ) -> Arrays.stream(arr).map(
+				(d) -> d * scalar
+			).toArray()
+		).toArray(double[][]::new);
+		return this;
+	}
+
+	@Override
+	public MLMatrix scaleInPlace( final MLMatrix S ) {
+		coefficients = IntStream.range(0, coefficients.length).parallel().mapToObj(
+			( i ) -> IntStream.range(0, coefficients[i].length).mapToDouble(
+				( j ) -> coefficients[i][j] * S.get(i, j)
+			).toArray()
+		).toArray(double[][]::new);
+		return this;
+	}
+
+	@Override
+	public MLMatrix apply( DoubleUnaryOperator op ) {
+		return new DoubleMatrix(Arrays.stream(coefficients).parallel().map(
+			( arr ) -> Arrays.stream(arr).map(op).toArray()
+		).toArray(double[][]::new));
+	}
+
+	@Override
+	public MLMatrix applyInPlace( DoubleUnaryOperator op ) {
+		this.coefficients = Arrays.stream(coefficients).parallel().map(
+			( arr ) -> Arrays.stream(arr).map(op).toArray()
+		).toArray(double[][]::new);
 		return this;
 	}
 
@@ -82,66 +196,4 @@ public class DoubleMatrix implements MLMatrix {
 		return sb.toString();
 	}
 
-	@Override
-	public MLMatrix transpose() {
-		coefficients = MLMath.matrixTranspose(coefficients);
-		return this;
-	}
-
-	@Override
-	public MLMatrix multiply( MLMatrix B ) {
-		coefficients = MLMath.matrixMultiply(coefficients, B.coefficients());
-		return this;
-	}
-
-	@Override
-	public MLMatrix multiplyAddBias( MLMatrix B, MLMatrix C ) {
-		double[] biases = Arrays.stream(C.coefficients()).mapToDouble(( arr) -> arr[0]).toArray();
-		coefficients = MLMath.biasAdd(
-			MLMath.matrixMultiply(coefficients, B.coefficients()),
-			biases
-		);
-		return this;
-	}
-
-	@Override
-	public MLMatrix multiplyLeft( MLMatrix B ) {
-		coefficients = MLMath.matrixMultiply(B.coefficients(), coefficients);
-		return this;
-	}
-
-	@Override
-	public MLMatrix add( MLMatrix B ) {
-		coefficients = MLMath.matrixAdd(coefficients, B.coefficients());
-		return this;
-	}
-
-	@Override
-	public MLMatrix sub( MLMatrix B ) {
-		coefficients = MLMath.matrixSub(coefficients, B.coefficients());
-		return this;
-	}
-
-	@Override
-	public MLMatrix scale( double scalar ) {
-		return this;
-	}
-
-	@Override
-	public MLMatrix scale( MLMatrix S ) {
-		coefficients = MLMath.matrixScale(coefficients, S.coefficients());
-		return this;
-	}
-
-	@Override
-	public MLMatrix apply( DoubleUnaryOperator op ) {
-		this.coefficients = MLMath.matrixApply(coefficients, op);
-		return this;
-	}
-
-	@Override
-	public MLMatrix duplicate() {
-		return new DoubleMatrix(MLMath.copyMatrix(coefficients));
-	}
-
 }

src/main/java/schule/ngb/zm/ml/MLMatrix.java

@@ -22,29 +22,123 @@ public interface MLMatrix {
 
 	MLMatrix initializeZero();
 
+	//MLMatrix transpose();
+
+	//MLMatrix multiply( MLMatrix B );
+
 	/**
-	 * Erzeugt die transponierte Matrix zu dieser.
+	 * Erzeugt eine neue Matrix <em>C</em> mit dem Ergebnis der Matrixoperation
+	 * <pre>
+	 * C = A.B + V
+	 * </pre>
+	 * wobei <em>A</em> dieses Matrixobjekt ist und {@code .} für die
+	 * Matrixmultiplikation steht.
+	 *
+	 * @param B
+	 * @param V
 	 * @return
 	 */
-	MLMatrix transpose();
+	MLMatrix multiplyAddBias( MLMatrix B, MLMatrix V );
 
-	MLMatrix multiply( MLMatrix B );
+	/**
+	 * Erzeugt eine neue Matrix <em>C</em> mit dem Ergebnis der Matrixoperation
+	 * <pre>
+	 * C = A.t(B)
+	 * </pre>
+	 * wobei <em>A</em> dieses Matrixobjekt ist und {@code t(B)} für die
+	 * Transposition der Matrix <em>B</em>> steht.
+	 *
+	 * @param B
+	 * @return
+	 */
+	MLMatrix multiplyTransposed( MLMatrix B );
 
-	MLMatrix multiplyAddBias( MLMatrix B, MLMatrix C );
-
-	MLMatrix multiplyLeft( MLMatrix B );
+	MLMatrix transposedMultiplyAndScale( MLMatrix B, double scalar );
 
+	/**
+	 * Erzeugt eine neue Matrix <em>C</em> mit dem Ergebnis der
+	 * komponentenweisen Matrix-Addition
+	 * <pre>
+	 * C = A+B
+	 * </pre>
+	 * wobei <em>A</em> dieses Matrixobjekt ist. Für ein Element
+	 * <em>C_ij</em> in <em>C</em> gilt
+	 * <pre>
+	 * C_ij = A_ij + B_ij
+	 * </pre>
+	 *
+	 * @param B Die zweite Matrix.
+	 * @return Ein neues Matrixobjekt mit dem Ergebnis.
+	 */
 	MLMatrix add( MLMatrix B );
 
+	/**
+	 * Setzt dies Matrix auf das Ergebnis der
+	 * komponentenweisen Matrix-Addition
+	 * <pre>
+	 * A = A+B
+	 * </pre>
+	 * wobei <em>A</em> dieses Matrixobjekt ist. Für ein Element
+	 * <em>A_ij</em> in <em>A</em> gilt
+	 * <pre>
+	 * A_ij = A_ij + B_ij
+	 * </pre>
+	 *
+	 * @param B Die zweite Matrix.
+	 * @return Diese Matrix selbst (method chaining).
+	 */
+	MLMatrix addInPlace( MLMatrix B );
+
+	/**
+	 * Erzeugt eine neue Matrix <em>C</em> mit dem Ergebnis der
+	 * komponentenweisen Matrix-Subtraktion
+	 * <pre>
+	 * C = A-B
+	 * </pre>
+	 * wobei <em>A</em> dieses Matrixobjekt ist. Für ein Element
+	 * <em>C_ij</em> in <em>C</em> gilt
+	 * <pre>
+	 * C_ij = A_ij - B_ij
+	 * </pre>
+	 *
+	 * @param B
+	 * @return
+	 */
 	MLMatrix sub( MLMatrix B );
 
+	MLMatrix scaleInPlace( double scalar );
 
-	MLMatrix scale( double scalar );
+	MLMatrix scaleInPlace( MLMatrix S );
 
-	MLMatrix scale( MLMatrix S );
+	/**
+	 * Berechnet eine neue Matrix mit nur einer Zeile, die die Spaltensummen
+	 * dieser Matrix enthalten.
+	 * @return
+	 */
+	MLMatrix colSums();
 
+	/**
+	 * Endet die gegebene Funktion auf jeden Wert der Matrix an.
+	 *
+	 * @param op
+	 * @return
+	 */
 	MLMatrix apply( DoubleUnaryOperator op );
 
+	/**
+	 * Endet die gegebene Funktion auf jeden Wert der Matrix an.
+	 *
+	 * @param op
+	 * @return
+	 */
+	MLMatrix applyInPlace( DoubleUnaryOperator op );
+
+	/**
+	 * Erzeugt eine neue Matrix mit denselben Dimenstionen und Koeffizienten wie
+	 * diese Matrix.
+	 *
+	 * @return
+	 */
 	MLMatrix duplicate();
 
 	String toString();

src/main/java/schule/ngb/zm/ml/MatrixFactory.java

@@ -1,7 +1,6 @@
 package schule.ngb.zm.ml;
 
-import cern.colt.matrix.DoubleMatrix2D;
-import cern.colt.matrix.impl.DenseDoubleMatrix2D;
+import cern.colt.matrix.DoubleFactory2D;
 import schule.ngb.zm.Constants;
 import schule.ngb.zm.util.Log;
 
@@ -11,7 +10,7 @@ public class MatrixFactory {
 
 	public static void main( String[] args ) {
 		System.out.println(
-			MatrixFactory.create(new double[][]{ {1.0, 0.0}, {0.0, 1.0} }).toString()
+			MatrixFactory.create(new double[][]{{1.0, 0.0}, {0.0, 1.0}}).toString()
 		);
 	}
 
@@ -26,14 +25,14 @@ public class MatrixFactory {
 
 	public static final MLMatrix create( double[][] values ) {
 		try {
-			return getMatrixType().getDeclaredConstructor(double[][].class).newInstance((Object)values);
+			return getMatrixType().getDeclaredConstructor(double[][].class).newInstance((Object) values);
 		} catch( Exception ex ) {
 			LOG.error(ex, "Could not initialize matrix implementation for class <%s>. Using internal implementation.", matrixType);
 		}
 		return new DoubleMatrix(values);
 	}
 
-	private static Class<? extends MLMatrix> matrixType = null;
+	static Class<? extends MLMatrix> matrixType = null;
 
 	private static final Class<? extends MLMatrix> getMatrixType() {
 		if( matrixType == null ) {
@@ -63,6 +62,10 @@ public class MatrixFactory {
 			matrix = new cern.colt.matrix.impl.DenseDoubleMatrix2D(rows, cols);
 		}
 
+		public ColtMatrix( ColtMatrix matrix ) {
+			this.matrix = matrix.matrix.copy();
+		}
+
 		@Override
 		public int columns() {
 			return matrix.columns();
@@ -91,13 +94,12 @@ public class MatrixFactory {
 
 		@Override
 		public MLMatrix initializeRandom() {
-			matrix.assign((d) -> Constants.randomGaussian());
-			return this;
+			return initializeRandom(-1.0, 1.0);
 		}
 
 		@Override
 		public MLMatrix initializeRandom( double lower, double upper ) {
-			matrix.assign((d) -> ((upper-lower) * (Constants.randomGaussian()+1) * .5) + lower);
+			matrix.assign(( d ) -> ((upper - lower) * Constants.random()) + lower);
 			return this;
 		}
 
@@ -114,72 +116,97 @@ public class MatrixFactory {
 		}
 
 		@Override
-		public MLMatrix apply( DoubleUnaryOperator op ) {
-			this.matrix.assign((d) -> op.applyAsDouble(d));
-			return this;
+		public MLMatrix duplicate() {
+			ColtMatrix newMatrix = new ColtMatrix(matrix.rows(), matrix.columns());
+			newMatrix.matrix.assign(this.matrix);
+			return newMatrix;
 		}
 
 		@Override
-		public MLMatrix transpose() {
-			this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.transpose(this.matrix);
-			return this;
-		}
-
-		@Override
-		public MLMatrix multiply( MLMatrix B ) {
-			ColtMatrix CB = (ColtMatrix)B;
-			this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(matrix, CB.matrix);
-			return this;
-		}
-
-		@Override
-		public MLMatrix multiplyLeft( MLMatrix B ) {
-			ColtMatrix CB = (ColtMatrix)B;
-			this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(CB.matrix, matrix);
-			return this;
+		public MLMatrix multiplyTransposed( MLMatrix B ) {
+			ColtMatrix CB = (ColtMatrix) B;
+			ColtMatrix newMatrix = new ColtMatrix(0, 0);
+			newMatrix.matrix = matrix.zMult(CB.matrix, null, 1.0, 0.0, false, true);
+			return newMatrix;
 		}
 
 		@Override
 		public MLMatrix multiplyAddBias( MLMatrix B, MLMatrix C ) {
-			ColtMatrix CB = (ColtMatrix)B;
-			this.matrix = cern.colt.matrix.linalg.Algebra.DEFAULT.mult(matrix, CB.matrix);
-			// TODO: add bias
-			return this;
+			ColtMatrix CB = (ColtMatrix) B;
+			ColtMatrix newMatrix = new ColtMatrix(0, 0);
+			newMatrix.matrix = DoubleFactory2D.dense.repeat(((ColtMatrix) C).matrix, rows(), 1);
+			matrix.zMult(CB.matrix, newMatrix.matrix, 1.0, 1.0, false, false);
+			return newMatrix;
+		}
+
+		@Override
+		public MLMatrix transposedMultiplyAndScale( final MLMatrix B, final double scalar ) {
+			ColtMatrix CB = (ColtMatrix) B;
+			ColtMatrix newMatrix = new ColtMatrix(0, 0);
+			newMatrix.matrix = matrix.zMult(CB.matrix, null, scalar, 0.0, true, false);
+			return newMatrix;
 		}
 
 		@Override
 		public MLMatrix add( MLMatrix B ) {
-			ColtMatrix CB = (ColtMatrix)B;
-			matrix.assign(CB.matrix, (d1,d2) -> d1+d2);
+			ColtMatrix CB = (ColtMatrix) B;
+			ColtMatrix newMatrix = new ColtMatrix(this);
+			newMatrix.matrix.assign(CB.matrix, ( d1, d2 ) -> d1 + d2);
+			return newMatrix;
+		}
+
+		@Override
+		public MLMatrix addInPlace( MLMatrix B ) {
+			ColtMatrix CB = (ColtMatrix) B;
+			matrix.assign(CB.matrix, ( d1, d2 ) -> d1 + d2);
 			return this;
 		}
 
 		@Override
 		public MLMatrix sub( MLMatrix B ) {
-			ColtMatrix CB = (ColtMatrix)B;
-			matrix.assign(CB.matrix, (d1,d2) -> d1-d2);
-			return this;
-		}
-
-		@Override
-		public MLMatrix scale( double scalar ) {
-			this.matrix.assign((d) -> d*scalar);
-			return this;
-		}
-
-		@Override
-		public MLMatrix scale( MLMatrix S ) {
-			this.matrix.forEachNonZero((r, c, d) -> d * S.get(r, c));
-			return this;
-		}
-
-		@Override
-		public MLMatrix duplicate() {
-			ColtMatrix newMatrix = new ColtMatrix(matrix.rows(), matrix.columns());
-			newMatrix.matrix.assign(this.matrix);
+			ColtMatrix CB = (ColtMatrix) B;
+			ColtMatrix newMatrix = new ColtMatrix(this);
+			newMatrix.matrix.assign(CB.matrix, ( d1, d2 ) -> d1 - d2);
 			return newMatrix;
 		}
 
+		@Override
+		public MLMatrix colSums() {
+			double[][] sums = new double[1][matrix.columns()];
+			for( int c = 0; c < matrix.columns(); c++ ) {
+				for( int r = 0; r < matrix.rows(); r++ ) {
+					sums[0][c] += matrix.getQuick(r, c);
+				}
+			}
+			return new ColtMatrix(sums);
+		}
+
+		@Override
+		public MLMatrix scaleInPlace( double scalar ) {
+			this.matrix.assign(( d ) -> d * scalar);
+			return this;
+		}
+
+		@Override
+		public MLMatrix scaleInPlace( MLMatrix S ) {
+			this.matrix.forEachNonZero(( r, c, d ) -> d * S.get(r, c));
+			return this;
+		}
+
+		@Override
+		public MLMatrix apply( DoubleUnaryOperator op ) {
+			ColtMatrix newMatrix = new ColtMatrix(matrix.rows(), matrix.columns());
+			newMatrix.matrix.assign(matrix);
+			newMatrix.matrix.assign(( d ) -> op.applyAsDouble(d));
+			return newMatrix;
+		}
+
+		@Override
+		public MLMatrix applyInPlace( DoubleUnaryOperator op ) {
+			this.matrix.assign(( d ) -> op.applyAsDouble(d));
+			return this;
+		}
+
 		@Override
 		public String toString() {
 			return matrix.toString();

src/main/java/schule/ngb/zm/ml/NeuronLayer.java

@@ -41,10 +41,10 @@ public class NeuronLayer implements Function<MLMatrix, MLMatrix> {
 	public NeuronLayer( int neurons, int inputs ) {
 		weights = MatrixFactory
 			.create(inputs, neurons)
-			.initializeRandom(-1, 1);
+			.initializeRandom();
 
 		biases = MatrixFactory
-			.create(neurons, 1)
+			.create(1, neurons)
 			.initializeZero();
 
 		activationFunction = MLMath::sigmoid;
@@ -110,10 +110,6 @@ public class NeuronLayer implements Function<MLMatrix, MLMatrix> {
 		weights = newWeights.duplicate();
 	}
 
-	public void adjustWeights( MLMatrix adjustment ) {
-		weights.add(adjustment);
-	}
-
 	@Override
 	public String toString() {
 		return weights.toString() + "\n" + biases.toString();
@@ -121,10 +117,10 @@ public class NeuronLayer implements Function<MLMatrix, MLMatrix> {
 
 	@Override
 	public MLMatrix apply( MLMatrix inputs ) {
-		lastInput = inputs;
+		lastInput = inputs.duplicate();
 		lastOutput = inputs
 			.multiplyAddBias(weights, biases)
-			.apply(activationFunction);
+			.applyInPlace(activationFunction);
 
 		if( next != null ) {
 			return next.apply(lastOutput);
@@ -144,14 +140,16 @@ public class NeuronLayer implements Function<MLMatrix, MLMatrix> {
 	}
 
 	public void backprop( MLMatrix expected, double learningRate ) {
-		MLMatrix error, delta, adjustment;
+		MLMatrix error, adjustment;
 		if( next == null ) {
-			error = expected.duplicate().sub(lastOutput);
+			error = expected.sub(lastOutput);
 		} else {
-			error = expected.duplicate().multiply(next.weights.transpose());
+			error = expected.multiplyTransposed(next.weights);
 		}
 
-		error.scale(lastOutput.duplicate().apply(this.activationFunctionDerivative));
+		error.scaleInPlace(
+			lastOutput.apply(this.activationFunctionDerivative)
+		);
 		// Hier schon leraningRate anwenden?
 		// See https://towardsdatascience.com/understanding-and-implementing-neural-networks-in-java-from-scratch-61421bb6352c
 		//delta = MLMath.matrixApply(delta, ( x ) -> learningRate * x);
@@ -159,10 +157,14 @@ public class NeuronLayer implements Function<MLMatrix, MLMatrix> {
 			previous.backprop(error, learningRate);
 		}
 
-		// biases = MLMath.biasAdjust(biases, MLMath.matrixApply(delta, ( x ) -> learningRate * x));
+		biases.addInPlace(
+			error.colSums().scaleInPlace(
+				-learningRate / (double) error.rows()
+			)
+		);
 
-		adjustment = lastInput.duplicate().transpose().multiply(error).apply((d) -> learningRate*d);
-		this.adjustWeights(adjustment);
+		adjustment = lastInput.transposedMultiplyAndScale(error, learningRate);
+		weights.addInPlace(adjustment);
 	}
 
 }