Fix ELU gradient (#774)

Author: ishii-norimi

lib/model/nns/layer/elu.js

@@ -14,14 +14,15 @@ export default class ELULayer extends Layer {
 	}
 
 	calc(x) {
-		this._o = x.copy()
-		this._o.map(v => (v > 0 ? v : this._a * (Math.exp(v) - 1)))
-		return this._o
+		this._i = x
+		const o = x.copy()
+		o.map(v => (v > 0 ? v : this._a * (Math.exp(v) - 1)))
+		return o
 	}
 
 	grad(bo) {
 		const bi = bo.copy()
-		bi.broadcastOperate(this._o, (a, b) => a * (b > 0 ? 1 : this._a * Math.exp(b)))
+		bi.broadcastOperate(this._i, (a, b) => a * (b > 0 ? 1 : this._a * Math.exp(b)))
 		return bi
 	}
 

tests/lib/model/nns/layer/elu.test.js

@@ -41,17 +41,35 @@ describe('layer', () => {
 	})
 
 	describe('grad', () => {
+		test('numeric calculation', () => {
+			const layer = new ELULayer({})
+			const e = 1.0e-8
+
+			const x = Matrix.fromArray([Array.from({ length: 100 }, (_, i) => (i - 50) / 50)])
+			const y = layer.calc(x)
+
+			const bo = Matrix.ones(1, 100)
+			const bi = layer.grad(bo)
+
+			const d = layer.calc(Matrix.add(x, e))
+			for (let i = 0; i < x.rows; i++) {
+				for (let j = 0; j < x.cols; j++) {
+					expect(bi.at(i, j)).toBeCloseTo((d.at(i, j) - y.at(i, j)) / e)
+				}
+			}
+		})
+
 		test('matrix', () => {
 			const layer = new ELULayer({})
 
 			const x = Matrix.randn(100, 10)
-			const y = layer.calc(x)
+			layer.calc(x)
 
 			const bo = Matrix.ones(100, 10)
 			const bi = layer.grad(bo)
 			for (let i = 0; i < x.rows; i++) {
 				for (let j = 0; j < x.cols; j++) {
-					expect(bi.at(i, j)).toBeCloseTo(x.at(i, j) > 0 ? 1 : Math.exp(y.at(i, j)))
+					expect(bi.at(i, j)).toBeCloseTo(x.at(i, j) > 0 ? 1 : Math.exp(x.at(i, j)))
 				}
 			}
 		})
@@ -60,14 +78,14 @@ describe('layer', () => {
 			const layer = new ELULayer({})
 
 			const x = Tensor.randn([15, 10, 7])
-			const y = layer.calc(x)
+			layer.calc(x)
 
 			const bo = Tensor.ones([15, 10, 7])
 			const bi = layer.grad(bo)
 			for (let i = 0; i < x.sizes[0]; i++) {
 				for (let j = 0; j < x.sizes[1]; j++) {
 					for (let k = 0; k < x.sizes[2]; k++) {
-						expect(bi.at(i, j, k)).toBeCloseTo(x.at(i, j, k) > 0 ? 1 : Math.exp(y.at(i, j, k)))
+						expect(bi.at(i, j, k)).toBeCloseTo(x.at(i, j, k) > 0 ? 1 : Math.exp(x.at(i, j, k)))
 					}
 				}
 			}