From 6a0ea165390f881469a0565fa77beae4d0fb7920 Mon Sep 17 00:00:00 2001 From: ishii-norimi Date: Fri, 10 Nov 2023 19:56:11 +0900 Subject: [PATCH] Use common layer builder for identity-like layer --- lib/model/nns/layer/base.js | 28 ++++++++++------------ lib/model/nns/layer/detach.js | 18 -------------- lib/model/nns/layer/identity.js | 16 ------------- lib/model/nns/layer/index.js | 2 -- tests/lib/model/nns/layer/detach.test.js | 18 +++++++------- tests/lib/model/nns/layer/identity.test.js | 18 +++++++------- 6 files changed, 30 insertions(+), 70 deletions(-) delete mode 100644 lib/model/nns/layer/detach.js delete mode 100644 lib/model/nns/layer/identity.js diff --git a/lib/model/nns/layer/base.js b/lib/model/nns/layer/base.js index b2482cf08..d2e4d43da 100644 --- a/lib/model/nns/layer/base.js +++ b/lib/model/nns/layer/base.js @@ -143,12 +143,18 @@ const buildUnaryLayer = (name, calcFunc, gradFunc) => { class TempLayer extends Layer { calc(x) { this._i = x + if (!calcFunc) { + return (this._o = x) + } this._o = x.copy() this._o.map(calcFunc) return this._o } grad(bo) { + if (!gradFunc) { + return bo + } const bi = bo.copy() for (let i = 0; i < bi.length; i++) { bi.value[i] *= gradFunc(this._i.value[i], this._o.value[i]) @@ -199,10 +205,7 @@ const unaryLayers = { calc: v => ~v, grad: () => 0, }, - ceil: { - calc: Math.ceil, - grad: () => 1, - }, + ceil: { calc: Math.ceil }, cloglog: { calc: v => 1 - Math.exp(-Math.exp(v)), grad: (i, o) => Math.exp(i) * (1 - o), @@ -219,6 +222,7 @@ const unaryLayers = { calc: Math.cosh, grad: i => Math.sinh(i), }, + detach: { grad: () => 0 }, elish: { calc: v => (v >= 0 ? v : Math.exp(v) - 1) / (1 + Math.exp(-v)), grad: (i, o) => ((i >= 0 ? 1 : Math.exp(i)) + o * Math.exp(-i)) / (1 + Math.exp(-i)), @@ -247,10 +251,7 @@ const unaryLayers = { calc: Math.exp, grad: (i, o) => o, }, - floor: { - calc: Math.floor, - grad: () => 1, - }, + floor: { calc: Math.floor }, gelu: { calc: v => { const erf = v => { @@ -278,6 +279,7 @@ const unaryLayers = { calc: v => (v <= -3 ? 0 : 3 <= v ? v : (v * (v + 3)) / 6), grad: i => (i <= -3 ? 0 : 3 <= i ? 1 : (2 * i + 3) / 6), }, + identity: {}, is_inf: { calc: v => v === Infinity || v === -Infinity, grad: () => 0, @@ -336,14 +338,8 @@ const unaryLayers = { calc: v => v / (1 + Math.sqrt(1 + v ** 2)), grad: i => 1 / (Math.sqrt(i ** 2 + 1) + i ** 2 + 1), }, - round: { - calc: Math.round, - grad: () => 1, - }, - sign: { - calc: Math.sign, - grad: () => 1, - }, + round: { calc: Math.round }, + sign: { calc: Math.sign }, silu: { calc: v => v / (1 + Math.exp(-v)), grad: (i, o) => 1 / (1 + Math.exp(-i)) + o * (1 - o / i), diff --git a/lib/model/nns/layer/detach.js b/lib/model/nns/layer/detach.js deleted file mode 100644 index 9acde693e..000000000 --- a/lib/model/nns/layer/detach.js +++ /dev/null @@ -1,18 +0,0 @@ -import Layer from './base.js' - -/** - * Detach layer - */ -export default class DetachLayer extends Layer { - calc(x) { - return x - } - - grad(bo) { - const bi = bo.copy() - bi.map(() => 0) - return bi - } -} - -DetachLayer.registLayer() diff --git a/lib/model/nns/layer/identity.js b/lib/model/nns/layer/identity.js deleted file mode 100644 index 8d1cf7ec8..000000000 --- a/lib/model/nns/layer/identity.js +++ /dev/null @@ -1,16 +0,0 @@ -import Layer from './base.js' - -/** - * Identity layer - */ -export default class IdentityLayer extends Layer { - calc(x) { - return x - } - - grad(bo) { - return bo - } -} - -IdentityLayer.registLayer() diff --git a/lib/model/nns/layer/index.js b/lib/model/nns/layer/index.js index 69655d981..5c5505803 100644 --- a/lib/model/nns/layer/index.js +++ b/lib/model/nns/layer/index.js @@ -17,7 +17,6 @@ export { default as CondLayer } from './cond.js' export { default as ConstLayer } from './const.js' export { default as ConvLayer } from './conv.js' export { default as ConcatenatedReLULayer } from './crelu.js' -export { default as DetachLayer } from './detach.js' export { default as DropoutLayer } from './dropout.js' export { default as ElasticELULayer } from './eelu.js' export { default as ELULayer } from './elu.js' @@ -38,7 +37,6 @@ export { default as HardSigmoidLayer } from './hard_sigmoid.js' export { default as HardTanhLayer } from './hard_tanh.js' export { default as HexpoLayer } from './hexpo.js' export { default as HuberLayer } from './huber.js' -export { default as IdentityLayer } from './identity.js' export { default as IncludeLayer } from './include.js' export { default as InputLayer } from './input.js' export { default as ImprovedSigmoidLayer } from './isigmoid.js' diff --git a/tests/lib/model/nns/layer/detach.test.js b/tests/lib/model/nns/layer/detach.test.js index 8ad839e10..3c897a34a 100644 --- a/tests/lib/model/nns/layer/detach.test.js +++ b/tests/lib/model/nns/layer/detach.test.js @@ -2,17 +2,17 @@ import NeuralNetwork from '../../../../../lib/model/neuralnetwork.js' import Matrix from '../../../../../lib/util/matrix.js' import Tensor from '../../../../../lib/util/tensor.js' -import DetachLayer from '../../../../../lib/model/nns/layer/detach.js' +import Layer from '../../../../../lib/model/nns/layer/base.js' describe('layer', () => { test('construct', () => { - const layer = new DetachLayer({}) + const layer = Layer.fromObject({ type: 'detach' }) expect(layer).toBeDefined() }) describe('calc', () => { test('matrix', () => { - const layer = new DetachLayer({}) + const layer = Layer.fromObject({ type: 'detach' }) const x = Matrix.randn(100, 10) const y = layer.calc(x) @@ -24,7 +24,7 @@ describe('layer', () => { }) test('tensor', () => { - const layer = new DetachLayer({}) + const layer = Layer.fromObject({ type: 'detach' }) const x = Tensor.randn([15, 10, 7]) const y = layer.calc(x) @@ -40,7 +40,7 @@ describe('layer', () => { describe('grad', () => { test('matrix', () => { - const layer = new DetachLayer({}) + const layer = Layer.fromObject({ type: 'detach' }) const x = Matrix.randn(100, 10) layer.calc(x) @@ -55,7 +55,7 @@ describe('layer', () => { }) test('tensor', () => { - const layer = new DetachLayer({}) + const layer = Layer.fromObject({ type: 'detach' }) const x = Tensor.randn([15, 10, 7]) layer.calc(x) @@ -73,15 +73,15 @@ describe('layer', () => { }) test('toObject', () => { - const layer = new DetachLayer({}) + const layer = Layer.fromObject({ type: 'detach' }) const obj = layer.toObject() expect(obj).toEqual({ type: 'detach' }) }) test('fromObject', () => { - const layer = DetachLayer.fromObject({ type: 'detach' }) - expect(layer).toBeInstanceOf(DetachLayer) + const layer = Layer.fromObject({ type: 'detach' }) + expect(layer).toBeDefined() }) }) diff --git a/tests/lib/model/nns/layer/identity.test.js b/tests/lib/model/nns/layer/identity.test.js index 3972e8682..1393f349b 100644 --- a/tests/lib/model/nns/layer/identity.test.js +++ b/tests/lib/model/nns/layer/identity.test.js @@ -2,17 +2,17 @@ import NeuralNetwork from '../../../../../lib/model/neuralnetwork.js' import Matrix from '../../../../../lib/util/matrix.js' import Tensor from '../../../../../lib/util/tensor.js' -import IdentityLayer from '../../../../../lib/model/nns/layer/identity.js' +import Layer from '../../../../../lib/model/nns/layer/base.js' describe('layer', () => { test('construct', () => { - const layer = new IdentityLayer({}) + const layer = Layer.fromObject({ type: 'identity' }) expect(layer).toBeDefined() }) describe('calc', () => { test('matrix', () => { - const layer = new IdentityLayer({}) + const layer = Layer.fromObject({ type: 'identity' }) const x = Matrix.randn(100, 10) const y = layer.calc(x) @@ -24,7 +24,7 @@ describe('layer', () => { }) test('tensor', () => { - const layer = new IdentityLayer({}) + const layer = Layer.fromObject({ type: 'identity' }) const x = Tensor.randn([15, 10, 7]) const y = layer.calc(x) @@ -40,7 +40,7 @@ describe('layer', () => { describe('grad', () => { test('matrix', () => { - const layer = new IdentityLayer({}) + const layer = Layer.fromObject({ type: 'identity' }) const x = Matrix.randn(100, 10) layer.calc(x) @@ -55,7 +55,7 @@ describe('layer', () => { }) test('tensor', () => { - const layer = new IdentityLayer({}) + const layer = Layer.fromObject({ type: 'identity' }) const x = Tensor.randn([15, 10, 7]) layer.calc(x) @@ -73,15 +73,15 @@ describe('layer', () => { }) test('toObject', () => { - const layer = new IdentityLayer({}) + const layer = Layer.fromObject({ type: 'identity' }) const obj = layer.toObject() expect(obj).toEqual({ type: 'identity' }) }) test('fromObject', () => { - const layer = IdentityLayer.fromObject({ type: 'identity' }) - expect(layer).toBeInstanceOf(IdentityLayer) + const layer = Layer.fromObject({ type: 'identity' }) + expect(layer).toBeDefined() }) })