From 0bd0c0b017dc4c857de3926eae62f0811b0341be Mon Sep 17 00:00:00 2001 From: ishii-norimi Date: Thu, 11 Jan 2024 19:48:54 +0900 Subject: [PATCH] Simplify MLP implementation --- js/view/mlp.js | 70 +++-------- js/view/worker/mlp_worker.js | 31 ----- lib/model/mlp.js | 232 ++++++++++++++++++++++++++++++----- lib/model/ranknet.js | 3 +- tests/gui/view/mlp.test.js | 4 +- tests/lib/model/mlp.test.js | 163 +++++++++++++++++++----- 6 files changed, 350 insertions(+), 153 deletions(-) delete mode 100644 js/view/worker/mlp_worker.js diff --git a/js/view/mlp.js b/js/view/mlp.js index c70f39ff6..b3970f5d4 100644 --- a/js/view/mlp.js +++ b/js/view/mlp.js @@ -1,32 +1,13 @@ import Matrix from '../../lib/util/matrix.js' import Controller from '../controller.js' -import { BaseWorker } from '../utils.js' - -class MLPWorker extends BaseWorker { - constructor() { - super('js/view/worker/mlp_worker.js', { type: 'module' }) - } - - initialize(type, hidden_sizes, activation, optimizer) { - return this._postMessage({ mode: 'init', type, hidden_sizes, activation, optimizer }) - } - - fit(train_x, train_y, iteration, rate, batch) { - return this._postMessage({ mode: 'fit', x: train_x, y: train_y, iteration, rate, batch }) - } - - predict(x) { - return this._postMessage({ mode: 'predict', x: x }) - } -} +import { MLPClassifier, MLPRegressor } from '../../lib/model/mlp.js' export default function (platform) { platform.setting.ml.usage = 'Click and add data point. Next, click "Initialize". Finally, click "Fit" button repeatedly.' const controller = new Controller(platform) const mode = platform.task - const model = new MLPWorker() - let epoch = 0 + let model = null const fitModel = async cb => { const dim = getInputDim() @@ -44,9 +25,8 @@ export default function (platform) { if (mode === 'CF') { ty = ty.map(v => v[0]) } - const e = await model.fit(tx, ty, +iteration.value, rate.value, batch.value) - epoch = e.data.epoch - platform.plotLoss(e.data.loss) + const loss = model.fit(tx, ty, +iteration.value, rate.value, batch.value) + platform.plotLoss(loss) if (mode === 'TP') { let lx = x.slice(x.rows - dim).value const p = [] @@ -57,14 +37,13 @@ export default function (platform) { cb && cb() return } - const e = await model.predict([lx]) - p.push(e.data[0]) + const data = model.predict([lx]) + p.push(data[0]) lx = lx.slice(x.cols) - lx.push(...e.data[0]) + lx.push(...data[0]) } } else { - const e = await model.predict(platform.testInput(dim === 1 ? 2 : 4)) - const data = e.data + const data = model.predict(platform.testInput(dim === 1 ? 2 : 4)) platform.testResult(data) cb && cb() @@ -90,42 +69,25 @@ export default function (platform) { }) const activation = controller.select({ label: ' Activation ', - values: [ - 'sigmoid', - 'tanh', - 'relu', - 'elu', - 'leaky_relu', - 'rrelu', - 'prelu', - 'gaussian', - 'softplus', - 'softsign', - 'identity', - ], + values: ['sigmoid', 'tanh', 'relu', 'elu', 'leaky_relu', 'gaussian', 'softplus', 'softsign', 'identity'], }) - const optimizer = controller.select({ label: ' Optimizer ', values: ['sgd', 'adam', 'momentum', 'rmsprop'] }) - const slbConf = controller.stepLoopButtons().init(done => { + const slbConf = controller.stepLoopButtons().init(() => { if (platform.datas.length === 0) { - done() return } - model - .initialize( - mode === 'CF' ? 'classifier' : 'regressor', - hidden_sizes.value, - activation.value, - optimizer.value - ) - .then(done) + if (mode === 'CF') { + model = new MLPClassifier(hidden_sizes.value, activation.value) + } else { + model = new MLPRegressor(hidden_sizes.value, activation.value) + } platform.init() }) const iteration = controller.select({ label: ' Iteration ', values: [1, 10, 100, 1000, 10000] }) const rate = controller.input.number({ label: ' Learning rate ', min: 0, max: 100, step: 0.01, value: 0.001 }) const batch = controller.input.number({ label: ' Batch size ', min: 1, max: 100, value: 10 }) - slbConf.step(fitModel).epoch(() => epoch) + slbConf.step(fitModel).epoch(() => model.epoch) let predCount if (mode === 'TP') { predCount = controller.input.number({ label: ' predict count', min: 1, max: 1000, value: 100 }) diff --git a/js/view/worker/mlp_worker.js b/js/view/worker/mlp_worker.js deleted file mode 100644 index 3dc3fd4d3..000000000 --- a/js/view/worker/mlp_worker.js +++ /dev/null @@ -1,31 +0,0 @@ -import { MLPClassifier, MLPRegressor } from '../../../lib/model/mlp.js' - -self.model = null - -self.addEventListener( - 'message', - function (e) { - const data = e.data - if (data.mode === 'init') { - if (data.type === 'classifier') { - self.model = new MLPClassifier(data.hidden_sizes, data.activation, data.optimizer) - } else { - self.model = new MLPRegressor(data.hidden_sizes, data.activation, data.optimizer) - } - self.postMessage(null) - } else if (data.mode === 'fit') { - const samples = data.x.length - if (samples === 0) { - self.postMessage(null) - return - } - - const loss = self.model.fit(data.x, data.y, data.iteration, data.rate, data.batch) - self.postMessage({ epoch: self.model.epoch, loss }) - } else if (data.mode === 'predict') { - const pred = self.model.predict(data.x) - self.postMessage(pred) - } - }, - false -) diff --git a/lib/model/mlp.js b/lib/model/mlp.js index ea4651919..a42a0e09f 100644 --- a/lib/model/mlp.js +++ b/lib/model/mlp.js @@ -1,4 +1,85 @@ -import NeuralNetwork from './neuralnetwork.js' +import Matrix from '../util/matrix.js' +import { AdamOptimizer } from './nns/optimizer.js' + +/** + * @ignore + * @typedef {import("./nns/graph").LayerObject} LayerObject + */ + +const ActivationFunctions = { + identity: { calc: i => i, grad: () => 1 }, + elu: { calc: i => (i > 0 ? i : Math.exp(i) - 1), grad: i => (i > 0 ? 1 : Math.exp(i)) }, + gaussian: { calc: i => Math.exp(-(i ** 2) / 2), grad: (i, o) => -o * i }, + leaky_relu: { calc: i => (i > 0 ? i : 0.01 * i), grad: i => (i > 0 ? 1 : 0.01) }, + relu: { calc: i => Math.max(0, i), grad: i => (i > 0 ? 1 : 0) }, + sigmoid: { calc: i => 1 / (1 + Math.exp(-i)), grad: (i, o) => o * (1 - o) }, + softplus: { calc: i => Math.log(1 + Math.exp(i)), grad: i => 1 / (1 + Math.exp(-i)) }, + softsign: { calc: v => v / (1 + Math.abs(v)), grad: i => 1 / (1 + Math.abs(i)) ** 2 }, + tanh: { calc: Math.tanh, grad: (i, o) => 1 - o ** 2 }, +} + +class MLP { + constructor(layer_sizes, activations) { + this._layer_sizes = layer_sizes + this._activations = activations + this._a = [] + + this._w = [] + this._b = [] + for (let i = 0; i < layer_sizes.length - 1; i++) { + this._a[i] = ActivationFunctions[activations[i]] + this._w[i] = Matrix.randn(layer_sizes[i], layer_sizes[i + 1], 0, 0.1) + this._b[i] = Matrix.zeros(1, layer_sizes[i + 1]) + } + this._optimizer = new AdamOptimizer() + this._optimizer_mng = this._optimizer.manager() + } + + calc(x) { + this._i = [x] + this._o = [x] + for (let i = 0; i < this._w.length; i++) { + this._i[i + 1] = x = x.dot(this._w[i]) + x.add(this._b[i]) + this._o[i + 1] = x = x.copy() + if (this._a[i]) { + x.map(this._a[i].calc) + } + } + return x + } + + update(e, r) { + this._optimizer.learningRate = r + for (let i = this._w.length - 1; i >= 0; i--) { + if (this._a[i]) { + for (let k = 0; k < e.length; k++) { + e.value[k] *= this._a[i].grad(this._i[i + 1].value[k], this._o[i + 1].value[k]) + } + } + const dw = this._o[i].tDot(e) + dw.div(this._i[i].rows) + const db = e.mean(0) + e = e.dot(this._w[i].t) + this._w[i].sub(this._optimizer_mng.delta(`w${i}`, dw)) + this._b[i].sub(this._optimizer_mng.delta(`b${i}`, db)) + } + } + + toObject() { + const layers = [{ type: 'input' }] + for (let i = 0; i < this._layer_sizes.length - 1; i++) { + layers.push({ + type: 'full', + out_size: this._layer_sizes[i + 1], + activation: this._activations[i], + w: this._w[i]?.toArray(), + b: this._b[i]?.toArray(), + }) + } + return layers + } +} /** * Multi layer perceptron classifier @@ -6,21 +87,26 @@ import NeuralNetwork from './neuralnetwork.js' export class MLPClassifier { /** * @param {number[]} hidden_sizes Sizes of hidden layers - * @param {string} activation Activation name - * @param {string} optimizer Optimizer of the network + * @param {'identity', 'elu', 'gaussian', 'leaky_relu', 'relu', 'sigmoid', 'softplus', 'softsign', 'tanh'} [activation=tanh] Activation name */ - constructor(hidden_sizes, activation, optimizer) { - this._layers = [{ type: 'input' }] - for (let i = 0; i < hidden_sizes.length; i++) { - this._layers.push({ type: 'full', out_size: hidden_sizes[i], activation: activation }) - } + constructor(hidden_sizes, activation = 'tanh') { + this._hidden_sizes = hidden_sizes + this._activations = Array(this._hidden_sizes.length).fill(activation) this._model = null this._classes = null - this._optimizer = optimizer this._epoch = 0 } + /** + * Category list + * + * @type {*[]} + */ + get categories() { + return this._classes + } + /** * Epoch * @@ -30,30 +116,80 @@ export class MLPClassifier { return this._epoch } + /** + * Returns object representation. + * + * @returns {LayerObject[]} Object represented this neuralnetwork + */ + toObject() { + return [...this._model.toObject(), { type: 'softmax' }, { type: 'output' }] + } + /** * Fit model. * * @param {Array>} train_x Training data * @param {*[]} train_y Target values * @param {number} iteration Iteration count - * @param {number} rate Learning rate - * @param {number} batch Batch size + * @param {number} [rate] Learning rate + * @param {number} [batch] Batch size * @returns {number} Loss value */ - fit(train_x, train_y, iteration, rate, batch) { + fit(train_x, train_y, iteration, rate = 0.001, batch = 0) { if (!this._model) { this._classes = [...new Set(train_y)] - this._layers.push({ type: 'full', out_size: this._classes.length }) - this._model = NeuralNetwork.fromObject(this._layers, 'mse', this._optimizer) + const layer_sizes = [train_x[0].length, ...this._hidden_sizes, this._classes.length] + this._model = new MLP(layer_sizes, this._activations) } const y = train_y.map(v => { const yi = Array(this._classes.length).fill(0) yi[this._classes.indexOf(v)] = 1 return yi }) - const loss = this._model.fit(train_x, y, iteration, rate, batch) + const xs = [] + const ys = [] + if (batch > 0) { + for (let k = 0; k < train_x.length; k += batch) { + xs.push(Matrix.fromArray(train_x.slice(k, k + batch))) + ys.push(Matrix.fromArray(y.slice(k, k + batch))) + } + } else { + xs.push(Matrix.fromArray(train_x)) + ys.push(Matrix.fromArray(y)) + } + let e + for (let i = 0; i < iteration; i++) { + for (let k = 0; k < xs.length; k++) { + e = this._fitonce(xs[k], ys[k], rate) + } + } this._epoch += iteration - return loss[0] + e.map(v => v ** 2) + return e.mean() + } + + _fitonce(x, y, r) { + const p = this._model.calc(x) + p.sub(p.max(1)) + p.map(Math.exp) + p.div(p.sum(1)) + const e = Matrix.sub(p, y) + this._model.update(e, r) + return e + } + + /** + * Returns predicted probabilities. + * + * @param {Array>} x Sample data + * @returns {Array>} Predicted values + */ + probability(x) { + const p = this._model.calc(Matrix.fromArray(x)) + p.sub(p.max(1)) + p.map(Math.exp) + p.div(p.sum(1)) + return p.toArray() } /** @@ -64,7 +200,7 @@ export class MLPClassifier { */ predict(x) { return this._model - .calc(x) + .calc(Matrix.fromArray(x)) .argmax(1) .value.map(v => this._classes[v]) } @@ -76,17 +212,12 @@ export class MLPClassifier { export class MLPRegressor { /** * @param {number[]} hidden_sizes Sizes of hidden layers - * @param {string} activation Activation name - * @param {string} optimizer Optimizer of the network + * @param {'identity', 'elu', 'gaussian', 'leaky_relu', 'relu', 'sigmoid', 'softplus', 'softsign', 'tanh'} [activation=tanh] Activation name */ - constructor(hidden_sizes, activation, optimizer) { - this._layers = [{ type: 'input' }] - for (let i = 0; i < hidden_sizes.length; i++) { - this._layers.push({ type: 'full', out_size: hidden_sizes[i], activation: activation }) - } - + constructor(hidden_sizes, activation = 'tanh') { + this._hidden_sizes = hidden_sizes + this._activations = Array(hidden_sizes.length).fill(activation) this._model = null - this._optimizer = optimizer this._epoch = 0 } @@ -99,24 +230,58 @@ export class MLPRegressor { return this._epoch } + /** + * Returns object representation. + * + * @returns {LayerObject[]} Object represented this neuralnetwork + */ + toObject() { + return [...this._model.toObject(), { type: 'output' }] + } + /** * Fit model. * * @param {Array>} train_x Training data * @param {Array>} train_y Target values * @param {number} iteration Iteration count - * @param {number} rate Learning rate - * @param {number} batch Batch size + * @param {number} [rate] Learning rate + * @param {number} [batch] Batch size * @returns {number} Loss value */ - fit(train_x, train_y, iteration, rate, batch) { + fit(train_x, train_y, iteration, rate = 0.001, batch = 0) { if (!this._model) { - this._layers.push({ type: 'full', out_size: train_y[0].length }) - this._model = NeuralNetwork.fromObject(this._layers, 'mse', this._optimizer) + const layer_sizes = [train_x[0].length, ...this._hidden_sizes, train_y[0].length] + this._model = new MLP(layer_sizes, this._activations) + } + const xs = [] + const ys = [] + if (batch > 0) { + for (let k = 0; k < train_x.length; k += batch) { + xs.push(Matrix.fromArray(train_x.slice(k, k + batch))) + ys.push(Matrix.fromArray(train_y.slice(k, k + batch))) + } + } else { + xs.push(Matrix.fromArray(train_x)) + ys.push(Matrix.fromArray(train_y)) + } + let e + for (let i = 0; i < iteration; i++) { + for (let k = 0; k < xs.length; k++) { + e = this._fitonce(xs[k], ys[k], rate) + } } - const loss = this._model.fit(train_x, train_y, iteration, rate, batch) this._epoch += iteration - return loss[0] + e.map(v => v ** 2) + return e.mean() + } + + _fitonce(x, y, r) { + const p = this._model.calc(x) + const e = Matrix.sub(p, y) + e.div(2) + this._model.update(e, r) + return e } /** @@ -126,6 +291,7 @@ export class MLPRegressor { * @returns {Array>} Predicted values */ predict(x) { + x = Matrix.fromArray(x) const pred = this._model.calc(x) return pred.toArray() } diff --git a/lib/model/ranknet.js b/lib/model/ranknet.js index c97575c04..d9c707d19 100644 --- a/lib/model/ranknet.js +++ b/lib/model/ranknet.js @@ -108,10 +108,9 @@ export default class RankNet { } const dw = outs1[i].tDot(e1) dw.add(outs2[i].tDot(e2)) - dw.mult(this._rate / n) + dw.div(n) const db = e1.mean(0) db.add(e2.mean(0)) - db.mult(this._rate) e1 = e1.dot(this._w[i].t) e2 = e2.dot(this._w[i].t) this._w[i].sub(this._optimizer.delta(`w${i}`, dw)) diff --git a/tests/gui/view/mlp.test.js b/tests/gui/view/mlp.test.js index 875da03a8..e49ed1e93 100644 --- a/tests/gui/view/mlp.test.js +++ b/tests/gui/view/mlp.test.js @@ -23,9 +23,7 @@ describe('classification', () => { await expect((await size.getProperty('value')).jsonValue()).resolves.toBe('10') const activation = await buttons.waitForSelector('select:nth-of-type(1)') await expect((await activation.getProperty('value')).jsonValue()).resolves.toBe('sigmoid') - const optimizer = await buttons.waitForSelector('select:nth-of-type(2)') - await expect((await optimizer.getProperty('value')).jsonValue()).resolves.toBe('sgd') - const iteration = await buttons.waitForSelector('select:nth-of-type(3)') + const iteration = await buttons.waitForSelector('select:nth-of-type(2)') await expect((await iteration.getProperty('value')).jsonValue()).resolves.toBe('1') const rate = await buttons.waitForSelector('input:nth-of-type(3)') await expect((await rate.getProperty('value')).jsonValue()).resolves.toBe('0.001') diff --git a/tests/lib/model/mlp.test.js b/tests/lib/model/mlp.test.js index cdc604ffd..e8e4be9c6 100644 --- a/tests/lib/model/mlp.test.js +++ b/tests/lib/model/mlp.test.js @@ -1,39 +1,142 @@ +import { jest } from '@jest/globals' +jest.retryTimes(5) + import Matrix from '../../../lib/util/matrix.js' import { MLPClassifier, MLPRegressor } from '../../../lib/model/mlp.js' +import NeuralNetwork from '../../../lib/model/neuralnetwork.js' import { accuracy } from '../../../lib/evaluate/classification.js' import { rmse } from '../../../lib/evaluate/regression.js' -test('regression', () => { - const model = new MLPRegressor([10, 10], 'tanh', 'adam') - const x = Matrix.randn(50, 2, 0, 5).toArray() - const t = [] - for (let i = 0; i < x.length; i++) { - t[i] = [x[i][0] + x[i][1] + (Math.random() - 0.5) / 10] - } - for (let i = 0; i < 1000; i++) { - model.fit(x, t, 1, 0.01, 10) - expect(model.epoch).toBe(i + 1) - } - const y = model.predict(x) - const err = rmse(y, t)[0] - expect(err).toBeLessThan(0.5) +describe('regression', () => { + test.each([ + undefined, + 'elu', + 'gaussian', + 'leaky_relu', + 'relu', + 'sigmoid', + 'softplus', + 'softsign', + 'tanh', + 'identity', + ])('%s', activation => { + const model = new MLPRegressor([5], activation) + const x = Matrix.randn(30, 2, 0, 5).toArray() + const t = [] + for (let i = 0; i < x.length; i++) { + t[i] = [x[i][0] + x[i][1] + (Math.random() - 0.5) / 10] + } + for (let i = 0; i < 40; i++) { + const loss = model.fit(x, t, 10, 0.01, 10) + expect(model.epoch).toBe((i + 1) * 10) + if (loss < 1.0e-3) { + break + } + } + const y = model.predict(x) + const err = rmse(y, t)[0] + expect(err).toBeLessThan(0.5) + }) + + test('toObject', () => { + const model = new MLPRegressor([10, 7]) + const x = Matrix.randn(50, 2, 0, 5).toArray() + const t = Matrix.randn(50, 1).toArray() + model.fit(x, t, 1) + const y = model.predict(x) + + const obj = model.toObject() + expect(obj).toHaveLength(5) + const nn = NeuralNetwork.fromObject(obj) + const p = nn.predict(x) + for (let i = 0; i < y.length; i++) { + for (let j = 0; j < y[i].length; j++) { + expect(p[i][j]).toBeCloseTo(y[i][j]) + } + } + }) }) -test('classifier', () => { - const model = new MLPClassifier([10], 'tanh', 'adam') - const x = Matrix.concat(Matrix.randn(50, 2, 0, 0.2), Matrix.randn(50, 2, 5, 0.2)).toArray() - const t = [] - for (let i = 0; i < x.length; i++) { - t[i] = String.fromCharCode('a'.charCodeAt(0) + Math.floor(i / 50)) - } - - for (let i = 0; i < 1000; i++) { - model.fit(x, t, 1, 0.01, 10) - expect(model.epoch).toBe(i + 1) - } - const y = model.predict(x) - expect(y).toHaveLength(x.length) - const acc = accuracy(y, t) - expect(acc).toBeGreaterThan(0.95) +describe('classifier', () => { + test.each([ + undefined, + 'elu', + 'gaussian', + 'leaky_relu', + 'relu', + 'sigmoid', + 'softplus', + 'softsign', + 'tanh', + 'identity', + ])('%s', activation => { + const model = new MLPClassifier([3], activation) + const x = Matrix.concat(Matrix.randn(20, 2, 0, 0.2), Matrix.randn(20, 2, 5, 0.2)).toArray() + const t = [] + for (let i = 0; i < x.length; i++) { + t[i] = String.fromCharCode('a'.charCodeAt(0) + Math.floor(i / 20)) + } + + for (let i = 0; i < 10; i++) { + const loss = model.fit(x, t, 10, 0.01, 10) + expect(model.epoch).toBe((i + 1) * 10) + if (loss < 1.0e-3) { + break + } + } + expect(model.categories.toSorted()).toEqual(['a', 'b']) + const y = model.predict(x) + expect(y).toHaveLength(x.length) + const acc = accuracy(y, t) + expect(acc).toBeGreaterThan(0.95) + }) + + test('probability', () => { + const model = new MLPClassifier([3]) + const x = Matrix.concat(Matrix.randn(20, 2, 0, 0.2), Matrix.randn(20, 2, 5, 0.2)).toArray() + const t = [] + for (let i = 0; i < x.length; i++) { + t[i] = String.fromCharCode('a'.charCodeAt(0) + Math.floor(i / 20)) + } + + for (let i = 0; i < 100; i++) { + model.fit(x, t, 1, 0.01, 10) + expect(model.epoch).toBe(i + 1) + } + const y = model.predict(x) + const p = model.probability(x) + const c = model.categories + for (let i = 0; i < p.length; i++) { + let pi = -Infinity + let cat = null + for (let j = 0; j < p[i].length; j++) { + expect(p[i][j]).toBeGreaterThanOrEqual(0) + expect(p[i][j]).toBeLessThanOrEqual(1) + if (pi < p[i][j]) { + pi = p[i][j] + cat = j + } + } + expect(c[cat]).toBe(y[i]) + } + }) + + test('toObject', () => { + const model = new MLPClassifier([10, 7]) + const x = Matrix.randn(50, 2, 0, 5).toArray() + const t = [...Array(25).fill(1), ...Array(25).fill(2)] + model.fit(x, t, 1) + const y = model.probability(x) + + const obj = model.toObject() + expect(obj).toHaveLength(6) + const nn = NeuralNetwork.fromObject(obj) + const p = nn.predict(x) + for (let i = 0; i < y.length; i++) { + for (let j = 0; j < y[i].length; j++) { + expect(p[i][j]).toBeCloseTo(y[i][j]) + } + } + }) })