From 667b38e88c5cfd6e8a1347fb4caf69b6e6f7fef5 Mon Sep 17 00:00:00 2001 From: ishii-norimi Date: Sun, 30 Jan 2022 20:14:53 +0900 Subject: [PATCH 1/4] Add std layer and improve some layers --- lib/model/nns/graph.js | 2 +- lib/model/nns/layer/index.js | 1 + lib/model/nns/layer/random.js | 8 ++- lib/model/nns/layer/split.js | 15 ++++- lib/model/nns/layer/std.js | 41 ++++++++++++++ tests/lib/model/nns/layer/std.test.js | 80 +++++++++++++++++++++++++++ 6 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 lib/model/nns/layer/std.js create mode 100644 tests/lib/model/nns/layer/std.test.js diff --git a/lib/model/nns/graph.js b/lib/model/nns/graph.js index 799654d7d..4a9620da7 100644 --- a/lib/model/nns/graph.js +++ b/lib/model/nns/graph.js @@ -3,7 +3,7 @@ import { NeuralnetworkException } from '../neuralnetwork.js' import { Layer } from './layer/index.js' /** - * @typedef Node + * @typedef {object} Node * @property {Layer} layer * @property {string} name * @property {string[]} [input] diff --git a/lib/model/nns/layer/index.js b/lib/model/nns/layer/index.js index 16ba0538e..d3b358369 100644 --- a/lib/model/nns/layer/index.js +++ b/lib/model/nns/layer/index.js @@ -149,6 +149,7 @@ export * from './softmax.js' export * from './softplus.js' export * from './sparse.js' export * from './split.js' +export * from './std.js' export * from './sub.js' export * from './sum.js' export * from './supervisor.js' diff --git a/lib/model/nns/layer/random.js b/lib/model/nns/layer/random.js index 8304bf3b1..adcdde814 100644 --- a/lib/model/nns/layer/random.js +++ b/lib/model/nns/layer/random.js @@ -2,9 +2,11 @@ import Layer from './base.js' import Matrix from '../../../util/matrix.js' export default class RandomLayer extends Layer { - constructor({ size, ...rest }) { + constructor({ size, mean = 0, variance = 1, ...rest }) { super(rest) this._size = size + this._mean = mean + this._variance = variance this._rows = 1 } @@ -15,9 +17,9 @@ export default class RandomLayer extends Layer { calc() { if (typeof this._size === 'string') { const sizes = this.graph.getNode(this._size).lastOutputSize - return Matrix.randn(sizes[0], sizes[1]) + return Matrix.randn(sizes[0], sizes[1], this._mean, this._variance) } - return Matrix.randn(this._rows, this._size) + return Matrix.randn(this._rows, this._size, this._mean, this._variance) } grad() {} diff --git a/lib/model/nns/layer/split.js b/lib/model/nns/layer/split.js index c202b56d6..8de253bd2 100644 --- a/lib/model/nns/layer/split.js +++ b/lib/model/nns/layer/split.js @@ -11,9 +11,18 @@ export default class SplitLayer extends Layer { calc(x) { let c = 0 this._o = [] - for (let i = 0; i < this._size.length; i++) { - this._o.push(x.slice(c, c + this._size[i], this._axis)) - c += this._size[i] + if (typeof this._size === 'number') { + const s = x.sizes[this._axis] / this._size + for (let i = 0; i < this._size; i++) { + const n = Math.round((i + 1) * s) + this._o.push(x.slice(c, n, this._axis)) + c = n + } + } else { + for (let i = 0; i < this._size.length; i++) { + this._o.push(x.slice(c, c + this._size[i], this._axis)) + c += this._size[i] + } } return this._o } diff --git a/lib/model/nns/layer/std.js b/lib/model/nns/layer/std.js new file mode 100644 index 000000000..d8470ecf1 --- /dev/null +++ b/lib/model/nns/layer/std.js @@ -0,0 +1,41 @@ +import Layer from './base.js' +import Matrix from '../../../util/matrix.js' + +export default class StdLayer extends Layer { + constructor({ axis = -1, ...rest }) { + super(rest) + this._axis = axis + } + + calc(x) { + this._i = x + this._m = x.mean(this._axis) + if (this._axis < 0) { + this._o = x.std() + return new Matrix(1, 1, this._o) + } + this._o = x.std(this._axis) + return this._o + } + + grad(bo) { + if (this._axis < 0) { + return this._i.copyMap(v => (bo.toScaler() * (v - this._m)) / (this._o * this._i.length)) + } + const bi = this._i.copySub(this._m) + bi.sub(this._m) + bi.mult(1 / this._i.sizes[this._axis]) + bi.mult(this._o.copyMap(v => 1 / v)) + bi.mult(bo) + return bi + } + + toObject() { + return { + type: 'std', + axis: this._axis, + } + } +} + +StdLayer.registLayer('std') diff --git a/tests/lib/model/nns/layer/std.test.js b/tests/lib/model/nns/layer/std.test.js new file mode 100644 index 000000000..6dbcc0e7e --- /dev/null +++ b/tests/lib/model/nns/layer/std.test.js @@ -0,0 +1,80 @@ +import { jest } from '@jest/globals' +jest.retryTimes(3) + +import NeuralNetwork from '../../../../../lib/model/neuralnetwork.js' +import Matrix from '../../../../../lib/util/matrix.js' + +describe('std', () => { + describe('axis -1', () => { + test('calc', () => { + const net = NeuralNetwork.fromObject([{ type: 'input' }, { type: 'std' }]) + const x = Matrix.randn(10, 10) + + const y = net.calc(x) + const s = x.std() + expect(y.sizes).toEqual([1, 1]) + expect(y.at(0, 0)).toBeCloseTo(s) + }) + + test('grad', () => { + const net = NeuralNetwork.fromObject( + [{ type: 'input' }, { type: 'full', out_size: 3 }, { type: 'std' }], + 'mse', + 'adam' + ) + const x = Matrix.random(4, 5, -0.1, 0.1) + const t = Matrix.random(1, 1, 0.1, 1) + + for (let i = 0; i < 100; i++) { + const loss = net.fit(x, t, 1000, 0.01) + if (loss[0] < 1.0e-8) { + break + } + } + + const y = net.calc(x) + expect(y.at(0, 0)).toBeCloseTo(t.at(0, 0)) + }) + }) + + describe.each([0, 1])('axis %i', axis => { + test('calc', () => { + const net = NeuralNetwork.fromObject([{ type: 'input' }, { type: 'std', axis }]) + const x = Matrix.randn(10, 10) + + const y = net.calc(x) + const s = x.std(axis) + expect(y.sizes).toEqual(s.sizes) + for (let i = 0; i < s.rows; i++) { + for (let j = 0; j < s.cols; j++) { + expect(y.at(i, j)).toBeCloseTo(s.at(i, j)) + } + } + }) + + test('grad', () => { + const net = NeuralNetwork.fromObject( + [{ type: 'input' }, { type: 'full', out_size: 3 }, { type: 'std', axis }], + 'mse', + 'adam' + ) + const x = Matrix.random(4, 5, -0.1, 0.1) + const size = axis === 0 ? [1, 3] : [4, 1] + const t = Matrix.random(...size, 0.1, 1) + + for (let i = 0; i < 100; i++) { + const loss = net.fit(x, t, 1000, 0.01) + if (loss[0] < 1.0e-8) { + break + } + } + + const y = net.calc(x) + for (let i = 0; i < t.rows; i++) { + for (let j = 0; j < t.cols; j++) { + expect(y.at(i, j)).toBeCloseTo(t.at(i, j)) + } + } + }) + }) +}) From f8f7de6e39ea1da81c4f825a8e30a2e4ed4c997c Mon Sep 17 00:00:00 2001 From: ishii-norimi Date: Sun, 30 Jan 2022 20:18:25 +0900 Subject: [PATCH 2/4] Add Ladder network --- README.md | 2 +- js/model_selector.js | 1 + js/view/ladder_network.js | 163 +++++++++++++++++++ js/view/worker/ladder_network_worker.js | 27 ++++ lib/model/ladder_network.js | 201 ++++++++++++++++++++++++ lib/model/neuralnetwork.js | 4 +- tests/lib/model/ladder_network.test.js | 23 +++ 7 files changed, 418 insertions(+), 3 deletions(-) create mode 100644 js/view/ladder_network.js create mode 100644 js/view/worker/ladder_network_worker.js create mode 100644 lib/model/ladder_network.js create mode 100644 tests/lib/model/ladder_network.test.js diff --git a/README.md b/README.md index d84792672..da1b9827e 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ for (let i = 0; i < n; i++) { | ---- | ----- | | clustering | k-means, k-means++, k-medois, k-medians, x-means, G-means, LBG, ISODATA, Soft k-means, Fuzzy c-means, Possibilistic c-means, Kernel k-means, Agglomerative (complete linkage, single linkage, group average, Ward's, centroid, weighted average, median), DIANA, Mean shift, DBSCAN, OPTICS, PAM, CLARA, CLARANS, BIRCH, CURE, ROCK, PLSA, Latent dirichlet allocation, GMM, VBGMM, Affinity propagation, Spectral clustering, Mountain, SOM, GTM, (Growing) Neural gas, Growing cell structures, LVQ, ART, SVC, CAST, NMF, Autoencoder | | classification | Linear discriminant (FLD, LDA), Quadratic discriminant, Mixture discriminant, Least squares, Ridge, (Complement / Negation / Universal-set / Selective) Naive Bayes (gaussian), AODE, k-nearest neighbor, Radius neighbor, Fuzzy k-nearest neighbor, Nearest centroid, DANN, Decision tree, Random forest, GBDT, XGBoost, ALMA, ROMMA, Online gradient descent, Passive aggressive, RLS, Second order perceptron, AROW, NAROW, Confidence weighted, CELLIP, IELLIP, Normal herd, (Multinomial) Logistic regression, (Multinomial) Probit, SVM, Gaussian process, HMM, CRF, Bayesian Network, LVQ, Perceptron, ADALINE, MLP, LMNN | -| semi-supervised classification | k-nearest neighbor, Radius neighbor, Label propagation, Label spreading, k-means, GMM | +| semi-supervised classification | k-nearest neighbor, Radius neighbor, Label propagation, Label spreading, k-means, GMM, Ladder network | | regression | Least squares, Ridge, Lasso, Elastic net, RLS, Bayesian linear, Poisson, Least absolute deviations, Huber, Tukey, Least trimmed squares, Least median squares, Lp norm linear, SMA, Deming, Segmented, LOWESS, spline, Gaussian process, Principal components, Partial least squares, Projection pursuit, Quantile regression, k-nearest neighbor, Radius neighbor, IDW, Nadaraya Watson, Priestley Chao, Gasser Muller, RBF Network, RVM, Decision tree, Random forest, GBDT, XGBoost, SVR, MLP, GMR, Isotonic, Ramer Douglas Peucker, Passing-Bablok, Repeated median | | interpolation | Nearest neighbor, IDW, Linear, Spherical linear, Brahmagupta, Logarithmic, Cosine, (Inverse) Smoothstep, Cubic, (Centripetal) Catmull-Rom, Hermit, Polynomial, Lagrange, Trigonometric, Spline, RBF Network, Akima, Natural neighbor, Delaunay | | anomaly detection | Percentile, MAD, Tukey's fences, Grubbs's test, Thompson test, Tietjen Moore test, Generalized ESD, Hotelling, MT, MCD, k-nearest neighbor, LOF, PCA, OCSVM, KDE, GMM, Isolation forest, Autoencoder, GAN | diff --git a/js/model_selector.js b/js/model_selector.js index 248f4d0cd..e70e6de50 100644 --- a/js/model_selector.js +++ b/js/model_selector.js @@ -167,6 +167,7 @@ const AIMethods = [ { value: 'label_spreading', title: 'Label spreading' }, { value: 'kmeans', title: 'K-Means' }, { value: 'gmm', title: 'Gaussian mixture model' }, + { value: 'ladder_network', title: 'Ladder network' }, ], }, { diff --git a/js/view/ladder_network.js b/js/view/ladder_network.js new file mode 100644 index 000000000..1df3effae --- /dev/null +++ b/js/view/ladder_network.js @@ -0,0 +1,163 @@ +class LadderNetworkWorker extends BaseWorker { + constructor() { + super('js/view/worker/ladder_network_worker.js', { type: 'module' }) + } + + initialize(hidden_sizes, lambdas, activation, optimizer, cb) { + this._postMessage({ mode: 'init', hidden_sizes, lambdas, activation, optimizer }, cb) + } + + fit(train_x, train_y, iteration, rate, batch, cb) { + this._postMessage({ mode: 'fit', x: train_x, y: train_y, iteration, rate, batch }, cb) + } + + predict(x, cb) { + this._postMessage({ mode: 'predict', x: x }, cb) + } +} + +var dispLadder = function (elm, platform) { + const model = new LadderNetworkWorker() + const hidden_sizes = [10] + let epoch = 0 + + const fitModel = cb => { + const iteration = +elm.select('[name=iteration]').property('value') + const batch = +elm.select('[name=batch]').property('value') + const rate = +elm.select('[name=rate]').property('value') + const dim = platform.datas.dimension + + platform.fit((tx, ty, pred_cb) => { + ty = ty.map(v => v[0]) + model.fit(tx, ty, iteration, rate, batch, e => { + epoch = e.data.epoch + platform.predict( + (px, pred_cb) => { + model.predict(px, e => { + const data = e.data + pred_cb(data) + + cb && cb() + }) + }, + dim === 1 ? 2 : 4 + ) + }) + }) + } + + elm.append('span').text(' Hidden Layers ') + + const hsElm = elm.append('span') + const createHsElms = () => { + hsElm.selectAll('*').remove() + for (let i = 0; i < hidden_sizes.length; i++) { + const hsi = hsElm + .append('input') + .attr('type', 'number') + .attr('min', 1) + .attr('max', 100) + .attr('value', hidden_sizes[i]) + .on('change', () => { + hidden_sizes[i] = +hsi.property('value') + }) + } + if (hidden_sizes.length > 0) { + hsElm + .append('input') + .attr('type', 'button') + .attr('value', '-') + .on('click', () => { + hidden_sizes.pop() + createHsElms() + }) + } + } + elm.append('input') + .attr('type', 'button') + .attr('value', '+') + .on('click', () => { + hidden_sizes.push(10) + createHsElms() + }) + createHsElms() + elm.append('span').text(' Activation ') + elm.append('select') + .attr('name', 'activation') + .selectAll('option') + .data([ + 'sigmoid', + 'tanh', + 'relu', + 'elu', + 'leaky_relu', + 'rrelu', + 'prelu', + 'gaussian', + 'softplus', + 'softsign', + 'linear', + ]) + .enter() + .append('option') + .property('value', d => d) + .text(d => d) + + elm.append('span').text(' Optimizer ') + elm.append('select') + .attr('name', 'optimizer') + .selectAll('option') + .data(['sgd', 'adam', 'momentum', 'rmsprop']) + .enter() + .append('option') + .property('value', d => d) + .text(d => d) + const slbConf = platform.setting.ml.controller.stepLoopButtons().init(() => { + if (platform.datas.length === 0) { + return + } + + const activation = elm.select('[name=activation]').property('value') + const optimizer = elm.select('[name=optimizer]').property('value') + const lambdas = Array(hidden_sizes.length + 2).fill(0.001) + + model.initialize([...hidden_sizes], lambdas, activation, optimizer) + platform.init() + }) + elm.append('span').text(' Iteration ') + elm.append('select') + .attr('name', 'iteration') + .selectAll('option') + .data([1, 10, 100, 1000, 10000]) + .enter() + .append('option') + .property('value', d => d) + .text(d => d) + elm.append('span').text(' Learning rate ') + elm.append('input') + .attr('type', 'number') + .attr('name', 'rate') + .attr('min', 0) + .attr('max', 100) + .attr('step', 0.01) + .attr('value', 0.001) + elm.append('span').text(' Batch size ') + elm.append('input') + .attr('type', 'number') + .attr('name', 'batch') + .attr('value', 1000) + .attr('min', 1) + .attr('max', 1000) + .attr('step', 1) + slbConf.step(fitModel).epoch(() => epoch) + + return () => { + model.terminate() + } +} + +export default function (platform) { + platform.setting.ml.usage = + 'Click and add data point. Next, click "Initialize". Finally, click "Fit" button repeatedly.' + platform.setting.ternimate = dispLadder(platform.setting.ml.configElement, platform) +} diff --git a/js/view/worker/ladder_network_worker.js b/js/view/worker/ladder_network_worker.js new file mode 100644 index 000000000..a713d9be9 --- /dev/null +++ b/js/view/worker/ladder_network_worker.js @@ -0,0 +1,27 @@ +import LadderNetwork from '../../../lib/model/ladder_network.js' + +self.model = null + +self.addEventListener( + 'message', + function (e) { + const data = e.data + if (data.mode === 'init') { + self.model = new LadderNetwork(data.hidden_sizes, data.lambdas, data.activation, data.optimizer) + self.postMessage(null) + } else if (data.mode === 'fit') { + const samples = data.x.length + if (samples === 0) { + self.postMessage(null) + return + } + + self.model.fit(data.x, data.y, data.iteration, data.rate, data.batch) + self.postMessage({ epoch: self.model.epoch }) + } else if (data.mode === 'predict') { + const pred = self.model.predict(data.x) + self.postMessage(pred) + } + }, + false +) diff --git a/lib/model/ladder_network.js b/lib/model/ladder_network.js new file mode 100644 index 000000000..81d992c7c --- /dev/null +++ b/lib/model/ladder_network.js @@ -0,0 +1,201 @@ +import NeuralNetwork from './neuralnetwork.js' + +/** + * Ladder network + */ +export default class LadderNetwork { + // https://www.kabuku.co.jp/developers/semi-supervised_learning_with_ladder_networks + /** + * @param {number[]} hidden_sizes + * @param {number[]} lambdas + * @param {string} activation + * @param {string} optimizer + */ + constructor(hidden_sizes, lambdas, activation, optimizer) { + this._hidden_sizes = hidden_sizes + this._lambdas = lambdas + this._activation = activation + this._optimizer = optimizer + this._noise_std = Array(this._hidden_sizes.length + 2).fill(0.001) + + this._model = null + this._classes = null + this._optimizer = optimizer + this._epoch = 0 + } + + /** + * Epoch + * + * @type {number} + */ + get epoch() { + return this._epoch + } + + _build() { + const hidden_sizes = [...this._hidden_sizes, this._classes.length] + this._layers = [ + { type: 'input', name: 'x' }, + { type: 'linear', name: 'z_0' }, + { type: 'random', size: 'x', variance: this._noise_std[0] ** 2, name: 'noize' }, + { type: 'add', input: ['x', 'noize'], name: 'zt_0' }, + + { type: 'mean', input: 'x', name: 'mean_0' }, + { type: 'std', input: 'x', name: 'std_0' }, + + { type: 'concat', input: ['z_0', 'zt_0'], axis: 0 }, + ] + for (let i = 0; i < hidden_sizes.length; i++) { + const k = i + 1 + this._layers.push( + { type: 'full', out_size: hidden_sizes[i] }, + { type: 'split', size: 2, axis: 0, name: `zpre_${k}` }, + + { type: 'batch_normalization', input: `zpre_${k}[1]`, name: `ztbn_${k}` }, + { type: 'random', size: `ztbn_${k}`, variance: this._noise_std[k] ** 2, name: `noize_${k}` }, + { type: 'add', input: [`ztbn_${k}`, `noize_${k}`], name: `zt_${k}` }, + + { type: 'mean', input: `zpre_${k}[0]`, name: `mean_${k}` }, + { type: 'std', input: `zpre_${k}[0]`, name: `std_${k}` }, + { type: 'batch_normalization', input: `zpre_${k}[0]`, name: `z_${k}` }, + + { type: 'concat', input: [`z_${k}`, `zt_${k}`], axis: 0, name: `zcon_${k}` }, + + { type: 'variable', size: [1, hidden_sizes[i]], name: `b_${k}` }, + { type: 'add', input: [`zcon_${k}`, `b_${k}`], name: `hadd_${k}` }, + { type: 'variable', size: [1, hidden_sizes[i]], name: `g_${k}` }, + { type: 'mult', input: [`hadd_${k}`, `g_${k}`] }, + { type: this._activation } + ) + } + this._layers.push( + { type: 'split', size: 2, axis: 0, name: 'reduced' }, + { type: 'softmax', input: 'reduced[0]', name: 'predict' }, + { type: 'linear', input: 'reduced[1]' } + ) + + for (let k = hidden_sizes.length; k >= 0; k--) { + const size = k === 0 ? 'x' : hidden_sizes[k - 1] + if (k === hidden_sizes.length) { + this._layers.push({ type: 'batch_normalization', name: `u_${k}` }) + } else { + this._layers.push( + { type: 'full', out_size: size, input: `zh_${k + 1}` }, + { type: 'batch_normalization', name: `u_${k}` } + ) + } + this._layers.push( + { type: 'variable', size: `mean_${k}`, name: `a1_${k}` }, + { type: 'variable', size: `mean_${k}`, name: `a2_${k}` }, + { type: 'variable', size: `mean_${k}`, name: `a3_${k}` }, + { type: 'variable', size: `mean_${k}`, name: `a4_${k}` }, + { type: 'variable', size: `mean_${k}`, name: `a5_${k}` }, + { type: 'variable', size: `mean_${k}`, name: `a6_${k}` }, + { type: 'variable', size: `mean_${k}`, name: `a7_${k}` }, + { type: 'variable', size: `mean_${k}`, name: `a8_${k}` }, + { type: 'variable', size: `mean_${k}`, name: `a9_${k}` }, + { type: 'variable', size: `mean_${k}`, name: `a0_${k}` }, + + { type: 'mult', input: [`a2_${k}`, `u_${k}`], name: `m1_${k}` }, + { type: 'add', input: [`a3_${k}`, `m1_${k}`] }, + { type: 'sigmoid', name: `m2_${k}` }, + { type: 'mult', input: [`a1_${k}`, `m2_${k}`], name: `m3_${k}` }, + { type: 'mult', input: [`a4_${k}`, `u_${k}`], name: `m4_${k}` }, + { type: 'add', input: [`a5_${k}`, `m3_${k}`, `m4_${k}`], name: `m_${k}` }, + + { type: 'mult', input: [`a7_${k}`, `u_${k}`], name: `v1_${k}` }, + { type: 'add', input: [`a8_${k}`, `v1_${k}`] }, + { type: 'sigmoid', name: `v2_${k}` }, + { type: 'mult', input: [`a6_${k}`, `v2_${k}`], name: `v3_${k}` }, + { type: 'mult', input: [`a9_${k}`, `u_${k}`], name: `v4_${k}` }, + { type: 'add', input: [`a0_${k}`, `v3_${k}`, `v4_${k}`], name: `v_${k}` }, + + { type: 'sub', input: [`zt_${k}`, `m_${k}`], name: `zh1_${k}` }, + { type: 'mult', input: [`zh1_${k}`, `v_${k}`], name: `zh2_${k}` }, + { type: 'add', input: [`zh2_${k}`, `m_${k}`], name: `zh_${k}` }, + + { type: 'sub', input: [`zh_${k}`, `mean_${k}`], name: `zhbn1_${k}` }, + { type: 'div', input: [`zhbn1_${k}`, `std_${k}`], name: `zhbn_${k}` } + ) + } + this._layers.push( + { type: 'output', name: 'reconstruct' }, + + { type: 'supervisor', name: 't' }, + { type: 'sub', input: ['t', 'predict'] }, + { type: 'square' }, + { type: 'sum', axis: 1 }, + { type: 'mean', name: 'cc0' }, + { type: 'input', name: 'is_labeled' }, + { type: 'mult', input: ['cc0', 'is_labeled'], name: 'cc' } + ) + const costs = ['cc'] + for (let i = 0; i <= hidden_sizes.length; i++) { + this._layers.push( + { type: 'sub', input: [`z_${i}`, `zhbn_${i}`] }, + { type: 'square' }, + { type: 'sum', axis: 1 }, + { type: 'mean', name: `cl_${i}` }, + { type: 'mult', input: [this._lambdas[i], `cl_${i}`], name: `wcl_${i}` } + ) + costs.push(`wcl_${i}`) + } + this._layers.push({ type: 'add', input: costs }) + + return NeuralNetwork.fromObject(this._layers, null, 'adam') + } + + /** + * Fit model. + * + * @param {Array>} train_x + * @param {(* | null)[]} train_y + * @param {number} iteration + * @param {number} rate + * @param {number} batch + */ + fit(train_x, train_y, iteration, rate, batch) { + if (!this._classes) { + const classes = new Set() + for (let i = 0; i < train_y.length; i++) { + if (train_y[i] != null) { + classes.add(train_y[i]) + } + } + this._classes = [...classes] + } + if (!this._model) { + this._model = this._build() + } + const labeled_x = train_x.filter((v, i) => train_y[i] != null) + if (labeled_x.length > 0) { + const labeled_y = train_y.filter(v => v != null) + const y = labeled_y.map(v => { + const yi = Array(this._classes.length).fill(0) + yi[this._classes.indexOf(v)] = 1 + return yi + }) + this._model.fit({ x: labeled_x, is_labeled: [[1]] }, y, iteration, rate, batch) + } + + const unlabeled_x = train_x.filter((v, i) => train_y[i] == null) + if (unlabeled_x.length > 0) { + this._model.fit({ x: unlabeled_x, is_labeled: [[0]] }, [[0]], iteration, rate, batch) + } + this._epoch += iteration + } + + /** + * Returns predicted values. + * + * @param {Array>} x + * @returns {*[]} + */ + predict(x) { + return this._model + .calc({ x, is_labeled: null }, null, ['predict']) + .predict.argmax(1) + .value.map(v => this._classes[v]) + } +} diff --git a/lib/model/neuralnetwork.js b/lib/model/neuralnetwork.js index 1d98a798f..a958e5b1f 100644 --- a/lib/model/neuralnetwork.js +++ b/lib/model/neuralnetwork.js @@ -265,11 +265,11 @@ export default class NeuralNetwork { const last = Math.min(t.rows, i + batch_size) let xi if (x instanceof Matrix || x instanceof Tensor) { - xi = x instanceof Matrix ? x.slice(i, last) : x.slice(i, last) + xi = x.slice(i, last) } else { xi = {} for (const k of Object.keys(x)) { - xi[k] = x[k] instanceof Matrix ? x[k].slice(i, last) : x[k].slice(i, last) + xi[k] = x[k].sizes[0] < t.rows ? x[k] : x[k].slice(i, last) } } e = this.calc(xi, t.slice(i, last), null, options) diff --git a/tests/lib/model/ladder_network.test.js b/tests/lib/model/ladder_network.test.js new file mode 100644 index 000000000..6d329b45e --- /dev/null +++ b/tests/lib/model/ladder_network.test.js @@ -0,0 +1,23 @@ +import Matrix from '../../../lib/util/matrix.js' +import LadderNetwork from '../../../lib/model/ladder_network.js' + +import { accuracy } from '../../../lib/evaluate/classification.js' + +test('semi-classifier', () => { + const model = new LadderNetwork([5], [0.001, 0.0001, 0.0001], 'tanh', 'adam') + const x = Matrix.randn(50, 2, 0, 0.2).concat(Matrix.randn(50, 2, 5, 0.2)).toArray() + const t = [] + const t_org = [] + for (let i = 0; i < x.length; i++) { + t_org[i] = t[i] = String.fromCharCode('a'.charCodeAt(0) + Math.floor(i / 50)) + if (Math.random() < 0.5) { + t[i] = null + } + } + for (let i = 0; i < 5; i++) { + model.fit(x, t, 100, 0.001) + } + const y = model.predict(x) + const acc = accuracy(y, t_org) + expect(acc).toBeGreaterThan(0.95) +}) From 83189991ad6133c7ba18b5da18a47e4b624c0179 Mon Sep 17 00:00:00 2001 From: ishii-norimi Date: Sun, 30 Jan 2022 20:33:23 +0900 Subject: [PATCH 3/4] Improve test --- js/view/ladder_network.js | 2 +- tests/lib/model/nns/layer/std.test.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/js/view/ladder_network.js b/js/view/ladder_network.js index 1df3effae..44e73aa55 100644 --- a/js/view/ladder_network.js +++ b/js/view/ladder_network.js @@ -121,7 +121,7 @@ var dispLadder = function (elm, platform) { const optimizer = elm.select('[name=optimizer]').property('value') const lambdas = Array(hidden_sizes.length + 2).fill(0.001) - model.initialize([...hidden_sizes], lambdas, activation, optimizer) + model.initialize(hidden_sizes, lambdas, activation, optimizer) platform.init() }) elm.append('span').text(' Iteration ') diff --git a/tests/lib/model/nns/layer/std.test.js b/tests/lib/model/nns/layer/std.test.js index 6dbcc0e7e..e2bed22df 100644 --- a/tests/lib/model/nns/layer/std.test.js +++ b/tests/lib/model/nns/layer/std.test.js @@ -1,5 +1,5 @@ import { jest } from '@jest/globals' -jest.retryTimes(3) +jest.retryTimes(5) import NeuralNetwork from '../../../../../lib/model/neuralnetwork.js' import Matrix from '../../../../../lib/util/matrix.js' @@ -60,7 +60,7 @@ describe('std', () => { ) const x = Matrix.random(4, 5, -0.1, 0.1) const size = axis === 0 ? [1, 3] : [4, 1] - const t = Matrix.random(...size, 0.1, 1) + const t = Matrix.random(...size, 0.5, 1) for (let i = 0; i < 100; i++) { const loss = net.fit(x, t, 1000, 0.01) From a74d9755f6ea60f13d139eb7e13cde3bd5a95e83 Mon Sep 17 00:00:00 2001 From: ishii-norimi Date: Sun, 30 Jan 2022 20:39:19 +0900 Subject: [PATCH 4/4] Improve test --- tests/lib/model/ladder_network.test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/lib/model/ladder_network.test.js b/tests/lib/model/ladder_network.test.js index 6d329b45e..7e9b8a1c1 100644 --- a/tests/lib/model/ladder_network.test.js +++ b/tests/lib/model/ladder_network.test.js @@ -1,3 +1,6 @@ +import { jest } from '@jest/globals' +jest.retryTimes(3) + import Matrix from '../../../lib/util/matrix.js' import LadderNetwork from '../../../lib/model/ladder_network.js'