Add ordered logistic regression and rename ordered probit model (#732)

* Add ordered logistic regression and rename ordered probit model

* Small modification

Author: ishii-norimi

README.md

@@ -122,7 +122,7 @@ for (let i = 0; i < n; i++) {
 | task | model |
 | ---- | ----- |
 | clustering | (Soft / Kernel / Genetic / Weighted) k-means, k-means++, k-medois, k-medians, x-means, G-means, LBG, ISODATA, Fuzzy c-means, Possibilistic c-means, Agglomerative (complete linkage, single linkage, group average, Ward's, centroid, weighted average, median), DIANA, Monothetic, Mutual kNN, Mean shift, DBSCAN, OPTICS, HDBSCAN, DENCLUE, DBCLASD, CLUES, PAM, CLARA, CLARANS, BIRCH, CURE, ROCK, C2P, PLSA, Latent dirichlet allocation, GMM, VBGMM, Affinity propagation, Spectral clustering, Mountain, (Growing) SOM, GTM, (Growing) Neural gas, Growing cell structures, LVQ, ART, SVC, CAST, CHAMELEON, COLL, CLIQUE, PROCLUS, ORCLUS, FINDIT, NMF, Autoencoder |
-| classification | (Fisher's) Linear discriminant, Quadratic discriminant, Mixture discriminant, Least squares, (Multiclass / Kernel) Ridge, (Complement / Negation / Universal-set / Selective) Naive Bayes (gaussian), AODE, (Fuzzy / Weighted) k-nearest neighbor, Radius neighbor, Nearest centroid, ENN, ENaN, NNBCA, ADAMENN, DANN, IKNN, Decision tree, Random forest, Extra trees, GBDT, XGBoost, ALMA, (Aggressive) ROMMA, (Bounded) Online gradient descent, (Budgeted online) Passive aggressive, RLS, (Selective-sampling) Second order perceptron, AROW, NAROW, Confidence weighted, CELLIP, IELLIP, Normal herd, Stoptron, (Kernelized) Pegasos, MIRA, Forgetron, Projectron, Projectron++, Banditron, Ballseptron, (Multiclass) BSGD, ILK, SILK, (Multinomial) Logistic regression, (Multinomial) Probit, Ordinal regression, SVM, Gaussian process, HMM, CRF, Bayesian Network, LVQ, (Average / Multiclass / Voted / Kernelized / Selective-sampling / Margin / Shifting / Budget / Tighter / Tightest) Perceptron, PAUM, RBP, ADALINE, MADALINE, MLP, LMNN |
+| classification | (Fisher's) Linear discriminant, Quadratic discriminant, Mixture discriminant, Least squares, (Multiclass / Kernel) Ridge, (Complement / Negation / Universal-set / Selective) Naive Bayes (gaussian), AODE, (Fuzzy / Weighted) k-nearest neighbor, Radius neighbor, Nearest centroid, ENN, ENaN, NNBCA, ADAMENN, DANN, IKNN, Decision tree, Random forest, Extra trees, GBDT, XGBoost, ALMA, (Aggressive) ROMMA, (Bounded) Online gradient descent, (Budgeted online) Passive aggressive, RLS, (Selective-sampling) Second order perceptron, AROW, NAROW, Confidence weighted, CELLIP, IELLIP, Normal herd, Stoptron, (Kernelized) Pegasos, MIRA, Forgetron, Projectron, Projectron++, Banditron, Ballseptron, (Multiclass) BSGD, ILK, SILK, (Multinomial) Logistic regression, (Multinomial) Probit, Ordered logistic, Ordered probit, SVM, Gaussian process, HMM, CRF, Bayesian Network, LVQ, (Average / Multiclass / Voted / Kernelized / Selective-sampling / Margin / Shifting / Budget / Tighter / Tightest) Perceptron, PAUM, RBP, ADALINE, MADALINE, MLP, LMNN |
 | semi-supervised classification | k-nearest neighbor, Radius neighbor, Label propagation, Label spreading, k-means, GMM, S3VM, Ladder network |
 | regression | Least squares, Ridge, Lasso, Elastic net, RLS, Bayesian linear, Poisson, Least absolute deviations, Huber, Tukey, Least trimmed squares, Least median squares, Lp norm linear, SMA, Deming, Segmented, LOWESS, LOESS, spline, Naive Bayes, Gaussian process, Principal components, Partial least squares, Projection pursuit, Quantile regression, k-nearest neighbor, Radius neighbor, IDW, Nadaraya Watson, Priestley Chao, Gasser Muller, RBF Network, RVM, Decision tree, Random forest, Extra trees, GBDT, XGBoost, SVR, MLP, GMR, Isotonic, Ramer Douglas Peucker, Theil-Sen, Passing-Bablok, Repeated median |
 | interpolation | Nearest neighbor, IDW, (Spherical) Linear, Brahmagupta, Logarithmic, Cosine, (Inverse) Smoothstep, Cubic, (Centripetal) Catmull-Rom, Hermit, Polynomial, Lagrange, Trigonometric, Spline, RBF Network, Akima, Natural neighbor, Delaunay |

js/model_selector.js

@@ -207,13 +207,16 @@ const AIMethods = [
 				{ value: 'mlp', title: 'Multi-layer perceptron' },
 				{ value: 'neuralnetwork', title: 'Neuralnetwork' },
 			],
+			Ranking: [
+				{ value: 'ordered_logistic', title: 'Ordered logistic regression' },
+				{ value: 'ordered_probit', title: 'Ordered probit regression' },
+			],
 			'': [
 				{ value: 'least_square', title: 'Least squares' },
 				{ value: 'ridge', title: 'Ridge' },
 				{ value: 'nearest_centroid', title: 'Nearest Centroid' },
 				{ value: 'logistic', title: 'Logistic regression' },
 				{ value: 'probit', title: 'Probit' },
-				{ value: 'ordinal_regression', title: 'Ordinal regression' },
 				{ value: 'svm', title: 'Support vector machine' },
 				{ value: 'gaussian_process', title: 'Gaussian Process' },
 				{ value: 'hmm', title: 'HMM' },

js/view/ordered_logistic.js

@@ -0,0 +1,36 @@
+import OrderedLogisticRegression from '../../lib/model/ordered_logistic.js'
+import Controller from '../controller.js'
+
+export default function (platform) {
+	platform.setting.ml.usage =
+		'Click and add data point. Next, click "Initialize". Finally, click "Fit" button repeatedly.'
+	platform.setting.ml.reference = {
+		title: 'Ordinal regression (Wikipedia)',
+		url: 'https://en.wikipedia.org/wiki/Ordinal_regression',
+	}
+	const controller = new Controller(platform)
+
+	let model = null
+	const fitModel = () => {
+		if (!model) {
+			return
+		}
+
+		model.fit(
+			platform.trainInput,
+			platform.trainOutput.map(v => v[0])
+		)
+		const pred = model.predict(platform.testInput(4))
+		platform.testResult(pred)
+	}
+
+	const rate = controller.input.number({ label: ' Learning rate ', value: 0.1, min: 0, max: 100, step: 0.1 })
+	controller
+		.stepLoopButtons()
+		.init(() => {
+			model = new OrderedLogisticRegression(rate.value)
+			platform.init()
+		})
+		.step(fitModel)
+		.epoch()
+}

js/view/ordinal_regression.js renamed to js/view/ordered_probit.js

@@ -1,4 +1,4 @@
-import OrdinalRegression from '../../lib/model/ordinal_regression.js'
+import OrderedProbitRegression from '../../lib/model/ordered_probit.js'
 import Controller from '../controller.js'
 
 export default function (platform) {
@@ -24,11 +24,11 @@ export default function (platform) {
 		platform.testResult(pred)
 	}
 
-	const rate = controller.input.number({ label: ' Learning rate ', value: 0.001, min: 0, max: 100, step: 0.001 })
+	const rate = controller.input.number({ label: ' Learning rate ', value: 0.1, min: 0, max: 100, step: 0.1 })
 	controller
 		.stepLoopButtons()
 		.init(() => {
-			model = new OrdinalRegression(rate.value)
+			model = new OrderedProbitRegression(rate.value)
 			platform.init()
 		})
 		.step(fitModel)

lib/model/ordered_logistic.js

@@ -0,0 +1,86 @@
+import Matrix from '../util/matrix.js'
+
+/**
+ * Ordered logistic regression
+ */
+export default class OrderedLogisticRegression {
+	// https://en.wikipedia.org/wiki/Ordered_logit
+	// https://en.wikipedia.org/wiki/Ordinal_regression
+	/**
+	 * @param {number} [rate=0.1] Learning rate
+	 */
+	constructor(rate = 0.1) {
+		this._w = null
+		this._a = rate
+	}
+
+	_s(x) {
+		return 1 / (1 + Math.exp(-x))
+	}
+
+	/**
+	 * Fit model.
+	 *
+	 * @param {Array<Array<number>>} x Training data
+	 * @param {Array<number>} y Target values
+	 */
+	fit(x, y) {
+		x = Matrix.fromArray(x)
+		if (!this._response) {
+			this._response = [...new Set(y)]
+			this._response.sort((a, b) => a - b)
+			this._t = Array.from(this._response, (_, i) => i - 1)
+			this._t[0] = -Infinity
+			this._t.push(Infinity)
+			this._w = Matrix.randn(x.cols, 1, 0, 0.1)
+		}
+
+		const pstar = x.dot(this._w).value
+		const dt = Array(this._t.length).fill(0)
+		const dw = Matrix.zeros(this._w.rows, this._w.cols)
+		for (let i = 0; i < y.length; i++) {
+			const k = this._response.indexOf(y[i])
+			const pk0 = this._s(this._t[k] - pstar[i])
+			const pk1 = this._s(this._t[k + 1] - pstar[i])
+			if (pk0 === pk1) {
+				continue
+			}
+
+			dt[k] += (pk0 * (1 - pk0)) / (pk0 - pk1)
+			dt[k + 1] -= (pk1 * (1 - pk1)) / (pk0 - pk1)
+
+			const dwi = x.row(i).t
+			dwi.mult((pk1 * (1 - pk1) - pk0 * (1 - pk0)) / (pk0 - pk1))
+			dw.add(dwi)
+		}
+		dw.mult(this._a / y.length)
+
+		this._w.add(dw)
+		for (let i = 1; i < this._t.length - 1; i++) {
+			this._t[i] += (this._a / y.length) * dt[i]
+		}
+		this._t.sort((a, b) => a - b)
+	}
+
+	/**
+	 * Returns predicted values.
+	 *
+	 * @param {Array<Array<number>>} x Sample data
+	 * @returns {Array<number>} Predicted values
+	 */
+	predict(x) {
+		x = Matrix.fromArray(x)
+		const pstar = x.dot(this._w).value
+		const p = []
+		for (let i = 0; i < pstar.length; i++) {
+			p[i] = null
+			for (let k = 0; k < this._response.length; k++) {
+				if (this._t[k] < pstar[i] && pstar[i] <= this._t[k + 1]) {
+					p[i] = this._response[k]
+					break
+				}
+			}
+		}
+		return p
+	}
+}

lib/model/ordinal_regression.js renamed to lib/model/ordered_probit.js

@@ -1,9 +1,9 @@
 import Matrix from '../util/matrix.js'
 
 /**
- * Ordinal regression
+ * Ordered probit regression
  */
-export default class OrdinalRegression {
+export default class OrderedProbitRegression {
 	// https://en.wikipedia.org/wiki/Ordinal_regression
 	/**
 	 * @param {number} [rate=0.001] Learning rate
@@ -76,11 +76,11 @@ export default class OrdinalRegression {
 			dwi.mult((this._gaussian(this._t[k + 1] - pstar[i]) - this._gaussian(this._t[k] - pstar[i])) / (pk0 - pk1))
 			dw.add(dwi)
 		}
-		dw.mult(this._a)
+		dw.mult(this._a / y.length)
 
 		this._w.add(dw)
 		for (let i = 1; i < this._t.length - 1; i++) {
-			this._t[i] += this._a * dt[i]
+			this._t[i] += (this._a / y.length) * dt[i]
 		}
 		this._t.sort((a, b) => a - b)
 	}

tests/gui/view/ordinal_regression.test.js renamed to tests/gui/view/ordered_logistic.test.js

@@ -15,19 +15,19 @@ describe('classification', () => {
 		const taskSelectBox = await page.waitForSelector('#ml_selector dl:first-child dd:nth-child(5) select')
 		await taskSelectBox.selectOption('CF')
 		const modelSelectBox = await page.waitForSelector('#ml_selector .model_selection #mlDisp')
-		await modelSelectBox.selectOption('ordinal_regression')
+		await modelSelectBox.selectOption('ordered_logistic')
 		const methodMenu = await page.waitForSelector('#ml_selector #method_menu')
 		const buttons = await methodMenu.waitForSelector('.buttons')
 
 		const rate = await buttons.waitForSelector('input:nth-of-type(1)')
-		await expect((await rate.getProperty('value')).jsonValue()).resolves.toBe('0.001')
+		await expect((await rate.getProperty('value')).jsonValue()).resolves.toBe('0.1')
 	})
 
 	test('learn', async () => {
 		const taskSelectBox = await page.waitForSelector('#ml_selector dl:first-child dd:nth-child(5) select')
 		await taskSelectBox.selectOption('CF')
 		const modelSelectBox = await page.waitForSelector('#ml_selector .model_selection #mlDisp')
-		await modelSelectBox.selectOption('ordinal_regression')
+		await modelSelectBox.selectOption('ordered_logistic')
 		const methodMenu = await page.waitForSelector('#ml_selector #method_menu')
 		const buttons = await methodMenu.waitForSelector('.buttons')
 

tests/gui/view/ordered_probit.test.js

@@ -0,0 +1,47 @@
+import { getPage } from '../helper/browser'
+
+describe('classification', () => {
+	/** @type {Awaited<ReturnType<getPage>>} */
+	let page
+	beforeEach(async () => {
+		page = await getPage()
+	})
+
+	afterEach(async () => {
+		await page?.close()
+	})
+
+	test('initialize', async () => {
+		const taskSelectBox = await page.waitForSelector('#ml_selector dl:first-child dd:nth-child(5) select')
+		await taskSelectBox.selectOption('CF')
+		const modelSelectBox = await page.waitForSelector('#ml_selector .model_selection #mlDisp')
+		await modelSelectBox.selectOption('ordered_probit')
+		const methodMenu = await page.waitForSelector('#ml_selector #method_menu')
+		const buttons = await methodMenu.waitForSelector('.buttons')
+
+		const rate = await buttons.waitForSelector('input:nth-of-type(1)')
+		await expect((await rate.getProperty('value')).jsonValue()).resolves.toBe('0.1')
+	})
+
+	test('learn', async () => {
+		const taskSelectBox = await page.waitForSelector('#ml_selector dl:first-child dd:nth-child(5) select')
+		await taskSelectBox.selectOption('CF')
+		const modelSelectBox = await page.waitForSelector('#ml_selector .model_selection #mlDisp')
+		await modelSelectBox.selectOption('ordered_probit')
+		const methodMenu = await page.waitForSelector('#ml_selector #method_menu')
+		const buttons = await methodMenu.waitForSelector('.buttons')
+
+		const epoch = await buttons.waitForSelector('[name=epoch]')
+		await expect(epoch.evaluate(el => el.textContent)).resolves.toBe('0')
+		const methodFooter = await page.waitForSelector('#method_footer', { state: 'attached' })
+		await expect(methodFooter.evaluate(el => el.textContent)).resolves.toBe('')
+
+		const initButton = await buttons.waitForSelector('input[value=Initialize]')
+		await initButton.evaluate(el => el.click())
+		const stepButton = await buttons.waitForSelector('input[value=Step]:enabled')
+		await stepButton.evaluate(el => el.click())
+
+		await expect(epoch.evaluate(el => el.textContent)).resolves.toBe('1')
+		await expect(methodFooter.evaluate(el => el.textContent)).resolves.toMatch(/^Accuracy:[0-9.]+$/)
+	})
+})

tests/lib/model/ordinal_regression.test.js renamed to tests/lib/model/ordered_logistic.test.js

@@ -1,11 +1,11 @@
 import Matrix from '../../../lib/util/matrix.js'
-import OrdinalRegression from '../../../lib/model/ordinal_regression.js'
+import OrderedLogisticRegression from '../../../lib/model/ordered_logistic.js'
 
 import { rmse } from '../../../lib/evaluate/regression.js'
 
 describe('ordinal', () => {
 	test('fit', () => {
-		const model = new OrdinalRegression()
+		const model = new OrderedLogisticRegression()
 		const x = Matrix.concat(
 			Matrix.concat(Matrix.randn(50, 2, 0, 0.2), Matrix.randn(50, 2, 2, 0.2)),
 			Matrix.randn(50, 2, 4, 0.2)

tests/lib/model/ordered_probit.test.js

@@ -0,0 +1,27 @@
+import { jest } from '@jest/globals'
+jest.retryTimes(3)
+
+import Matrix from '../../../lib/util/matrix.js'
+import OrderedProbitRegression from '../../../lib/model/ordered_probit.js'
+
+import { rmse } from '../../../lib/evaluate/regression.js'
+
+describe('ordinal', () => {
+	test('fit', () => {
+		const model = new OrderedProbitRegression()
+		const x = Matrix.concat(
+			Matrix.concat(Matrix.randn(50, 2, 0, 0.2), Matrix.randn(50, 2, 2, 0.2)),
+			Matrix.randn(50, 2, 4, 0.2)
+		).toArray()
+		const t = []
+		for (let i = 0; i < x.length; i++) {
+			t[i] = Math.floor(i / 50)
+		}
+		for (let i = 0; i < 100; i++) {
+			model.fit(x, t)
+		}
+		const y = model.predict(x)
+		const err = rmse(y, t)
+		expect(err).toBeLessThan(0.7)
+	})
+})