Skip to content

Commit 5a29b07

Browse files
jc21jc21
committed
Postgres Support
- Combines #4086 and #4087 PRs - Adds authentik in CI stack
1 parent 805968a commit 5a29b07

22 files changed

+437
-67
lines changed

‎Jenkinsfile‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,44 @@ pipeline {
167167
}
168168
}
169169
}
170+
stage('Test Postgres') {
171+
environment {
172+
COMPOSE_PROJECT_NAME = "npm_${BRANCH_LOWER}_${BUILD_NUMBER}_postgres"
173+
COMPOSE_FILE = 'docker/docker-compose.ci.yml:docker/docker-compose.ci.postgres.yml'
174+
}
175+
when {
176+
not {
177+
equals expected: 'UNSTABLE', actual: currentBuild.result
178+
}
179+
}
180+
steps {
181+
sh 'rm -rf ./test/results/junit/*'
182+
sh './scripts/ci/fulltest-cypress'
183+
}
184+
post {
185+
always {
186+
// Dumps to analyze later
187+
sh 'mkdir -p debug/postgres'
188+
sh 'docker logs $(docker-compose ps --all -q fullstack) > debug/postgres/docker_fullstack.log 2>&1'
189+
sh 'docker logs $(docker-compose ps --all -q stepca) > debug/postgres/docker_stepca.log 2>&1'
190+
sh 'docker logs $(docker-compose ps --all -q pdns) > debug/postgres/docker_pdns.log 2>&1'
191+
sh 'docker logs $(docker-compose ps --all -q pdns-db) > debug/postgres/docker_pdns-db.log 2>&1'
192+
sh 'docker logs $(docker-compose ps --all -q dnsrouter) > debug/postgres/docker_dnsrouter.log 2>&1'
193+
sh 'docker logs $(docker-compose ps --all -q db-postgres) > debug/postgres/docker_db-postgres.log 2>&1'
194+
sh 'docker logs $(docker-compose ps --all -q authentik) > debug/postgres/docker_authentik.log 2>&1'
195+
sh 'docker logs $(docker-compose ps --all -q authentik-redis) > debug/postgres/docker_authentik-redis.log 2>&1'
196+
sh 'docker logs $(docker-compose ps --all -q authentik-ldap) > debug/postgres/docker_authentik-ldap.log 2>&1'
197+
198+
junit 'test/results/junit/*'
199+
sh 'docker-compose down --remove-orphans --volumes -t 30 || true'
200+
}
201+
unstable {
202+
dir(path: 'testing/results') {
203+
archiveArtifacts(allowEmptyArchive: true, artifacts: '**/*', excludes: '**/*.xml')
204+
}
205+
}
206+
}
207+
}
170208
stage('MultiArch Build') {
171209
when {
172210
not {

‎backend/internal/access-list.js‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ const internalAccessList = {
8181

8282
return internalAccessList.build(row)
8383
.then(() => {
84-
if (row.proxy_host_count) {
84+
if (parseInt(row.proxy_host_count, 10)) {
8585
return internalNginx.bulkGenerateConfigs('proxy_host', row.proxy_hosts);
8686
}
8787
})
@@ -223,7 +223,7 @@ const internalAccessList = {
223223
.then((row) => {
224224
return internalAccessList.build(row)
225225
.then(() => {
226-
if (row.proxy_host_count) {
226+
if (parseInt(row.proxy_host_count, 10)) {
227227
return internalNginx.bulkGenerateConfigs('proxy_host', row.proxy_hosts);
228228
}
229229
}).then(internalNginx.reload)
@@ -252,7 +252,10 @@ const internalAccessList = {
252252
let query = accessListModel
253253
.query()
254254
.select('access_list.*', accessListModel.raw('COUNT(proxy_host.id) as proxy_host_count'))
255-
.joinRaw('LEFT JOIN `proxy_host` ON `proxy_host`.`access_list_id` = `access_list`.`id` AND `proxy_host`.`is_deleted` = 0')
255+
.leftJoin('proxy_host', function() {
256+
this.on('proxy_host.access_list_id', '=', 'access_list.id')
257+
.andOn('proxy_host.is_deleted', '=', 0);
258+
})
256259
.where('access_list.is_deleted', 0)
257260
.andWhere('access_list.id', data.id)
258261
.allowGraph('[owner,items,clients,proxy_hosts.[certificate,access_list.[clients,items]]]')
@@ -373,7 +376,10 @@ const internalAccessList = {
373376
let query = accessListModel
374377
.query()
375378
.select('access_list.*', accessListModel.raw('COUNT(proxy_host.id) as proxy_host_count'))
376-
.joinRaw('LEFT JOIN `proxy_host` ON `proxy_host`.`access_list_id` = `access_list`.`id` AND `proxy_host`.`is_deleted` = 0')
379+
.leftJoin('proxy_host', function() {
380+
this.on('proxy_host.access_list_id', '=', 'access_list.id')
381+
.andOn('proxy_host.is_deleted', '=', 0);
382+
})
377383
.where('access_list.is_deleted', 0)
378384
.groupBy('access_list.id')
379385
.allowGraph('[owner,items,clients]')

‎backend/internal/audit-log.js‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
const error = require('../lib/error');
2-
const auditLogModel = require('../models/audit-log');
1+
const error = require('../lib/error');
2+
const auditLogModel = require('../models/audit-log');
3+
const {castJsonIfNeed} = require('../lib/helpers');
34

45
const internalAuditLog = {
56

@@ -22,9 +23,9 @@ const internalAuditLog = {
2223
.allowGraph('[user]');
2324

2425
// Query is used for searching
25-
if (typeof search_query === 'string') {
26+
if (typeof search_query === 'string' && search_query.length > 0) {
2627
query.where(function () {
27-
this.where('meta', 'like', '%' + search_query + '%');
28+
this.where(castJsonIfNeed('meta'), 'like', '%' + search_query + '%');
2829
});
2930
}
3031

‎backend/internal/dead-host.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ const internalHost = require('./host');
66
const internalNginx = require('./nginx');
77
const internalAuditLog = require('./audit-log');
88
const internalCertificate = require('./certificate');
9+
const {castJsonIfNeed} = require('../lib/helpers');
910

1011
function omissions () {
1112
return ['is_deleted'];
@@ -409,16 +410,16 @@ const internalDeadHost = {
409410
.where('is_deleted', 0)
410411
.groupBy('id')
411412
.allowGraph('[owner,certificate]')
412-
.orderBy('domain_names', 'ASC');
413+
.orderBy(castJsonIfNeed('domain_names'), 'ASC');
413414

414415
if (access_data.permission_visibility !== 'all') {
415416
query.andWhere('owner_user_id', access.token.getUserId(1));
416417
}
417418

418419
// Query is used for searching
419-
if (typeof search_query === 'string') {
420+
if (typeof search_query === 'string' && search_query.length > 0) {
420421
query.where(function () {
421-
this.where('domain_names', 'like', '%' + search_query + '%');
422+
this.where(castJsonIfNeed('domain_names'), 'like', '%' + search_query + '%');
422423
});
423424
}
424425

‎backend/internal/host.js‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ const _ = require('lodash');
22
const proxyHostModel = require('../models/proxy_host');
33
const redirectionHostModel = require('../models/redirection_host');
44
const deadHostModel = require('../models/dead_host');
5+
const {castJsonIfNeed} = require('../lib/helpers');
56

67
const internalHost = {
78

@@ -17,7 +18,7 @@ const internalHost = {
1718
cleanSslHstsData: function (data, existing_data) {
1819
existing_data = existing_data === undefined ? {} : existing_data;
1920

20-
let combined_data = _.assign({}, existing_data, data);
21+
const combined_data = _.assign({}, existing_data, data);
2122

2223
if (!combined_data.certificate_id) {
2324
combined_data.ssl_forced = false;
@@ -73,7 +74,7 @@ const internalHost = {
7374
* @returns {Promise}
7475
*/
7576
getHostsWithDomains: function (domain_names) {
76-
let promises = [
77+
const promises = [
7778
proxyHostModel
7879
.query()
7980
.where('is_deleted', 0),
@@ -125,19 +126,19 @@ const internalHost = {
125126
* @returns {Promise}
126127
*/
127128
isHostnameTaken: function (hostname, ignore_type, ignore_id) {
128-
let promises = [
129+
const promises = [
129130
proxyHostModel
130131
.query()
131132
.where('is_deleted', 0)
132-
.andWhere('domain_names', 'like', '%' + hostname + '%'),
133+
.andWhere(castJsonIfNeed('domain_names'), 'like', '%' + hostname + '%'),
133134
redirectionHostModel
134135
.query()
135136
.where('is_deleted', 0)
136-
.andWhere('domain_names', 'like', '%' + hostname + '%'),
137+
.andWhere(castJsonIfNeed('domain_names'), 'like', '%' + hostname + '%'),
137138
deadHostModel
138139
.query()
139140
.where('is_deleted', 0)
140-
.andWhere('domain_names', 'like', '%' + hostname + '%')
141+
.andWhere(castJsonIfNeed('domain_names'), 'like', '%' + hostname + '%')
141142
];
142143

143144
return Promise.all(promises)

‎backend/internal/proxy-host.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ const internalHost = require('./host');
66
const internalNginx = require('./nginx');
77
const internalAuditLog = require('./audit-log');
88
const internalCertificate = require('./certificate');
9+
const {castJsonIfNeed} = require('../lib/helpers');
910

1011
function omissions () {
1112
return ['is_deleted', 'owner.is_deleted'];
@@ -416,16 +417,16 @@ const internalProxyHost = {
416417
.where('is_deleted', 0)
417418
.groupBy('id')
418419
.allowGraph('[owner,access_list,certificate]')
419-
.orderBy('domain_names', 'ASC');
420+
.orderBy(castJsonIfNeed('domain_names'), 'ASC');
420421

421422
if (access_data.permission_visibility !== 'all') {
422423
query.andWhere('owner_user_id', access.token.getUserId(1));
423424
}
424425

425426
// Query is used for searching
426-
if (typeof search_query === 'string') {
427+
if (typeof search_query === 'string' && search_query.length > 0) {
427428
query.where(function () {
428-
this.where('domain_names', 'like', '%' + search_query + '%');
429+
this.where(castJsonIfNeed('domain_names'), 'like', `%${search_query}%`);
429430
});
430431
}
431432

‎backend/internal/redirection-host.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ const internalHost = require('./host');
66
const internalNginx = require('./nginx');
77
const internalAuditLog = require('./audit-log');
88
const internalCertificate = require('./certificate');
9+
const {castJsonIfNeed} = require('../lib/helpers');
910

1011
function omissions () {
1112
return ['is_deleted'];
@@ -409,16 +410,16 @@ const internalRedirectionHost = {
409410
.where('is_deleted', 0)
410411
.groupBy('id')
411412
.allowGraph('[owner,certificate]')
412-
.orderBy('domain_names', 'ASC');
413+
.orderBy(castJsonIfNeed('domain_names'), 'ASC');
413414

414415
if (access_data.permission_visibility !== 'all') {
415416
query.andWhere('owner_user_id', access.token.getUserId(1));
416417
}
417418

418419
// Query is used for searching
419-
if (typeof search_query === 'string') {
420+
if (typeof search_query === 'string' && search_query.length > 0) {
420421
query.where(function () {
421-
this.where('domain_names', 'like', '%' + search_query + '%');
422+
this.where(castJsonIfNeed('domain_names'), 'like', `%${search_query}%`);
422423
});
423424
}
424425

‎backend/internal/stream.js‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ const utils = require('../lib/utils');
44
const streamModel = require('../models/stream');
55
const internalNginx = require('./nginx');
66
const internalAuditLog = require('./audit-log');
7+
const {castJsonIfNeed} = require('../lib/helpers');
78

89
function omissions () {
910
return ['is_deleted'];
@@ -293,21 +294,21 @@ const internalStream = {
293294
getAll: (access, expand, search_query) => {
294295
return access.can('streams:list')
295296
.then((access_data) => {
296-
let query = streamModel
297+
const query = streamModel
297298
.query()
298299
.where('is_deleted', 0)
299300
.groupBy('id')
300301
.allowGraph('[owner]')
301-
.orderBy('incoming_port', 'ASC');
302+
.orderByRaw('CAST(incoming_port AS INTEGER) ASC');
302303

303304
if (access_data.permission_visibility !== 'all') {
304305
query.andWhere('owner_user_id', access.token.getUserId(1));
305306
}
306307

307308
// Query is used for searching
308-
if (typeof search_query === 'string') {
309+
if (typeof search_query === 'string' && search_query.length > 0) {
309310
query.where(function () {
310-
this.where('incoming_port', 'like', '%' + search_query + '%');
311+
this.where(castJsonIfNeed('incoming_port'), 'like', `%${search_query}%`);
311312
});
312313
}
313314

@@ -327,9 +328,9 @@ const internalStream = {
327328
* @returns {Promise}
328329
*/
329330
getCount: (user_id, visibility) => {
330-
let query = streamModel
331+
const query = streamModel
331332
.query()
332-
.count('id as count')
333+
.count('id AS count')
333334
.where('is_deleted', 0);
334335

335336
if (visibility !== 'all') {

‎backend/lib/config.js‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@ const fs = require('fs');
22
const NodeRSA = require('node-rsa');
33
const logger = require('../logger').global;
44

5-
const keysFile = '/data/keys.json';
5+
const keysFile = '/data/keys.json';
6+
const mysqlEngine = 'mysql2';
7+
const postgresEngine = 'pg';
8+
const sqliteClientName = 'sqlite3';
69

710
let instance = null;
811

@@ -14,7 +17,7 @@ const configure = () => {
1417
let configData;
1518
try {
1619
configData = require(filename);
17-
} catch (err) {
20+
} catch (_) {
1821
// do nothing
1922
}
2023

@@ -34,7 +37,7 @@ const configure = () => {
3437
logger.info('Using MySQL configuration');
3538
instance = {
3639
database: {
37-
engine: 'mysql2',
40+
engine: mysqlEngine,
3841
host: envMysqlHost,
3942
port: process.env.DB_MYSQL_PORT || 3306,
4043
user: envMysqlUser,
@@ -46,13 +49,33 @@ const configure = () => {
4649
return;
4750
}
4851

52+
const envPostgresHost = process.env.DB_POSTGRES_HOST || null;
53+
const envPostgresUser = process.env.DB_POSTGRES_USER || null;
54+
const envPostgresName = process.env.DB_POSTGRES_NAME || null;
55+
if (envPostgresHost && envPostgresUser && envPostgresName) {
56+
// we have enough postgres creds to go with postgres
57+
logger.info('Using Postgres configuration');
58+
instance = {
59+
database: {
60+
engine: postgresEngine,
61+
host: envPostgresHost,
62+
port: process.env.DB_POSTGRES_PORT || 5432,
63+
user: envPostgresUser,
64+
password: process.env.DB_POSTGRES_PASSWORD,
65+
name: envPostgresName,
66+
},
67+
keys: getKeys(),
68+
};
69+
return;
70+
}
71+
4972
const envSqliteFile = process.env.DB_SQLITE_FILE || '/data/database.sqlite';
5073
logger.info(`Using Sqlite: ${envSqliteFile}`);
5174
instance = {
5275
database: {
5376
engine: 'knex-native',
5477
knex: {
55-
client: 'sqlite3',
78+
client: sqliteClientName,
5679
connection: {
5780
filename: envSqliteFile
5881
},
@@ -143,7 +166,27 @@ module.exports = {
143166
*/
144167
isSqlite: function () {
145168
instance === null && configure();
146-
return instance.database.knex && instance.database.knex.client === 'sqlite3';
169+
return instance.database.knex && instance.database.knex.client === sqliteClientName;
170+
},
171+
172+
/**
173+
* Is this a mysql configuration?
174+
*
175+
* @returns {boolean}
176+
*/
177+
isMysql: function () {
178+
instance === null && configure();
179+
return instance.database.engine === mysqlEngine;
180+
},
181+
182+
/**
183+
* Is this a postgres configuration?
184+
*
185+
* @returns {boolean}
186+
*/
187+
isPostgres: function () {
188+
instance === null && configure();
189+
return instance.database.engine === postgresEngine;
147190
},
148191

149192
/**

0 commit comments

Comments
 (0)