Compare commits

...

10 Commits

Author SHA1 Message Date
Rainer Killinger
1515e7778f 1.2.0 2022-06-08 18:20:48 +02:00
Rainer Killinger
139e405bd3 refactor: update dependencies 2022-06-08 17:47:10 +02:00
Rainer Killinger
5522ac55ac feat: added prometheus metrics support 2022-06-08 17:44:57 +02:00
Rainer Killinger
39e710e685 test: expand test coverage 2022-05-10 12:58:22 +02:00
Rainer Killinger
4bb46d8a06 feat: support docker swarm deployments 2022-05-10 12:32:38 +02:00
Rainer Killinger
ac144095bf refactor: apply @openstapps/eslint-config rules 2022-05-09 15:45:25 +02:00
Rainer Killinger
1fcf7340d4 feat: reload nginx on proxyconfig change 2022-05-06 12:55:10 +02:00
openstappsbot
ed7b5eaf65 refactor: update all 2022-05-05 07:08:18 +00:00
Rainer Killinger
6756f6ccd4 ci: add docker image build to all tests 2022-03-10 17:01:52 +01:00
Rainer Killinger
9990759de0 docs: update changelog 2022-03-10 16:55:19 +01:00
16 changed files with 1827 additions and 656 deletions

3
.eslintrc.json Normal file
View File

@@ -0,0 +1,3 @@
{
"extends": "@openstapps"
}

3
.gitignore vendored
View File

@@ -92,4 +92,5 @@ docs/
# Certificates # Certificates
*.crt *.crt
*.key *.key
test/certs

View File

@@ -38,7 +38,6 @@ scheduled-audit:
stage: audit stage: audit
docker image build: docker image build:
before_script: []
image: registry.gitlab.com/openstapps/projectmanagement/builder image: registry.gitlab.com/openstapps/projectmanagement/builder
stage: test stage: test
dependencies: dependencies:
@@ -48,10 +47,7 @@ docker image build:
services: services:
- docker:dind - docker:dind
script: script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker build -t $CI_REGISTRY_IMAGE/$REGISTRY_BRANCH$CI_COMMIT_REF_NAME:latest . - docker build -t $CI_REGISTRY_IMAGE/$REGISTRY_BRANCH$CI_COMMIT_REF_NAME:latest .
except:
- /(^v[0-9]+\.[0-9]+\.[0-9]+$|^master$|^develop$)/
tags: tags:
- docker - docker

View File

@@ -1,3 +1,7 @@
# [1.1.0](https://gitlab.com/openstapps/proxy/compare/v1.0.1...v1.1.0) (2022-03-10)
## [1.0.1](https://gitlab.com/openstapps/proxy/compare/v1.0.0...v1.0.1) (2022-03-10) ## [1.0.1](https://gitlab.com/openstapps/proxy/compare/v1.0.0...v1.0.1) (2022-03-10)

View File

@@ -6,6 +6,7 @@ RUN apk update && \
apk upgrade && \ apk upgrade && \
apk add openssl && \ apk add openssl && \
apk add nginx && \ apk add nginx && \
apk add nginx-mod-http-vts && \
rm -rf /var/cache/apk/* rm -rf /var/cache/apk/*
ADD . /app ADD . /app

View File

@@ -16,8 +16,9 @@
import {ConfigFile} from '../src/common'; import {ConfigFile} from '../src/common';
const config: ConfigFile = { const config: ConfigFile = {
activeVersions: ['1\\.0\\.\\d+','2\\.0\\.\\d+'], activeVersions: ['1\\.0\\.\\d+', '2\\.0\\.\\d+'],
hiddenRoutes: ['/bulk'], hiddenRoutes: ['/bulk'],
metrics: false,
outdatedVersions: ['0\\.8\\.\\d+', '0\\.5\\.\\d+', '0\\.6\\.\\d+', '0\\.7\\.\\d+'], outdatedVersions: ['0\\.8\\.\\d+', '0\\.5\\.\\d+', '0\\.6\\.\\d+', '0\\.7\\.\\d+'],
output: '/etc/nginx/http.d/default.conf', output: '/etc/nginx/http.d/default.conf',
rateLimitAllowList: ['127.0.0.1/32'], rateLimitAllowList: ['127.0.0.1/32'],

View File

@@ -0,0 +1,7 @@
server {
listen 8080;
location /metrics {
vhost_traffic_status_display;
vhost_traffic_status_display_format prometheus;
}
}

View File

@@ -1,3 +1,5 @@
load_module modules/ngx_http_vhost_traffic_status_module.so;
worker_processes 1; worker_processes 1;
error_log stderr; error_log stderr;
@@ -9,6 +11,7 @@ events {
} }
http { http {
vhost_traffic_status_zone;
include mime.types; include mime.types;
default_type application/octet-stream; default_type application/octet-stream;
access_log /dev/stdout; access_log /dev/stdout;

View File

@@ -1,3 +1,5 @@
{{{ metrics }}}
{{{ dockerVersionMap }}} {{{ dockerVersionMap }}}
# create a custom request limit zone which can handle 160,000 IP-Addresses at the same time # create a custom request limit zone which can handle 160,000 IP-Addresses at the same time

1907
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,49 +1,60 @@
{ {
"name": "@openstapps/proxy", "name": "@openstapps/proxy",
"version": "1.1.0", "version": "1.2.0",
"description": "NGINX proxy that is dynamically configured by a Node.js script", "description": "NGINX proxy that is dynamically configured by a Node.js script",
"main": "./lib/cli.js", "main": "./lib/cli.js",
"dependencies": { "dependencies": {
"@openstapps/logger": "0.8.0", "@openstapps/logger": "0.8.1",
"@types/config": "0.0.41", "@types/config": "0.0.41",
"@types/dockerode": "3.3.3", "@types/dockerode": "3.3.9",
"@types/node": "14.18.12", "@types/node": "14.18.16",
"@types/sha1": "1.1.3", "@types/sha1": "1.1.3",
"config": "3.3.7", "config": "3.3.7",
"dockerode": "3.3.1", "dockerode": "3.3.2",
"is-cidr": "4.0.2", "is-cidr": "4.0.2",
"mustache": "4.2.0", "mustache": "4.2.0",
"semver": "7.3.5" "node-port-scanner": "3.0.1",
"semver": "7.3.7"
}, },
"devDependencies": { "devDependencies": {
"@openstapps/configuration": "0.29.0", "@openstapps/configuration": "0.31.0",
"@openstapps/eslint-config": "1.0.0",
"@testdeck/mocha": "0.2.0", "@testdeck/mocha": "0.2.0",
"@types/chai": "4.3.0", "@types/chai": "4.3.1",
"@types/chai-spies": "1.0.3", "@types/chai-spies": "1.0.3",
"@types/mustache": "4.1.2", "@types/mustache": "4.1.3",
"@types/proxyquire": "1.3.28",
"@typescript-eslint/eslint-plugin": "5.27.1",
"@typescript-eslint/parser": "5.27.1",
"chai": "4.3.6", "chai": "4.3.6",
"chai-spies": "1.0.0", "chai-spies": "1.0.0",
"conventional-changelog-cli": "2.2.2", "conventional-changelog-cli": "2.2.2",
"mocha": "9.2.1", "eslint": "8.17.0",
"eslint-config-prettier": "8.5.0",
"eslint-plugin-jsdoc": "39.3.2",
"eslint-plugin-prettier": "4.0.0",
"eslint-plugin-unicorn": "42.0.0",
"mocha": "9.2.2",
"nyc": "15.1.0", "nyc": "15.1.0",
"prepend-file-cli": "1.0.6", "prepend-file-cli": "1.0.6",
"prettier": "2.6.2",
"proxyquire": "2.1.3",
"rimraf": "3.0.2", "rimraf": "3.0.2",
"ts-node": "10.6.0", "ts-node": "10.8.1",
"tslint": "6.1.3", "typedoc": "0.22.17",
"typedoc": "0.22.12", "typescript": "4.4.4"
"typescript": "4.4.2"
}, },
"scripts": { "scripts": {
"build": "npm run tslint && npm run compile", "build": "npm run lint && npm run compile",
"changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 0 && git add CHANGELOG.md && git commit -m 'docs: update changelog'", "changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 0 && git add CHANGELOG.md && git commit -m 'docs: update changelog'",
"check-configuration": "openstapps-configuration", "check-configuration": "openstapps-configuration",
"compile": "rimraf lib && tsc && prepend lib/cli.js '#!/usr/bin/env node\n'", "compile": "rimraf lib && tsc && prepend lib/cli.js '#!/usr/bin/env node\n'",
"documentation": "typedoc --includeVersion --out docs --readme README.md --entryPointStrategy expand src", "documentation": "typedoc --includeVersion --out docs --readme README.md --entryPointStrategy expand src",
"lint": "eslint --ext .ts src/",
"postversion": "npm run changelog", "postversion": "npm run changelog",
"prepublishOnly": "npm ci && npm run build", "prepublishOnly": "npm ci && npm run build",
"preversion": "npm run prepublishOnly", "preversion": "npm run prepublishOnly",
"push": "git push && git push origin \"v$npm_package_version\"", "push": "git push && git push origin \"v$npm_package_version\"",
"tslint": "tslint -p tsconfig.json -c tslint.json 'src/**/*.ts'",
"test": "nyc mocha --require ts-node/register 'test/**/*.spec.ts'" "test": "nyc mocha --require ts-node/register 'test/**/*.spec.ts'"
}, },
"repository": { "repository": {

View File

@@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2019 StApps * Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as * it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the * published by the Free Software Foundation, either version 3 of the
@@ -17,16 +17,20 @@ import {Logger} from '@openstapps/logger';
import {execSync} from 'child_process'; import {execSync} from 'child_process';
import * as Dockerode from 'dockerode'; import * as Dockerode from 'dockerode';
import {render} from 'mustache'; import {render} from 'mustache';
import {asyncReadFile, asyncWriteFile, configFile} from './common'; import {asyncReadFile, asyncWriteFile} from './common';
import {getContainers, getTemplateView} from './main'; import {getContainers, getTemplateView} from './main';
/* eslint-disable unicorn/prefer-module */
// handle unhandled promise rejections // handle unhandled promise rejections
process.on('unhandledRejection', async (error) => { process.on('unhandledRejection', async error => {
await Logger.error(error); await Logger.error(error);
process.exit(1); process.exit(1);
}); });
let containerHashCache = ''; let containerHashCache = '';
let configHashCache = '';
const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
/** /**
* Reads the container information from the docker socket and updates the nginx config if necessary * Reads the container information from the docker socket and updates the nginx config if necessary
@@ -40,16 +44,25 @@ async function updateNginxConfig() {
}) })
.join(','); .join(',');
delete require.cache[require.resolve('config')];
// eslint-disable-next-line @typescript-eslint/no-var-requires
const configFile = require('config');
const configHash = JSON.stringify(configFile);
// if containers changed -> write config file, reload nginx // if containers changed -> write config file, reload nginx
if (containerHash !== containerHashCache) { if (containerHash !== containerHashCache || configHash !== configHashCache) {
Logger.log('Generating new NGINX configuration'); Logger.log('Generating new NGINX configuration');
Logger.log('Waiting for Docker network to settle...');
await delay(10_000);
// render nginx config file // render nginx config file
const nginxConfig = render(await asyncReadFile('nginx.conf.template', 'utf8'), await getTemplateView(containers)); const nginxConfig = render(
await asyncReadFile('nginx.conf.template', 'utf8'),
Logger.log(`containers (${containerHash}) matched the configuration.`); await getTemplateView(containers),
);
containerHashCache = containerHash; containerHashCache = containerHash;
configHashCache = configHash;
Logger.log(`Writing new config file "${configFile.output}"`); Logger.log(`Writing new config file "${configFile.output}"`);
@@ -61,9 +74,9 @@ async function updateNginxConfig() {
execSync('nginx -s reload'); execSync('nginx -s reload');
} }
// tslint:disable-next-line:no-magic-numbers - set timeout to update configuration again in 30s // set timeout to update configuration again in 30s
setTimeout(updateNginxConfig, 30000); setTimeout(updateNginxConfig, 30_000);
} }
// tslint:disable-next-line:no-floating-promises - start the process that checks the docker socket periodically // start the process that checks the docker socket periodically
updateNginxConfig(); updateNginxConfig();

View File

@@ -59,12 +59,16 @@ export interface ConfigFile {
* List of hidden routes * List of hidden routes
*/ */
hiddenRoutes: string[]; hiddenRoutes: string[];
/**
* Enables metrics on /metrics route
*/
metrics: boolean;
/** /**
* List of outdated versions * List of outdated versions
*/ */
outdatedVersions: string[]; outdatedVersions: string[];
/** /**
* Output?! TODO * Path the generated config will be written to
*/ */
output: string; output: string;
/** /**
@@ -93,6 +97,10 @@ export interface TemplateView {
* Listener * Listener
*/ */
listener: string; listener: string;
/**
* Local server with listener for /metrics route
*/
metrics: string;
/** /**
* Allow list for rate limiting * Allow list for rate limiting
*/ */

View File

@@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2019 StApps * Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as * it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the * published by the Free Software Foundation, either version 3 of the
@@ -17,11 +17,11 @@ import {Logger} from '@openstapps/logger';
import Dockerode from 'dockerode'; import Dockerode from 'dockerode';
import isCidr from 'is-cidr'; import isCidr from 'is-cidr';
import {render} from 'mustache'; import {render} from 'mustache';
import {join} from 'path'; import path from 'path';
import * as semver from 'semver'; import * as semver from 'semver';
import { import {
asyncReadFile, asyncReadFile,
configFile, ConfigFile,
isFileType, isFileType,
protocolHardeningParameters, protocolHardeningParameters,
SSLFilePaths, SSLFilePaths,
@@ -29,6 +29,12 @@ import {
TemplateView, TemplateView,
} from './common'; } from './common';
/* eslint-disable unicorn/prefer-module */
/* eslint-disable unicorn/no-await-expression-member */
// eslint-disable-next-line @typescript-eslint/no-var-requires
const nodePortScanner = require('node-port-scanner');
/** /**
* Checks if a ContainerInfo matches a name and version regex * Checks if a ContainerInfo matches a name and version regex
* *
@@ -36,11 +42,20 @@ import {
* @param versionRegex Version regex to check * @param versionRegex Version regex to check
* @param container Container info for check * @param container Container info for check
*/ */
export function containerMatchesRegex(name: string, versionRegex: RegExp, container: Dockerode.ContainerInfo): boolean { export function containerMatchesRegex(
return typeof container.Labels['stapps.version'] === 'string' name: string,
&& container.Labels['stapps.version'].match(versionRegex) !== null versionRegex: RegExp,
&& typeof container.Labels['com.docker.compose.service'] === 'string' container: Dockerode.ContainerInfo,
&& container.Labels['com.docker.compose.service'] === name; ): boolean {
return (
typeof container.Labels['stapps.version'] === 'string' &&
container.Labels['stapps.version'].match(versionRegex) !== null &&
((typeof container.Labels['com.docker.compose.service'] === 'string' &&
container.Labels['com.docker.compose.service'] === name) ||
(typeof container.Labels['com.docker.stack.namespace'] === 'string' &&
container.Labels['com.docker.swarm.service.name'] ===
`${container.Labels['com.docker.stack.namespace']}_${name}`))
);
} }
/** /**
@@ -53,14 +68,44 @@ export function containerMatchesRegex(name: string, versionRegex: RegExp, contai
*/ */
export async function getGatewayOfStAppsBackend(container: Dockerode.ContainerInfo): Promise<string> { export async function getGatewayOfStAppsBackend(container: Dockerode.ContainerInfo): Promise<string> {
if (container.Ports.length === 0) { if (container.Ports.length === 0) {
await Logger.error(`Container ${container.Id} does not advertise any port. await Logger.error(`Container ${container.Names[0]} does not advertise any port.
Please expose a port if the container should be accessible by NGINX.`); Please expose a port if the container should be accessible by NGINX.`);
return ''; return '';
} }
// ip:port // Basic Docker network
return `${container.Ports[0].IP}:${container.Ports[0].PublicPort}`; if (typeof container.Ports[0].IP !== 'undefined' && typeof container.Ports[0].PublicPort !== 'undefined') {
// ip:port
return `${container.Ports[0].IP}:${container.Ports[0].PublicPort}`;
}
// Docker Swarm network
if (
/* istanbul ignore next */
typeof container.NetworkSettings?.Networks?.ingress?.IPAddress !== 'undefined' &&
typeof container.Ports[0].PrivatePort !== 'undefined'
) {
const port = container.Ports[0].PrivatePort;
// Get a routable network connection
for (const network in container.NetworkSettings.Networks) {
const scan = await nodePortScanner(container.NetworkSettings.Networks[network].IPAddress, [port]);
if ((scan.ports.open as Array<number>).includes(port)) {
Logger.info(
`${container.Names[0]} reachable via ${container.NetworkSettings.Networks[network].IPAddress}:${port}`,
);
return `${container.NetworkSettings.Networks[network].IPAddress}:${port}`;
}
}
}
await Logger.error(
`Couldn't infer ${container.Names[0]} network reachability. It's possible your current Docker network setup isn't supported yet.`,
);
return '';
} }
/** /**
@@ -82,56 +127,71 @@ export async function generateUpstreamMap(
let foundMatchingContainer = false; let foundMatchingContainer = false;
// const backendContainer = containers.filter(container => container.Image.includes('backend'));
// // eslint-disable-next-line no-console
//console.log(JSON.stringify(backendContainer, undefined, 2));
// active versions // active versions
result += (await Promise.all( result += (
activeVersions await Promise.all(
.map(async (activeVersionRegex) => { activeVersions.map(async activeVersionRegex => {
const upstreamName = activeVersionRegex.replace(/[\\|.+]/g, '_'); const upstreamName = activeVersionRegex.replace(/[\\|.+]/g, '_');
let activeBackends = containers.filter((container) => { let activeBackends = containers.filter(container => {
return containerMatchesRegex('backend', new RegExp(activeVersionRegex), container); return containerMatchesRegex('backend', new RegExp(activeVersionRegex), container);
}); });
// .Labels['stapps.version'] is available // .Labels['stapps.version'] is available
if (activeBackends.length > 0) { if (activeBackends.length > 0) {
activeBackends = activeBackends.sort((a, b) =>
activeBackends = activeBackends.sort((a, b) => semver.rcompare(a.Labels['stapps.version'],b.Labels['stapps.version'])); semver.rcompare(a.Labels['stapps.version'], b.Labels['stapps.version']),
const activeBackendsVersions = activeBackends.map((container) => container.Labels['stapps.version']) );
// tslint:disable-next-line: strict-boolean-expressions const activeBackendsVersions = activeBackends
.reduce((map, e) => map.set(e, (map.get(e) || 0) + 1), new Map<string, number>()); .map(container => container.Labels['stapps.version'])
// eslint-disable-next-line unicorn/no-array-reduce
.reduce(
(map, element) => map.set(element, (map.get(element) || 0) + 1),
new Map<string, number>(),
);
for (const [version, occurrences] of activeBackendsVersions) { for (const [version, occurrences] of activeBackendsVersions) {
if (occurrences > 1) { if (occurrences > 1) {
await Logger.error(`Omitting running version ${version} ! Multiple backends with this exact version are running`); await Logger.error(
activeBackends = activeBackends.filter((container) => container.Labels['stapps.version'] !== version); `Omitting running version ${version} ! Multiple backends with this exact version are running.`,
);
activeBackends = activeBackends.filter(
container => container.Labels['stapps.version'] !== version,
);
} }
} }
if (activeBackends.length !== 0) { if (activeBackends.length > 0) {
// not only dublicates // not only duplicates
foundMatchingContainer = true; foundMatchingContainer = true;
const gateWayOfContainer = await getGatewayOfStAppsBackend(activeBackends[0]); const gatewayOfContainer = await getGatewayOfStAppsBackend(activeBackends[0]);
if (gateWayOfContainer.length !== 0) { if (gatewayOfContainer.length > 0) {
upstreams += `\nupstream ${upstreamName} {\n server ${gateWayOfContainer};\n}`; upstreams += `\nupstream ${upstreamName} {\n server ${gatewayOfContainer};\n}`;
return ` \"~${activeVersionRegex}\" ${upstreamName};\n`; return ` \"~${activeVersionRegex}\" ${upstreamName};\n`;
} }
} }
} }
await Logger.error('No backend for version', activeVersionRegex, 'found'); Logger.warn('No backend for version', activeVersionRegex, 'found');
return ` \"~${activeVersionRegex}\" unavailable;\n`; return ` \"~${activeVersionRegex}\" unavailable;\n`;
}), }),
)).join(''); )
).join('');
// outdated versions // outdated versions
result += outdatedVersions result += outdatedVersions
.map((outdatedVersionRegex) => { .map(outdatedVersionRegex => {
return ` \"~${outdatedVersionRegex}\" outdated;`; return ` \"~${outdatedVersionRegex}\" outdated;`;
}) })
.join(''); .join('\n');
// eslint-disable-next-line prettier/prettier
result += '\n\}'; result += '\n\}';
if (!foundMatchingContainer) { if (!foundMatchingContainer) {
@@ -149,19 +209,24 @@ export async function generateUpstreamMap(
export function generateListener(sslFilePaths: SSLFilePaths) { export function generateListener(sslFilePaths: SSLFilePaths) {
let listener = ''; let listener = '';
if (typeof sslFilePaths !== 'undefined' && if (
typeof sslFilePaths.certificate !== 'undefined' && isFileType(sslFilePaths.certificate,'crt') && typeof sslFilePaths !== 'undefined' &&
typeof sslFilePaths.certificateChain !== 'undefined' && isFileType(sslFilePaths.certificateChain,'crt') && typeof sslFilePaths.certificate !== 'undefined' &&
typeof sslFilePaths.certificateKey !== 'undefined' && isFileType(sslFilePaths.certificateKey,'key') && isFileType(sslFilePaths.certificate, 'crt') &&
typeof sslFilePaths.dhparam !== 'undefined' && isFileType(sslFilePaths.dhparam,'pem') typeof sslFilePaths.certificateChain !== 'undefined' &&
isFileType(sslFilePaths.certificateChain, 'crt') &&
typeof sslFilePaths.certificateKey !== 'undefined' &&
isFileType(sslFilePaths.certificateKey, 'key') &&
typeof sslFilePaths.dhparam !== 'undefined' &&
isFileType(sslFilePaths.dhparam, 'pem')
) { ) {
// https listener // https listener
listener = `listen 443 ssl default_server; listener = ` listen 443 ssl default_server;
ssl_certificate ${sslFilePaths.certificate}; ssl_certificate ${sslFilePaths.certificate};
ssl_certificate_key ${sslFilePaths.certificateKey}; ssl_certificate_key ${sslFilePaths.certificateKey};
ssl_trusted_certificate ${sslFilePaths.certificateChain}; ssl_trusted_certificate ${sslFilePaths.certificateChain};
ssl_dhparam ${sslFilePaths.dhparam}; ssl_dhparam ${sslFilePaths.dhparam};
${sslHardeningParameters}`; ${sslHardeningParameters}`;
} else { } else {
// default http listener // default http listener
listener = 'listen 80 default_server;'; listener = 'listen 80 default_server;';
@@ -176,6 +241,16 @@ ${protocolHardeningParameters}
return listener; return listener;
} }
/**
* Reads predefined server entry with metrics location
*/
export async function generateMetricsServer(enableMetrics: boolean): Promise<string> {
if (!enableMetrics) {
return '';
}
return asyncReadFile('./fixtures/metrics.template', 'utf8');
}
/** /**
* Render a mustache template file with given view object * Render a mustache template file with given view object
* *
@@ -194,7 +269,8 @@ async function renderTemplate(path: string, view: unknown): Promise<string> {
* @param entries Allow list entries that should be in CIDR notation * @param entries Allow list entries that should be in CIDR notation
*/ */
function generateRateLimitAllowList(entries: string[]): string { function generateRateLimitAllowList(entries: string[]): string {
return entries.filter(entry => isCidr(entry)) return entries
.filter(entry => isCidr(entry))
.map(entry => `${entry} 0;`) .map(entry => `${entry} 0;`)
.join('\n'); .join('\n');
} }
@@ -205,28 +281,38 @@ function generateRateLimitAllowList(entries: string[]): string {
* @param containers List of container info * @param containers List of container info
*/ */
export async function getTemplateView(containers: Dockerode.ContainerInfo[]): Promise<TemplateView> { export async function getTemplateView(containers: Dockerode.ContainerInfo[]): Promise<TemplateView> {
delete require.cache[require.resolve('config')];
// eslint-disable-next-line @typescript-eslint/no-var-requires
const config = require('config');
const configFile = config as ConfigFile;
const cors = await asyncReadFile('./fixtures/cors.template', 'utf8'); const cors = await asyncReadFile('./fixtures/cors.template', 'utf8');
const visibleRoutesPromises = ['/'].map(async (route) => { const visibleRoutesPromises = ['/'].map(async route => {
return renderTemplate(join('fixtures', 'visibleRoute.template'), { return renderTemplate(path.join('fixtures', 'visibleRoute.template'), {
cors, cors,
route, route,
}); });
}); });
const hiddenRoutesPromises = configFile.hiddenRoutes.map(async (route) => { const hiddenRoutesPromises = configFile.hiddenRoutes.map(async route => {
return renderTemplate(join('fixtures', 'hiddenRoute.template'), { return renderTemplate(path.join('fixtures', 'hiddenRoute.template'), {
cors, cors,
route, route,
}); });
}); });
return { return {
dockerVersionMap: await generateUpstreamMap(configFile.activeVersions, configFile.outdatedVersions, containers), dockerVersionMap: await generateUpstreamMap(
configFile.activeVersions,
configFile.outdatedVersions,
containers,
),
hiddenRoutes: (await Promise.all(hiddenRoutesPromises)).join(''), hiddenRoutes: (await Promise.all(hiddenRoutesPromises)).join(''),
listener: generateListener(configFile.sslFilePaths), listener: generateListener(configFile.sslFilePaths),
metrics: await generateMetricsServer(configFile.metrics),
rateLimitAllowList: generateRateLimitAllowList(configFile.rateLimitAllowList), rateLimitAllowList: generateRateLimitAllowList(configFile.rateLimitAllowList),
staticRoute: await renderTemplate(join('fixtures', 'staticRoute.template'), {cors}), staticRoute: await renderTemplate(path.join('fixtures', 'staticRoute.template'), {cors}),
visibleRoutes: (await Promise.all(visibleRoutesPromises)).join(''), visibleRoutes: (await Promise.all(visibleRoutesPromises)).join(''),
}; };
} }
@@ -236,7 +322,9 @@ export async function getTemplateView(containers: Dockerode.ContainerInfo[]): Pr
* *
* @param pathToDockerSocket Path to docker socket * @param pathToDockerSocket Path to docker socket
*/ */
export async function getContainers(pathToDockerSocket = '/var/run/docker.sock'): Promise<Dockerode.ContainerInfo[]> { export async function getContainers(
pathToDockerSocket = '/var/run/docker.sock',
): Promise<Dockerode.ContainerInfo[]> {
const docker = new Dockerode({ const docker = new Dockerode({
socketPath: pathToDockerSocket, socketPath: pathToDockerSocket,
}); });

View File

@@ -24,12 +24,23 @@ import {expect} from 'chai';
import chaiSpies from 'chai-spies'; import chaiSpies from 'chai-spies';
import {ContainerInfo} from 'dockerode'; import {ContainerInfo} from 'dockerode';
import {slow, suite, test, timeout} from '@testdeck/mocha'; import {slow, suite, test, timeout} from '@testdeck/mocha';
import {sslHardeningParameters, protocolHardeningParameters, SSLFilePaths } from './../src/common'; import {sslHardeningParameters, protocolHardeningParameters, SSLFilePaths} from './../src/common';
import {containerMatchesRegex, generateUpstreamMap, getGatewayOfStAppsBackend, getTemplateView, generateListener, getContainers} from '../src/main'; import {
import { resolve } from 'path'; containerMatchesRegex,
import { mkdirSync, writeFileSync, unlinkSync, rmdirSync } from 'fs'; generateUpstreamMap,
getGatewayOfStAppsBackend,
getTemplateView,
generateListener,
generateMetricsServer,
getContainers,
} from '../src/main';
import {resolve} from 'path';
import {mkdirSync, writeFileSync, unlinkSync, rmdirSync} from 'fs';
import proxyquire from 'proxyquire';
process.on('unhandledRejection', async (error) => { proxyquire.callThru().preserveCache();
process.on('unhandledRejection', async error => {
await Logger.error(error); await Logger.error(error);
process.exit(1); process.exit(1);
@@ -40,9 +51,9 @@ chai.use(chaiSpies);
@suite(timeout(1000), slow(500)) @suite(timeout(1000), slow(500))
export class MainSpec { export class MainSpec {
static anyContainerWithExposedPorts: ContainerInfo = { static 'anyContainerWithExposedPorts': ContainerInfo = {
Command: 'sh', Command: 'sh',
Created: 1524669882, Created: 1_524_669_882,
HostConfig: { HostConfig: {
NetworkMode: 'default', NetworkMode: 'default',
}, },
@@ -51,9 +62,7 @@ export class MainSpec {
ImageID: 'sha256:ef9f0c8c4b6f99dd208948c7aae1d042590aa18e05ebeae4f586e4b4beebeac9', ImageID: 'sha256:ef9f0c8c4b6f99dd208948c7aae1d042590aa18e05ebeae4f586e4b4beebeac9',
Labels: {}, Labels: {},
Mounts: [], Mounts: [],
Names: [ Names: ['/container_name_1'],
'/container_name_1',
],
NetworkSettings: { NetworkSettings: {
Networks: { Networks: {
bridge: { bridge: {
@@ -84,7 +93,7 @@ export class MainSpec {
Status: 'Up 3 minutes', Status: 'Up 3 minutes',
}; };
static backendContainerWithExposedPorts: ContainerInfo = { static 'backendContainerWithExposedPorts': ContainerInfo = {
Command: 'node ./bin/www', Command: 'node ./bin/www',
Created: 1524669882, Created: 1524669882,
HostConfig: { HostConfig: {
@@ -103,9 +112,7 @@ export class MainSpec {
'stapps.version': '1.0.0', 'stapps.version': '1.0.0',
}, },
Mounts: [], Mounts: [],
Names: [ Names: ['/deployment_backend_1'],
'/deployment_backend_1',
],
NetworkSettings: { NetworkSettings: {
Networks: { Networks: {
deployment_default: { deployment_default: {
@@ -136,36 +143,87 @@ export class MainSpec {
Status: 'Up 3 minutes', Status: 'Up 3 minutes',
}; };
static sandbox = chai.spy.sandbox(); static 'swarmBackendContainerWithExposedPorts': ContainerInfo = {
Command: 'node ./bin/www',
Created: 1524669882,
HostConfig: {
NetworkMode: 'swarm_default',
},
Id: 'e3d3f4d18aceac2780bdb95523845d066ed25c04fc65168a5ddbd37a85671bb7',
Image: 'registry.gitlab.com/openstapps/backend/b-tu-typescript-refactor-for-new-tslint-config',
ImageID: 'sha256:ef9f0c8c4b6f99dd208948c7aae1d042590aa18e05ebeae4f586e4b4beebeac9',
Labels: {
'com.docker.compose.config-hash': '91c6e0cebad15951824162c93392b6880b69599692f07798ae8de659c1616a03',
'com.docker.compose.container-number': '1',
'com.docker.compose.oneoff': 'False',
'com.docker.stack.namespace': 'deployment',
'com.docker.swarm.service.name': 'deployment_backend',
'com.docker.compose.version': '1.21.0',
'stapps.version': '1.0.0',
},
Mounts: [],
Names: ['/deployment_backend_1'],
NetworkSettings: {
Networks: {
ingress: {
Aliases: null,
EndpointID: 'da17549a086ff2c9f622e80de833e6f334afda52c8f07080428640c1716dcd14',
Gateway: '172.18.0.1',
GlobalIPv6Address: '',
GlobalIPv6PrefixLen: 0,
IPAMConfig: null,
IPAddress: '172.18.0.3',
IPPrefixLen: 16,
IPv6Gateway: '',
Links: null,
MacAddress: '03:41:ac:11:00:23',
NetworkID: '947ea5247cc7429e1fdebd5404fa4d15f7c05e6765f2b93ddb3bdb6aaffd1193',
},
},
},
Ports: [
{
IP: 'delete me',
PrivatePort: 3000,
PublicPort: 3000,
Type: 'tcp',
},
],
State: 'running',
Status: 'Up 3 minutes',
};
before() { static 'sandbox' = chai.spy.sandbox();
'before'() {
MainSpec.sandbox.restore(); MainSpec.sandbox.restore();
} }
@test @test
'check if container does not match any container'() { 'check if container does not match any container'() {
expect(containerMatchesRegex( expect(
'anyName', containerMatchesRegex('anyName', new RegExp('d+'), MainSpec.anyContainerWithExposedPorts),
new RegExp('d+'),
MainSpec.anyContainerWithExposedPorts),
).to.be.equal(false); ).to.be.equal(false);
} }
@test @test
'check if container does not match if version is incorrect'() { 'check if container does not match if version is incorrect'() {
expect(containerMatchesRegex( expect(
'backend', containerMatchesRegex('backend', new RegExp('1\\.4\\.\\d+'), MainSpec.backendContainerWithExposedPorts),
new RegExp('1\\.4\\.\\d+'),
MainSpec.backendContainerWithExposedPorts),
).to.be.equal(false); ).to.be.equal(false);
} }
@test @test
'check if container matches'() { 'check if container matches'() {
expect(containerMatchesRegex( expect(
'backend', containerMatchesRegex('backend', new RegExp('1\\.0\\.\\d+'), MainSpec.backendContainerWithExposedPorts),
new RegExp('1\\.0\\.\\d+'), ).to.be.equal(true);
MainSpec.backendContainerWithExposedPorts), expect(
containerMatchesRegex(
'backend',
new RegExp('1\\.0\\.\\d+'),
MainSpec.swarmBackendContainerWithExposedPorts,
),
).to.be.equal(true); ).to.be.equal(true);
} }
@@ -183,27 +241,98 @@ export class MainSpec {
const containerWithoutPorts: Partial<ContainerInfo> = { const containerWithoutPorts: Partial<ContainerInfo> = {
Id: 'Foo', Id: 'Foo',
Ports: [], Ports: [],
Names: ['/container_name_1'],
}; };
expect(await getGatewayOfStAppsBackend(containerWithoutPorts as ContainerInfo)).to.be.equal(''); expect(await getGatewayOfStAppsBackend(containerWithoutPorts as ContainerInfo)).to.be.equal('');
expect(spy.__spy.calls[0][0]).to.contain('Container Foo does not advertise any port.'); expect(spy.__spy.calls[0][0]).to.contain('Container /container_name_1 does not advertise any port.');
} }
@test @test
async 'get gateway of backend container without ports'() { async 'get gateway of backend container without ports'() {
expect(await getGatewayOfStAppsBackend(MainSpec.backendContainerWithExposedPorts)).to.be.equal('127.0.0.1:3000'); expect(await getGatewayOfStAppsBackend(MainSpec.backendContainerWithExposedPorts)).to.be.equal(
'127.0.0.1:3000',
);
}
@test
async 'get gateway of backend container within docker swarm'() {
const backendContainer = MainSpec.swarmBackendContainerWithExposedPorts as any;
delete backendContainer.Ports[0].IP;
const main = proxyquire('../src/main', {
'node-port-scanner': (_host: unknown, _ports: unknown) => {
return new Promise((resolve, _reject) => {
resolve({
ports: {
open: [3000],
},
});
});
},
});
expect(await main.getGatewayOfStAppsBackend(backendContainer)).to.be.equal('172.18.0.3:3000');
}
@test
async 'fail to get gateway of backend container if unreachable'() {
const backendContainer = MainSpec.swarmBackendContainerWithExposedPorts as any;
delete backendContainer.Ports[0].IP;
const spy = MainSpec.sandbox.on(console, 'error', () => {
// noop
});
const main = proxyquire('../src/main', {
'node-port-scanner': (_host: unknown, _ports: unknown) => {
return new Promise((resolve, _reject) => {
resolve({
ports: {
open: [],
},
});
});
},
});
expect(await main.getGatewayOfStAppsBackend(MainSpec.swarmBackendContainerWithExposedPorts)).to.be.equal(
'',
);
expect(spy.__spy.calls[0][0]).to.contain(
"It's possible your current Docker network setup isn't supported yet.",
);
}
@test
async 'fail to get gateway of backend container network mode is unsupported'() {
const backendContainer = MainSpec.swarmBackendContainerWithExposedPorts as any;
delete backendContainer.Ports[0].IP;
delete backendContainer.Ports[0].PublicPort;
delete backendContainer.Ports[0].PrivatePort;
const spy = MainSpec.sandbox.on(console, 'error', () => {
// noop
});
expect(await getGatewayOfStAppsBackend(MainSpec.swarmBackendContainerWithExposedPorts)).to.be.equal('');
expect(spy.__spy.calls[0][0]).to.contain(
"It's possible your current Docker network setup isn't supported yet.",
);
} }
@test @test
async 'upstream map calls logger error when no matching container is found'() { async 'upstream map calls logger error when no matching container is found'() {
const spy = MainSpec.sandbox.on(console, 'error', () => { const spy = MainSpec.sandbox.on(console, 'warn', () => {
// noop
}); });
expect(await generateUpstreamMap( expect(
['0\\.8\\.\\d+'], await generateUpstreamMap(
['1\\.1\\.\\d+'], ['0\\.8\\.\\d+'],
[MainSpec.backendContainerWithExposedPorts], ['1\\.1\\.\\d+'],
)).to.be.equal(`map $http_x_stapps_version $proxyurl { [MainSpec.backendContainerWithExposedPorts],
),
).to.be.equal(`map $http_x_stapps_version $proxyurl {
default unsupported; default unsupported;
"~0\\.8\\.\\d+" unavailable; "~0\\.8\\.\\d+" unavailable;
"~1\\.1\\.\\d+" outdated; "~1\\.1\\.\\d+" outdated;
@@ -215,11 +344,13 @@ export class MainSpec {
@test @test
async 'upstream map with one active version and no outdated ones'() { async 'upstream map with one active version and no outdated ones'() {
expect(await generateUpstreamMap( expect(
['1\\.0\\.\\d+'], await generateUpstreamMap(
['0\\.8\\.\\d+'], ['1\\.0\\.\\d+'],
[MainSpec.backendContainerWithExposedPorts], ['0\\.8\\.\\d+'],
)).to.be.equal(`map $http_x_stapps_version $proxyurl { [MainSpec.backendContainerWithExposedPorts],
),
).to.be.equal(`map $http_x_stapps_version $proxyurl {
default unsupported; default unsupported;
"~1\\.0\\.\\d+" 1__0___d_; "~1\\.0\\.\\d+" 1__0___d_;
"~0\\.8\\.\\d+" outdated; "~0\\.8\\.\\d+" outdated;
@@ -255,33 +386,43 @@ Please check if docker is running and Node.js can access the docker socket (/var
@test @test
async 'get template view'() { async 'get template view'() {
try { try {
let containersWithSameVersion = [MainSpec.backendContainerWithExposedPorts, MainSpec.backendContainerWithExposedPorts]; let containersWithSameVersion = [
MainSpec.backendContainerWithExposedPorts,
MainSpec.backendContainerWithExposedPorts,
];
await getTemplateView(containersWithSameVersion); await getTemplateView(containersWithSameVersion);
return false; return false;
} catch (e) { } catch (e) {
expect((e as Error).message).to.equal( expect((e as Error).message).to.equal(`Multiple backends for one version found.`);
`Multiple backends for one version found.`);
} }
return true; return true;
} }
@test @test
'create listener faulty config'() { async 'include metrics config'() {
expect(await generateMetricsServer(true)).length.to.be.greaterThan(1);
}
expect(generateListener({ @test
certificate: 'faultyTest', async 'omit metrics config'() {
certificateChain: 'faultyTest', expect(await generateMetricsServer(false)).to.equal('');
certificateKey: 'faultyTest', }
dhparam: 'faultyTest',
})).to @test
.equal(`listen 80 default_server; 'create listener faulty config'() {
expect(
generateListener({
certificate: 'faultyTest',
certificateChain: 'faultyTest',
certificateKey: 'faultyTest',
dhparam: 'faultyTest',
}),
).to.equal(`listen 80 default_server;
${protocolHardeningParameters} ${protocolHardeningParameters}
`); `);
} }
@test @test
@@ -294,10 +435,10 @@ ${protocolHardeningParameters}
const certificateChainFile = resolve(testCertDir, 'chain.crt'); const certificateChainFile = resolve(testCertDir, 'chain.crt');
const dhparamFile = resolve(testCertDir, 'dhparam.pem'); const dhparamFile = resolve(testCertDir, 'dhparam.pem');
writeFileSync(certificateFile,'Test'); writeFileSync(certificateFile, 'Test');
writeFileSync(certificateKeyFile,'Test'); writeFileSync(certificateKeyFile, 'Test');
writeFileSync(certificateChainFile,'Test'); writeFileSync(certificateChainFile, 'Test');
writeFileSync(dhparamFile,'Test'); writeFileSync(dhparamFile, 'Test');
const sslFilePaths: SSLFilePaths = { const sslFilePaths: SSLFilePaths = {
certificate: certificateFile, certificate: certificateFile,
@@ -306,12 +447,12 @@ ${protocolHardeningParameters}
dhparam: dhparamFile, dhparam: dhparamFile,
}; };
expect(generateListener(sslFilePaths)).to.equal(`listen 443 ssl default_server; expect(generateListener(sslFilePaths)).to.equal(` listen 443 ssl default_server;
ssl_certificate ${sslFilePaths.certificate}; ssl_certificate ${sslFilePaths.certificate};
ssl_certificate_key ${sslFilePaths.certificateKey}; ssl_certificate_key ${sslFilePaths.certificateKey};
ssl_trusted_certificate ${sslFilePaths.certificateChain}; ssl_trusted_certificate ${sslFilePaths.certificateChain};
ssl_dhparam ${sslFilePaths.dhparam}; ssl_dhparam ${sslFilePaths.dhparam};
${sslHardeningParameters} ${sslHardeningParameters}
${protocolHardeningParameters} ${protocolHardeningParameters}
`); `);

View File

@@ -1,3 +0,0 @@
{
"extends": "./node_modules/@openstapps/configuration/tslint.json"
}