refactor: move proxy to monorepo

This commit is contained in:
2023-03-14 17:13:57 +01:00
parent 49d6db178c
commit ae053147d4
34 changed files with 0 additions and 0 deletions

83
backend/proxy/src/cli.ts Normal file
View File

@@ -0,0 +1,83 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Logger} from '@openstapps/logger';
import {execSync} from 'child_process';
import * as Dockerode from 'dockerode';
import {render} from 'mustache';
import {asyncReadFile, asyncWriteFile} from './common';
import {getContainers, getTemplateView} from './main';
/* eslint-disable unicorn/prefer-module */
// handle unhandled promise rejections
process.on('unhandledRejection', async error => {
await Logger.error(error);
process.exit(1);
});
let containerHashCache = '';
let configHashCache = '';
const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
/**
* Reads the container information from the docker socket and updates the nginx config if necessary
*/
async function updateNginxConfig() {
const containers = await getContainers();
const containerHash = containers
.map((container: Dockerode.ContainerInfo) => {
return container.Id;
})
.join(',');
delete require.cache[require.resolve('config')];
// eslint-disable-next-line @typescript-eslint/no-var-requires
const configFile = require('config');
const configHash = JSON.stringify(configFile);
// if containers changed -> write config file, reload nginx
if (containerHash !== containerHashCache || configHash !== configHashCache) {
Logger.log('Generating new NGINX configuration');
Logger.log('Waiting for Docker network to settle...');
await delay(10_000);
// render nginx config file
const nginxConfig = render(
await asyncReadFile('nginx.conf.template', 'utf8'),
await getTemplateView(containers),
);
containerHashCache = containerHash;
configHashCache = configHash;
Logger.log(`Writing new config file "${configFile.output}"`);
// overwrite nginx config file with our rendered one
await asyncWriteFile(configFile.output, nginxConfig, 'utf8');
Logger.log('Executing "nginx -s reload" to tell nginx to reload the configuration file');
execSync('nginx -s reload');
}
// set timeout to update configuration again in 30s
setTimeout(updateNginxConfig, 30_000);
}
// start the process that checks the docker socket periodically
// eslint-disable-next-line unicorn/prefer-top-level-await
updateNginxConfig();

180
backend/proxy/src/common.ts Normal file
View File

@@ -0,0 +1,180 @@
/*
* Copyright (C) 2019 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Logger} from '@openstapps/logger';
import {SMTP} from '@openstapps/logger/lib/smtp';
import config from 'config';
import {existsSync, readFile, writeFile} from 'fs';
import {promisify} from 'util';
// set transport on logger
Logger.setTransport(SMTP.getInstance());
export const asyncReadFile = promisify(readFile);
export const asyncWriteFile = promisify(writeFile);
/**
* A representation of the file paths of the needed ssl certificates
*/
export interface SSLFilePaths {
/**
* Path to SSL certificate
*/
certificate: string;
/**
* Path to SSL certificate chain
*/
certificateChain: string;
/**
* Path to SSL certificate key
*/
certificateKey: string;
/**
* Path to SSL DHParam
*/
dhparam: string;
}
/**
* Supported log formats for config
*/
type SupportedLogFormatsKeys = 'default' | 'combined' | 'json';
/**
* Map supported formats to stings used in template view
*/
export const SupportedLogFormats: {[key in SupportedLogFormatsKeys]: string} = {
default: 'combined',
combined: 'combined',
json: 'json',
};
/**
* A representation of the config file
*/
export interface ConfigFile {
/**
* List of active version
*/
activeVersions: string[];
/**
* List of hidden routes
*/
hiddenRoutes: string[];
/**
* Sets log format (default or json)
*/
logFormat: SupportedLogFormatsKeys;
/**
* Enables metrics on /metrics route
*/
metrics?: boolean;
/**
* List of outdated versions
*/
outdatedVersions: string[];
/**
* Path the generated config will be written to
*/
output: string;
/**
* Allow list for rate limiting
*/
rateLimitAllowList: string[];
/**
* SSL file paths
*/
sslFilePaths: SSLFilePaths;
}
/**
* A view object to render the nginx config template
*/
export interface TemplateView {
/**
* Docker version map
*/
dockerVersionMap: string;
/**
* Hidden routes
*/
hiddenRoutes: string;
/**
* Listener
*/
listener: string;
/**
* Log format to use
*/
logFormat: string;
/**
* Custom Log formatters
*/
logFormatters: string;
/**
* Local server with listener for /metrics route
*/
metrics: string;
/**
* Allow list for rate limiting
*/
rateLimitAllowList: string;
/**
* Static route
*/
staticRoute: string;
/**
* Visible routes
*/
visibleRoutes: string;
}
/**
* Nginx protocol parameters to harden serverside settings
*/
export const protocolHardeningParameters = `
add_header Strict-Transport-Security "max-age=63072000; includeSubDomains;";
add_header X-Frame-Options DENY;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";`;
// tslint:disable:max-line-length
/**
* Nginx ssl parameters to harden serverside settings
*/
export const sslHardeningParameters = `
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
ssl_prefer_server_ciphers on;
ssl_ecdh_curve X25519:secp384r1;
ssl_session_timeout 10m;
ssl_session_cache shared:SSL:10m;
ssl_session_tickets off;
ssl_stapling on;
ssl_stapling_verify on;`;
/**
* Config file
*/
export const configFile: ConfigFile = config.util.toObject();
/**
* Check if path is a specific file type
*/
export function isFileType(path: string, fileType: string) {
const regExp = new RegExp(`.*\.${fileType}$`);
return existsSync(path) && regExp.test(path);
}

358
backend/proxy/src/main.ts Normal file
View File

@@ -0,0 +1,358 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Logger} from '@openstapps/logger';
import Dockerode from 'dockerode';
import isCidr from 'is-cidr';
import {render} from 'mustache';
import path from 'path';
import * as semver from 'semver';
import {
asyncReadFile,
ConfigFile,
isFileType,
protocolHardeningParameters,
SSLFilePaths,
sslHardeningParameters,
SupportedLogFormats,
TemplateView,
} from './common';
/* eslint-disable unicorn/prefer-module */
/* eslint-disable unicorn/no-await-expression-member */
// eslint-disable-next-line @typescript-eslint/no-var-requires
const nodePortScanner = require('node-port-scanner');
/**
* Checks if a ContainerInfo matches a name and version regex
*
* @param name Name to check
* @param versionRegex Version regex to check
* @param container Container info for check
*/
export function containerMatchesRegex(
name: string,
versionRegex: RegExp,
container: Dockerode.ContainerInfo,
): boolean {
return (
typeof container.Labels['stapps.version'] === 'string' &&
container.Labels['stapps.version'].match(versionRegex) !== null &&
((typeof container.Labels['com.docker.compose.service'] === 'string' &&
container.Labels['com.docker.compose.service'] === name) ||
(typeof container.Labels['com.docker.stack.namespace'] === 'string' &&
container.Labels['com.docker.swarm.service.name'] ===
`${container.Labels['com.docker.stack.namespace']}_${name}`))
);
}
/**
* Gets Gateway (ip:port) of given ContainerInfo
*
* Returns an empty String if there is no Gateway.
* This assumes that a backend runs in the container and it exposes one port.
*
* @param container Container info of which to get address from
*/
export async function getGatewayOfStAppsBackend(container: Dockerode.ContainerInfo): Promise<string> {
if (container.Ports.length === 0) {
await Logger.error(`Container ${container.Names[0]} does not advertise any port.
Please expose a port if the container should be accessible by NGINX.`);
return '';
}
// Basic Docker network
if (typeof container.Ports[0].IP !== 'undefined' && typeof container.Ports[0].PublicPort !== 'undefined') {
// ip:port
return `${container.Ports[0].IP}:${container.Ports[0].PublicPort}`;
}
// Docker Swarm network
if (
/* istanbul ignore next */
typeof container.NetworkSettings?.Networks?.ingress?.IPAddress !== 'undefined' &&
typeof container.Ports[0].PrivatePort !== 'undefined'
) {
const port = container.Ports[0].PrivatePort;
// Get a routable network connection
for (const network in container.NetworkSettings.Networks) {
const scan = await nodePortScanner(container.NetworkSettings.Networks[network].IPAddress, [port]);
if ((scan.ports.open as Array<number>).includes(port)) {
Logger.info(
`${container.Names[0]} reachable via ${container.NetworkSettings.Networks[network].IPAddress}:${port}`,
);
return `${container.NetworkSettings.Networks[network].IPAddress}:${port}`;
}
}
}
await Logger.error(
`Couldn't infer ${container.Names[0]} network reachability. It's possible your current Docker network setup isn't supported yet.`,
);
return '';
}
/**
* Generates an upstream map
*
* It maps all stapps-backend-containers to an gateway.
*
* @param activeVersions List of active versions
* @param outdatedVersions List of outdated versions
* @param containers List of container infos
*/
export async function generateUpstreamMap(
activeVersions: string[],
outdatedVersions: string[],
containers: Dockerode.ContainerInfo[],
): Promise<string> {
let result = 'map $http_x_stapps_version $proxyurl {\n default unsupported;\n';
let upstreams = '';
let foundMatchingContainer = false;
// const backendContainer = containers.filter(container => container.Image.includes('backend'));
// // eslint-disable-next-line no-console
//console.log(JSON.stringify(backendContainer, undefined, 2));
// active versions
result += (
await Promise.all(
activeVersions.map(async activeVersionRegex => {
const upstreamName = activeVersionRegex.replace(/[\\|.+]/g, '_');
let activeBackends = containers.filter(container => {
return containerMatchesRegex('backend', new RegExp(activeVersionRegex), container);
});
// .Labels['stapps.version'] is available
if (activeBackends.length > 0) {
activeBackends = activeBackends.sort((a, b) =>
semver.rcompare(a.Labels['stapps.version'], b.Labels['stapps.version']),
);
const activeBackendsVersions = activeBackends
.map(container => container.Labels['stapps.version'])
// eslint-disable-next-line unicorn/no-array-reduce
.reduce(
(map, element) => map.set(element, (map.get(element) || 0) + 1),
new Map<string, number>(),
);
for (const [version, occurrences] of activeBackendsVersions) {
if (occurrences > 1) {
await Logger.error(
`Omitting running version ${version} ! Multiple backends with this exact version are running.`,
);
activeBackends = activeBackends.filter(
container => container.Labels['stapps.version'] !== version,
);
}
}
if (activeBackends.length > 0) {
// not only duplicates
foundMatchingContainer = true;
const gatewayOfContainer = await getGatewayOfStAppsBackend(activeBackends[0]);
if (gatewayOfContainer.length > 0) {
upstreams += `\nupstream ${upstreamName} {\n server ${gatewayOfContainer};\n}`;
return ` \"~${activeVersionRegex}\" ${upstreamName};\n`;
}
}
}
Logger.warn('No backend for version', activeVersionRegex, 'found');
return ` \"~${activeVersionRegex}\" unavailable;\n`;
}),
)
).join('');
// outdated versions
result += outdatedVersions
.map(outdatedVersionRegex => {
return ` \"~${outdatedVersionRegex}\" outdated;`;
})
.join('\n');
// eslint-disable-next-line prettier/prettier
result += '\n\}';
if (!foundMatchingContainer) {
await Logger.error(
'No container with matching version label found. Please start a container with a matching version Label.',
);
}
return `${result}${upstreams}\n`;
}
/**
* Generates http or https listener
*/
export function generateListener(sslFilePaths: SSLFilePaths) {
let listener = '';
if (
typeof sslFilePaths !== 'undefined' &&
typeof sslFilePaths.certificate !== 'undefined' &&
isFileType(sslFilePaths.certificate, 'crt') &&
typeof sslFilePaths.certificateChain !== 'undefined' &&
isFileType(sslFilePaths.certificateChain, 'crt') &&
typeof sslFilePaths.certificateKey !== 'undefined' &&
isFileType(sslFilePaths.certificateKey, 'key') &&
typeof sslFilePaths.dhparam !== 'undefined' &&
isFileType(sslFilePaths.dhparam, 'pem')
) {
// https listener
listener = ` listen 443 ssl default_server;
ssl_certificate ${sslFilePaths.certificate};
ssl_certificate_key ${sslFilePaths.certificateKey};
ssl_trusted_certificate ${sslFilePaths.certificateChain};
ssl_dhparam ${sslFilePaths.dhparam};
${sslHardeningParameters}`;
} else {
// default http listener
listener = 'listen 80 default_server;';
Logger.warn('Https usage is not setup properly, falling back to http!');
}
listener = `${listener}
${protocolHardeningParameters}
`;
return listener;
}
/**
* Reads predefined server entry with metrics location
*/
export async function generateMetricsServer(logFormat: string, enableMetrics?: boolean): Promise<string> {
if (!enableMetrics) {
return '';
}
return renderTemplate(path.join('fixtures', 'metrics.template'), {
logFormat: logFormat,
});
}
/**
* Render a mustache template file with given view object
*
* @param path Path to template file
* @param view Data to render template with
*/
async function renderTemplate(path: string, view: unknown): Promise<string> {
const content = await asyncReadFile(path, 'utf8');
return render(content, view);
}
/**
* Generate allow list entries in CIDR notation that pass thru rate limiting
*
* @param entries Allow list entries that should be in CIDR notation
*/
function generateRateLimitAllowList(entries: string[]): string {
return entries
.filter(entry => isCidr(entry))
.map(entry => `${entry} 0;`)
.join('\n');
}
/**
* Returns view for nginx config file
*
* @param containers List of container info
*/
export async function getTemplateView(containers: Dockerode.ContainerInfo[]): Promise<TemplateView> {
delete require.cache[require.resolve('config')];
// eslint-disable-next-line @typescript-eslint/no-var-requires
const config = require('config');
const configFile = config as ConfigFile;
const cors = await asyncReadFile('./fixtures/cors.template', 'utf8');
const visibleRoutesPromises = ['/'].map(async route => {
return renderTemplate(path.join('fixtures', 'visibleRoute.template'), {
cors,
route,
});
});
const hiddenRoutesPromises = configFile.hiddenRoutes.map(async route => {
return renderTemplate(path.join('fixtures', 'hiddenRoute.template'), {
cors,
route,
});
});
const logFormattingPromise = renderTemplate(path.join('fixtures', 'logFormatters.template'), {});
const logFormat =
configFile.logFormat in SupportedLogFormats
? SupportedLogFormats[configFile.logFormat]
: SupportedLogFormats.default;
return {
dockerVersionMap: await generateUpstreamMap(
configFile.activeVersions,
configFile.outdatedVersions,
containers,
),
hiddenRoutes: (await Promise.all(hiddenRoutesPromises)).join(''),
listener: generateListener(configFile.sslFilePaths),
logFormat: logFormat,
logFormatters: await logFormattingPromise,
metrics: await generateMetricsServer(logFormat, configFile.metrics),
rateLimitAllowList: generateRateLimitAllowList(configFile.rateLimitAllowList),
staticRoute: await renderTemplate(path.join('fixtures', 'staticRoute.template'), {cors}),
visibleRoutes: (await Promise.all(visibleRoutesPromises)).join(''),
};
}
/**
* Read the list of docker containers
*
* @param pathToDockerSocket Path to docker socket
*/
export async function getContainers(
pathToDockerSocket = '/var/run/docker.sock',
): Promise<Dockerode.ContainerInfo[]> {
const docker = new Dockerode({
socketPath: pathToDockerSocket,
});
const containers = await docker.listContainers();
/* istanbul ignore next */
if (containers.length === 0) {
throw new Error(
`No running docker containers found.
Please check if docker is running and Node.js can access the docker socket (${pathToDockerSocket})`,
);
}
/* istanbul ignore next */
return containers;
}