1.5.0

Closes #14

CHANGELOG.md

@@ -1,3 +1,25 @@
+## [1.4.1](https://gitlab.com/openstapps/proxy/compare/v1.4.0...v1.4.1) (2022-11-09)
+
+
+### Bug Fixes
+
+* include uri path in json log verbatim ([5f77877](https://gitlab.com/openstapps/proxy/commit/5f77877bb4239a437ff3f2eea1c0dfd51c7d4818))
+
+
+
+# [1.4.0](https://gitlab.com/openstapps/proxy/compare/v1.3.0...v1.4.0) (2022-11-09)
+
+
+### Features
+
+* add support for log aggregators ([8c49c31](https://gitlab.com/openstapps/proxy/commit/8c49c317603b1a2964708ed377a5a7c687f829cd))
+
+
+
+# [1.3.0](https://gitlab.com/openstapps/proxy/compare/v1.2.0...v1.3.0) (2022-08-22)
+
+
+
 # [1.2.0](https://gitlab.com/openstapps/proxy/compare/v1.1.0...v1.2.0) (2022-06-08)
 
 

config/default.ts

@@ -18,6 +18,7 @@ import {ConfigFile} from '../src/common';
 const config: ConfigFile = {
   activeVersions: ['1\\.0\\.\\d+', '2\\.0\\.\\d+'],
   hiddenRoutes: ['/bulk'],
+  logFormat: 'default',
   metrics: false,
   outdatedVersions: ['0\\.8\\.\\d+', '0\\.5\\.\\d+', '0\\.6\\.\\d+', '0\\.7\\.\\d+'],
   output: '/etc/nginx/http.d/default.conf',

fixtures/cors.template

@@ -3,4 +3,3 @@ add_header 'Access-Control-Allow-Credentials' 'true';
 add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
 add_header 'Access-Control-Allow-Headers' 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,X-StApps-Version';
 add_header 'Access-Control-Max-Age' 1728000;
-add_header 'Content-Type' 'text/plain charset=UTF-8';

fixtures/logFormatters.template

@@ -0,0 +1,32 @@
+map $time_iso8601 $time_iso8601_dateTime {
+  ~([^+]+) $1;
+}
+map $time_iso8601 $time_iso8601_TZ {
+  ~\+([0-9:]+)$ $1;
+}
+map $msec $millisec {
+  ~\.([0-9]+)$ $1;
+}
+
+log_format json escape=json '{ "nginx_timestamp": "$time_iso8601_dateTime.$millisec+$time_iso8601_TZ", '
+       '"remote_addr": "$remote_addr", '
+       '"connection": "$connection", '
+       '"connection_requests": $connection_requests, '
+       '"pipe": "$pipe", '
+       '"body_bytes_sent": $body_bytes_sent, '
+       '"request_length": $request_length, '
+       '"request_time": $request_time, '
+       '"response_status": $status, '
+       '"request_uri": "$request_uri", '
+       '"request_method": "$request_method", '
+       '"host": "$host", '
+       '"upstream_cache_status": "$upstream_cache_status", '
+       '"upstream_addr": "$upstream_addr", '
+       '"http_x_forwarded_for": "$http_x_forwarded_for", '
+       '"http_referrer": "$http_referer", '
+       '"http_user_agent": "$http_user_agent", '
+       '"http_version": "$server_protocol", '
+       '"remote_user": "$remote_user", '
+       '"http_x_forwarded_proto": "$http_x_forwarded_proto", '
+       '"upstream_response_time": "$upstream_response_time", '
+       '"nginx_access": true }';

fixtures/metrics.template

@@ -5,9 +5,10 @@ map $status $omitOKs {
 
 server {
     listen 8080;
-    access_log /dev/stdout combined if=$omitOKs;
+    error_log stderr;
+    access_log /dev/stdout {{{ logFormat }}} if=$omitOKs;
     location /metrics {
-      vhost_traffic_status_display;
-      vhost_traffic_status_display_format prometheus;
+        vhost_traffic_status_display;
+        vhost_traffic_status_display_format prometheus;
     }
 }

nginx.conf.template

@@ -1,3 +1,5 @@
+{{{ logFormatters }}}
+
 {{{ metrics }}}
 
 {{{ dockerVersionMap }}}
@@ -19,6 +21,10 @@ map $isRateLimited $rateLimit {
 limit_req_zone $rateLimit zone=customstappslimit:10m rate=20r/s;
 
 server {
+  charset utf-8;
+  error_log stderr;
+  access_log /dev/stdout {{{ logFormat }}};
+
   {{{ listener }}}
 
   {{{ visibleRoutes }}}

package.json

@@ -1,40 +1,40 @@
 {
   "name": "@openstapps/proxy",
-  "version": "1.3.0",
+  "version": "1.5.0",
   "description": "NGINX proxy that is dynamically configured by a Node.js script",
   "main": "./lib/cli.js",
   "dependencies": {
-    "@openstapps/logger": "1.0.0",
+    "@openstapps/logger": "1.1.1",
     "@types/config": "3.3.0",
-    "@types/dockerode": "3.3.9",
+    "@types/dockerode": "3.3.12",
     "@types/node": "14.18.24",
     "@types/sha1": "1.1.3",
-    "config": "3.3.7",
-    "dockerode": "3.3.3",
+    "config": "3.3.8",
+    "dockerode": "3.3.4",
     "is-cidr": "4.0.2",
     "mustache": "4.2.0",
     "node-port-scanner": "3.0.1",
-    "semver": "7.3.7"
+    "semver": "7.3.8"
   },
   "devDependencies": {
     "@openstapps/configuration": "0.33.0",
     "@openstapps/eslint-config": "1.1.0",
-    "@testdeck/mocha": "0.2.0",
-    "@types/chai": "4.3.3",
+    "@testdeck/mocha": "0.3.0",
+    "@types/chai": "4.3.4",
     "@types/chai-spies": "1.0.3",
     "@types/mustache": "4.2.1",
     "@types/proxyquire": "1.3.28",
-    "@typescript-eslint/eslint-plugin": "5.33.1",
-    "@typescript-eslint/parser": "5.33.1",
-    "chai": "4.3.6",
+    "@typescript-eslint/eslint-plugin": "5.42.1",
+    "@typescript-eslint/parser": "5.42.1",
+    "chai": "4.3.7",
     "chai-spies": "1.0.0",
     "conventional-changelog-cli": "2.2.2",
-    "eslint": "8.22.0",
+    "eslint": "8.27.0",
     "eslint-config-prettier": "8.5.0",
-    "eslint-plugin-jsdoc": "39.3.6",
+    "eslint-plugin-jsdoc": "39.6.2",
     "eslint-plugin-prettier": "4.2.1",
-    "eslint-plugin-unicorn": "43.0.2",
-    "mocha": "9.2.2",
+    "eslint-plugin-unicorn": "44.0.2",
+    "mocha": "10.1.0",
     "nyc": "15.1.0",
     "prepend-file-cli": "1.0.6",
     "prettier": "2.7.1",

src/common.ts

@@ -47,6 +47,20 @@ export interface SSLFilePaths {
   dhparam: string;
 }
 
+/**
+ * Supported log formats for config
+ */
+type SupportedLogFormatsKeys = 'default' | 'combined' | 'json';
+
+/**
+ * Map supported formats to stings used in template view
+ */
+export const SupportedLogFormats: {[key in SupportedLogFormatsKeys]: string} = {
+  default: 'combined',
+  combined: 'combined',
+  json: 'json',
+};
+
 /**
  * A representation of the config file
  */
@@ -59,10 +73,14 @@ export interface ConfigFile {
    * List of hidden routes
    */
   hiddenRoutes: string[];
+  /**
+   * Sets log format (default or json)
+   */
+  logFormat: SupportedLogFormatsKeys;
   /**
    * Enables metrics on /metrics route
    */
-  metrics: boolean;
+  metrics?: boolean;
   /**
    * List of outdated versions
    */
@@ -97,6 +115,14 @@ export interface TemplateView {
    * Listener
    */
   listener: string;
+  /**
+   * Log format to use
+   */
+  logFormat: string;
+  /**
+   * Custom Log formatters
+   */
+  logFormatters: string;
   /**
    * Local server with listener for /metrics route
    */

src/main.ts

@@ -26,6 +26,7 @@ import {
   protocolHardeningParameters,
   SSLFilePaths,
   sslHardeningParameters,
+  SupportedLogFormats,
   TemplateView,
 } from './common';
 
@@ -244,11 +245,14 @@ ${protocolHardeningParameters}
 /**
  * Reads predefined server entry with metrics location
  */
-export async function generateMetricsServer(enableMetrics: boolean): Promise<string> {
+export async function generateMetricsServer(logFormat: string, enableMetrics?: boolean): Promise<string> {
   if (!enableMetrics) {
     return '';
   }
-  return asyncReadFile('./fixtures/metrics.template', 'utf8');
+
+  return renderTemplate(path.join('fixtures', 'metrics.template'), {
+    logFormat: logFormat,
+  });
 }
 
 /**
@@ -302,6 +306,13 @@ export async function getTemplateView(containers: Dockerode.ContainerInfo[]): Pr
     });
   });
 
+  const logFormattingPromise = renderTemplate(path.join('fixtures', 'logFormatters.template'), {});
+
+  const logFormat =
+    configFile.logFormat in SupportedLogFormats
+      ? SupportedLogFormats[configFile.logFormat]
+      : SupportedLogFormats.default;
+
   return {
     dockerVersionMap: await generateUpstreamMap(
       configFile.activeVersions,
@@ -310,7 +321,9 @@ export async function getTemplateView(containers: Dockerode.ContainerInfo[]): Pr
     ),
     hiddenRoutes: (await Promise.all(hiddenRoutesPromises)).join(''),
     listener: generateListener(configFile.sslFilePaths),
-    metrics: await generateMetricsServer(configFile.metrics),
+    logFormat: logFormat,
+    logFormatters: await logFormattingPromise,
+    metrics: await generateMetricsServer(logFormat, configFile.metrics),
     rateLimitAllowList: generateRateLimitAllowList(configFile.rateLimitAllowList),
     staticRoute: await renderTemplate(path.join('fixtures', 'staticRoute.template'), {cors}),
     visibleRoutes: (await Promise.all(visibleRoutesPromises)).join(''),

test/main.spec.ts

@@ -402,12 +402,12 @@ Please check if docker is running and Node.js can access the docker socket (/var
 
   @test
   async 'include metrics config'() {
-    expect(await generateMetricsServer(true)).length.to.be.greaterThan(1);
+    expect(await generateMetricsServer('test', true)).length.to.be.greaterThan(1);
   }
 
   @test
   async 'omit metrics config'() {
-    expect(await generateMetricsServer(false)).to.equal('');
+    expect(await generateMetricsServer('test', false)).to.equal('');
   }
 
   @test