feat: add pack script

Fixes #4

src/cli.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018 StApps
+ * Copyright (C) 2018-2019 StApps
  * This program is free software: you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation, version 3.
@@ -15,7 +15,8 @@
 import * as commander from 'commander';
 import {existsSync, readFileSync, writeFileSync} from 'fs';
 import {join, resolve} from 'path';
-import {getProjectReflection, logger, mkdirPromisified, readFilePromisifed} from './common';
+import {getProjectReflection, logger, mkdirPromisified, readFilePromisified} from './common';
+import {pack} from './pack';
 import {gatherRouteInformation, generateDocumentationForRoute, getNodeMetaInformationMap} from './routes';
 import {Converter, getValidatableTypesFromReflection} from './schema';
 import {validateFiles, writeReport} from './validate';
@@ -91,7 +92,7 @@ commander
 
     logger.info(`Using ${corePackageJsonPath} to determine version for schemas.`);
 
-    const buffer = await readFilePromisifed(corePackageJsonPath);
+    const buffer = await readFilePromisified(corePackageJsonPath);
     const corePackageJson = JSON.parse(buffer.toString());
     const coreVersion = corePackageJson.version;
 
@@ -141,6 +142,12 @@ commander
     }
   });
 
+commander
+  .command('pack')
+  .action(async () => {
+    await pack();
+  });
+
 commander
   .parse(process.argv);
 

src/common.ts

@@ -13,22 +13,20 @@
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
 import {Logger} from '@openstapps/logger';
-import {mkdir, PathLike, readFile, writeFile} from 'fs';
+import {mkdir, PathLike, readFile, unlink, writeFile} from 'fs';
 import * as glob from 'glob';
 import {Schema as JSONSchema, ValidationError} from 'jsonschema';
 import {Definition} from 'ts-json-schema-generator';
 import {Application, ProjectReflection} from 'typedoc';
 import {promisify} from 'util';
 
-/**
- * Initialized logger
- */
 export const logger = new Logger();
 
-export const globPromisfied = promisify(glob);
+export const globPromisified = promisify(glob);
 export const mkdirPromisified = promisify(mkdir);
-export const readFilePromisifed = promisify(readFile);
+export const readFilePromisified = promisify(readFile);
 export const writeFilePromisified = promisify(writeFile);
+export const unlinkPromisified = promisify(unlink);
 
 /**
  * A route instance with its relevant meta information

src/pack.ts

@@ -0,0 +1,463 @@
+/*
+ * Copyright (C) 2018-2019 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import * as del from 'del';
+import {existsSync} from 'fs';
+import {basename, dirname, join} from 'path';
+import {cwd} from 'process';
+import {globPromisified, logger, readFilePromisified, unlinkPromisified, writeFilePromisified} from './common';
+
+const PACK_IDENTIFIER = '/* PACKED BY @openstapps/pack */';
+
+/**
+ * A JavaScript module representation to sort a list of them by dependencies
+ */
+interface JavaScriptModule {
+  /**
+   * Content of the module
+   */
+  content: string;
+
+  /**
+   * List of names of dependencies
+   */
+  dependencies: string[];
+
+  /**
+   * Directory the module is in
+   */
+  directory: string;
+
+  /**
+   * The name of the module
+   */
+  name: string;
+}
+
+/**
+ * Pack cli.js
+ *
+ * This finds all internal requires and replaces the paths with `./index` or internal requires if it has been
+ * required already.
+ *
+ * Furthermore it checks that no shebang line is present and that it does not export anything.
+ */
+async function packCliJs(): Promise<void> {
+  const path = join(cwd(), 'lib', 'cli.js');
+
+  if (!existsSync(path)) {
+    return;
+  }
+
+  logger.info('Adjusting JavaScript CLI...');
+
+  const buffer = await readFilePromisified(path);
+  const content = buffer.toString();
+
+  if (content.indexOf('#!/') === 0) {
+    throw new Error('`cli.js` must not contain a shebang line! It is added by this script.');
+  }
+
+  let internalRequire: string | null = null;
+
+  const adjustedContent = '#!/usr/bin/env node\n\n' + content
+    .split('\n')
+    .map((line, lineNumber) => {
+
+      // check for exports (cli.js is not allowed to export anything)
+      if (Array.isArray(line.match(/^\s*((exports)|(module\.exports))/))) {
+        throw new Error(
+          'Line ' +
+          lineNumber +
+          ' in cli.js has a reference to the exports object. cli.js is not for exporting. Line was: "'
+          + line
+          + '"',
+        );
+      }
+
+      // replace lines with internal requires
+      // extract module name from line
+      const match = line.match(/^(\s*)(const|var) ([a-z0-9_]*) = require\(("[^"]+"|'[^']+')\);$/i);
+
+      if (match !== null) {
+        const importedName = match[3];
+        const moduleName = match[4].substring(1, match[4].length - 1);
+
+        // if it begins with '.' and not ends with json
+        if (/^[.]{1,2}\/(?!.*\.json$).*$/i.test(moduleName)) {
+
+          // is the first internal require
+          if (internalRequire !== null) {
+            return 'const ' + importedName + ' = ' + internalRequire + ';';
+          }
+
+          // only the first import needs a require
+          internalRequire = importedName;
+          return 'const ' + importedName + ' = require("./index");';
+        }
+      }
+      return line;
+    })
+    .join('\n');
+
+  return await writeFilePromisified(path, adjustedContent);
+}
+
+/**
+ * Get a list containing the contents of all type definition files
+ */
+async function getAllTypeDefinitions(): Promise<string[]> {
+  const fileNames = await globPromisified(join(cwd(), '*(lib|src)', '**', '*.d.ts'), {
+    ignore: [
+      join(cwd(), 'lib', 'doc', '**', '*.d.ts'),
+      join(cwd(), 'lib', 'test', '**', '*.d.ts'),
+      join(cwd(), 'lib', 'cli.d.ts'),
+    ],
+  });
+
+  const promises = fileNames.map((fileName) => {
+    return readFilePromisified(fileName, 'utf8');
+  });
+
+  return await Promise.all(promises);
+}
+
+/**
+ * Pack a list of type definitions into one file
+ */
+async function packTypeDefinitions(): Promise<void> {
+  logger.info('Packing TypeScript definition files...');
+
+  const path = join(cwd(), 'lib', 'index.d.ts');
+
+  await deleteFileIfExistingAndPacked(path);
+
+  const typeDefinitions = await getAllTypeDefinitions();
+
+  // pack TypeScript definition files
+  const imports: { [k: string]: string[] } = {};
+
+  const referenceLines: string[] = [];
+
+  let allDefinitions = typeDefinitions
+  // concat them separated by new lines
+    .join('\n\n\n\n\n')
+    // split all lines
+    .split('\n')
+    .map((line) => {
+      if (line.indexOf('export =') !== -1) {
+        throw new Error('`export =` is not allowed by pack. Use named imports instead.');
+      }
+
+      if (line.indexOf('/// <reference') === 0) {
+        referenceLines.push(line);
+        return '// moved referenced line';
+      }
+
+      // match import lines
+      const match = line.match(/^import {([^}].*)} from '([^'].*)';$/);
+
+      if (match !== null) {
+        // extract module
+        const module = match[2];
+
+        // extract imported objects
+        const importedObjects = match[1].split(',').map((object) => {
+          return object.trim();
+        });
+
+        // add list of already imported objects for module
+        if (typeof imports[module] === 'undefined') {
+          imports[module] = [];
+        }
+
+        // count already imported objects and objects to import now
+        const objectsToImport: string[] = [];
+        importedObjects.forEach((object) => {
+          if (imports[module].indexOf(object) === -1) {
+            imports[module].push(object);
+            objectsToImport.push(object);
+          }
+        });
+
+        // replace import line
+        if (objectsToImport.length === 0) {
+          return '// extraneous removed import';
+        } else {
+          return 'import { ' + objectsToImport.join(', ') + ' } from \'' + module + '\';';
+        }
+      }
+
+      return line;
+    })
+    // filter lines which contain "local" imports
+    .filter((line) => {
+      return !line.match(/^import .* from '\./);
+    })
+    // concat all lines separated by new lines
+    .join('\n');
+
+  if (allDefinitions.length > 0) {
+    if (referenceLines.length > 0) {
+      allDefinitions = referenceLines.join('\n') + '\n\n' + allDefinitions;
+    }
+
+    // write packed TypeScript definition files
+    return await writeFilePromisified(path, PACK_IDENTIFIER + '\n\n' + allDefinitions);
+  }
+}
+
+/**
+ * Get all JavaScript modules
+ */
+async function getAllJavaScriptModules(): Promise<JavaScriptModule[]> {
+  const fileNames = await globPromisified(join(cwd(), 'lib', '**', '*.js'), {
+    ignore: [
+      join(cwd(), 'lib', 'doc', '**', '*.js'),
+      join(cwd(), 'lib', 'test', '*.js'),
+      join(cwd(), 'lib', 'cli.js'),
+    ],
+  });
+
+  const promises = fileNames.map(async (fileName) => {
+    const fileContent = await readFilePromisified(fileName, 'utf8');
+    const directory = dirname(fileName).replace(new RegExp('^' + join(cwd(), 'lib')), '');
+
+    return {
+      content: '(function() {\n' + fileContent + '\n})();\n',
+      dependencies: getAllInternalDependencies(fileContent),
+      directory: directory,
+      name: basename(fileName, '.js'),
+    };
+  });
+
+  return await Promise.all(promises);
+}
+
+/**
+ * Pack all javascript files
+ */
+async function packJavaScriptFiles(): Promise<void> {
+  const path = join(cwd(), 'lib', 'index.js');
+
+  logger.info('Packing JavaScript files...');
+
+  await deleteFileIfExistingAndPacked(path);
+
+  // topologically sort the modules (sort by dependencies)
+  const jsModules = topologicalSort(await getAllJavaScriptModules());
+
+  let wholeCode = jsModules
+  // convert modules to strings
+    .map((module) => {
+      module.content = module.content
+        .split('\n')
+        .map((line) => {
+          const match = line.match(
+            /^(\s*)(const|var) ([a-z0-9_]*) = ((require\("([^"]+)"\))|(require\('([^']+)'\)));$/i,
+          );
+
+          // replace lines with internal requires
+          if (match !== null) {
+            // match[6] or match[8] contain the modulePath
+            if (typeof match[6] === 'undefined') {
+              match[6] = match[8];
+            }
+
+            const whiteSpace = match[1] ? match[1] : '';
+            const importedName = match[3];
+            const modulePath = match[6];
+
+            // leave line unchanged if it is a "global" import
+            if (modulePath.match(/^[.]{1,2}\//) === null) {
+              return line;
+            }
+
+            // replace internal requires with `module.exports`
+            if (existsSync(join(cwd(), 'lib', module.directory, modulePath + '.js'))) {
+              return whiteSpace + 'const ' + importedName + ' = module.exports;';
+            }
+
+            if (existsSync(join(cwd(), 'src', module.directory, modulePath))) {
+              return whiteSpace + 'const ' + importedName + ' = require(\'../src/' + modulePath + '\');';
+            }
+
+            logger.warn('Import ' + importedName + ' could not be found in module.directory ' + modulePath);
+          }
+
+          return line;
+        })
+        .join('\n');
+
+      return '// Module: ' + module.name + '\n' + module.content;
+    })
+    // concat them separated by new lines
+    .join('\n\n\n\n\n')
+    // split all lines
+    .split('\n')
+    // filter lines
+    .filter((line) => {
+      // remove strict usage
+      if (line === '"use strict";') {
+        return false;
+      }
+
+      // remove esModule property
+      if (line === 'Object.defineProperty(exports, "__esModule", { value: true });') {
+        return false;
+      }
+
+      // remove source map references
+      if (line.indexOf('//# sourceMappingURL=') === 0) {
+        return false;
+      }
+
+      // keep all other lines
+      return true;
+    })
+    // concat all lines separated by new lines
+    .join('\n');
+
+  if (wholeCode.length > 0) {
+    // add meta lines to the file
+    wholeCode = '"use strict";\nObject.defineProperty(exports, "__esModule", { value: true });\n\n' + wholeCode;
+
+    // write packed JavaScript files
+    return await writeFilePromisified(path, PACK_IDENTIFIER + '\n\n' + wholeCode);
+  }
+}
+
+/**
+ * Delete file if it exists and is packed by this script
+ *
+ * @param path Path to file to check/delete
+ */
+async function deleteFileIfExistingAndPacked(path: string): Promise<void> {
+  try {
+    const buffer = await readFilePromisified(path);
+    const content = buffer.toString();
+
+    // check if packed by this script
+    if (content.indexOf(PACK_IDENTIFIER) === 0) {
+      logger.log('Found `' + path + '` which is packed by this script. Deleting it...');
+      return await unlinkPromisified(path);
+    }
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return;
+    }
+  }
+}
+
+/**
+ * Get all internal dependencies from the content of a module
+ *
+ * @param moduleContent Module content to analyze
+ */
+function getAllInternalDependencies(moduleContent: string): string[] {
+  // match all const <name> = require(<moduleName>);
+  const requireLines =
+    moduleContent.match(/^\s*(const|var) [a-z0-9_]* = require\("([^"]+)"\)|require\('([^']+)'\);$/gmi);
+
+  if (Array.isArray(requireLines)) {
+    return requireLines.map((requireLine) => {
+      const matches = requireLine.match(/require\("([^"]+)"\)|require\('([^']+)'\);$/i);
+
+      // previously matched require line does not contain a require?!
+      if (matches === null) {
+        throw new Error();
+      }
+
+      // return only the moduleName
+      return matches[1];
+    }).filter((moduleName) => {
+      // filter out internal modules beginning with './' and not ending with '.json'
+      return /^[.]{1,2}\/(?!.*\.json$).*$/i.test(moduleName);
+    }).map((internalModuleName) => {
+      // cut './' from the name
+      return internalModuleName.substring(2);
+    });
+  }
+
+  return [];
+}
+
+/**
+ * Sort modules by their dependencies
+ *
+ * @param modules Modules to sort
+ */
+function topologicalSort(modules: JavaScriptModule[]): JavaScriptModule[] {
+  const topoSort = require('toposort');
+
+  // vertices are modules, an edge from a to b means that b depends on a
+  const edges: string[][] = [];
+  const nodes: string[] = [];
+
+  // add all edges
+  modules.forEach((module) => {
+    module.dependencies.forEach((dependenciePath) => {
+      // add edge from dependency to our module
+      edges.push([basename(dependenciePath), module.name]);
+    });
+
+    nodes.push(module.name);
+  });
+
+  // sort graph and return as an array of sorted modules
+  return topoSort.array(nodes, edges).map((moduleName: string) => {
+    return modules.find((module) => {
+      return module.name === moduleName;
+    });
+  });
+}
+
+/**
+ * Pack
+ */
+export async function pack() {
+  logger.log(`Packing project in ${process.cwd()}...`);
+
+  // run all tasks in parallel
+  const promises: Array<Promise<void>> = [
+    packCliJs(),
+    packTypeDefinitions(),
+    packJavaScriptFiles(),
+  ];
+
+  await Promise.all(promises);
+
+  // clean up afterwards
+  logger.info('Deleting extraneous files...');
+
+  await del([
+    // delete all transpiled files
+    'lib/*',
+
+    // keep packed files
+    '!lib/index.d.ts', '!lib/index.js',
+
+    // keep converted schema files
+    '!lib/schema', '!lib/schema/*.json',
+
+    // keep documentation
+    '!lib/doc', '!lib/doc/*', '!lib/doc/**/*',
+
+    // keep cli
+    '!lib/cli.js',
+
+    // keep tests
+    '!lib/test', '!lib/test/*', '!lib/test/**/*',
+  ]);
+}

src/validate.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018 StApps
+ * Copyright (C) 2018-2019 StApps
  * This program is free software: you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation, version 3.
@@ -19,10 +19,10 @@ import * as mustache from 'mustache';
 import {basename, join, resolve} from 'path';
 import {
   ExpectableValidationErrors,
-  globPromisfied,
+  globPromisified,
   isThingWithType,
   logger,
-  readFilePromisifed,
+  readFilePromisified,
   writeFilePromisified,
 } from './common';
 
@@ -53,7 +53,7 @@ export class Validator {
    * @param schemaDir Path to directory that contains schema files
    */
   public async addSchemas(schemaDir: PathLike): Promise<string[]> {
-    const schemaFiles = await globPromisfied(join(schemaDir.toString(), '*.json'));
+    const schemaFiles = await globPromisified(join(schemaDir.toString(), '*.json'));
 
     if (schemaFiles.length === 0) {
       throw new Error(`No schema files in ${schemaDir.toString()}!`);
@@ -64,7 +64,7 @@ export class Validator {
     // Iterate over schema files
     await asyncPool(2, schemaFiles, async (file) => {
       // read schema file
-      const buffer = await readFilePromisifed(file);
+      const buffer = await readFilePromisified(file);
       const schema = JSON.parse(buffer.toString());
 
       // add schema to validator
@@ -124,7 +124,7 @@ export async function validateFiles(schemaDir: string, resourcesDir: string): Pr
   await v.addSchemas(schemaDir);
 
   // get list of files to test
-  const testFiles = await globPromisfied(join(resourcesDir, '*.json'));
+  const testFiles = await globPromisified(join(resourcesDir, '*.json'));
 
   if (testFiles.length === 0) {
     throw new Error(`No test files in ${resourcesDir}!`);
@@ -139,7 +139,7 @@ export async function validateFiles(schemaDir: string, resourcesDir: string): Pr
   await asyncPool(2, testFiles, async (testFile) => {
     const testFileName = basename(testFile);
 
-    const buffer = await readFilePromisifed(join(resourcesDir, testFileName));
+    const buffer = await readFilePromisified(join(resourcesDir, testFileName));
 
     // read test description from file
     const testDescription = JSON.parse(buffer.toString());
@@ -208,10 +208,10 @@ export async function validateFiles(schemaDir: string, resourcesDir: string): Pr
  * @param errors Errors that occurred in validation
  */
 export async function writeReport(reportPath: PathLike, errors: ExpectableValidationErrors): Promise<void> {
-  let buffer = await readFilePromisifed(resolve(__dirname, '..', 'resources', 'file.html.mustache'));
+  let buffer = await readFilePromisified(resolve(__dirname, '..', 'resources', 'file.html.mustache'));
   const fileTemplate = buffer.toString();
 
-  buffer = await readFilePromisifed(resolve(__dirname, '..', 'resources', 'error.html.mustache'));
+  buffer = await readFilePromisified(resolve(__dirname, '..', 'resources', 'error.html.mustache'));
   const errorTemplate = buffer.toString();
 
   let output = '';
@@ -236,7 +236,7 @@ export async function writeReport(reportPath: PathLike, errors: ExpectableValida
     });
   });
 
-  buffer = await readFilePromisifed(resolve(__dirname, '..', 'resources', 'report.html.mustache'));
+  buffer = await readFilePromisified(resolve(__dirname, '..', 'resources', 'report.html.mustache'));
   const reportTemplate = buffer.toString();
 
   await writeFilePromisified(reportPath, mustache.render(reportTemplate, {