openstapps/src/pack.ts

/*
 * Copyright (C) 2019 StApps
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation, version 3.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program. If not, see <https://www.gnu.org/licenses/>.
 */
import {Logger} from '@openstapps/logger';
import del from 'del';
import {existsSync} from 'fs';
import {basename, dirname, join} from 'path';
import {cwd} from 'process';
import {globPromisified, readFilePromisified, unlinkPromisified, writeFilePromisified} from './common';

const PACK_IDENTIFIER = '/* PACKED BY @openstapps/pack */';

/**
 * A JavaScript module representation to sort a list of them by dependencies
 */
interface JavaScriptModule {
  /**
   * Content of the module
   */
  content: string;

  /**
   * List of names of dependencies
   */
  dependencies: string[];

  /**
   * Directory the module is in
   */
  directory: string;

  /**
   * The name of the module
   */
  name: string;
}

/**
 * Pack cli.js
 *
 * This finds all internal requires and replaces the paths with `./index` or internal requires if it has been
 * required already.
 *
 * Furthermore it checks that no shebang line is present and that it does not export anything.
 */
async function packCliJs(): Promise<void> {
  const path = join(cwd(), 'lib', 'cli.js');

  if (!existsSync(path)) {
    return;
  }

  Logger.info('Adjusting JavaScript CLI...');

  const buffer = await readFilePromisified(path);
  const content = buffer.toString();

  if (content.indexOf('#!/') === 0) {
    throw new Error('`cli.js` must not contain a shebang line! It is added by this script.');
  }

  let internalRequire: string | null = null;

  const adjustedContent = content
    .split('\n')
    .map((line, lineNumber) => {

      // check for exports (cli.js is not allowed to export anything)
      if (Array.isArray(line.match(/^\s*((exports)|(module\.exports))/))) {
        throw new Error(
          `Line '${lineNumber}' in 'cli.js' exports something. cli.js is not for exporting. Line was:
${line}`,
        );
      }

      // replace lines with internal requires
      // extract module name from line
      const match = line.match(/^(\s*)(const|var) ([a-z0-9_]*) = require\(("[^"]+"|'[^']+')\);$/i);

      if (match !== null) {
        // tslint:disable-next-line:no-magic-numbers
        const importedName = match[3];
        // tslint:disable-next-line:no-magic-numbers
        const moduleName = match[4].substring(1, match[4].length - 1);

        // if it begins with '.' and not ends with json
        if (/^[.]{1,2}\/(?!.*\.json$).*$/i.test(moduleName)) {

          // is the first internal require
          if (internalRequire !== null) {
            return `const ${importedName} = ${internalRequire};`;
          }

          // only the first import needs a require
          internalRequire = importedName;

          return `const ${importedName} = require("./index");`;
        }
      }

      return line;
    })
    .join('\n');

  return writeFilePromisified(path, `#!/usr/bin/env node

${adjustedContent}`);
}

/**
 * Get a list containing the contents of all type definition files
 */
async function getAllTypeDefinitions(): Promise<string[]> {
  const fileNames = await globPromisified(join(cwd(), '*(lib|src)', '**', '*.d.ts'), {
    ignore: [
      join(cwd(), 'lib', 'doc', '**', '*.d.ts'),
      join(cwd(), 'lib', 'test', '**', '*.d.ts'),
      join(cwd(), 'lib', 'cli.d.ts'),
    ],
  });

  const promises = fileNames.map(async (fileName: string) => {
    return readFilePromisified(fileName, 'utf8');
  });

  return Promise.all(promises);
}

/**
 * Pack a list of type definitions into one file
 */
async function packTypeDefinitions(): Promise<void> {
  Logger.info('Packing TypeScript definition files...');

  const path = join(cwd(), 'lib', 'index.d.ts');

  await deleteFileIfExistingAndPacked(path);

  const typeDefinitions = await getAllTypeDefinitions();

  // pack TypeScript definition files
  const imports: { [k: string]: string[]; } = {};

  const referenceLines: string[] = [];

  let allDefinitions = typeDefinitions
  // concat them separated by new lines
    .join('\n\n\n\n\n')
    // split all lines
    .split('\n')
    .map((line) => {
      if (line.indexOf('export =') !== -1) {
        throw new Error('`export =` is not allowed by pack. Use named imports instead.');
      }

      if (line.indexOf('/// <reference') === 0) {
        referenceLines.push(line);

        return '// moved referenced line';
      }

      // match import lines
      const match = line.match(/^import {([^}].*)} from '([^'].*)';$/);

      if (match !== null) {
        // tslint:disable-next-line:no-magic-numbers - extract module
        const module = match[2];

        // extract imported objects
        const importedObjects = match[1]
          .split(',')
          .map((object) => {
            return object.trim();
          });

        // add list of already imported objects for module
        if (typeof imports[module] === 'undefined') {
          imports[module] = [];
        }

        // count already imported objects and objects to import now
        const objectsToImport: string[] = [];
        importedObjects.forEach((object) => {
          if (imports[module].indexOf(object) === -1) {
            imports[module].push(object);
            objectsToImport.push(object);
          }
        });

        // replace import line
        if (objectsToImport.length === 0) {
          return '// extraneous removed import';
        }

        return `import {${objectsToImport.join(', ')}} from '${module}';`;
      }

      return line;
    })
    // filter lines which contain "local" imports
    .filter((line) => {
      return line.match(/^import .* from '\./) === null;
    })
    // concat all lines separated by new lines
    .join('\n');

  if (allDefinitions.length > 0) {
    if (referenceLines.length > 0) {
      allDefinitions = `${referenceLines.join('\n')}

${allDefinitions}`;
    }

    // write packed TypeScript definition files
    return writeFilePromisified(path, `${PACK_IDENTIFIER}

${allDefinitions}`);
  }
}

/**
 * Get all JavaScript modules
 */
async function getAllJavaScriptModules(): Promise<JavaScriptModule[]> {
  const fileNames = await globPromisified(join(cwd(), 'lib', '**', '*.js'), {
    ignore: [
      join(cwd(), 'lib', 'doc', '**', '*.js'),
      join(cwd(), 'lib', 'test', '*.js'),
      join(cwd(), 'lib', 'cli.js'),
    ],
  });

  const promises = fileNames.map(async (fileName: string) => {
    const fileContent = await readFilePromisified(fileName, 'utf8');
    const directory = dirname(fileName)
      .replace(new RegExp(`^${join(cwd(), 'lib')}`), '');

    return {
      content: `(function() {
${fileContent}
})();
`,
      dependencies: getAllInternalDependencies(fileContent),
      directory: directory,
      name: basename(fileName, '.js'),
    };
  });

  return Promise.all(promises);
}

/**
 * Pack all javascript files
 */
async function packJavaScriptFiles(): Promise<void> {
  const path = join(cwd(), 'lib', 'index.js');

  Logger.info('Packing JavaScript files...');

  await deleteFileIfExistingAndPacked(path);

  // topologically sort the modules (sort by dependencies)
  const jsModules = topologicalSort(await getAllJavaScriptModules());

  let wholeCode = jsModules
  // convert modules to strings
    .map((module) => {
      module.content = module.content
        .split('\n')
        .map((line) => {
          const match = line.match(
            /^(\s*)(const|var) ([a-z0-9_]*) = ((require\("([^"]+)"\))|(require\('([^']+)'\)));$/i,
          );

          // replace lines with internal requires
          if (match !== null) {
            // tslint:disable-next-line:no-magic-numbers - match[6] or match[8] contain the modulePath
            if (typeof match[6] === 'undefined') {
              // tslint:disable-next-line:no-magic-numbers
              match[6] = match[8];
            }

            const whiteSpace = (typeof match[1] === 'string' && match[1].length > 0) ? match[1] : '';
            // tslint:disable-next-line:no-magic-numbers
            const importedName = match[3];
            // tslint:disable-next-line:no-magic-numbers
            const modulePath = match[6];

            // leave line unchanged if it is a "global" import
            if (modulePath.match(/^[.]{1,2}\//) === null) {
              return line;
            }

            // replace internal requires with `module.exports`
            if (existsSync(join(cwd(), 'lib', module.directory, `${modulePath}.js`))) {
              return `${whiteSpace}const ${importedName} = module.exports;`;
            }

            if (existsSync(join(cwd(), 'src', module.directory, modulePath))) {
              return `${whiteSpace} const ${importedName} = require(../src/${modulePath});`;
            }

            Logger.warn(`Import ${importedName} could not be found in module.directory ${modulePath}.`);
          }

          return line;
        })
        .join('\n');

      return `// Module: ${module.name}
${module.content}`;
    })
    // concat them separated by new lines
    .join('\n\n\n\n\n')
    // split all lines
    .split('\n')
    // filter lines
    .filter((line) => {
      // remove strict usage
      if (line === '"use strict";') {
        return false;
      }

      // remove esModule property
      if (line === 'Object.defineProperty(exports, "__esModule", { value: true });') {
        return false;
      }

      // remove source map references
      if (line.indexOf('//# sourceMappingURL=') === 0) {
        return false;
      }

      // keep all other lines
      return true;
    })
    // concat all lines separated by new lines
    .join('\n');

  if (wholeCode.length > 0) {
    // add meta lines to the file
    wholeCode = `"use strict";
Object.defineProperty(exports, "__esModule", { value: true });

${wholeCode}`;

    // write packed JavaScript files
    return writeFilePromisified(path, `${PACK_IDENTIFIER}

${wholeCode}`);
  }
}

/**
 * Delete file if it exists and is packed by this script
 *
 * @param path Path to file to check/delete
 */
async function deleteFileIfExistingAndPacked(path: string): Promise<void> {
  try {
    const buffer = await readFilePromisified(path);
    const content = buffer.toString();

    // check if packed by this script
    if (content.indexOf(PACK_IDENTIFIER) === 0) {
      Logger.log(`Found '${path}' which is packed by this script. Deleting it...`);

      return unlinkPromisified(path);
    }
  } catch (err) {
    if (err.code === 'ENOENT') {
      return;
    }
  }
}

/**
 * Get all internal dependencies from the content of a module
 *
 * @param moduleContent Module content to analyze
 */
function getAllInternalDependencies(moduleContent: string): string[] {
  // match all const <name> = require(<moduleName>);
  const requireLines =
    moduleContent.match(/^\s*(const|var) [a-z0-9_]* = require\("([^"]+)"\)|require\('([^']+)'\);$/gmi);

  if (Array.isArray(requireLines)) {
    return requireLines
      .map((requireLine) => {
        const matches = requireLine.match(/require\("([^"]+)"\)|require\('([^']+)'\);$/i);

        // previously matched require line does not contain a require?!
        if (matches === null) {
          throw new Error();
        }

        // return only the moduleName
        return matches[1];
      })
      .filter((moduleName) => {
        // filter out internal modules beginning with './' and not ending with '.json'
        return /^[.]{1,2}\/(?!.*\.json$).*$/i.test(moduleName);
      })
      .map((internalModuleName) => {
        // cut './' from the name
        return internalModuleName.substring('./'.length);
      });
  }

  return [];
}

/**
 * Sort modules by their dependencies
 *
 * @param modules Modules to sort
 */
function topologicalSort(modules: JavaScriptModule[]): JavaScriptModule[] {
  const topoSort = require('toposort');

  // vertices are modules, an edge from a to b means that b depends on a
  const edges: string[][] = [];
  const nodes: string[] = [];

  // add all edges
  modules.forEach((module) => {
    module.dependencies.forEach((dependencyPath) => {
      // add edge from dependency to our module
      edges.push([basename(dependencyPath), module.name]);
    });

    nodes.push(module.name);
  });

  // sort graph and return as an array of sorted modules
  return topoSort
    .array(nodes, edges)
    .map((moduleName: string) => {
      return modules.find((module) => {
        return module.name === moduleName;
      });
    });
}

/**
 * Pack
 */
export async function pack() {
  Logger.log(`Packing project in ${process.cwd()}...`);

  // run all tasks in parallel
  const promises: Array<Promise<void>> = [
    packCliJs(),
    packTypeDefinitions(),
    packJavaScriptFiles(),
  ];

  await Promise.all(promises);

  // clean up afterwards
  Logger.info('Deleting extraneous files...');

  await del([
    // delete all transpiled files
    'lib/*',

    // keep packed files
    '!lib/index.d.ts', '!lib/index.js',

    // keep converted schema files
    '!lib/schema', '!lib/schema/*.json',

    // keep documentation
    '!lib/doc', '!lib/doc/*', '!lib/doc/**/*',

    // keep cli
    '!lib/cli.js',

    // keep tests
    '!lib/test', '!lib/test/*', '!lib/test/**/*',
  ]);
}