openstapps/src/pack.ts

/* eslint-disable unicorn/error-message */
/*
 * Copyright (C) 2021 StApps
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation, version 3.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program. If not, see <https://www.gnu.org/licenses/>.
 */
import {Logger} from '@openstapps/logger';
import del from 'del';
import {existsSync} from 'fs';
import {cwd} from 'process';
import {globPromisified, readFilePromisified, unlinkPromisified, writeFilePromisified} from './common';
import {JavaScriptModule} from './types/pack';
import path from 'path';

const PACK_IDENTIFIER = '/* PACKED BY @openstapps/pack */';

/**
 * Pack cli.js
 *
 * This finds all internal requires and replaces the paths with `./index` or internal requires if it has been
 * required already.
 *
 * Furthermore it checks that no shebang line is present and that it does not export anything.
 */
async function packCliJs(): Promise<void> {
  const cliPath = path.join(cwd(), 'lib', 'cli.js');

  if (!existsSync(cliPath)) {
    return;
  }

  Logger.info('Adjusting JavaScript CLI...');

  const buffer = await readFilePromisified(cliPath);
  const content = buffer.toString();

  if (content.indexOf('#!/') === 0) {
    throw new Error('`cli.js` must not contain a shebang line! It is added by this script.');
  }

  let internalRequire: string | undefined;

  const adjustedContent = content
    .split('\n')
    .map((line, lineNumber) => {
      // check for exports (cli.js is not allowed to export anything)
      if (Array.isArray(line.match(/^\s*((exports)|(module\.exports))/))) {
        throw new TypeError(
          `Line '${lineNumber}' in 'cli.js' exports something. cli.js is not for exporting. Line was:\n${line}`,
        );
      }

      // replace lines with internal requires
      // extract module name from line
      const match = line.match(/^(\s*)(const|var) ([a-z0-9_]*) = require\(("[^"]+"|'[^']+')\);$/i);

      if (match !== null) {
        const importedName = match[3];
        // eslint-disable-next-line unicorn/prefer-string-slice
        const moduleName = match[4].substring(1, match[4].length - 1);

        // if it begins with '.' and not ends with json
        if (/^[.]{1,2}\/(?!.*\.json$).*$/i.test(moduleName)) {
          // is the first internal require
          if (internalRequire) {
            return `const ${importedName} = ${internalRequire};`;
          }

          // only the first import needs a require
          internalRequire = importedName;

          return `const ${importedName} = require("./index");`;
        }
      }

      return line;
    })
    .join('\n');

  return writeFilePromisified(cliPath, `#!/usr/bin/env node\n\n${adjustedContent}`);
}

/**
 * Get a list containing the contents of all type definition files
 */
async function getAllTypeDefinitions(): Promise<string[]> {
  const fileNames = await globPromisified(path.join(cwd(), '*(lib|src)', '**', '*.d.ts'), {
    ignore: [
      path.join(cwd(), 'lib', 'doc', '**', '*.d.ts'),
      path.join(cwd(), 'lib', 'test', '**', '*.d.ts'),
      path.join(cwd(), 'lib', 'cli.d.ts'),
    ],
  });

  const promises = fileNames.map(async (fileName: string) => {
    return readFilePromisified(fileName, 'utf8');
  });

  return Promise.all(promises);
}

/**
 * Pack a list of type definitions into one file
 */
async function packTypeDefinitions(): Promise<void> {
  Logger.info('Packing TypeScript definition files...');

  const indexPath = path.join(cwd(), 'lib', 'index.d.ts');

  await deleteFileIfExistingAndPacked(indexPath);

  const typeDefinitions = await getAllTypeDefinitions();

  // pack TypeScript definition files
  const imports: {[k: string]: string[]} = {};

  const referenceLines: string[] = [];

  let allDefinitions = typeDefinitions
    // concat them separated by new lines
    .join('\n\n\n\n\n')
    // split all lines
    .split('\n')
    .map(line => {
      if (line.includes('export =')) {
        throw new Error('`export =` is not allowed by pack. Use named imports instead.');
      }

      if (line.indexOf('/// <reference') === 0) {
        referenceLines.push(line);

        return '// moved referenced line';
      }

      // match import lines
      const match = line.match(/^import {([^}].*)} from '([^'].*)';$/);

      if (match !== null) {
        const module = match[2];

        // extract imported objects
        const importedObjects = match[1].split(',').map(object => {
          return object.trim();
        });

        // add list of already imported objects for module
        if (typeof imports[module] === 'undefined') {
          imports[module] = [];
        }

        // count already imported objects and objects to import now
        const objectsToImport: string[] = [];
        for (const object of importedObjects) {
          if (!imports[module].includes(object)) {
            imports[module].push(object);
            objectsToImport.push(object);
          }
        }

        // replace import line
        if (objectsToImport.length === 0) {
          return '// extraneous removed import';
        }

        return `import {${objectsToImport.join(', ')}} from '${module}';`;
      }

      return line;
    })
    // filter lines which contain "local" imports
    .filter(line => {
      return line.match(/^import .* from '\./) === null;
    })
    // concat all lines separated by new lines
    .join('\n');

  if (allDefinitions.length > 0) {
    if (referenceLines.length > 0) {
      allDefinitions = `${referenceLines.join('\n')}

${allDefinitions}`;
    }

    // write packed TypeScript definition files
    return writeFilePromisified(
      indexPath,
      `${PACK_IDENTIFIER}

${allDefinitions}`,
    );
  }
}

/**
 * Get all JavaScript modules
 */
async function getAllJavaScriptModules(): Promise<JavaScriptModule[]> {
  const fileNames = await globPromisified(path.join(cwd(), 'lib', '**', '*.js'), {
    ignore: [
      path.join(cwd(), 'lib', 'doc', '**', '*.js'),
      path.join(cwd(), 'lib', 'test', '*.js'),
      path.join(cwd(), 'lib', 'cli.js'),
    ],
  });

  const promises = fileNames.map(async (fileName: string) => {
    const fileContent = await readFilePromisified(fileName, 'utf8');
    const directory = path.dirname(fileName).replace(new RegExp(`^${path.join(cwd(), 'lib')}`), '');

    return {
      content: `(function() {
${fileContent}
})();
`,
      dependencies: getAllInternalDependencies(fileContent),
      directory: directory,
      name: path.basename(fileName, '.js'),
    };
  });

  return Promise.all(promises);
}

/**
 * Pack all javascript files
 */
async function packJavaScriptFiles(): Promise<void> {
  const indexPath = path.join(cwd(), 'lib', 'index.js');

  Logger.info('Packing JavaScript files...');

  await deleteFileIfExistingAndPacked(indexPath);

  // topologically sort the modules (sort by dependencies)
  const jsModules = topologicalSort(await getAllJavaScriptModules());

  let wholeCode = jsModules
    // convert modules to strings
    .map(module => {
      module.content = module.content
        .split('\n')
        .map(line => {
          const match = line.match(
            /^(\s*)(const|var) ([a-z0-9_]*) = ((require\("([^"]+)"\))|(require\('([^']+)'\)));$/i,
          );

          // replace lines with internal requires
          if (match !== null) {
            if (typeof match[6] === 'undefined') {
              match[6] = match[8];
            }

            const whiteSpace = typeof match[1] === 'string' && match[1].length > 0 ? match[1] : '';
            const importedName = match[3];
            const modulePath = match[6];

            // leave line unchanged if it is a "global" import
            if (modulePath.match(/^[.]{1,2}\//) === null) {
              return line;
            }

            // replace internal requires with `module.exports`
            if (existsSync(path.join(cwd(), 'lib', module.directory, `${modulePath}.js`))) {
              return `${whiteSpace}const ${importedName} = module.exports;`;
            }

            if (existsSync(path.join(cwd(), 'src', module.directory, modulePath))) {
              return `${whiteSpace} const ${importedName} = require(../src/${modulePath});`;
            }

            Logger.warn(`Import ${importedName} could not be found in module.directory ${modulePath}.`);
          }

          return line;
        })
        .join('\n');

      return `// Module: ${module.name}
${module.content}`;
    })
    // concat them separated by new lines
    .join('\n\n\n\n\n')
    // split all lines
    .split('\n')
    // filter lines
    .filter(line => {
      // remove strict usage
      if (line === '"use strict";') {
        return false;
      }

      // remove esModule property
      if (line === 'Object.defineProperty(exports, "__esModule", { value: true });') {
        return false;
      }

      // remove source map references
      if (line.indexOf('//# sourceMappingURL=') === 0) {
        return false;
      }

      // keep all other lines
      return true;
    })
    // concat all lines separated by new lines
    .join('\n');

  if (wholeCode.length > 0) {
    // add meta lines to the file
    wholeCode = `"use strict";
Object.defineProperty(exports, "__esModule", { value: true });

${wholeCode}`;

    // write packed JavaScript files
    return writeFilePromisified(
      indexPath,
      `${PACK_IDENTIFIER}

${wholeCode}`,
    );
  }
}

/**
 * Delete file if it exists and is packed by this script
 *
 * @param path Path to file to check/delete
 */
async function deleteFileIfExistingAndPacked(path: string): Promise<void> {
  try {
    const buffer = await readFilePromisified(path);
    const content = buffer.toString();

    // check if packed by this script
    if (content.indexOf(PACK_IDENTIFIER) === 0) {
      Logger.log(`Found '${path}' which is packed by this script. Deleting it...`);

      return unlinkPromisified(path);
    }
  } catch (error) {
    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
      return;
    }
  }
}

/**
 * Get all internal dependencies from the content of a module
 *
 * @param moduleContent Module content to analyze
 */
function getAllInternalDependencies(moduleContent: string): string[] {
  // match all const <name> = require(<moduleName>);
  const requireLines = moduleContent.match(
    /^\s*(const|var) [a-z0-9_]* = require\("([^"]+)"\)|require\('([^']+)'\);$/gim,
  );

  if (Array.isArray(requireLines)) {
    return requireLines
      .map(requireLine => {
        const matches = requireLine.match(/require\("([^"]+)"\)|require\('([^']+)'\);$/i);

        // previously matched require line does not contain a require?!
        if (matches === null) {
          throw new Error();
        }

        // return only the moduleName
        return matches[1];
      })
      .filter(moduleName => {
        // filter out internal modules beginning with './' and not ending with '.json'
        return /^[.]{1,2}\/(?!.*\.json$).*$/i.test(moduleName);
      })
      .map(internalModuleName => {
        // cut './' from the name
        return internalModuleName.slice('./'.length);
      });
  }

  return [];
}

/**
 * Sort modules by their dependencies
 *
 * @param modules Modules to sort
 */
function topologicalSort(modules: JavaScriptModule[]): JavaScriptModule[] {
  // eslint-disable-next-line unicorn/prefer-module,@typescript-eslint/no-var-requires
  const topoSort = require('toposort');

  // vertices are modules, an edge from a to b means that b depends on a
  const edges: string[][] = [];
  const nodes: string[] = [];

  // add all edges
  for (const module of modules) {
    for (const dependencyPath of module.dependencies) {
      // add edge from dependency to our module
      edges.push([path.basename(dependencyPath), module.name]);
    }

    nodes.push(module.name);
  }

  // sort graph and return as an array of sorted modules
  return topoSort.array(nodes, edges).map((moduleName: string) => {
    return modules.find(module => {
      return module.name === moduleName;
    });
  });
}

/**
 * Pack
 */
export async function pack() {
  Logger.log(`Packing project in ${process.cwd()}...`);

  // run all tasks in parallel
  const promises: Array<Promise<void>> = [packCliJs(), packTypeDefinitions(), packJavaScriptFiles()];

  await Promise.all(promises);

  // clean up afterwards
  Logger.info('Deleting extraneous files...');

  await del([
    // delete all transpiled files
    'lib/*',

    // keep packed files
    '!lib/index.d.ts',
    '!lib/index.js',

    // keep converted schema files
    '!lib/schema',
    '!lib/schema/*.json',

    // keep documentation
    '!lib/doc',
    '!lib/doc/*',
    '!lib/doc/**/*',

    // keep cli
    '!lib/cli.js',

    // keep tests
    '!lib/test',
    '!lib/test/*',
    '!lib/test/**/*',
  ]);
}