Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-21 17:12:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: modernize core-tools

Browse Source
This commit is contained in:
Wieland Schöbl
2021-08-25 09:47:36 +00:00
parent 106dd26f89
commit fe59204b42
106 changed files with 4131 additions and 6216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
src/pack.ts
View File

@@ -1,5 +1,6 @@
/* eslint-disable unicorn/error-message */
/*
* Copyright (C) 2019 StApps
* Copyright (C) 2021 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
@@ -15,37 +16,13 @@
import {Logger} from '@openstapps/logger';
import del from 'del';
import {existsSync} from 'fs';
import {basename, dirname, join} from 'path';
import {cwd} from 'process';
import {globPromisified, readFilePromisified, unlinkPromisified, writeFilePromisified} from './common';
import {JavaScriptModule} from './types/pack';
import path from 'path';
const PACK_IDENTIFIER = '/* PACKED BY @openstapps/pack */';
/**
* A JavaScript module representation to sort a list of them by dependencies
*/
interface JavaScriptModule {
/**
* Content of the module
*/
content: string;
/**
* List of names of dependencies
*/
dependencies: string[];
/**
* Directory the module is in
*/
directory: string;
/**
* The name of the module
*/
name: string;
}
/**
* Pack cli.js
*
@@ -55,32 +32,30 @@ interface JavaScriptModule {
* Furthermore it checks that no shebang line is present and that it does not export anything.
*/
async function packCliJs(): Promise<void> {
const path = join(cwd(), 'lib', 'cli.js');
const cliPath = path.join(cwd(), 'lib', 'cli.js');
if (!existsSync(path)) {
if (!existsSync(cliPath)) {
return;
}
Logger.info('Adjusting JavaScript CLI...');
const buffer = await readFilePromisified(path);
const buffer = await readFilePromisified(cliPath);
const content = buffer.toString();
if (content.indexOf('#!/') === 0) {
throw new Error('`cli.js` must not contain a shebang line! It is added by this script.');
}
let internalRequire: string | null = null;
let internalRequire: string | undefined;
const adjustedContent = content
.split('\n')
.map((line, lineNumber) => {
// check for exports (cli.js is not allowed to export anything)
if (Array.isArray(line.match(/^\s*((exports)|(module\.exports))/))) {
throw new Error(
`Line '${lineNumber}' in 'cli.js' exports something. cli.js is not for exporting. Line was:
${line}`,
throw new TypeError(
`Line '${lineNumber}' in 'cli.js' exports something. cli.js is not for exporting. Line was:\n${line}`,
);
}
@@ -89,16 +64,14 @@ ${line}`,
const match = line.match(/^(\s*)(const|var) ([a-z0-9_]*) = require\(("[^"]+"|'[^']+')\);$/i);
if (match !== null) {
// tslint:disable-next-line:no-magic-numbers
const importedName = match[3];
// tslint:disable-next-line:no-magic-numbers
// eslint-disable-next-line unicorn/prefer-string-slice
const moduleName = match[4].substring(1, match[4].length - 1);
// if it begins with '.' and not ends with json
if (/^[.]{1,2}\/(?!.*\.json$).*$/i.test(moduleName)) {
// is the first internal require
if (internalRequire !== null) {
if (internalRequire) {
return `const ${importedName} = ${internalRequire};`;
}
@@ -113,20 +86,18 @@ ${line}`,
})
.join('\n');
return writeFilePromisified(path, `#!/usr/bin/env node
${adjustedContent}`);
return writeFilePromisified(cliPath, `#!/usr/bin/env node\n\n${adjustedContent}`);
}
/**
* Get a list containing the contents of all type definition files
*/
async function getAllTypeDefinitions(): Promise<string[]> {
const fileNames = await globPromisified(join(cwd(), '*(lib|src)', '**', '*.d.ts'), {
const fileNames = await globPromisified(path.join(cwd(), '*(lib|src)', '**', '*.d.ts'), {
ignore: [
join(cwd(), 'lib', 'doc', '**', '*.d.ts'),
join(cwd(), 'lib', 'test', '**', '*.d.ts'),
join(cwd(), 'lib', 'cli.d.ts'),
path.join(cwd(), 'lib', 'doc', '**', '*.d.ts'),
path.join(cwd(), 'lib', 'test', '**', '*.d.ts'),
path.join(cwd(), 'lib', 'cli.d.ts'),
],
});
@@ -143,24 +114,24 @@ async function getAllTypeDefinitions(): Promise<string[]> {
async function packTypeDefinitions(): Promise<void> {
Logger.info('Packing TypeScript definition files...');
const path = join(cwd(), 'lib', 'index.d.ts');
const indexPath = path.join(cwd(), 'lib', 'index.d.ts');
await deleteFileIfExistingAndPacked(path);
await deleteFileIfExistingAndPacked(indexPath);
const typeDefinitions = await getAllTypeDefinitions();
// pack TypeScript definition files
const imports: { [k: string]: string[]; } = {};
const imports: {[k: string]: string[]} = {};
const referenceLines: string[] = [];
let allDefinitions = typeDefinitions
// concat them separated by new lines
// concat them separated by new lines
.join('\n\n\n\n\n')
// split all lines
.split('\n')
.map((line) => {
if (line.indexOf('export =') !== -1) {
.map(line => {
if (line.includes('export =')) {
throw new Error('`export =` is not allowed by pack. Use named imports instead.');
}
@@ -174,15 +145,12 @@ async function packTypeDefinitions(): Promise<void> {
const match = line.match(/^import {([^}].*)} from '([^'].*)';$/);
if (match !== null) {
// tslint:disable-next-line:no-magic-numbers - extract module
const module = match[2];
// extract imported objects
const importedObjects = match[1]
.split(',')
.map((object) => {
return object.trim();
});
const importedObjects = match[1].split(',').map(object => {
return object.trim();
});
// add list of already imported objects for module
if (typeof imports[module] === 'undefined') {
@@ -191,12 +159,12 @@ async function packTypeDefinitions(): Promise<void> {
// count already imported objects and objects to import now
const objectsToImport: string[] = [];
importedObjects.forEach((object) => {
if (imports[module].indexOf(object) === -1) {
for (const object of importedObjects) {
if (!imports[module].includes(object)) {
imports[module].push(object);
objectsToImport.push(object);
}
});
}
// replace import line
if (objectsToImport.length === 0) {
@@ -209,7 +177,7 @@ async function packTypeDefinitions(): Promise<void> {
return line;
})
// filter lines which contain "local" imports
.filter((line) => {
.filter(line => {
return line.match(/^import .* from '\./) === null;
})
// concat all lines separated by new lines
@@ -223,9 +191,12 @@ ${allDefinitions}`;
}
// write packed TypeScript definition files
return writeFilePromisified(path, `${PACK_IDENTIFIER}
return writeFilePromisified(
indexPath,
`${PACK_IDENTIFIER}
${allDefinitions}`);
${allDefinitions}`,
);
}
}
@@ -233,18 +204,17 @@ ${allDefinitions}`);
* Get all JavaScript modules
*/
async function getAllJavaScriptModules(): Promise<JavaScriptModule[]> {
const fileNames = await globPromisified(join(cwd(), 'lib', '**', '*.js'), {
const fileNames = await globPromisified(path.join(cwd(), 'lib', '**', '*.js'), {
ignore: [
join(cwd(), 'lib', 'doc', '**', '*.js'),
join(cwd(), 'lib', 'test', '*.js'),
join(cwd(), 'lib', 'cli.js'),
path.join(cwd(), 'lib', 'doc', '**', '*.js'),
path.join(cwd(), 'lib', 'test', '*.js'),
path.join(cwd(), 'lib', 'cli.js'),
],
});
const promises = fileNames.map(async (fileName: string) => {
const fileContent = await readFilePromisified(fileName, 'utf8');
const directory = dirname(fileName)
.replace(new RegExp(`^${join(cwd(), 'lib')}`), '');
const directory = path.dirname(fileName).replace(new RegExp(`^${path.join(cwd(), 'lib')}`), '');
return {
content: `(function() {
@@ -253,7 +223,7 @@ ${fileContent}
`,
dependencies: getAllInternalDependencies(fileContent),
directory: directory,
name: basename(fileName, '.js'),
name: path.basename(fileName, '.js'),
};
});
@@ -264,37 +234,33 @@ ${fileContent}
* Pack all javascript files
*/
async function packJavaScriptFiles(): Promise<void> {
const path = join(cwd(), 'lib', 'index.js');
const indexPath = path.join(cwd(), 'lib', 'index.js');
Logger.info('Packing JavaScript files...');
await deleteFileIfExistingAndPacked(path);
await deleteFileIfExistingAndPacked(indexPath);
// topologically sort the modules (sort by dependencies)
const jsModules = topologicalSort(await getAllJavaScriptModules());
let wholeCode = jsModules
// convert modules to strings
.map((module) => {
// convert modules to strings
.map(module => {
module.content = module.content
.split('\n')
.map((line) => {
.map(line => {
const match = line.match(
/^(\s*)(const|var) ([a-z0-9_]*) = ((require\("([^"]+)"\))|(require\('([^']+)'\)));$/i,
);
// replace lines with internal requires
if (match !== null) {
// tslint:disable-next-line:no-magic-numbers - match[6] or match[8] contain the modulePath
if (typeof match[6] === 'undefined') {
// tslint:disable-next-line:no-magic-numbers
match[6] = match[8];
}
const whiteSpace = (typeof match[1] === 'string' && match[1].length > 0) ? match[1] : '';
// tslint:disable-next-line:no-magic-numbers
const whiteSpace = typeof match[1] === 'string' && match[1].length > 0 ? match[1] : '';
const importedName = match[3];
// tslint:disable-next-line:no-magic-numbers
const modulePath = match[6];
// leave line unchanged if it is a "global" import
@@ -303,11 +269,11 @@ async function packJavaScriptFiles(): Promise<void> {
}
// replace internal requires with `module.exports`
if (existsSync(join(cwd(), 'lib', module.directory, `${modulePath}.js`))) {
if (existsSync(path.join(cwd(), 'lib', module.directory, `${modulePath}.js`))) {
return `${whiteSpace}const ${importedName} = module.exports;`;
}
if (existsSync(join(cwd(), 'src', module.directory, modulePath))) {
if (existsSync(path.join(cwd(), 'src', module.directory, modulePath))) {
return `${whiteSpace} const ${importedName} = require(../src/${modulePath});`;
}
@@ -326,7 +292,7 @@ ${module.content}`;
// split all lines
.split('\n')
// filter lines
.filter((line) => {
.filter(line => {
// remove strict usage
if (line === '"use strict";') {
return false;
@@ -356,9 +322,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
${wholeCode}`;
// write packed JavaScript files
return writeFilePromisified(path, `${PACK_IDENTIFIER}
return writeFilePromisified(
indexPath,
`${PACK_IDENTIFIER}
${wholeCode}`);
${wholeCode}`,
);
}
}
@@ -378,8 +347,8 @@ async function deleteFileIfExistingAndPacked(path: string): Promise<void> {
return unlinkPromisified(path);
}
} catch (err) {
if (err.code === 'ENOENT') {
} catch (error) {
if (error.code === 'ENOENT') {
return;
}
}
@@ -392,12 +361,13 @@ async function deleteFileIfExistingAndPacked(path: string): Promise<void> {
*/
function getAllInternalDependencies(moduleContent: string): string[] {
// match all const <name> = require(<moduleName>);
const requireLines =
moduleContent.match(/^\s*(const|var) [a-z0-9_]* = require\("([^"]+)"\)|require\('([^']+)'\);$/gmi);
const requireLines = moduleContent.match(
/^\s*(const|var) [a-z0-9_]* = require\("([^"]+)"\)|require\('([^']+)'\);$/gim,
);
if (Array.isArray(requireLines)) {
return requireLines
.map((requireLine) => {
.map(requireLine => {
const matches = requireLine.match(/require\("([^"]+)"\)|require\('([^']+)'\);$/i);
// previously matched require line does not contain a require?!
@@ -408,13 +378,13 @@ function getAllInternalDependencies(moduleContent: string): string[] {
// return only the moduleName
return matches[1];
})
.filter((moduleName) => {
.filter(moduleName => {
// filter out internal modules beginning with './' and not ending with '.json'
return /^[.]{1,2}\/(?!.*\.json$).*$/i.test(moduleName);
})
.map((internalModuleName) => {
.map(internalModuleName => {
// cut './' from the name
return internalModuleName.substring('./'.length);
return internalModuleName.slice('./'.length);
});
}
@@ -427,6 +397,7 @@ function getAllInternalDependencies(moduleContent: string): string[] {
* @param modules Modules to sort
*/
function topologicalSort(modules: JavaScriptModule[]): JavaScriptModule[] {
// eslint-disable-next-line unicorn/prefer-module,@typescript-eslint/no-var-requires
const topoSort = require('toposort');
// vertices are modules, an edge from a to b means that b depends on a
@@ -434,23 +405,21 @@ function topologicalSort(modules: JavaScriptModule[]): JavaScriptModule[] {
const nodes: string[] = [];
// add all edges
modules.forEach((module) => {
module.dependencies.forEach((dependencyPath) => {
for (const module of modules) {
for (const dependencyPath of module.dependencies) {
// add edge from dependency to our module
edges.push([basename(dependencyPath), module.name]);
});
edges.push([path.basename(dependencyPath), module.name]);
}
nodes.push(module.name);
});
}
// sort graph and return as an array of sorted modules
return topoSort
.array(nodes, edges)
.map((moduleName: string) => {
return modules.find((module) => {
return module.name === moduleName;
});
return topoSort.array(nodes, edges).map((moduleName: string) => {
return modules.find(module => {
return module.name === moduleName;
});
});
}
/**
@@ -460,11 +429,7 @@ export async function pack() {
Logger.log(`Packing project in ${process.cwd()}...`);
// run all tasks in parallel
const promises: Array<Promise<void>> = [
packCliJs(),
packTypeDefinitions(),
packJavaScriptFiles(),
];
const promises: Array<Promise<void>> = [packCliJs(), packTypeDefinitions(), packJavaScriptFiles()];
await Promise.all(promises);
@@ -476,18 +441,24 @@ export async function pack() {
'lib/*',
// keep packed files
'!lib/index.d.ts', '!lib/index.js',
'!lib/index.d.ts',
'!lib/index.js',
// keep converted schema files
'!lib/schema', '!lib/schema/*.json',
'!lib/schema',
'!lib/schema/*.json',
// keep documentation
'!lib/doc', '!lib/doc/*', '!lib/doc/**/*',
'!lib/doc',
'!lib/doc/*',
'!lib/doc/**/*',
// keep cli
'!lib/cli.js',
// keep tests
'!lib/test', '!lib/test/*', '!lib/test/**/*',
'!lib/test',
'!lib/test/*',
'!lib/test/**/*',
]);
}