feat: add projectmanagement and pack

This commit is contained in:
Karl-Philipp Wulfert
2018-11-29 13:36:34 +01:00
commit 19fd0f6e4c
28 changed files with 4801 additions and 0 deletions

160
src/common.ts Normal file
View File

@@ -0,0 +1,160 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Api} from '@openstapps/gitlab-api';
import {Project} from '@openstapps/gitlab-api/lib/types';
import {Logger} from '@openstapps/logger';
import {asyncPool} from 'async-pool-native/dist/async-pool';
import {readFile, unlink, writeFile} from 'fs';
import * as glob from 'glob';
import {basename} from 'path';
import {promisify} from 'util';
import {PACK_IDENTIFIER} from './configuration';
import {JavaScriptModule} from './pack/types';
/**
* Instantiated logger
*/
export const logger = new Logger();
/**
* Get projects for a list of groups
*
* @param api GitLab API to make requests with
* @param groups List of groups
*/
export async function getProjects(api: Api, groups: number[]): Promise<Project[]> {
logger.info('Fetching all projects for specified groups (' + groups.length + ')...');
const projectResults = await asyncPool(3, groups, (groupId) => {
return api.getProjectsForGroup(groupId);
});
const projects = flatten2dArray(projectResults);
logger.log('Fetched ' + projects.length + ' project(s).');
return projects;
}
/**
* Flatten 2d array
*
* @param arr Flattened array
*/
export function flatten2dArray<T>(arr: T[][]): T[] {
return [].concat.apply([], arr);
}
/**
* Promisified version of readFile
*/
export const readFilePromisified = promisify(readFile);
/**
* Promisified version of glob
*/
export const globPromisified = promisify(glob);
/**
* Promisified version of writeFile
*/
export const writeFilePromisified = promisify(writeFile);
/**
* Promisified version of unlink
*/
export const unlinkPromisified = promisify(unlink);
/**
* Delete file if it exists and is packed by this script
*
* @param path Path to file to check/delete
*/
export async function deleteFileIfExistingAndPacked(path: string): Promise<void> {
try {
const buffer = await readFilePromisified(path);
const content = buffer.toString();
// check if packed by this script
if (content.indexOf(PACK_IDENTIFIER) === 0) {
logger.log('Found `' + path + '` which is packed by this script. Deleting it...');
return await unlinkPromisified(path);
}
} catch (err) {
if (err.code === 'ENOENT') {
return;
}
}
}
/**
* Get all internal dependencies from the content of a module
*
* @param moduleContent Module content to analyze
*/
export function getAllInternalDependecies(moduleContent: string): string[] {
// match all const <name> = require(<moduleName>);
const requireLines =
moduleContent.match(/^\s*(const|var) [a-z0-9_]* = require\("([^"]+)"\)|require\('([^']+)'\);$/gmi);
if (Array.isArray(requireLines)) {
return requireLines.map((requireLine) => {
const matches = requireLine.match(/require\("([^"]+)"\)|require\('([^']+)'\);$/i);
// previously matched require line does not contain a require?!
if (matches === null) {
throw new Error();
}
// return only the moduleName
return matches[1];
}).filter((moduleName) => {
// filter out internal modules beginning with './' and not ending with '.json'
return /^[.]{1,2}\/(?!.*\.json$).*$/i.test(moduleName);
}).map((internalModuleName) => {
// cut './' from the name
return internalModuleName.substring(2);
});
}
return [];
}
/**
* Sort modules by their dependencies
*
* @param modules Modules to sort
*/
export function topologicalSort(modules: JavaScriptModule[]): JavaScriptModule[] {
const topoSort = require('toposort');
// vertices are modules, an edge from a to b means that b depends on a
const edges: string[][] = [];
const nodes: string[] = [];
// add all edges
modules.forEach((module) => {
module.dependencies.forEach((dependenciePath) => {
// add edge from dependency to our module
edges.push([basename(dependenciePath), module.name]);
});
nodes.push(module.name);
});
// sort graph and return as an array of sorted modules
return topoSort.array(nodes, edges).map((moduleName: string) => {
return modules.find((module) => {
return module.name === moduleName;
});
});
}

156
src/configuration.ts Normal file
View File

@@ -0,0 +1,156 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Label} from '@openstapps/gitlab-api/lib/types';
// list of school with their IDs
export const SCHOOLS: { [school: string]: number } = {};
// list of group IDs to fetch issues for - 4088298 is openstapps main group
export const GROUPS: number[] = [4088298].concat(Object.keys(SCHOOLS).map((school) => {
return SCHOOLS[school];
}));
// weight for labels to order issues by
export const LABEL_WEIGHTS: any = {
'bug': 1,
'critical': 2,
};
// list of labels to print bold
export const BOLD_LABELS: string[] = [
'bug',
'critical',
];
// GitLab API URL
export const GITLAB_API_URL = 'https://gitlab.com/api/v4/';
// needed milestones
export const NEEDED_MILESTONES = [
'Backlog',
];
// branches to protect
export const PROTECTED_BRANCHES = [
'develop',
'master',
];
// needed labels
export const NEEDED_LABELS: Label[] = [
{
color: '#FF0000',
description: 'An error/something that is not working as expected',
name: 'bug',
},
{
color: '#5CB85C',
name: 'consistency',
},
{
color: '#FF0000',
name: 'confirmed',
},
{
color: '#FF0000',
description: 'A blocking issue/something that needs to be fixed ASAP',
name: 'critical',
},
{
color: '#428BCA',
name: 'design',
},
{
color: '#0033CC',
description: 'An issue about the documentation of the software',
name: 'documentation',
},
{
color: '#5CB85C',
name: 'Doing',
},
{
color: '#5CB85C',
description: 'A feature proposal/something that will be developed',
name: 'feature',
},
{
color: '#7F8C8D',
description: 'An issue that is unimportant or invalid',
name: 'invalid',
},
{
color: '#FFFF88',
name: 'Meeting',
},
{
color: '#8E44AD',
name: 'organization',
},
{
color: '#FF0000',
description: 'An issue with the performance of the software',
name: 'performance',
},
{
color: '#69D100',
name: 'refactoring',
},
{
color: '#FF0000',
description: 'An issue with the security of the software',
name: 'security',
},
{
color: '#D1D100',
description: 'An issue about the testing procedure of the software',
name: 'testing',
},
{
color: '#F0AD4E',
name: 'To Do',
},
{
color: '#A8D695',
description: 'An issue with low priority',
name: 'unimportant',
},
{
color: '#D10069',
description: 'An issue with the usability of the software',
name: 'usability',
},
{
color: '#428BCA',
description: 'Feedback from the feedback-module of the app',
name: 'user-feedback',
},
].concat(Object.keys(SCHOOLS).map((school) => {
return {
color: '#F0AD4E',
description: 'An issue that specifically applies to this school',
name: 'school-' + school,
};
})).concat(['android', 'iOS', 'web', 'node'].map((platform) => {
return {
color: '#FFECDB',
description: 'An issue that specifically applies to this platform',
name: 'platform-' + platform,
};
}));
export const NOTE_PREFIX = '`openstapps/projectmanagement`';
export const PACK_IDENTIFIER = '/* PACKED BY @openstapps/pack */';

65
src/pack.cli.ts Normal file
View File

@@ -0,0 +1,65 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import * as del from 'del';
import {logger} from './common';
import {packCliJs} from './pack/pack-cli';
import {packTypeDefinitions} from './pack/pack-definitions';
import {packJavaScriptFiles} from './pack/pack-modules';
/**
* Pack
*/
async function pack() {
logger.log(`Packing project in ${process.cwd()}...`);
// run all tasks in parallel
const promises: Array<Promise<void>> = [
packCliJs(),
packTypeDefinitions(),
packJavaScriptFiles(),
];
await Promise.all(promises);
// clean up afterwards
logger.info('Deleting extraneous files...');
await del([
// delete all transpiled files
'lib/*',
// keep packed files
'!lib/index.d.ts', '!lib/index.js',
// keep converted schema files
'!lib/schema', '!lib/schema/*.json',
// keep documentation
'!lib/doc', '!lib/doc/*', '!lib/doc/**/*',
// keep cli
'!lib/cli.js',
// keep tests
'!lib/test', '!lib/test/*', '!lib/test/**/*',
]);
}
pack().then(() => {
logger.ok('Packed library!');
}, (err) => {
logger.error(err);
process.exit(-1);
});

88
src/pack/pack-cli.ts Normal file
View File

@@ -0,0 +1,88 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {existsSync} from 'fs';
import {join} from 'path';
import {cwd} from 'process';
import {logger, writeFilePromisified} from '../common';
import {readFilePromisified} from '../common';
/**
* Pack cli.js
*
* This finds all internal requires and replaces the paths with `./index` or internal requires if it has been
* required already.
*
* Furthermore it checks that no shebang line is present and that it does not export anything.
*/
export async function packCliJs(): Promise<void> {
const path = join(cwd(), 'lib', 'cli.js');
if (!existsSync(path)) {
return;
}
logger.info('Adjusting JavaScript CLI...');
const buffer = await readFilePromisified(path);
const content = buffer.toString();
if (content.indexOf('#!/') === 0) {
throw new Error('`cli.js` must not contain a shebang line! It is added by this script.');
}
let internalRequire: string | null = null;
const adjustedContent = '#!/usr/bin/env node\n\n' + content
.split('\n')
.map((line, lineNumber) => {
// check for exports (cli.js is not allowed to export anything)
if (Array.isArray(line.match(/^\s*((exports)|(module\.exports))/))) {
throw new Error(
'Line ' +
lineNumber +
' in cli.js has a reference to the exports object. cli.js is not for exporting. Line was: "'
+ line
+ '"',
);
}
// replace lines with internal requires
// extract module name from line
const match = line.match(/^(\s*)(const|var) ([a-z0-9_]*) = require\(("[^"]+"|'[^']+')\);$/i);
if (match !== null) {
const importedName = match[3];
const moduleName = match[4].substring(1, match[4].length - 1);
// if it begins with '.' and not ends with json
if (/^[.]{1,2}\/(?!.*\.json$).*$/i.test(moduleName)) {
// is the first internal require
if (internalRequire !== null) {
return 'const ' + importedName + ' = ' + internalRequire + ';';
}
// only the first import needs a require
internalRequire = importedName;
return 'const ' + importedName + ' = require("./index");';
}
}
return line;
})
.join('\n');
return await writeFilePromisified(path, adjustedContent);
}

View File

@@ -0,0 +1,128 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {join} from 'path';
import {cwd} from 'process';
import {
deleteFileIfExistingAndPacked,
globPromisified,
logger,
readFilePromisified,
writeFilePromisified,
} from '../common';
import {PACK_IDENTIFIER} from '../configuration';
/**
* Get a list containing the contents of all type definition files
*/
async function getAllTypeDefinitions(): Promise<string[]> {
const fileNames = await globPromisified(join(cwd(), '*(lib|src)', '**', '*.d.ts'), {
ignore: [
join(cwd(), 'lib', 'doc', '**', '*.d.ts'),
join(cwd(), 'lib', 'test', '**', '*.d.ts'),
join(cwd(), 'lib', 'cli.d.ts'),
],
});
const promises = fileNames.map((fileName) => {
return readFilePromisified(fileName, 'utf8');
});
return await Promise.all(promises);
}
/**
* Pack a list of type definitions into one file
*/
export async function packTypeDefinitions(): Promise<void> {
logger.info('Packing TypeScript definition files...');
const path = join(cwd(), 'lib', 'index.d.ts');
await deleteFileIfExistingAndPacked(path);
const typeDefinitions = await getAllTypeDefinitions();
// pack TypeScript definition files
const imports: { [k: string]: string[] } = {};
const referenceLines: string[] = [];
let allDefinitions = typeDefinitions
// concat them separated by new lines
.join('\n\n\n\n\n')
// split all lines
.split('\n')
.map((line) => {
if (line.indexOf('export =') !== -1) {
throw new Error('`export =` is not allowed by pack. Use named imports instead.');
}
if (line.indexOf('/// <reference') === 0) {
referenceLines.push(line);
return '// moved referenced line';
}
// match import lines
const match = line.match(/^import {([^}].*)} from '([^'].*)';$/);
if (match !== null) {
// extract module
const module = match[2];
// extract imported objects
const importedObjects = match[1].split(',').map((object) => {
return object.trim();
});
// add list of already imported objects for module
if (typeof imports[module] === 'undefined') {
imports[module] = [];
}
// count already imported objects and objects to import now
const objectsToImport: string[] = [];
importedObjects.forEach((object) => {
if (imports[module].indexOf(object) === -1) {
imports[module].push(object);
objectsToImport.push(object);
}
});
// replace import line
if (objectsToImport.length === 0) {
return '// extraneous removed import';
} else {
return 'import { ' + objectsToImport.join(', ') + ' } from \'' + module + '\';';
}
}
return line;
})
// filter lines which contain "local" imports
.filter((line) => {
return !line.match(/^import .* from '\./);
})
// concat all lines separated by new lines
.join('\n');
if (allDefinitions.length > 0) {
if (referenceLines.length > 0) {
allDefinitions = referenceLines.join('\n') + '\n\n' + allDefinitions;
}
// write packed TypeScript definition files
return await writeFilePromisified(path, PACK_IDENTIFIER + '\n\n' + allDefinitions);
}
}

148
src/pack/pack-modules.ts Normal file
View File

@@ -0,0 +1,148 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {existsSync} from 'fs';
import {basename, dirname, join} from 'path';
import {cwd} from 'process';
import {
deleteFileIfExistingAndPacked,
getAllInternalDependecies,
globPromisified,
logger,
readFilePromisified,
topologicalSort,
writeFilePromisified,
} from '../common';
import {PACK_IDENTIFIER} from '../configuration';
import {JavaScriptModule} from './types';
/**
* Get all JavaScript modules
*/
async function getAllJavaScriptModules(): Promise<JavaScriptModule[]> {
const fileNames = await globPromisified(join(cwd(), 'lib', '**', '*.js'), {
ignore: [
join(cwd(), 'lib', 'doc', '**', '*.js'),
join(cwd(), 'lib', 'test', '*.js'),
join(cwd(), 'lib', 'cli.js'),
],
});
const promises = fileNames.map(async (fileName) => {
const fileContent = await readFilePromisified(fileName, 'utf8');
const directory = dirname(fileName).replace(new RegExp('^' + join(cwd(), 'lib')), '');
return {
content: '(function() {\n' + fileContent + '\n})();\n',
dependencies: getAllInternalDependecies(fileContent),
directory: directory,
name: basename(fileName, '.js'),
};
});
return await Promise.all(promises);
}
/**
* Pack all javascript files
*/
export async function packJavaScriptFiles(): Promise<void> {
const path = join(cwd(), 'lib', 'index.js');
logger.info('Packing JavaScript files...');
await deleteFileIfExistingAndPacked(path);
// topologically sort the modules (sort by dependencies)
const jsModules = topologicalSort(await getAllJavaScriptModules());
let wholeCode = jsModules
// convert modules to strings
.map((module) => {
module.content = module.content
.split('\n')
.map((line) => {
const match = line.match(
/^(\s*)(const|var) ([a-z0-9_]*) = ((require\("([^"]+)"\))|(require\('([^']+)'\)));$/i,
);
// replace lines with internal requires
if (match !== null) {
// match[6] or match[8] contain the modulePath
if (typeof match[6] === 'undefined') {
match[6] = match[8];
}
const whiteSpace = match[1] ? match[1] : '';
const importedName = match[3];
const modulePath = match[6];
// leave line unchanged if it is a "global" import
if (modulePath.match(/^[.]{1,2}\//) === null) {
return line;
}
// replace internal requires with `module.exports`
if (existsSync(join(cwd(), 'lib', module.directory, modulePath + '.js'))) {
return whiteSpace + 'const ' + importedName + ' = module.exports;';
}
if (existsSync(join(cwd(), 'src', module.directory, modulePath))) {
return whiteSpace + 'const ' + importedName + ' = require(\'../src/' + modulePath + '\');';
}
logger.warn('Import ' + importedName + ' could not be found in module.directory ' + modulePath);
}
return line;
})
.join('\n');
return '// Module: ' + module.name + '\n' + module.content;
})
// concat them separated by new lines
.join('\n\n\n\n\n')
// split all lines
.split('\n')
// filter lines
.filter((line) => {
// remove strict usage
if (line === '"use strict";') {
return false;
}
// remove esModule property
if (line === 'Object.defineProperty(exports, "__esModule", { value: true });') {
return false;
}
// remove source map references
if (line.indexOf('//# sourceMappingURL=') === 0) {
return false;
}
// keep all other lines
return true;
})
// concat all lines separated by new lines
.join('\n');
if (wholeCode.length > 0) {
// add meta lines to the file
wholeCode = '"use strict";\nObject.defineProperty(exports, "__esModule", { value: true });\n\n' + wholeCode;
// write packed JavaScript files
return await writeFilePromisified(path, PACK_IDENTIFIER + '\n\n' + wholeCode);
}
}

38
src/pack/types.ts Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* A JavaScript module representation to sort a list of them by dependencies
*/
export interface JavaScriptModule {
/**
* Content of the module
*/
content: string;
/**
* List of names of dependencies
*/
dependencies: string[];
/**
* Directory the module is in
*/
directory: string;
/**
* The name of the module
*/
name: string;
}

View File

@@ -0,0 +1,71 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Api} from '@openstapps/gitlab-api';
import {Logger} from '@openstapps/logger';
import * as commander from 'commander';
import {existsSync, readFileSync} from 'fs';
import {join} from 'path';
import {GITLAB_API_URL} from './configuration';
import {move} from './tasks/move';
import {report} from './tasks/report';
import {tidy} from './tasks/tidy';
const logger = new Logger();
// add default handler for unhandled rejections
process.on('unhandledRejection', (err) => {
logger.error('UNHANDLED REJECTION', err.stack);
process.exit(1);
});
// check that environment variable GITLAB_PRIVATE_TOKEN is set
if (typeof process.env.GITLAB_PRIVATE_TOKEN !== 'string' || process.env.GITLAB_PRIVATE_TOKEN.length === 0) {
logger.error('Environment variable GITLAB_PRIVATE_TOKEN is not set!');
process.exit(1);
}
const gitlabApi = new Api(GITLAB_API_URL, process.env.GITLAB_PRIVATE_TOKEN as string);
if (existsSync(join(__dirname, 'package.json'))) {
commander.version(JSON.parse(readFileSync(join(__dirname, '..', 'package.json')).toString()).version);
}
commander
.command('report <milestone>')
.action(async (milestone: string) => {
await report(gitlabApi, milestone);
logger.ok('Done!');
});
commander
.command('move')
.action(async () => {
await move(gitlabApi);
logger.ok('Done!');
});
commander
.command('tidy')
.action(async () => {
await tidy(gitlabApi);
logger.ok('Done!');
});
commander
.parse(process.argv);
if (commander.args.length < 1) {
commander.help();
}

99
src/tasks/move.ts Normal file
View File

@@ -0,0 +1,99 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Api} from '@openstapps/gitlab-api';
import {Milestone} from '@openstapps/gitlab-api/lib/types';
import {asyncPool} from 'async-pool-native/dist/async-pool';
import {prompt} from 'inquirer';
import {flatten2dArray, logger} from '../common';
import {GROUPS, NOTE_PREFIX} from '../configuration';
/**
* Move closed issues away from Meeting milestone
*
* @param api Instance of GitLabAPI to send requests with
*/
export async function move(api: Api) {
const issueResults = await asyncPool(3, GROUPS, (groupId) => {
return api.getIssues({
groupId: groupId,
state: 'closed',
});
});
const issues = flatten2dArray(issueResults);
logger.log('Fetched ' + issues.length + ' closed issue(s).');
const milestoneCache: { [s: number]: Milestone[] } = {};
await asyncPool(1, issues, async (issue) => {
const selectedMilestone = issue.milestone;
let milestoneId: number | null = null;
if (selectedMilestone !== null) {
milestoneId = selectedMilestone.id;
}
if (typeof milestoneCache[issue.project_id] === 'undefined') {
const milestones = await api.getMilestonesForProject(issue.project_id);
milestones.sort((a, b) => {
return a.title.localeCompare(b.title);
});
milestoneCache[issue.project_id] = milestones;
}
const selectableMilestones: Array<{ name: string, value: number | null }> = [
{
name: '>skip<',
value: milestoneId,
}, {
name: '>none<',
value: null,
},
];
milestoneCache[issue.project_id].forEach((milestone) => {
selectableMilestones.push({value: milestone.id, name: milestone.title});
});
const answer: any = await prompt({
choices: selectableMilestones,
message: '(' + issue.web_url + '): ' + issue.title,
name: 'Milestone',
type: 'list',
});
const chosenMilestoneId = answer[Object.keys(answer)[0]];
if (chosenMilestoneId === milestoneId) {
logger.info('Milestone unchanged...');
return;
}
await api.setMilestoneForIssue(issue, chosenMilestoneId);
await api.createNote(
issue.project_id,
issue.iid,
`${NOTE_PREFIX} Issue was moved automatically.`,
);
logger.log('Milestone has been updated...');
});
logger.ok('Closed issues have been moved.');
}

358
src/tasks/report.ts Normal file
View File

@@ -0,0 +1,358 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Api} from '@openstapps/gitlab-api';
import {Issue, Project, User} from '@openstapps/gitlab-api/lib/types';
import {asyncPool} from 'async-pool-native/dist/async-pool';
import {readFileSync, writeFile} from 'fs';
import * as moment from 'moment';
import {join} from 'path';
import {cwd} from 'process';
import {promisify} from 'util';
import {flatten2dArray, getProjects, logger} from '../common';
import {BOLD_LABELS, GROUPS, LABEL_WEIGHTS} from '../configuration';
const asyncWriteFile = promisify(writeFile);
/**
* Check if issue state is opened or closed
*
* @param state State to check
*/
export function issueStateIsOpenedOrClosed(state: string): state is 'opened' | 'closed' {
return ['opened', 'closed'].indexOf(state) >= 0;
}
/**
* Get merge request URLs from given data
*
* @param projectMergeRequests Merge requests data (object containing array of objects)
* @param projectId Project ID to get data about merge requests for
* @param issueIid Issue IID in certain project (relative ID, and not issue's GitLab API ID)
*/
export function getMergeRequestUrls(projectMergeRequests: MergeRequestData,
projectId: number,
issueIid: number): string[] {
if (typeof projectMergeRequests[projectId] === 'undefined' || projectMergeRequests[projectId].length === 0) {
return [];
}
return projectMergeRequests[projectId].filter((obj) => {
return obj.issue_iid === issueIid;
}).map((obj) => {
return obj.web_url;
});
}
/**
* An assignee with his/her issues
*/
export interface AssigneeWithIssues {
assignee: User;
issueCounts: {
closed: number;
opened: number;
};
issues: Issue[];
quota: number;
}
export interface MergeRequestData {
[k: string]: Array<{
issue_iid: number;
web_url: string;
}>;
}
/**
* Tangular compiled template
*/
export type TangularCompiled = (model?: any, $?: any) => string;
/**
* Convert an issue to a markdown table row
*
* @param issue Issue to convert
* @param projectBranches Map of branches for project IDs
* @param projectMergeRequests Map of merge requests for project IDs
* @param template Template to render this issue with
*/
function issueToString(issue: any,
projectBranches: any,
projectMergeRequests: any,
template: any): string {
issue._labels = issue.labels.map((label: string) => {
// print specific labels bold
if (BOLD_LABELS.indexOf(label) >= 0) {
label = '__' + label + '__';
}
return label;
}).join(', ');
/* tslint:disable: max-line-length */
return template({
branchExists: typeof projectBranches[issue.project_id] !== 'undefined' && projectBranches[issue.project_id].indexOf(issue.iid) >= 0,
issue: issue,
// take the first URL from the merge request urls array (usually if there are URLs, then there is that only one)
mergeRequestUrl: getMergeRequestUrls(projectMergeRequests, issue.project_id, issue.iid)[0],
project: Api.getProjectPath(issue),
weeksOpen: moment().diff(moment(issue.created_at), 'weeks'),
});
/* tslint:enable */
}
/**
* Get issues from all groups with a specific milestone
*
* @param api GitLab API to make requests with
* @param milestone Milestone to filter by
*/
export async function getIssues(api: Api, milestone: string): Promise<Issue[]> {
const issueResults = await asyncPool(2, GROUPS, (groupId) => {
return api.getIssues({
groupId: groupId,
milestone: milestone as any,
});
});
const issues = flatten2dArray(issueResults);
logger.log('Fetched ' + issues.length + ' issue(s).');
return issues;
}
/**
* Get IDs of issues with branches for projects
*
* @param api GitLab API To make requests with
* @param projects List of projects
*/
export async function getIdsOfIssuesWithBranchesForProjects(
api: Api,
projects: Project[]): Promise<{ [k: string]: number[] }> {
const projectBranches: { [k: string]: number[] } = {};
await asyncPool(2, projects, async (project) => {
const branches = await api.getBranchesForProject(project.id);
// extract issue number from branch
projectBranches[project.id] = branches.map((branch) => {
return branch.name.split('-')[0];
}).filter((branchNameStart) => {
return branchNameStart.match(/^[0-9]+$/);
}).map((branchNameStart) => {
return parseInt(branchNameStart, 10);
});
});
return projectBranches;
}
/**
* Group a list of issues by their assignees
*
* @param issues List of issues to group
*/
export function groupIssuesByAssignee(issues: Issue[]): { [k: number]: AssigneeWithIssues } {
const issuesByAssignee: {
[k: number]: AssigneeWithIssues;
} = {};
issues.forEach((issue) => {
if (issue.assignee === null) {
logger.warn('Issue without assignee!', issue.web_url);
return;
}
if (typeof issuesByAssignee[issue.assignee.id] === 'undefined') {
issuesByAssignee[issue.assignee.id] = {
assignee: issue.assignee,
issueCounts: {
closed: 0,
opened: 0,
},
issues: [],
quota: 0,
};
}
if (issue.state === 'reopened') {
issue.state = 'opened';
}
if (issueStateIsOpenedOrClosed(issue.state)) {
issuesByAssignee[issue.assignee.id].issueCounts[issue.state]++;
issuesByAssignee[issue.assignee.id].issues.push(issue);
}
});
Object.keys(issuesByAssignee).forEach((_assigneeId) => {
const assigneeId = parseInt(_assigneeId, 10);
issuesByAssignee[assigneeId].quota = Math.floor(
issuesByAssignee[assigneeId].issueCounts.closed
/ (issuesByAssignee[assigneeId].issueCounts.opened
+ issuesByAssignee[assigneeId].issueCounts.closed) * 100,
);
});
return issuesByAssignee;
}
/**
* Compile templates
*/
export function compileTemplates(): { [k: string]: TangularCompiled } {
const tangular: { compile: (template: string) => TangularCompiled } = require('tangular');
const templates = {
assigneeFooter: tangular.compile(readFileSync('templates/md/assigneeFooter.md').toString()),
assigneeHeader: tangular.compile(readFileSync('templates/md/assigneeHeader.md').toString()),
header: tangular.compile(readFileSync('templates/md/header.md').toString()),
issue: tangular.compile(readFileSync('templates/md/issue.md').toString()),
};
logger.log('Compiled templates.');
return templates;
}
/**
* Get next meeting day
*/
export function getNextMeetingDay() {
// get "now"
const now = moment();
// get first wednesday of month
const meetingDayMoment = moment().startOf('month').hour(10).isoWeekday(3);
while (meetingDayMoment.isBefore(now)) {
// add one week until meeting day is after now
meetingDayMoment.add(1, 'weeks');
}
const meetingDay = meetingDayMoment.format('YYYY-MM-DD');
// log found meeting day
logger.info('Generating report for ' + meetingDay + ' of ' + GROUPS.length + ' group(s)...');
return meetingDay;
}
/**
* Get a list of merge requests for projects
*
* @param api GitLab API to make requests with
* @param projects List of projects
*/
export async function getMergeRequestsForProjects(api: Api,
projects: Project[]): Promise<MergeRequestData> {
const projectMergeRequests: MergeRequestData = {};
// iterate over projects
await asyncPool(2, projects, async (project) => {
// check if project can have merge requests
if (!project.merge_requests_enabled) {
return;
}
// get all merge requests for project
const mergeRequests = await api.getMergeRequestsForProject(project.id);
// extract issue number from merge request
projectMergeRequests[project.id] = mergeRequests.map((mergeRequest) => {
// keep information about web url too
return {issue_iid: mergeRequest.source_branch.split('-')[0], web_url: mergeRequest.web_url};
}).filter((branchNameStartAndUrl) => {
return branchNameStartAndUrl.issue_iid.match(/^[0-9]+$/);
}).map((branchNameStartAndUrl) => {
return {
issue_iid: parseInt(branchNameStartAndUrl.issue_iid, 10),
web_url: branchNameStartAndUrl.web_url,
};
});
});
return projectMergeRequests;
}
export async function report(api: Api, milestone: string) {
const templates = compileTemplates();
const projects = await getProjects(api, GROUPS);
const issues = await getIssues(api, milestone);
const issuesGroupedByAssignee = groupIssuesByAssignee(issues);
const issueBranches = await getIdsOfIssuesWithBranchesForProjects(api, projects);
const mergeRequests = await getMergeRequestsForProjects(api, projects);
const meetingDay = getNextMeetingDay();
let allMarkdown = templates.header();
Object.keys(issuesGroupedByAssignee).forEach((_assigneeId) => {
const assigneeId = parseInt(_assigneeId, 10);
allMarkdown += templates.assigneeHeader(issuesGroupedByAssignee[assigneeId]);
issuesGroupedByAssignee[assigneeId].issues.sort((a, b) => {
let weightA = 0;
let weightB = 0;
Object.keys(LABEL_WEIGHTS).forEach((label: string) => {
if (a.labels.indexOf(label) >= 0) {
weightA += LABEL_WEIGHTS[label];
}
if (b.labels.indexOf(label) >= 0) {
weightB += LABEL_WEIGHTS[label];
}
});
if (a.state === 'closed') {
weightA -= 10;
}
if (b.state === 'closed') {
weightB -= 10;
}
return weightB - weightA;
}).forEach((issue) => {
allMarkdown += issueToString(
issue,
issueBranches,
mergeRequests,
templates.issue,
);
});
allMarkdown += templates.assigneeFooter(issuesGroupedByAssignee[assigneeId]);
});
let filename = join(cwd(), 'reports', meetingDay + '.md');
if (milestone === 'Backlog') {
filename = join(cwd(), 'reports', 'Backlog.md');
}
await asyncWriteFile(filename, allMarkdown);
logger.ok('Wrote file `' + filename + '`.');
}

268
src/tasks/tidy.ts Normal file
View File

@@ -0,0 +1,268 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Api} from '@openstapps/gitlab-api';
import {AccessLevel, Label, Milestone, Project} from '@openstapps/gitlab-api/lib/types';
import {asyncPool} from 'async-pool-native/dist/async-pool';
import {flatten2dArray, getProjects, logger} from '../common';
import {GROUPS, NEEDED_LABELS, NEEDED_MILESTONES, NOTE_PREFIX, PROTECTED_BRANCHES, SCHOOLS} from '../configuration';
/**
* Tidy issues without milestone
*
* This will set the milestone of issues without milestone to 'Meeting'.
*
* @param api GitLab API instance to use for the requests
*/
export async function tidyIssuesWithoutMilestone(api: Api): Promise<void> {
// fetch issues without milestone from all groups
const issueResults = await asyncPool(3, GROUPS, (groupId) => {
return api.getIssues({
groupId: groupId,
milestone: 'No Milestone',
state: 'opened',
});
});
// flatten structure, e.g. put all issues in one array
const issuesWithoutMilestone = flatten2dArray(issueResults);
logger.info('Found `' + issuesWithoutMilestone.length + '` issue(s) without milestone.');
const milestoneCache: { [s: number]: Milestone[] } = {};
await asyncPool(5, issuesWithoutMilestone, async (issue) => {
if (typeof milestoneCache[issue.project_id] === 'undefined') {
milestoneCache[issue.project_id] = await api.getMilestonesForProject(issue.project_id);
}
let milestoneId = null;
milestoneCache[issue.project_id].forEach((milestone) => {
if (milestone.title === 'Meeting') {
milestoneId = milestone.id;
}
});
if (milestoneId === null) {
logger.warn('Milestone `Meeting` was not available for issue ' + issue.title + ' (' + issue.web_url + ').');
return;
}
await api.setMilestoneForIssue(issue, milestoneId);
logger.log('Milestone was set to `Meeting` for issue ' + issue.title + ' (' + issue.web_url + ').');
await api.createNote(
issue.project_id,
issue.iid,
`${NOTE_PREFIX} Milestone was set automatically to \`Meeting\`.`,
);
});
logger.ok('Tidied issues without milestones.');
}
/**
* Tidy labels in a list of projects
*
* @param api GitLab API instance to use for the requests
* @param projects List of projects to tidy labels on
*/
export async function tidyLabels(api: Api, projects: Project[]): Promise<void> {
await asyncPool(5, projects, async (project) => {
const labels = await api.getLabels(project.id);
const neededLabels = NEEDED_LABELS.slice(0);
const extraneousLabels: Label[] = [];
labels.forEach((label) => {
let needed = false;
neededLabels.forEach((neededLabel, neededLabelIdx) => {
if (neededLabel.name.toLowerCase() === label.name.toLowerCase()) {
neededLabels.splice(neededLabelIdx, 1);
needed = true;
}
});
if (!needed) {
extraneousLabels.push(label);
}
});
await asyncPool(2, neededLabels, async (neededLabel) => {
await api.createLabel(project.id, neededLabel.name, neededLabel.description, neededLabel.color);
logger.log('Created label `' + neededLabel.name + '` in ' + project.name_with_namespace + '.');
});
await asyncPool(2, extraneousLabels, async (extraneousLabel) => {
await api.deleteLabel(project.id, extraneousLabel.name);
logger.log('Deleted label `' + extraneousLabel.name + '` from ' + project.name_with_namespace + '.');
});
});
logger.ok('Tidied labels.');
}
/**
* Tidy milestones in a list of projects
*
* @param api GitLab API instance to use for the requests
* @param projects List of projects to tidy milestones on
*/
export async function tidyMilestones(api: Api, projects: Project[]): Promise<void> {
await asyncPool(5, projects, async (project) => {
const milestones = await api.getMilestonesForProject(project.id);
const missingMilestones = NEEDED_MILESTONES.slice(0);
milestones.forEach((milestone) => {
const idx = missingMilestones.indexOf(milestone.title);
if (idx >= 0) {
missingMilestones.splice(idx, 1);
}
});
if (missingMilestones.length > 0) {
await asyncPool(2, missingMilestones, async (milestone) => {
await api.createMilestone(project.id, milestone);
logger.log('Created milestone ' + milestone + ' for project ' + project.name_with_namespace + '.');
});
}
});
logger.ok('Tidied milestones.');
}
/**
* Tidy protected branches in a list of projects
*
* @param api GitLab API instance to use for the requests
* @param projects List of projects to tidy milestones on
*/
export async function tidyProtectedBranches(api: Api, projects: Project[]): Promise<void> {
await asyncPool(2, projects, async (project) => {
const branches = await api.getBranchesForProject(project.id);
const protectableBranches = branches.filter((branch) => {
return PROTECTED_BRANCHES.indexOf(branch.name) >= 0;
});
const unprotectedBranches = protectableBranches.filter((branch) => {
return !branch.protected;
});
await asyncPool(2, unprotectedBranches, async (branch) => {
await api.protectBranch(project.id, branch.name);
logger.log('Added protected branch `' + branch.name + '` in project `' + project.name_with_namespace + '`...');
});
});
logger.ok('Tidied protected branches.');
}
/**
* Tidy "sub" group members
*
* @param api GitLab API instance to use for the requests
*/
export async function tidySubGroupMembers(api: Api): Promise<void> {
const stappsMembers = await api.getMembers('groups', GROUPS[0]);
const stappsMemberIds = stappsMembers.map((member) => member.id);
const groupIdsToSchool: any = {};
Object.keys(SCHOOLS).map((school) => {
groupIdsToSchool[SCHOOLS[school]] = school;
});
await asyncPool(2, GROUPS.slice(1), async (groupId) => {
const members = await api.getMembers('groups', groupId);
const memberIds = members.map((member) => member.id);
await asyncPool(2, stappsMembers, async (stappsMember) => {
if (memberIds.indexOf(stappsMember.id) === -1) {
await api.addMember('groups', groupId, stappsMember.id, AccessLevel.Developer);
logger.log('Added ' + stappsMember.name + ' to group ' + groupIdsToSchool[groupId] + '.');
}
});
await asyncPool(2, members, async (member) => {
if (stappsMemberIds.indexOf(member.id) === -1) {
await api.deleteMember('groups', groupId, member.id);
logger.log('Deleted member ' + member.name + ' from group ' + groupIdsToSchool[groupId] + '.');
}
});
});
logger.ok('Tidied "sub" group members.');
}
export async function tidyIssuesWithoutAssignee(api: Api): Promise<void> {
// fetch issues without milestone from all groups
const issueResults = await asyncPool(3, GROUPS, (groupId) => {
return api.getIssues({
groupId: groupId,
state: 'opened',
});
});
// flatten structure, e.g. put all issues in one array
const issues = flatten2dArray(issueResults);
const issuesWithoutAssignee = issues.filter((issue) => {
return issue.assignee === null;
});
logger.info('Found `' + issuesWithoutAssignee.length + '` issue(s) without assignee.');
await asyncPool(3, issuesWithoutAssignee, async (issue) => {
await api.setAssigneeForIssue(issue, issue.author.id);
logger.log('Set assignee for `' + issue.title + '` to ' + issue.author.name + '.');
await api.createNote(
issue.project_id,
issue.iid,
`${NOTE_PREFIX} Assignee was set automatically to author.`,
);
});
logger.ok('Tidied issues without assignee.');
}
/**
* Tidy
*
* @param api GitLab API instance to use for the requests
*/
export async function tidy(api: Api) {
const projects = await getProjects(api, GROUPS);
await Promise.all([
tidyLabels(api, projects),
tidyMilestones(api, projects),
tidyProtectedBranches(api, projects),
tidySubGroupMembers(api),
]);
await tidyIssuesWithoutMilestone(api);
await tidyIssuesWithoutAssignee(api);
}