refactor: migrate @krlwlfrt/async-pool to @openstapps/collection-utils

This commit is contained in:
2023-06-29 17:29:49 +02:00
parent 64caebafe5
commit 98546a97a3
58 changed files with 621 additions and 1175 deletions

View File

@@ -1,7 +1,7 @@
{
"name": "@openstapps/projectmanagement",
"description": "Main documentation and scripts for maintenance.",
"version": "3.0.0",
"version": "2.0.0",
"private": true,
"type": "module",
"license": "GPL-3.0-only",
@@ -36,6 +36,7 @@
"test": "c8 mocha"
},
"dependencies": {
"@openstapps/collection-utils": "workspace:*",
"@openstapps/gitlab-api": "workspace:*",
"@openstapps/logger": "workspace:*",
"@slack/web-api": "6.8.1",

View File

@@ -22,7 +22,7 @@ import {GITLAB_API_URL} from './configuration.js';
import {getUsedVersionMajorMinor} from './tasks/get-used-version.js';
import {remind} from './tasks/remind.js';
import {tidy} from './tasks/tidy.js';
import {unlabel} from './tasks/unlabel.js';
import {removeLabel} from './tasks/remove-label.js';
// add default handler for unhandled rejections
process.on('unhandledRejection', async reason => {
@@ -57,7 +57,7 @@ if (existsSync(path.join(__dirname, 'package.json'))) {
}
commander.command('unlabel').action(async () => {
await unlabel(gitlabApi);
await removeLabel(gitlabApi);
Logger.ok('Done!');
});

View File

@@ -14,6 +14,8 @@
*/
import {Api, Project} from '@openstapps/gitlab-api';
import {Logger} from '@openstapps/logger';
import {mapAsyncLimit} from '@openstapps/collection-utils';
import {CONCURRENCY} from './configuration.js';
/**
* Get projects for a list of groups
@@ -23,8 +25,7 @@ import {Logger} from '@openstapps/logger';
export async function getProjects(api: Api, groups: number[]): Promise<Project[]> {
Logger.info(`Fetching all projects for specified groups (${groups.length})...`);
const projectResults = await Promise.all(groups.map(api.getProjectsForGroup));
const projects = projectResults.flat();
const projects = await mapAsyncLimit(groups, api.getProjectsForGroup, CONCURRENCY).then(it => it.flat());
Logger.log(`Fetched ${projects.length} project(s).`);

View File

@@ -185,6 +185,11 @@ export const NOTE_PREFIX = '`openstapps/projectmanagement`';
*/
export const SLACK_CHANNEL = 'C762UG76Z';
/**
* Concurrency for network requests
*/
export const CONCURRENCY = 3;
/**
* Maximum depth for merge request reminders
*/

View File

@@ -23,14 +23,15 @@ import {
} from '@openstapps/gitlab-api';
import {Logger} from '@openstapps/logger';
import {WebClient} from '@slack/web-api';
import {GROUPS, MAX_DEPTH_FOR_REMINDER, NOTE_PREFIX, SLACK_CHANNEL} from '../configuration.js';
import {CONCURRENCY, GROUPS, MAX_DEPTH_FOR_REMINDER, NOTE_PREFIX, SLACK_CHANNEL} from '../configuration.js';
import {mapAsyncLimit} from '@openstapps/collection-utils';
/**
* Remind people of open merge requests
* @param api GitLab API to make requests with
*/
export async function remind(api: Api): Promise<void> {
// get list of open merge requests
// get a list of open merge requests
const allMergeRequests = await api.getMergeRequests(
MembershipScope.GROUPS,
GROUPS[0],
@@ -55,7 +56,7 @@ export async function remind(api: Api): Promise<void> {
const client =
process.env.SLACK_API_TOKEN === undefined ? undefined : new WebClient(process.env.SLACK_API_TOKEN);
// get members of main group
// get members of the main group
const members = await api.getMembers(MembershipScope.GROUPS, GROUPS[0]);
// filter members with at least maintainer status
@@ -71,8 +72,9 @@ export async function remind(api: Api): Promise<void> {
Logger.info(`Found ${maintainers.length} maintainer(s).`);
await Promise.all(
mergeRequests.map(async mergeRequest => {
await mapAsyncLimit(
mergeRequests,
async mergeRequest => {
// check if merge request is WIP
if (mergeRequest.work_in_progress) {
Logger.info(`Merge request '${mergeRequest.title}' is WIP.`);
@@ -118,7 +120,7 @@ export async function remind(api: Api): Promise<void> {
// get possible appropers, prefixed with '@' and joined with commas
const possibleApprovers = maintainerUsernames
.filter(username => {
if (mergeRequest.assignee.username === username) {
if (mergeRequest!.assignee.username === username) {
return false;
}
if (username.includes('openstapps') || username.includes('kphilipp')) {
@@ -168,7 +170,7 @@ export async function remind(api: Api): Promise<void> {
// prefix maintainers with '@' and join with commas
const possibleMergers = maintainerUsernames
.filter(username => {
return mergeRequest.assignee.username !== username;
return mergeRequest!.assignee.username !== username;
})
.map(username => `@${username}`)
.join(', ');
@@ -182,6 +184,7 @@ export async function remind(api: Api): Promise<void> {
);
}
}
}),
},
CONCURRENCY,
);
}

View File

@@ -14,27 +14,31 @@
*/
import {Api, IssueState, Scope} from '@openstapps/gitlab-api';
import {Logger} from '@openstapps/logger';
import {GROUPS, LAST_MEETING, NOTE_PREFIX} from '../configuration.js';
import {CONCURRENCY, GROUPS, LAST_MEETING, NOTE_PREFIX} from '../configuration.js';
import isBefore from 'date-fns/isBefore';
import {mapAsyncLimit} from '@openstapps/collection-utils';
/**
* Remove label `meeting` from closed issues
* @param api Instance of GitLabAPI to send requests with
*/
export async function unlabel(api: Api) {
const issues = await Promise.all(
GROUPS.map(async groupId => {
export async function removeLabel(api: Api) {
const issues = await mapAsyncLimit(
GROUPS,
async groupId => {
return api.getIssues({
groupId: groupId,
state: IssueState.CLOSED,
});
}),
},
CONCURRENCY,
).then(it => it.flat());
Logger.log(`Fetched ${issues.length} closed issue(s).`);
await Promise.all(
issues.map(async issue => {
await mapAsyncLimit(
issues,
async issue => {
if (
issue.labels.includes('meeting') &&
issue.closed_at !== null &&
@@ -52,7 +56,8 @@ export async function unlabel(api: Api) {
/unlabel ~meeting`,
);
}
}),
},
CONCURRENCY,
);
Logger.ok('Label `meeting` has been removed from closed issues.');

View File

@@ -26,12 +26,13 @@ import mustache from 'mustache';
import path from 'path';
import {cwd} from 'process';
import {getProjects} from '../common.js';
import {BOLD_LABELS, GROUPS, LABEL_WEIGHTS, NEXT_MEETING} from '../configuration.js';
import {BOLD_LABELS, CONCURRENCY, GROUPS, LABEL_WEIGHTS, NEXT_MEETING} from '../configuration.js';
import differenceInWeeks from 'date-fns/differenceInWeeks';
import formatISO from 'date-fns/formatISO';
import format from 'date-fns/format';
import de from 'date-fns/locale/de';
import {readFile, writeFile} from 'fs/promises';
import {mapAsyncLimit} from '@openstapps/collection-utils';
/**
* A structure for template compilation
@@ -94,11 +95,11 @@ export interface AssigneeWithIssues {
*/
export interface IssueWithMeta extends Issue {
/**
* Whether or not an issue branch exists
* Whether an issue branch exists
*/
$branchExists: boolean;
/**
* Whether or not the issue is closed
* Whether the issue is closed
*/
$closed: boolean;
/**
@@ -106,7 +107,7 @@ export interface IssueWithMeta extends Issue {
*/
$labels: Array<{
/**
* Whether or not to print the label bold
* Whether to print the label bold
*/
bold: boolean;
/**
@@ -119,7 +120,7 @@ export interface IssueWithMeta extends Issue {
*/
$mergeRequestUrl: string;
/**
* Name of project
* Name of the project
*/
$project: string;
/**
@@ -158,7 +159,7 @@ export function issueStateIsOpenedOrClosed(
* Get merge request URLs from given data
* @param projectMergeRequests Merge requests data (object containing array of objects)
* @param projectId Project ID to get data about merge requests for
* @param issueIid Issue IID in certain project (relative ID, and not issue's GitLab API ID)
* @param issueIid Issue IID in a specific project (relative ID, and not issue's GitLab API ID)
*/
export function getMergeRequestUrls(
projectMergeRequests: MergeRequestsForProjects,
@@ -170,12 +171,8 @@ export function getMergeRequestUrls(
}
return projectMergeRequests[projectId]
.filter(object => {
return object.issue_iid === issueIid;
})
.map(object => {
return object.web_url;
});
.filter(object => object.issue_iid === issueIid)
.map(object => object.web_url);
}
/**
@@ -193,9 +190,7 @@ export async function getIssues(api: Api, label: string, groups: number[]): Prom
),
);
const issues = issueResults.flat().filter(issue => {
return issue.labels.includes(label);
});
const issues = issueResults.flat().filter(issue => issue.labels.includes(label));
Logger.log(`Fetched ${issues.length} issue(s).`);
@@ -210,8 +205,9 @@ export async function getIssues(api: Api, label: string, groups: number[]): Prom
export async function getIssueBranches(api: Api, projects: Project[]): Promise<{[k: string]: number[]}> {
const projectBranches: {[k: string]: number[]} = {};
await Promise.all(
projects.map(async project => {
await mapAsyncLimit(
projects,
async project => {
const branches = await api.getBranchesForProject(project.id);
// extract issue number from branch
@@ -225,7 +221,8 @@ export async function getIssueBranches(api: Api, projects: Project[]): Promise<{
.map(branchNameStart => {
return Number.parseInt(branchNameStart, 10);
});
}),
},
CONCURRENCY,
);
return projectBranches;
@@ -234,18 +231,20 @@ export async function getIssueBranches(api: Api, projects: Project[]): Promise<{
/**
* Get issues grouped by assignees
* @param api GitLab API to make requests with
* @param label Label to generate report for
* @param label Label to generate the report for
*/
export async function getIssuesGroupedByAssignees(api: Api, label: string): Promise<AssigneeWithIssues[]> {
const issuesByAssignee: IssuesGroupedByAssigneeId = {};
const groups = await Promise.all(
GROUPS.map(async groupId => {
const groups = await mapAsyncLimit(
GROUPS,
async groupId => {
const subGroups = await api.getSubGroupsForGroup(groupId);
return subGroups.map(group => {
return group.id;
});
}),
},
CONCURRENCY,
).then(it => it.flat());
groups.push(...groups, ...GROUPS);
@@ -385,14 +384,15 @@ export async function getMergeRequests(api: Api, projects: Project[]): Promise<M
const projectMergeRequests: MergeRequestsForProjects = {};
// iterate over projects
await Promise.all(
projects.map(async project => {
// check if project can have merge requests
await mapAsyncLimit(
projects,
async project => {
// check if the project can have merge requests
if (!project.merge_requests_enabled) {
return;
}
// get all merge requests for project
// get all merge requests for the project
const mergeRequests = await api.getMergeRequests(
MembershipScope.PROJECTS,
project.id,
@@ -414,7 +414,8 @@ export async function getMergeRequests(api: Api, projects: Project[]): Promise<M
web_url: branchNameStartAndUrl.web_url,
};
});
}),
},
CONCURRENCY,
);
return projectMergeRequests;
@@ -423,8 +424,8 @@ export async function getMergeRequests(api: Api, projects: Project[]): Promise<M
/**
* Generate a report
* @param api GitLab API to make requests with
* @param label Label to generate report for
* @param template Template to generate report with
* @param label Label to generate the report for
* @param template Template to generate the report with
*/
export async function generateReport(api: Api, label: string, template: string): Promise<string> {
const issuesGroupedByAssignee = await getIssuesGroupedByAssignees(api, label);
@@ -441,7 +442,7 @@ export async function generateReport(api: Api, label: string, template: string):
/**
* Generate a markdown report
* @param api GitLab API to make requests with
* @param label Label to generate report for
* @param label Label to generate the report for
*/
export async function report(api: Api, label: string) {
const meetingDay = getNextMeetingDay();

View File

@@ -25,6 +25,7 @@ import {
import {Logger} from '@openstapps/logger';
import {getProjects} from '../common.js';
import {
CONCURRENCY,
GROUPS,
NEEDED_LABELS,
NEEDED_MILESTONES,
@@ -32,31 +33,34 @@ import {
PROTECTED_BRANCHES,
SCHOOLS,
} from '../configuration.js';
import {mapAsyncLimit} from '@openstapps/collection-utils';
/**
* Tidy issues without milestone
* Tidy issues without a milestone
*
* This will set the milestone of issues without milestone to 'Meeting'.
* @param api GitLab API instance to use for the requests
* This will set the milestone of issues without a milestone to 'Meeting'.
* @param api GitLab API instance for the requests
*/
export async function tidyIssuesWithoutMilestone(api: Api): Promise<void> {
// fetch issues without milestone from all groups
const issuesWithoutMilestone = await Promise.all(
GROUPS.map(groupId =>
// fetch issues without a milestone from all groups
const issuesWithoutMilestone = await mapAsyncLimit(
GROUPS,
groupId =>
api.getIssues({
groupId: groupId,
milestone: 'No Milestone',
state: IssueState.OPENED,
}),
),
CONCURRENCY,
).then(it => it.flat());
Logger.info(`Found '${issuesWithoutMilestone.length}' issue(s) without milestone.`);
const milestoneCache: {[s: number]: Milestone[]} = {};
await Promise.all(
issuesWithoutMilestone.map(async issue => {
await mapAsyncLimit(
issuesWithoutMilestone,
async issue => {
if (milestoneCache[issue.project_id] === undefined) {
milestoneCache[issue.project_id] = await api.getMilestonesForProject(issue.project_id);
}
@@ -85,7 +89,8 @@ export async function tidyIssuesWithoutMilestone(api: Api): Promise<void> {
issue.iid,
`${NOTE_PREFIX} Milestone was set automatically to 'Meeting'.`,
);
}),
},
CONCURRENCY,
);
Logger.ok('Tidied issues without milestones.');
@@ -95,17 +100,18 @@ export async function tidyIssuesWithoutMilestone(api: Api): Promise<void> {
* Tidy open issues without meeting label
*
* This adds the label 'meeting' to all open issues that do not have this label.
* @param api GitLab API instante to use for requests
* @param api GitLab API instance for requests
*/
export async function tidyOpenIssuesWithoutMeetingLabel(api: Api): Promise<void> {
// fetch all open issues
const openIssues = await Promise.all(
GROUPS.map(groupId =>
const openIssues = await mapAsyncLimit(
GROUPS,
groupId =>
api.getIssues({
groupId: groupId,
state: IssueState.OPENED,
}),
),
CONCURRENCY,
).then(it => it.flat());
Logger.info(`Found ${openIssues.length} open issue(s).`);
@@ -117,8 +123,9 @@ export async function tidyOpenIssuesWithoutMeetingLabel(api: Api): Promise<void>
Logger.info(`Filtered ${openIssuesWithoutMeetingLabel.length} open issue(s) without label 'meeting'.`);
await Promise.all(
openIssuesWithoutMeetingLabel.map(async issue => {
await mapAsyncLimit(
openIssuesWithoutMeetingLabel,
async issue => {
if (issue.milestone !== null && issue.milestone.title === 'Backlog') {
Logger.info(`Skipping issue "${issue.title}" because it is in backlog.`);
@@ -131,7 +138,8 @@ export async function tidyOpenIssuesWithoutMeetingLabel(api: Api): Promise<void>
issue.iid,
`${NOTE_PREFIX} Automatically adding label 'meeting'\n\n/label ~meeting`,
);
}),
},
CONCURRENCY,
);
Logger.ok(`Tidied open issues without label 'meeting'.`);
@@ -143,8 +151,9 @@ export async function tidyOpenIssuesWithoutMeetingLabel(api: Api): Promise<void>
* @param projects List of projects to tidy labels on
*/
export async function tidyLabels(api: Api, projects: Project[]): Promise<void> {
await Promise.all(
projects.map(async project => {
await mapAsyncLimit(
projects,
async project => {
const labels = await api.getLabels(project.id);
const neededLabels = [...NEEDED_LABELS];
@@ -165,12 +174,14 @@ export async function tidyLabels(api: Api, projects: Project[]): Promise<void> {
} */
}
await Promise.all(
neededLabels.map(async neededLabel => {
await mapAsyncLimit(
neededLabels,
async neededLabel => {
await api.createLabel(project.id, neededLabel.name, neededLabel.description, neededLabel.color);
Logger.log(`Created label '${neededLabel.name}' in '${project.name_with_namespace}'.`);
}),
},
CONCURRENCY,
);
// await asyncPool(2, extraneousLabels, async (extraneousLabel) => {
@@ -178,7 +189,8 @@ export async function tidyLabels(api: Api, projects: Project[]): Promise<void> {
//
// Logger.log('Deleted label `' + extraneousLabel.name + '` from ' + project.name_with_namespace + '.');
// });
}),
},
CONCURRENCY,
);
Logger.ok('Tidied labels.');
@@ -190,8 +202,9 @@ export async function tidyLabels(api: Api, projects: Project[]): Promise<void> {
* @param projects List of projects to tidy milestones on
*/
export async function tidyMilestones(api: Api, projects: Project[]): Promise<void> {
await Promise.all(
projects.map(async project => {
await mapAsyncLimit(
projects,
async project => {
const milestones = await api.getMilestonesForProject(project.id);
const missingMilestones = [...NEEDED_MILESTONES];
@@ -211,7 +224,8 @@ export async function tidyMilestones(api: Api, projects: Project[]): Promise<voi
}),
);
}
}),
},
CONCURRENCY,
);
Logger.ok('Tidied milestones.');
@@ -223,8 +237,9 @@ export async function tidyMilestones(api: Api, projects: Project[]): Promise<voi
* @param projects List of projects to tidy milestones on
*/
export async function tidyProtectedBranches(api: Api, projects: Project[]): Promise<void> {
await Promise.all(
projects.map(async project => {
await mapAsyncLimit(
projects,
async project => {
const branches = await api.getBranchesForProject(project.id);
const protectableBranches = branches.filter(branch => {
@@ -244,7 +259,8 @@ export async function tidyProtectedBranches(api: Api, projects: Project[]): Prom
);
}),
);
}),
},
CONCURRENCY,
);
Logger.ok('Tidied protected branches.');
@@ -256,8 +272,9 @@ export async function tidyProtectedBranches(api: Api, projects: Project[]): Prom
* @param projects List of projects to tidy protected tags on
*/
export async function tidyProtectedTags(api: Api, projects: Project[]): Promise<void> {
await Promise.all(
projects.map(async project => {
await mapAsyncLimit(
projects,
async project => {
// TODO: move this to GitLab API
const protectedTags = (await api.makeGitLabAPIRequest(
`projects/${project.id}/protected_tags`,
@@ -296,7 +313,8 @@ export async function tidyProtectedTags(api: Api, projects: Project[]): Promise<
Logger.log(`Added protected version tag in project '${project.name_with_namespace}'.`);
}
}),
},
CONCURRENCY,
);
Logger.ok('Tidied protected tags.');
@@ -316,31 +334,37 @@ export async function tidySubGroupMembers(api: Api): Promise<void> {
groupIdsToSchool[SCHOOLS[school]] = school;
});
await Promise.all(
GROUPS.slice(1).map(async groupId => {
await mapAsyncLimit(
GROUPS.slice(1),
async groupId => {
const members = await api.getMembers(MembershipScope.GROUPS, groupId);
const memberIds = new Set(members.map(member => member.id));
await Promise.all(
stappsMembers.map(async stappsMember => {
await mapAsyncLimit(
stappsMembers,
async stappsMember => {
if (!memberIds.has(stappsMember.id)) {
await api.addMember(MembershipScope.GROUPS, groupId, stappsMember.id, AccessLevel.Developer);
Logger.log(`Added '${stappsMember.name}' to group '${groupIdsToSchool[groupId]}'.`);
}
}),
},
CONCURRENCY,
);
await Promise.all(
members.map(async member => {
await mapAsyncLimit(
members,
async member => {
if (!stappsMemberIds.has(member.id)) {
await api.deleteMember(MembershipScope.GROUPS, groupId, member.id);
Logger.log(`Deleted member '${member.name}' from group '${groupIdsToSchool[groupId]}'.`);
}
}),
},
CONCURRENCY,
);
}),
},
CONCURRENCY,
);
Logger.ok(`Tidied 'sub' group members.`);
@@ -350,17 +374,19 @@ export async function tidySubGroupMembers(api: Api): Promise<void> {
* Tidy issues without assignee
*
* Set assignee to author if no assignee is set.
* @param api GitLab API instance to use for the requests
* @param api GitLab API instance for the requests
*/
export async function tidyIssuesWithoutAssignee(api: Api): Promise<void> {
// fetch issues without milestone from all groups
const issues = await Promise.all(
GROUPS.map(async groupId => {
// fetch issues without a milestone from all groups
const issues = await mapAsyncLimit(
GROUPS,
async groupId => {
return api.getIssues({
groupId: groupId,
state: IssueState.OPENED,
});
}),
},
CONCURRENCY,
).then(it => it.flat());
const issuesWithoutAssignee = issues.filter(issue => {
@@ -369,8 +395,9 @@ export async function tidyIssuesWithoutAssignee(api: Api): Promise<void> {
Logger.info(`Found '${issuesWithoutAssignee.length}' issue(s) without assignee.`);
await Promise.all(
issuesWithoutAssignee.map(async issue => {
await mapAsyncLimit(
issuesWithoutAssignee,
async issue => {
await api.setAssigneeForIssue(issue, issue.author.id);
Logger.log(`Set assignee for '${issue.title}' to '${issue.author.name}'.`);
@@ -381,7 +408,8 @@ export async function tidyIssuesWithoutAssignee(api: Api): Promise<void> {
issue.iid,
`${NOTE_PREFIX} Assignee was set automatically to author.`,
);
}),
},
CONCURRENCY,
);
Logger.ok('Tidied issues without assignee.');
@@ -391,13 +419,15 @@ export async function tidyIssuesWithoutAssignee(api: Api): Promise<void> {
* Tidy merge requests without assignee
*
* Set assignee to author if no assignee is set.
* @param api GitLab API instance to use for the requests
* @param api GitLab API instance for the requests
*/
export async function tidyMergeRequestsWithoutAssignee(api: Api): Promise<void> {
const mergeRequests = await Promise.all(
GROUPS.map(async groupId => {
const mergeRequests = await mapAsyncLimit(
GROUPS,
async groupId => {
return api.getMergeRequests(MembershipScope.GROUPS, groupId, MergeRequestState.OPENED);
}),
},
CONCURRENCY,
).then(it => it.flat());
const mergeRequestsWithoutAssignee = mergeRequests.filter(mergeRequest => {
@@ -406,8 +436,9 @@ export async function tidyMergeRequestsWithoutAssignee(api: Api): Promise<void>
Logger.info(`Found '${mergeRequestsWithoutAssignee.length}' merge requests without assignee.`);
await Promise.all(
mergeRequestsWithoutAssignee.map(async mergeRequest => {
await mapAsyncLimit(
mergeRequestsWithoutAssignee,
async mergeRequest => {
await api.setAssigneeForMergeRequest(mergeRequest, mergeRequest.author.id);
Logger.log(`Set assignee for '${mergeRequest.title}' to '${mergeRequest.author.name}'.`);
@@ -418,7 +449,8 @@ export async function tidyMergeRequestsWithoutAssignee(api: Api): Promise<void>
mergeRequest.iid,
`${NOTE_PREFIX} Assignee was set automatically to author.`,
);
}),
},
CONCURRENCY,
);
Logger.ok('Tidied merge requests without assignee.');
@@ -429,7 +461,7 @@ export async function tidyMergeRequestsWithoutAssignee(api: Api): Promise<void>
* @param api GitLab API instance to use for the requests
*/
export async function tidy(api: Api) {
// get first level sub groups
// get first level subgroups
const groups = [...GROUPS];
const subGroups = await api.getSubGroupsForGroup(groups[0]);
groups.push(...groups, ...subGroups.map(group => group.id));