refactor: app deployment

This commit is contained in:
2023-06-05 10:54:18 +02:00
parent b21833de40
commit 3b8a344d73
104 changed files with 583 additions and 13494 deletions

View File

@@ -1,4 +0,0 @@
.idea/
.git/
.vscode/
Dockerfile

View File

@@ -1,91 +0,0 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
# parcel-bundler cache (https://parceljs.org/)
.cache
# next.js build output
.next
# nuxt.js build output
.nuxt
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
#DynamoDB Local files
.dynamodb/
########## end of https://github.com/github/gitignore/blob/master/Node.gitignore
# ignore ide files
.idea
.vscode
# ignore lib
lib
# ignore docs
docs

View File

@@ -1,168 +0,0 @@
image: registry.gitlab.com/openstapps/projectmanagement/node
default:
tags:
- performance
stages:
- build
- test
- audit
- publish
- deploy
build:
stage: build
script:
- npm ci
- npm run build
artifacts:
expire_in: 1 day
untracked: true
paths:
- node_modules/
unit:
stage: test
dependencies:
- build
script:
- npm run test-unit
coverage: '/Statements[^:]*\:[^:]*\s+([\d\.]+)%/'
artifacts:
expire_in: 6 month
reports:
coverage_report:
coverage_format: cobertura
path: coverage/cobertura-coverage.xml
integration:
image: registry.gitlab.com/openstapps/projectmanagement/builder
stage: test
dependencies:
- build
variables:
DOCKER_DRIVER: overlay2
services:
- docker:dind
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker-compose -f integration-test.yml pull && docker-compose -f integration-test.yml up --abort-on-container-exit --exit-code-from apicli
tags:
- gitlab-org-docker
audit:
stage: audit
dependencies:
- build
script:
- npm audit --production
pages:
stage: deploy
script:
- npm run documentation
- mv docs/openapi public
only:
- master
artifacts:
paths:
- public
ci:
stage: test
dependencies:
- build
script:
- .gitlab/ci/testCIScripts.sh
# Anchor templates for publishing the image in the docker registry
# Automatically publishing for versions in tags (eg v1.0.0 as 1.0.0), master and develop
# Manual publishing for all other branches
.publish_template_auto: &publish_template_auto
image: registry.gitlab.com/openstapps/projectmanagement/builder
stage: publish
dependencies:
- build
variables:
DOCKER_DRIVER: overlay2
services:
- docker:dind
script:
- echo "YOU SHOULD OVERRIDE THIS SECTION"
- exit 1
only:
- master
tags:
- secrecy
.publish_template_manual: &publish_template_manual
image: registry.gitlab.com/openstapps/projectmanagement/builder
stage: publish
dependencies:
- build
variables:
DOCKER_DRIVER: overlay2
services:
- docker:dind
script:
- export CORE_VERSION=$(openstapps-projectmanagement get-used-version @openstapps/core)
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME
- export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
- export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $IMAGETAG_LATEST -t $IMAGETAG_CORE_VERSION .
- docker push $IMAGETAG_BASE
except:
- /(^v[0-9]+\.[0-9]+\.[0-9]+$|^master$|^develop$)/
only:
- branches
when: manual
tags:
- secrecy
.publish_version_template: &publish_version_template
script:
- export CORE_VERSION=$(openstapps-projectmanagement get-used-version @openstapps/core)
- export VERSION=$(echo -n "$CI_COMMIT_REF_NAME" | cut -c 2-)
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE
- export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
- export IMAGETAG_VERSION=$IMAGETAG_BASE:$VERSION
- export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $IMAGETAG_LATEST -t $IMAGETAG_VERSION -t $IMAGETAG_CORE_VERSION .
- docker push $IMAGETAG_BASE
.publish_branch_template: &publish_branch_template
script:
- export CORE_VERSION=$(openstapps-projectmanagement get-used-version @openstapps/core)
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME
- export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
- export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $IMAGETAG_LATEST -t $IMAGETAG_CORE_VERSION .
- docker push $IMAGETAG_BASE
# Jobs joining anchor templates with automatic publishing behaviour
# ! The jobname must end with ":" followed by the name for the registry branch
image:version:
<<: *publish_template_auto
<<: *publish_version_template
only:
- /^v[0-9]+\.[0-9]+\.[0-9]+$/
image:master:
<<: *publish_template_auto
<<: *publish_branch_template
only:
- master
image:develop:
<<: *publish_template_auto
<<: *publish_branch_template
only:
- develop
# Jobs joining anchor templates with manual publishing behaviour
image:branch:
<<: *publish_template_manual

View File

@@ -1,4 +0,0 @@
#!/usr/bin/env sh
# script returns string with everything after the last colon of $1 input
echo -n $1 | grep -oE '[^:]+$'

View File

@@ -1,8 +0,0 @@
#!/usr/bin/env sh
# script returns semantical versioning string linke 2.0.0 (if $1 is v2.0.0) or $1
if echo -n $1 | grep -Eq 'v[0-9]+\.[0-9]+\.[0-9]+'; then
echo $(echo -n "$1" | cut -c 2-);
else
echo $1;
fi

View File

@@ -1,7 +0,0 @@
#!/usr/bin/env sh
# If this is a pipeline of a version tag, also push this version
# as latest to the registry alias $2:latest
if echo -n $1 | grep -Eq 'v[0-9]+\.[0-9]+\.[0-9]+'; then
docker push $2:latest;
fi

View File

@@ -1,29 +0,0 @@
#!/usr/bin/env sh
# test all CI scripts
SCRIPT_DIR=$(dirname "$0")
TAG_VERSION=$(sh $SCRIPT_DIR/getRegistryTag.sh "v1.0.0")
TAG_TEST=$(sh $SCRIPT_DIR/getRegistryTag.sh "TEST")
BRANCH_NAME=$(sh $SCRIPT_DIR/getRegistryBranch.sh "very:first:test")
# Leaving out pushAsLatestVersion.sh as its controll flow
# is based on the same condition as getRegistryTag.sh
if [ $TAG_VERSION != "1.0.0" ]; then
echo "ERROR in CI SCRIPT: $SCRIPT_DIR/getRegistryTag.sh"
return 1
fi
if [ $TAG_TEST != "TEST" ]; then
echo "ERROR in CI SCRIPT: $SCRIPT_DIR/getRegistryTag.sh"
return 2
fi
if [ $BRANCH_NAME != "test" ]; then
echo "ERROR in CI SCRIPT: $SCRIPT_DIR/getRegistryBranch.sh"
return 3
fi
return 0

View File

@@ -1,36 +0,0 @@
## Summary
(Summarize the bug encountered concisely)
## Steps to reproduce
(How one can reproduce the issue - this is very important)
## Example Project
(If possible, please create an example project here on GitLab.com that exhibits the problematic behaviour, and link to it here in the bug report)
(If you are using an older version of GitLab, this will also determine whether the bug has been fixed in a more recent version)
## Which version of the software did you use ?
(Version numbers of used software or commit references)
## What is the current bug behavior?
(What actually happens)
## What is the expected correct behavior?
(What you should see instead)
## Relevant logs and/or screenshots
(Paste any relevant logs - please use code blocks (```) to format console output,
logs, and code as it's very hard to read otherwise.)
## Possible fixes
(If you can, link to the line of code that might be responsible for the problem)
/label ~bug ~meeting

View File

@@ -1,17 +0,0 @@
## Description
(Describe the feature that you're requesting concisely)
## Explanation
(Explain why the feature is necessary)
## Mockups/Data
(If possible, provide mockups or examples of communication, which demonstrate the feature)
## Dependencies, issues to be resolved beforehand
(List issues or dependencies that need to be resolved before this feature can be implemented)
/label ~feature ~meeting

View File

@@ -1,3 +0,0 @@
{
"port": 9200
}

View File

@@ -1,10 +0,0 @@
# Ignore all files/folders by default
# See https://stackoverflow.com/a/29932318
/*
# Except these files/folders
!lib
!LICENSE
!package.json
!package-lock.json
!README.md
!src

View File

@@ -1,13 +1,14 @@
FROM registry.gitlab.com/openstapps/projectmanagement/node
FROM openstapps:workspace as pruned
USER root
RUN apk add --update python3 py3-pip make g++
RUN pnpm --filter @openstapps/backend --prod deploy /pruned
FROM registry.gitlab.com/openstapps/projectmanagement/node:v18
USER node
ADD --chown=node:node . /app
WORKDIR /app
RUN npm ci && npm run build
ENV NODE_ENV=production
COPY --chown=node:node --from=pruned pruned .
EXPOSE 3000
CMD ["node", "app"]
ENTRYPOINT ["node", "app.js"]

View File

@@ -1,9 +1,9 @@
version: '3'
services:
backend:
image: openstapps:backend
ports:
- '3000:3000'
build: .
environment:
STAPPS_LOG_LEVEL: '31'
STAPPS_EXIT_LEVEL: '8'
@@ -13,12 +13,12 @@ services:
ES_ADDR: 'http://elasticsearch:9200'
elasticsearch:
image: openstapps:database
ports:
- '9200:9200'
image: 'registry.gitlab.com/openstapps/database:latest'
apicli:
image: 'registry.gitlab.com/openstapps/api/cli:latest'
api-cli:
image: openstapps:api-cli
environment:
STAPPS_LOG_LEVEL: '31'
STAPPS_EXIT_LEVEL: '8'

File diff suppressed because it is too large Load Diff

View File

@@ -1,7 +1,7 @@
{
"name": "@openstapps/backend",
"description": "A reference implementation for a StApps backend",
"version": "2.1.0",
"version": "3.0.0",
"type": "module",
"license": "AGPL-3.0-only",
"author": "André Bierlein <andre.mt.bierlein@gmail.com>",
@@ -17,8 +17,16 @@
],
"main": "lib/app.js",
"bin": "app.js",
"files": [
"app.js",
"lib",
"config",
"README.md",
"ROUTES.md"
],
"scripts": {
"build": "tsup",
"build": "tsup-node",
"build:docker": "docker build -t openstapps:backend .",
"dev": "tsup --watch --onSuccess \"pnpm run start\"",
"format": "prettier . -c --ignore-path ../../.gitignore",
"format:fix": "prettier --write . --ignore-path ../../.gitignore",
@@ -27,14 +35,24 @@
"start": "cross-env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node app.js",
"start:debug": "cross-env STAPPS_LOG_LEVEL=31 NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node app.js",
"test": "pnpm run test:unit",
"test:integration": "docker-compose -f integration-test.yml pull && docker-compose -f integration-test.yml up --build --abort-on-container-exit --exit-code-from apicli",
"test:integration": "docker-compose -f integration-test.yml up --abort-on-container-exit --exit-code-from api-cli",
"test:unit": "cross-env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true STAPPS_LOG_LEVEL=0 mocha --exit"
},
"dependencies": {
"@elastic/elasticsearch": "8.4.0",
"@openstapps/core": "workspace:*",
"@openstapps/core-tools": "workspace:*",
"@openstapps/es-mapping-generator": "workspace:*",
"@openstapps/logger": "workspace:*",
"@types/body-parser": "1.19.2",
"@types/cors": "2.8.13",
"@types/express": "4.17.17",
"@types/geojson": "1.0.6",
"@types/node": "18.15.3",
"@types/node-cron": "3.0.7",
"@types/nodemailer": "6.4.7",
"@types/promise-queue": "2.2.0",
"@types/uuid": "8.3.4",
"body-parser": "1.20.2",
"cors": "2.8.5",
"cosmiconfig": "8.1.3",
@@ -50,38 +68,25 @@
"nodemailer": "6.9.1",
"prom-client": "14.1.1",
"promise-queue": "2.2.5",
"ts-node": "10.9.1",
"uuid": "8.3.2"
},
"devDependencies": {
"@openstapps/es-mapping-generator": "workspace:*",
"@openstapps/eslint-config": "workspace:*",
"@openstapps/prettier-config": "workspace:*",
"@openstapps/tsconfig": "workspace:*",
"@testdeck/mocha": "0.3.3",
"@types/body-parser": "1.19.2",
"@types/chai": "4.3.5",
"@types/chai-as-promised": "7.1.5",
"@types/config": "3.3.0",
"@types/cors": "2.8.13",
"@types/express": "4.17.17",
"@types/geojson": "1.0.6",
"@types/mocha": "10.0.1",
"@types/morgan": "1.9.4",
"@types/node": "18.15.3",
"@types/node-cron": "3.0.7",
"@types/nodemailer": "6.4.7",
"@types/promise-queue": "2.2.0",
"@types/sinon": "10.0.14",
"@types/sinon-express-mock": "1.3.9",
"@types/supertest": "2.0.12",
"@types/uuid": "8.3.4",
"c8": "7.14.0",
"chai": "4.3.7",
"chai-as-promised": "7.1.1",
"conventional-changelog-cli": "2.2.2",
"cross-env": "7.0.3",
"esmock": "2.2.3",
"get-port": "5.1.1",
"mocha": "10.2.0",
"mocked-env": "1.3.5",
@@ -89,6 +94,7 @@
"sinon": "15.0.4",
"sinon-express-mock": "2.2.1",
"supertest": "6.3.3",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"typescript": "4.9.5"
},

View File

@@ -0,0 +1,12 @@
{
"extends": ["//"],
"pipeline": {
"test:integration": {
"dependsOn": [
"@openstapps/database#build:docker",
"@openstapps/backend#build:docker",
"@openstapps/api-cli#build:docker"
]
}
}
}

View File

@@ -1,55 +0,0 @@
image: registry.gitlab.com/openstapps/projectmanagement/builder
default:
tags:
- docker
stages:
- test
- publish
image build:
stage: test
variables:
DOCKER_DRIVER: overlay2
services:
- docker:dind
script:
- export CI_ES_VERSION=$(sed -n '/FROM elasticsearch/p' Dockerfile | cut -d ":" -f 2)
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $CI_REGISTRY_IMAGE:$CI_ES_VERSION .
- docker tag $CI_REGISTRY_IMAGE:$CI_ES_VERSION $CI_REGISTRY_IMAGE:latest
only:
- branches
except:
- main
- develop
main:
stage: publish
variables:
DOCKER_DRIVER: overlay2
services:
- docker:dind
script:
- export CI_ES_VERSION=$(sed -n '/FROM elasticsearch/p' Dockerfile | cut -d ":" -f 2)
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $CI_REGISTRY_IMAGE:$CI_ES_VERSION .
- docker tag $CI_REGISTRY_IMAGE:$CI_ES_VERSION $CI_REGISTRY_IMAGE:latest
- docker push $CI_REGISTRY_IMAGE:latest
- docker push $CI_REGISTRY_IMAGE:$CI_ES_VERSION
only:
- main
develop:
stage: publish
variables:
DOCKER_DRIVER: overlay2
services:
- docker:dind
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME .
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
only:
- develop

View File

@@ -1,7 +1,3 @@
# Open StApps Monorepo
========
# Database docker image
[![pipeline status](https://img.shields.io/gitlab/pipeline/openstapps/database.svg?style=flat-square)](https://gitlab.com/openstapps/database/commits/main)

View File

@@ -0,0 +1,9 @@
{
"name": "@openstapps/database",
"version": "3.0.0",
"license": "AGPL-3.0-only",
"scripts": {
"build:docker": "docker build -t openstapps:database ."
},
"private": true
}

View File

@@ -1,96 +0,0 @@
# Logs
logs/
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov/
# Coverage directory used by tools like istanbul
coverage/
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components/
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# Optional npm cache directory
.npm/
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
# parcel-bundler cache (https://parceljs.org/)
.cache
# next.js build output
.next
# nuxt.js build output
.nuxt
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
#DynamoDB Local files
.dynamodb/
########## end of https://github.com/github/gitignore/blob/master/Node.gitignore
# ignore ide files
.idea/
.vscode/
# ignore lib
lib/
# ignore docs
docs/
# Certificates
*.crt
*.key
test/certs

View File

@@ -1,91 +0,0 @@
image: registry.gitlab.com/openstapps/projectmanagement/node
stages:
- build
- test
- audit
- publish
before_script:
- npm ci
build:
stage: build
script:
- npm run build
artifacts:
paths:
- lib
unit:
stage: test
script:
- npm test
coverage: '/Statements[^:]*\:[^:]*\s+([\d\.]+)%/'
artifacts:
reports:
coverage_report:
coverage_format: cobertura
path: coverage/cobertura-coverage.xml
audit:
allow_failure: true
except:
- schedules
script:
- npm audit
stage: audit
scheduled-audit:
only:
- schedules
script:
- npm audit --audit-level=high
stage: audit
docker image build:
image: registry.gitlab.com/openstapps/projectmanagement/builder
stage: test
dependencies:
- build
variables:
DOCKER_DRIVER: overlay2
services:
- docker:dind
script:
- docker build -t $CI_REGISTRY_IMAGE/$REGISTRY_BRANCH$CI_COMMIT_REF_NAME:latest .
tags:
- docker
docker image:
image: registry.gitlab.com/openstapps/projectmanagement/builder
stage: publish
dependencies:
- build
variables:
DOCKER_DRIVER: overlay2
services:
- docker:dind
script:
- export VERSION=$(echo -n "$CI_COMMIT_REF_NAME" | cut -c 2-)
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE
- export IMAGETAG_VERSION=$IMAGETAG_BASE:$VERSION
- export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $IMAGETAG_VERSION -t $IMAGETAG_LATEST .
- docker push $IMAGETAG_BASE
only:
- /^v[0-9]+\.[0-9]+\.[0-9]+$/
tags:
- docker
pages:
artifacts:
paths:
- public
only:
- /^v[0-9]+\.[0-9]+\.[0-9]+$/
script:
- npm run documentation
- mv docs public
stage: publish

View File

@@ -1,32 +0,0 @@
## Summary
(Summarize the bug encountered concisely)
## Steps to reproduce
(How one can reproduce the issue - this is very important)
## Example Project
(If possible, please create an example project here on GitLab.com that exhibits the problematic behaviour, and link to it here in the bug report)
(If you are using an older version of GitLab, this will also determine whether the bug has been fixed in a more recent version)
## What is the current bug behavior?
(What actually happens)
## What is the expected correct behavior?
(What you should see instead)
## Relevant logs and/or screenshots
(Paste any relevant logs - please use code blocks (```) to format console output,
logs, and code as it's very hard to read otherwise.)
## Possible fixes
(If you can, link to the line of code that might be responsible for the problem)
/label ~meeting ~bug

View File

@@ -1,13 +0,0 @@
## Description
(Describe the feature that you're requesting concisely)
## Explanation
(Explain why the feature is necessary)
## Dependencies, issues to be resolved beforehand
(List issues or dependencies that need to be resolved before this feature can be implemented)
/label ~meeting ~feature

View File

@@ -1,10 +0,0 @@
# Ignore all files/folders by default
# See https://stackoverflow.com/a/29932318
/*
# Except these files/folders
!lib
!LICENSE
!package.json
!package-lock.json
!README.md
!src

View File

@@ -1,7 +1,7 @@
{
"name": "@openstapps/proxy",
"description": "NGINX proxy that is dynamically configured by a Node.js script",
"version": "2.1.0",
"version": "3.0.0",
"type": "module",
"license": "AGPL-3.0-only",
"repository": "git@gitlab.com:openstapps/proxy.git",
@@ -15,8 +15,21 @@
"Rainer Killinger <mail-openstapps@killinger.co>"
],
"bin": "app.js",
"files": [
"app.js",
"lib",
"bin",
"config",
"fixtures",
"systemd",
"Dockerfile",
".dockerignore",
"nginx.conf",
"nginx.conf.template",
"README.md"
],
"scripts": {
"build": "tsup --dts",
"build": "tsup-node --dts",
"format": "prettier . -c --ignore-path ../../.gitignore",
"format:fix": "prettier --write . --ignore-path ../../.gitignore",
"lint": "eslint --ext .ts src/",
@@ -37,8 +50,8 @@
"typescript": "4.9.5"
},
"devDependencies": {
"@openstapps/api-cli": "workspace:*",
"@openstapps/eslint-config": "workspace:*",
"@openstapps/nyc-config": "workspace:*",
"@openstapps/prettier-config": "workspace:*",
"@openstapps/tsconfig": "workspace:*",
"@types/chai": "4.3.5",