refactor: app deployment

2026-01-20 16:42:56 +00:00 · 2023-06-05 10:54:18 +02:00
parent b21833de40
commit 3b8a344d73
104 changed files with 583 additions and 13494 deletions
--- a/backend/backend/.dockerignore
+++ b/backend/backend/.dockerignore
@@ -1,4 +0,0 @@
-.idea/
-.git/
-.vscode/
-Dockerfile
--- a/backend/backend/.gitignore
+++ b/backend/backend/.gitignore
@@ -1,91 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-
-# Runtime data
-pids
-*.pid
-*.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
-
-# Dependency directories
-node_modules/
-jspm_packages/
-
-# TypeScript v1 declaration files
-typings/
-
-# Optional npm cache directory
-.npm
-
-# Optional eslint cache
-.eslintcache
-
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
-
-# dotenv environment variables file
-.env
-
-# parcel-bundler cache (https://parceljs.org/)
-.cache
-
-# next.js build output
-.next
-
-# nuxt.js build output
-.nuxt
-
-# vuepress build output
-.vuepress/dist
-
-# Serverless directories
-.serverless/
-
-# FuseBox cache
-.fusebox/
-
-#DynamoDB Local files
-.dynamodb/
-
-########## end of https://github.com/github/gitignore/blob/master/Node.gitignore
-
-# ignore ide files
-.idea
-.vscode
-
-# ignore lib
-lib
-
-# ignore docs
-docs
--- a/backend/backend/.gitlab-ci.yml
+++ b/backend/backend/.gitlab-ci.yml
@@ -1,168 +0,0 @@
-image: registry.gitlab.com/openstapps/projectmanagement/node
-
-default:
-  tags:
-    - performance
-
-stages:
-  - build
-  - test
-  - audit
-  - publish
-  - deploy
-
-build:
-  stage: build
-  script:
-    - npm ci
-    - npm run build
-  artifacts:
-    expire_in: 1 day
-    untracked: true
-    paths:
-      - node_modules/
-
-unit:
-  stage: test
-  dependencies:
-    - build
-  script:
-    - npm run test-unit
-  coverage: '/Statements[^:]*\:[^:]*\s+([\d\.]+)%/'
-  artifacts:
-    expire_in: 6 month
-    reports:
-      coverage_report:
-        coverage_format: cobertura
-        path: coverage/cobertura-coverage.xml
-
-integration:
-  image: registry.gitlab.com/openstapps/projectmanagement/builder
-  stage: test
-  dependencies:
-    - build
-  variables:
-    DOCKER_DRIVER: overlay2
-  services:
-    - docker:dind
-  script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - docker-compose -f integration-test.yml pull && docker-compose -f integration-test.yml up --abort-on-container-exit --exit-code-from apicli
-  tags:
-    - gitlab-org-docker
-
-audit:
-  stage: audit
-  dependencies:
-    - build
-  script:
-    - npm audit --production
-
-pages:
-  stage: deploy
-  script:
-    - npm run documentation
-    - mv docs/openapi public
-  only:
-    - master
-  artifacts:
-    paths:
-      - public
-
-ci:
-  stage: test
-  dependencies:
-    - build
-  script:
-    - .gitlab/ci/testCIScripts.sh
-
-# Anchor templates for publishing the image in the docker registry
-# Automatically publishing for versions in tags (eg v1.0.0 as 1.0.0), master and develop
-# Manual publishing for all other branches
-.publish_template_auto: &publish_template_auto
-  image: registry.gitlab.com/openstapps/projectmanagement/builder
-  stage: publish
-  dependencies:
-    - build
-  variables:
-    DOCKER_DRIVER: overlay2
-  services:
-    - docker:dind
-  script:
-    - echo "YOU SHOULD OVERRIDE THIS SECTION"
-    - exit 1
-  only:
-    - master
-  tags:
-    - secrecy
-
-.publish_template_manual: &publish_template_manual
-  image: registry.gitlab.com/openstapps/projectmanagement/builder
-  stage: publish
-  dependencies:
-    - build
-  variables:
-    DOCKER_DRIVER: overlay2
-  services:
-    - docker:dind
-  script:
-    - export CORE_VERSION=$(openstapps-projectmanagement get-used-version @openstapps/core)
-    - export IMAGETAG_BASE=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME
-    - export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
-    - export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - docker build -t $IMAGETAG_LATEST -t $IMAGETAG_CORE_VERSION .
-    - docker push $IMAGETAG_BASE
-  except:
-    - /(^v[0-9]+\.[0-9]+\.[0-9]+$|^master$|^develop$)/
-  only:
-    - branches
-  when: manual
-  tags:
-    - secrecy
-
-.publish_version_template: &publish_version_template
-  script:
-    - export CORE_VERSION=$(openstapps-projectmanagement get-used-version @openstapps/core)
-    - export VERSION=$(echo -n "$CI_COMMIT_REF_NAME" | cut -c 2-)
-    - export IMAGETAG_BASE=$CI_REGISTRY_IMAGE
-    - export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
-    - export IMAGETAG_VERSION=$IMAGETAG_BASE:$VERSION
-    - export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - docker build -t $IMAGETAG_LATEST -t $IMAGETAG_VERSION -t $IMAGETAG_CORE_VERSION .
-    - docker push $IMAGETAG_BASE
-
-.publish_branch_template: &publish_branch_template
-  script:
-    - export CORE_VERSION=$(openstapps-projectmanagement get-used-version @openstapps/core)
-    - export IMAGETAG_BASE=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME
-    - export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
-    - export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - docker build -t $IMAGETAG_LATEST -t $IMAGETAG_CORE_VERSION .
-    - docker push $IMAGETAG_BASE
-
-# Jobs joining anchor templates with automatic publishing behaviour
-# ! The jobname must end with ":" followed by the name for the registry branch
-image:version:
-  <<: *publish_template_auto
-  <<: *publish_version_template
-  only:
-    - /^v[0-9]+\.[0-9]+\.[0-9]+$/
-
-image:master:
-  <<: *publish_template_auto
-  <<: *publish_branch_template
-  only:
-    - master
-
-image:develop:
-  <<: *publish_template_auto
-  <<: *publish_branch_template
-  only:
-    - develop
-
-# Jobs joining anchor templates with manual publishing behaviour
-image:branch:
-  <<: *publish_template_manual
--- a/backend/backend/.gitlab/ci/getRegistryBranch.sh
+++ b/backend/backend/.gitlab/ci/getRegistryBranch.sh
@@ -1,4 +0,0 @@
-#!/usr/bin/env sh
-
-# script returns string with everything after the last colon of $1 input
-echo -n $1 | grep -oE '[^:]+$'
--- a/backend/backend/.gitlab/ci/getRegistryTag.sh
+++ b/backend/backend/.gitlab/ci/getRegistryTag.sh
@@ -1,8 +0,0 @@
-#!/usr/bin/env sh
-
-# script returns semantical versioning string linke 2.0.0 (if $1 is v2.0.0) or $1
-if echo -n $1 | grep -Eq 'v[0-9]+\.[0-9]+\.[0-9]+'; then 
-  echo $(echo -n "$1" | cut -c 2-);
-else 
-  echo $1;
-fi
--- a/backend/backend/.gitlab/ci/pushAsLatestVersion.sh
+++ b/backend/backend/.gitlab/ci/pushAsLatestVersion.sh
@@ -1,7 +0,0 @@
-#!/usr/bin/env sh
-
-# If this is a pipeline of a version tag, also push this version 
-# as latest to the registry alias $2:latest
-if echo -n $1 | grep -Eq 'v[0-9]+\.[0-9]+\.[0-9]+'; then
-  docker push $2:latest;
-fi
--- a/backend/backend/.gitlab/ci/testCIScripts.sh
+++ b/backend/backend/.gitlab/ci/testCIScripts.sh
@@ -1,29 +0,0 @@
-#!/usr/bin/env sh
-
-# test all CI scripts
-
-SCRIPT_DIR=$(dirname "$0")
-
-TAG_VERSION=$(sh $SCRIPT_DIR/getRegistryTag.sh "v1.0.0")
-TAG_TEST=$(sh $SCRIPT_DIR/getRegistryTag.sh "TEST")
-BRANCH_NAME=$(sh $SCRIPT_DIR/getRegistryBranch.sh "very:first:test")
-
-# Leaving out pushAsLatestVersion.sh as its controll flow
-# is based on the same condition as getRegistryTag.sh
-
-if [ $TAG_VERSION != "1.0.0" ]; then
-	echo "ERROR in CI SCRIPT: $SCRIPT_DIR/getRegistryTag.sh"
-	return 1
-fi
-
-if [ $TAG_TEST != "TEST" ]; then
-	echo "ERROR in CI SCRIPT: $SCRIPT_DIR/getRegistryTag.sh"
-	return 2
-fi
-
-if [ $BRANCH_NAME != "test" ]; then
-	echo "ERROR in CI SCRIPT: $SCRIPT_DIR/getRegistryBranch.sh"
-	return 3
-fi
-
-return 0
--- a/backend/backend/.gitlab/issue_templates/bug.md
+++ b/backend/backend/.gitlab/issue_templates/bug.md
@@ -1,36 +0,0 @@
-## Summary
-
-(Summarize the bug encountered concisely)
-
-## Steps to reproduce
-
-(How one can reproduce the issue - this is very important)
-
-## Example Project
-
-(If possible, please create an example project here on GitLab.com that exhibits the problematic behaviour, and link to it here in the bug report)
-
-(If you are using an older version of GitLab, this will also determine whether the bug has been fixed in a more recent version)
-
-## Which version of the software did you use ?
-
-(Version numbers of used software or commit references)
-
-## What is the current bug behavior?
-
-(What actually happens)
-
-## What is the expected correct behavior?
-
-(What you should see instead)
-
-## Relevant logs and/or screenshots
-
-(Paste any relevant logs - please use code blocks (```) to format console output,
-logs, and code as it's very hard to read otherwise.)
-
-## Possible fixes
-
-(If you can, link to the line of code that might be responsible for the problem)
-
-/label ~bug ~meeting
--- a/backend/backend/.gitlab/issue_templates/feature.md
+++ b/backend/backend/.gitlab/issue_templates/feature.md
@@ -1,17 +0,0 @@
-## Description
-
-(Describe the feature that you're requesting concisely)
-
-## Explanation
-
-(Explain why the feature is necessary)
-
-## Mockups/Data
-
-(If possible, provide mockups or examples of communication, which demonstrate the feature)
-
-## Dependencies, issues to be resolved beforehand
-
-(List issues or dependencies that need to be resolved before this feature can be implemented)
-
-/label ~feature ~meeting
--- a/backend/backend/.mock-yeah
+++ b/backend/backend/.mock-yeah
@@ -1,3 +0,0 @@
-{
-  "port": 9200
-}
--- a/backend/backend/.npmignore
+++ b/backend/backend/.npmignore
@@ -1,10 +0,0 @@
-# Ignore all files/folders by default
-# See https://stackoverflow.com/a/29932318
-/*
-# Except these files/folders
-!lib
-!LICENSE
-!package.json
-!package-lock.json
-!README.md
-!src
--- a/backend/backend/Dockerfile
+++ b/backend/backend/Dockerfile
@@ -1,13 +1,14 @@
-FROM registry.gitlab.com/openstapps/projectmanagement/node
+FROM openstapps:workspace as pruned

 USER root
-RUN apk add --update python3 py3-pip make g++
+RUN pnpm --filter @openstapps/backend --prod deploy /pruned
+
+FROM registry.gitlab.com/openstapps/projectmanagement/node:v18

 USER node
-ADD --chown=node:node . /app
-WORKDIR /app
-RUN npm ci && npm run build
+ENV NODE_ENV=production
+
+COPY --chown=node:node --from=pruned pruned .

 EXPOSE 3000
-
-CMD ["node", "app"]
+ENTRYPOINT ["node", "app.js"]
--- a/backend/backend/integration-test.yml
+++ b/backend/backend/integration-test.yml
@@ -1,9 +1,9 @@
 version: '3'
 services:
  backend:
+    image: openstapps:backend
    ports:
      - '3000:3000'
-    build: .
    environment:
      STAPPS_LOG_LEVEL: '31'
      STAPPS_EXIT_LEVEL: '8'
@@ -13,12 +13,12 @@ services:
      ES_ADDR: 'http://elasticsearch:9200'

  elasticsearch:
+    image: openstapps:database
    ports:
      - '9200:9200'
-    image: 'registry.gitlab.com/openstapps/database:latest'

-  apicli:
-    image: 'registry.gitlab.com/openstapps/api/cli:latest'
+  api-cli:
+    image: openstapps:api-cli
    environment:
      STAPPS_LOG_LEVEL: '31'
      STAPPS_EXIT_LEVEL: '8'
--- a/backend/backend/package-lock.json
+++ b/backend/backend/package-lock.json
--- a/backend/backend/package.json
+++ b/backend/backend/package.json
@@ -1,7 +1,7 @@
 {
  "name": "@openstapps/backend",
  "description": "A reference implementation for a StApps backend",
-  "version": "2.1.0",
+  "version": "3.0.0",
  "type": "module",
  "license": "AGPL-3.0-only",
  "author": "André Bierlein <andre.mt.bierlein@gmail.com>",
@@ -17,8 +17,16 @@
  ],
  "main": "lib/app.js",
  "bin": "app.js",
+  "files": [
+    "app.js",
+    "lib",
+    "config",
+    "README.md",
+    "ROUTES.md"
+  ],
  "scripts": {
-    "build": "tsup",
+    "build": "tsup-node",
+    "build:docker": "docker build -t openstapps:backend .",
    "dev": "tsup --watch --onSuccess \"pnpm run start\"",
    "format": "prettier . -c --ignore-path ../../.gitignore",
    "format:fix": "prettier --write . --ignore-path ../../.gitignore",
@@ -27,14 +35,24 @@
    "start": "cross-env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node app.js",
    "start:debug": "cross-env STAPPS_LOG_LEVEL=31 NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node app.js",
    "test": "pnpm run test:unit",
-    "test:integration": "docker-compose -f integration-test.yml pull && docker-compose -f integration-test.yml up --build --abort-on-container-exit --exit-code-from apicli",
+    "test:integration": "docker-compose -f integration-test.yml up --abort-on-container-exit --exit-code-from api-cli",
    "test:unit": "cross-env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true STAPPS_LOG_LEVEL=0 mocha --exit"
  },
  "dependencies": {
    "@elastic/elasticsearch": "8.4.0",
    "@openstapps/core": "workspace:*",
    "@openstapps/core-tools": "workspace:*",
+    "@openstapps/es-mapping-generator": "workspace:*",
    "@openstapps/logger": "workspace:*",
+    "@types/body-parser": "1.19.2",
+    "@types/cors": "2.8.13",
+    "@types/express": "4.17.17",
+    "@types/geojson": "1.0.6",
+    "@types/node": "18.15.3",
+    "@types/node-cron": "3.0.7",
+    "@types/nodemailer": "6.4.7",
+    "@types/promise-queue": "2.2.0",
+    "@types/uuid": "8.3.4",
    "body-parser": "1.20.2",
    "cors": "2.8.5",
    "cosmiconfig": "8.1.3",
@@ -50,38 +68,25 @@
    "nodemailer": "6.9.1",
    "prom-client": "14.1.1",
    "promise-queue": "2.2.5",
-    "ts-node": "10.9.1",
    "uuid": "8.3.2"
  },
  "devDependencies": {
-    "@openstapps/es-mapping-generator": "workspace:*",
    "@openstapps/eslint-config": "workspace:*",
    "@openstapps/prettier-config": "workspace:*",
    "@openstapps/tsconfig": "workspace:*",
    "@testdeck/mocha": "0.3.3",
-    "@types/body-parser": "1.19.2",
    "@types/chai": "4.3.5",
    "@types/chai-as-promised": "7.1.5",
-    "@types/config": "3.3.0",
-    "@types/cors": "2.8.13",
-    "@types/express": "4.17.17",
-    "@types/geojson": "1.0.6",
    "@types/mocha": "10.0.1",
    "@types/morgan": "1.9.4",
-    "@types/node": "18.15.3",
-    "@types/node-cron": "3.0.7",
-    "@types/nodemailer": "6.4.7",
-    "@types/promise-queue": "2.2.0",
    "@types/sinon": "10.0.14",
    "@types/sinon-express-mock": "1.3.9",
    "@types/supertest": "2.0.12",
-    "@types/uuid": "8.3.4",
    "c8": "7.14.0",
    "chai": "4.3.7",
    "chai-as-promised": "7.1.1",
    "conventional-changelog-cli": "2.2.2",
    "cross-env": "7.0.3",
-    "esmock": "2.2.3",
    "get-port": "5.1.1",
    "mocha": "10.2.0",
    "mocked-env": "1.3.5",
@@ -89,6 +94,7 @@
    "sinon": "15.0.4",
    "sinon-express-mock": "2.2.1",
    "supertest": "6.3.3",
+    "ts-node": "10.9.1",
    "tsup": "6.7.0",
    "typescript": "4.9.5"
  },
--- a/backend/backend/turbo.json
+++ b/backend/backend/turbo.json
@@ -0,0 +1,12 @@
+{
+  "extends": ["//"],
+  "pipeline": {
+    "test:integration": {
+      "dependsOn": [
+        "@openstapps/database#build:docker",
+        "@openstapps/backend#build:docker",
+        "@openstapps/api-cli#build:docker"
+      ]
+    }
+  }
+}