From 16bbb7e9e36b7adf27452e1b09f7970e98aa27df Mon Sep 17 00:00:00 2001 From: Anselm Stordeur Date: Tue, 8 Jan 2019 12:26:15 +0100 Subject: [PATCH] feat: add backend --- .dockerignore | 4 + .editorconfig | 17 + .gitattributes | 17 + .gitignore | 7 + .gitlab-ci.yml | 155 ++ .gitlab/ci/getRegistryBranch.sh | 4 + .gitlab/ci/getRegistryTag.sh | 8 + .gitlab/ci/pushAsLatestVersion.sh | 7 + .gitlab/ci/testCIScripts.sh | 29 + .gitlab/issue_templates/bug.md | 41 + .gitlab/issue_templates/feature.md | 20 + .mock-yeah | 3 + Dockerfile | 8 + LICENSE | 619 +++++ README.md | 70 + config/default.ts | 175 ++ config/elasticsearch-b-tu.ts | 30 + config/elasticsearch.ts | 25 + package-lock.json | 2372 +++++++++++++++++ package.json | 65 + src/app.ts | 157 ++ src/cli.ts | 93 + src/common.ts | 21 + src/notification/BackendTransport.ts | 77 + src/notification/MailQueue.ts | 96 + src/routes/BulkAddRoute.ts | 46 + src/routes/BulkDoneRoute.ts | 46 + src/routes/BulkRoute.ts | 31 + src/routes/HTTPTypes.ts | 43 + src/routes/IndexRoute.ts | 32 + src/routes/MultiSearchRoute.ts | 56 + src/routes/Route.ts | 159 ++ src/routes/SearchRoute.ts | 28 + src/routes/ThingUpdateRoute.ts | 32 + src/storage/BulkStorage.ts | 191 ++ src/storage/Database.ts | 78 + src/storage/elasticsearch/Elasticsearch.ts | 508 ++++ src/storage/elasticsearch/aggregations.ts | 88 + src/storage/elasticsearch/common.ts | 208 ++ src/storage/elasticsearch/monitoring.ts | 141 + src/storage/elasticsearch/query.ts | 408 +++ .../templates/address.field.template.json | 19 + .../templates/article.sc-type.template.json | 25 + .../templates/base.template.json | 91 + .../templates/book.sc-type.template.json | 31 + .../templates/catalog.sc-type.template.json | 22 + .../templates/date.sc-type.template.json | 47 + .../templates/diff.sc-type.template.json | 10 + .../templates/dish.sc-type.template.json | 22 + .../templates/event.sc-type.template.json | 47 + .../eventSubProperties.field.template.json | 16 + .../filterableDate.field.template.json | 8 + .../filterableKeyword.field.template.json | 8 + .../templates/floorplan.sc-type.template.json | 10 + .../templates/jobs.field.template.json | 26 + .../templates/offers.sc-type.template.json | 24 + .../organization.sc-type.template.json | 7 + .../templates/person.sc-type.template.json | 47 + .../templates/place.sc-type.template.json | 33 + .../placeSubProperties.field.template.json | 38 + .../templates/prices.field.template.json | 14 + .../sortableKeyword.field.template.json | 15 + .../templates/text.field.template.json | 10 + src/storage/elasticsearch/templating.ts | 145 + tsconfig.json | 6 + tslint.json | 3 + 66 files changed, 6939 insertions(+) create mode 100644 .dockerignore create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100755 .gitlab/ci/getRegistryBranch.sh create mode 100755 .gitlab/ci/getRegistryTag.sh create mode 100755 .gitlab/ci/pushAsLatestVersion.sh create mode 100755 .gitlab/ci/testCIScripts.sh create mode 100644 .gitlab/issue_templates/bug.md create mode 100644 .gitlab/issue_templates/feature.md create mode 100644 .mock-yeah create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 config/default.ts create mode 100644 config/elasticsearch-b-tu.ts create mode 100644 config/elasticsearch.ts create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 src/app.ts create mode 100644 src/cli.ts create mode 100644 src/common.ts create mode 100644 src/notification/BackendTransport.ts create mode 100644 src/notification/MailQueue.ts create mode 100644 src/routes/BulkAddRoute.ts create mode 100644 src/routes/BulkDoneRoute.ts create mode 100644 src/routes/BulkRoute.ts create mode 100644 src/routes/HTTPTypes.ts create mode 100644 src/routes/IndexRoute.ts create mode 100644 src/routes/MultiSearchRoute.ts create mode 100644 src/routes/Route.ts create mode 100644 src/routes/SearchRoute.ts create mode 100644 src/routes/ThingUpdateRoute.ts create mode 100644 src/storage/BulkStorage.ts create mode 100644 src/storage/Database.ts create mode 100644 src/storage/elasticsearch/Elasticsearch.ts create mode 100644 src/storage/elasticsearch/aggregations.ts create mode 100644 src/storage/elasticsearch/common.ts create mode 100644 src/storage/elasticsearch/monitoring.ts create mode 100644 src/storage/elasticsearch/query.ts create mode 100644 src/storage/elasticsearch/templates/address.field.template.json create mode 100644 src/storage/elasticsearch/templates/article.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/base.template.json create mode 100644 src/storage/elasticsearch/templates/book.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/catalog.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/date.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/diff.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/dish.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/event.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/eventSubProperties.field.template.json create mode 100644 src/storage/elasticsearch/templates/filterableDate.field.template.json create mode 100644 src/storage/elasticsearch/templates/filterableKeyword.field.template.json create mode 100644 src/storage/elasticsearch/templates/floorplan.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/jobs.field.template.json create mode 100644 src/storage/elasticsearch/templates/offers.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/organization.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/person.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/place.sc-type.template.json create mode 100644 src/storage/elasticsearch/templates/placeSubProperties.field.template.json create mode 100644 src/storage/elasticsearch/templates/prices.field.template.json create mode 100644 src/storage/elasticsearch/templates/sortableKeyword.field.template.json create mode 100644 src/storage/elasticsearch/templates/text.field.template.json create mode 100644 src/storage/elasticsearch/templating.ts create mode 100644 tsconfig.json create mode 100644 tslint.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..5579ca6b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.idea/ +.git/ +.vscode/ +Dockerfile diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..251c64e6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +# EditorConfig helps developers define and maintain consistent coding styles between different editors and IDEs +# editorconfig.org + +root = true + +[*] +indent_style = space +indent_size = 2 + +# We recommend you to keep these unchanged +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md] +trim_trailing_whitespace = false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..bdb0cabc --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Custom for Visual Studio +*.cs diff=csharp + +# Standard to msysgit +*.doc diff=astextplain +*.DOC diff=astextplain +*.docx diff=astextplain +*.DOCX diff=astextplain +*.dot diff=astextplain +*.DOT diff=astextplain +*.pdf diff=astextplain +*.PDF diff=astextplain +*.rtf diff=astextplain +*.RTF diff=astextplain diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e8c8414f --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.vscode/ +.idea/ +node_modules/ +coverage/ +*.js +*.js.map +lib/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..78988d3d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,155 @@ +image: registry.gitlab.com/openstapps/projectmanagement/node + +stages: + - build + - test + - publish + +build: + stage: build + script: + - npm install + - npm run build + artifacts: + untracked: true + paths: + - node_modules/ + tags: + - docker + +lint: + stage: test + dependencies: + - build + script: + - npm run tslint + tags: + - docker + +audit: + stage: test + dependencies: + - build + script: + - npm audit + tags: + - docker + +test:ci: + stage: test + dependencies: + - build + script: + - .gitlab/ci/testCIScripts.sh + tags: + - docker + +# Anchor templates for publishing the image in the docker registry +# Automatically publishing for versions in tags (eg v1.0.0 as 1.0.0), master and develop +# Manual publishing for all other branches +.publish_template_auto: &publish_template_auto + image: registry.gitlab.com/openstapps/projectmanagement/builder + stage: publish + dependencies: + - build + artifacts: + untracked: true + script: + - export REGISTRY_BRANCH=$(.gitlab/ci/getRegistryBranch.sh "$CI_JOB_NAME") + - export TAGNAME=$(.gitlab/ci/getRegistryTag.sh "$CI_BUILD_REF_NAME") + - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY + - docker build -t $CI_REGISTRY_IMAGE/$REGISTRY_BRANCH:$TAGNAME . + - docker tag $CI_REGISTRY_IMAGE/$REGISTRY_BRANCH:$TAGNAME $CI_REGISTRY_IMAGE/$REGISTRY_BRANCH:latest + - docker push $CI_REGISTRY_IMAGE/$REGISTRY_BRANCH:$TAGNAME + - .gitlab/ci/pushAsLatestVersion.sh "$CI_BUILD_REF_NAME" "$CI_REGISTRY_IMAGE/$REGISTRY_BRANCH" + only: + - /(^v[0-9]+\.[0-9]+\.[0-9]+$|^master$|^develop$)/ + tags: + - docker + +.publish_template_manual: &publish_template_manual + image: registry.gitlab.com/openstapps/projectmanagement/builder + stage: publish + dependencies: + - build + artifacts: + untracked: true + script: + - export REGISTRY_BRANCH=$(.gitlab/ci/getRegistryBranch.sh "$CI_JOB_NAME") + - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY + - docker build -t $CI_REGISTRY_IMAGE/$REGISTRY_BRANCH/$CI_COMMIT_REF_NAME:latest . + - docker push $CI_REGISTRY_IMAGE/$REGISTRY_BRANCH/$CI_COMMIT_REF_NAME:latest + except: + - /(^v[0-9]+\.[0-9]+\.[0-9]+$|^master$|^develop$)/ + only: + - branches + when: manual + tags: + - docker + + +# Anchor templates for custom build processes +.build_template_b-tu: &build_template_b-tu + # before_script: + # - npm install "@stapps/b-tu-feedback@0.13.1" + # - npm install "@stapps/b-tu-tickets@0.13.1" + # - npm install "@stapps/b-tu-isbn-availability@0.13.1" + tags: + - docker + +.build_template_f-u: &build_template_f-u + # before_script: + # - npm install "git+ssh://git@gitlab.tubit.tu-berlin.de:stapps-f-u/feedback.git#1.0.0" + # - npm install "git+ssh://git@gitlab.tubit.tu-berlin.de:stapps-f-u/dish-feedback.git#1.0.0" + tags: + - docker + + +# Jobs joining anchor templates with automatic publishing behaviour +# ! The jobname must end with ":" followed by the name for the registry branch +publish:default: + <<: *publish_template_auto + +publish:ab-fh: + <<: *publish_template_auto + +publish:b-tu: + <<: *publish_template_auto + <<: *build_template_b-tu + +publish:f-u: + <<: *publish_template_auto + <<: *build_template_f-u + +publish:gi-fh: + <<: *publish_template_auto + +publish:gi-u: + <<: *publish_template_auto + +publish:ks-ug: + <<: *publish_template_auto + +# Jobs joining anchor templates with manual publishing behaviour +custom:default: + <<: *publish_template_manual + +custom:ab-fh: + <<: *publish_template_manual + +custom:b-tu: + <<: *publish_template_manual + <<: *build_template_b-tu + +custom:f-u: + <<: *publish_template_manual + <<: *build_template_f-u + +custom:gi-fh: + <<: *publish_template_manual + +custom:gi-u: + <<: *publish_template_manual + +custom:ks-ug: + <<: *publish_template_manual diff --git a/.gitlab/ci/getRegistryBranch.sh b/.gitlab/ci/getRegistryBranch.sh new file mode 100755 index 00000000..0b71943f --- /dev/null +++ b/.gitlab/ci/getRegistryBranch.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env sh + +# script returns string with everything after the last colon of $1 input +echo -n $1 | grep -oE '[^:]+$' diff --git a/.gitlab/ci/getRegistryTag.sh b/.gitlab/ci/getRegistryTag.sh new file mode 100755 index 00000000..d325ce19 --- /dev/null +++ b/.gitlab/ci/getRegistryTag.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +# script returns semantical versioning string linke 2.0.0 (if $1 is v2.0.0) or $1 +if echo -n $1 | grep -Eq 'v[0-9]+\.[0-9]+\.[0-9]+'; then + echo $(echo -n "$1" | cut -c 2-); +else + echo $1; +fi diff --git a/.gitlab/ci/pushAsLatestVersion.sh b/.gitlab/ci/pushAsLatestVersion.sh new file mode 100755 index 00000000..22141ee2 --- /dev/null +++ b/.gitlab/ci/pushAsLatestVersion.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# If this is a pipeline of a version tag, also push this version +# as latest to the registry alias $2:latest +if echo -n $1 | grep -Eq 'v[0-9]+\.[0-9]+\.[0-9]+'; then + docker push $2:latest; +fi diff --git a/.gitlab/ci/testCIScripts.sh b/.gitlab/ci/testCIScripts.sh new file mode 100755 index 00000000..6cd7218a --- /dev/null +++ b/.gitlab/ci/testCIScripts.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env sh + +# test all CI scripts + +SCRIPT_DIR=$(dirname "$0") + +TAG_VERSION=$(sh $SCRIPT_DIR/getRegistryTag.sh "v1.0.0") +TAG_TEST=$(sh $SCRIPT_DIR/getRegistryTag.sh "TEST") +BRANCH_NAME=$(sh $SCRIPT_DIR/getRegistryBranch.sh "very:first:test") + +# Leaving out pushAsLatestVersion.sh as its controll flow +# is based on the same condition as getRegistryTag.sh + +if [ $TAG_VERSION != "1.0.0" ]; then + echo "ERROR in CI SCRIPT: $SCRIPT_DIR/getRegistryTag.sh" + return 1 +fi + +if [ $TAG_TEST != "TEST" ]; then + echo "ERROR in CI SCRIPT: $SCRIPT_DIR/getRegistryTag.sh" + return 2 +fi + +if [ $BRANCH_NAME != "test" ]; then + echo "ERROR in CI SCRIPT: $SCRIPT_DIR/getRegistryBranch.sh" + return 3 +fi + +return 0 diff --git a/.gitlab/issue_templates/bug.md b/.gitlab/issue_templates/bug.md new file mode 100644 index 00000000..b31312d7 --- /dev/null +++ b/.gitlab/issue_templates/bug.md @@ -0,0 +1,41 @@ +## Summary + +(Summarize the bug encountered concisely) + + +## Steps to reproduce + +(How one can reproduce the issue - this is very important) + + +## Example Project + +(If possible, please create an example project here on GitLab.com that exhibits the problematic behaviour, and link to it here in the bug report) + +(If you are using an older version of GitLab, this will also determine whether the bug has been fixed in a more recent version) + +## Which version of the software did you use ? + +(Version numbers of used software or commit references) + +## What is the current bug behavior? + +(What actually happens) + + +## What is the expected correct behavior? + +(What you should see instead) + + +## Relevant logs and/or screenshots + +(Paste any relevant logs - please use code blocks (```) to format console output, +logs, and code as it's very hard to read otherwise.) + + +## Possible fixes + +(If you can, link to the line of code that might be responsible for the problem) + +/label ~bug ~meeting diff --git a/.gitlab/issue_templates/feature.md b/.gitlab/issue_templates/feature.md new file mode 100644 index 00000000..e601cd15 --- /dev/null +++ b/.gitlab/issue_templates/feature.md @@ -0,0 +1,20 @@ +## Description + +(Describe the feature that you're requesting concisely) + + +## Explanation + +(Explain why the feature is necessary) + + +## Mockups/Data + +(If possible, provide mockups or examples of communication, which demonstrate the feature) + + +## Dependencies, issues to be resolved beforehand + +(List issues or dependencies that need to be resolved before this feature can be implemented) + +/label ~feature ~meeting \ No newline at end of file diff --git a/.mock-yeah b/.mock-yeah new file mode 100644 index 00000000..2c6f3aed --- /dev/null +++ b/.mock-yeah @@ -0,0 +1,3 @@ +{ + "port": 9200 +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a0d83f26 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM registry.gitlab.com/openstapps/projectmanagement/node + +ADD . /app +WORKDIR /app + +EXPOSE 3000 + +CMD ["node", "./lib/cli"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..4f3a4263 --- /dev/null +++ b/LICENSE @@ -0,0 +1,619 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +Preamble + +The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + +The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + +When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + +Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + +A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + +The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + +An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + +The precise terms and conditions for copying, distribution and +modification follow. + +TERMS AND CONDITIONS + +0. Definitions. + +"This License" refers to version 3 of the GNU Affero General Public License. + +"Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + +"The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + +To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + +A "covered work" means either the unmodified Program or a work based +on the Program. + +To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + +To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + +An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +1. Source Code. + +The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + +A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + +The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + +The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + +The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + +The Corresponding Source for a work in source code form is that +same work. + +2. Basic Permissions. + +All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + +3. Protecting Users' Legal Rights From Anti-Circumvention Law. + +No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + +When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + +4. Conveying Verbatim Copies. + +You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + +5. Conveying Modified Source Versions. + +You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + +A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + +6. Conveying Non-Source Forms. + +You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + +A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + +"Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + +If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + +The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + +"Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + +All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + +8. Termination. + +You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + +However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + +9. Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + +10. Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + +An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + +11. Patents. + +A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + +A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + +In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + +If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + +A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + +12. No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + +13. Remote Network Interaction; Use with the GNU General Public License. + +Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + +Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + +14. Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + +If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + +Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + +15. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +16. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + +17. Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + +END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md new file mode 100644 index 00000000..80eaaa88 --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +[![pipeline status](https://img.shields.io/gitlab/pipeline/openstapps/backend.svg?style=flat-square)](https://gitlab.com/openstapps/backend/commits/master) +# backend (a reference implementation of a StApps backend) +This project is a reference implementation for a StApps backend. It provides an HTTP API to index data into a database, +perform full text search, sorts and filters. It also delivers the configuration needed by the app. The API is specified +within the [@openstapps/core](https://gitlab.com/openstapps/core). + +If you want to perform requests, index data or search within JavaScript or TypeScript you should consider using +[@openstapps/api](https://gitlab.com/openstapps/api) + +# Usage +This backend is not a standalone software. It needs a database like Elasticsearch to work. + +If you just want to use the backend you should consider using +[minimal-deployment](https://gitlab.com/openstapps/minimal-deployment). The minimal-deployment will provide +you with everything you need to run this backend. + +# Local usage for development purposes +## Requirements +* Elasticsearch (5.5) +* Node.js (~10) / NPM +* Docker + +## Start Database (Elasticsearch) +Elasticsearch needs some configuration and plugins to be able to work +with the backend. To save you some work we provide a +[docker image](https://gitlab.com/openstapps/database) which +only needs to be executed to work with the backend. + +Run `docker run -d -p 9200:9200 registry.gitlab.com/openstapps/database:master` + +Elasticsearch should be running at port 9200 now. If you have problems with +getting elasticsearch to work, have a look in the +[README](https://gitlab.com/openstapps/database) of the image +first. + +## Start backend +Run `npm install` and `npm run build`, then start with `npm start`. The server should now be accepting connections at `http://localhost:3000`. + +# Environment Variables +To select a database implementation you have to set the `NODE_CONFIG_ENV` variable. At the time only `NODE_CONFIG_ENV=elasticsearch` is supported. +Set `NODE_ENV=production` to run backend for production usages. In production the backend expects some kind of monitoring to be set via the +environment. At the time only SMTP is being implemented. The backend wouldn't start if you don't provide SMTP authentification. Alternatively +you can set `ALLOW_NO_TRANSPORT=true`. To set up an SMTP configuration have a look at +[@openstapps/logger](https://gitlab.com/openstapps/logger). + +## Config files +Each university can have it's specific config for the general backend and app and for all databases. + +All config files can be found in `./config/`. There is a `default.ts` which is used by default. You can create an +university specific file with following naming scheme: `default-.ts` + +A university specific file will only overwrite all properties of the `default.ts` that are set in the file itself. +To start the backend using your configuration you have to provide the `NODE_APP_INSTANCE` environment variable +with your university license plate. + +To set a database you have to provide the `NODE_CONFIG_ENV` environment variable with the name of the database. +At the time only Elasticsearch is implemented. + +To create your university specific config file for the elasticsearch you have to create a file with following naming +scheme: `elasticsearch-.ts`. + +## Debugging +Set `ES_DEBUG=true` to enable verbose Elasticsearch tracing information. +This can be useful to debug some issues between backend and elasticsearch. + +## Setting a different url for elasticsearch +Set `ES_PORT_9200_TCP_ADDR` to change the elasticsearch-http-address which by default is `localhost`. +Set `ES_PORT_9200_TCP_PORT` to change the elasticsearch port which by default is `9200` . + +## [Contributing](https://gitlab.com/openstapps/projectmanagement/blob/master/CONTRIBUTING.md) diff --git a/config/default.ts b/config/default.ts new file mode 100644 index 00000000..370cf204 --- /dev/null +++ b/config/default.ts @@ -0,0 +1,175 @@ +import { SCConfigFile } from '@openstapps/core'; + +/** + * This is the default configuration for app and backend + * + * University specific files can be created with following naming scheme: default-.ts + * + * To select your university specific configuration which is merged from this default file and your university specific + * file, you have to supply the `NODE_APP_INSTANCE` environment variable with your license plate + * + * To get more information about the meaning of specific fields please have a look at `@openstapps/core` or use your + * IDE to read the TSDoc documentation. + */ +const config: Partial = { + app: { + campusPolygon: { + coordinates: [ + [ + [ + 13.31916332244873, + 52.50796756998264, + ], + [ + 13.336544036865234, + 52.50796756998264, + ], + [ + 13.336544036865234, + 52.51726547416385, + ], + [ + 13.31916332244873, + 52.51726547416385, + ], + [ + 13.31916332244873, + 52.50796756998264, + ], + ], + ], + type: 'Polygon', + }, + features: { + widgets: true, + }, + menus: [], + name: 'StApps - Technische Universität Berlin', + privacyPolicyUrl: 'https://stappsbe01.innocampus.tu-berlin.de/_static/privacy.md', + settings: [], + }, + backend: { + SCVersion: '1.0.0', + hiddenTypes: [ + 'date series', + 'diff', + 'floor', + ], + name: 'Technische Universität Berlin', + namespace: '909a8cbc-8520-456c-b474-ef1525f14209', + sortableFields: [ + { + fieldName: 'name', + sortTypes: ['ducet'], + }, + { + fieldName: 'type', + sortTypes: ['ducet'], + }, + { + fieldName: 'categories', + onlyOnTypes: ['academic event', 'building', 'catalog', 'dish', 'point of interest', 'room'], + sortTypes: ['ducet'], + }, + { + fieldName: 'geo.point.coordinates', + onlyOnTypes: ['building', 'point of interest', 'room'], + sortTypes: ['distance'], + }, + { + fieldName: 'geo.point.coordinates', + onlyOnTypes: ['building', 'point of interest', 'room'], + sortTypes: ['distance'], + }, + { + fieldName: 'inPlace.geo.point.coordinates', + onlyOnTypes: ['date series', 'dish', 'floor', 'organization', 'point of interest', 'room', 'ticket'], + sortTypes: ['distance'], + }, + { + fieldName: 'offers', + onlyOnTypes: ['dish'], + sortTypes: ['price'], + }, + ], + }, + internal: { + aggregations: [ + { + fieldName: 'categories', + onlyOnTypes: ['academic event', 'article', 'building', 'catalog', 'dish', 'point of interest', 'room'], + }, + { + fieldName: 'inPlace.name', + onlyOnTypes: ['date series', 'dish', 'floor', 'organization', 'point of interest', 'room', 'ticket'], + }, + { + fieldName: 'academicTerms.acronym', + onlyOnTypes: ['academic event', 'sport course'], + }, + { + fieldName: 'academicTerm.acronym', + onlyOnTypes: ['catalog'], + }, + { + fieldName: 'majors', + onlyOnTypes: ['academic event'], + }, + { + fieldName: 'keywords', + onlyOnTypes: ['article', 'book', 'message', 'video'], + }, + { + fieldName: 'type', + }, + ], + boostings: [ + { + factor: 1, + fields: { + 'academicTerms.acronym': { + 'SS 2018': 1.05, + 'WS 2018/19': 1.1, + }, + categories: { + 'course': 1.08, + 'integrated course': 1.08, + 'introductory class': 1.05, + 'lecture': 1.1, + 'seminar': 1.01, + 'tutorial': 1.05, + }, + }, + type: 'academic event', + }, + { + factor: 1.6, + type: 'building', + }, + { + factor: 1, + fields: { + 'categories': { + 'cafe': 1.1, + 'learn': 1.1, + 'library': 1.2, + 'restaurant': 1.1, + }, + }, + type: 'point of interest', + }, + { + factor: 1, + fields: { + 'categories': { + 'main dish': 2, + }, + }, + type: 'dish', + }, + ], + }, + uid: 'b-tu', +}; + +export default config; diff --git a/config/elasticsearch-b-tu.ts b/config/elasticsearch-b-tu.ts new file mode 100644 index 00000000..fbd5217f --- /dev/null +++ b/config/elasticsearch-b-tu.ts @@ -0,0 +1,30 @@ +import { ElasticsearchConfigFile } from '../src/storage/elasticsearch/Elasticsearch'; + +/** + * A partial type which is recursive + * + * Copied and only modified array type from `[]` to `Array<>` from https://stackoverflow.com/a/51365037 + */ +type RecursivePartial = { + [P in keyof T]?: + T[P] extends Array<(infer U)> ? Array> : + T[P] extends object ? RecursivePartial : + T[P]; +}; + +/** + * This is the database configuration for the technical university of berlin + */ +const config: RecursivePartial = { + internal: { + database: { + name: 'elasticsearch', + query: { + minMatch: '60%', + queryType: 'query_string', + }, + }, + }, +}; + +export default config; diff --git a/config/elasticsearch.ts b/config/elasticsearch.ts new file mode 100644 index 00000000..3bf645ed --- /dev/null +++ b/config/elasticsearch.ts @@ -0,0 +1,25 @@ +import { ElasticsearchConfigFile } from '../src/storage/elasticsearch/common'; + +/** + * This is the default configuration for elasticsearch (a database) + * + * University specific files can be created with following naming scheme: elasticsearch-.ts + * + * To select your university specific configuration which is merged from this default file and your university specific + * file, you have to supply the `NODE_APP_INSTANCE` environment variable with your license plate + * + * To select a differen database you have to supply the `NODE_CONFIG_ENV` environment variable with a database name that + * is implemented in the backend + * + * To get more information about the meaning of specific fields please use your IDE to read the TSDoc documentation. + */ +const config: ElasticsearchConfigFile = { + internal: { + database: { + name: 'elasticsearch', + version: '5.5', + }, + }, +}; + +export default config; diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..324a4b5c --- /dev/null +++ b/package-lock.json @@ -0,0 +1,2372 @@ +{ + "name": "@openstapps/backend", + "version": "0.0.1", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@openstapps/core-validator": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@openstapps/core-validator/-/core-validator-0.0.1.tgz", + "integrity": "sha512-g48PGH6T82e45C/f2QDvwvaSxoiJUbCsCaonEyM5GazxN8D9hAjbk8DCYarcPmhEhrw7aUem3hxECwCn2/po1A==", + "requires": { + "@openstapps/logger": "0.0.3", + "@types/node": "10.12.10", + "commander": "2.19.0", + "jsonschema": "1.2.4" + }, + "dependencies": { + "@types/node": { + "version": "10.12.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.10.tgz", + "integrity": "sha512-8xZEYckCbUVgK8Eg7lf5Iy4COKJ5uXlnIOnePN0WUwSQggy9tolM+tDJf7wMOnT/JT/W9xDYIaYggt3mRV2O5w==" + } + } + }, + "@openstapps/logger": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@openstapps/logger/-/logger-0.0.3.tgz", + "integrity": "sha512-Q1kghyVNIXepfuLcdy2gFygI6jpxTBV0oqwM46hqzST4w/DNmDnzpScVQNQf5C0PhLUihPNhpjLnu6i7ujIX3g==", + "requires": { + "@types/circular-json": "0.4.0", + "@types/node": "10.12.10", + "@types/nodemailer": "4.6.5", + "circular-json": "0.5.9", + "nodemailer": "4.7.0" + }, + "dependencies": { + "@types/node": { + "version": "10.12.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.10.tgz", + "integrity": "sha512-8xZEYckCbUVgK8Eg7lf5Iy4COKJ5uXlnIOnePN0WUwSQggy9tolM+tDJf7wMOnT/JT/W9xDYIaYggt3mRV2O5w==" + } + } + }, + "@types/body-parser": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.17.0.tgz", + "integrity": "sha512-a2+YeUjPkztKJu5aIF2yArYFQQp8d51wZ7DavSHjFuY1mqVgidGyzEQ41JIVNy82fXj8yPgy2vJmfIywgESW6w==", + "dev": true, + "requires": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "@types/circular-json": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@types/circular-json/-/circular-json-0.4.0.tgz", + "integrity": "sha512-7+kYB7x5a7nFWW1YPBh3KxhwKfiaI4PbZ1RvzBU91LZy7lWJO822CI+pqzSre/DZ7KsCuMKdHnLHHFu8AyXbQg==" + }, + "@types/config": { + "version": "0.0.34", + "resolved": "http://registry.npmjs.org/@types/config/-/config-0.0.34.tgz", + "integrity": "sha512-jWi9DXx77hnzN4kHCNEvP/kab+nchRLTg9yjXYxjTcMBkuc5iBb3QuwJ4sPrb+nzy1GQjrfyfMqZOdR4i7opRQ==", + "dev": true + }, + "@types/connect": { + "version": "3.4.32", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.32.tgz", + "integrity": "sha512-4r8qa0quOvh7lGD0pre62CAb1oni1OO6ecJLGCezTmhQ8Fz50Arx9RUszryR8KlgK6avuSXvviL6yWyViQABOg==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/cors": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.4.tgz", + "integrity": "sha512-ipZjBVsm2tF/n8qFGOuGBkUij9X9ZswVi9G3bx/6dz7POpVa6gVHcj1wsX/LVEn9MMF41fxK/PnZPPoTD1UFPw==", + "dev": true, + "requires": { + "@types/express": "*" + } + }, + "@types/events": { + "version": "1.2.0", + "resolved": "http://registry.npmjs.org/@types/events/-/events-1.2.0.tgz", + "integrity": "sha512-KEIlhXnIutzKwRbQkGWb/I4HFqBuUykAdHgDED6xqwXJfONCjF5VoE0cXEiurh3XauygxzeDzgtXUqvLkxFzzA==" + }, + "@types/express": { + "version": "4.16.0", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.16.0.tgz", + "integrity": "sha512-TtPEYumsmSTtTetAPXlJVf3kEqb6wZK0bZojpJQrnD/djV4q1oB6QQ8aKvKqwNPACoe02GNiy5zDzcYivR5Z2w==", + "dev": true, + "requires": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "*", + "@types/serve-static": "*" + } + }, + "@types/express-serve-static-core": { + "version": "4.16.0", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.16.0.tgz", + "integrity": "sha512-lTeoCu5NxJU4OD9moCgm0ESZzweAx0YqsAcab6OB0EB3+As1OaHtKnaGJvcngQxYsi9UNv0abn4/DRavrRxt4w==", + "dev": true, + "requires": { + "@types/events": "*", + "@types/node": "*", + "@types/range-parser": "*" + } + }, + "@types/fs-extra": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.0.4.tgz", + "integrity": "sha512-DsknoBvD8s+RFfSGjmERJ7ZOP1HI0UZRA3FSI+Zakhrc/Gy26YQsLI+m5V5DHxroHRJqCDLKJp7Hixn8zyaF7g==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/mime": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.0.tgz", + "integrity": "sha512-A2TAGbTFdBw9azHbpVd+/FkdW2T6msN1uct1O9bH3vTerEHKZhTXJUQXy+hNq1B0RagfU8U+KBdqiZpxjhOUQA==", + "dev": true + }, + "@types/morgan": { + "version": "1.7.35", + "resolved": "https://registry.npmjs.org/@types/morgan/-/morgan-1.7.35.tgz", + "integrity": "sha512-E9qFi0seOkdlQnCTPv54brNfGWeFdRaEhI5tSue4pdx/V+xfxvMETsxXhOEcj1cYL+0n/jcTEmj/jD2gjzCwMg==", + "dev": true, + "requires": { + "@types/express": "*" + } + }, + "@types/node": { + "version": "10.12.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.11.tgz", + "integrity": "sha512-3iIOhNiPGTdcUNVCv9e5G7GotfvJJe2pc9w2UgDXlUwnxSZ3RgcUocIU+xYm+rTU54jIKih998QE4dMOyMN1NQ==", + "dev": true + }, + "@types/node-cache": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@types/node-cache/-/node-cache-4.1.1.tgz", + "integrity": "sha512-dFZY8H8uaf4qtd+a18jS5VChiI1apGXh6N6SXdRf4SPfdLG/D+oNvsEM2Ic0ee+nFqJLAsRy4+R0qkvO45CIWA==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/node-cron": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-2.0.0.tgz", + "integrity": "sha512-JGP9lBkMYc/a1/RdwgP9latBZw9lHz6W36CnjRZB7dADWD/g820gu9hIaVvD/+9f+XwhbGhnMwDrpJLfhBpVeg==", + "dev": true + }, + "@types/nodemailer": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-4.6.5.tgz", + "integrity": "sha512-cbs2HFLj33TBqzcCqTrs+6/mgTX3xl0odbApv3vTdF2+JERLxh5rDZCasXhvy+YqaiUNBr2I1RjNCdbKGs1Bnw==", + "requires": { + "@types/events": "*", + "@types/node": "*" + }, + "dependencies": { + "@types/node": { + "version": "10.12.12", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.12.tgz", + "integrity": "sha512-Pr+6JRiKkfsFvmU/LK68oBRCQeEg36TyAbPhc2xpez24OOZZCuoIhWGTd39VZy6nGafSbxzGouFPTFD/rR1A0A==" + } + } + }, + "@types/promise-queue": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@types/promise-queue/-/promise-queue-2.2.0.tgz", + "integrity": "sha512-9QLtid6GxEWqpF+QImxBRG6bSVOHtpAm2kXuIyEvZBbSOupLvqhhJv8uaHbS8kUL8FDjzH3RWcSyC/52WOVtGw==", + "dev": true + }, + "@types/range-parser": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.2.tgz", + "integrity": "sha512-HtKGu+qG1NPvYe1z7ezLsyIaXYyi8SoAVqWDZgDQ8dLrsZvSzUNCwZyfX33uhWxL/SU0ZDQZ3nwZ0nimt507Kw==", + "dev": true + }, + "@types/serve-static": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.2.tgz", + "integrity": "sha512-/BZ4QRLpH/bNYgZgwhKEh+5AsboDBcUdlBYgzoLX0fpj3Y2gp6EApyOlM3bK53wQS/OE1SrdSYBAbux2D1528Q==", + "dev": true, + "requires": { + "@types/express-serve-static-core": "*", + "@types/mime": "*" + } + }, + "@types/sha1": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@types/sha1/-/sha1-1.1.1.tgz", + "integrity": "sha512-Yrz4TPsm/xaw7c39aTISskNirnRJj2W9OVeHv8ooOR9SG8NHEfh4lwvGeN9euzxDyPfBdFkvL/VHIY3kM45OpQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/uuid": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.4.tgz", + "integrity": "sha512-tPIgT0GUmdJQNSHxp0X2jnpQfBSTfGxUMc/2CXBU2mnyTFVYVa2ojpoQ74w0U2yn2vw3jnC640+77lkFFpdVDw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "dev": true, + "requires": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + } + }, + "accepts": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", + "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=", + "requires": { + "mime-types": "~2.1.18", + "negotiator": "0.6.1" + } + }, + "add-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/add-stream/-/add-stream-1.0.0.tgz", + "integrity": "sha1-anmQQ3ynNtXhKI25K9MmbV9csqo=", + "dev": true + }, + "agentkeepalive": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", + "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", + "requires": { + "humanize-ms": "^1.2.1" + } + }, + "ajv": { + "version": "6.6.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.6.1.tgz", + "integrity": "sha512-ZoJjft5B+EJBjUyu9C9Hc0OZyPZSSlOF+plzouTrg6UlA8f+e/n8NIgBFG/9tppJtpPWfthHakK7juJdNDODww==", + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" + }, + "array-find-index": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz", + "integrity": "sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E=", + "dev": true + }, + "array-flatten": { + "version": "1.1.1", + "resolved": "http://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "array-ify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/array-ify/-/array-ify-1.0.0.tgz", + "integrity": "sha1-nlKHYrSpBmrRY6aWKjZEGOlibs4=", + "dev": true + }, + "arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=" + }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, + "async": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz", + "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==", + "dev": true, + "requires": { + "lodash": "^4.17.10" + } + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" + }, + "basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "requires": { + "safe-buffer": "5.1.2" + } + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "requires": { + "tweetnacl": "^0.14.3" + } + }, + "body-parser": { + "version": "1.18.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", + "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "requires": { + "bytes": "3.0.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "~1.6.3", + "iconv-lite": "0.4.23", + "on-finished": "~2.3.0", + "qs": "6.5.2", + "raw-body": "2.3.3", + "type-is": "~1.6.16" + } + }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, + "builtin-modules": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", + "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", + "dev": true + }, + "bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + }, + "camelcase": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", + "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=", + "dev": true + }, + "camelcase-keys": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-4.2.0.tgz", + "integrity": "sha1-oqpfsa9oh1glnDLBQUJteJI7m3c=", + "dev": true, + "requires": { + "camelcase": "^4.1.0", + "map-obj": "^2.0.0", + "quick-lru": "^1.0.0" + } + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" + }, + "circular-json": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.5.9.tgz", + "integrity": "sha512-4ivwqHpIFJZBuhN3g/pEcdbnGUywkBblloGbkglyloVjjR3uT6tieI89MVOfbP2tHX5sgb01FuLgAOzebNlJNQ==" + }, + "clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=" + }, + "combined-stream": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", + "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "commander": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz", + "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==" + }, + "compare-func": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/compare-func/-/compare-func-1.3.2.tgz", + "integrity": "sha1-md0LpFfh+bxyKxLAjsM+6rMfpkg=", + "dev": true, + "requires": { + "array-ify": "^1.0.0", + "dot-prop": "^3.0.0" + } + }, + "content-disposition": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", + "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "conventional-changelog": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/conventional-changelog/-/conventional-changelog-3.0.5.tgz", + "integrity": "sha512-JYSVGJbnOl9S2gkZwmoJ+wX2gxNVHodUmEiv+eIykeJBNX0zN5vJ3oa2xCvk2HiF7TZ+Les0eq/aX49dcymONA==", + "dev": true, + "requires": { + "conventional-changelog-angular": "^5.0.2", + "conventional-changelog-atom": "^2.0.1", + "conventional-changelog-codemirror": "^2.0.1", + "conventional-changelog-core": "^3.1.5", + "conventional-changelog-ember": "^2.0.2", + "conventional-changelog-eslint": "^3.0.1", + "conventional-changelog-express": "^2.0.1", + "conventional-changelog-jquery": "^3.0.4", + "conventional-changelog-jshint": "^2.0.1", + "conventional-changelog-preset-loader": "^2.0.2" + } + }, + "conventional-changelog-angular": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/conventional-changelog-angular/-/conventional-changelog-angular-5.0.2.tgz", + "integrity": "sha512-yx7m7lVrXmt4nKWQgWZqxSALEiAKZhOAcbxdUaU9575mB0CzXVbgrgpfSnSP7OqWDUTYGD0YVJ0MSRdyOPgAwA==", + "dev": true, + "requires": { + "compare-func": "^1.3.1", + "q": "^1.5.1" + } + }, + "conventional-changelog-atom": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/conventional-changelog-atom/-/conventional-changelog-atom-2.0.1.tgz", + "integrity": "sha512-9BniJa4gLwL20Sm7HWSNXd0gd9c5qo49gCi8nylLFpqAHhkFTj7NQfROq3f1VpffRtzfTQp4VKU5nxbe2v+eZQ==", + "dev": true, + "requires": { + "q": "^1.5.1" + } + }, + "conventional-changelog-cli": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/conventional-changelog-cli/-/conventional-changelog-cli-2.0.11.tgz", + "integrity": "sha512-00Z4EZfpuQxvStA5fjJXdixXCtRd5/AUMUOhYKOomhH3cRFqzF/P0MP8vavT9wnGkR0eba9mrWsMuqeVszPRxQ==", + "dev": true, + "requires": { + "add-stream": "^1.0.0", + "conventional-changelog": "^3.0.5", + "lodash": "^4.2.1", + "meow": "^4.0.0", + "tempfile": "^1.1.1" + } + }, + "conventional-changelog-codemirror": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/conventional-changelog-codemirror/-/conventional-changelog-codemirror-2.0.1.tgz", + "integrity": "sha512-23kT5IZWa+oNoUaDUzVXMYn60MCdOygTA2I+UjnOMiYVhZgmVwNd6ri/yDlmQGXHqbKhNR5NoXdBzSOSGxsgIQ==", + "dev": true, + "requires": { + "q": "^1.5.1" + } + }, + "conventional-changelog-core": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/conventional-changelog-core/-/conventional-changelog-core-3.1.5.tgz", + "integrity": "sha512-iwqAotS4zk0wA4S84YY1JCUG7X3LxaRjJxuUo6GI4dZuIy243j5nOg/Ora35ExT4DOiw5dQbMMQvw2SUjh6moQ==", + "dev": true, + "requires": { + "conventional-changelog-writer": "^4.0.2", + "conventional-commits-parser": "^3.0.1", + "dateformat": "^3.0.0", + "get-pkg-repo": "^1.0.0", + "git-raw-commits": "2.0.0", + "git-remote-origin-url": "^2.0.0", + "git-semver-tags": "^2.0.2", + "lodash": "^4.2.1", + "normalize-package-data": "^2.3.5", + "q": "^1.5.1", + "read-pkg": "^3.0.0", + "read-pkg-up": "^3.0.0", + "through2": "^2.0.0" + } + }, + "conventional-changelog-ember": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/conventional-changelog-ember/-/conventional-changelog-ember-2.0.2.tgz", + "integrity": "sha512-qtZbA3XefO/n6DDmkYywDYi6wDKNNc98MMl2F9PKSaheJ25Trpi3336W8fDlBhq0X+EJRuseceAdKLEMmuX2tg==", + "dev": true, + "requires": { + "q": "^1.5.1" + } + }, + "conventional-changelog-eslint": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/conventional-changelog-eslint/-/conventional-changelog-eslint-3.0.1.tgz", + "integrity": "sha512-yH3+bYrtvgKxSFChUBQnKNh9/U9kN2JElYBm253VpYs5wXhPHVc9ENcuVGWijh24nnOkei7wEJmnmUzgZ4ok+A==", + "dev": true, + "requires": { + "q": "^1.5.1" + } + }, + "conventional-changelog-express": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/conventional-changelog-express/-/conventional-changelog-express-2.0.1.tgz", + "integrity": "sha512-G6uCuCaQhLxdb4eEfAIHpcfcJ2+ao3hJkbLrw/jSK/eROeNfnxCJasaWdDAfFkxsbpzvQT4W01iSynU3OoPLIw==", + "dev": true, + "requires": { + "q": "^1.5.1" + } + }, + "conventional-changelog-jquery": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/conventional-changelog-jquery/-/conventional-changelog-jquery-3.0.4.tgz", + "integrity": "sha512-IVJGI3MseYoY6eybknnTf9WzeQIKZv7aNTm2KQsiFVJH21bfP2q7XVjfoMibdCg95GmgeFlaygMdeoDDa+ZbEQ==", + "dev": true, + "requires": { + "q": "^1.5.1" + } + }, + "conventional-changelog-jshint": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/conventional-changelog-jshint/-/conventional-changelog-jshint-2.0.1.tgz", + "integrity": "sha512-kRFJsCOZzPFm2tzRHULWP4tauGMvccOlXYf3zGeuSW4U0mZhk5NsjnRZ7xFWrTFPlCLV+PNmHMuXp5atdoZmEg==", + "dev": true, + "requires": { + "compare-func": "^1.3.1", + "q": "^1.5.1" + } + }, + "conventional-changelog-preset-loader": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/conventional-changelog-preset-loader/-/conventional-changelog-preset-loader-2.0.2.tgz", + "integrity": "sha512-pBY+qnUoJPXAXXqVGwQaVmcye05xi6z231QM98wHWamGAmu/ghkBprQAwmF5bdmyobdVxiLhPY3PrCfSeUNzRQ==", + "dev": true + }, + "conventional-changelog-writer": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/conventional-changelog-writer/-/conventional-changelog-writer-4.0.2.tgz", + "integrity": "sha512-d8/FQY/fix2xXEBUhOo8u3DCbyEw3UOQgYHxLsPDw+wHUDma/GQGAGsGtoH876WyNs32fViHmTOUrgRKVLvBug==", + "dev": true, + "requires": { + "compare-func": "^1.3.1", + "conventional-commits-filter": "^2.0.1", + "dateformat": "^3.0.0", + "handlebars": "^4.0.2", + "json-stringify-safe": "^5.0.1", + "lodash": "^4.2.1", + "meow": "^4.0.0", + "semver": "^5.5.0", + "split": "^1.0.0", + "through2": "^2.0.0" + } + }, + "conventional-commits-filter": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/conventional-commits-filter/-/conventional-commits-filter-2.0.1.tgz", + "integrity": "sha512-92OU8pz/977udhBjgPEbg3sbYzIxMDFTlQT97w7KdhR9igNqdJvy8smmedAAgn4tPiqseFloKkrVfbXCVd+E7A==", + "dev": true, + "requires": { + "is-subset": "^0.1.1", + "modify-values": "^1.0.0" + } + }, + "conventional-commits-parser": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-3.0.1.tgz", + "integrity": "sha512-P6U5UOvDeidUJ8ebHVDIoXzI7gMlQ1OF/id6oUvp8cnZvOXMt1n8nYl74Ey9YMn0uVQtxmCtjPQawpsssBWtGg==", + "dev": true, + "requires": { + "JSONStream": "^1.0.4", + "is-text-path": "^1.0.0", + "lodash": "^4.2.1", + "meow": "^4.0.0", + "split2": "^2.0.0", + "through2": "^2.0.0", + "trim-off-newlines": "^1.0.0" + } + }, + "cookie": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", + "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "requires": { + "object-assign": "^4", + "vary": "^1" + } + }, + "crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" + }, + "currently-unhandled": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz", + "integrity": "sha1-mI3zP+qxke95mmE2nddsF635V+o=", + "dev": true, + "requires": { + "array-find-index": "^1.0.1" + } + }, + "dargs": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/dargs/-/dargs-4.1.0.tgz", + "integrity": "sha1-A6nbtLXC8Tm/FK5T8LiipqhvThc=", + "dev": true, + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "dateformat": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz", + "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==", + "dev": true + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true + }, + "decamelize-keys": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/decamelize-keys/-/decamelize-keys-1.1.0.tgz", + "integrity": "sha1-0XGoeTMlKAfrPLYdwcFEXQeN8tk=", + "dev": true, + "requires": { + "decamelize": "^1.1.0", + "map-obj": "^1.0.0" + }, + "dependencies": { + "map-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", + "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=", + "dev": true + } + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "diff": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==" + }, + "dot-prop": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-3.0.0.tgz", + "integrity": "sha1-G3CK8JSknJoOfbyteQq6U52sEXc=", + "dev": true, + "requires": { + "is-obj": "^1.0.0" + } + }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "elasticsearch": { + "version": "15.2.0", + "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-15.2.0.tgz", + "integrity": "sha512-jOFcBoEh3Sn3gjUTozInODZTLriJtfppAUC7jnQCUE+OUj8o7GoAyC+L4h/L3ZxmXNFbQCunqVR+nmSofHdo9A==", + "requires": { + "agentkeepalive": "^3.4.1", + "chalk": "^1.0.0", + "lodash": "^4.17.10" + }, + "dependencies": { + "chalk": { + "version": "1.1.3", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + } + } + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, + "error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "requires": { + "is-arrayish": "^0.2.1" + } + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + }, + "express": { + "version": "4.16.4", + "resolved": "https://registry.npmjs.org/express/-/express-4.16.4.tgz", + "integrity": "sha512-j12Uuyb4FMrd/qQAm6uCHAkPtO8FDTRJZBDd5D2KOL2eLaz1yUNdUB/NOIyq0iU4q4cFarsUCrnFDPBcnksuOg==", + "requires": { + "accepts": "~1.3.5", + "array-flatten": "1.1.1", + "body-parser": "1.18.3", + "content-disposition": "0.5.2", + "content-type": "~1.0.4", + "cookie": "0.3.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.1.1", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.2", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.4", + "qs": "6.5.2", + "range-parser": "~1.2.0", + "safe-buffer": "5.1.2", + "send": "0.16.2", + "serve-static": "1.13.2", + "setprototypeof": "1.1.0", + "statuses": "~1.4.0", + "type-is": "~1.6.16", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "dependencies": { + "statuses": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + } + } + }, + "express-promise-router": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/express-promise-router/-/express-promise-router-3.0.3.tgz", + "integrity": "sha1-Xm0ipaPwE9cYMxcv6NereAw/a3A=", + "requires": { + "is-promise": "^2.1.0", + "lodash.flattendeep": "^4.0.0", + "methods": "^1.0.0" + } + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + }, + "finalhandler": { + "version": "1.1.1", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.2", + "statuses": "~1.4.0", + "unpipe": "~1.0.0" + }, + "dependencies": { + "statuses": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + } + } + }, + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "requires": { + "locate-path": "^2.0.0" + } + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "forwarded": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + }, + "fs-extra": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", + "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, + "get-pkg-repo": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/get-pkg-repo/-/get-pkg-repo-1.4.0.tgz", + "integrity": "sha1-xztInAbYDMVTbCyFP54FIyBWly0=", + "dev": true, + "requires": { + "hosted-git-info": "^2.1.4", + "meow": "^3.3.0", + "normalize-package-data": "^2.3.0", + "parse-github-repo-url": "^1.3.0", + "through2": "^2.0.0" + }, + "dependencies": { + "camelcase": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", + "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=", + "dev": true + }, + "camelcase-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", + "dev": true, + "requires": { + "camelcase": "^2.0.0", + "map-obj": "^1.0.0" + } + }, + "find-up": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", + "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=", + "dev": true, + "requires": { + "path-exists": "^2.0.0", + "pinkie-promise": "^2.0.0" + } + }, + "indent-string": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz", + "integrity": "sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=", + "dev": true, + "requires": { + "repeating": "^2.0.0" + } + }, + "load-json-file": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0", + "strip-bom": "^2.0.0" + } + }, + "map-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", + "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=", + "dev": true + }, + "meow": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", + "dev": true, + "requires": { + "camelcase-keys": "^2.0.0", + "decamelize": "^1.1.2", + "loud-rejection": "^1.0.0", + "map-obj": "^1.0.1", + "minimist": "^1.1.3", + "normalize-package-data": "^2.3.4", + "object-assign": "^4.0.1", + "read-pkg-up": "^1.0.1", + "redent": "^1.0.0", + "trim-newlines": "^1.0.0" + } + }, + "parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "dev": true, + "requires": { + "error-ex": "^1.2.0" + } + }, + "path-exists": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", + "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=", + "dev": true, + "requires": { + "pinkie-promise": "^2.0.0" + } + }, + "path-type": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", + "integrity": "sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + } + }, + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true + }, + "read-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", + "integrity": "sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=", + "dev": true, + "requires": { + "load-json-file": "^1.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^1.0.0" + } + }, + "read-pkg-up": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz", + "integrity": "sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI=", + "dev": true, + "requires": { + "find-up": "^1.0.0", + "read-pkg": "^1.0.0" + } + }, + "redent": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz", + "integrity": "sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94=", + "dev": true, + "requires": { + "indent-string": "^2.1.0", + "strip-indent": "^1.0.1" + } + }, + "strip-bom": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=", + "dev": true, + "requires": { + "is-utf8": "^0.2.0" + } + }, + "strip-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz", + "integrity": "sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI=", + "dev": true, + "requires": { + "get-stdin": "^4.0.1" + } + }, + "trim-newlines": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz", + "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM=", + "dev": true + } + } + }, + "get-stdin": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", + "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=", + "dev": true + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "git-raw-commits": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/git-raw-commits/-/git-raw-commits-2.0.0.tgz", + "integrity": "sha512-w4jFEJFgKXMQJ0H0ikBk2S+4KP2VEjhCvLCNqbNRQC8BgGWgLKNCO7a9K9LI+TVT7Gfoloje502sEnctibffgg==", + "dev": true, + "requires": { + "dargs": "^4.0.1", + "lodash.template": "^4.0.2", + "meow": "^4.0.0", + "split2": "^2.0.0", + "through2": "^2.0.0" + } + }, + "git-remote-origin-url": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/git-remote-origin-url/-/git-remote-origin-url-2.0.0.tgz", + "integrity": "sha1-UoJlna4hBxRaERJhEq0yFuxfpl8=", + "dev": true, + "requires": { + "gitconfiglocal": "^1.0.0", + "pify": "^2.3.0" + }, + "dependencies": { + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true + } + } + }, + "git-semver-tags": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/git-semver-tags/-/git-semver-tags-2.0.2.tgz", + "integrity": "sha512-34lMF7Yo1xEmsK2EkbArdoU79umpvm0MfzaDkSNYSJqtM5QLAVTPWgpiXSVI5o/O9EvZPSrP4Zvnec/CqhSd5w==", + "dev": true, + "requires": { + "meow": "^4.0.0", + "semver": "^5.5.0" + } + }, + "gitconfiglocal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz", + "integrity": "sha1-QdBF84UaXqiPA/JMocYXgRRGS5s=", + "dev": true, + "requires": { + "ini": "^1.3.2" + } + }, + "graceful-fs": { + "version": "4.1.15", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==" + }, + "handlebars": { + "version": "4.0.12", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.12.tgz", + "integrity": "sha512-RhmTekP+FZL+XNhwS1Wf+bTTZpdLougwt5pcgA1tuz6Jcx0fpH/7z0qd71RKnZHBCxIRBHfBOnio4gViPemNzA==", + "dev": true, + "requires": { + "async": "^2.5.0", + "optimist": "^0.6.1", + "source-map": "^0.6.1", + "uglify-js": "^3.1.4" + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "hosted-git-info": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz", + "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==", + "dev": true + }, + "http-errors": { + "version": "1.6.3", + "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + } + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=", + "requires": { + "ms": "^2.0.0" + } + }, + "iconv-lite": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", + "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "indent-string": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", + "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=", + "dev": true + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "dev": true + }, + "ipaddr.js": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz", + "integrity": "sha1-6qM9bd16zo9/b+DJygRA5wZzix4=" + }, + "is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", + "dev": true + }, + "is-builtin-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", + "dev": true, + "requires": { + "builtin-modules": "^1.0.0" + } + }, + "is-finite": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.0.2.tgz", + "integrity": "sha1-zGZ3aVYCvlUO8R6LSqYwU0K20Ko=", + "dev": true, + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "is-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=", + "dev": true + }, + "is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", + "dev": true + }, + "is-promise": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", + "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=" + }, + "is-subset": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-subset/-/is-subset-0.1.1.tgz", + "integrity": "sha1-ilkRfZMt4d4A8kX83TnOQ/HpOaY=", + "dev": true + }, + "is-text-path": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-text-path/-/is-text-path-1.0.1.tgz", + "integrity": "sha1-Thqg+1G/vLPpJogAE5cgLBd1tm4=", + "dev": true, + "requires": { + "text-extensions": "^1.0.0" + } + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "is-utf8": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=", + "dev": true + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "requires": { + "graceful-fs": "^4.1.6" + } + }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", + "dev": true + }, + "jsonschema": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/jsonschema/-/jsonschema-1.2.4.tgz", + "integrity": "sha512-lz1nOH69GbsVHeVgEdvyavc/33oymY1AZwtePMiMj4HZPMbP5OIKK3zT9INMWjwua/V4Z4yq7wSlBbSG+g4AEw==" + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + } + }, + "locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "requires": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + } + }, + "lodash": { + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + }, + "lodash._reinterpolate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz", + "integrity": "sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0=", + "dev": true + }, + "lodash.flattendeep": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", + "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=" + }, + "lodash.template": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.template/-/lodash.template-4.4.0.tgz", + "integrity": "sha1-5zoDhcg1VZF0bgILmWecaQ5o+6A=", + "dev": true, + "requires": { + "lodash._reinterpolate": "~3.0.0", + "lodash.templatesettings": "^4.0.0" + } + }, + "lodash.templatesettings": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/lodash.templatesettings/-/lodash.templatesettings-4.1.0.tgz", + "integrity": "sha1-K01OlbpEDZFf8IvImeRVNmZxMxY=", + "dev": true, + "requires": { + "lodash._reinterpolate": "~3.0.0" + } + }, + "loud-rejection": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz", + "integrity": "sha1-W0b4AUft7leIcPCG0Eghz5mOVR8=", + "dev": true, + "requires": { + "currently-unhandled": "^0.4.1", + "signal-exit": "^3.0.0" + } + }, + "make-error": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz", + "integrity": "sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==" + }, + "map-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-2.0.0.tgz", + "integrity": "sha1-plzSkIepJZi4eRJXpSPgISIqwfk=", + "dev": true + }, + "media-typer": { + "version": "0.3.0", + "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, + "meow": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/meow/-/meow-4.0.1.tgz", + "integrity": "sha512-xcSBHD5Z86zaOc+781KrupuHAzeGXSLtiAOmBsiLDiPSaYSB6hdew2ng9EBAnZ62jagG9MHAOdxpDi/lWBFJ/A==", + "dev": true, + "requires": { + "camelcase-keys": "^4.0.0", + "decamelize-keys": "^1.0.0", + "loud-rejection": "^1.0.0", + "minimist": "^1.1.3", + "minimist-options": "^3.0.1", + "normalize-package-data": "^2.3.4", + "read-pkg-up": "^3.0.0", + "redent": "^2.0.0", + "trim-newlines": "^2.0.0" + } + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, + "mime": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", + "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==" + }, + "mime-db": { + "version": "1.37.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", + "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==" + }, + "mime-types": { + "version": "2.1.21", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", + "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", + "requires": { + "mime-db": "~1.37.0" + } + }, + "minimist": { + "version": "1.2.0", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + }, + "minimist-options": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-3.0.2.tgz", + "integrity": "sha512-FyBrT/d0d4+uiZRbqznPXqw3IpZZG3gl3wKWiX784FycUKVwBt0uLBFkQrtE4tZOrgo78nZp2jnKz3L65T5LdQ==", + "dev": true, + "requires": { + "arrify": "^1.0.1", + "is-plain-obj": "^1.1.0" + } + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "requires": { + "minimist": "0.0.8" + }, + "dependencies": { + "minimist": { + "version": "0.0.8", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + } + } + }, + "modify-values": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/modify-values/-/modify-values-1.0.1.tgz", + "integrity": "sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw==", + "dev": true + }, + "morgan": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.9.1.tgz", + "integrity": "sha512-HQStPIV4y3afTiCYVxirakhlCfGkI161c76kKFca7Fk1JusM//Qeo1ej2XaMniiNeaZklMVrh3vTtIzpzwbpmA==", + "requires": { + "basic-auth": "~2.0.0", + "debug": "2.6.9", + "depd": "~1.1.2", + "on-finished": "~2.3.0", + "on-headers": "~1.0.1" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "negotiator": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", + "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" + }, + "node-cache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-4.2.0.tgz", + "integrity": "sha512-obRu6/f7S024ysheAjoYFEEBqqDWv4LOMNJEuO8vMeEw2AT4z+NCzO4hlc2lhI4vATzbCQv6kke9FVdx0RbCOw==", + "requires": { + "clone": "2.x", + "lodash": "4.x" + } + }, + "node-cron": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-2.0.3.tgz", + "integrity": "sha512-eJI+QitXlwcgiZwNNSRbqsjeZMp5shyajMR81RZCqeW0ZDEj4zU9tpd4nTh/1JsBiKbF8d08FCewiipDmVIYjg==", + "requires": { + "opencollective-postinstall": "^2.0.0", + "tz-offset": "0.0.1" + } + }, + "nodemailer": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-4.7.0.tgz", + "integrity": "sha512-IludxDypFpYw4xpzKdMAozBSkzKHmNBvGanUREjJItgJ2NYcK/s8+PggVhj7c2yGFQykKsnnmv1+Aqo0ZfjHmw==" + }, + "normalize-package-data": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz", + "integrity": "sha512-9jjUFbTPfEy3R/ad/2oNbKtW9Hgovl5O1FvFWKkKblNXoN/Oou6+9+KKohPK13Yc3/TyunyWhJp6gvRNR/PPAw==", + "dev": true, + "requires": { + "hosted-git-info": "^2.1.4", + "is-builtin-module": "^1.0.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", + "dev": true + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, + "on-headers": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz", + "integrity": "sha1-ko9dD0cNSTQmUepnlLCFfBAGk/c=" + }, + "opencollective-postinstall": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.1.tgz", + "integrity": "sha512-saQQ9hjLwu/oS0492eyYotoh+bra1819cfAT5rjY/e4REWwuc8IgZ844Oo44SiftWcJuBiqp0SA0BFVbmLX0IQ==" + }, + "optimist": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "dev": true, + "requires": { + "minimist": "~0.0.1", + "wordwrap": "~0.0.2" + }, + "dependencies": { + "minimist": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=", + "dev": true + } + } + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "dev": true + }, + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "requires": { + "p-try": "^1.0.0" + } + }, + "p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "requires": { + "p-limit": "^1.1.0" + } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true + }, + "parse-github-repo-url": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/parse-github-repo-url/-/parse-github-repo-url-1.4.1.tgz", + "integrity": "sha1-nn2LslKmy2ukJZUGC3v23z28H1A=", + "dev": true + }, + "parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", + "dev": true, + "requires": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + } + }, + "parseurl": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", + "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + }, + "pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=", + "dev": true + }, + "pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "dev": true, + "requires": { + "pinkie": "^2.0.0" + } + }, + "pluralize": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-7.0.0.tgz", + "integrity": "sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==" + }, + "prepend-file": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/prepend-file/-/prepend-file-1.3.1.tgz", + "integrity": "sha1-g7FuC0rBkB/OiNvZRaIvTMgd9Xk=", + "dev": true, + "requires": { + "tmp": "0.0.31" + } + }, + "prepend-file-cli": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/prepend-file-cli/-/prepend-file-cli-1.0.6.tgz", + "integrity": "sha1-/34RbJMU24XCLEOioH8/k4epp08=", + "dev": true, + "requires": { + "minimist": "^1.2.0", + "prepend-file": "1.3.1" + } + }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", + "dev": true + }, + "promise-queue": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/promise-queue/-/promise-queue-2.2.5.tgz", + "integrity": "sha1-L29ffA9tCBCelnZZx5uIqe1ek7Q=" + }, + "proxy-addr": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", + "integrity": "sha512-5erio2h9jp5CHGwcybmxmVqHmnCBZeewlfJ0pex+UW7Qny7OOZXTtH56TGNyBizkgiOwhJtMKrVzDTeKcySZwA==", + "requires": { + "forwarded": "~0.1.2", + "ipaddr.js": "1.8.0" + } + }, + "psl": { + "version": "1.1.29", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz", + "integrity": "sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ==" + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, + "q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=", + "dev": true + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "quick-lru": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-1.1.0.tgz", + "integrity": "sha1-Q2CxfGETatOAeDl/8RQW4Ybc+7g=", + "dev": true + }, + "range-parser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", + "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" + }, + "raw-body": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", + "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==", + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.3", + "iconv-lite": "0.4.23", + "unpipe": "1.0.0" + } + }, + "read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", + "dev": true, + "requires": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + } + }, + "read-pkg-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz", + "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=", + "dev": true, + "requires": { + "find-up": "^2.0.0", + "read-pkg": "^3.0.0" + } + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "redent": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-2.0.0.tgz", + "integrity": "sha1-wbIAe0LVfrE4kHmzyDM2OdXhzKo=", + "dev": true, + "requires": { + "indent-string": "^3.0.0", + "strip-indent": "^2.0.0" + } + }, + "repeating": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/repeating/-/repeating-2.0.1.tgz", + "integrity": "sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo=", + "dev": true, + "requires": { + "is-finite": "^1.0.0" + } + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==" + }, + "send": { + "version": "0.16.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", + "integrity": "sha512-E64YFPUssFHEFBvpbbjr44NCLtI1AohxQ8ZSiJjQLskAdKuriYEP6VyGEsRDH8ScozGpkaX1BGvhanqCwkcEZw==", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.6.2", + "mime": "1.4.1", + "ms": "2.0.0", + "on-finished": "~2.3.0", + "range-parser": "~1.2.0", + "statuses": "~1.4.0" + }, + "dependencies": { + "statuses": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + } + } + }, + "serve-static": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz", + "integrity": "sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.2", + "send": "0.16.2" + } + }, + "setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + }, + "sha1": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/sha1/-/sha1-1.1.1.tgz", + "integrity": "sha1-rdqnqTFo85PxnrKxUJFhjicA+Eg=", + "requires": { + "charenc": ">= 0.0.1", + "crypt": ">= 0.0.1" + } + }, + "signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "dev": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + }, + "source-map-support": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.9.tgz", + "integrity": "sha512-gR6Rw4MvUlYy83vP0vxoVNzM6t8MUXqNuRsuBmBHQDu1Fh6X015FrLdgoDKcNdkwGubozq0P4N0Q37UyFVr1EA==", + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "spdx-correct": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", + "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==", + "dev": true, + "requires": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-exceptions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz", + "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==", + "dev": true + }, + "spdx-expression-parse": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", + "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "dev": true, + "requires": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-license-ids": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.3.tgz", + "integrity": "sha512-uBIcIl3Ih6Phe3XHK1NqboJLdGfwr1UN3k6wSD1dZpmPsIkb8AGNbZYJ1fOBk834+Gxy8rpfDxrS6XLEMZMY2g==", + "dev": true + }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "dev": true, + "requires": { + "through": "2" + } + }, + "split2": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-2.2.0.tgz", + "integrity": "sha512-RAb22TG39LhI31MbreBgIuKiIKhVsawfTgEGqKHTK87aG+ul/PB8Sqoi3I7kVdRWiCfrKxK3uo4/YUkpNvhPbw==", + "dev": true, + "requires": { + "through2": "^2.0.2" + } + }, + "sshpk": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.15.2.tgz", + "integrity": "sha512-Ra/OXQtuh0/enyl4ETZAfTaeksa6BXks5ZcjpSUNrjBr0DvrJKX+1fsKDPpT9TBXgHAFsa4510aNVgI8g/+SzA==", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "dev": true + }, + "strip-indent": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-2.0.0.tgz", + "integrity": "sha1-XvjbKV0B5u1sv3qrlpmNeCJSe2g=", + "dev": true + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" + }, + "tempfile": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tempfile/-/tempfile-1.1.1.tgz", + "integrity": "sha1-W8xOrsxKsscH2LwR2ZzMmiyyh/I=", + "dev": true, + "requires": { + "os-tmpdir": "^1.0.0", + "uuid": "^2.0.1" + }, + "dependencies": { + "uuid": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=", + "dev": true + } + } + }, + "text-extensions": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-1.9.0.tgz", + "integrity": "sha512-wiBrwC1EhBelW12Zy26JeOUkQ5mRu+5o8rpsJk5+2t+Y5vE7e842qtZDQ2g1NpX/29HdyFeJ4nSIhI47ENSxlQ==", + "dev": true + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", + "dev": true + }, + "through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "dev": true, + "requires": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "tmp": { + "version": "0.0.31", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.31.tgz", + "integrity": "sha1-jzirlDjhcxXl29izZX6L+yd65Kc=", + "dev": true, + "requires": { + "os-tmpdir": "~1.0.1" + } + }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + } + } + }, + "trim-newlines": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-2.0.0.tgz", + "integrity": "sha1-tAPQuRvlDDMd/EuC7s6yLD3hbSA=", + "dev": true + }, + "trim-off-newlines": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz", + "integrity": "sha1-n5up2e+odkw4dpi8v+sshI8RrbM=", + "dev": true + }, + "ts-node": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-7.0.1.tgz", + "integrity": "sha512-BVwVbPJRspzNh2yfslyT1PSbl5uIk03EZlb493RKHN4qej/D06n1cEhjlOJG69oFsE7OT8XjpTUcYf6pKTLMhw==", + "requires": { + "arrify": "^1.0.0", + "buffer-from": "^1.1.0", + "diff": "^3.1.0", + "make-error": "^1.1.1", + "minimist": "^1.2.0", + "mkdirp": "^0.5.1", + "source-map-support": "^0.5.6", + "yn": "^2.0.0" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, + "type-is": { + "version": "1.6.16", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", + "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.18" + } + }, + "tz-offset": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/tz-offset/-/tz-offset-0.0.1.tgz", + "integrity": "sha512-kMBmblijHJXyOpKzgDhKx9INYU4u4E1RPMB0HqmKSgWG8vEcf3exEfLh4FFfzd3xdQOw9EuIy/cP0akY6rHopQ==" + }, + "uglify-js": { + "version": "3.4.9", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.4.9.tgz", + "integrity": "sha512-8CJsbKOtEbnJsTyv6LE6m6ZKniqMiFWmm9sRbopbkGs3gMPPfd3Fh8iIA4Ykv5MgaTbqHr4BaoGLJLZNhsrW1Q==", + "dev": true, + "optional": true, + "requires": { + "commander": "~2.17.1", + "source-map": "~0.6.1" + }, + "dependencies": { + "commander": { + "version": "2.17.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz", + "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==", + "dev": true, + "optional": true + } + } + }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "requires": { + "punycode": "^2.1.0" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "dev": true + }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dev": true, + "requires": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", + "dev": true + }, + "xtend": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", + "dev": true + }, + "yn": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yn/-/yn-2.0.0.tgz", + "integrity": "sha1-5a2ryKz0CPY4X8dklWhMiOavaJo=" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..e93756dd --- /dev/null +++ b/package.json @@ -0,0 +1,65 @@ +{ + "name": "@openstapps/backend", + "version": "0.0.1", + "description": "A reference implementation for a StApps backend", + "license": "AGPL-3.0-only", + "author": "André Bierlein ", + "contributors": [ + "Anselm Stordeur ", + "Benjamin Joeckel", + "Jovan Krunic ", + "Karl-Philipp Wulfert " + ], + "scripts": { + "build": "npm run tslint && npm run compile", + "compile": "rimraf lib && tsc --outDir lib && prepend lib/cli.js '#!/usr/bin/env node\n'", + "changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 0", + "start": "NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node ./lib/cli.js", + "tslint": "tslint 'src/**/*.ts'" + }, + "dependencies": { + "@openstapps/core": "0.1.0", + "@openstapps/core-validator": "0.0.1", + "@openstapps/logger": "0.0.3", + "body-parser": "1.18.3", + "config": "3.0.1", + "cors": "2.8.5", + "elasticsearch": "15.2.0", + "express": "4.16.4", + "express-promise-router": "3.0.3", + "fs-extra": "7.0.1", + "moment": "2.23.0", + "morgan": "1.9.1", + "node-cache": "4.2.0", + "node-cron": "2.0.3", + "pluralize": "7.0.0", + "promise-queue": "2.2.5", + "request": "2.88.0", + "semver": "5.6.0", + "sha1": "1.1.1", + "ts-node": "7.0.1", + "uuid": "3.3.2" + }, + "devDependencies": { + "@openstapps/configuration": "0.5.0", + "@types/body-parser": "1.17.0", + "@types/config": "0.0.34", + "@types/cors": "2.8.4", + "@types/elasticsearch": "5.0.30", + "@types/express": "4.16.0", + "@types/fs-extra": "5.0.4", + "@types/morgan": "1.7.35", + "@types/node": "10.12.18", + "@types/node-cache": "4.1.1", + "@types/node-cron": "2.0.0", + "@types/nodemailer": "4.6.5", + "@types/promise-queue": "2.2.0", + "@types/sha1": "1.1.1", + "@types/uuid": "3.4.4", + "conventional-changelog-cli": "2.0.11", + "get-port": "4.1.0", + "prepend-file-cli": "1.0.6", + "rimraf": "2.6.3", + "typescript": "3.2.2" + } +} diff --git a/src/app.ts b/src/app.ts new file mode 100644 index 00000000..ac404908 --- /dev/null +++ b/src/app.ts @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { SCNotFoundErrorResponse, SCUnsupportedMediaTypeErrorResponse } from '@openstapps/core'; +import { SCValidator } from '@openstapps/core-validator'; +import * as bodyParser from 'body-parser'; +import * as config from 'config'; +import * as cors from 'cors'; +import * as express from 'express'; +import * as morgan from 'morgan'; +import { logger, mailer } from './common'; +import { MailQueue } from './notification/MailQueue'; +import { bulkAddRouter } from './routes/BulkAddRoute'; +import { bulkDoneRouter } from './routes/BulkDoneRoute'; +import { bulkRouter } from './routes/BulkRoute'; +import { indexRouter } from './routes/IndexRoute'; +import { multiSearchRouter } from './routes/MultiSearchRoute'; +import { searchRouter } from './routes/SearchRoute'; +import { thingUpdateRouter } from './routes/ThingUpdateRoute'; +import { BulkStorage } from './storage/BulkStorage'; +import { DatabaseConstructor } from './storage/Database'; +import { Elasticsearch } from './storage/elasticsearch/Elasticsearch'; + +export const app = express(); + +// only accept json as content type for all requests +app.use(bodyParser.json({ + limit: '500kb', + type: (req) => { + const contentType = typeof req.headers['Content-Type'] === 'string' ? + req.headers['Content-Type'] : req.headers['content-type']; + if (typeof contentType === 'string' && contentType.match(/^application\/json/)) { + return true; + } else { + throw new SCUnsupportedMediaTypeErrorResponse(process.env.NODE_ENV !== 'production'); + } + }, +})); // 500kb should be reasonably large + +// use morgan as a request logger +// request loggers have to be the first middleware to be set in express +app.use(morgan('dev')); + +const databases: {[name: string]: DatabaseConstructor} = { + elasticsearch: Elasticsearch, +}; + +// validate config file +export const scValidator = new SCValidator('./node_modules/@openstapps/core/lib/schema/'); +scValidator.feedValidator(); + +// validate the config file +const configValidation = scValidator.validate(config.util.toObject(), 'ConfigFile'); + +// validation failed +if (configValidation.errors.length > 0) { + throw new Error( + 'Validation of config file failed. Errors were: ' + + JSON.stringify(configValidation.errors), + ); +} + +// check if a database name was given +if (!config.has('internal.database.name')) { + throw new Error('You have to configure a database'); +} + +if (typeof mailer !== 'undefined') { + // set a mailQueue to use the backend mailer + if (config.has('internal.monitoring')) { + app.set('mailQueue', new MailQueue(mailer)); + } +} + +const database = + new databases[config.get('internal.database.name')]( + config.util.toObject(), + app.get('mailQueue'), + ); + +if (typeof database === 'undefined') { + throw new Error('No implementation for configured database found. Please check your configuration.'); +} + +logger.ok('Validated config file sucessfully'); + +// make the validator available on the app +app.set('validator', scValidator); + +// treats /foo and /foo/ as two different routes +// see http://expressjs.com/en/api.html#app.set +app.set('strict routing', true); + +// make the bulk storage available to all http middlewares/routes +app.set( + 'bulk', + new BulkStorage(database), +); + +const corsOptions = { + allowedHeaders: [ + 'DNT', + 'Keep-Alive', + 'User-Agent', + 'X-Requested-With', + 'If-Modified-Since', + 'Cache-Control', + 'Content-Type', + 'X-StApps-Version', + ], + credentials: true, + maxAge: 1728000, + methods: ['GET', 'POST', 'PUT', 'OPTIONS'], + optionsSuccessStatus: 204, +}; + +// allow all origins on all routes +app.use(cors(corsOptions)); +// TODO: See if it can handle options request with no content-type + +// allow cors preflight requests on every route +app.options('*', cors(corsOptions)); + +app.set('isProductiveEnvironment', process.env.NODE_ENV !== 'production'); + +// load routes before plugins +// they now can be used or overwritten by any plugin +app.use( + bulkAddRouter, + bulkDoneRouter, + bulkRouter, + indexRouter, + multiSearchRouter, + searchRouter, + thingUpdateRouter, +); + +// add a route for a missing resource (404) +app.use((_req, res) => { + const errorResponse = new SCNotFoundErrorResponse(process.env.NODE_ENV !== 'production'); + res.status(errorResponse.statusCode); + res.json(errorResponse); +}); + +// TODO: implement a route to register plugins diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 00000000..a0667f6a --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import * as http from 'http'; +import { app } from './app'; +import { logger } from './common'; + +/** + * Get port from environment and store in Express. + */ +const port = normalizePort(process.env.PORT || '3000'); +app.set('port', port); + +/** + * Create HTTP server. + */ +const server = http.createServer(app); + +/** + * Listen on provided port, on all network interfaces. + */ +server.listen(port); +server.on('error', onError); +server.on('listening', onListening); + +/** + * Normalize a port into a number, string, or false. + */ +function normalizePort(value: string) { + const portNumber = parseInt(value, 10); + + if (isNaN(portNumber)) { + // named pipe + return value; + } + + if (portNumber >= 0) { + // port number + return portNumber; + } + + return false; +} + +/** + * Event listener for HTTP server "error" event. + */ +function onError(error: Error | any) { + if (error.syscall !== 'listen') { + throw error; + } + + const bind = typeof port === 'string' + ? 'Pipe ' + port + : 'Port ' + port; + + // handle specific listen errors with friendly messages + switch (error.code) { + case 'EACCES': + logger.error(bind + ' requires elevated privileges'); + process.exit(1); + break; + case 'EADDRINUSE': + logger.error(bind + ' is already in use'); + process.exit(1); + break; + default: + throw error; + } +} + +/** + * Event listener for HTTP server "listening" event. + */ +function onListening() { + const addr = server.address(); + const bind = typeof addr === 'string' + ? 'pipe ' + addr + : 'port ' + addr.port; + logger.ok('Listening on ' + bind); +} diff --git a/src/common.ts b/src/common.ts new file mode 100644 index 00000000..f5247cea --- /dev/null +++ b/src/common.ts @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { Logger } from '@openstapps/logger'; +import { BackendTransport } from './notification/BackendTransport'; + +export const mailer = BackendTransport.getTransportInstance(); + +export const logger = new Logger(mailer); diff --git a/src/notification/BackendTransport.ts b/src/notification/BackendTransport.ts new file mode 100644 index 00000000..07f19c3e --- /dev/null +++ b/src/notification/BackendTransport.ts @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import {SMTP} from '@openstapps/logger/lib/SMTP'; +import {Transport, TransportWithVerification} from '@openstapps/logger/lib/Transport'; + +export function isTransportWithVerification(instance: Transport): instance is TransportWithVerification { + return typeof (instance as TransportWithVerification).verify === 'function'; +} + +/** + * Singleton for getting only one transport service + * + * In the future this may support more than loading SMTP as a transport. + */ +export class BackendTransport { + + private static _instance: BackendTransport; + private waitingForVerification: boolean; + protected transport: SMTP | undefined; + + public static getTransportInstance(): SMTP | undefined { + if (this._instance) { + return this._instance.transport; + } + + this._instance = new this(); + return this._instance.transport; + } + + private constructor() { + // get SMTP instance for the time + // in the future we may implement some other transport services which can be selected + // via the configuration files + try { + this.transport = SMTP.getInstance(); + } catch (error) { + if (process.env.ALLOW_NO_TRANSPORT === 'true') { + /* tslint:disable-next-line:no-console */ + console.warn('SMTP error was ignored.'); + } else { + throw error; + } + } + + if (typeof this.transport !== 'undefined' && isTransportWithVerification(this.transport)) { + this.waitingForVerification = true; + + this.transport.verify().then((message) => { + if (typeof message === 'string') { + // tslint:disable-next-line:no-console + console.log(message); + } + }).catch((err) => { + throw err; + }); + } else { + this.waitingForVerification = false; + } + } + + public isWaitingForVerification(): boolean { + return this.waitingForVerification; + } +} diff --git a/src/notification/MailQueue.ts b/src/notification/MailQueue.ts new file mode 100644 index 00000000..a739b2ed --- /dev/null +++ b/src/notification/MailQueue.ts @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see .nse along with + * this program. If not, see . + */ +import { SMTP } from '@openstapps/logger/lib/SMTP'; +import { MailOptions } from 'nodemailer/lib/sendmail-transport'; +import * as Queue from 'promise-queue'; +import { logger } from '../common'; +/** + * A queue that can send mails in serial + */ +export class MailQueue { + + /** + * A queue that saves mails, before the transport is ready. When + * the transport gets ready this mails are getting pushed in to + * the normal queue. + */ + dryQueue: MailOptions[]; + + /** + * A queue that saves mails, that are being sent in series + */ + queue: Queue; + + /** + * A counter for the number of verifications that failed + */ + verificationCounter: number; + + /** + * Creates a mail queue + * @param transport + */ + constructor(private transport: SMTP) { + + this.queue = new Queue(1); + + // this queue saves all request when the transport is not ready yet + this.dryQueue = []; + + this.verificationCounter = 0; + + // if the transport can be verified it should check if it was done... + this.checkForVerification(); + } + + /** + * Verify the given transport + */ + private checkForVerification() { + + if (this.verificationCounter > 5) { + throw new Error('Failed to initialize the SMTP transport for the mail queue'); + } + + if (!this.transport.isVerified()) { + this.verificationCounter++; + setTimeout(() => { + logger.warn('Transport not verified yet. Trying to send mails here...'); + this.checkForVerification(); + }, 5000); + } else { + logger.ok('Transport for mail queue was verified. We can send mails now'); + // if the transport finally was verified send all our mails from the dry queue + this.dryQueue.forEach((mail) => { + this.queue.add(() => (this.transport as SMTP).sendMail(mail)); + }); + } + } + + /** + * Push a mail into the queue so it gets send when the queue is ready + * @param mail + */ + public push(mail: MailOptions) { + if (!this.transport.isVerified()) { // the transport has verification, but is not verified yet + // push to a dry queue which gets pushed to the real queue when the transport is verified + this.dryQueue.push(mail); + } else { + this.queue.add(() => (this.transport as SMTP).sendMail(mail)); + } + } +} diff --git a/src/routes/BulkAddRoute.ts b/src/routes/BulkAddRoute.ts new file mode 100644 index 00000000..9782a717 --- /dev/null +++ b/src/routes/BulkAddRoute.ts @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { SCBulkAddRequest, SCBulkAddResponse, SCBulkAddRoute, SCNotFoundErrorResponse } from '@openstapps/core'; +import { logger } from '../common'; +import { BulkStorage } from '../storage/BulkStorage'; +import { createRoute } from './Route'; + +const bulkRouteModel = new SCBulkAddRoute(); + +/** + * Implementation of the bulk add route (SCBulkAddRoute) + */ +export const bulkAddRouter = createRoute( + bulkRouteModel, + async (request: SCBulkAddRequest, app, params) => { + + if (!params || typeof params.UID !== 'string') { + throw new Error('UID of Bulk was not given, but route with obligatory parameter was called'); + } + + const bulkMemory: BulkStorage = app.get('bulk'); + const bulk = await bulkMemory.read(params.UID); + + if (typeof bulk === 'undefined') { + logger.warn(`Bulk with ${params.UID} not found.`); + throw new SCNotFoundErrorResponse(app.get('isProductiveEnvironment')); + } + + await bulkMemory.database.post(request, bulk); + + return {}; + }, +); diff --git a/src/routes/BulkDoneRoute.ts b/src/routes/BulkDoneRoute.ts new file mode 100644 index 00000000..62fab8a6 --- /dev/null +++ b/src/routes/BulkDoneRoute.ts @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { SCBulkDoneRequest, SCBulkDoneResponse, SCBulkDoneRoute, SCNotFoundErrorResponse } from '@openstapps/core'; +import { logger } from '../common'; +import { BulkStorage } from '../storage/BulkStorage'; +import { createRoute } from './Route'; + +const bulkDoneRouteModel = new SCBulkDoneRoute(); + +/** + * Implementation of the bulk done request route (SCBulkDoneRoute) + */ +export const bulkDoneRouter = createRoute( + bulkDoneRouteModel, + async (_request: SCBulkDoneRequest, app, params) => { + + if (!params || typeof params.UID !== 'string') { + throw new Error('UID of Bulk was not given, but route with obligatory parameter was called'); + } + + const bulkMemory: BulkStorage = app.get('bulk'); + const bulk = await bulkMemory.read(params.UID); + + if (typeof bulk === 'undefined') { + logger.warn(`Bulk with ${params.UID} not found.`); + throw new SCNotFoundErrorResponse(app.get('isProductiveEnvironment')); + } + + bulk.state = 'done'; + await bulkMemory.markAsDone(bulk); + return {}; + }, +); diff --git a/src/routes/BulkRoute.ts b/src/routes/BulkRoute.ts new file mode 100644 index 00000000..ad41427b --- /dev/null +++ b/src/routes/BulkRoute.ts @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { SCBulkRequest, SCBulkResponse, SCBulkRoute } from '@openstapps/core'; +import { BulkStorage } from '../storage/BulkStorage'; +import { createRoute } from './Route'; + +const bulkRouteModel = new SCBulkRoute(); + +/** + * Implementation of the bulk request route (SCBulkRoute) + */ +export const bulkRouter = createRoute( + bulkRouteModel, + async (request: SCBulkRequest, app) => { + const bulkMemory: BulkStorage = app.get('bulk'); + return await bulkMemory.create(request); + }, +); diff --git a/src/routes/HTTPTypes.ts b/src/routes/HTTPTypes.ts new file mode 100644 index 00000000..c15baa9d --- /dev/null +++ b/src/routes/HTTPTypes.ts @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +export type HTTPVerb = 'all' | + 'get' | + 'post' | + 'put' | + 'delete' | + 'patch' | + 'options' | + 'head' | + 'checkout' | + 'copy' | + 'lock' | + 'merge' | + 'mkactivity' | + 'mkcol' | + 'move' | + 'm-search' | + 'notify' | + 'purge' | + 'report' | + 'search' | + 'subscribe' | + 'trace' | + 'unlock' | + 'unsubscribe'; + +export function isHttpMethod(method: string): method is HTTPVerb { + return ['get', 'post', 'put'].indexOf(method) > -1; +} diff --git a/src/routes/IndexRoute.ts b/src/routes/IndexRoute.ts new file mode 100644 index 00000000..277b34e5 --- /dev/null +++ b/src/routes/IndexRoute.ts @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { SCConfigFile, SCIndexResponse, SCIndexRoute } from '@openstapps/core'; +import * as config from 'config'; +import { createRoute } from './Route'; + +const indexRouteModel = new SCIndexRoute(); + +/** + * Implementation of the index route (SCIndexRoute) + */ +export const indexRouter = createRoute( + indexRouteModel, + async (_request: SCIndexRoute, _app) => { + const configObject: SCConfigFile = config.util.toObject(); + delete configObject.internal; + return configObject; + }, +); diff --git a/src/routes/MultiSearchRoute.ts b/src/routes/MultiSearchRoute.ts new file mode 100644 index 00000000..ece207b3 --- /dev/null +++ b/src/routes/MultiSearchRoute.ts @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { + SCMultiSearchRequest, + SCMultiSearchResponse, + SCMultiSearchRoute, + SCSearchResponse, + SCTooManyRequestsErrorResponse, +} from '@openstapps/core'; +import { BulkStorage } from '../storage/BulkStorage'; +import { createRoute } from './Route'; + +const multiSearchRouteModel = new SCMultiSearchRoute(); + +/** + * Implementation of the multi search route (SCMultiSearchRoute) + */ +export const multiSearchRouter = createRoute( + multiSearchRouteModel, + async (request: SCMultiSearchRequest, app) => { + + const bulkMemory: BulkStorage = app.get('bulk'); + const queryNames = Object.keys(request); + + if (queryNames.length > 5) { + return new SCTooManyRequestsErrorResponse(app.get('isProductiveEnvironment')); + } + + // get a map of promises for each query + const searchRequests = queryNames.map((queryName) => { + return bulkMemory.database.search(request[queryName]); + }); + + const listOfSearchResponses = await Promise.all(searchRequests); + + const response: { [queryName: string]: SCSearchResponse } = {}; + queryNames.forEach((queryName, index) => { + response[queryName] = listOfSearchResponses[index]; + }); + + return response; + }, +); diff --git a/src/routes/Route.ts b/src/routes/Route.ts new file mode 100644 index 00000000..11fbe235 --- /dev/null +++ b/src/routes/Route.ts @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { + SCInternalServerErrorResponse, + SCMethodNotAllowedErrorResponse, + SCRoute, + SCValidationErrorResponse, +} from '@openstapps/core'; +import { SCValidator } from '@openstapps/core-validator'; +import { Application, Router } from 'express'; +import PromiseRouter from 'express-promise-router'; +import { logger } from '../common'; +import { isHttpMethod } from './HTTPTypes'; + +/** + * Creates a router from a route class (model of a route) and a handler function which implements the logic + * + * The given router performs a request and respone validation, sets status codes and checks if the given handler + * only returns errors that are allowed for the client to see + * + * @param routeClass + * @param handler + */ +export function createRoute( + routeClass: SCRoute, + handler: (validatedBody: any, app: Application, params?: { [parameterName: string]: string }) => Promise, +): Router { + // create router + const router = PromiseRouter({ mergeParams: true }); + + // create route + // the given type has no index signature so we have to cast to get the IRouteHandler when a HTTP method is given + const route = router.route(routeClass.urlFragment); + + // get route parameters (path parameters) + if (Array.isArray(routeClass.obligatoryParameters) && routeClass.obligatoryParameters.length > 0) { + routeClass.obligatoryParameters.forEach((parameterName) => { + router.param(parameterName, async (_req, _res, next, _parameterValue: string) => { + + // if (typeof req.params === 'undefined') { + // req.params = {}; + // } + + // set parameter values on request object + // req.params[parameterName] = parameterValue; + // hand over the request to the next handler (our method route handler) + next(); + }); + }); + } + + const verb = routeClass.method.toLowerCase(); + + // check if route has a valid http verb + if (isHttpMethod(verb)) { + // create a route handler for the given HTTP method + route[verb](async (req, res) => { + + try { + // get the core validator from the app + const validator: SCValidator = req.app.get('validator'); + + // validate request + const requestValidation = validator.validate(req.body, routeClass.requestBodyName.substring(2)); + + if (requestValidation.errors.length > 0) { + const error = new SCValidationErrorResponse( + requestValidation.errors, + req.app.get('isProductiveEnvironment'), + ); + res.status(error.statusCode); + res.json(error); + logger.warn(error); + return; + } + + const params: { [parameterName: string]: string } = {}; + + if (Array.isArray(routeClass.obligatoryParameters) && routeClass.obligatoryParameters) { + // copy over parameter values from request object + // the parameter values were set in the parameter handler of the route + routeClass.obligatoryParameters.forEach((parameterName) => { + params[parameterName] = req.params[parameterName]; + }); + } + + // hand over request to handler with path parameters + const response = await handler(req.body, req.app, params); + + // validate response generated by handler + const responseValidation = validator.validate(response, routeClass.responseBodyName.substring(2)); + + if (responseValidation.errors.length > 0) { + const validationError = new SCValidationErrorResponse( + responseValidation.errors, + req.app.get('isProductiveEnvironment'), + ); + const internalServerError = new SCInternalServerErrorResponse( + validationError, + req.app.get('isProductiveEnvironment'), + ); + res.status(internalServerError.statusCode); + res.json(internalServerError); + logger.warn(internalServerError); + return; + } + + // set status code + res.status(routeClass.statusCodeSuccess); + + // respond + res.json(response); + } catch (error) { + // if the error response is allowed on the route + if (routeClass.errorNames.indexOf(error.constructor.name) > -1) { + // respond with the error from the handler + res.status(error.statusCode); + res.json(error); + logger.warn(error); + } else { + // the error is not allowed so something went wrong + const internalServerError = new SCInternalServerErrorResponse( + error, + req.app.get('isProductiveEnvironment'), + ); + res.status(internalServerError.statusCode); + res.json(internalServerError); + logger.error(error); + } + } + }); + } else { + throw new Error('Invalid HTTP verb in route definition. Please check route definitions in `@openstapps/core`'); + } + + // return a SCMethodNotAllowedErrorResponse on all other HTTP methods + route.all((req, res) => { + const error = new SCMethodNotAllowedErrorResponse(req.app.get('isProductiveEnvironment')); + res.status(error.statusCode); + res.json(error); + logger.warn(error); + }); + + // return router + return router; +} diff --git a/src/routes/SearchRoute.ts b/src/routes/SearchRoute.ts new file mode 100644 index 00000000..1b1aa47a --- /dev/null +++ b/src/routes/SearchRoute.ts @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { SCSearchRequest, SCSearchResponse, SCSearchRoute } from '@openstapps/core'; +import { BulkStorage } from '../storage/BulkStorage'; +import { createRoute } from './Route'; + +const searchRouteModel = new SCSearchRoute(); + +/** + * Implementation of the search route (SCSearchRoute) + */ +export const searchRouter = createRoute(searchRouteModel, async ( request: SCSearchRequest, app) => { + const bulkMemory: BulkStorage = app.get('bulk'); + return await bulkMemory.database.search(request); +}); diff --git a/src/routes/ThingUpdateRoute.ts b/src/routes/ThingUpdateRoute.ts new file mode 100644 index 00000000..2b6424e5 --- /dev/null +++ b/src/routes/ThingUpdateRoute.ts @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { SCThingUpdateRequest, SCThingUpdateResponse, SCThingUpdateRoute } from '@openstapps/core'; +import { BulkStorage } from '../storage/BulkStorage'; +import { createRoute } from './Route'; + +const thingUpdateRouteModel = new SCThingUpdateRoute(); + +/** + * Implementation of the thing update route (SCThingUpdateRoute) + */ +export const thingUpdateRouter = createRoute( + thingUpdateRouteModel, + async (request: SCThingUpdateRequest, app) => { + const bulkMemory: BulkStorage = app.get('bulk'); + await bulkMemory.database.put(request); + return {}; + }, +); diff --git a/src/storage/BulkStorage.ts b/src/storage/BulkStorage.ts new file mode 100644 index 00000000..d5b28c2e --- /dev/null +++ b/src/storage/BulkStorage.ts @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { SCBulkRequest, SCThingTypes } from '@openstapps/core'; +import * as moment from 'moment'; +import * as NodeCache from 'node-cache'; +import { promisify } from 'util'; +import { v4 } from 'uuid'; +import { logger } from '../common'; +import { Database } from './Database'; + +export type BulkOperation = 'create' | 'expired' | 'update'; + +/** + * Describes an indexing process + */ +export class Bulk implements SCBulkRequest { + + /** + * Expiration of the bulk + * + * If the bulk is not finished before the expiration date is hit, the bulk + * and all data associated with it will be deleted + */ + expiration: string; + + /** + * The data source of the bulk + * + * Bulks with same type and source will be replaced and the data will be + * updated when the bulk is marked as done + */ + source: string; + + /** + * State of the bulk + * + * Data can be indexed for this bulk as long as the state is `in progress` + * and the bulk is not expired + * + * When the bulk is marked as `done` it replaces the previous bulk with + * the same source and type. The data will be availabe to the user when + * the bulk switches to done + */ + state: 'in progress' | 'done'; + + /** + * Type of data in the bulk + */ + type: SCThingTypes; + + /** + * Unique identifier of the bulk + */ + uid: string; + + /** + * Creates a new bulk process + * @param request + */ + constructor(request: SCBulkRequest) { + this.uid = v4(); + this.state = 'in progress'; + + if (typeof request.expiration === 'string') { + this.expiration = request.expiration; + } else { + this.expiration = moment().add(1, 'hour').toISOString(); + } + // when should this process be finished + // where does the process come from + this.source = request.source; + // which type of data is this process about to index + this.type = request.type; + } +} + +/** + * Cache for bulk-processes + */ +export class BulkStorage { + + private cache: NodeCache; + + /** + * Creates a new BulkStorage + * @param database the database that is controlled by this bulk storage + */ + constructor(public database: Database) { + + // a bulk lives 60 minutes if no expiration is given + // the cache is checked every 60 seconds + this.cache = new NodeCache({ stdTTL: 3600, checkperiod: 60 }); + + this.cache.on('expired', (_key, bulk: Bulk) => { + // if the bulk is not done + if (bulk.state !== 'done') { + // the database can delete the data associated with this bulk + this.database.bulkExpired(bulk); + } + }); + } + + /** + * Saves a bulk process and assigns to it a user-defined ttl (time-to-live) + * @param bulk the bulk process to save + * @returns the bulk process that was saved + */ + private async save(bulk: Bulk): Promise { + const expirationInSeconds = moment(bulk.expiration).diff(moment.now()) / 1000; + logger.info('Bulk expires in ', expirationInSeconds, 'seconds'); + + // save the item in the cache with it's expected expiration + await promisify(this.cache.set)(bulk.uid, bulk, expirationInSeconds); + return bulk; + } + + /** + * Create and save a new bulk process + * @param bulkRequest a request for a new bulk process + * @returns a promise that contains the new bulk process + */ + public async create(bulkRequest: SCBulkRequest): Promise { + const bulk = new Bulk(bulkRequest); + bulk.source = bulkRequest.source; + bulk.type = bulkRequest.type; + + await this.save(bulk); + + // tell the database that the bulk was created + await this.database.bulkCreated(bulk); + return bulk; + } + + /** + * Delete a bulk process + * @param uid uid of the bulk process + * @returns a promise that contains the deleted bulk process + */ + public async delete(uid: string): Promise { + const bulk = await this.read(uid); + + if (typeof bulk === 'undefined') { + throw new Error(`Bulk that should be deleted was not found. UID was "${uid}"`); + } + + // delete the bulk process from the cache + await promisify(this.cache.del)(uid); + + // tell the database to handle the expiration of the bulk + await this.database.bulkExpired(bulk); + + return bulk; + } + + /** + * Update an old bulk process (replace it with the new one) + * @param bulk new bulk process + * @returns an empty promise + */ + public async markAsDone(bulk: Bulk): Promise { + bulk.state = 'done'; + await this.save(bulk); + + // tell the database that this is the new bulk + this.database.bulkUpdated(bulk); + return; + } + + /** + * Read an existing bulk process + * @param uid uid of the bulk process + * @returns a promise that contains a bulk + */ + public async read(uid: string): Promise { + return await promisify(this.cache.get)(uid); + } + +} diff --git a/src/storage/Database.ts b/src/storage/Database.ts new file mode 100644 index 00000000..b90f8dfa --- /dev/null +++ b/src/storage/Database.ts @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { SCSearchQuery, SCSearchResponse, SCThings, SCUuid } from '@openstapps/core'; +import {Bulk} from './BulkStorage'; + +export interface DatabaseConstructor { + new (...args: any): Database; +} + +export interface Database { + + /** + * Gets called if a bulk was created + * + * The database should + * @param bulk + */ + bulkCreated(bulk: Bulk): Promise; + + /** + * Gets called if a bulk expires + * + * The database should delete all data that is associtated with this bulk + * @param bulk + */ + bulkExpired(bulk: Bulk): Promise; + + /** + * Gets called if a bulk was updated + * + * If the database holds a bulk with the same type and source as the given + * bulk it should be replaced by the given one + * + * @param bulk + */ + bulkUpdated(bulk: Bulk): Promise; + + /** + * Get a single document + * @param uid + */ + get(uid: SCUuid): Promise; + + /** + * Add a thing to an existing bulk + * @param object + * @param bulkId + */ + post(thing: SCThings, bulk: Bulk): Promise; + + /** + * Replace an existing thing in any Bulk + * + * Currently it is not possible to put an non-existing object + * + * @param thing + */ + put(thing: SCThings): Promise; + + /** + * Search for things + * @param params + */ + search(params: SCSearchQuery): Promise; +} diff --git a/src/storage/elasticsearch/Elasticsearch.ts b/src/storage/elasticsearch/Elasticsearch.ts new file mode 100644 index 00000000..ff4eb83d --- /dev/null +++ b/src/storage/elasticsearch/Elasticsearch.ts @@ -0,0 +1,508 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { + SCBulkResponse, + SCConfigFile, + SCFacet, + SCSearchQuery, + SCSearchResponse, + SCThings, + SCThingTypes, + SCUuid, +} from '@openstapps/core'; +import * as ES from 'elasticsearch'; +import * as moment from 'moment'; +import { logger } from '../../common'; +import { MailQueue } from '../../notification/MailQueue'; +import { Bulk } from '../BulkStorage'; +import { Database } from '../Database'; +import { buildAggregations, parseAggregations } from './aggregations'; +import { AggregationSchema, ElasticsearchConfig, ElasticsearchObject } from './common'; +import * as Monitoring from './monitoring'; +import { buildQuery, buildSort } from './query'; +import { putTemplate } from './templating'; + +// this will match index names such as stapps___ +const indexRegex = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/; + +/** + * A database interface for elasticsearch + */ +export class Elasticsearch implements Database { + + aggregationsSchema: AggregationSchema; + + /** + * Holds a map of all elasticsearch indices that are available to search + */ + aliasMap: { + // each scType has a alias which can contain multiple sources + [scType: string]: { + // each source is assigned a index name in elasticsearch + [source: string]: string; + }, + }; + client: ES.Client; + ready: boolean; + + /** + * Create a new interface for elasticsearch + * @param config an assembled config file + * @param mailQueue a mailqueue for monitoring + */ + constructor(private config: SCConfigFile, mailQueue?: MailQueue) { + + if (!config.internal.database || typeof config.internal.database.version === 'undefined') { + throw new Error('Database version is undefined. Check you config file'); + } + + const options = { + apiVersion: config.internal.database.version, + host: this.getElasticsearchUrl(), + log: 'error', + }; + + // enable verbose logging for all request to elasticsearch + if (process.env.ES_DEBUG === 'true') { + options.log = 'trace'; + } + + this.client = new ES.Client(options); + this.aliasMap = {}; + this.ready = false; + + this.aggregationsSchema = buildAggregations(this.config.internal.aggregations); + + this.getAliasMap(); + + const monitoringConfiguration = this.config.internal.monitoring; + + if (typeof monitoringConfiguration !== 'undefined') { + if (typeof mailQueue === 'undefined') { + throw new Error('Monitoring is defined, but MailQueue is undefined. A MailQueue is obligatory for monitoring.'); + } + // read all watches and schedule searches on the client + Monitoring.setUp(monitoringConfiguration, this.client, mailQueue); + } + + } + + /** + * Tests if an object already exists + * + * Returns Elasticsearch Object if it exists + */ + private async doesItemExist(object: SCThings): Promise<{exists: boolean; object?: ElasticsearchObject}> { + const searchResponse = await this.client.search({ + body: { + query: { + term: { + 'uid.raw': { + value: object.uid, + }, + }, + }, + }, + from: 0, + index: this.getListOfAllIndices(), + size: 1, + }); + + if (searchResponse.hits.total > 1) { + return { + exists: true, + object: searchResponse.hits.hits[0], + }; + } + + return { + exists: false, + }; + } + + /** + * Gets a map which contains each alias and all indices that are associated with each alias + */ + private async getAliasMap() { + + // create a list of old indices that are not in use + const oldIndicesToDelete: string[] = []; + + let aliases: { + [index: string]: { + aliases: { + [K in SCThingTypes]: any + }, + }, + }; + + try { + aliases = await this.client.indices.getAlias({}); + } catch (error) { + logger.error('Failed getting alias map:', error); + setTimeout(() => { + this.getAliasMap(); + }, 5000); // retry in 5 seconds + return; + } + + for (const index in aliases) { + if (aliases.hasOwnProperty(index)) { + + const matches = indexRegex.exec(index); + if (matches !== null) { + const type = matches[1]; + const source = matches[2]; + + // check if there is an alias for the current index + // check that alias equals type + const hasAlias = type in aliases[index].aliases; + if (hasAlias) { + if (typeof this.aliasMap[type] === 'undefined') { + this.aliasMap[type] = {}; + } + this.aliasMap[type][source] = index; + } else { + oldIndicesToDelete.push(index); + } + } + } + } + + this.ready = true; + + // delete old indices that are not used in any alias + if (oldIndicesToDelete.length > 0) { + await this.client.indices.delete({ + index: oldIndicesToDelete, + }); + logger.warn('Deleted old indices: ' + oldIndicesToDelete); + } + + logger.ok('Read alias map from elasticsearch: ' + JSON.stringify(this.aliasMap, null, 2)); + } + + /** + * Get the url of elasticsearch + */ + private getElasticsearchUrl(): string { + // check if we have a docker link + if (process.env.ES_PORT_9200_TCP_ADDR !== undefined && process.env.ES_PORT_9200_TCP_PORT !== undefined) { + return process.env.ES_PORT_9200_TCP_ADDR + ':' + process.env.ES_PORT_9200_TCP_PORT; + } + + // default + return 'localhost:9200'; + } + + /** + * Gets the index name in elasticsearch for one SCThingType + * @param type SCThingType of data in the index + * @param source source of data in the index + * @param bulk bulk process which created this index + */ + private getIndex(type: SCThingTypes, source: string, bulk: SCBulkResponse) { + return `stapps_${type.toLowerCase().replace(' ', '_')}_${source}_${bulk.uid.substring(0, 8)}`; + } + + /** + * Generates a string which matches all indices + */ + private getListOfAllIndices(): string { + // map each SC type in upper camel case + return 'stapps_*_*_*'; + } + + /** + * Should be called, when a new bulk was created. Creates a new index and applies a the mapping to the index + * @param bulk the bulk process that was created + */ + public async bulkCreated(bulk: Bulk): Promise { + // if our es instance is not ready yet, we cannot serve this request + if (!this.ready) { + throw new Error('No connection to elasticsearch established yet.'); + } + + // index name for elasticsearch + const index: string = this.getIndex(bulk.type, bulk.source, bulk); + + // there already is an index with this type and source. We will index the new one and switch the alias to it + // the old one is deleted + const alias = bulk.type; + + if (typeof this.aliasMap[alias] === 'undefined') { + this.aliasMap[alias] = {}; + } + + if (!indexRegex.test(index)) { + throw new Error( + 'Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.\n' + + 'Make sure to set the bulk "source" and "type" to names consisting of the characters above.', + ); + } + + // re-apply the index template before each new bulk operation + await putTemplate(this.client); + await this.client.indices.create({ + index, + }); + + logger.info('Created index', index); + } + + /** + * Should be called when a bulk process is expired. The index that was created with this bulk gets deleted + * @param bulk the bulk process that is expired + */ + public async bulkExpired(bulk: Bulk): Promise { + // index name for elasticsearch + const index: string = this.getIndex(bulk.type, bulk.source, bulk); + + logger.info('Bulk expired. Deleting index', index); + + // don't delete indices that are in use already + if (bulk.state !== 'done') { + logger.info('deleting obsolete index', index); + return await this.client.indices.delete({ index }); + } + } + + /** + * Should be called when a bulk process is updated (replaced by a newer bulk). This will replace the old + * index and publish all data, that was index in the new instead + * @param bulk the new bulk process that should replace the old one with same type and source + */ + public async bulkUpdated(bulk: Bulk): Promise { + // if our es instance is not ready yet, we cannot serve this request + if (!this.ready) { + throw new Error('Elasticsearch not ready'); + } + + // index name for elasticsearch + const index: string = this.getIndex(bulk.type, bulk.source, bulk); + + // alias for the indices + const alias = bulk.type; + + if (typeof this.aliasMap[alias] === 'undefined') { + this.aliasMap[alias] = {}; + } + + if (!indexRegex.test(index)) { + throw new Error( + 'Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.\n' + + 'Make sure to set the bulk "source" and "type" to names consisting of the characters above.', + ); + } + + // create the new index if it does not exists + if (!(await this.client.indices.exists({ index }))) { + // re-apply the index template before each new bulk operation + await putTemplate(this.client); + await this.client.indices.create({ + index, + }); + } + + // get the old index from our aliasMap + const oldIndex: string = this.aliasMap[alias][bulk.source]; + + // add our new index to the alias + const actions: ES.IndicesUpdateAliasesParamsAction[] = [ + { + add: { index: index, alias: alias }, + }, + ]; + + // remove our old index if it exists + if (typeof oldIndex === 'string') { + actions.push({ + remove: { index: oldIndex, alias: alias }, + }); + } + + // refresh the index (fsync changes) + await this.client.indices.refresh({ + index, + }); + + // execute our alias actions + await this.client.indices.updateAliases({ + body: { + actions, + }, + }); + + // swap the index in our aliasMap + this.aliasMap[alias][bulk.source] = index; + if (typeof oldIndex === 'string') { + // delete the old index + await this.client.indices.delete({ index: oldIndex }); + logger.info('deleted old index', oldIndex); + } + logger.info('swapped alias index alias', oldIndex, '=>', index); + } + + /** + * Gets an SCThing from all indexed data + * @param uid uid of an SCThing + */ + public async get(uid: SCUuid): Promise { + const searchResponse = await this.client.search({ + body: { + query: { + term: { + uid, + }, + }, + }, + index: this.getListOfAllIndices(), + }); + + // get data from response + const hits = searchResponse.hits.hits; + + if (hits.length !== 1) { + throw new Error('No unique item found.'); + } else { + return hits[0]._source as SCThings; + } + } + + /** + * Add an item to an index + * @param object the SCThing to add to the index + * @param bulk the bulk process which item belongs to + */ + public async post(object: SCThings, bulk: Bulk): Promise { + + const obj: SCThings & {creation_date: string} = { + ...object, + creation_date: moment().format(), + }; + + const itemMeta = await this.doesItemExist(obj); + + // we have to check that the item will get replaced if the index is rolled over + if (itemMeta.exists && typeof itemMeta.object !== 'undefined') { + const indexOfNew = this.getIndex(obj.type, bulk.source, bulk); + const oldIndex = itemMeta.object._index; + + // new item doesn't replace the old one + if (oldIndex.substring(0, oldIndex.length - 9) !== indexOfNew.substring(0, indexOfNew.length - 9)) { + throw new Error( + 'Object \"' + obj.uid + '\" already exists. Object was: ' + + JSON.stringify(obj, null, 2), + ); + } + } + + // regular bulk update (item gets replaced when bulk is updated) + const searchResponse = await this.client.create({ + body: obj, + id: obj.uid, + index: this.getIndex(obj.type, bulk.source, bulk), + timeout: '90s', + type: obj.type, + }); + + if (!searchResponse.created) { + throw new Error('Object creation Error: Instance was: ' + JSON.stringify(obj)); + } + } + + /** + * Put (update) an existing item + * @param object SCThing to put + */ + public async put(object: SCThings) { + + const itemMeta = await this.doesItemExist(object); + + if (itemMeta.exists && typeof itemMeta.object !== 'undefined') { + return await this.client.update({ + body: { + doc: object, + }, + id: object.uid, + index: itemMeta.object._index, + type: object.type.toLowerCase(), + }); + } + + throw new Error('You tried to PUT an non-existing object. PUT is only supported on existing objects.'); + } + + /** + * Search all indexed data + * @param params search query + */ + public async search(params: SCSearchQuery): Promise { + + if (typeof this.config.internal.database === 'undefined') { + throw new Error('Database is undefined. You have to configure the query build'); + } + + const searchRequest: ES.SearchParams = { + body: { + aggs: this.aggregationsSchema, // use cached version of aggregations (they only change if config changes) + query: buildQuery(params, this.config, this.config.internal.database as ElasticsearchConfig), + }, + from: params.from, + index: this.getListOfAllIndices(), + size: params.size, + }; + + if (typeof params.sort !== 'undefined') { + searchRequest.body.sort = buildSort(params.sort); + } + + // perform the search against elasticsearch + const response = await this.client.search(searchRequest); + + // gather pagination information + const pagination = { + count: response.hits.hits.length, + offset: (typeof params.from === 'number') ? params.from : 0, + total: response.hits.total, + }; + + // gather statistics about this search + const stats = { + time: response.took, + }; + + // we only directly return the _source documents + // elasticsearch provides much more information, the user shouldn't see + const data = response.hits.hits.map((hit) => { + return hit._source; // SCThing + }); + + let facets: SCFacet[] = []; + + // read the aggregations from elasticsearch and parse them to facets by our configuration + if (typeof response.aggregations !== 'undefined') { + facets = parseAggregations(this.aggregationsSchema, response.aggregations); + } + + return { + data, + facets, + pagination, + stats, + }; + } +} diff --git a/src/storage/elasticsearch/aggregations.ts b/src/storage/elasticsearch/aggregations.ts new file mode 100644 index 00000000..a73b4163 --- /dev/null +++ b/src/storage/elasticsearch/aggregations.ts @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { SCBackendAggregationConfiguration, SCFacet, SCThingTypes } from '@openstapps/core'; +import { AggregationSchema } from './common'; + +export type aggregationType = SCThingTypes | '@all'; + +/** + * Builds the aggregation + * @returns a schema to tell elasticsearch which aggregations to collect + */ +export function buildAggregations(aggsConfig: SCBackendAggregationConfiguration[]): AggregationSchema { + + const result: AggregationSchema = {}; + + aggsConfig.forEach((aggregation) => { + + result[aggregation.fieldName] = { + terms: { + field: aggregation.fieldName + '.raw', + size: 1000, + }, + }; + }); + + return result; +} + +/** + * An elasticsearch aggregation bucket + */ +interface Bucket { + doc_count: number; + key: string; +} + +/** + * An elasticsearch aggregation response + */ +interface AggregationResponse { + [field: string]: { + buckets: Bucket[]; + doc_count?: number; + }; +} + +/** + * Parses elasticsearch aggregations (response from es) to facets for the app + * @param aggregationSchema - aggregation-schema for elasticsearch + * @param aggregations - aggregations response from elasticsearch + */ +export function parseAggregations( + aggregationSchema: AggregationSchema, + aggregations: AggregationResponse): SCFacet[] { + + const facets: SCFacet[] = []; + + const aggregationNames = Object.keys(aggregations); + + aggregationNames.forEach((aggregationName) => { + const buckets = aggregations[aggregationName].buckets; + + const facet: SCFacet = { + buckets: buckets.map((bucket) => { + const facetBucket: { [value: string]: number } = {}; + facetBucket[bucket.key] = bucket.doc_count; + return facetBucket; + }), + field: aggregationSchema[aggregationName].terms.field + '.raw', + }; + + facets.push(facet); + }); + return facets; +} diff --git a/src/storage/elasticsearch/common.ts b/src/storage/elasticsearch/common.ts new file mode 100644 index 00000000..1a9143da --- /dev/null +++ b/src/storage/elasticsearch/common.ts @@ -0,0 +1,208 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { SCThingTypes } from '@openstapps/core'; +import { SCThing } from '@openstapps/core'; + +/** + * An elasticsearch bucket aggregation + * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.5/search-aggregations-bucket.html + */ +export interface AggregationSchema { + [aggregationName: string]: ESTermsFilter; +} + +/** + * A configuration for using the Dis Max Query + * + * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further + * explanation of what the parameters mean + */ +export interface ElasticsearchQueryDisMaxConfig { + cutoffFrequency: number; + fuzziness: number; + matchBoosting: number; + minMatch: string; + queryType: 'dis_max'; + tieBreaker: number; +} + +/** + * A configuration for using Query String Query + * + * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further + * explanation of what the parameters mean + */ +export interface ElasticsearchQueryQueryStringConfig { + minMatch: string; + queryType: 'query_string'; +} + +/** + * A hit in an elastiscsearch search result + */ +export interface ElasticsearchObject { + _id: string; + _index: string; + _score: number; + _source: T; + _type: string; + _version?: number; + fields?: any; + highlight?: any; + inner_hits?: any; + matched_queries?: string[]; + sort?: string[]; +} + +/** + * An config file for the elasticsearch database interface + * + * The config file extends the SCConfig file by further defining how the database property + */ +export interface ElasticsearchConfigFile { + internal: { + database: ElasticsearchConfig; + }; +} + +/** + * An elasticsearch configuration + */ +export interface ElasticsearchConfig { + name: 'elasticsearch'; + query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig; + version: string; +} + +/** + * An elasticsearch term filter + */ +export interface ESTermFilter { + term: { + [fieldName: string]: string; + }; +} + +/** + * An elasticsearch terms filter + */ +export interface ESTermsFilter { + terms: { + field: string; + size?: number; + }; +} + +/** + * An elasticsearch type filter + */ +export interface ESTypeFilter { + type: { + value: SCThingTypes; + }; +} + +/** + * Filter arguments for an elasticsearch geo distance filter + */ +export interface ESGeoDistanceFilterArguments { + distance: string; + [fieldName: string]: { + lat: number; + lon: number; + } | string; +} + +/** + * An elasticsearch geo distance filter + */ +export interface ESGeoDistanceFilter { + geo_distance: ESGeoDistanceFilterArguments; +} + +/** + * Filter arguments for an elasticsearch boolean filter + */ +export interface ESBooleanFilterArguments { + minimum_should_match?: number; + must?: T[]; + must_not?: T[]; + should?: T[]; +} + +/** + * An elasticsearch boolean filter + */ +export interface ESBooleanFilter { + bool: ESBooleanFilterArguments; +} + +/** + * An elasticsearch function score query + */ +export interface ESFunctionScoreQuery { + function_score: { + functions: ESFunctionScoreQueryFunction[]; + query: ESBooleanFilter; + score_mode: 'multiply'; + }; +} + +/** + * An function for an elasticsearch functions score query + */ +export interface ESFunctionScoreQueryFunction { + filter: ESTermFilter | ESTypeFilter | ESBooleanFilter; + weight: number; +} + +/** + * An elasticsearch ducet sort + */ +export interface ESDucetSort { + [field: string]: string; +} + +/** + * Sort arguments for an elasticsearch geo distance sort + */ +export interface ESGeoDistanceSortArguments { + mode: 'avg' | 'max' | 'median' | 'min'; + order: 'asc' | 'desc'; + unit: 'm'; + [field: string]: { + lat: number; + lon: number; + } | string; +} + +/** + * An elasticsearch geo distance sort + */ +export interface ESGeoDistanceSort { + _geo_distance: ESGeoDistanceSortArguments; +} + +/** + * An elasticsearch script sort + */ +export interface ScriptSort { + _script: { + order: 'asc' | 'desc'; + script: string; + type: 'number' | 'string'; + }; +} diff --git a/src/storage/elasticsearch/monitoring.ts b/src/storage/elasticsearch/monitoring.ts new file mode 100644 index 00000000..89fc5163 --- /dev/null +++ b/src/storage/elasticsearch/monitoring.ts @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { + SCMonitoringConfiguration, + SCMonitoringLogAction, + SCMonitoringMailAction, + SCMonitoringMaximumLengthCondition, + SCMonitoringMinimumLengthCondition, +} from '@openstapps/core'; +import * as ES from 'elasticsearch'; +import * as cron from 'node-cron'; +import { logger } from '../../common'; +import { MailQueue } from '../../notification/MailQueue'; + +/** + * Check if the given condition fails on the given number of results and the condition + * @param condition condition + * @param total number of results + */ +function conditionFails( + condition: SCMonitoringMaximumLengthCondition | SCMonitoringMinimumLengthCondition, + total: number, +) { + if (condition.type === 'MaximumLength') { + return maxConditionFails(condition.length, total); + } + return minConditionFails(condition.length, total); +} + +/** + * Check if the min condition fails + * @param minimumLength + * @param total + */ +function minConditionFails(minimumLength: number, total: number) { + return typeof minimumLength === 'number' && minimumLength > total; +} + +/** + * Check if the max condition fails + * @param maximumLength + * @param total + */ +function maxConditionFails(maximumLength: number, total: number) { + return typeof maximumLength === 'number' && maximumLength < total; +} + +/** + * Run all the given actions + * @param actions actions to perform + * @param watcherName name of watcher that wants to perform them + * @param triggerName name of trigger that triggered the watcher + * @param total total number of results of the query + * @param mailQueue mailQueue to execute mail actions + */ +export function runActions( + actions: Array, + watcherName: string, + triggerName: string, + total: number, + mailQueue: MailQueue, +) { + + actions.forEach((action) => { + if (action.type === 'log') { + logger.error( + action.prefix, + `Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'`, `Found ${total} hits instead`, + action.message, + ); + } else { + mailQueue.push({ + subject: action.subject, + text: `Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'\n` + + action.message + + `Found ${total} hits instead`, + to: action.recipients, + }); + } + }); +} + +/** + * Set up the triggers for the configured watchers + * @param monitoringConfig configuration of the monitoring + * @param esClient elasticsearch client + * @param mailQueue mailQueue for mail actions + */ +export function setUp(monitoringConfig: SCMonitoringConfiguration, esClient: ES.Client, mailQueue: MailQueue) { + + // set up Watches + monitoringConfig.watchers.forEach((watcher) => { + + // make a schedule for each trigger + watcher.triggers.forEach((trigger) => { + switch (trigger.executionTime) { + case 'hourly': + trigger.executionTime = '5 * * * *'; + break; + case 'daily': + trigger.executionTime = '5 0 * * *'; + break; + case 'weekly': + trigger.executionTime = '5 0 * * 0'; + break; + case 'monthly': + trigger.executionTime = '5 0 * 0 * *'; + } + + cron.schedule(trigger.executionTime, async () => { + // execute watch (search->condition->action) + const result = await esClient.search(watcher.query); + + // check conditions + const total = result.hits.total; + + watcher.conditions.forEach((condition) => { + if (conditionFails(condition, total)) { + runActions(watcher.actions, watcher.name, trigger.name, total, mailQueue); + } + }); + }); + }); + + }); + + logger.log('Scheduled ' + monitoringConfig.watchers.length + ' watches'); +} diff --git a/src/storage/elasticsearch/query.ts b/src/storage/elasticsearch/query.ts new file mode 100644 index 00000000..c66c38b4 --- /dev/null +++ b/src/storage/elasticsearch/query.ts @@ -0,0 +1,408 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { + SCBackendConfigurationSearchBoosting, + SCConfigFile, + SCSearchBooleanFilter, + SCSearchFilter, + SCSearchQuery, + SCSearchSort, + SCSportCoursePriceGroup, + SCThingsField, + SCThingTypes, +} from '@openstapps/core'; +import { ElasticsearchConfig, ScriptSort } from './common'; +import { + ESBooleanFilter, + ESBooleanFilterArguments, + ESDucetSort, + ESFunctionScoreQuery, + ESFunctionScoreQueryFunction, + ESGeoDistanceFilter, + ESGeoDistanceFilterArguments, + ESGeoDistanceSort, + ESGeoDistanceSortArguments, + ESTermFilter, + ESTypeFilter, +} from './common'; + +/** + * Builds a boolean filter. Returns an elasticsearch boolean filter + */ +export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBooleanFilterArguments { + + const result: ESBooleanFilterArguments = { + minimum_should_match: 0, + must: [], + must_not: [], + should: [], + }; + + if (booleanFilter.arguments.operation === 'and') { + result.must = booleanFilter.arguments.filters.map((filter) => buildFilter(filter)); + } + + if (booleanFilter.arguments.operation === 'or') { + result.should = booleanFilter.arguments.filters.map((filter) => buildFilter(filter)); + result.minimum_should_match = 1; + } + + if (booleanFilter.arguments.operation === 'not') { + result.must_not = booleanFilter.arguments.filters.map((filter) => buildFilter(filter)); + } + + return result; +} + +/** + * Converts Array of Filters to elasticsearch query-syntax + * @param filter + */ +export function buildFilter(filter: SCSearchFilter): ESTermFilter | ESGeoDistanceFilter | ESBooleanFilter { + + switch (filter.type) { + case 'value': + const filterObj: { [field: string]: string } = {}; + filterObj[filter.arguments.field + '.raw'] = filter.arguments.value; + return { + term: filterObj, + }; + case 'availability': + const startRangeFilter: { [field: string]: { lte: string } } = {}; + startRangeFilter[filter.arguments.fromField] = { + lte: 'now', + }; + + const endRangeFilter: { [field: string]: { gte: string } } = {}; + endRangeFilter[filter.arguments.toField] = { + gte: 'now', + }; + + return { + bool: { + should: [ + { + bool: { + must: [ + { + range: startRangeFilter, + }, + { + range: endRangeFilter, + }, + ], + }, + }, + { + bool: { + must_not: [ + { + exists: { + field: filter.arguments.fromField, + }, + }, + { + exists: { + field: filter.arguments.toField, + }, + }, + ], + }, + }, + ], + }, + }; + case 'distance': + const geoObject: ESGeoDistanceFilterArguments = { + distance: filter.arguments.distanceInM + 'm', + }; + geoObject[filter.arguments.field] = { + lat: filter.arguments.lat, + lon: filter.arguments.lon, + }; + return { + geo_distance: geoObject, + }; + case 'boolean': + return { + bool: buildBooleanFilter(filter), + }; + default: + throw new Error('Unknown Filter type'); + } +} + +/** + * Builds scorings functions from boosting config + * @param boosting + * @returns + */ +export function buildFunctions(boosting: SCBackendConfigurationSearchBoosting[]): ESFunctionScoreQueryFunction[] { + + const functions: ESFunctionScoreQueryFunction[] = []; + + // add a good scoring subset from config file + boosting.forEach((boostingForOneSCType) => { + const typeFilter: ESTypeFilter = { + type: { + value: boostingForOneSCType.type, + }, + }; + + functions.push({ + filter: typeFilter, + weight: boostingForOneSCType.factor, + }); + + if (typeof boostingForOneSCType.fields !== 'undefined') { + + const fields = boostingForOneSCType.fields; + + Object.keys(boostingForOneSCType.fields).forEach((fieldName) => { + + const boostingForOneField = fields[fieldName]; + + Object.keys(boostingForOneField).forEach((value) => { + const factor = boostingForOneField[value]; + + // build term filter + const termFilter: ESTermFilter = { + term: {}, + }; + termFilter.term[fieldName + '.raw'] = value; + + functions.push({ + filter: { + bool: { + must: [ + typeFilter, + termFilter, + ], + should: [], + }, + }, + weight: factor, + }); + }); + }); + } + }); + + return functions; +} +/** + * Builds body for Elasticsearch requests + * @param params + * @param defaultConfig + * @returns ElasticsearchQuery (body of a search-request) + */ +export function buildQuery( + params: SCSearchQuery, + defaultConfig: SCConfigFile, + elasticsearchConfig: ElasticsearchConfig, +): ESFunctionScoreQuery { + + // if a sort is used it, we may have to narrow down the types so the sort is executable + let typeFiltersToAppend: ESTypeFilter[] = []; + + if (typeof params.sort !== 'undefined') { + params.sort.forEach((sort) => { + // types that the sort is supported on + const types: SCThingTypes[] = []; + + defaultConfig.backend.sortableFields + .filter((sortableField) => { + return sortableField.fieldName === sort.arguments.field && sortableField.sortTypes.indexOf(sort.type) > -1; + }) + .forEach((sortableField) => { + if (typeof sortableField.onlyOnTypes !== 'undefined') { + sortableField.onlyOnTypes.forEach((scType) => { + if (types.indexOf(scType) === -1) { + types.push(scType); + } + }); + } + }); + + if (types.length > 0) { + typeFiltersToAppend = types.map((type) => { + return { + type: { + value: type, + }, + }; + }); + } + }); + } + + // if config provides an minMatch parameter we use query_string instead of match query + let query; + if (typeof elasticsearchConfig.query === 'undefined') { + query = { + query_string: { + analyzer: 'search_german', + default_field: 'name', + minimum_should_match: '90%', + query: (typeof params.query !== 'string') ? '*' : params.query, + }, + }; + } else if (elasticsearchConfig.query.queryType === 'query_string') { + query = { + query_string: { + analyzer: 'search_german', + default_field: 'name', + minimum_should_match: elasticsearchConfig.query.minMatch, + query: (typeof params.query !== 'string') ? '*' : params.query, + }, + }; + } else if (elasticsearchConfig.query.queryType === 'dis_max') { + if (params.query !== '*') { + query = { + dis_max: { + boost: 1.2, + queries: [ + { + match: { + name: { + boost: elasticsearchConfig.query.matchBoosting, + cutoff_frequency: elasticsearchConfig.query.cutoffFrequency, + fuzziness: elasticsearchConfig.query.fuzziness, + query: (typeof params.query !== 'string') ? '*' : params.query, + }, + }, + }, + { + query_string: { + analyzer: 'search_german', + default_field: 'name', + minimum_should_match: elasticsearchConfig.query.fuzziness, + query: (typeof params.query !== 'string') ? '*' : params.query, + }, + }, + ], + tie_breaker: elasticsearchConfig.query.tieBreaker, + }, + + }; + } else { + throw new Error('Query Type is not supported. Check your config file and reconfigure your elasticsearch query'); + } + } + + const functionScoreQuery: ESFunctionScoreQuery = { + function_score: { + functions: buildFunctions(defaultConfig.internal.boostings), + query: { + bool: { + minimum_should_match: 0, // if we have no should, nothing can match + must: [], + should: [], + }, + }, + score_mode: 'multiply', + }, + }; + + const mustMatch = functionScoreQuery.function_score.query.bool.must; + + if (Array.isArray(mustMatch)) { + if (typeof query !== 'undefined') { + mustMatch.push(query); + } + + if (typeof params.filter !== 'undefined') { + mustMatch.push(buildFilter(params.filter)); + } + + // add type filters for sorts + mustMatch.push.apply(mustMatch, typeFiltersToAppend); + } + return functionScoreQuery; +} + +/** + * converts query to + * @param params + * @param sortableFields + * @returns an array of sort queries + */ +export function buildSort( + sorts: SCSearchSort[], +): Array { + return sorts.map((sort) => { + switch (sort.type) { + case 'ducet': + const ducetSort: ESDucetSort = {}; + ducetSort[sort.arguments.field + '.sort'] = sort.order; + return ducetSort; + case 'distance': + const args: ESGeoDistanceSortArguments = { + mode: 'avg', + order: sort.order, + unit: 'm', + }; + + args[sort.arguments.field] = { + lat: sort.arguments.lat, + lon: sort.arguments.lon, + }; + + return { + _geo_distance: args, + }; + case 'price': + return { + _script: { + order: sort.order, + script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field), + type: 'number' as 'number', + }, + }; + } + }); +} + +export function buildPriceSortScript(universityRole: keyof SCSportCoursePriceGroup, field: SCThingsField): string { + return ` + // initialize the sort value with the maximum + double price = Double.MAX_VALUE; + + // if we have any offers + if (params._source.containsKey('${field}')) { + // iterate through all offers + for (offer in params._source.${field}) { + // if this offer contains a role specific price + if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) { + // if the role specific price is smaller than the cheapest we found + if (offer.prices.${universityRole} < price) { + // set the role specific price as cheapest for now + price = offer.prices.${universityRole}; + } + } else { // we have no role specific price for our role in this offer + // if the default price of this offer is lower than the cheapest we found + if (offer.price < price) { + // set this price as the cheapest + price = offer.price; + } + } + } + } + + // return cheapest price for our role + return price; + `; +} diff --git a/src/storage/elasticsearch/templates/address.field.template.json b/src/storage/elasticsearch/templates/address.field.template.json new file mode 100644 index 00000000..28f9f26f --- /dev/null +++ b/src/storage/elasticsearch/templates/address.field.template.json @@ -0,0 +1,19 @@ +{ + "properties": { + "addressCountry": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "addressLocality": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "addressRegion": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "streetAddress": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "postalCode": { + "_fieldRef": "filterableKeyword.field.template.json" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/article.sc-type.template.json b/src/storage/elasticsearch/templates/article.sc-type.template.json new file mode 100644 index 00000000..f04deafb --- /dev/null +++ b/src/storage/elasticsearch/templates/article.sc-type.template.json @@ -0,0 +1,25 @@ +{ + "properties": { + "authors": { + "_typeRef": "person.sc-type.template.json" + }, + "publishers": { + "_typeRef": [ + "person.sc-type.template.json", + "organization.sc-type.template.json" + ] + }, + "datePublished": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "inLanguages": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "keywords": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "articleBody": { + "_fieldRef": "text.field.template.json" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/base.template.json b/src/storage/elasticsearch/templates/base.template.json new file mode 100644 index 00000000..22c2bc5a --- /dev/null +++ b/src/storage/elasticsearch/templates/base.template.json @@ -0,0 +1,91 @@ +{ + "template": "stapps_*", + "settings": { + "max_result_window": 30000, + "mapping.total_fields.limit": 2000, + "number_of_shards": 1, + "number_of_replicas": 0, + "analysis": { + "filter": { + "german_stemmer": { + "type": "stemmer", + "language": "german" + }, + "german_stop": { + "type": "stop", + "stopwords": "_german_" + }, + "german_phonebook": { + "type": "icu_collation", + "language": "de", + "country": "DE", + "variant": "@collation=phonebook" + } + }, + "tokenizer": { + "stapps_ngram": { + "type": "ngram", + "min_gram": 4, + "max_gram": 7 + } + }, + "analyzer": { + "search_german": { + "tokenizer": "stapps_ngram", + "filter": [ + "lowercase", + "german_stop", + "german_stemmer" + ] + }, + "ducet_sort": { + "tokenizer": "keyword", + "filter": [ + "german_phonebook" + ] + } + } + } + }, + "mappings": { + "_default_": { + "properties": { + "creation_date": { + "type": "date", + "store": true + }, + "uid": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "type": { + "_fieldRef": "sortableKeyword.field.template.json" + }, + "name": { + "_fieldRef": "text.field.template.json" + }, + "alternateNames": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "categories": { + "_fieldRef": "sortableKeyword.field.template.json" + }, + "url": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "description": { + "_fieldRef": "text.field.template.json" + }, + "image": { + "_fieldRef": "filterableKeyword.field.template.json" + } + }, + "_source": { + "excludes": [ + "creation_date" + ] + }, + "date_detection": false, + "dynamic_templates": [] + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/book.sc-type.template.json b/src/storage/elasticsearch/templates/book.sc-type.template.json new file mode 100644 index 00000000..bdf31647 --- /dev/null +++ b/src/storage/elasticsearch/templates/book.sc-type.template.json @@ -0,0 +1,31 @@ +{ + "properties": { + "authors": { + "_typeRef": "person.sc-type.template.json" + }, + "bookEdition": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "isbn": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "numberOfPages": { + "type": "integer" + }, + "publishers": { + "_typeRef": "organization.sc-type.template.json" + }, + "datePublished": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "inLanguages": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "keywords": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "articleBody": { + "_fieldRef": "text.field.template.json" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/catalog.sc-type.template.json b/src/storage/elasticsearch/templates/catalog.sc-type.template.json new file mode 100644 index 00000000..526d3610 --- /dev/null +++ b/src/storage/elasticsearch/templates/catalog.sc-type.template.json @@ -0,0 +1,22 @@ +{ + "properties": { + "level": { + "type": "integer", + "fields": { + "raw": { + "type": "integer" + } + } + }, + "semester": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "superCatalog": { + "_typeRef": "catalog.sc-type.template.json" + }, + "superCatalogs": { + "type": "nested", + "_typeRef": "catalog.sc-type.template.json" + } + } +} diff --git a/src/storage/elasticsearch/templates/date.sc-type.template.json b/src/storage/elasticsearch/templates/date.sc-type.template.json new file mode 100644 index 00000000..8429b131 --- /dev/null +++ b/src/storage/elasticsearch/templates/date.sc-type.template.json @@ -0,0 +1,47 @@ +{ + "properties": { + "startDate": { + "_fieldRef": "filterableDate.field.template.json" + }, + "endDate": { + "_fieldRef": "filterableDate.field.template.json" + }, + "startTime": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "endTime": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "dayOfWeek": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "place": { + "_typeRef": "place.sc-type.template.json" + }, + "performers": { + "type": "nested", + "_typeRef": "person.sc-type.template.json" + }, + "duration": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "frequency": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "frequencyMultiplier": { + "type": "float" + }, + "repeatFrequency": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "superEvent": { + "_typeRef": "event.sc-type.template.json" + }, + "exceptions": { + "_fieldRef": "filterableDate.field.template.json" + }, + "dates": { + "_fieldRef": "filterableDate.field.template.json" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/diff.sc-type.template.json b/src/storage/elasticsearch/templates/diff.sc-type.template.json new file mode 100644 index 00000000..fe0163ba --- /dev/null +++ b/src/storage/elasticsearch/templates/diff.sc-type.template.json @@ -0,0 +1,10 @@ +{ + "properties": { + "dateCreated": { + "_fieldRef": "filterableDate.field.template.json" + }, + "action": { + "_fieldRef": "filterableKeyword.field.template.json" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/dish.sc-type.template.json b/src/storage/elasticsearch/templates/dish.sc-type.template.json new file mode 100644 index 00000000..2c65219a --- /dev/null +++ b/src/storage/elasticsearch/templates/dish.sc-type.template.json @@ -0,0 +1,22 @@ +{ + "properties": { + "availabilityStarts": { + "_fieldRef": "filterableDate.field.template.json" + }, + "availabilityEnds": { + "_fieldRef": "filterableDate.field.template.json" + }, + "offers": { + "_typeRef": "offers.sc-type.template.json" + }, + "characteristics": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "additives": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "place": { + "_typeRef": "place.sc-type.template.json" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/event.sc-type.template.json b/src/storage/elasticsearch/templates/event.sc-type.template.json new file mode 100644 index 00000000..39f0830f --- /dev/null +++ b/src/storage/elasticsearch/templates/event.sc-type.template.json @@ -0,0 +1,47 @@ +{ + "properties": { + "subType": { + "_fieldRef": "sortableKeyword.field.template.json" + }, + "categories": { + "_fieldRef": "sortableKeyword.field.template.json" + }, + "previousStartDate": { + "_fieldRef": "filterableDate.field.template.json" + }, + "place": { + "_typeRef": "place.sc-type.template.json" + }, + "organizers": { + "type": "nested", + "_typeRef": "person.sc-type.template.json" + }, + "performers": { + "type": "nested", + "_typeRef": "person.sc-type.template.json" + }, + "attendees": { + "type": "nested", + "_typeRef": "person.sc-type.template.json" + }, + "catalogs": { + "type": "nested", + "_typeRef": "catalog.sc-type.template.json" + }, + "maximumParticipants": { + "type": "integer" + }, + "superEvent": { + "_typeRef": "event.sc-type.template.json" + }, + "subProperties": { + "_fieldRef": "eventSubProperties.field.template.json" + }, + "startDate": { + "_fieldRef": "filterableDate.field.template.json" + }, + "endDate": { + "_fieldRef": "filterableDate.field.template.json" + } + } +} diff --git a/src/storage/elasticsearch/templates/eventSubProperties.field.template.json b/src/storage/elasticsearch/templates/eventSubProperties.field.template.json new file mode 100644 index 00000000..28bc7eca --- /dev/null +++ b/src/storage/elasticsearch/templates/eventSubProperties.field.template.json @@ -0,0 +1,16 @@ +{ + "properties": { + "id": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "semester": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "majors": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "originalCategory": { + "_fieldRef": "filterableKeyword.field.template.json" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/filterableDate.field.template.json b/src/storage/elasticsearch/templates/filterableDate.field.template.json new file mode 100644 index 00000000..9710b951 --- /dev/null +++ b/src/storage/elasticsearch/templates/filterableDate.field.template.json @@ -0,0 +1,8 @@ +{ + "type": "date", + "fields": { + "raw": { + "type": "keyword" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/filterableKeyword.field.template.json b/src/storage/elasticsearch/templates/filterableKeyword.field.template.json new file mode 100644 index 00000000..ec2e522f --- /dev/null +++ b/src/storage/elasticsearch/templates/filterableKeyword.field.template.json @@ -0,0 +1,8 @@ +{ + "type": "keyword", + "fields": { + "raw": { + "type": "keyword" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/floorplan.sc-type.template.json b/src/storage/elasticsearch/templates/floorplan.sc-type.template.json new file mode 100644 index 00000000..ee37e92a --- /dev/null +++ b/src/storage/elasticsearch/templates/floorplan.sc-type.template.json @@ -0,0 +1,10 @@ +{ + "properties": { + "place": { + "_typeRef": "place.sc-type.template.json" + }, + "floor": { + "_fieldRef": "filterableKeyword.field.template.json" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/jobs.field.template.json b/src/storage/elasticsearch/templates/jobs.field.template.json new file mode 100644 index 00000000..f995f028 --- /dev/null +++ b/src/storage/elasticsearch/templates/jobs.field.template.json @@ -0,0 +1,26 @@ +{ + "properties": { + "jobTitle": { + "type": "text" + }, + "worksFor": { + "_typeRef": "organization.sc-type.template.json" + }, + "workLocation": { + "properties": { + "email": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "faxNumber": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "telephone": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "openingHours": { + "_fieldRef": "filterableKeyword.field.template.json" + } + } + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/offers.sc-type.template.json b/src/storage/elasticsearch/templates/offers.sc-type.template.json new file mode 100644 index 00000000..5163b4d1 --- /dev/null +++ b/src/storage/elasticsearch/templates/offers.sc-type.template.json @@ -0,0 +1,24 @@ +{ + "properties": { + "price": { + "type": "double" + }, + "prices": { + "type": "nested", + "properties": { + "alumni": { + "type": "double" + }, + "student": { + "type": "double" + }, + "employee": { + "type": "double" + }, + "guest": { + "type": "double" + } + } + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/organization.sc-type.template.json b/src/storage/elasticsearch/templates/organization.sc-type.template.json new file mode 100644 index 00000000..93a16c76 --- /dev/null +++ b/src/storage/elasticsearch/templates/organization.sc-type.template.json @@ -0,0 +1,7 @@ +{ + "properties": { + "address": { + "_fieldRef": "address.field.template.json" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/person.sc-type.template.json b/src/storage/elasticsearch/templates/person.sc-type.template.json new file mode 100644 index 00000000..6e3f54d7 --- /dev/null +++ b/src/storage/elasticsearch/templates/person.sc-type.template.json @@ -0,0 +1,47 @@ +{ + "properties": { + "honorificPrefix": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "honorificSuffix": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "givenName": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "additionalName": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "familyName": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "birthDate": { + "_fieldRef": "filterableDate.field.template.json" + }, + "gender": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "email": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "faxNumber": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "telephone": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "adress": { + "type": "text" + }, + "affiliations": { + "type": "nested", + "_typeRef": "organization.sc-type.template.json" + }, + "homeLocation": { + "_typeRef": "place.sc-type.template.json" + }, + "nationality": { + "_fieldRef": "filterableKeyword.field.template.json" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/place.sc-type.template.json b/src/storage/elasticsearch/templates/place.sc-type.template.json new file mode 100644 index 00000000..0bca00f2 --- /dev/null +++ b/src/storage/elasticsearch/templates/place.sc-type.template.json @@ -0,0 +1,33 @@ +{ + "properties": { + "subType": { + "_fieldRef": "sortableKeyword.field.template.json" + }, + "address": { + "_fieldRef": "address.field.template.json" + }, + "openingHours": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "geo": { + "properties": { + "point": { + "properties": { + "coordinates": { + "type": "geo_point" + } + } + }, + "polygon": { + "type": "geo_shape" + } + } + }, + "superPlace": { + "_typeRef": "place.sc-type.template.json" + }, + "subProperties": { + "_fieldRef": "placeSubProperties.field.template.json" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/placeSubProperties.field.template.json b/src/storage/elasticsearch/templates/placeSubProperties.field.template.json new file mode 100644 index 00000000..f508e477 --- /dev/null +++ b/src/storage/elasticsearch/templates/placeSubProperties.field.template.json @@ -0,0 +1,38 @@ +{ + "properties": { + "floors": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "floor": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "paymentAccepted": { + "_fieldRef": "filterableKeyword.field.template.json" + }, + "roomCharacterization": { + "type": "nested", + "properties": { + "inventory": { + "properties": { + "key": { + "type": "keyword", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "value": { + "type": "integer", + "fields": { + "raw": { + "type": "integer" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/prices.field.template.json b/src/storage/elasticsearch/templates/prices.field.template.json new file mode 100644 index 00000000..e83dd296 --- /dev/null +++ b/src/storage/elasticsearch/templates/prices.field.template.json @@ -0,0 +1,14 @@ +{ + "type": "nested", + "properties": { + "student": { + "type": "float" + }, + "employee": { + "type": "float" + }, + "guest": { + "type": "float" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/sortableKeyword.field.template.json b/src/storage/elasticsearch/templates/sortableKeyword.field.template.json new file mode 100644 index 00000000..32834e6b --- /dev/null +++ b/src/storage/elasticsearch/templates/sortableKeyword.field.template.json @@ -0,0 +1,15 @@ +{ + "type": "text", + "analyzer": "search_german", + "fields": { + "raw": { + "type": "keyword", + "ignore_above": 10000 + }, + "sort": { + "fielddata": true, + "type": "text", + "analyzer": "ducet_sort" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/text.field.template.json b/src/storage/elasticsearch/templates/text.field.template.json new file mode 100644 index 00000000..6d4310c7 --- /dev/null +++ b/src/storage/elasticsearch/templates/text.field.template.json @@ -0,0 +1,10 @@ +{ + "type": "text", + "fielddata": true, + "analyzer": "search_german", + "fields": { + "raw": { + "type": "keyword" + } + } +} \ No newline at end of file diff --git a/src/storage/elasticsearch/templating.ts b/src/storage/elasticsearch/templating.ts new file mode 100644 index 00000000..11297aa7 --- /dev/null +++ b/src/storage/elasticsearch/templating.ts @@ -0,0 +1,145 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import {Client} from 'elasticsearch'; +import {readdir, readFile} from 'fs-extra'; +import { resolve } from 'path'; +import { logger } from '../../common'; + +/** + * Assembles an elasticsearch template with all resolved subType-references + * @param templateType + * @param templates + * @param inline + * @returns + */ +function assembleElasticsearchTemplate(templateType: string, templates: {[key: string]: any}, inline: number): any { + + const templateBase = JSON.parse(JSON.stringify(templates[templateType])); + + if (inline) { + delete templateBase.dynamic_templates; + } + + // these have no properties to replace + const excludeBaseFields = [ + 'filterableKeyword.field.template.json', + 'sortableKeyword.field.template.json', + 'text.field.template.json', + 'filterableDate.field.template.json', + ]; + + if (excludeBaseFields.indexOf(templateType) === -1) { + + try { + // extend the template by the properties of the basetemplate + templateBase.properties = Object.assign( + templateBase.properties, + templates['base.template.json'].mappings._default_.properties, + ); + } catch (e) { + logger.error('Failed to merge properties on: ' + templateType); + throw e; + } + const fieldKeys = Object.keys(templateBase.properties); + + fieldKeys.forEach((fieldKey) => { + + const field = templateBase.properties[fieldKey]; + const keys = Object.keys(field); + + // we have subtype-references to replace + if (keys.indexOf('_typeRef') > -1) { + // if we are already inline of a superObject, we don't resolve types + if (inline > 1) { + delete templateBase.properties[fieldKey]; + } else { + // we have more than one reference + if (Array.isArray(field._typeRef)) { + let obj = {}; + field._typeRef.forEach((subType: string) => { + obj = Object.assign(obj, assembleElasticsearchTemplate(subType, templates, inline + 1)); + }); + templateBase.properties[fieldKey] = obj; + } else { + templateBase.properties[fieldKey] = assembleElasticsearchTemplate(field._typeRef, templates, inline + 1); + } + } + } else if (keys.indexOf('_fieldRef') > -1) { + templateBase.properties[fieldKey] = assembleElasticsearchTemplate(field._fieldRef, templates, inline + 1); + } + }); + } + return templateBase; +} + +/** + * Reads all template files and returns the assembled template + */ +export async function getElasticsearchTemplate(): Promise { + + // readIM all templates + const elasticsearchFolder = resolve('.', 'src', 'storage', 'elasticsearch', 'templates'); + const templates: {[key: string]: any} = {}; + + const fileNames = await readdir(elasticsearchFolder); + + const availableTypes = fileNames.filter((fileName) => { + return Array.isArray(fileName.match(/\w*\.sc-type\.template\.json/i)); + }).map((fileName) => { + return fileName.substring(0, fileName.indexOf('.sc-type.template.json')); + }); + + const promises = fileNames.map(async (fileName) => { + const file = await readFile(resolve(elasticsearchFolder, fileName), 'utf8'); + + try { + templates[fileName] = JSON.parse(file.toString()); + } catch (jsonParsingError) { + logger.error('Failed parsing file: ' + fileName); + throw jsonParsingError; + } + }); + + await Promise.all(promises); + + const template = templates['base.template.json']; + + availableTypes.forEach((configType) => { + template.mappings[configType.toLowerCase()] = + assembleElasticsearchTemplate(configType + '.sc-type.template.json', templates, 0); + }); + + // this is like the base type (StappsCoreThing) + const baseProperties = template.mappings._default_.properties; + Object.keys(baseProperties).forEach((basePropertyName) => { + let field = baseProperties[basePropertyName]; + field = templates[field._fieldRef]; + template.mappings._default_.properties[basePropertyName] = field; + }); + + return template; +} + +/** + * Puts a new global template + * @param client + */ +export async function putTemplate(client: Client): Promise { + return client.indices.putTemplate({ + body: await getElasticsearchTemplate(), + name: 'global', + }); +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000..3c8c1bc0 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "./node_modules/@openstapps/configuration/tsconfig.json", + "exclude": [ + "./config/" + ] +} diff --git a/tslint.json b/tslint.json new file mode 100644 index 00000000..96870d13 --- /dev/null +++ b/tslint.json @@ -0,0 +1,3 @@ +{ + "extends": "./node_modules/@openstapps/configuration/tslint.json" +} \ No newline at end of file