From 87c3cae5e6b99ddd4c790507651f487c50a1b969 Mon Sep 17 00:00:00 2001
From: Mathieu Leguay <mathieu.leguay@teamdlab.com>
Date: Wed, 20 May 2020 15:20:17 +0200
Subject: [PATCH] [DLAB-4126] Upgrade mongo init
[DLAB-4126] Fix documentation
---
.../group_vars/all/mongodb_vars.yml | 13 +++-
.../group_vars/all/vitamui_vars.yml | 7 --
deployment/mongo.yml | 3 +-
deployment/roles/mongo_init/README.md | 44 +++++++++++
.../roles/mongo_init/tasks/check_auth.yml | 4 +-
.../roles/mongo_init/tasks/execute_script.yml | 77 ++++++++-----------
deployment/roles/mongo_init/tasks/main.yml | 2 +-
.../templates/versionned_script.js.j2 | 11 +++
tools/docker/mongo/README.md | 2 +-
tools/docker/mongo/mongo_vars_dev.yml | 2 +
10 files changed, 106 insertions(+), 59 deletions(-)
create mode 100644 deployment/roles/mongo_init/README.md
create mode 100644 deployment/roles/mongo_init/templates/versionned_script.js.j2
diff --git a/deployment/environments/group_vars/all/mongodb_vars.yml b/deployment/environments/group_vars/all/mongodb_vars.yml
index 799fc1eb..925b6c1f 100755
--- a/deployment/environments/group_vars/all/mongodb_vars.yml
+++ b/deployment/environments/group_vars/all/mongodb_vars.yml
@@ -5,8 +5,19 @@ mongodb:
host: "mongo-vitamui-mongod.service.consul"
mongod_port: 27017
check_consul: 10 # in seconds
- drop_info_log: false # Drop mongo (I)nformational log, for Verbosity Level of 0
+ drop_info_log: false # Drop mongo (I)nformational log, for Verbosity Level of 0
+ versioning:
+ enable: true # Enable or not the versioning of the scripts.
+ included_scripts: # List of regexs allowing to determine which scripts will be included and applied on database.
+ - ".*"
+ excluded_scripts: # List of regexs allowing to determine which scripts will be excluded.
+ - ".*_dev.js.*"
+ - ".*_demo.js.*"
mongod_replicaset_name: "shard{{ mongo_shard_id }}"
mongod_client_connect_timeout_ms: 2000
+
+
+mongodb:
+
\ No newline at end of file
diff --git a/deployment/environments/group_vars/all/vitamui_vars.yml b/deployment/environments/group_vars/all/vitamui_vars.yml
index 612837c1..82fc596a 100755
--- a/deployment/environments/group_vars/all/vitamui_vars.yml
+++ b/deployment/environments/group_vars/all/vitamui_vars.yml
@@ -200,10 +200,3 @@ vitamui_platform_informations:
theme_colors:
vitamui_primary: "#ff4a54"
vitamui_secondary: "#241f63"
-
-mongodb:
- included_scripts:
- - ".*"
- excluded_scripts:
- - ".*_dev.js.*"
- - ".*_demo.js.*"
diff --git a/deployment/mongo.yml b/deployment/mongo.yml
index 5fb651fb..6b2cdf36 100644
--- a/deployment/mongo.yml
+++ b/deployment/mongo.yml
@@ -6,7 +6,8 @@
- mongo
- {
role: mongo_init,
- mongod_source_template_dir: "{{playbook_dir}}/scripts/mongod/"
+ mongod_source_template_dir: "{{playbook_dir}}/scripts/mongod/",
+ tags: mongo_init
}
- mongo_configure
- { role: mongo-express, when: "mongo_express is defined and mongo_express|lower == 'true'"}
diff --git a/deployment/roles/mongo_init/README.md b/deployment/roles/mongo_init/README.md
new file mode 100644
index 00000000..bded25cc
--- /dev/null
+++ b/deployment/roles/mongo_init/README.md
@@ -0,0 +1,44 @@
+Init of mongo
+=========
+
+Script allowing to apply scripts on database.
+Scripts are mutualized between dev's docker and deployment.
+
+All scripts are stored into the directory `{PROJECT_ROOT}/tools/docker/mongo/database_scripts/`.
+Each version of the application, embedding Mongo scripts, is represented by a folder.
+Into each version's folder, scripts are indexed according the following format: {index}_{name_of_the_script}.js(.j2).
+
+- **database_scripts**
+ - 0.0.0
+ - 1_init-database.js.j2
+ - 2_create-users.js.j2
+ - 1.0.0
+ - 1_init-security-context.js.j2
+ - 100_init-security-context_dev.js.j2
+
+The role creates an ordered list of scripts will be executed on database.
+For the sort , we use an option of the Unix command `sort`. The option `-V` allows to sort versions numbers.
+
+Once the list of eligibles scripts is set, we must version them (if the feature is enabled) before their execution.
+For this purpose, we inject the content of the script into the template `versionned_script.js.j2`.
+According to this script, we check by its filename if the script has already been executed. If not, the original script is executed and an entry is added into the collection **changelog**.
+
+Requirements
+------------
+
+- Mongo database must be deployed and ready.
+- The following variables must be defined:
+ - **mongod_source_template_dir** (source: playbook): Path where scripts are located.
+ - **mongodb.versioning.enable** (source: mongodb_vars.yml): Boolean indicating if a script must be versioned (i.n its execution is recorded into a database. On the next execution of this role, this script will be ignored because it will have already been executed.)
+ - **mongodb.included_scripts** (source: mongodb_vars.yml): List of regexs allowing to determine which scripts will be included and applied on database. By default, we accept all.
+ - **mongodb.excluded_scripts** (source: mongodb_vars.yml): List of regexs allowing to determine which scripts will be excluded. By default, dev and demo scripts are excluded.
+
+- The following variable(s) can be defined:
+ - **mock_insert_data** (source: extra-vars): Flag used in a versioning context, scripts won't be executed. It can be used for a data recovery - the collection *changelog* will be provided with scripts' information.
+ - **mongodb.docker.enable** (source: mongodb_vars.yml): Boolean indicating if we use Docker for the database. In this case, a volume must be mounted on the output directory **mongod_output_dir_entry_point**.
+ - **mongodb.docker.image_name** (source: mongodb_vars.yml): Name of the docker image.
+ - **mongodb.docker.internal_dir** (source: mongodb_vars.yml): Internal path into the container where the host's directory **mongod_output_dir_entry_point** is mapped.
+
+Author Information
+------------------
+Projet VITAMUI
diff --git a/deployment/roles/mongo_init/tasks/check_auth.yml b/deployment/roles/mongo_init/tasks/check_auth.yml
index 2afac0e3..f5486f98 100644
--- a/deployment/roles/mongo_init/tasks/check_auth.yml
+++ b/deployment/roles/mongo_init/tasks/check_auth.yml
@@ -1,7 +1,7 @@
---
- name: Check if authent is enabled
- command: "mongo --host {{ mongod_uri }}/admin -u {{ mongodb.localadmin.user }} -p {{ mongodb.localadmin.password }} --quiet --eval 'db.help()'"
+ command: "mongo --host {{ mongod_uri }}/admin -u {{ mongodb.admin.user }} -p {{ mongodb.admin.password }} --quiet --eval 'db.help()'"
register: mongo_authent_enabled
failed_when: false
no_log: "{{ hide_passwords_during_deploy }}"
@@ -27,7 +27,7 @@
# When authentication is required, we set mongodb admin credentials
- name: Set mongodb authentication credentials
set_fact:
- mongo_credentials: " -u {{ mongodb.localadmin.user }} -p {{ mongodb.localadmin.password }} --authenticationDatabase {{ mongodb.localadmin.db }} "
+ mongo_credentials: " -u {{ mongodb.admin.user }} -p {{ mongodb.admin.password }} --authenticationDatabase {{ mongodb.admin.db }} "
when: "mongo_authent_enabled.rc == 0"
no_log: "{{ hide_passwords_during_deploy }}"
diff --git a/deployment/roles/mongo_init/tasks/execute_script.yml b/deployment/roles/mongo_init/tasks/execute_script.yml
index 03ec476f..c2ecfceb 100644
--- a/deployment/roles/mongo_init/tasks/execute_script.yml
+++ b/deployment/roles/mongo_init/tasks/execute_script.yml
@@ -5,61 +5,46 @@
- name:
debug:
- msg: ">>>> Execution of the file {{ mongo_file.finalname }} <<<<"
+ msg: ">>>> Execution of the file {{ mongo_file.finalname }}<<<<"
- name: Check if the script exists
stat:
path: "{{ mongod_output_dir_entry_point }}/{{ mongo_file.finalname }}"
register: stat_result
-- fail: msg="The file '{{ mongo_file.finalname }}' is not exist"
- when: not stat_result.stat.exists
-
- block:
+ - fail: msg="The file '{{ mongo_file.finalname }}' is not exist"
+ when: not stat_result.stat.exists
+
+ - name: Compute versionned script files
+ template:
+ src: "versionned_script.js.j2"
+ dest: "{{ mongod_output_dir_entry_point }}/versionned_{{ mongo_file.finalname }}"
+ owner: "{{ vitamui_defaults.users.vitamuidb }}"
+ group: "{{ vitamui_defaults.users.group }}"
+ mode: 0755
+
+ - name: Insert original script into versionned script.
+ blockinfile:
+ path: "{{ mongod_output_dir_entry_point }}/versionned_{{ mongo_file.finalname }}"
+ insertafter: "// Insert data"
+ marker: "// {mark} of the script"
+ block: |
+ {{lookup('file', '{{ mongod_output_dir_entry_point }}/{{ mongo_file.finalname }}') }}
+ when: mock_insert_data is not defined
- - name: Check if the script has already been executed
- shell: "mongo \"mongodb://{{ mongod_uri }}/admin\" {{ mongo_credentials }} --quiet --eval \"db.changelog.find({filename: '{{ mongo_file.finalname }}', checksum: '{{ stat_result.stat.checksum}}'});\""
- no_log: "{{ hide_passwords_during_deploy }}"
- register: mongo_versionning_result
-
- - name: Debug mongo return
- debug:
- msg: "{{ mongo_versionning_result.stdout }}"
-
- - block:
- - name: Load script in database
- shell: "mongo \"mongodb://{{ mongod_uri }}/admin\" {{ mongo_credentials }} {{ mongod_output_dir_entry_point }}/{{ mongo_file.finalname }}"
- no_log: "{{ hide_passwords_during_deploy }}"
- when: mock_insert_data is not defined
-
- - name: Update changelog
- shell: "mongo \"mongodb://{{ mongod_uri }}/versioning\" {{ mongo_credentials }} --eval \"db.changelog.insertOne({filename: '{{ mongo_file.finalname }}', date: new Date(), version: '{{ mongo_file.version }}', checksum: '{{ stat_result.stat.checksum}}'});\""
- no_log: "{{ hide_passwords_during_deploy }}"
- when:
- - "'_id' not in mongo_versionning_result.stdout"
+ - name: Update finalname script
+ set_fact:
+ mongo_file: "{{ mongo_file | combine( { 'finalname': 'versionned_{{ mongo_file.finalname }}' } ) }}"
+ when: mongodb.versioning is defined and mongodb.versioning.enable
+
+- name: Load script in database
+ shell: "mongo \"mongodb://{{ mongod_uri }}/admin\" {{ mongo_credentials }} {{ mongod_output_dir_entry_point }}/{{ mongo_file.finalname }}"
+ no_log: "{{ hide_passwords_during_deploy }}"
when: mongodb.docker is not defined or not mongodb.docker.enable
-- block:
- - name: Check if the script has already been executed (docker)
- shell: "docker exec --tty vitamui-mongo /bin/bash -c \"mongo \\\"mongodb://{{ mongod_uri }}/versioning\\\" {{ mongo_credentials }} --quiet --eval \\\"db.changelog.find({filename: '{{ mongo_file.finalname }}', checksum: '{{ stat_result.stat.checksum}}'});\\\"\""
- no_log: "{{ hide_passwords_during_deploy }}"
- register: mongo_versionning_result
-
- - name: Debug mongo return (Docker)
- debug:
- msg: "{{ mongo_versionning_result.stdout }}"
-
- - block:
- - name: Load script in database (docker)
- command: "docker exec --tty vitamui-mongo /bin/bash -c \"mongo \\\"mongodb://{{ mongod_uri }}/admin\\\" {{ mongo_credentials }} {{ mongodb.docker.internal_dir}}/app/mongod/{{ mongo_file.finalname }}\""
- no_log: "{{ hide_passwords_during_deploy }}"
- when: mock_insert_data is not defined
-
- - name: Update changelog (docker)
- shell: "docker exec --tty vitamui-mongo /bin/bash -c \"mongo \\\"mongodb://{{ mongod_uri }}/versioning\\\" {{ mongo_credentials }} --quiet --eval \\\"db.changelog.insertOne({filename: '{{ mongo_file.finalname }}', date: new Date(), version: '{{ mongo_file.version }}', checksum: '{{ stat_result.stat.checksum}}'});\\\"\""
- no_log: "{{ hide_passwords_during_deploy }}"
-
- when:
- - "'_id' not in mongo_versionning_result.stdout"
+- name: Load script in database (docker)
+ command: "docker exec --tty {{ mongodb.docker.image_name }} /bin/bash -c \"mongo \\\"mongodb://{{ mongod_uri }}/admin\\\" {{ mongo_credentials }} {{ mongodb.docker.internal_dir}}/app/mongod/{{ mongo_file.finalname }}\""
+ no_log: "{{ hide_passwords_during_deploy }}"
when: mongodb.docker is defined and mongodb.docker.enable
diff --git a/deployment/roles/mongo_init/tasks/main.yml b/deployment/roles/mongo_init/tasks/main.yml
index 36d383de..62280541 100644
--- a/deployment/roles/mongo_init/tasks/main.yml
+++ b/deployment/roles/mongo_init/tasks/main.yml
@@ -82,4 +82,4 @@
include_tasks: "execute_script.yml"
loop: "{{ mongod_eligible_files | unique }}"
loop_control:
- loop_var: mongo_file
\ No newline at end of file
+ loop_var: mongo_file
diff --git a/deployment/roles/mongo_init/templates/versionned_script.js.j2 b/deployment/roles/mongo_init/templates/versionned_script.js.j2
new file mode 100644
index 00000000..6d41a907
--- /dev/null
+++ b/deployment/roles/mongo_init/templates/versionned_script.js.j2
@@ -0,0 +1,11 @@
+versioningDb = db.getSiblingDB('versioning')
+
+var commitChangelog = versioningDb.changelog.find({filename: '{{ mongo_file.finalname }}'}).toArray();
+
+if (commitChangelog.length == 0) {
+
+ // Insert data
+
+ // Insert changelog
+ versioningDb.changelog.insertOne({filename: '{{ mongo_file.finalname }}', date: new Date(), version: '{{ mongo_file.version }}', checksum: '{{ stat_result.stat.checksum}}'});
+}
\ No newline at end of file
diff --git a/tools/docker/mongo/README.md b/tools/docker/mongo/README.md
index eac4b8ed..a6e256fa 100644
--- a/tools/docker/mongo/README.md
+++ b/tools/docker/mongo/README.md
@@ -96,7 +96,7 @@ Afin d'injecter les bonnes valeurs aux variables des templates, le fichier **mon
Afin d'intégrer une surcharge extérieure des variables par défaut, il est possible de renseigner le chemin d'un autre fichier de variable à travers la variable d'environnements **ADDITIONNAL_VITAMUI_CONFIG_FILE**.
Attention, toute variable définie dans ce fichier additionnel écrasera la valeur existante.
-## Versionning des scripts Mongo
+## Versioning des scripts Mongo
Avant l'exécution d'un script, on vérifie que ce dernier a été exécuté. Pour ce faire, nous nous basons sur deux éléments:
- le nom du fichier
diff --git a/tools/docker/mongo/mongo_vars_dev.yml b/tools/docker/mongo/mongo_vars_dev.yml
index 3d25358a..73328188 100644
--- a/tools/docker/mongo/mongo_vars_dev.yml
+++ b/tools/docker/mongo/mongo_vars_dev.yml
@@ -19,6 +19,7 @@ mongodb:
- "nothing"
docker:
enable: true
+ image_name: vitamui-mongo
internal_dir: /scripts/mongo/data
mongod_port: 27018
passphrase: mongogo
@@ -48,6 +49,7 @@ mongodb:
password: "mongod_dbpwd_cas"
roles: '[{ role: "readWrite", db: "cas" }]'
versioning:
+ enable: true
db: versioning
user: "mongod_dbuser_versioning"
password: "mongod_dbpwd_versioning"
--
GitLab