[DLAB-4126] Upgrade mongo init

[DLAB-4126] Fix documentation

deployment/environments/group_vars/all/mongodb_vars.yml

@@ -5,8 +5,19 @@ mongodb:
     host: "mongo-vitamui-mongod.service.consul"
     mongod_port: 27017
     check_consul: 10 # in seconds
-    drop_info_log: false # Drop mongo (I)nformational log, for Verbosity Level of 0
+    drop_info_log: false # Drop mongo (I)nformational log, for Verbosity Level of 0    
+    versioning:
+        enable: true # Enable or not the versioning of the scripts.
+    included_scripts: # List of regexs allowing to determine which scripts will be included and applied on database.
+        - ".*"
+    excluded_scripts: # List of regexs allowing to determine which scripts will be excluded.
+        - ".*_dev.js.*"
+        - ".*_demo.js.*"
 
 mongod_replicaset_name: "shard{{ mongo_shard_id }}"
 
 mongod_client_connect_timeout_ms: 2000
+
+
+mongodb:
+  
\ No newline at end of file

deployment/environments/group_vars/all/vitamui_vars.yml

@@ -200,10 +200,3 @@ vitamui_platform_informations:
   theme_colors:
     vitamui_primary: "#ff4a54"
     vitamui_secondary: "#241f63"
-
-mongodb:
-  included_scripts:
-    - ".*"
-  excluded_scripts:
-    - ".*_dev.js.*"
-    - ".*_demo.js.*"

deployment/mongo.yml

@@ -6,7 +6,8 @@
         - mongo
         - { 
             role: mongo_init, 
-            mongod_source_template_dir: "{{playbook_dir}}/scripts/mongod/"
+            mongod_source_template_dir: "{{playbook_dir}}/scripts/mongod/",
+            tags: mongo_init
           }
         - mongo_configure
         - { role: mongo-express, when: "mongo_express is defined and mongo_express|lower == 'true'"}

deployment/roles/mongo_init/README.md

+Init of mongo
+=========
+
+Script allowing to apply scripts on database.
+Scripts are mutualized between dev's docker and deployment.
+
+All scripts are stored into the directory `{PROJECT_ROOT}/tools/docker/mongo/database_scripts/`.
+Each version of the application, embedding Mongo scripts, is represented by a folder.
+Into each version's folder, scripts are indexed according the following format: {index}_{name_of_the_script}.js(.j2).
+
+- **database_scripts**
+    - 0.0.0
+        - 1_init-database.js.j2
+        - 2_create-users.js.j2
+    - 1.0.0
+        - 1_init-security-context.js.j2
+        - 100_init-security-context_dev.js.j2
+
+The role creates an ordered list of scripts will be executed on database.
+For the sort , we use an option of the Unix command `sort`. The option `-V` allows to sort versions numbers.
+
+Once the list of eligibles scripts is set, we must version them (if the feature is enabled) before their execution.
+For this purpose, we inject the content of the script into the template `versionned_script.js.j2`. 
+According to this script, we check by its filename if the script has already been executed. If not, the original script is executed and an entry is added into the collection **changelog**.
+
+Requirements
+------------
+
+- Mongo database must be deployed and ready.
+- The following variables must be defined:
+    - **mongod_source_template_dir** (source: playbook): Path where scripts are located.
+    - **mongodb.versioning.enable** (source: mongodb_vars.yml): Boolean indicating if a script must be versioned (i.n its execution is recorded into a database. On the next execution of this role, this script will be ignored because it will have already been executed.)
+    - **mongodb.included_scripts** (source: mongodb_vars.yml): List of regexs allowing to determine which scripts will be included and applied on database. By default, we accept all.
+    - **mongodb.excluded_scripts** (source: mongodb_vars.yml): List of regexs allowing to determine which scripts will be excluded. By default, dev and demo scripts are excluded.
+
+- The following variable(s) can be defined:
+    - **mock_insert_data** (source: extra-vars): Flag used in a versioning context, scripts won't be executed. It can be used for a data recovery - the collection *changelog* will be provided with scripts' information.
+    - **mongodb.docker.enable** (source: mongodb_vars.yml): Boolean indicating if we use Docker for the database. In this case, a volume must be mounted on the output directory **mongod_output_dir_entry_point**.
+    - **mongodb.docker.image_name** (source: mongodb_vars.yml): Name of the docker image.
+    - **mongodb.docker.internal_dir** (source: mongodb_vars.yml): Internal path into the container where the host's directory **mongod_output_dir_entry_point** is mapped.
+
+Author Information
+------------------
+Projet VITAMUI

deployment/roles/mongo_init/tasks/check_auth.yml

 ---
 
 - name: Check if authent is enabled
-  command: "mongo --host {{ mongod_uri }}/admin -u {{ mongodb.localadmin.user }} -p {{ mongodb.localadmin.password }} --quiet --eval 'db.help()'"
+  command: "mongo --host {{ mongod_uri }}/admin -u {{ mongodb.admin.user }} -p {{ mongodb.admin.password }} --quiet --eval 'db.help()'"
   register: mongo_authent_enabled
   failed_when: false
   no_log: "{{ hide_passwords_during_deploy }}"
@@ -27,7 +27,7 @@
 # When authentication is required, we set mongodb admin credentials
 - name: Set mongodb authentication credentials
   set_fact:
-    mongo_credentials: " -u {{ mongodb.localadmin.user }} -p {{ mongodb.localadmin.password }} --authenticationDatabase {{ mongodb.localadmin.db }} "
+    mongo_credentials: " -u {{ mongodb.admin.user }} -p {{ mongodb.admin.password }} --authenticationDatabase {{ mongodb.admin.db }} "
   when: "mongo_authent_enabled.rc == 0"
   no_log: "{{ hide_passwords_during_deploy }}"
 

deployment/roles/mongo_init/tasks/execute_script.yml

@@ -5,61 +5,46 @@
 
 - name: 
   debug:
-    msg: ">>>> Execution of the file {{ mongo_file.finalname }} <<<<"
+    msg: ">>>> Execution of the file {{ mongo_file.finalname }}<<<<"
 
 - name: Check if the script exists
   stat:
     path: "{{ mongod_output_dir_entry_point }}/{{ mongo_file.finalname }}"
   register: stat_result
 
-- fail: msg="The file '{{ mongo_file.finalname }}' is not exist"
-  when: not stat_result.stat.exists
-
 - block:
+    - fail: msg="The file '{{ mongo_file.finalname }}' is not exist"
+      when: not stat_result.stat.exists
+
+    - name: Compute versionned script files
+      template:
+        src: "versionned_script.js.j2"
+        dest: "{{ mongod_output_dir_entry_point }}/versionned_{{ mongo_file.finalname }}"
+        owner: "{{ vitamui_defaults.users.vitamuidb }}"
+        group: "{{ vitamui_defaults.users.group }}"
+        mode: 0755
+
+    - name: Insert original script into versionned script.
+      blockinfile:
+        path: "{{ mongod_output_dir_entry_point }}/versionned_{{ mongo_file.finalname }}"
+        insertafter: "// Insert data"
+        marker: "// {mark} of the script"
+        block: |
+          {{lookup('file', '{{ mongod_output_dir_entry_point }}/{{ mongo_file.finalname }}') }}
+      when: mock_insert_data is not defined
 
-  - name: Check if the script has already been executed
-    shell: "mongo \"mongodb://{{ mongod_uri }}/admin\" {{ mongo_credentials }} --quiet --eval \"db.changelog.find({filename: '{{ mongo_file.finalname }}', checksum: '{{ stat_result.stat.checksum}}'});\""
-    no_log: "{{ hide_passwords_during_deploy }}"
-    register: mongo_versionning_result
-
-  - name: Debug mongo return
-    debug:
-      msg: "{{ mongo_versionning_result.stdout }}" 
-
-  - block:
-      - name: Load script in database
-        shell: "mongo \"mongodb://{{ mongod_uri }}/admin\" {{ mongo_credentials }} {{ mongod_output_dir_entry_point }}/{{ mongo_file.finalname }}"
-        no_log: "{{ hide_passwords_during_deploy }}"
-        when: mock_insert_data is not defined
-      
-      - name: Update changelog
-        shell: "mongo \"mongodb://{{ mongod_uri }}/versioning\" {{ mongo_credentials }} --eval \"db.changelog.insertOne({filename: '{{ mongo_file.finalname }}', date: new Date(), version: '{{ mongo_file.version }}', checksum: '{{ stat_result.stat.checksum}}'});\""
-        no_log: "{{ hide_passwords_during_deploy }}"
-    when: 
-      - "'_id' not in mongo_versionning_result.stdout"
+    - name: Update finalname script
+      set_fact:
+        mongo_file: "{{ mongo_file | combine( { 'finalname': 'versionned_{{ mongo_file.finalname }}' } ) }}"
 
+  when: mongodb.versioning is defined and mongodb.versioning.enable
+    
+- name: Load script in database
+  shell: "mongo \"mongodb://{{ mongod_uri }}/admin\" {{ mongo_credentials }} {{ mongod_output_dir_entry_point }}/{{ mongo_file.finalname }}"
+  no_log: "{{ hide_passwords_during_deploy }}"
   when: mongodb.docker is not defined or not mongodb.docker.enable
 
-- block:
-  - name: Check if the script has already been executed (docker)
-    shell: "docker exec --tty vitamui-mongo /bin/bash -c \"mongo \\\"mongodb://{{ mongod_uri }}/versioning\\\" {{ mongo_credentials }} --quiet --eval \\\"db.changelog.find({filename: '{{ mongo_file.finalname }}', checksum: '{{ stat_result.stat.checksum}}'});\\\"\""
-    no_log: "{{ hide_passwords_during_deploy }}"
-    register: mongo_versionning_result
-
-  - name: Debug mongo return (Docker)
-    debug:
-      msg: "{{ mongo_versionning_result.stdout }}" 
-
-  - block:
-    - name: Load script in database (docker)
-      command: "docker exec --tty vitamui-mongo /bin/bash -c \"mongo \\\"mongodb://{{ mongod_uri }}/admin\\\" {{ mongo_credentials }} {{ mongodb.docker.internal_dir}}/app/mongod/{{ mongo_file.finalname }}\""
-      no_log: "{{ hide_passwords_during_deploy }}"
-      when: mock_insert_data is not defined
-
-    - name: Update changelog (docker)
-      shell: "docker exec --tty vitamui-mongo /bin/bash -c \"mongo \\\"mongodb://{{ mongod_uri }}/versioning\\\" {{ mongo_credentials }} --quiet --eval \\\"db.changelog.insertOne({filename: '{{ mongo_file.finalname }}', date: new Date(), version: '{{ mongo_file.version }}', checksum: '{{ stat_result.stat.checksum}}'});\\\"\""
-      no_log: "{{ hide_passwords_during_deploy }}"
-
-    when: 
-      - "'_id' not in mongo_versionning_result.stdout"
+- name: Load script in database (docker)
+  command: "docker exec --tty {{ mongodb.docker.image_name }} /bin/bash -c \"mongo \\\"mongodb://{{ mongod_uri }}/admin\\\" {{ mongo_credentials }} {{ mongodb.docker.internal_dir}}/app/mongod/{{ mongo_file.finalname }}\""
+  no_log: "{{ hide_passwords_during_deploy }}"
   when: mongodb.docker is defined and mongodb.docker.enable

deployment/roles/mongo_init/tasks/main.yml

@@ -82,4 +82,4 @@
   include_tasks: "execute_script.yml"
   loop: "{{ mongod_eligible_files | unique }}"
   loop_control:
-    loop_var: mongo_file
+    loop_var: mongo_file
\ No newline at end of file

deployment/roles/mongo_init/templates/versionned_script.js.j2

+versioningDb = db.getSiblingDB('versioning')
+
+var commitChangelog = versioningDb.changelog.find({filename: '{{ mongo_file.finalname }}'}).toArray();
+
+if (commitChangelog.length  == 0) {
+
+    // Insert data
+
+    // Insert changelog
+    versioningDb.changelog.insertOne({filename: '{{ mongo_file.finalname }}', date: new Date(), version: '{{ mongo_file.version }}', checksum: '{{ stat_result.stat.checksum}}'});
+}
\ No newline at end of file

tools/docker/mongo/README.md

@@ -96,7 +96,7 @@ Afin d'injecter les bonnes valeurs aux variables des templates, le fichier **mon
 Afin d'intégrer une surcharge extérieure des variables par défaut, il est possible de renseigner le chemin d'un autre fichier de variable à travers la variable d'environnements **ADDITIONNAL_VITAMUI_CONFIG_FILE**.
 Attention, toute variable définie dans ce fichier additionnel écrasera la valeur existante.
 
-## Versionning des scripts Mongo
+## Versioning des scripts Mongo
 
 Avant l'exécution d'un script, on vérifie que ce dernier a été exécuté. Pour ce faire, nous nous basons sur deux éléments:
 - le nom du fichier

tools/docker/mongo/mongo_vars_dev.yml

@@ -19,6 +19,7 @@ mongodb:
     - "nothing"
   docker:
     enable: true
+    image_name: vitamui-mongo
     internal_dir: /scripts/mongo/data
   mongod_port: 27018
   passphrase: mongogo
@@ -48,6 +49,7 @@ mongodb:
     password: "mongod_dbpwd_cas"
     roles: '[{ role: "readWrite", db: "cas" }]'
   versioning:
+    enable: true
     db: versioning
     user: "mongod_dbuser_versioning"
     password: "mongod_dbpwd_versioning"