From 14f96444fb5a8fea2b4b1f00de25302d260cc5c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9o=20-=20Le=20Filament?= <theo@le-filament.com>
Date: Thu, 10 Apr 2025 15:10:59 +0200
Subject: [PATCH] change: improve docker compose and CRON for backups

---
 tasks/main.yml                     | 41 ++++++++++++--
 templates/backup.yaml.j2           | 85 +++++++++++++++--------------
 templates/restore-odootest.yaml.j2 | 87 +++++++++++++++++-------------
 3 files changed, 132 insertions(+), 81 deletions(-)

diff --git a/tasks/main.yml b/tasks/main.yml
index 155fedb..0fc67f3 100644
--- a/tasks/main.yml
+++ b/tasks/main.yml
@@ -394,6 +394,7 @@
     template_backup_credentials: "{{ swift_odoo_credentials[template_backup_account.key] }}"
     template_odoo_instance: "{{ odoo_instance }}"
     template_database_name: "{{ template_odoo_instance.value.db }}"
+    template_source_database_name: "{{ hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].db }}"
     template_odoo_source_instance: "{{ {'key': template_odoo_instance.value.prod_instance | default(template_odoo_instance.key ), 'value': odoo_instances[template_odoo_instance.value.prod_instance | default(template_odoo_instance.key )]} }}"
   ansible.builtin.template:
     src: "restore-odootest.yaml.j2"
@@ -415,6 +416,25 @@
 # --------------------------------------------------
 # prod backup section
 # --------------------------------------------------
+- name: "Set empty lists to filtered instances"
+  tags:
+    - "odoo_backup"
+  set_fact:
+    odoo_instances_filtered: []
+
+- name: "Filter Odoo instance that needs backup"
+  tags:
+    - "odoo_backup"
+  set_fact:
+    odoo_instances_filtered: "{{ odoo_instances_filtered + [odoo_instance] }}"
+  loop: "{{ odoo_instances | dict2items }}"
+  loop_control:
+    label: "{{ odoo_instance.key }}"
+    loop_var: item
+  when: >
+    (odoo_instance.value.backup_host | default(inventory_hostname) == inventory_hostname)
+    and test_instance_is_prod
+
 - name: "Copy docker compose for backup"
   tags:
     - "odoo_backup"
@@ -431,7 +451,7 @@
     owner: "root"
     group: "root"
     mode: "0400"
-  loop: "{{ swift_odoo_accounts | dict2items | product(odoo_instances | dict2items) }}"
+  loop: "{{ swift_odoo_accounts | dict2items | product(odoo_instances_filtered) }}"
   loop_control:
     label: "account {{ template_backup_account.key }} on {{ template_odoo_instance.key }}"
     loop_var: item_account_instance
@@ -449,15 +469,28 @@
     item: "{{ item_account_instance.1 }}"
     template_backup_account: "{{ item_account_instance.0 }}"
     template_odoo_instance: "{{ odoo_instance }}"
+    template_backup_timestamp: >
+      {{
+        (
+          ('1970-01-01 ' + odoo_backup_config.time_start) | to_datetime
+        ).timestamp()
+        + ansible_loop.index0 * (
+          (
+            (odoo_backup_config.time_slot_duration | community.general.to_seconds)
+            - (odoo_backup_config.time_max_duration | community.general.to_seconds)
+          ) / (ansible_loop.length - 1)
+        )
+      }}
   ansible.builtin.cron:
     name: "backup {{ template_odoo_instance.key }}{{ template_backup_account.key }}"
-    minute: "{{ '%H' | strftime((('1970-01-01 ' + backup_time_start) | to_datetime).timestamp() + (swift_odoo_accounts | length - template_backup_account.key) * ((backup_time_slot_duration | community.general.to_seconds - swift_odoo_accounts | length * backup_time_max_duration | community.general.to_seconds) / (swift_odoo_accounts | length - 1) + backup_time_max_duration | community.general.to_seconds) | int) }}"
-    hour: "{{ '%H' | strftime((('1970-01-01 ' + backup_time_start) | to_datetime).timestamp() + (swift_odoo_accounts | length - template_backup_account.key) * ((backup_time_slot_duration | community.general.to_seconds - swift_odoo_accounts | length * backup_time_max_duration | community.general.to_seconds) / (swift_odoo_accounts | length - 1) + backup_time_max_duration | community.general.to_seconds) | int) }}"
+    minute: "{{ '%M' | strftime(template_backup_timestamp | int) }}"
+    hour: "{{ '%H' | strftime(template_backup_timestamp | int) }}"
     job: "/usr/bin/docker compose -f /home/docker/backups/backup-{{ template_odoo_instance.key }}{{ template_backup_account.key }}.yaml run --rm backup_odoo"
-  loop: "{{ swift_odoo_accounts | dict2items | product(odoo_instances | dict2items) }}"
+  loop: "{{ swift_odoo_accounts | dict2items | product(odoo_instances_filtered) }}"
   loop_control:
     label: "account {{ template_backup_account.key }} on {{ template_odoo_instance.key }}"
     loop_var: item_account_instance
+    extended: true
   when: >
     (odoo_instance.value.backup_host | default(inventory_hostname) == inventory_hostname)
     and (inventory_hostname in groups.maintenance_contract)
diff --git a/templates/backup.yaml.j2 b/templates/backup.yaml.j2
index 76ae5a0..e6fa30a 100644
--- a/templates/backup.yaml.j2
+++ b/templates/backup.yaml.j2
@@ -1,50 +1,53 @@
-version: "2.1"
 services:
-    backup_odoo:
-        image: ghcr.io/tecnativa/docker-duplicity-postgres:master
-        hostname: {{ inventory_hostname_short | lower | regex_replace('_','') }}-{{ template_odoo_instance.key }}
-        environment:
-            DB_VERSION: "{{ [template_odoo_instance_setup.postgres_version | split('.') | first | int, 10] | max }}"
-            DST: "swift://{{ template_odoo_instance.key }}_{{ inventory_hostname | lower }}"
-            PGDATABASE: "{{ template_odoo_instance.value.db }}"
-            PGUSER: "{{ template_odoo_instance.value.db_user }}"
-            PGPASSWORD: "{{ template_odoo_instance.value.db_pass }}"
-            PASSPHRASE: "{{ template_odoo_instance.value.odoo_backup_pass | default(template_odoo_instance.value.master_pass) }}"
-            SWIFT_USERNAME: "{{ template_backup_credentials.username }}"
-            SWIFT_PASSWORD: "{{ template_backup_credentials.password }}"
-            SWIFT_AUTHURL: "{{ template_backup_account.value.authurl }}"
-            SWIFT_AUTHVERSION: {{ template_backup_account.value.authversion }}
-            SWIFT_TENANTNAME: "{{ template_backup_account.value.tenantname }}"
-            SWIFT_TENANTID: "{{ template_backup_account.value.tenantid }}"
-            SWIFT_REGIONNAME: "{{ template_backup_account.value.regionname }}"
-            JOB_200_WHEN: "never"
-            JOB_300_WHAT: "pg_dump --no-owner --format c --file $$SRC/$$PGDATABASE.pgdump && backup --full-if-older-than 6D"
-            JOB_302_WHAT: "dup remove-all-but-n-full 5 --force $$DST $$@"
-            JOB_302_WHEN: "daily"
-        volumes:
-            - {{ template_odoo_instance.key }}{{ template_backup_account.key }}_backup_cache:/root/.cache/duplicity/:z
-            - {{ template_odoo_instance.key }}_filestore:/mnt/backup/src/odoo:z
+  backup_odoo:
+    image: sources.le-filament.com:5050/lefilament/duplicity_image/duplicity:latest
+    hostname: {{ inventory_hostname_short | lower | regex_replace('_','') }}-{{ template_odoo_instance.key }}
+    environment:
+      SRC: "/mnt/backup/src"
+      DST: "swift://{{ template_odoo_instance.key }}_{{ inventory_hostname | lower }}"
+      PGHOST: "db"
+      PGDATABASE: "{{ template_odoo_instance.value.db }}"
+      PGUSER: "{{ template_odoo_instance.value.db_user }}"
+      PGPASSWORD: "{{ template_odoo_instance.value.db_pass }}"
+      PASSPHRASE: "{{ template_odoo_instance.value.odoo_backup_pass | default(template_odoo_instance.value.master_pass) }}"
+      SWIFT_USERNAME: "{{ template_backup_credentials.username }}"
+      SWIFT_PASSWORD: "{{ template_backup_credentials.password }}"
+      SWIFT_AUTHURL: "{{ template_backup_account.value.authurl }}"
+      SWIFT_AUTHVERSION: {{ template_backup_account.value.authversion }}
+      SWIFT_TENANTNAME: "{{ template_backup_account.value.tenantname }}"
+      SWIFT_TENANTID: "{{ template_backup_account.value.tenantid }}"
+      SWIFT_REGIONNAME: "{{ template_backup_account.value.regionname }}"
+    volumes:
+      - {{ template_odoo_instance.key }}{{ template_backup_account.key }}_backup_cache:/root/.cache/duplicity/:rw
+      - {{ template_odoo_instance.key }}_filestore:/mnt/backup/src/odoo:ro
 {% if template_odoo_instance.value.metabase | default(false) %}
-            - {{ template_odoo_instance.key }}_metabase:/mnt/backup/src/metabase:z
+      - {{ template_odoo_instance.key }}_metabase:/mnt/backup/src/metabase:ro
 {% endif %}
-        networks:
-            - {{ template_odoo_instance.key }}_default
-            - public
-        command:
-            - /etc/periodic/daily/jobrunner
+    networks:
+      - {{ template_odoo_instance.key }}_default
+      - public
+    command: >
+      /bin/ash -c "
+      echo \"info: dumping '$$PGDATABASE' into '$$SRC/$$PGDATABASE.pgdump'...\"
+      && pg_dump --no-owner --format c --file $$SRC/$$PGDATABASE.pgdump
+      && echo \"info: backuping '$$SRC' to '$$DST'...\"
+      && duplicity backup --full-if-older-than 6D $$SRC $$DST
+      && echo \"info: removing all but 5 full backups in '$$DST'\"
+      && duplicity remove-all-but-n-full 5 --force $$DST
+      && echo \"info: done\""
 
 networks:
-    {{ template_odoo_instance.key }}_default:
-        external: true
-    public:
-        driver_opts:
-            encrypted: 1
+  {{ template_odoo_instance.key }}_default:
+    external: true
+  public:
+    driver_opts:
+      encrypted: 1
 
 volumes:
-    {{ template_odoo_instance.key }}{{ template_backup_account.key }}_backup_cache:
-    {{ template_odoo_instance.key }}_filestore:
-        external: true
+  {{ template_odoo_instance.key }}{{ template_backup_account.key }}_backup_cache:
+  {{ template_odoo_instance.key }}_filestore:
+    external: true
 {% if template_odoo_instance.value.metabase | default(false) %}
-    {{ template_odoo_instance.key }}_metabase:
-        external: true
+  {{ template_odoo_instance.key }}_metabase:
+    external: true
 {% endif %}
diff --git a/templates/restore-odootest.yaml.j2 b/templates/restore-odootest.yaml.j2
index f313781..8aacf4e 100644
--- a/templates/restore-odootest.yaml.j2
+++ b/templates/restore-odootest.yaml.j2
@@ -1,43 +1,58 @@
-version: "2.1"
 services:
-    restore_test:
-        image: ghcr.io/tecnativa/docker-duplicity-postgres:master
-        hostname: {{ template_odoo_instance.value.backup_host | default(inventory_hostname_short) | lower | regex_replace('_','') }}-odoo
-        networks:
-            - {{ template_odoo_instance.key }}_default
-            - public
-        volumes:
-            - backups_{{ template_odoo_instance.value.backup_instance }}{{ template_backup_account.key }}_backup_cache:/root/.cache/duplicity/:z
-            - {{ template_odoo_instance.key }}_filestore:/mnt/backup/src/odoo:z
-            - ./post_restore-{{ template_odoo_instance.key }}.sql:/tmp/post-restore.sql:ro
-            - ./pre_restore-{{ template_odoo_instance.key }}.sql:/tmp/pre-restore.sql:ro
-        environment:
-            PGDATABASE: "{{ template_database_name }}"
-            SWIFT_USERNAME: "{{ template_backup_credentials.username }}"
-            SWIFT_PASSWORD: "{{ template_backup_credentials.password }}"
-            SWIFT_AUTHURL: "{{ template_backup_account.value.authurl }}"
-            SWIFT_AUTHVERSION: {{ template_backup_account.value.authversion }}
-            SWIFT_TENANTNAME: "{{ template_backup_account.value.tenantname }}"
-            SWIFT_TENANTID: "{{ template_backup_account.value.tenantid }}"
-            SWIFT_REGIONNAME: "{{ template_backup_account.value.regionname }}"
-            OPTIONS: "--force"
-            DST: "swift://{{ template_odoo_instance.value.backup_instance }}_{{ template_odoo_instance.value.backup_host | default(inventory_hostname) | lower }}"
-            PGUSER: "{{ template_odoo_source_instance.value.db_user }}"
-            PGPASSWORD: "{{ template_odoo_source_instance.value.db_pass }}"
-            PASSPHRASE: "{{ hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].odoo_backup_pass | default(hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].master_pass) }}"
-        command: [sh, -c, "psql -a -f /tmp/pre-restore.sql postgres ; echo 'remove existing dir' && rm -rf /mnt/backup/src/odoo/filestore/$$PGDATABASE && restore && echo 'move repo to final dest' && mv /mnt/backup/src/odoo/filestore/{{ hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].db }} /mnt/backup/src/odoo/filestore/$$PGDATABASE && echo 'create database' && createdb -T template0 $$PGDATABASE && echo 'restore database' && pg_restore -d $$PGDATABASE $$SRC/{{ hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].db }}.pgdump ; psql -a -f /tmp/post-restore.sql $$PGDATABASE"]
+  restore_test:
+    image: sources.le-filament.com:5050/lefilament/duplicity_image/duplicity:latest
+    hostname: {{ template_odoo_instance.value.backup_host | default(inventory_hostname_short) | lower | regex_replace('_','') }}-odoo
+    networks:
+      - {{ template_odoo_instance.key }}_default
+      - public
+    volumes:
+      - backups_{{ template_odoo_instance.value.backup_instance }}{{ template_backup_account.key }}_backup_cache:/root/.cache/duplicity/:rw
+      - {{ template_odoo_instance.key }}_filestore:/mnt/backup/src/odoo:rw
+      - ./post_restore-{{ template_odoo_instance.key }}.sql:/tmp/post-restore.sql:ro
+      - ./pre_restore-{{ template_odoo_instance.key }}.sql:/tmp/pre-restore.sql:ro
+    environment:
+      SRC: "swift://{{ template_odoo_instance.value.backup_instance }}_{{ template_odoo_instance.value.backup_host | default(inventory_hostname) | lower }}"
+      DST: "/mnt/backup/src"
+      PGHOST: "db"
+      PGDATABASE: "{{ template_database_name }}"
+      PGUSER: "{{ template_odoo_source_instance.value.db_user }}"
+      PGPASSWORD: "{{ template_odoo_source_instance.value.db_pass }}"
+      PASSPHRASE: "{{ hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].odoo_backup_pass | default(hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].master_pass) }}"
+      SWIFT_USERNAME: "{{ template_backup_credentials.username }}"
+      SWIFT_PASSWORD: "{{ template_backup_credentials.password }}"
+      SWIFT_AUTHURL: "{{ template_backup_account.value.authurl }}"
+      SWIFT_AUTHVERSION: {{ template_backup_account.value.authversion }}
+      SWIFT_TENANTNAME: "{{ template_backup_account.value.tenantname }}"
+      SWIFT_TENANTID: "{{ template_backup_account.value.tenantid }}"
+      SWIFT_REGIONNAME: "{{ template_backup_account.value.regionname }}"
+    command: >
+      /bin/ash -c "
+      echo \"info: executing pre-restore SQL script...\"
+      && psql -a -f /tmp/pre-restore.sql postgres
+      && echo \"info: removing existing Odoo filestore of database '$$PGDATABASE'...\"
+      && rm -rf /mnt/backup/src/odoo/filestore/$$PGDATABASE
+      && echo \"info: restoring database dump and filestore backup...\"
+      && duplicity restore --force $$SRC $$DST
+      && mv /mnt/backup/src/odoo/filestore/{{ template_source_database_name }} /mnt/backup/src/odoo/filestore/$$PGDATABASE
+      && echo \"info: creating database '$$PGDATABASE'\"
+      && createdb -T template0 $$PGDATABASE
+      && echo \"info: restoring database '$$PGDATABASE' from dump '$$DST/{{ template_source_database_name }}.pgdump'\"
+      && pg_restore -d $$PGDATABASE $$DST/{{ template_source_database_name }}.pgdump
+      ; echo \"info: executing post-restore SQL script...\"
+      && psql -a -f /tmp/post-restore.sql $$PGDATABASE
+      && echo \"info: done\""
 
 networks:
-    {{ template_odoo_instance.key }}_default:
-        external: true
-    public:
-        driver_opts:
-            encrypted: 1
+  {{ template_odoo_instance.key }}_default:
+    external: true
+  public:
+    driver_opts:
+      encrypted: 1
 
 volumes:
-    backups_{{ template_odoo_instance.value.backup_instance }}{{ template_backup_account.key }}_backup_cache:
+  backups_{{ template_odoo_instance.value.backup_instance }}{{ template_backup_account.key }}_backup_cache:
 {% if template_odoo_instance.value.backup_host | default(inventory_hostname) == inventory_hostname %}
-        external: true
+    external: true
 {% endif %}
-    {{ template_odoo_instance.key }}_filestore:
-        external: true
+  {{ template_odoo_instance.key }}_filestore:
+    external: true
-- 
GitLab