diff --git a/tasks/main.yml b/tasks/main.yml index 155fedb948818bb62e1164b089db7ea67da28c00..0fc67f3e8459ed90de11039753977357cada8dae 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -394,6 +394,7 @@ template_backup_credentials: "{{ swift_odoo_credentials[template_backup_account.key] }}" template_odoo_instance: "{{ odoo_instance }}" template_database_name: "{{ template_odoo_instance.value.db }}" + template_source_database_name: "{{ hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].db }}" template_odoo_source_instance: "{{ {'key': template_odoo_instance.value.prod_instance | default(template_odoo_instance.key ), 'value': odoo_instances[template_odoo_instance.value.prod_instance | default(template_odoo_instance.key )]} }}" ansible.builtin.template: src: "restore-odootest.yaml.j2" @@ -415,6 +416,25 @@ # -------------------------------------------------- # prod backup section # -------------------------------------------------- +- name: "Set empty lists to filtered instances" + tags: + - "odoo_backup" + set_fact: + odoo_instances_filtered: [] + +- name: "Filter Odoo instance that needs backup" + tags: + - "odoo_backup" + set_fact: + odoo_instances_filtered: "{{ odoo_instances_filtered + [odoo_instance] }}" + loop: "{{ odoo_instances | dict2items }}" + loop_control: + label: "{{ odoo_instance.key }}" + loop_var: item + when: > + (odoo_instance.value.backup_host | default(inventory_hostname) == inventory_hostname) + and test_instance_is_prod + - name: "Copy docker compose for backup" tags: - "odoo_backup" @@ -431,7 +451,7 @@ owner: "root" group: "root" mode: "0400" - loop: "{{ swift_odoo_accounts | dict2items | product(odoo_instances | dict2items) }}" + loop: "{{ swift_odoo_accounts | dict2items | product(odoo_instances_filtered) }}" loop_control: label: "account {{ template_backup_account.key }} on {{ template_odoo_instance.key }}" loop_var: item_account_instance @@ -449,15 +469,28 @@ item: "{{ item_account_instance.1 }}" template_backup_account: "{{ item_account_instance.0 }}" template_odoo_instance: "{{ odoo_instance }}" + template_backup_timestamp: > + {{ + ( + ('1970-01-01 ' + odoo_backup_config.time_start) | to_datetime + ).timestamp() + + ansible_loop.index0 * ( + ( + (odoo_backup_config.time_slot_duration | community.general.to_seconds) + - (odoo_backup_config.time_max_duration | community.general.to_seconds) + ) / (ansible_loop.length - 1) + ) + }} ansible.builtin.cron: name: "backup {{ template_odoo_instance.key }}{{ template_backup_account.key }}" - minute: "{{ '%H' | strftime((('1970-01-01 ' + backup_time_start) | to_datetime).timestamp() + (swift_odoo_accounts | length - template_backup_account.key) * ((backup_time_slot_duration | community.general.to_seconds - swift_odoo_accounts | length * backup_time_max_duration | community.general.to_seconds) / (swift_odoo_accounts | length - 1) + backup_time_max_duration | community.general.to_seconds) | int) }}" - hour: "{{ '%H' | strftime((('1970-01-01 ' + backup_time_start) | to_datetime).timestamp() + (swift_odoo_accounts | length - template_backup_account.key) * ((backup_time_slot_duration | community.general.to_seconds - swift_odoo_accounts | length * backup_time_max_duration | community.general.to_seconds) / (swift_odoo_accounts | length - 1) + backup_time_max_duration | community.general.to_seconds) | int) }}" + minute: "{{ '%M' | strftime(template_backup_timestamp | int) }}" + hour: "{{ '%H' | strftime(template_backup_timestamp | int) }}" job: "/usr/bin/docker compose -f /home/docker/backups/backup-{{ template_odoo_instance.key }}{{ template_backup_account.key }}.yaml run --rm backup_odoo" - loop: "{{ swift_odoo_accounts | dict2items | product(odoo_instances | dict2items) }}" + loop: "{{ swift_odoo_accounts | dict2items | product(odoo_instances_filtered) }}" loop_control: label: "account {{ template_backup_account.key }} on {{ template_odoo_instance.key }}" loop_var: item_account_instance + extended: true when: > (odoo_instance.value.backup_host | default(inventory_hostname) == inventory_hostname) and (inventory_hostname in groups.maintenance_contract) diff --git a/templates/backup.yaml.j2 b/templates/backup.yaml.j2 index 76ae5a00e79021cb5de66e07021011e175228e5b..e6fa30ac4450f1595b2f4882090b0b6f11bb5ae6 100644 --- a/templates/backup.yaml.j2 +++ b/templates/backup.yaml.j2 @@ -1,50 +1,53 @@ -version: "2.1" services: - backup_odoo: - image: ghcr.io/tecnativa/docker-duplicity-postgres:master - hostname: {{ inventory_hostname_short | lower | regex_replace('_','') }}-{{ template_odoo_instance.key }} - environment: - DB_VERSION: "{{ [template_odoo_instance_setup.postgres_version | split('.') | first | int, 10] | max }}" - DST: "swift://{{ template_odoo_instance.key }}_{{ inventory_hostname | lower }}" - PGDATABASE: "{{ template_odoo_instance.value.db }}" - PGUSER: "{{ template_odoo_instance.value.db_user }}" - PGPASSWORD: "{{ template_odoo_instance.value.db_pass }}" - PASSPHRASE: "{{ template_odoo_instance.value.odoo_backup_pass | default(template_odoo_instance.value.master_pass) }}" - SWIFT_USERNAME: "{{ template_backup_credentials.username }}" - SWIFT_PASSWORD: "{{ template_backup_credentials.password }}" - SWIFT_AUTHURL: "{{ template_backup_account.value.authurl }}" - SWIFT_AUTHVERSION: {{ template_backup_account.value.authversion }} - SWIFT_TENANTNAME: "{{ template_backup_account.value.tenantname }}" - SWIFT_TENANTID: "{{ template_backup_account.value.tenantid }}" - SWIFT_REGIONNAME: "{{ template_backup_account.value.regionname }}" - JOB_200_WHEN: "never" - JOB_300_WHAT: "pg_dump --no-owner --format c --file $$SRC/$$PGDATABASE.pgdump && backup --full-if-older-than 6D" - JOB_302_WHAT: "dup remove-all-but-n-full 5 --force $$DST $$@" - JOB_302_WHEN: "daily" - volumes: - - {{ template_odoo_instance.key }}{{ template_backup_account.key }}_backup_cache:/root/.cache/duplicity/:z - - {{ template_odoo_instance.key }}_filestore:/mnt/backup/src/odoo:z + backup_odoo: + image: sources.le-filament.com:5050/lefilament/duplicity_image/duplicity:latest + hostname: {{ inventory_hostname_short | lower | regex_replace('_','') }}-{{ template_odoo_instance.key }} + environment: + SRC: "/mnt/backup/src" + DST: "swift://{{ template_odoo_instance.key }}_{{ inventory_hostname | lower }}" + PGHOST: "db" + PGDATABASE: "{{ template_odoo_instance.value.db }}" + PGUSER: "{{ template_odoo_instance.value.db_user }}" + PGPASSWORD: "{{ template_odoo_instance.value.db_pass }}" + PASSPHRASE: "{{ template_odoo_instance.value.odoo_backup_pass | default(template_odoo_instance.value.master_pass) }}" + SWIFT_USERNAME: "{{ template_backup_credentials.username }}" + SWIFT_PASSWORD: "{{ template_backup_credentials.password }}" + SWIFT_AUTHURL: "{{ template_backup_account.value.authurl }}" + SWIFT_AUTHVERSION: {{ template_backup_account.value.authversion }} + SWIFT_TENANTNAME: "{{ template_backup_account.value.tenantname }}" + SWIFT_TENANTID: "{{ template_backup_account.value.tenantid }}" + SWIFT_REGIONNAME: "{{ template_backup_account.value.regionname }}" + volumes: + - {{ template_odoo_instance.key }}{{ template_backup_account.key }}_backup_cache:/root/.cache/duplicity/:rw + - {{ template_odoo_instance.key }}_filestore:/mnt/backup/src/odoo:ro {% if template_odoo_instance.value.metabase | default(false) %} - - {{ template_odoo_instance.key }}_metabase:/mnt/backup/src/metabase:z + - {{ template_odoo_instance.key }}_metabase:/mnt/backup/src/metabase:ro {% endif %} - networks: - - {{ template_odoo_instance.key }}_default - - public - command: - - /etc/periodic/daily/jobrunner + networks: + - {{ template_odoo_instance.key }}_default + - public + command: > + /bin/ash -c " + echo \"info: dumping '$$PGDATABASE' into '$$SRC/$$PGDATABASE.pgdump'...\" + && pg_dump --no-owner --format c --file $$SRC/$$PGDATABASE.pgdump + && echo \"info: backuping '$$SRC' to '$$DST'...\" + && duplicity backup --full-if-older-than 6D $$SRC $$DST + && echo \"info: removing all but 5 full backups in '$$DST'\" + && duplicity remove-all-but-n-full 5 --force $$DST + && echo \"info: done\"" networks: - {{ template_odoo_instance.key }}_default: - external: true - public: - driver_opts: - encrypted: 1 + {{ template_odoo_instance.key }}_default: + external: true + public: + driver_opts: + encrypted: 1 volumes: - {{ template_odoo_instance.key }}{{ template_backup_account.key }}_backup_cache: - {{ template_odoo_instance.key }}_filestore: - external: true + {{ template_odoo_instance.key }}{{ template_backup_account.key }}_backup_cache: + {{ template_odoo_instance.key }}_filestore: + external: true {% if template_odoo_instance.value.metabase | default(false) %} - {{ template_odoo_instance.key }}_metabase: - external: true + {{ template_odoo_instance.key }}_metabase: + external: true {% endif %} diff --git a/templates/restore-odootest.yaml.j2 b/templates/restore-odootest.yaml.j2 index f313781d930f715b011bf9c0279f36592f58e8e9..8aacf4e15ab41c4e9031ce951b386c153ed4263f 100644 --- a/templates/restore-odootest.yaml.j2 +++ b/templates/restore-odootest.yaml.j2 @@ -1,43 +1,58 @@ -version: "2.1" services: - restore_test: - image: ghcr.io/tecnativa/docker-duplicity-postgres:master - hostname: {{ template_odoo_instance.value.backup_host | default(inventory_hostname_short) | lower | regex_replace('_','') }}-odoo - networks: - - {{ template_odoo_instance.key }}_default - - public - volumes: - - backups_{{ template_odoo_instance.value.backup_instance }}{{ template_backup_account.key }}_backup_cache:/root/.cache/duplicity/:z - - {{ template_odoo_instance.key }}_filestore:/mnt/backup/src/odoo:z - - ./post_restore-{{ template_odoo_instance.key }}.sql:/tmp/post-restore.sql:ro - - ./pre_restore-{{ template_odoo_instance.key }}.sql:/tmp/pre-restore.sql:ro - environment: - PGDATABASE: "{{ template_database_name }}" - SWIFT_USERNAME: "{{ template_backup_credentials.username }}" - SWIFT_PASSWORD: "{{ template_backup_credentials.password }}" - SWIFT_AUTHURL: "{{ template_backup_account.value.authurl }}" - SWIFT_AUTHVERSION: {{ template_backup_account.value.authversion }} - SWIFT_TENANTNAME: "{{ template_backup_account.value.tenantname }}" - SWIFT_TENANTID: "{{ template_backup_account.value.tenantid }}" - SWIFT_REGIONNAME: "{{ template_backup_account.value.regionname }}" - OPTIONS: "--force" - DST: "swift://{{ template_odoo_instance.value.backup_instance }}_{{ template_odoo_instance.value.backup_host | default(inventory_hostname) | lower }}" - PGUSER: "{{ template_odoo_source_instance.value.db_user }}" - PGPASSWORD: "{{ template_odoo_source_instance.value.db_pass }}" - PASSPHRASE: "{{ hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].odoo_backup_pass | default(hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].master_pass) }}" - command: [sh, -c, "psql -a -f /tmp/pre-restore.sql postgres ; echo 'remove existing dir' && rm -rf /mnt/backup/src/odoo/filestore/$$PGDATABASE && restore && echo 'move repo to final dest' && mv /mnt/backup/src/odoo/filestore/{{ hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].db }} /mnt/backup/src/odoo/filestore/$$PGDATABASE && echo 'create database' && createdb -T template0 $$PGDATABASE && echo 'restore database' && pg_restore -d $$PGDATABASE $$SRC/{{ hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].db }}.pgdump ; psql -a -f /tmp/post-restore.sql $$PGDATABASE"] + restore_test: + image: sources.le-filament.com:5050/lefilament/duplicity_image/duplicity:latest + hostname: {{ template_odoo_instance.value.backup_host | default(inventory_hostname_short) | lower | regex_replace('_','') }}-odoo + networks: + - {{ template_odoo_instance.key }}_default + - public + volumes: + - backups_{{ template_odoo_instance.value.backup_instance }}{{ template_backup_account.key }}_backup_cache:/root/.cache/duplicity/:rw + - {{ template_odoo_instance.key }}_filestore:/mnt/backup/src/odoo:rw + - ./post_restore-{{ template_odoo_instance.key }}.sql:/tmp/post-restore.sql:ro + - ./pre_restore-{{ template_odoo_instance.key }}.sql:/tmp/pre-restore.sql:ro + environment: + SRC: "swift://{{ template_odoo_instance.value.backup_instance }}_{{ template_odoo_instance.value.backup_host | default(inventory_hostname) | lower }}" + DST: "/mnt/backup/src" + PGHOST: "db" + PGDATABASE: "{{ template_database_name }}" + PGUSER: "{{ template_odoo_source_instance.value.db_user }}" + PGPASSWORD: "{{ template_odoo_source_instance.value.db_pass }}" + PASSPHRASE: "{{ hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].odoo_backup_pass | default(hostvars[template_odoo_instance.value.backup_host | default(inventory_hostname)]['odoo_instances'][template_odoo_instance.value.backup_instance].master_pass) }}" + SWIFT_USERNAME: "{{ template_backup_credentials.username }}" + SWIFT_PASSWORD: "{{ template_backup_credentials.password }}" + SWIFT_AUTHURL: "{{ template_backup_account.value.authurl }}" + SWIFT_AUTHVERSION: {{ template_backup_account.value.authversion }} + SWIFT_TENANTNAME: "{{ template_backup_account.value.tenantname }}" + SWIFT_TENANTID: "{{ template_backup_account.value.tenantid }}" + SWIFT_REGIONNAME: "{{ template_backup_account.value.regionname }}" + command: > + /bin/ash -c " + echo \"info: executing pre-restore SQL script...\" + && psql -a -f /tmp/pre-restore.sql postgres + && echo \"info: removing existing Odoo filestore of database '$$PGDATABASE'...\" + && rm -rf /mnt/backup/src/odoo/filestore/$$PGDATABASE + && echo \"info: restoring database dump and filestore backup...\" + && duplicity restore --force $$SRC $$DST + && mv /mnt/backup/src/odoo/filestore/{{ template_source_database_name }} /mnt/backup/src/odoo/filestore/$$PGDATABASE + && echo \"info: creating database '$$PGDATABASE'\" + && createdb -T template0 $$PGDATABASE + && echo \"info: restoring database '$$PGDATABASE' from dump '$$DST/{{ template_source_database_name }}.pgdump'\" + && pg_restore -d $$PGDATABASE $$DST/{{ template_source_database_name }}.pgdump + ; echo \"info: executing post-restore SQL script...\" + && psql -a -f /tmp/post-restore.sql $$PGDATABASE + && echo \"info: done\"" networks: - {{ template_odoo_instance.key }}_default: - external: true - public: - driver_opts: - encrypted: 1 + {{ template_odoo_instance.key }}_default: + external: true + public: + driver_opts: + encrypted: 1 volumes: - backups_{{ template_odoo_instance.value.backup_instance }}{{ template_backup_account.key }}_backup_cache: + backups_{{ template_odoo_instance.value.backup_instance }}{{ template_backup_account.key }}_backup_cache: {% if template_odoo_instance.value.backup_host | default(inventory_hostname) == inventory_hostname %} - external: true + external: true {% endif %} - {{ template_odoo_instance.key }}_filestore: - external: true + {{ template_odoo_instance.key }}_filestore: + external: true