From b613bdd5240b2730a6b4b71e0420436ca094ec05 Mon Sep 17 00:00:00 2001
From: Julien Ortet <julien@le-filament.com>
Date: Tue, 18 Mar 2025 16:12:46 +0100
Subject: [PATCH 1/8] [ADD] generate agg data when import cdc
---
wizard/acc_operation_import_wizard.py | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/wizard/acc_operation_import_wizard.py b/wizard/acc_operation_import_wizard.py
index dfa2e28..b4a753f 100644
--- a/wizard/acc_operation_import_wizard.py
+++ b/wizard/acc_operation_import_wizard.py
@@ -126,6 +126,10 @@ class AccOperationImportWizard(models.TransientModel):
+ self.model
+ "</h1>"
)
+
+ start_dates = []
+ end_dates = []
+
for file in self.attachment_ids:
message += (
"<p><strong>Fichier "
@@ -133,7 +137,10 @@ class AccOperationImportWizard(models.TransientModel):
+ "</strong><br/>Début Import ... <br/>"
)
data_filename = file.name.split("_")
+ date_format = "%d%m%Y"
id_pdm = data_filename[0]
+ start_dates.append(datetime.strptime(data_filename[1], date_format))
+ end_dates.append(datetime.strptime(data_filename[2], date_format))
counter_id = self.env["acc.counter"].search([("name", "=", id_pdm)])
data_filename[3] = data_filename[3].lower()
@@ -188,6 +195,24 @@ class AccOperationImportWizard(models.TransientModel):
)
self.update_partner_id(data_filename, counter_id)
+ message += "Génération des données agrégées<br/>"
+ domain = [
+ ("acc_operation_id", "=", self.operation_id.id),
+ ("date_slot", ">=", min(start_dates)),
+ ("date_slot", "<", max(end_dates)),
+ ]
+
+ # Vérification si des données existent déjà
+ # pour cet intervalle de date
+ rec_ids = self.env["acc.enedis.cdc.agg"].search(domain)
+ if rec_ids:
+ rec_ids.unlink()
+
+ self.env["acc.enedis.cdc.agg"].generate(
+ acc_operation_id=self.operation_id.id,
+ start_date=min(start_dates),
+ end_date=max(end_dates),
+ )
message += "Fin de l'import des données OK<br/>"
# Suppression du fichier après création des enregistrements
--
GitLab
From 4f0b0c98c6dcb617ff47e6cfc07f9b4ad73394cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20-=20Le=20Filament?= <remi@le-filament.com>
Date: Tue, 18 Mar 2025 16:14:05 +0100
Subject: [PATCH 2/8] [IMP] update partner_id only for new curves
---
wizard/acc_operation_import_wizard.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/wizard/acc_operation_import_wizard.py b/wizard/acc_operation_import_wizard.py
index b4a753f..58ffb4c 100644
--- a/wizard/acc_operation_import_wizard.py
+++ b/wizard/acc_operation_import_wizard.py
@@ -115,7 +115,11 @@ class AccOperationImportWizard(models.TransientModel):
]
self.env["acc.counter.period"]._get_periods_from_interval(
domain, date_begin_obj, date_end_obj
- )._update_cdc_partner_id(model=self.model)
+ )._update_cdc_partner_id(
+ model=self.model,
+ date_start=date_begin_obj.date(),
+ date_end=date_end_obj.date(),
+ )
def valid_import(self):
message = ""
--
GitLab
From c5b216c11d7a0d3c1b0f5f48ba2e2429ede95bff Mon Sep 17 00:00:00 2001
From: Julien Ortet <julien@le-filament.com>
Date: Tue, 18 Mar 2025 16:28:13 +0100
Subject: [PATCH 3/8] [ADD] generate agg data when import cdc
---
wizard/acc_operation_import_wizard.py | 37 ++++++++++++++-------------
1 file changed, 19 insertions(+), 18 deletions(-)
diff --git a/wizard/acc_operation_import_wizard.py b/wizard/acc_operation_import_wizard.py
index 58ffb4c..b962dab 100644
--- a/wizard/acc_operation_import_wizard.py
+++ b/wizard/acc_operation_import_wizard.py
@@ -199,24 +199,6 @@ class AccOperationImportWizard(models.TransientModel):
)
self.update_partner_id(data_filename, counter_id)
- message += "Génération des données agrégées<br/>"
- domain = [
- ("acc_operation_id", "=", self.operation_id.id),
- ("date_slot", ">=", min(start_dates)),
- ("date_slot", "<", max(end_dates)),
- ]
-
- # Vérification si des données existent déjà
- # pour cet intervalle de date
- rec_ids = self.env["acc.enedis.cdc.agg"].search(domain)
- if rec_ids:
- rec_ids.unlink()
-
- self.env["acc.enedis.cdc.agg"].generate(
- acc_operation_id=self.operation_id.id,
- start_date=min(start_dates),
- end_date=max(end_dates),
- )
message += "Fin de l'import des données OK<br/>"
# Suppression du fichier après création des enregistrements
@@ -224,6 +206,25 @@ class AccOperationImportWizard(models.TransientModel):
file.unlink()
message += "Suppression OK </p>"
+ message += "Génération des données agrégées<br/>"
+ domain = [
+ ("acc_operation_id", "=", self.operation_id.id),
+ ("date_slot", ">=", min(start_dates)),
+ ("date_slot", "<", max(end_dates)),
+ ]
+
+ # Vérification si des données existent déjà
+ # pour cet intervalle de date
+ rec_ids = self.env["acc.enedis.cdc.agg"].search(domain)
+ if rec_ids:
+ rec_ids.unlink()
+
+ self.env["acc.enedis.cdc.agg"].generate(
+ acc_operation_id=self.operation_id.id,
+ start_date=min(start_dates),
+ end_date=max(end_dates),
+ )
+
message += "<h1>Fin import manuel: " + str(fields.Datetime.now()) + "</h1>"
# Logs information logs
log_id = self.env["acc.logs"].create(
--
GitLab
From 1ed8d9d1226fe287ceede8e079a79a67fa1a6ef0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20-=20Le=20Filament?= <remi@le-filament.com>
Date: Wed, 19 Mar 2025 08:45:17 +0100
Subject: [PATCH 4/8] [MOV] agg table name
---
wizard/acc_operation_import_wizard.py | 94 +++++++++++++++------------
1 file changed, 52 insertions(+), 42 deletions(-)
diff --git a/wizard/acc_operation_import_wizard.py b/wizard/acc_operation_import_wizard.py
index b962dab..02996ce 100644
--- a/wizard/acc_operation_import_wizard.py
+++ b/wizard/acc_operation_import_wizard.py
@@ -8,8 +8,10 @@ from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from odoo import fields, models
+from odoo.osv import expression
from odoo.addons.api_connector.tools.date_utils import local_to_utc
+from odoo.addons.oacc.models.acc_enedis_cdc_day import COMP_DATA_TYPE_MAP
class AccOperationImportWizard(models.TransientModel):
@@ -55,7 +57,7 @@ class AccOperationImportWizard(models.TransientModel):
# ------------------------------------------------------
# Business methods
# ------------------------------------------------------
- def delete_existing_data(self, data_filename, counter_id):
+ def _delete_existing_data(self, data_filename, counter_id):
message = ""
# Suppression des données si déjà existantes
date_begin_str = data_filename[1]
@@ -78,35 +80,51 @@ class AccOperationImportWizard(models.TransientModel):
computed_data_type = "autocons"
domain_all = [
+ ("acc_operation_id", "=", self.operation_id.id),
+ ("acc_counter_id", "=", counter_id),
+ ]
+ domain_cdc = expression.AND([domain_all, [
("date_slot", ">=", start_datetime),
("date_slot", "<", end_datetime),
- ("acc_operation_id", "=", self.operation_id.id),
("comp_data_type", "=", computed_data_type),
- ("acc_counter_id", "=", counter_id.id),
- ]
- rec_ids = self.env[self.model].search(domain_all)
+ ]])
+ domain_cdc_day = expression.AND([domain_all, [
+ ("date_slot", ">=", date_begin_obj.date()),
+ ("date_slot", "<", date_end_obj.date()),
+ ("comp_data_type", "=", COMP_DATA_TYPE_MAP[computed_data_type]),
+ ]])
+ rec_ids = self.env[self.model].search(domain_cdc)
if rec_ids:
- message += "Suppression des enregistrements existants ... <br/>"
+ message += "Suppression des enregistrements existants de courbes ... <br/>"
rec_ids.unlink()
- message += "Suppression des enregistrements existants OK <br/>"
+ message += "Suppression des enregistrements existants de courbes OK <br/>"
+
+ if self.model == "acc.enedis.cdc":
+ day_rec_ids = self.env["acc.enedis.cdc.day"].search(domain_cdc_day)
+ if day_rec_ids:
+ message += "Suppression des enregistrements existants de courbes "\
+ "agrégées au jour ... <br/>"
+ day_rec_ids.unlink()
+ message += "Suppression des enregistrements existants de courbes "\
+ "agrégées au jour OK <br/>"
return message
def create_curve(self, curve_data):
self.env[self.model].create(curve_data)
- def update_partner_id(self, data_filename, counter_id):
+ def _update_partner_id(self, data_filename, counter_id):
date_begin_str = data_filename[1]
date_end_str = data_filename[2]
date_format = "%d%m%Y"
- date_begin_obj = datetime.strptime(date_begin_str, date_format)
+ date_begin_obj = datetime.strptime(date_begin_str, date_format).date()
date_end_obj = datetime.strptime(date_end_str, date_format) + relativedelta(
days=1
- )
+ ).date()
# Update partner_id for retrieved cdc
domain = [
("acc_operation_id", "=", self.operation_id.id),
- ("acc_counter_id", "=", counter_id.id or False),
+ ("acc_counter_id", "=", counter_id),
(
"prm_type",
"=",
@@ -115,10 +133,10 @@ class AccOperationImportWizard(models.TransientModel):
]
self.env["acc.counter.period"]._get_periods_from_interval(
domain, date_begin_obj, date_end_obj
- )._update_cdc_partner_id(
+ ).update_cdc_partner_id(
model=self.model,
- date_start=date_begin_obj.date(),
- date_end=date_end_obj.date(),
+ date_start=date_begin_obj,
+ date_end=date_end_obj,
)
def valid_import(self):
@@ -131,9 +149,6 @@ class AccOperationImportWizard(models.TransientModel):
+ "</h1>"
)
- start_dates = []
- end_dates = []
-
for file in self.attachment_ids:
message += (
"<p><strong>Fichier "
@@ -143,16 +158,16 @@ class AccOperationImportWizard(models.TransientModel):
data_filename = file.name.split("_")
date_format = "%d%m%Y"
id_pdm = data_filename[0]
- start_dates.append(datetime.strptime(data_filename[1], date_format))
- end_dates.append(datetime.strptime(data_filename[2], date_format))
+ start_date = datetime.strptime(data_filename[1], date_format)
+ end_date = datetime.strptime(data_filename[2], date_format)
- counter_id = self.env["acc.counter"].search([("name", "=", id_pdm)])
+ counter_id = self.env["acc.counter"].search([("name", "=", id_pdm)]).id
data_filename[3] = data_filename[3].lower()
if data_filename[3] not in ["prod", "surplus"]:
data_filename[3] = data_filename[3][:-1]
- message += self.delete_existing_data(
+ message += self._delete_existing_data(
data_filename=data_filename, counter_id=counter_id
)
@@ -163,6 +178,8 @@ class AccOperationImportWizard(models.TransientModel):
csv_reader = csv.reader(file_decode, delimiter=";")
file_reader.extend(csv_reader)
+ curves_to_create = []
+
# Create Data for the CDC
message += "Lecture et import des données ... <br/>"
for row in file_reader:
@@ -186,11 +203,11 @@ class AccOperationImportWizard(models.TransientModel):
else:
timestamp = timestamp + timedelta(minutes=timestep)
- self.create_curve(
+ curves_to_create.append(
{
"name": file.name,
"acc_operation_id": self.operation_id.id,
- "acc_counter_id": counter_id.id,
+ "acc_counter_id": counter_id,
"comp_data_type": data_filename[3],
"power": power,
"date_slot": timestamp,
@@ -198,7 +215,18 @@ class AccOperationImportWizard(models.TransientModel):
}
)
- self.update_partner_id(data_filename, counter_id)
+ message += "Création des courbes<br/>"
+ self.create_curve(curves_to_create)
+ message += "Mise à jour du contact associé aux courbes<br/>"
+ self._update_partner_id(data_filename, counter_id)
+
+ message += "Génération des données agrégées au jour<br/>"
+ self.env["acc.enedis.cdc.day"].generate(
+ acc_operation_id=self.operation_id.id,
+ acc_counter_id=counter_id,
+ start_date=start_date,
+ end_date=end_date,
+ )
message += "Fin de l'import des données OK<br/>"
# Suppression du fichier après création des enregistrements
@@ -206,24 +234,6 @@ class AccOperationImportWizard(models.TransientModel):
file.unlink()
message += "Suppression OK </p>"
- message += "Génération des données agrégées<br/>"
- domain = [
- ("acc_operation_id", "=", self.operation_id.id),
- ("date_slot", ">=", min(start_dates)),
- ("date_slot", "<", max(end_dates)),
- ]
-
- # Vérification si des données existent déjà
- # pour cet intervalle de date
- rec_ids = self.env["acc.enedis.cdc.agg"].search(domain)
- if rec_ids:
- rec_ids.unlink()
-
- self.env["acc.enedis.cdc.agg"].generate(
- acc_operation_id=self.operation_id.id,
- start_date=min(start_dates),
- end_date=max(end_dates),
- )
message += "<h1>Fin import manuel: " + str(fields.Datetime.now()) + "</h1>"
# Logs information logs
--
GitLab
From 11488b6fe0c869bb34354bf80a1387c177583f66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20-=20Le=20Filament?= <remi@le-filament.com>
Date: Wed, 19 Mar 2025 11:03:26 +0100
Subject: [PATCH 5/8] [ADD] debug logger for file import
---
wizard/acc_operation_import_wizard.py | 53 +++++++++++++++++++--------
1 file changed, 37 insertions(+), 16 deletions(-)
diff --git a/wizard/acc_operation_import_wizard.py b/wizard/acc_operation_import_wizard.py
index 02996ce..2a160a8 100644
--- a/wizard/acc_operation_import_wizard.py
+++ b/wizard/acc_operation_import_wizard.py
@@ -3,6 +3,7 @@
import base64
import csv
import io
+import logging
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
@@ -13,6 +14,8 @@ from odoo.osv import expression
from odoo.addons.api_connector.tools.date_utils import local_to_utc
from odoo.addons.oacc.models.acc_enedis_cdc_day import COMP_DATA_TYPE_MAP
+_logger = logging.getLogger(__name__)
+
class AccOperationImportWizard(models.TransientModel):
_name = "acc.operation.import.wizard"
@@ -83,16 +86,26 @@ class AccOperationImportWizard(models.TransientModel):
("acc_operation_id", "=", self.operation_id.id),
("acc_counter_id", "=", counter_id),
]
- domain_cdc = expression.AND([domain_all, [
- ("date_slot", ">=", start_datetime),
- ("date_slot", "<", end_datetime),
- ("comp_data_type", "=", computed_data_type),
- ]])
- domain_cdc_day = expression.AND([domain_all, [
- ("date_slot", ">=", date_begin_obj.date()),
- ("date_slot", "<", date_end_obj.date()),
- ("comp_data_type", "=", COMP_DATA_TYPE_MAP[computed_data_type]),
- ]])
+ domain_cdc = expression.AND(
+ [
+ domain_all,
+ [
+ ("date_slot", ">=", start_datetime),
+ ("date_slot", "<", end_datetime),
+ ("comp_data_type", "=", computed_data_type),
+ ],
+ ]
+ )
+ domain_cdc_day = expression.AND(
+ [
+ domain_all,
+ [
+ ("date_slot", ">=", date_begin_obj.date()),
+ ("date_slot", "<", date_end_obj.date()),
+ ("comp_data_type", "=", COMP_DATA_TYPE_MAP[computed_data_type]),
+ ],
+ ]
+ )
rec_ids = self.env[self.model].search(domain_cdc)
if rec_ids:
message += "Suppression des enregistrements existants de courbes ... <br/>"
@@ -102,11 +115,15 @@ class AccOperationImportWizard(models.TransientModel):
if self.model == "acc.enedis.cdc":
day_rec_ids = self.env["acc.enedis.cdc.day"].search(domain_cdc_day)
if day_rec_ids:
- message += "Suppression des enregistrements existants de courbes "\
+ message += (
+ "Suppression des enregistrements existants de courbes "
"agrégées au jour ... <br/>"
+ )
day_rec_ids.unlink()
- message += "Suppression des enregistrements existants de courbes "\
+ message += (
+ "Suppression des enregistrements existants de courbes "
"agrégées au jour OK <br/>"
+ )
return message
@@ -118,9 +135,9 @@ class AccOperationImportWizard(models.TransientModel):
date_end_str = data_filename[2]
date_format = "%d%m%Y"
date_begin_obj = datetime.strptime(date_begin_str, date_format).date()
- date_end_obj = datetime.strptime(date_end_str, date_format) + relativedelta(
- days=1
- ).date()
+ date_end_obj = (
+ datetime.strptime(date_end_str, date_format) + relativedelta(days=1).date()
+ )
# Update partner_id for retrieved cdc
domain = [
("acc_operation_id", "=", self.operation_id.id),
@@ -150,6 +167,7 @@ class AccOperationImportWizard(models.TransientModel):
)
for file in self.attachment_ids:
+ _logger.debug(f"Processing file {file.name} - Start")
message += (
"<p><strong>Fichier "
+ file.name
@@ -215,11 +233,14 @@ class AccOperationImportWizard(models.TransientModel):
}
)
+ _logger.debug(f"Processing file {file.name} - Curves creation")
message += "Création des courbes<br/>"
self.create_curve(curves_to_create)
+ _logger.debug(f"Processing file {file.name} - Update partner_id on curves")
message += "Mise à jour du contact associé aux courbes<br/>"
self._update_partner_id(data_filename, counter_id)
+ _logger.debug(f"Processing file {file.name} - Generating daily curves")
message += "Génération des données agrégées au jour<br/>"
self.env["acc.enedis.cdc.day"].generate(
acc_operation_id=self.operation_id.id,
@@ -230,11 +251,11 @@ class AccOperationImportWizard(models.TransientModel):
message += "Fin de l'import des données OK<br/>"
# Suppression du fichier après création des enregistrements
+ _logger.debug(f"Processing file {file.name} - Remove file")
message += "Suppression du fichier " + file.name + " ...<br/>"
file.unlink()
message += "Suppression OK </p>"
-
message += "<h1>Fin import manuel: " + str(fields.Datetime.now()) + "</h1>"
# Logs information logs
log_id = self.env["acc.logs"].create(
--
GitLab
From eaf86cd4058c05e0fb34e80aa97fc8dcf12911a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20-=20Le=20Filament?= <remi@le-filament.com>
Date: Wed, 19 Mar 2025 12:09:50 +0100
Subject: [PATCH 6/8] [FIX] incorrect .date()
---
wizard/acc_operation_import_wizard.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/wizard/acc_operation_import_wizard.py b/wizard/acc_operation_import_wizard.py
index 2a160a8..6563d23 100644
--- a/wizard/acc_operation_import_wizard.py
+++ b/wizard/acc_operation_import_wizard.py
@@ -136,8 +136,8 @@ class AccOperationImportWizard(models.TransientModel):
date_format = "%d%m%Y"
date_begin_obj = datetime.strptime(date_begin_str, date_format).date()
date_end_obj = (
- datetime.strptime(date_end_str, date_format) + relativedelta(days=1).date()
- )
+ datetime.strptime(date_end_str, date_format) + relativedelta(days=1)
+ ).date()
# Update partner_id for retrieved cdc
domain = [
("acc_operation_id", "=", self.operation_id.id),
--
GitLab
From d78bea635d73b3e64336015c2998d8ddf37291b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20-=20Le=20Filament?= <remi@le-filament.com>
Date: Wed, 19 Mar 2025 14:29:54 +0100
Subject: [PATCH 7/8] [FIX] use date iso datetime
---
wizard/acc_operation_import_wizard.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/wizard/acc_operation_import_wizard.py b/wizard/acc_operation_import_wizard.py
index 6563d23..cc16417 100644
--- a/wizard/acc_operation_import_wizard.py
+++ b/wizard/acc_operation_import_wizard.py
@@ -176,8 +176,10 @@ class AccOperationImportWizard(models.TransientModel):
data_filename = file.name.split("_")
date_format = "%d%m%Y"
id_pdm = data_filename[0]
- start_date = datetime.strptime(data_filename[1], date_format)
- end_date = datetime.strptime(data_filename[2], date_format)
+ start_date = datetime.strptime(data_filename[1], date_format).date()
+ end_date = (
+ datetime.strptime(data_filename[2], date_format) + relativedelta(days=1)
+ ).date()
counter_id = self.env["acc.counter"].search([("name", "=", id_pdm)]).id
data_filename[3] = data_filename[3].lower()
--
GitLab
From 53739aee377f7017e797842e37aae33049ddd36a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20-=20Le=20Filament?= <remi@le-filament.com>
Date: Thu, 20 Mar 2025 04:45:58 +0100
Subject: [PATCH 8/8] [FIX] call functions depending on source table Update
partner_id and generation of by-day aggregated curves should only happen for
acc.enedis.cdc (not for raw curves for instance)
---
wizard/acc_operation_import_wizard.py | 26 ++++++++++++++------------
1 file changed, 14 insertions(+), 12 deletions(-)
diff --git a/wizard/acc_operation_import_wizard.py b/wizard/acc_operation_import_wizard.py
index cc16417..2c356a0 100644
--- a/wizard/acc_operation_import_wizard.py
+++ b/wizard/acc_operation_import_wizard.py
@@ -238,18 +238,20 @@ class AccOperationImportWizard(models.TransientModel):
_logger.debug(f"Processing file {file.name} - Curves creation")
message += "Création des courbes<br/>"
self.create_curve(curves_to_create)
- _logger.debug(f"Processing file {file.name} - Update partner_id on curves")
- message += "Mise à jour du contact associé aux courbes<br/>"
- self._update_partner_id(data_filename, counter_id)
-
- _logger.debug(f"Processing file {file.name} - Generating daily curves")
- message += "Génération des données agrégées au jour<br/>"
- self.env["acc.enedis.cdc.day"].generate(
- acc_operation_id=self.operation_id.id,
- acc_counter_id=counter_id,
- start_date=start_date,
- end_date=end_date,
- )
+
+ if self.model == "acc.enedis.cdc":
+ _logger.debug(f"Processing file {file.name} - Update partner_id on curves")
+ message += "Mise à jour du contact associé aux courbes<br/>"
+ self._update_partner_id(data_filename, counter_id)
+
+ _logger.debug(f"Processing file {file.name} - Generating daily curves")
+ message += "Génération des données agrégées au jour<br/>"
+ self.env["acc.enedis.cdc.day"].generate(
+ acc_operation_id=self.operation_id.id,
+ acc_counter_id=counter_id,
+ start_date=start_date,
+ end_date=end_date,
+ )
message += "Fin de l'import des données OK<br/>"
# Suppression du fichier après création des enregistrements
--
GitLab