[PERF] repartition data: reduce number of SQL calls

By fetching first all partners and counters with necessary fields (with browse + read on specific fields)
On ACC119, we reduce from 549 SQL requests to 24 SQL requests, reducing time from 5s to less than 1s

models/acc_operation.py

@@ -112,28 +112,17 @@ class AccOperation(models.Model):
             energy_dict["prod"]["partner_ids"], filter_partner_id, filter_prm_id
         )
 
-        del_partner_ids = energy_dict["cons"]["partner_ids"]
-        (
-            total_autocons,
-            sorted_partner,
-        ) = prepare_repartition_data.sort_partners_prorata(
-            del_partner_ids, filter_partner_id, filter_prm_id
-        )
-
         tPartners = prepare_repartition_data.get_partners_data_prorata(
             self.env,
-            sorted_partner,
-            del_partner_ids,
+            energy_dict["cons"]["partner_ids"],
             total_prod,
             filter_partner_id,
             filter_prm_id,
         )
-
-        percentage_surplus = 100 * total_surplus / total_prod
-
-        tSurplus = {}
-        tSurplus["surplus"] = total_surplus
-        tSurplus["percentage"] = percentage_surplus
+        tSurplus = {
+            "surplus": total_surplus,
+            "percentage": 100 * total_surplus / total_prod if total_prod else 0,
+        }
 
         render_values = {"tPartners": tPartners, "tSurplus": tSurplus}
         html = self.env["ir.ui.view"]._render_template(

tools/prepare_repartition_data.py

-def get_prod_datas_prorata(partner_ids, filter_partner_id=None, filter_counter_id=None):
+def get_prod_datas_prorata(
+    inj_partners_node, filter_partner_id=None, filter_counter_id=None
+):
     total_surplus = 0
     total_prod = 0
-    for partner_id in partner_ids:  # producteur
+    for partner_id in inj_partners_node:  # producteur
         if filter_partner_id is not None and partner_id != filter_partner_id:
             continue
-        partner = partner_ids[partner_id]
-        counter_ids = partner["acc_counter_ids"]
-        for counter_id in counter_ids:  # injection
+        inj_counters_node = inj_partners_node[partner_id]["acc_counter_ids"]
+        for counter_id in inj_counters_node:  # injection
             if filter_counter_id is not None and counter_id != filter_counter_id:
                 continue
-            total_surplus = total_surplus + counter_ids[counter_id]["surplus"]
-            total_prod = total_prod + counter_ids[counter_id]["prod"]
+            total_surplus = total_surplus + inj_counters_node[counter_id].get(
+                "surplus", 0.0
+            )
+            total_prod = total_prod + inj_counters_node[counter_id].get("prod", 0.0)
     return total_prod, total_surplus
 
 
-def sort_partners_prorata(partner_ids, filter_partner_id=None, filter_counter_id=None):
+def _sort_partners_prorata(
+    del_partners_node, filter_partner_id=None, filter_counter_id=None
+):
     tPartners = {}
-    total = 0
-    for partner_id in partner_ids:  # consommateur
-        partner = partner_ids[partner_id]
-        counter_ids = partner["acc_counter_ids"]
+    for partner_id in del_partners_node:  # consommateur
+        del_counters_node = del_partners_node[partner_id]["acc_counter_ids"]
         autocons = 0
-        for counter_id in counter_ids:  # soutirage
+        for counter_id in del_counters_node:  # soutirage
             if filter_counter_id is not None:
-                autocons = (
-                    autocons
-                    + counter_ids[counter_id]["autocons_per_inj_counter"][
-                        filter_counter_id
-                    ]
-                )
+                autocons += del_counters_node[counter_id][
+                    "autocons_per_inj_counter"
+                ].get(filter_counter_id, 0.0)
             elif filter_partner_id is not None:
-                autocons = (
-                    autocons
-                    + counter_ids[counter_id]["autocons_per_inj_partner"][
-                        filter_partner_id
-                    ]
-                )
+                autocons += del_counters_node[counter_id][
+                    "autocons_per_inj_partner"
+                ].get(filter_partner_id, 0.0)
             else:
-                autocons = autocons + counter_ids[counter_id]["autocons"]
+                autocons += del_counters_node[counter_id].get("autocons")
         tPartners[partner_id] = autocons
-        total = total + autocons
 
     sorted_partner = dict(
         sorted(tPartners.items(), key=lambda item: item[1], reverse=True)
     )
-    return total, sorted_partner
+    return sorted_partner
 
 
-def _sort_counters_prorata(counter_ids, filter_partner_id=None, filter_counter_id=None):
+def _sort_counters_prorata(
+    del_counters_node, filter_partner_id=None, filter_counter_id=None
+):
     tCounters = {}
-    for counter_id in counter_ids:  # soutirage
+    for counter_id in del_counters_node:  # soutirage
         autocons = 0
-        if filter_counter_id is not None:
-            autocons = (
-                autocons
-                + counter_ids[counter_id]["autocons_per_inj_counter"][filter_counter_id]
+        if filter_counter_id:
+            autocons = del_counters_node[counter_id]["autocons_per_inj_counter"].get(
+                filter_counter_id, 0.0
             )
-        elif filter_partner_id is not None:
-            autocons = (
-                autocons
-                + counter_ids[counter_id]["autocons_per_inj_partner"][filter_partner_id]
+        elif filter_partner_id:
+            autocons = del_counters_node[counter_id]["autocons_per_inj_partner"].get(
+                filter_partner_id, 0.0
             )
         else:
-            autocons = autocons + counter_ids[counter_id]["autocons"]
+            autocons = del_counters_node[counter_id].get("autocons", 0.0)
         tCounters[counter_id] = autocons
 
     sorted_counter = dict(
@@ -72,69 +68,74 @@ def _sort_counters_prorata(counter_ids, filter_partner_id=None, filter_counter_i
 
 
 def _get_counters_data_prorata(
-    env,
-    sorted_counter,
-    counter_ids,
+    del_counters_dict,
+    del_counters_ndoe,
     total,
     filter_partner_id=None,
     filter_counter_id=None,
 ):
     tCounters = {}
     subtotal = 0
-    for counter_id in sorted_counter:  # soutirage
-        tCounter = {}
-        dbCounter = env["acc.counter"].browse(counter_id)
-        tCounter["name"] = dbCounter.name
-        tCounter["street"] = dbCounter.street
-        tCounter["street2"] = dbCounter.street2
-        tCounter["zip"] = dbCounter.zip
-        tCounter["city"] = dbCounter.city
 
-        autocons = 0
-        if filter_counter_id is not None:
-            autocons = (
-                autocons
-                + counter_ids[counter_id]["autocons_per_inj_counter"][filter_counter_id]
-            )
-        elif filter_partner_id is not None:
-            autocons = (
-                autocons
-                + counter_ids[counter_id]["autocons_per_inj_partner"][filter_partner_id]
+    sorted_counter = _sort_counters_prorata(
+        del_counters_ndoe, filter_partner_id, filter_counter_id
     )
-        else:
-            autocons = autocons + counter_ids[counter_id]["autocons"]
 
-        subtotal = subtotal + autocons
-        tCounter["autocons"] = autocons
-        tCounter["percentage"] = 100 * autocons / total
+    for counter_id in sorted_counter:  # soutirage
+        tCounter = {
+            "name": del_counters_dict[counter_id]["name"],
+            "street": del_counters_dict[counter_id]["street"],
+            "street2": del_counters_dict[counter_id]["street2"],
+            "zip": del_counters_dict[counter_id]["zip"],
+            "city": del_counters_dict[counter_id]["city"],
+            "autocons": sorted_counter[counter_id],
+            "percentage": 100 * sorted_counter[counter_id] / total,
+        }
+        subtotal = subtotal + sorted_counter[counter_id]
         tCounters[counter_id] = tCounter
     return subtotal, tCounters
 
 
 def get_partners_data_prorata(
     env,
-    sorted_partner,
-    partner_ids,
+    del_partners_node,
     total,
     filter_partner_id=None,
     filter_counter_id=None,
 ):
     tPartners = {}
+    dbPartners = env["res.partner"].browse(del_partners_node.keys())
+    del_partners_dict = {
+        partner["id"]: partner["name"] for partner in dbPartners.read(["name"])
+    }
+    del_counter_ids = {
+        counter_id: del_partners_node[partner_id]["acc_counter_ids"][counter_id]
+        for partner_id in dbPartners.ids
+        for counter_id in del_partners_node[partner_id]["acc_counter_ids"]
+    }
+    dbCounters = env["acc.counter"].browse(del_counter_ids.keys())
+    del_counters_dict = {
+        counter["id"]: {
+            "name": counter["name"],
+            "street": counter["street"],
+            "street2": counter["street2"],
+            "zip": counter["zip"],
+            "city": counter["city"],
+        }
+        for counter in dbCounters.read(["name", "street", "street2", "zip", "city"])
+    }
+
+    sorted_partner = _sort_partners_prorata(
+        del_partners_node, filter_partner_id, filter_counter_id
+    )
     for partner_id in sorted_partner:  # consommateur
         tPartner = {}
-        partner = env["res.partner"].browse(partner_id)
-        tPartner["name"] = partner.name
-        partner = partner_ids[partner_id]
-        counter_ids = partner["acc_counter_ids"]
-
-        sorted_counter = _sort_counters_prorata(
-            counter_ids, filter_partner_id, filter_counter_id
-        )
+        tPartner["name"] = del_partners_dict[partner_id]
+        del_counters_node = del_partners_node[partner_id]["acc_counter_ids"]
 
         (subtotal, tCounters) = _get_counters_data_prorata(
-            env,
-            sorted_counter,
-            counter_ids,
+            del_counters_dict,
+            del_counters_node,
             total,
             filter_partner_id,
             filter_counter_id,