From c2058ca9158cca9d3bd8ae66d9173bbc66ce13f5 Mon Sep 17 00:00:00 2001 From: lishengbao Date: Fri, 3 Mar 2023 16:30:06 +0800 Subject: [PATCH] Modify the MAX_BULK_UPDATE_SIZE to 5000 Signed-off-by: lishengbao --- compass_contributor/contributor_dev_org_repo.py | 3 ++- compass_metrics_model/metrics_model.py | 2 +- compass_metrics_model/metrics_model_summary.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/compass_contributor/contributor_dev_org_repo.py b/compass_contributor/contributor_dev_org_repo.py index 06601b3..c2dbbb8 100644 --- a/compass_contributor/contributor_dev_org_repo.py +++ b/compass_contributor/contributor_dev_org_repo.py @@ -15,6 +15,7 @@ logger = logging.getLogger(__name__) urllib3.disable_warnings() page_size = 1000 +MAX_BULK_UPDATE_SIZE = 5000 exclude_field_list = ["unknown", "-- undefined --"] @@ -227,7 +228,7 @@ def processing_data(self, repo): } } all_bulk_data.append(contributor_data) - if len(all_bulk_data) > 100: + if len(all_bulk_data) > MAX_BULK_UPDATE_SIZE: helpers.bulk(client=self.client, actions=all_bulk_data) all_bulk_data = [] helpers.bulk(client=self.client, actions=all_bulk_data) diff --git a/compass_metrics_model/metrics_model.py b/compass_metrics_model/metrics_model.py index e541b4c..edc02ab 100755 --- a/compass_metrics_model/metrics_model.py +++ b/compass_metrics_model/metrics_model.py @@ -55,7 +55,7 @@ logger = logging.getLogger(__name__) -MAX_BULK_UPDATE_SIZE = 10 +MAX_BULK_UPDATE_SIZE = 5000 # [Fixme] In fact, origin should not be distinguished by this form of string. # Maybe pass parameters through configuration file is better. diff --git a/compass_metrics_model/metrics_model_summary.py b/compass_metrics_model/metrics_model_summary.py index eea2b1e..fbcce77 100644 --- a/compass_metrics_model/metrics_model_summary.py +++ b/compass_metrics_model/metrics_model_summary.py @@ -13,7 +13,7 @@ from .utils import (get_uuid, get_date_list) -MAX_BULK_UPDATE_SIZE = 10 +MAX_BULK_UPDATE_SIZE = 5000 logger = logging.getLogger(__name__)