refactor(looker): optimize view-to-explore mapping and enhance logging

Author: askumar27

metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py

@@ -794,7 +794,6 @@ def get_internal_workunits(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:
                             if self.source_config.emit_reachable_views_only:
                                 explore_reachable_views.add(view_name.include)
 
-                            # Build view-to-explores mapping efficiently
                             view_to_explores[view_name.include].add(explore.name)
                             explore_to_views[explore.name].add(view_name.include)
                 except Exception as e:
@@ -810,6 +809,16 @@ def get_internal_workunits(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:
                 model.connection, set()
             )
 
+            view_to_explore_map = {}
+            if view_to_explores and explore_to_views:
+                view_to_explore_map = self._optimize_views_by_common_explore(
+                    view_to_explores, explore_to_views
+                )
+            else:
+                logger.warning(
+                    f"Either view_to_explores: {view_to_explores} or explore_to_views: {explore_to_views} is empty"
+                )
+
             project_name = self.get_project_name(model_name)
 
             looker_view_id_cache: LookerViewIdCache = LookerViewIdCache(
@@ -894,9 +903,7 @@ def get_internal_workunits(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:
                                 config=self.source_config,
                                 ctx=self.ctx,
                                 looker_client=self.looker_client,
-                                view_to_explore_map=self._optimize_views_by_common_explore(
-                                    view_to_explores, explore_to_views
-                                ),
+                                view_to_explore_map=view_to_explore_map,
                             )
                         except Exception as e:
                             self.reporter.report_warning(
@@ -1074,10 +1081,7 @@ def _optimize_views_by_common_explore(
         view_to_explore: Dict[str, str] = {}
 
         # For each view, find the explore with maximum size that contains it
-        for view_name in view_to_explores:
-            # Get all explores that contain this view from pre-built mapping
-            candidate_explores = view_to_explores[view_name]
-
+        for view_name, candidate_explores in view_to_explores.items():
             if candidate_explores:
                 # Find explore with maximum size using max() with key function
                 # This assings the view to the explore with the most views that contains it
@@ -1102,6 +1106,7 @@ def _optimize_views_by_common_explore(
                 f"View-explore optimization: No explores to optimize for {total_views} views"
             )
 
+        logger.debug(f"Final View-to-explore mapping: {view_to_explore}")
         return view_to_explore
 
     def get_report(self):

metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py

@@ -611,6 +611,9 @@ def _get_fields_from_looker_api(self, explore_name: str) -> List[str]:
             )
 
             if explore and explore.fields:
+                logger.debug(
+                    f"Looker API response for explore fields: {explore.fields}"
+                )
                 # Creating a map to de-dup dimension group fields - adding all of them adds to the query length, we dont need all of them for CLL
                 dimension_group_fields_mapping: Dict[str, str] = {}
                 # Get dimensions from API
@@ -654,10 +657,13 @@ def _get_fields_from_looker_api(self, explore_name: str) -> List[str]:
                     f"No fields found in explore '{explore_name}' from Looker API, falling back to view context"
                 )
 
-        except Exception as e:
+        except Exception:
             logger.warning(
-                f"Failed to get explore details from Looker API for explore '{explore_name}': {e}. Falling back to view context."
+                f"Failed to get explore details from Looker API for explore '{explore_name}'. Current view: {self.view_context.name()} and view_fields: {view_fields}. Falling back to view csontext.",
+                exc_info=True,
             )
+            # Resetting view_fields to trigger fallback to view context
+            view_fields = []
 
         return view_fields
 
@@ -682,7 +688,7 @@ def _get_fields_from_view_context(self) -> List[str]:
 
         for dim_group in self.view_context.dimension_groups():
             dim_group_type_str = dim_group.get(VIEW_FIELD_TYPE_ATTRIBUTE)
-            
+
             logger.debug(
                 f"Processing dimension group from view context: {dim_group.get(NAME, 'unknown')}, type: {dim_group_type_str}"
             )
@@ -710,8 +716,11 @@ def _get_fields_from_view_context(self) -> List[str]:
                             self._get_duration_dim_group_field_name(dim_group)
                         )
                     )
-            except Exception as e:
-                logger.error(f"View-name: {self.view_context.name()}: {e}")
+            except Exception:
+                logger.error(
+                    f"Failed to process dimension group for View-name: {self.view_context.name()}",
+                    exc_info=True,
+                )
                 # Continue processing other fields instead of failing completely
                 continue