feat(lookml): Use Looker API to get fields of a View #15060

askumar27 · 2025-10-21T00:00:13Z

Use Looker APIs to get fields for a Views

Current Approach

We use lkml parser to get View attributes, and this has limitation of when Views are very complex

Proposed Approach

Use Looker APIs to get fields of a View, this API need to be called via the Explore name.
As many Views exist within an Explore - number of API calls <= number of Views with local LRU cache
The code tries to groups as many Views to the same Explore as possible, so as to minimize the number of API calls. As a single View can exist in many Explore
Add reliability towards lineage extraction in the next step where SQL query is prepared based on the fields

Other changes

More debug logging
Automatic fallback to view_context based fields extraction with better error handling
Best effort fields extraction - continue on failure (atleast one field is required) instead of falling back to different upstream strategy

codecov · 2025-10-21T00:03:23Z

❌ 21 Tests Failed:

Tests completed	Failed	Passed	Skipped
4977	21	4956	38

View the top 3 failed test(s) by shortest run time

tests.unit.lookml.test_lookml_api_based_view_upstream.TestLookMLAPIBasedViewUpstream::test_api_failure_fallback

Stack Traces | 0.01s run time

self = <tests.unit.lookml.test_lookml_api_based_view_upstream.TestLookMLAPIBasedViewUpstream object at 0x7fefb56c3e20>
mock_view_context = <MagicMock spec='LookerViewContext' id='140664546232976'>
mock_looker_view_id_cache = <MagicMock spec='LookerViewIdCache' id='140664548719152'>
mock_config = <MagicMock spec='LookMLSourceConfig' id='140664545017376'>
mock_reporter = <MagicMock spec='LookMLSourceReport' id='140664548719392'>
mock_ctx = <MagicMock spec='PipelineContext' id='140664549614688'>
mock_looker_client = <MagicMock spec='LookerAPI' id='140664572475376'>
view_to_explore_map = {'test_view': 'test_explore'}

    def test_api_failure_fallback(
        self,
        mock_view_context,
        mock_looker_view_id_cache,
        mock_config,
        mock_reporter,
        mock_ctx,
        mock_looker_client,
        view_to_explore_map,
    ):
        """Test that API failures are handled gracefully."""
        # Mock the Looker client to raise an exception
        mock_looker_client.generate_sql_query.side_effect = Exception("API call failed")
    
        # This should not raise an exception, but should be handled by the fallback mechanism
        # in the factory function
        with pytest.raises(Exception, match="API call failed"):
>           LookerQueryAPIBasedViewUpstream(
                view_context=mock_view_context,
                looker_view_id_cache=mock_looker_view_id_cache,
                config=mock_config,
                reporter=mock_reporter,
                ctx=mock_ctx,
                looker_client=mock_looker_client,
                view_to_explore_map=view_to_explore_map,
            )

.../unit/lookml/test_lookml_api_based_view_upstream.py:498: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../source/looker/view_upstream.py:420: in __init__
    self._get_spr()
.../source/looker/view_upstream.py:468: in __get_spr
    if len(view_fields) > self.config.field_threshold_for_splitting:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <MagicMock spec='LookMLSourceConfig' id='140664545017376'>
name = 'field_threshold_for_splitting'

    def __getattr__(self, name):
        if name in {'_mock_methods', '_mock_unsafe'}:
            raise AttributeError(name)
        elif self._mock_methods is not None:
            if name not in self._mock_methods or name in _all_magics:
>               raise AttributeError("Mock object has no attribute %r" % name)
E               AttributeError: Mock object has no attribute 'field_threshold_for_splitting'

.../hostedtoolcache/Python/3.9.23.../x64/lib/python3.9/unittest/mock.py:630: AttributeError

During handling of the above exception, another exception occurred:

self = <tests.unit.lookml.test_lookml_api_based_view_upstream.TestLookMLAPIBasedViewUpstream object at 0x7fefb56c3e20>
mock_view_context = <MagicMock spec='LookerViewContext' id='140664546232976'>
mock_looker_view_id_cache = <MagicMock spec='LookerViewIdCache' id='140664548719152'>
mock_config = <MagicMock spec='LookMLSourceConfig' id='140664545017376'>
mock_reporter = <MagicMock spec='LookMLSourceReport' id='140664548719392'>
mock_ctx = <MagicMock spec='PipelineContext' id='140664549614688'>
mock_looker_client = <MagicMock spec='LookerAPI' id='140664572475376'>
view_to_explore_map = {'test_view': 'test_explore'}

    def test_api_failure_fallback(
        self,
        mock_view_context,
        mock_looker_view_id_cache,
        mock_config,
        mock_reporter,
        mock_ctx,
        mock_looker_client,
        view_to_explore_map,
    ):
        """Test that API failures are handled gracefully."""
        # Mock the Looker client to raise an exception
        mock_looker_client.generate_sql_query.side_effect = Exception("API call failed")
    
        # This should not raise an exception, but should be handled by the fallback mechanism
        # in the factory function
        with pytest.raises(Exception, match="API call failed"):
>           LookerQueryAPIBasedViewUpstream(
                view_context=mock_view_context,
                looker_view_id_cache=mock_looker_view_id_cache,
                config=mock_config,
                reporter=mock_reporter,
                ctx=mock_ctx,
                looker_client=mock_looker_client,
                view_to_explore_map=view_to_explore_map,
            )
E           AssertionError: Regex pattern did not match.
E            Regex: 'API call failed'
E            Input: "Mock object has no attribute 'field_threshold_for_splitting'"

.../unit/lookml/test_lookml_api_based_view_upstream.py:498: AssertionError

tests.unit.lookml.test_lookml_api_based_view_upstream.TestLookMLAPIBasedViewUpstream::test_get_sql_write_query_success

Stack Traces | 0.01s run time

self = <tests.unit.lookml.test_lookml_api_based_view_upstream.TestLookMLAPIBasedViewUpstream object at 0x7fefb56ab430>
mock_view_context = <MagicMock spec='LookerViewContext' id='140664546473392'>
mock_looker_view_id_cache = <MagicMock spec='LookerViewIdCache' id='140664499482336'>
mock_config = <MagicMock spec='LookMLSourceConfig' id='140664499662656'>
mock_reporter = <MagicMock spec='LookMLSourceReport' id='140664499564400'>
mock_ctx = <MagicMock spec='PipelineContext' id='140664499662320'>
mock_looker_client = <MagicMock spec='LookerAPI' id='140664499572800'>
view_to_explore_map = {'test_view': 'test_explore'}

    @pytest.fixture
    def upstream_instance(
        self,
        mock_view_context,
        mock_looker_view_id_cache,
        mock_config,
        mock_reporter,
        mock_ctx,
        mock_looker_client,
        view_to_explore_map,
    ):
        """Create a LookerQueryAPIBasedViewUpstream instance for testing."""
        # Mock the API response to prevent initialization errors
        mock_looker_client.generate_sql_query.return_value = [
            {"sql": "SELECT test_view.user_id FROM test_table"}
        ]
    
        # Mock the view ID cache
        mock_view_id = MagicMock(spec=LookerViewId)
        mock_view_id.get_urn.return_value = "urn:li:dataset:test"
        mock_looker_view_id_cache.get_looker_view_id.return_value = mock_view_id
    
        with patch(
            "datahub.ingestion.source.looker.view_upstream.create_lineage_sql_parsed_result"
        ) as mock_create_lineage:
            # Mock successful SQL parsing
            mock_spr = create_mock_sql_parsing_result()
            mock_create_lineage.return_value = mock_spr
    
>           return LookerQueryAPIBasedViewUpstream(
                view_context=mock_view_context,
                looker_view_id_cache=mock_looker_view_id_cache,
                config=mock_config,
                reporter=mock_reporter,
                ctx=mock_ctx,
                looker_client=mock_looker_client,
                view_to_explore_map=view_to_explore_map,
            )

.../unit/lookml/test_lookml_api_based_view_upstream.py:162: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../source/looker/view_upstream.py:420: in __init__
    self._get_spr()
.../source/looker/view_upstream.py:468: in __get_spr
    if len(view_fields) > self.config.field_threshold_for_splitting:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <MagicMock spec='LookMLSourceConfig' id='140664499662656'>
name = 'field_threshold_for_splitting'

    def __getattr__(self, name):
        if name in {'_mock_methods', '_mock_unsafe'}:
            raise AttributeError(name)
        elif self._mock_methods is not None:
            if name not in self._mock_methods or name in _all_magics:
>               raise AttributeError("Mock object has no attribute %r" % name)
E               AttributeError: Mock object has no attribute 'field_threshold_for_splitting'

.../hostedtoolcache/Python/3.9.23.../x64/lib/python3.9/unittest/mock.py:630: AttributeError

tests.unit.lookml.test_lookml_api_based_view_upstream.TestLookMLAPIBasedViewUpstream::test_create_fields_no_column_lineage