unifies ResourceHints typed dict

Author: rudolfix

dlt/extract/extractors.py

@@ -130,7 +130,7 @@ def write_items(self, resource: DltResource, items: TDataItems, meta: Any) -> No
             # convert to table meta if created table variant so item is assigned to this table
             if meta.create_table_variant:
                 # name in hints meta must be a string, otherwise merge_hints would fail
-                meta = TableNameMeta(meta.hints["name"])  # type: ignore[arg-type]
+                meta = TableNameMeta(meta.hints["table_name"])  # type: ignore[arg-type]
             self._reset_contracts_cache()
 
         if table_name := self._get_static_table_name(resource, meta):

dlt/extract/hints.py

@@ -42,23 +42,23 @@
 
 
 class TResourceHintsBase(TypedDict, total=False):
+    table_name: Optional[TTableHintTemplate[str]]
     write_disposition: Optional[TTableHintTemplate[TWriteDispositionConfig]]
     parent: Optional[TTableHintTemplate[str]]
     primary_key: Optional[TTableHintTemplate[TColumnNames]]
+    columns: Optional[TTableHintTemplate[TAnySchemaColumns]]
     schema_contract: Optional[TTableHintTemplate[TSchemaContract]]
     table_format: Optional[TTableHintTemplate[TTableFormat]]
+    file_format: TTableHintTemplate[TFileFormat]
     merge_key: Optional[TTableHintTemplate[TColumnNames]]
+    nested_hints: Optional[Dict[TColumnNames, "TResourceHintsBase"]]
 
 
 class TResourceHints(TResourceHintsBase, total=False):
-    name: TTableHintTemplate[str]
     # description: TTableHintTemplate[str]
-    # table_sealed: Optional[bool]
-    columns: TTableHintTemplate[TTableSchemaColumns]
-    incremental: Incremental[Any]
-    file_format: TTableHintTemplate[TFileFormat]
+    incremental: Optional[Incremental[Any]]
     validator: ValidateItem
-    original_columns: TTableHintTemplate[TAnySchemaColumns]
+    original_columns: Optional[TTableHintTemplate[TAnySchemaColumns]]
 
 
 class HintsMeta:
@@ -98,7 +98,7 @@ def make_hints(
         file_format=file_format,  # type: ignore
     )
     if not table_name:
-        new_template.pop("name")
+        new_template.pop("table_name")
     if not write_disposition and "write_disposition" in new_template:
         new_template.pop("write_disposition")
     # remember original columns and set template columns
@@ -139,7 +139,7 @@ def table_name(self) -> TTableHintTemplate[str]:
         if self._table_name_hint_fun:
             return self._table_name_hint_fun
         # get table name or default name
-        return self._hints.get("name") or self.name if self._hints else self.name
+        return self._hints.get("table_name") or self.name if self._hints else self.name
 
     @table_name.setter
     def table_name(self, value: TTableHintTemplate[str]) -> None:
@@ -158,7 +158,7 @@ def write_disposition(self, value: TTableHintTemplate[TWriteDispositionConfig])
     @property
     def columns(self) -> TTableHintTemplate[TTableSchemaColumns]:
         """Gets columns' schema that can be modified in place"""
-        return None if self._hints is None else self._hints.get("columns")
+        return None if self._hints is None else self._hints.get("columns")  # type: ignore[return-value]
 
     @property
     def schema_contract(self) -> TTableHintTemplate[TSchemaContract]:
@@ -187,8 +187,8 @@ def compute_table_schema(self, item: TDataItem = None, meta: Any = None) -> TTab
 
         # resolve a copy of a held template
         table_template = self._clone_hints(table_template)
-        if "name" not in table_template:
-            table_template["name"] = self.name
+        if "table_name" not in table_template:
+            table_template["table_name"] = self.name
 
         # if table template present and has dynamic hints, the data item must be provided.
         if self._table_name_hint_fun and item is None:
@@ -276,9 +276,9 @@ def apply_hints(
             t = self._clone_hints(t)
             if table_name is not None:
                 if table_name:
-                    t["name"] = table_name
+                    t["table_name"] = table_name
                 else:
-                    t.pop("name", None)
+                    t.pop("table_name", None)
             if parent_table_name is not None:
                 if parent_table_name:
                     t["parent"] = parent_table_name
@@ -296,6 +296,7 @@ def apply_hints(
                     # normalize columns
                     columns = ensure_table_schema_columns(columns)
                     # this updates all columns with defaults
+                    assert isinstance(t["columns"], dict)
                     t["columns"] = merge_columns(t["columns"], columns, merge_columns=True)
                 else:
                     # set to empty columns
@@ -354,7 +355,8 @@ def _set_hints(
         DltResourceHints.validate_dynamic_hints(hints_template)
         DltResourceHints.validate_write_disposition_hint(hints_template.get("write_disposition"))
         if create_table_variant:
-            table_name: str = hints_template["name"]  # type: ignore[assignment]
+            # for table variants, table name must be a str
+            table_name: str = hints_template["table_name"]  # type: ignore[assignment]
             # incremental cannot be specified in variant
             if hints_template.get("incremental"):
                 raise InconsistentTableTemplate(
@@ -388,7 +390,7 @@ def merge_hints(
         self, hints_template: TResourceHints, create_table_variant: bool = False
     ) -> None:
         self.apply_hints(
-            table_name=hints_template.get("name"),
+            table_name=hints_template.get("table_name"),
             parent_table_name=hints_template.get("parent"),
             write_disposition=hints_template.get("write_disposition"),
             columns=hints_template.get("original_columns"),

dlt/sources/rest_api/typing.py

@@ -254,9 +254,7 @@ class ProcessingSteps(TypedDict):
 class ResourceBase(TResourceHintsBase, total=False):
     """Defines hints that may be passed to `dlt.resource` decorator"""
 
-    table_name: Optional[TTableHintTemplate[str]]
     max_table_nesting: Optional[int]
-    columns: Optional[TTableHintTemplate[TAnySchemaColumns]]
     selected: Optional[bool]
     parallelized: Optional[bool]
     processing_steps: Optional[List[ProcessingSteps]]