[WIP] add extracting table info into a csv (#41)

* add extracting table info into a csv

* format

* _

* unused import

Author: jeromedockes

src/pubget/_coordinates.py

@@ -99,6 +99,8 @@ def _extract_coordinates_from_article_dir(
                 f"in article pmcid {pmcid}"
             )
             continue
+        if coordinates.empty:
+            continue
         coordinates["pmcid"] = pmcid
         coordinates["table_id"] = table_info["table_id"]
         coordinates["table_label"] = table_info["table_label"]

src/pubget/_data_extraction.py

@@ -26,6 +26,7 @@
 from pubget._coordinates import CoordinateExtractor
 from pubget._links import LinkExtractor, neurovault_id_extractors
 from pubget._metadata import MetadataExtractor
+from pubget._table_info import TableInfoExtractor
 from pubget._text import TextExtractor
 from pubget._typing import (
     ArgparseActions,
@@ -208,6 +209,7 @@ def _get_data_extractors() -> List[Extractor]:
         MetadataExtractor(),
         AuthorsExtractor(),
         TextExtractor(),
+        TableInfoExtractor(),
         CoordinateExtractor(),
         CoordinateSpaceExtractor(),
         LinkExtractor(),

src/pubget/_table_info.py

@@ -0,0 +1,51 @@
+"""Extracting table info from article directories."""
+import json
+from pathlib import Path
+from typing import Dict
+
+import pandas as pd
+from lxml import etree
+
+from pubget import _utils
+from pubget._typing import Extractor, Records
+
+_TABLE_FIELDS = (
+    "pmcid",
+    "table_id",
+    "table_label",
+    "table_caption",
+    "table_foot",
+    "n_header_rows",
+    "table_data_file",
+)
+
+
+class TableInfoExtractor(Extractor):
+    """Read table info JSON files so they can be assembled in a single CSV."""
+
+    fields = _TABLE_FIELDS
+    name = "tables"
+
+    def extract(
+        self,
+        article: etree.ElementTree,
+        article_dir: Path,
+        previous_extractors_output: Dict[str, Records],
+    ) -> pd.DataFrame:
+        del article, previous_extractors_output
+        all_tables_info = []
+        pmcid = _utils.get_pmcid_from_article_dir(article_dir)
+        for table_json in _utils.get_table_info_files_from_article_dir(
+            article_dir
+        ):
+            table_info = json.loads(table_json.read_text("UTF-8"))
+            table_info["pmcid"] = pmcid
+            table_info["table_data_file"] = str(
+                table_json.with_name(
+                    table_info["table_data_file"]
+                ).relative_to(article_dir.parents[2])
+            )
+            all_tables_info.append(table_info)
+        if all_tables_info:
+            return pd.DataFrame(all_tables_info)[list(self.fields)]
+        return pd.DataFrame(columns=list(self.fields))

src/pubget/_utils.py

@@ -10,7 +10,7 @@
 import shutil
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, Generator, Optional, Tuple, Union
+from typing import Any, Dict, Generator, List, Optional, Tuple, Union
 
 import pandas as pd
 from lxml import etree
@@ -160,13 +160,15 @@ def read_article_table(
     return table_info, table_data
 
 
+def get_table_info_files_from_article_dir(article_dir: Path) -> List[Path]:
+    return sorted(article_dir.joinpath("tables").glob("table_*_info.json"))
+
+
 def get_tables_from_article_dir(
     article_dir: Path,
 ) -> Generator[Tuple[Dict[str, Any], pd.DataFrame], None, None]:
     """Load information and data for all tables belonging to an article."""
-    for table_info_json in sorted(
-        article_dir.joinpath("tables").glob("table_*_info.json")
-    ):
+    for table_info_json in get_table_info_files_from_article_dir(article_dir):
         yield read_article_table(table_info_json)