Add faceted type-ahead search capability to CSET web interface

src/CSET/cset_workflow/app/finish_website/bin/finish_website.py

@@ -14,10 +14,11 @@
 # limitations under the License.
 
 """
-Write finished status to website front page.
+Create the CSET diagnostic viewing website.
 
-Constructs the plot index, and does the final update to the workflow status on
-the front page of the web interface.
+Copies the static files that make up the web interface, constructs the plot
+index, and updates the workflow status on the front page of the
+web interface.
 """
 
 import datetime
@@ -28,71 +29,95 @@
 from importlib.metadata import version
 from pathlib import Path
 
-from CSET._common import combine_dicts, sort_dict
+from CSET._common import sort_dict
 
 logging.basicConfig(
     level=os.getenv("LOGLEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s"
 )
-
-
-def construct_index():
-    """Construct the plot index.
-
-    Index should has the form ``{"Category Name": {"recipe_id": "Plot Name"}}``
-    where ``recipe_id`` is the name of the plot's directory.
-    """
-    index = {}
-    plots_dir = Path(os.environ["CYLC_WORKFLOW_SHARE_DIR"]) / "web/plots"
-    # Loop over all diagnostics and append to index.
-    for metadata_file in plots_dir.glob("**/*/meta.json"):
-        try:
-            with open(metadata_file, "rt", encoding="UTF-8") as fp:
-                plot_metadata = json.load(fp)
-
-            category = plot_metadata["category"]
-            case_date = plot_metadata.get("case_date", "")
-            relative_url = str(metadata_file.parent.relative_to(plots_dir))
-
-            record = {
-                category: {
-                    case_date if case_date else "Aggregation": {
-                        relative_url: plot_metadata["title"].strip()
-                    }
-                }
-            }
-        except (json.JSONDecodeError, KeyError, TypeError) as err:
-            logging.error("%s is invalid, skipping.\n%s", metadata_file, err)
-            continue
-        index = combine_dicts(index, record)
-
-    # Sort index of diagnostics.
-    index = sort_dict(index)
-
-    # Write out website index.
-    with open(plots_dir / "index.json", "wt", encoding="UTF-8") as fp:
-        json.dump(index, fp, indent=2)
-
-
-def update_workflow_status():
+logger = logging.getLogger(__name__)
+
+
+def install_website_skeleton(www_root_link: Path, www_content: Path):
+    """Copy static website files and create symlink from web document root."""
+    # Remove existing link to output ahead of creating new symlink.
+    logger.info("Removing any existing output link at %s.", www_root_link)
+    www_root_link.unlink(missing_ok=True)
+
+    logger.info("Installing website files to %s.", www_content)
+    # Create directory for web content.
+    www_content.mkdir(parents=True, exist_ok=True)
+    # Copy static HTML/CSS/JS.
+    html_source = Path.cwd() / "html"
+    shutil.copytree(html_source, www_content, dirs_exist_ok=True)
+    # Create directory for plots.
+    plot_dir = www_content / "plots"
+    plot_dir.mkdir(exist_ok=True)
+
+    logger.info("Linking %s to web content.", www_root_link)
+    # Ensure parent directories of WEB_DIR exist.
+    www_root_link.parent.mkdir(parents=True, exist_ok=True)
+    # Create symbolic link to web directory.
+    # NOTE: While good for space, it means `cylc clean` removes output.
+    www_root_link.symlink_to(www_content)
+
+
+def construct_index(www_content: Path):
+    """Construct the plot index."""
+    plots_dir = www_content / "plots"
+    with open(plots_dir / "index.jsonl", "wt", encoding="UTF-8") as index_fp:
+        # Loop over all diagnostics and append to index. The glob is sorted to
+        # ensure a consistent ordering.
+        for metadata_file in sorted(plots_dir.glob("**/*/meta.json")):
+            try:
+                with open(metadata_file, "rt", encoding="UTF-8") as plot_fp:
+                    plot_metadata = json.load(plot_fp)
+                plot_metadata["path"] = str(metadata_file.parent.relative_to(plots_dir))
+                # Remove keys that are not useful for the index.
+                plot_metadata.pop("description", None)
+                plot_metadata.pop("plots", None)
+                # Sort plot metadata.
+                plot_metadata = sort_dict(plot_metadata)
+                # Write metadata into website index.
+                json.dump(plot_metadata, index_fp, separators=(",", ":"))
+                index_fp.write("\n")
+            except (json.JSONDecodeError, KeyError, TypeError) as err:
+                logger.error("%s is invalid, skipping.\n%s", metadata_file, err)
+                continue
+
+
+def update_workflow_status(www_content: Path):
     """Update the workflow status on the front page of the web interface."""
-    web_dir = Path(os.environ["CYLC_WORKFLOW_SHARE_DIR"] + "/web")
-    with open(web_dir / "status.html", "wt") as fp:
+    with open(www_content / "placeholder.html", "r+t") as fp:
+        content = fp.read()
         finish_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
-        fp.write(f"<p>Completed at {finish_time} using CSET v{version('CSET')}</p>\n")
+        status = f"Completed at {finish_time} using CSET v{version('CSET')}"
+        new_content = content.replace(
+            '<p id="workflow-status">Unknown</p>',
+            f'<p id="workflow-status">{status}</p>',
+        )
+        fp.seek(0)
+        fp.truncate()
+        fp.write(new_content)
 
 
-def copy_rose_config():
+def copy_rose_config(www_content: Path):
     """Copy the rose-suite.conf file to add to output web directory."""
     rose_suite_conf = Path(os.environ["CYLC_WORKFLOW_RUN_DIR"]) / "rose-suite.conf"
-    web_conf_file = Path(os.environ["CYLC_WORKFLOW_SHARE_DIR"]) / "web/rose-suite.conf"
-    shutil.copy(rose_suite_conf, web_conf_file)
+    web_conf_file = www_content / "rose-suite.conf"
+    shutil.copyfile(rose_suite_conf, web_conf_file)
 
 
 def run():
     """Do the final steps to finish the website."""
-    construct_index()
-    update_workflow_status()
-    copy_rose_config()
+    # Strip trailing slashes in case they have been added in the config.
+    # Otherwise they break the symlinks.
+    www_root_link = Path(os.environ["WEB_DIR"].rstrip("/"))
+    www_content = Path(os.environ["CYLC_WORKFLOW_SHARE_DIR"] + "/web")
+
+    install_website_skeleton(www_root_link, www_content)
+    construct_index(www_content)
+    update_workflow_status(www_content)
+    copy_rose_config(www_content)
 
 
 if __name__ == "__main__":  # pragma: no cover

src/CSET/cset_workflow/app/install_website_skeleton/file/html/index.html → src/CSET/cset_workflow/app/finish_website/file/html/index.html

@@ -15,10 +15,20 @@
         <header>
             <h1>CSET</h1>
             <button id="clear-plots">⎚ Clear view</button>
+            <button id="clear-query">⌫ Clear search</button>
             <button id="description-toggle">⇲ Hide description</button>
+            <search>
+                <input type="search" name="query" id="filter-query" placeholder="Filter...">
+                <!-- TODO: Add help for query syntax. -->
+                <fieldset id="filter-facets">
+                    <legend>Search facets</legend>
+                </fieldset>
+            </search>
         </header>
-        <hr>
-        <!-- Links to diagnostics get inserted here. -->
+        <ul id="diagnostics">
+            <!-- Links to diagnostics get inserted here. -->
+            <loading-throbber></loading-throbber>
+        </ul>
     </nav>
     <main>
         <article id="single-frame">

src/CSET/cset_workflow/app/install_website_skeleton/file/html/placeholder.html → src/CSET/cset_workflow/app/finish_website/file/html/placeholder.html

@@ -22,35 +22,16 @@ <h1>Select a plot from the sidebar</h1>
             Your processed diagnostics can be accessed via the buttons on the sidebar.
         </p>
         <h3>Workflow status</h3>
-        <div id="workflow-status">
-            <p>Unknown</p>
-        </div>
+        <p id="workflow-status">Unknown</p>
         <h3>CSET configuration file</h3>
         <p><a href="rose-suite.conf">rose-suite.conf</a></p>
-	<h3>Send feedback</h3>
+    	<h3>Send feedback</h3>
         <p>
-            CSET is a new system, and we would love to hear about your
-            experiences. Please tell us your highlights, issues, or
-            suggestions:
+            CSET is a new system, and we would love to hear about your experiences.
+            Please tell us your highlights, issues, or suggestions:
             <a href="https://github.com/MetOffice/CSET/issues/new/choose">Feedback via GitHub</a>
             |
             <a href="mailto:[email protected]?subject=CSET%20Feedback&body=%3E%20Thanks%20for%20sharing%20your%20feedback!%20Please%20share%20your%20issue%2C%20suggestion%2C%20or%20highlight%20in%20this%20message.%20If%20you%20are%20having%20issues%2C%20please%20also%20include%20your%20CSET%20version.%0A%0A">Feedback via email</a>
         </p>
     </main>
-    <script>
-        // Display workflow status on placeholder page.
-        const workflow_status = document.getElementById("workflow-status");
-        fetch("status.html")
-            .then((response) => {
-                if (!response.ok) {
-                    console.warn("Could not fetch the workflow status.", response.status);
-                    return;
-                }
-                response.text().then((html) => { workflow_status.innerHTML = html; });
-            })
-            // Catch non-HTTP fetch errors.
-            .catch((err) => {
-                console.error("Workflow status could not be retrieved: ", err);
-            });
-    </script>
 </body>