Refactor/remove opengraph (#699)

* refactor: Remove opengraph from database schema, update seed sql scripts

* refactor: Remove all references to opengraph

* fix: Remove unneccesary file

* fix: Add back migration

* refactor: Remove `seed_connectors.sql`

* fix: Migration script

* refactor: Run whole migration in transaction

* refactor: Rewrite migration a bit

Can't seem to drop `generate_opengraph_value()`...

control_plane_readme.md

@@ -29,7 +29,7 @@ The agent is a non-user-facing component which lives under [crates/agent/](crate
 
 Today this includes:
 
-* Fetching connector details, such as open-graph metadata and endpoint / resource JSON-schemas.
+* Fetching connector details, such as endpoint / resource JSON-schemas.
 * Running connector discovery operations to produce proposed catalog specifications.
 * Publishing catalog drafts by testing and then activating them into the data-plane.
 
@@ -182,15 +182,6 @@ data-plane-gateway
 
 _Note: The gateway allows for configuring the port, the Flow service ports, the signing secret, and the CORS settings. The defaults should work out of the box._
 
-### Build `fetch-open-graph`:
-
-Build the fetch-open-graph helper to the same location where the flow binaries live. This is the same path that will be provided to the agent using `--bin--dir` argument:
-
-```console
-cd fetch-open-graph/
-go build -o ~/estuary/flow/.build/package/bin/
-```
-
 ### Start the `agent`:
 
 Again from within your checkout of this repo:

crates/agent-sql/src/connector_tags.rs

@@ -66,27 +66,6 @@ where
     Ok(())
 }
 
-pub async fn update_open_graph_raw(
-    connector_id: Id,
-    open_graph_raw: Box<RawValue>,
-    txn: &mut sqlx::Transaction<'_, sqlx::Postgres>,
-) -> sqlx::Result<()> {
-    sqlx::query!(
-        r#"update connectors set
-            open_graph_raw = $2,
-            updated_at = clock_timestamp()
-        where id = $1
-        returning 1 as "must_exist";
-        "#,
-        connector_id as Id,
-        Json(open_graph_raw) as Json<Box<RawValue>>,
-    )
-    .fetch_one(txn)
-    .await?;
-
-    Ok(())
-}
-
 pub async fn update_oauth2_spec(
     connector_id: Id,
     oauth2_spec: Box<RawValue>,

crates/agent/src/connector_tags.rs

@@ -112,29 +112,6 @@ impl TagHandler {
             return Ok((row.tag_id, JobStatus::SpecFailed));
         }
 
-        let fetch_open_graph =
-            tokio::process::Command::new(format!("{}/fetch-open-graph", &self.bindir))
-                .kill_on_drop(true)
-                .arg("-url")
-                .arg(&row.external_url)
-                .output()
-                .await
-                .context("fetching open graph metadata")?;
-
-        if !fetch_open_graph.status.success() {
-            return Ok((
-                row.tag_id,
-                JobStatus::OpenGraphFailed {
-                    error: String::from_utf8_lossy(&fetch_open_graph.stderr).into(),
-                },
-            ));
-        }
-        let open_graph_raw: Box<RawValue> = serde_json::from_slice(&fetch_open_graph.stdout)
-            .context("parsing open graph response")?;
-
-        agent_sql::connector_tags::update_open_graph_raw(row.connector_id, open_graph_raw, txn)
-            .await?;
-
         /// Spec is the output shape of the `flowctl api spec` command.
         #[derive(Deserialize)]
         #[serde(rename_all = "camelCase")]

local/start-component.sh

@@ -33,11 +33,11 @@ function must_run() {
 function wait_until_listening() {
     local port="$1"
     local desc="$2"
-    log Waiting for $desc to be listening on port $port
-    while ! nc -z localhost $port; do
+    log Waiting for "$desc" to be listening on port "$port"
+    while ! nc -z localhost "$port"; do
         sleep 1
     done
-    log $desc is now listening on port $port
+    log "$desc" is now listening on port "$port"
 }
 
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )";
@@ -90,8 +90,6 @@ function start_control_plane() {
 
 function start_control_plane_agent() {
     local flow_bin_dir="$(project_dir 'flow')/.build/package/bin"
-    cd "$(project_dir 'flow')/fetch-open-graph"
-    go build -o "$flow_bin_dir"
 
     cd "$(project_dir 'flow')"
     # Start building immediately, since it could take a while

scripts/healthcheck.sh

@@ -5,8 +5,7 @@ paths=(
   /usr/local/bin/flowctl-go 
   /usr/bin/gsutil 
   /usr/local/bin/sops 
-  /usr/bin/jq 
-  /usr/local/bin/fetch-open-graph)
+  /usr/bin/jq)
 
 for i in "${paths[@]}"
 do