feat(webserver, db): refactor gitlab and github integrations into unified integration / third party repository services (TabbyML#2088)

* draft: begin implementing integration / third party repository service

* Progress on refactor

* [autofix.ci] apply automated fixes

* Use join instead of adding kind as a field of repositories

* Update schema

* [autofix.ci] apply automated fixes

* Implement sync_repositories

* Make access_token field required

* [autofix.ci] apply automated fixes

* Implement IntegrationService

* Implement service layer

* [autofix.ci] apply automated fixes

* Switch graphQL endpoints to use new service

* [autofix.ci] apply automated fixes

* Fix queries

* Fully working

* Rebase

* Update submodules

* Cleanup imports

* [autofix.ci] apply automated fixes

* Apply suggestions

* [autofix.ci] apply automated fixes

* Add back updated_at

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

Author: boxbeam

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "crates/llama-cpp-bindings/llama.cpp"]
+[submodule "crates/llama-cpp-server/llama.cpp"]
 	path = crates/llama-cpp-server/llama.cpp
 	url = https://github.com/ggerganov/llama.cpp

ee/tabby-db-macros/src/lib.rs

@@ -109,7 +109,7 @@ pub fn query_paged_as(input: TokenStream) -> TokenStream {
     let backwards = input.backwards;
     quote! {
         sqlx::query_as(&crate::make_pagination_query_with_condition({
-            let _ = sqlx::query_as!(#typ, "SELECT " + #columns + " FROM " + #table_name);
+            let _ = sqlx::query_as!(#typ, "SELECT " + #columns + " FROM (SELECT * FROM " + #table_name + ")");
             &#table_name
         }, &[ #(#column_args),* ], #limit, #skip_id, #backwards, #condition))
     }

ee/tabby-db/migrations/0029_merged-provider-tables.up.sql

@@ -2,10 +2,11 @@ CREATE TABLE integration_access_tokens(
     id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
     kind TEXT NOT NULL,
     display_name TEXT NOT NULL,
-    access_token TEXT,
+    access_token TEXT NOT NULL,
     error TEXT,
     created_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now')),
-    updated_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now'))
+    updated_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now')),
+    synced_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now'))
 );
 
 CREATE TABLE provided_repositories(
@@ -20,3 +21,9 @@ CREATE TABLE provided_repositories(
     FOREIGN KEY (integration_access_token_id) REFERENCES integration_access_tokens(id) ON DELETE CASCADE,
     CONSTRAINT idx_unique_provider_id_vendor_id UNIQUE (integration_access_token_id, vendor_id)
 );
+
+INSERT INTO integration_access_tokens(kind, display_name, access_token)
+    SELECT 'github', display_name, access_token FROM github_repository_provider WHERE access_token IS NOT NULL;
+
+INSERT INTO integration_access_tokens(kind, display_name, access_token)
+    SELECT 'gitlab', display_name, access_token FROM gitlab_repository_provider WHERE access_token IS NOT NULL;

ee/tabby-db/schema.sqlite

@@ -176,10 +176,11 @@ CREATE TABLE integration_access_tokens(
   id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
   kind TEXT NOT NULL,
   display_name TEXT NOT NULL,
-  access_token TEXT,
+  access_token TEXT NOT NULL,
   error TEXT,
   created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')),
-  updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now'))
+  updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')),
+  synced_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now'))
 );
 CREATE TABLE provided_repositories(
   id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,

ee/tabby-db/schema/schema.sql

@@ -10,17 +10,18 @@ pub struct IntegrationAccessTokenDAO {
     pub kind: String,
     pub error: Option<String>,
     pub display_name: String,
-    pub access_token: Option<String>,
+    pub access_token: String,
     pub created_at: DateTimeUtc,
     pub updated_at: DateTimeUtc,
+    pub synced_at: DateTimeUtc,
 }
 
 impl DbConn {
     pub async fn create_integration_access_token(
         &self,
-        kind: &str,
-        name: &str,
-        access_token: &str,
+        kind: String,
+        name: String,
+        access_token: String,
     ) -> Result<i64> {
         let res = query!(
             "INSERT INTO integration_access_tokens(kind, display_name, access_token) VALUES (?, ?, ?);",
@@ -42,8 +43,9 @@ impl DbConn {
                 error,
                 display_name,
                 access_token,
-                created_at AS "created_at: DateTimeUtc",
-                updated_at AS "updated_at: DateTimeUtc"
+                updated_at,
+                created_at,
+                synced_at
             FROM integration_access_tokens WHERE id = ?;"#,
             id
         )
@@ -52,10 +54,14 @@ impl DbConn {
         Ok(provider)
     }
 
-    pub async fn delete_integration_access_token(&self, id: i64) -> Result<()> {
-        let res = query!("DELETE FROM integration_access_tokens WHERE id = ?;", id)
-            .execute(&self.pool)
-            .await?;
+    pub async fn delete_integration_access_token(&self, id: i64, kind: &str) -> Result<()> {
+        let res = query!(
+            "DELETE FROM integration_access_tokens WHERE id = ? AND kind = ?;",
+            id,
+            kind
+        )
+        .execute(&self.pool)
+        .await?;
         if res.rows_affected() != 1 {
             return Err(anyhow!("No integration access token to delete"));
         }
@@ -65,19 +71,21 @@ impl DbConn {
     pub async fn update_integration_access_token(
         &self,
         id: i64,
+        kind: &str,
         display_name: String,
         access_token: Option<String>,
     ) -> Result<()> {
         let access_token = match access_token {
-            Some(access_token) => Some(access_token),
+            Some(access_token) => access_token,
             None => self.get_integration_access_token(id).await?.access_token,
         };
 
         let res = query!(
-            "UPDATE integration_access_tokens SET display_name = ?, access_token=? WHERE id = ?;",
+            "UPDATE integration_access_tokens SET display_name = ?, access_token = ?, updated_at = DATETIME('now') WHERE id = ? AND kind = ?;",
             display_name,
             access_token,
-            id
+            id,
+            kind
         )
         .execute(&self.pool)
         .await?;
@@ -97,7 +105,7 @@ impl DbConn {
         error: Option<String>,
     ) -> Result<()> {
         query!(
-            "UPDATE integration_access_tokens SET updated_at = DATETIME('now'), error = ? WHERE id = ?",
+            "UPDATE integration_access_tokens SET synced_at = DATETIME('now'), error = ? WHERE id = ?",
             error,
             id
         )
@@ -126,7 +134,7 @@ impl DbConn {
         });
         conditions.extend(id_condition);
 
-        let kind_condition = kind.map(|kind| format!("kind = {kind}"));
+        let kind_condition = kind.map(|kind| format!("kind = '{kind}'"));
         conditions.extend(kind_condition);
 
         let condition = (!conditions.is_empty()).then(|| conditions.join(" AND "));
@@ -140,8 +148,9 @@ impl DbConn {
                 "error",
                 "display_name",
                 "access_token",
-                "created_at" as "created_at: DateTimeUtc",
-                "updated_at" as "updated_at: DateTimeUtc"
+                "created_at",
+                "updated_at",
+                "synced_at"
             ],
             limit,
             skip_id,

ee/tabby-db/schema/schema.svg

@@ -12,9 +12,11 @@ use chrono::{DateTime, Duration, NaiveDateTime, Utc};
 pub use email_setting::EmailSettingDAO;
 pub use github_repository_provider::{GithubProvidedRepositoryDAO, GithubRepositoryProviderDAO};
 pub use gitlab_repository_provider::{GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO};
+pub use integration_access_tokens::IntegrationAccessTokenDAO;
 pub use invitations::InvitationDAO;
 pub use job_runs::JobRunDAO;
 pub use oauth_credential::OAuthCredentialDAO;
+pub use provided_repositories::ProvidedRepositoryDAO;
 pub use repositories::RepositoryDAO;
 pub use server_setting::ServerSettingDAO;
 use sqlx::{

ee/tabby-db/src/integration_access_tokens.rs

@@ -61,7 +61,7 @@ impl DbConn {
 
     pub async fn list_provided_repositories(
         &self,
-        provider_ids: Vec<i64>,
+        integration_ids: Vec<i64>,
         kind: Option<String>,
         active: Option<bool>,
         limit: Option<usize>,
@@ -70,35 +70,37 @@ impl DbConn {
     ) -> Result<Vec<ProvidedRepositoryDAO>> {
         let mut conditions = vec![];
 
-        let provider_ids = provider_ids
+        let integration_ids = integration_ids
             .into_iter()
             .map(|id| id.to_string())
             .collect::<Vec<_>>()
             .join(", ");
-        if !provider_ids.is_empty() {
-            conditions.push(format!("access_token_provider_id IN ({provider_ids})"));
+        if !integration_ids.is_empty() {
+            conditions.push(format!(
+                "integration_access_token_id IN ({integration_ids})"
+            ));
         }
 
         let active_filter = active.map(|active| format!("active = {active}"));
         conditions.extend(active_filter);
 
-        let kind_filter = kind.map(|kind| format!("kind = {kind}"));
+        let kind_filter = kind.map(|kind| format!("kind = '{kind}'"));
         conditions.extend(kind_filter);
 
         let condition = (!conditions.is_empty()).then(|| conditions.join(" AND "));
 
         let repos = query_paged_as!(
             ProvidedRepositoryDAO,
-            "provided_repositories",
+            "provided_repositories JOIN integration_access_tokens ON integration_access_token_id = integration_access_tokens.id",
             [
                 "id",
                 "vendor_id",
                 "name",
                 "git_url",
                 "active",
                 "integration_access_token_id",
-                "created_at" as "created_at: DateTimeUtc",
-                "updated_at" as "updated_at: DateTimeUtc"
+                "created_at",
+                "updated_at"
             ],
             limit,
             skip_id,
@@ -128,3 +130,16 @@ impl DbConn {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::DbConn;
+
+    #[tokio::test]
+    async fn test_list_provided_repositories() {
+        let db = DbConn::new_in_memory().await.unwrap();
+        db.list_github_provided_repositories(vec![], None, None, None, false)
+            .await
+            .unwrap();
+    }
+}