feat(backend): implement scalable async task queue & webhook system (#71)

core/Cargo.toml

@@ -32,6 +32,6 @@ ed25519-dalek = "2"
 sha2 = "0.10"
 rand = "0.8"
 moka = { version = "0.12", features = ["future"] }
-dashmap = "6"
+sqlx = { version = "0.7", features = ["runtime-tokio", "tls-native-tls", "postgres", "sqlite", "uuid", "chrono", "migrate"] }
 uuid = { version = "1", features = ["v4", "serde"] }
 chrono = { version = "0.4", features = ["serde"] }

core/migrations/001_create_jobs_table.sql

@@ -0,0 +1,42 @@
+-- Create jobs table for async task queue
+CREATE TABLE IF NOT EXISTS jobs (
+    id UUID PRIMARY KEY,
+    job_type VARCHAR(50) NOT NULL,
+    status VARCHAR(20) NOT NULL DEFAULT 'queued',
+    payload JSONB NOT NULL,
+    result JSONB,
+    progress_percent INTEGER NOT NULL DEFAULT 0,
+    progress_message VARCHAR(255) NOT NULL DEFAULT 'Queued',
+    webhook_url VARCHAR(500),
+    webhook_headers JSONB,
+    webhook_secret VARCHAR(255),
+    error_message TEXT,
+    error_type VARCHAR(50),
+    timeout_secs INTEGER NOT NULL DEFAULT 300,
+    retry_count INTEGER NOT NULL DEFAULT 0,
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    started_at TIMESTAMP WITH TIME ZONE,
+    completed_at TIMESTAMP WITH TIME ZONE,
+    updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
+);
+
+-- Create indexes for efficient querying
+CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status);
+CREATE INDEX IF NOT EXISTS idx_jobs_created_at ON jobs(created_at);
+CREATE INDEX IF NOT EXISTS idx_jobs_status_created_at ON jobs(status, created_at);
+
+-- Create function to update updated_at timestamp
+CREATE OR REPLACE FUNCTION update_updated_at_column()
+RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated_at = NOW();
+    RETURN NEW;
+END;
+$$ language 'plpgsql';
+
+-- Create trigger to automatically update updated_at
+DROP TRIGGER IF EXISTS update_jobs_updated_at ON jobs;
+CREATE TRIGGER update_jobs_updated_at
+    BEFORE UPDATE ON jobs
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();