diff --git a/default_handlers.txt b/default_handlers.txt index d619d28bf39..f3700f8ae88 100644 --- a/default_handlers.txt +++ b/default_handlers.txt @@ -52,6 +52,7 @@ twillio twitter web youtube +xero zendesk zipcodebase zotero \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/README.md b/mindsdb/integrations/handlers/xero_handler/README.md new file mode 100644 index 00000000000..aaa629c6f2a --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/README.md @@ -0,0 +1,294 @@ +# Xero Handler for MindsDB + +This handler provides read-only access to Xero Accounting API data through MindsDB. It supports OAuth2 authentication with automatic token refresh and exposes 10 key accounting endpoints as queryable tables. + +## Installation + +The Xero handler is automatically installed with MindsDB. If you need to install it manually: + +```bash +pip install xero-python requests +``` + +## Supported Tables + +The handler provides access to the following Xero Accounting API endpoints: + +1. **budgets** - Budget information +2. **contacts** - Customer and supplier contacts +3. **invoices** - Invoice records +4. **items** - Products and services +5. **overpayments** - Overpayment records +6. **payments** - Payment records +7. **purchase_orders** - Purchase order records +8. **quotes** - Sales quote records +9. **repeating_invoices** - Repeating invoice templates +10. **accounts** - Chart of accounts + +All tables support SELECT queries with WHERE, ORDER BY, and LIMIT clauses. + +## Connection Setup + +### 1. Create a Xero App + +1. Go to [Xero Developer Portal](https://developer.xero.com) +2. Sign in with your Xero account +3. Click "My Apps" and create a new app +4. Choose "Web" as the app type +5. Fill in the app details: + - App Name + - Company URL + - Redirect URI (e.g., `http://localhost:3000/callback`) +6. Note your Client ID and Client Secret + +### 2. Configure the Connection in MindsDB + +#### Initial Connection (First Time) + +```sql +CREATE DATABASE xero_connection +WITH ENGINE = 'xero', +PARAMETERS = { + 'client_id': 'your_client_id', + 'client_secret': 'your_client_secret', + 'redirect_uri': 'http://localhost:3000/callback' +}; +``` + +When you run this for the first time, MindsDB will return an authorization URL. Visit this URL and authorize the application. You'll receive an authorization code. + +#### With Authorization Code + +After authorizing, provide the authorization code to complete the setup: + +```sql +CREATE DATABASE xero_connection +WITH ENGINE = 'xero', +PARAMETERS = { + 'client_id': 'your_client_id', + 'client_secret': 'your_client_secret', + 'redirect_uri': 'http://localhost:3000/callback', + 'code': 'your_authorization_code' +}; +``` + +#### Specifying Tenant (Organization) + +If you have access to multiple Xero organizations, you can specify which one to use: + +```sql +CREATE DATABASE xero_connection +WITH ENGINE = 'xero', +PARAMETERS = { + 'client_id': 'your_client_id', + 'client_secret': 'your_client_secret', + 'redirect_uri': 'http://localhost:3000/callback', + 'code': 'your_authorization_code', + 'tenant_id': 'your_tenant_id' +}; +``` + +If not specified, the handler will use the first accessible organization. + +## Token Injection (Backend Integration) + +For backend systems that manage OAuth2 tokens centrally, you can inject tokens directly without going through the authorization flow: + +### Basic Token Injection + +If you have a valid access token: + +```sql +CREATE DATABASE xero_backend +WITH ENGINE = 'xero', +PARAMETERS = { + 'client_id': 'your_client_id', + 'client_secret': 'your_client_secret', + 'access_token': 'xero_access_token', + 'tenant_id': 'xero_tenant_id' +}; +``` + +### Token Injection with Refresh + +For better reliability, provide both access and refresh tokens. The handler will automatically refresh the access token when needed: + +```sql +CREATE DATABASE xero_backend +WITH ENGINE = 'xero', +PARAMETERS = { + 'client_id': 'your_client_id', + 'client_secret': 'your_client_secret', + 'access_token': 'xero_access_token', + 'refresh_token': 'xero_refresh_token', + 'expires_at': '2024-11-30T12:00:00', -- Optional: ISO 8601 format or Unix timestamp + 'tenant_id': 'xero_tenant_id' +}; +``` + +### How Token Injection Works + +1. **Token Validation**: The handler checks if the provided access token is expired or expiring soon (within 5 minutes) +2. **Automatic Refresh**: If expired and a refresh_token is provided, the handler automatically refreshes the token +3. **Token Storage**: Refreshed tokens are stored for future use +4. **Grace Period**: Tokens are considered expired if they expire within 5 minutes, allowing proactive refresh + +**Note:** Token refresh requires valid `client_id` and `client_secret` in the connection parameters, even when using token injection. + +## Usage Examples + +### Select All Contacts + +```sql +SELECT * FROM xero_connection.contacts LIMIT 10; +``` + +### Find Recent Invoices + +```sql +SELECT invoice_number, contact_name, total, invoice_date +FROM xero_connection.invoices +WHERE status = 'DRAFT' +ORDER BY invoice_date DESC +LIMIT 20; +``` + +### Get All Active Accounts + +```sql +SELECT code, name, type, currency_code +FROM xero_connection.accounts +WHERE status = 'ACTIVE' +ORDER BY code; +``` + +### View Purchase Orders + +```sql +SELECT purchase_order_number, contact_name, total, order_date +FROM xero_connection.purchase_orders +WHERE status = 'AUTHORISED' +LIMIT 50; +``` + +### Check Available Items + +```sql +SELECT code, description, is_tracked_as_inventory +FROM xero_connection.items +LIMIT 30; +``` + +### Get Payments Summary + +```sql +SELECT invoice_id, amount, payment_type, reference +FROM xero_connection.payments +WHERE amount > 100 +ORDER BY updated_utc DESC; +``` + +### View Quotes + +```sql +SELECT quote_number, contact_name, total, quote_date, expiry_date +FROM xero_connection.quotes +WHERE status = 'DRAFT' +ORDER BY quote_date DESC; +``` + +## Authentication Details + +### OAuth2 Flow + +This handler implements the OAuth2 Authorization Code flow as documented in the [Xero OAuth2 Guide](https://developer.xero.com/documentation/guides/oauth2/auth-flow). + +1. **Authorization**: User is directed to Xero's authorization endpoint +2. **Code Exchange**: Authorization code is exchanged for access and refresh tokens +3. **Token Storage**: Tokens are stored securely in MindsDB's encrypted storage +4. **Token Refresh**: Access tokens are automatically refreshed when expired (Xero tokens expire every 30 minutes) + +### Scopes + +The handler requests the following OAuth2 scopes: +- `openid` - OpenID Connect +- `profile` - User profile information +- `email` - User email +- `accounting.transactions` - Read access to transactions +- `accounting.settings` - Read access to settings +- `offline_access` - Refresh token access + +## Limitations + +- **Read-only**: This handler provides read-only access to the Xero API. Create, update, and delete operations are not supported. +- **Rate Limiting**: Xero API has rate limits. Be mindful of the number of queries you execute. +- **Token Expiration**: Access tokens expire after 30 minutes. The handler automatically refreshes them as needed. +- **Tenant Selection**: If you have access to multiple Xero organizations, you must specify the `tenant_id` or the handler will use the first available organization. + +## Troubleshooting + +### "Authorization required" Error + +If you see this error during connection, you need to provide an authorization code: +1. Visit the URL provided in the error message +2. Authorize the application +3. Copy the authorization code +4. Update your connection parameters with the code + +### "No Xero tenants found" Error + +This typically means: +1. The user account has no Xero organizations/tenants +2. The user needs to create a Xero organization first +3. The user account permissions are restricted + +### Token Refresh Failures + +If token refresh fails: +1. Delete the current connection and create a new one +2. Go through the authorization flow again +3. This will generate new tokens + +## Advanced Usage + +### Query Filtering + +You can use WHERE clauses to filter data: + +```sql +SELECT * FROM xero_connection.invoices +WHERE invoice_date >= '2024-01-01' +AND status = 'DRAFT' +AND total > 1000; +``` + +### Ordering and Limiting + +```sql +SELECT contact_name, total +FROM xero_connection.payments +ORDER BY total DESC +LIMIT 5; +``` + +### Column Selection + +Only select the columns you need for better performance: + +```sql +SELECT invoice_number, total, due_date +FROM xero_connection.invoices; +``` + +## API Reference + +For detailed information about the Xero Accounting API, visit: +- [Xero API Documentation](https://developer.xero.com/documentation/api/accounting/overview) +- [Xero Python SDK](https://github.com/XeroAPI/xero-python) + +## Support + +For issues or questions: +1. Check the [Xero Developer Community](https://community.xero.com/developer) +2. Review the [xero-python SDK documentation](https://github.com/XeroAPI/xero-python) +3. Check MindsDB documentation for handler-specific issues diff --git a/mindsdb/integrations/handlers/xero_handler/TABLES_REFERENCE.md b/mindsdb/integrations/handlers/xero_handler/TABLES_REFERENCE.md new file mode 100644 index 00000000000..c904e24d40d --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/TABLES_REFERENCE.md @@ -0,0 +1,730 @@ +# Xero Connector Tables Reference + +This document provides comprehensive descriptions of all tables available in the MindsDB Xero connector. Each table description includes its purpose, the type of data it contains, and key columns that are most relevant for queries. + +## Table of Contents + +- [Data Tables (Transactional Data)](#data-tables-transactional-data) + - [accounts](#accounts) + - [bank_transactions](#bank_transactions) + - [bank_transfers](#bank_transfers) + - [budgets](#budgets) + - [contacts](#contacts) + - [contact_groups](#contact_groups) + - [credit_notes](#credit_notes) + - [invoices](#invoices) + - [items](#items) + - [journals](#journals) + - [manual_journals](#manual_journals) + - [organisations](#organisations) + - [overpayments](#overpayments) + - [payments](#payments) + - [prepayments](#prepayments) + - [purchase_orders](#purchase_orders) + - [quotes](#quotes) + - [repeating_invoices](#repeating_invoices) +- [Report Tables (Financial Reports)](#report-tables-financial-reports) + - [balance_sheet_report](#balance_sheet_report) + - [profit_loss_report](#profit_loss_report) + - [trial_balance_report](#trial_balance_report) + - [bank_summary_report](#bank_summary_report) + - [budget_summary_report](#budget_summary_report) + - [executive_summary_report](#executive_summary_report) +- [Usage Notes](#usage-notes) + +--- + +## Data Tables (Transactional Data) + +### accounts + +**Description:** Chart of accounts containing all account records used for categorizing transactions. Includes account codes, names, types (revenue, expense, asset, liability, equity, bank), tax types, currency information, and banking details. Essential for understanding how transactions are categorized across the organization's financial structure. + +**Key Columns:** +- `account_id` - Unique identifier for the account +- `code` - Account code used in the chart of accounts +- `name` - Display name of the account +- `type` - Account type (e.g., REVENUE, EXPENSE, BANK, CURRENT, FIXED) +- `account_class` - Classification (ASSET, LIABILITY, EQUITY, REVENUE, EXPENSE) +- `tax_type` - Default tax type for transactions on this account +- `status` - Account status (ACTIVE, ARCHIVED, DELETED) +- `currency_code` - Currency for the account +- `bank_account_number` - Bank account number if applicable +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Financial categorization, chart of accounts management, transaction classification, tax reporting + +--- + +### bank_transactions + +**Description:** All transactions recorded against bank accounts including money in and money out. Contains spend money, receive money, and bank transfer transactions with full contact details, line items, tax information, and reconciliation status. Critical for cash flow analysis and bank reconciliation. + +**Key Columns:** +- `bank_transaction_id` - Unique identifier for the transaction +- `type` - Transaction type (SPEND, RECEIVE, SPEND-OVERPAYMENT, RECEIVE-OVERPAYMENT, etc.) +- `contact_id` - Associated contact identifier +- `contact_name` - Name of the contact +- `date` - Transaction date +- `total` - Total transaction amount including tax +- `sub_total` - Transaction amount excluding tax +- `total_tax` - Total tax amount +- `status` - Transaction status (AUTHORISED, DELETED, VOIDED) +- `is_reconciled` - Whether the transaction has been reconciled +- `line_items` - JSON array of line item details +- `currency_code` - Transaction currency +- `reference` - Transaction reference number +- `bank_account_*` - Bank account details (code, name, type, etc.) + +**Use Cases:** Cash flow analysis, bank reconciliation, expense tracking, revenue analysis, tax reporting + +--- + +### bank_transfers + +**Description:** Internal transfers between bank accounts within Xero. Tracks money movement between different bank accounts, including transfer dates, amounts, and reconciliation status. Useful for understanding internal cash movements. + +**Key Columns:** +- `bank_transfer_id` - Unique identifier for the transfer +- `from_bank_account_*` - Source bank account details +- `to_bank_account_*` - Destination bank account details +- `amount` - Transfer amount +- `date` - Transfer date +- `has_attachments` - Whether supporting documents are attached +- `currency_code` - Transfer currency + +**Use Cases:** Internal fund transfers, cash management, bank reconciliation, liquidity management + +--- + +### budgets + +**Description:** Budget records containing planned financial targets organized by account and time period. Includes budget lines with tracking category breakdowns. Essential for comparing actual performance against planned targets and variance analysis. + +**Key Columns:** +- `budget_id` - Unique identifier for the budget +- `type` - Budget type (OVERALL or specific tracking option) +- `description` - Budget description +- `status` - Budget status (ACTIVE, DRAFT) +- `budget_lines` - JSON array of budget line items by account and period +- `tracking` - JSON array of tracking category assignments +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Budget vs. actual analysis, financial planning, variance reporting, performance management + +--- + +### contacts + +**Description:** Customer and supplier contact records including individuals and organizations. Contains names, contact details, addresses, phone numbers, tax information, bank details, and customer/supplier status flags. Central to managing business relationships and transaction associations. + +**Key Columns:** +- `contact_id` - Unique identifier for the contact +- `name` - Contact name (company or person) +- `first_name` - First name (for individuals) +- `last_name` - Last name (for individuals) +- `email_address` - Primary email address +- `is_customer` - Flag indicating if contact is a customer +- `is_supplier` - Flag indicating if contact is a supplier +- `contact_status` - Contact status (ACTIVE, ARCHIVED, GDPR_REQUEST) +- `addresses` - JSON array of postal and street addresses +- `phones` - JSON array of phone numbers +- `tax_number` - Tax identification number +- `default_currency` - Default currency for transactions +- `company_number` - Company registration number +- `bank_account_details` - Bank account information +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Customer/supplier management, contact information, CRM integration, transaction filtering + +--- + +### contact_groups + +**Description:** Groupings of contacts for organizational and reporting purposes. Allows categorization of contacts (e.g., "VIP Customers", "Local Suppliers") for targeted communication and analysis. + +**Key Columns:** +- `contact_group_id` - Unique identifier for the group +- `name` - Group name +- `status` - Group status (ACTIVE, DELETED) + +**Use Cases:** Contact segmentation, targeted reporting, customer categorization, marketing lists + +--- + +### credit_notes + +**Description:** Credit notes issued to customers or received from suppliers, representing refunds or corrections to invoices. Includes allocation details showing how credits are applied against invoices, remaining credit amounts, and full line item breakdowns. + +**Key Columns:** +- `credit_note_id` - Unique identifier for the credit note +- `credit_note_number` - Credit note number +- `type` - Type (ACCPAYCREDIT for bills, ACCRECCREDIT for sales) +- `contact_id` - Associated contact identifier +- `contact_name` - Contact name +- `date` - Credit note date +- `total` - Total credit amount including tax +- `sub_total` - Credit amount excluding tax +- `total_tax` - Total tax amount +- `remaining_credit` - Unallocated credit remaining +- `allocations` - JSON array showing how credit is allocated to invoices +- `line_items` - JSON array of line item details +- `status` - Credit note status (DRAFT, SUBMITTED, AUTHORISED, PAID, VOIDED, DELETED) +- `currency_code` - Transaction currency +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Returns processing, invoice corrections, credit management, refund tracking + +--- + +### invoices + +**Description:** Sales invoices (accounts receivable) and bills (accounts payable) representing amounts owed to or by the organization. Contains complete transaction details including line items, payments applied, credit notes, amounts due/paid, tax calculations, and due dates. Core to revenue and expense tracking. + +**Key Columns:** +- `invoice_id` - Unique identifier for the invoice +- `invoice_number` - Invoice number +- `type` - Invoice type (ACCPAY for bills, ACCREC for sales invoices) +- `contact_id` - Associated contact identifier +- `contact_name` - Contact name +- `date` - Invoice date +- `due_date` - Payment due date +- `total` - Total invoice amount including tax +- `sub_total` - Invoice amount excluding tax +- `total_tax` - Total tax amount +- `amount_due` - Outstanding amount still owed +- `amount_paid` - Amount already paid +- `amount_credited` - Amount credited via credit notes +- `status` - Invoice status (DRAFT, SUBMITTED, AUTHORISED, PAID, VOIDED, DELETED) +- `line_items` - JSON array of line item details +- `payments` - JSON array of payment records +- `credit_notes` - JSON array of applied credit notes +- `currency_code` - Transaction currency +- `currency_rate` - Exchange rate if multi-currency +- `reference` - Invoice reference +- `branding_theme_id` - Applied branding theme +- `has_attachments` - Whether supporting documents are attached +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Accounts receivable/payable, revenue recognition, expense tracking, aging reports, payment collection + +--- + +### items + +**Description:** Product and service catalog items that can be used on invoices, bills, and quotes. Includes item codes, descriptions, pricing, account codes for revenue/expense, and inventory tracking flags. Streamlines line item entry and ensures consistent pricing. + +**Key Columns:** +- `item_id` - Unique identifier for the item +- `code` - Item code/SKU +- `name` - Item name +- `description` - Item description for sales +- `purchase_description` - Description for purchases +- `sales_details_unit_price` - Default sales price +- `sales_details_account_code` - Revenue account code +- `sales_details_tax_type` - Sales tax type +- `purchase_details` - JSON object with purchase details +- `is_tracked_as_inventory` - Whether item quantity is tracked +- `is_sold` - Whether item can be sold +- `is_purchased` - Whether item can be purchased +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Product catalog, pricing management, inventory tracking, consistent invoicing, cost analysis + +--- + +### journals + +**Description:** System-generated journal entries showing the double-entry bookkeeping records for all transactions. Each journal contains debit and credit lines that maintain the accounting equation. Essential for audit trails and understanding the underlying accounting entries. + +**Key Columns:** +- `journal_id` - Unique identifier for the journal +- `journal_number` - Sequential journal number +- `journal_date` - Date of the journal entry +- `created_date_utc` - Creation timestamp +- `journal_lines` - JSON array of debit and credit line details including accounts, amounts, descriptions + +**Use Cases:** Audit trails, accounting verification, double-entry bookkeeping analysis, transaction investigation + +--- + +### manual_journals + +**Description:** User-created journal entries for adjustments, corrections, and transactions not captured through standard workflows (like depreciation, accruals, or corrections). Includes narration for documentation and complete debit/credit line details. + +**Key Columns:** +- `manual_journal_id` - Unique identifier for the manual journal +- `date` - Journal entry date +- `status` - Journal status (DRAFT, POSTED, DELETED, VOIDED) +- `narration` - Description/explanation of the journal entry +- `journal_lines` - JSON array of debit and credit lines +- `line_amount_types` - Whether amounts include tax (INCLUSIVE, EXCLUSIVE, NOTAX) +- `show_on_cash_basis_reports` - Whether to include in cash basis reporting +- `has_attachments` - Whether supporting documents are attached +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Accounting adjustments, period-end entries, depreciation, accruals, error corrections + +--- + +### organisations + +**Description:** Organization profile information including legal details, tax settings, financial year configuration, base currency, and contact information. Typically contains a single record representing the connected Xero organization. Important for understanding business context. + +**Key Columns:** +- `organisation_id` - Unique identifier (tenant ID) +- `name` - Organization trading name +- `legal_name` - Legal entity name +- `country_code` - Country of operation +- `base_currency` - Base currency code +- `tax_number` - Tax identification number +- `tax_number_name` - Tax number type (e.g., ABN, VAT, GST) +- `financial_year_end_month` - Month when financial year ends +- `financial_year_end_day` - Day when financial year ends +- `timezone` - Organization timezone +- `organisation_type` - Type of organization +- `edition` - Xero edition (BUSINESS, PARTNER, etc.) +- `addresses` - JSON array of addresses +- `phones` - JSON array of phone numbers +- `created_date_utc` - Organization creation date + +**Use Cases:** Organization context, multi-tenant management, configuration reference, financial year calculations + +--- + +### overpayments + +**Description:** Payments that exceed the invoice amount, creating a credit balance. Tracks the overpayment amount, how it's allocated against other invoices, and remaining credit available. Important for managing customer prepayments and supplier overpayments. + +**Key Columns:** +- `overpayment_id` - Unique identifier for the overpayment +- `contact_id` - Associated contact identifier +- `contact_name` - Contact name +- `date` - Overpayment date +- `total` - Total overpayment amount +- `sub_total` - Amount excluding tax +- `total_tax` - Tax amount +- `remaining_credit` - Unallocated credit remaining +- `allocations` - JSON array showing how overpayment is allocated +- `status` - Overpayment status (AUTHORISED, PAID, VOIDED) +- `currency_code` - Transaction currency +- `currency_rate` - Exchange rate if multi-currency +- `type` - Overpayment type (RECEIVE-OVERPAYMENT, SPEND-OVERPAYMENT) +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Credit management, overpayment tracking, allocation to invoices, customer account balances + +--- + +### payments + +**Description:** All payment records linking to invoices, credit notes, overpayments, or prepayments. Shows payment dates, amounts, bank accounts, payment methods, and reconciliation status. Critical for cash management and accounts receivable/payable tracking. + +**Key Columns:** +- `payment_id` - Unique identifier for the payment +- `date` - Payment date +- `amount` - Payment amount in invoice currency +- `bank_amount` - Payment amount in bank account currency +- `account_id` - Bank account identifier +- `account_code` - Bank account code +- `invoice_id` - Associated invoice identifier (if applicable) +- `invoice_number` - Invoice number +- `invoice_type` - Type of invoice (ACCPAY, ACCREC) +- `payment_type` - Payment method (e.g., ACCRECPAYMENT, ACCPAYPAYMENT) +- `status` - Payment status (AUTHORISED, DELETED) +- `is_reconciled` - Whether payment has been reconciled +- `reference` - Payment reference +- `currency_rate` - Exchange rate if multi-currency +- `batch_payment_id` - Batch payment identifier if part of a batch +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Payment tracking, cash receipts, payment reconciliation, accounts receivable/payable management + +--- + +### prepayments + +**Description:** Payments made before receiving goods/services or received before delivering goods/services. Tracks how prepayments are allocated over time as invoices are issued. Important for managing advance payments and deferred revenue. + +**Key Columns:** +- `prepayment_id` - Unique identifier for the prepayment +- `contact_id` - Associated contact identifier +- `contact_name` - Contact name +- `date` - Prepayment date +- `total` - Total prepayment amount +- `sub_total` - Amount excluding tax +- `total_tax` - Tax amount +- `remaining_credit` - Unallocated prepayment remaining +- `allocations` - JSON array showing how prepayment is allocated to invoices +- `status` - Prepayment status (AUTHORISED, PAID, VOIDED) +- `currency_code` - Transaction currency +- `currency_rate` - Exchange rate if multi-currency +- `type` - Prepayment type (RECEIVE-PREPAYMENT, SPEND-PREPAYMENT) +- `line_amount_types` - Tax treatment +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Advance payment tracking, deferred revenue management, prepayment allocation, customer deposits + +--- + +### purchase_orders + +**Description:** Purchase orders sent to suppliers representing commitments to purchase goods or services. Contains order details, delivery information, expected arrival dates, line items, and approval status. Essential for procurement tracking and three-way matching. + +**Key Columns:** +- `purchase_order_id` - Unique identifier for the purchase order +- `purchase_order_number` - PO number +- `contact_id` - Supplier contact identifier +- `contact_name` - Supplier name +- `date` - Purchase order date +- `delivery_date` - Expected delivery date +- `expected_arrival_date` - Expected arrival date +- `status` - PO status (DRAFT, SUBMITTED, AUTHORISED, BILLED, DELETED) +- `type` - PO type (PURCHASEORDER) +- `total` - Total PO amount including tax +- `sub_total` - Amount excluding tax +- `total_tax` - Tax amount +- `line_items` - JSON array of line item details +- `delivery_address` - Delivery address +- `delivery_instructions` - Special delivery instructions +- `attention_to` - Person to address PO to +- `reference` - PO reference +- `currency_code` - Transaction currency +- `has_attachments` - Whether supporting documents are attached +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Procurement management, purchase tracking, three-way matching, supplier orders, commitment reporting + +--- + +### quotes + +**Description:** Sales quotes/estimates sent to customers showing pricing before converting to invoices. Includes expiry dates, terms, line items, and quote status. Useful for sales pipeline tracking and conversion analysis. + +**Key Columns:** +- `quote_id` - Unique identifier for the quote +- `quote_number` - Quote number +- `contact` - JSON object with contact details +- `date` - Quote date +- `expiry_date` - Quote expiration date +- `status` - Quote status (DRAFT, SENT, ACCEPTED, DECLINED, INVOICED, DELETED) +- `total` - Total quoted amount including tax +- `sub_total` - Amount excluding tax +- `total_tax` - Tax amount +- `total_discount` - Total discount amount +- `line_items` - JSON array of line item details +- `title` - Quote title +- `summary` - Quote summary/description +- `terms` - Terms and conditions +- `reference` - Quote reference +- `currency_code` - Transaction currency +- `currency_rate` - Exchange rate if multi-currency +- `branding_theme_id` - Applied branding theme +- `updated_date_utc` - Last modification timestamp + +**Use Cases:** Sales pipeline, quote tracking, conversion analysis, proposal management, win/loss reporting + +--- + +### repeating_invoices + +**Description:** Templates for recurring invoices that automatically generate on a schedule. Contains schedule configuration (frequency, start/end dates), line items, and approval settings. Important for managing subscription revenue and recurring billing. + +**Key Columns:** +- `repeating_invoice_id` - Unique identifier for the template +- `id` - Alternative identifier +- `type` - Invoice type (ACCPAY, ACCREC) +- `status` - Template status (DRAFT, AUTHORISED, DELETED) +- `contact_id` - Associated contact identifier +- `contact_name` - Contact name +- `schedule_unit` - Schedule frequency unit (WEEKLY, MONTHLY, etc.) +- `schedule_period` - Number of units between invoices +- `schedule_due_date` - Days after invoice date payment is due +- `schedule_due_date_type` - Due date calculation method +- `schedule_start_date` - Date to start generating invoices +- `schedule_next_scheduled_date` - Next invoice generation date +- `schedule_end_date` - Date to stop generating invoices +- `line_items` - JSON array of line item details +- `currency_code` - Transaction currency +- `total` - Total invoice amount including tax +- `sub_total` - Amount excluding tax +- `total_tax` - Tax amount +- `reference` - Invoice reference +- `branding_theme_id` - Applied branding theme +- `approved_for_sending` - Whether template is approved + +**Use Cases:** Subscription billing, recurring revenue, automated invoicing, SaaS revenue management + +--- + +## Report Tables (Financial Reports) + +All report tables share a common structure with metadata columns and dynamic period columns (`period_1`, `period_2`, etc.) that contain financial values for different time periods. The `row_type` field distinguishes between section headers, subsection headers, and data rows. + +### Common Report Columns + +- `report_id` - Unique identifier for the report +- `report_name` - Name of the report +- `report_title` - Display title of the report +- `report_type` - Report type identifier +- `report_date` - Date the report was generated +- `updated_date_utc` - Last update timestamp +- `section` - Major section (e.g., Assets, Liabilities, Revenue, Expenses) +- `subsection` - Subsection within a section +- `depth` - Hierarchical depth level for nested accounts +- `row_type` - Type of row (Header, Section, SummaryRow, Row) +- `row_title` - Display title for the row +- `account_id` - Account identifier (for detail rows) +- `period_1`, `period_2`, ... `period_N` - Financial values for each period + +--- + +### balance_sheet_report + +**Description:** Balance sheet (statement of financial position) showing assets, liabilities, and equity at specific points in time. Organized into sections (Assets, Liabilities, Equity) with hierarchical account groupings. Essential for understanding financial position and net worth. Supports multi-period comparison. + +**Report Sections:** +- **Assets** - Current assets (cash, receivables, inventory) and fixed assets (property, equipment) +- **Liabilities** - Current liabilities (payables, short-term debt) and long-term liabilities +- **Equity** - Owner's equity, retained earnings, current year earnings + +**Key Use Cases:** +- Financial position analysis +- Net worth calculation +- Liquidity assessment +- Asset vs. liability comparison +- Period-over-period balance changes +- Working capital analysis + +**Query Example:** +```sql +SELECT * FROM xero.balance_sheet_report +WHERE report_date = '2024-12-31' +``` + +--- + +### profit_loss_report + +**Description:** Profit and loss statement (income statement) showing revenue, expenses, and net profit over time periods. Organized into Income and Expense sections with hierarchical groupings. Core report for understanding profitability, margins, and operational performance. Supports multi-period comparison for trend analysis. + +**Report Sections:** +- **Revenue/Income** - Sales revenue, service income, other income +- **Cost of Sales** - Direct costs of producing goods/services +- **Expenses** - Operating expenses (wages, rent, utilities, marketing, etc.) +- **Net Profit** - Bottom line profitability + +**Key Use Cases:** +- Profitability analysis +- Revenue trend analysis +- Expense management +- Margin calculation +- Period-over-period performance comparison +- Budget variance analysis + +**Query Example:** +```sql +SELECT * FROM xero.profit_loss_report +WHERE report_date BETWEEN '2024-01-01' AND '2024-12-31' +``` + +--- + +### trial_balance_report + +**Description:** Trial balance listing all accounts with their debit and credit balances at a point in time. Shows both balance sheet and profit & loss accounts in a single view, verifying that total debits equal total credits. Essential for month-end close processes and account reconciliation. + +**Report Sections:** +- Lists all active accounts with their current balances +- Separate debit and credit columns +- Typically includes both balance sheet and P&L accounts + +**Key Use Cases:** +- Month-end close verification +- Account reconciliation +- Identifying out-of-balance conditions +- Audit preparation +- General ledger verification +- Accounting period validation + +**Query Example:** +```sql +SELECT * FROM xero.trial_balance_report +WHERE report_date = '2024-12-31' +``` + +--- + +### bank_summary_report + +**Description:** Summary of all bank account balances and transactions over time. Shows opening balances, money in/out movements, and closing balances for each bank account. Critical for cash management, cash flow analysis, and bank reconciliation oversight. + +**Report Sections:** +- One section per bank account +- Opening balance for the period +- Total money in (receipts) +- Total money out (payments) +- Closing balance + +**Key Use Cases:** +- Cash flow monitoring +- Bank account reconciliation +- Liquidity management +- Cash position analysis +- Multi-account cash overview +- Cash movement tracking + +**Query Example:** +```sql +SELECT * FROM xero.bank_summary_report +WHERE report_date BETWEEN '2024-01-01' AND '2024-12-31' +``` + +--- + +### budget_summary_report + +**Description:** Comparison of actual financial results against budgeted amounts by account and time period. Shows budget vs. actual variances to identify areas over/under budget. Essential for performance management and financial control. + +**Report Sections:** +- Organized by account or account group +- Budget amount (planned) +- Actual amount (realized) +- Variance amount (actual - budget) +- Variance percentage + +**Key Use Cases:** +- Budget vs. actual analysis +- Variance investigation +- Performance management +- Financial control +- Identifying budget overruns +- Forecast accuracy assessment + +**Query Example:** +```sql +SELECT * FROM xero.budget_summary_report +WHERE report_date = '2024-12-31' +``` + +--- + +### executive_summary_report + +**Description:** High-level financial overview combining key metrics from balance sheet and profit & loss. Includes cash summary, revenue/expense totals, accounts receivable/payable aging, and profitability ratios. Designed for executive-level visibility into overall financial health. + +**Report Sections:** +- **Cash** - Cash position and movement +- **Revenue** - Total revenue for the period +- **Expenses** - Total expenses for the period +- **Accounts Receivable** - Outstanding invoices aging +- **Accounts Payable** - Outstanding bills aging +- **Net Profit** - Bottom line profitability + +**Key Use Cases:** +- Executive dashboard +- Financial health overview +- KPI monitoring +- Board reporting +- Quick financial snapshot +- Multi-metric analysis + +**Query Example:** +```sql +SELECT * FROM xero.executive_summary_report +WHERE report_date = '2024-12-31' +``` + +--- + +## Usage Notes + +### Data Tables vs. Report Tables + +- **Data Tables** are best for: + - Transaction-level analysis and detailed queries + - Building custom reports and dashboards + - Filtering by specific criteria (dates, contacts, statuses) + - Accessing granular data like line items and allocations + - Joining multiple tables for comprehensive analysis + +- **Report Tables** are best for: + - Standard financial reporting + - Period-over-period comparisons + - Executive dashboards and summaries + - Pre-aggregated financial data + - Quick insights without complex joins + +### Important Considerations + +1. **Nested Data**: Fields like `line_items`, `addresses`, `phones`, and `allocations` are returned as JSON strings and may require parsing for detailed analysis. + +2. **Currency**: All monetary amounts respect the organization's base currency unless a different `currency_code` is specified. Multi-currency transactions include `currency_rate` for conversion. + +3. **Filtering**: Most tables support filtering by: + - Date ranges (`date`, `updated_date_utc`) + - Contact IDs (`contact_id`) + - Statuses (`status`) + - Document types (`type`) + - Other table-specific criteria + +4. **Pagination**: The connector automatically handles pagination for large datasets up to 1000 records per page. + +5. **Report Periods**: Report tables generate dynamic `period_N` columns based on the date range requested. The number of periods varies by report type and date range. + +6. **Read-Only Access**: All tables are read-only. Insert, update, and delete operations are not supported. + +7. **Timestamps**: All timestamps are in UTC format. Use appropriate timezone conversion for local time analysis. + +8. **Reconciliation**: Fields like `is_reconciled` indicate whether transactions have been matched with bank statements. + +--- + +## Example Queries + +### Get all unpaid invoices +```sql +SELECT invoice_id, invoice_number, contact_name, date, due_date, amount_due +FROM xero.invoices +WHERE status = 'AUTHORISED' AND amount_due > 0 +ORDER BY due_date +``` + +### Analyze monthly revenue +```sql +SELECT contact_name, SUM(total) as revenue, COUNT(*) as invoice_count +FROM xero.invoices +WHERE type = 'ACCREC' + AND status IN ('AUTHORISED', 'PAID') + AND date BETWEEN '2024-01-01' AND '2024-12-31' +GROUP BY contact_name +ORDER BY revenue DESC +``` + +### Get current balance sheet +```sql +SELECT section, row_title, period_1 as current_balance +FROM xero.balance_sheet_report +WHERE report_date = '2024-12-31' + AND row_type = 'Row' +ORDER BY section, row_title +``` + +### Track payment reconciliation status +```sql +SELECT p.payment_id, p.date, p.amount, p.is_reconciled, + i.invoice_number, c.name as customer_name +FROM xero.payments p +JOIN xero.invoices i ON p.invoice_id = i.invoice_id +JOIN xero.contacts c ON i.contact_id = c.contact_id +WHERE p.is_reconciled = false +ORDER BY p.date DESC +``` + +--- + +## Additional Resources + +- [Xero Developer Documentation](https://developer.xero.com/documentation/api/accounting/overview) +- [Xero API Accounting Endpoints](https://developer.xero.com/documentation/api/accounting/endpoints) +- [MindsDB Xero Handler](https://github.com/mindsdb/mindsdb/tree/main/mindsdb/integrations/handlers/xero_handler) diff --git a/mindsdb/integrations/handlers/xero_handler/__about__.py b/mindsdb/integrations/handlers/xero_handler/__about__.py new file mode 100644 index 00000000000..82eb4caaa1c --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/__about__.py @@ -0,0 +1,9 @@ +__title__ = 'MindsDB Xero handler' +__package_name__ = 'mindsdb_xero_handler' +__version__ = '0.0.1' +__description__ = "MindsDB handler for the Xero Accounting API" +__author__ = 'MindsDB' +__github__ = 'https://github.com/mindsdb/mindsdb' +__pypi__ = 'https://pypi.org/project/mindsdb/' +__license__ = 'MIT' +__copyright__ = 'Copyright 2025 - mindsdb' diff --git a/mindsdb/integrations/handlers/xero_handler/__init__.py b/mindsdb/integrations/handlers/xero_handler/__init__.py new file mode 100644 index 00000000000..3e55956decc --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/__init__.py @@ -0,0 +1,25 @@ +from mindsdb.integrations.libs.const import HANDLER_TYPE +from .__about__ import __version__ as version, __description__ as description + +try: + from .xero_handler import XeroHandler as Handler + import_error = None +except Exception as e: + Handler = None + import_error = e + +title = "Xero" +name = "xero" +type = HANDLER_TYPE.DATA +icon_path = "icon.svg" + +__all__ = [ + "Handler", + "version", + "name", + "type", + "title", + "description", + "import_error", + "icon_path", +] diff --git a/mindsdb/integrations/handlers/xero_handler/connection_args.py b/mindsdb/integrations/handlers/xero_handler/connection_args.py new file mode 100644 index 00000000000..8e5a55a0253 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/connection_args.py @@ -0,0 +1,86 @@ +from collections import OrderedDict +from mindsdb.integrations.handlers.xero_handler.__about__ import __version__ as version +from mindsdb.integrations.libs.const import HANDLER_CONNECTION_ARG_TYPE as ARG_TYPE + + +connection_args = OrderedDict( + # OAuth2 Code Flow Parameters + client_id={ + 'type': ARG_TYPE.STR, + 'description': 'Xero OAuth2 Client ID', + 'label': 'OAuth Client ID', + 'required': True, + }, + client_secret={ + 'type': ARG_TYPE.STR, + 'description': 'Xero OAuth2 Client Secret', + 'label': 'OAuth Client Secret', + 'required': True, + 'secret': True, + }, + redirect_uri={ + 'type': ARG_TYPE.STR, + 'description': 'OAuth2 Redirect URI (must match your Xero app configuration). Required for code flow.', + 'label': 'Redirect URI', + 'required': False, + }, + code={ + 'type': ARG_TYPE.STR, + 'description': 'Authorization code obtained from OAuth flow (code flow only)', + 'label': 'Authorization Code', + 'required': False, + 'secret': True, + }, + + # Token Injection Parameters (for backend integration) + access_token={ + 'type': ARG_TYPE.STR, + 'description': 'Xero access token (for token injection from backend systems)', + 'label': 'Access Token', + 'required': False, + 'secret': True, + }, + refresh_token={ + 'type': ARG_TYPE.STR, + 'description': 'Xero refresh token (for automatic token refresh)', + 'label': 'Refresh Token', + 'required': False, + 'secret': True, + }, + expires_at={ + 'type': ARG_TYPE.STR, + 'description': 'Access token expiration time (ISO 8601 format or Unix timestamp)', + 'label': 'Token Expires At', + 'required': False, + }, + + # Organization/Tenant + tenant_id={ + 'type': ARG_TYPE.STR, + 'description': 'Xero Tenant ID (Organization ID). If not provided, the first accessible tenant will be used.', + 'label': 'Tenant ID', + 'required': False, + }, +) + +connection_args_strict = OrderedDict( + client_id={ + 'type': ARG_TYPE.STR, + 'description': 'Xero OAuth2 Client ID', + 'label': 'OAuth Client ID', + 'required': True, + }, + client_secret={ + 'type': ARG_TYPE.STR, + 'description': 'Xero OAuth2 Client Secret', + 'label': 'OAuth Client Secret', + 'required': True, + 'secret': True, + }, + redirect_uri={ + 'type': ARG_TYPE.STR, + 'description': 'OAuth2 Redirect URI', + 'label': 'Redirect URI', + 'required': True, + }, +) diff --git a/mindsdb/integrations/handlers/xero_handler/icon.svg b/mindsdb/integrations/handlers/xero_handler/icon.svg new file mode 100644 index 00000000000..8b63df067a0 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/icon.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/mindsdb/integrations/handlers/xero_handler/requirements.txt b/mindsdb/integrations/handlers/xero_handler/requirements.txt new file mode 100644 index 00000000000..24e4b595e45 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/requirements.txt @@ -0,0 +1,2 @@ +xero-python>=0.8.0 +requests>=2.28.0 diff --git a/mindsdb/integrations/handlers/xero_handler/tables/accounts_table.py b/mindsdb/integrations/handlers/xero_handler/tables/accounts_table.py new file mode 100644 index 00000000000..9e5b22e6050 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/accounts_table.py @@ -0,0 +1,107 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + + +class AccountsTable(XeroTable): + """Table for Xero Chart of Accounts""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "account_id": {"type": "where", "xero_field": "AccountID", "value_type": "guid"}, + "code": {"type": "where", "xero_field": "Code", "value_type": "string"}, + "name": {"type": "where", "xero_field": "Name", "value_type": "string"}, + "type": {"type": "where", "xero_field": "Type", "value_type": "string"}, + "status": {"type": "where", "xero_field": "Status", "value_type": "string"}, + "currency_code": {"type": "where", "xero_field": "CurrencyCode", "value_type": "string"}, + "account_class": {"type": "where", "xero_field": "Class", "value_type": "string"}, + "system_account": {"type": "where", "xero_field": "SystemAccount", "value_type": "string"}, + "tax_type": {"type": "where", "xero_field": "TaxType", "value_type": "string"}, + } + + def get_columns(self) -> List[str]: + return [ + "account_id", + "code", + "name", + "type", + "account_class", + "tax_type", + "description", + "currency_code", + "bank_account_number", + "bank_account_type", + "enable_payments_to_account", + "show_in_expense_claims", + "system_account", + "reporting_code", + "reporting_code_name", + "status", + "updated_utc", + "has_attachments", + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + try: + # Fetch accounts with optimized parameters + accounts = api.get_accounts(xero_tenant_id=self.handler.tenant_id, **api_params) + df = self._convert_response_to_dataframe(accounts.accounts or []) + except Exception as e: + raise Exception(f"Failed to fetch accounts: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Parse and execute query + parser = SELECTQueryParser( + query, "accounts", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/balance_sheet_report_table.py b/mindsdb/integrations/handlers/xero_handler/tables/balance_sheet_report_table.py new file mode 100644 index 00000000000..a2036e28cd9 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/balance_sheet_report_table.py @@ -0,0 +1,122 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_report_tables import XeroReportTable +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + + +class BalanceSheetReportTable(XeroReportTable): + """Table for Xero Balance Sheet Report""" + + # Define which parameters can be pushed to the Xero API + SUPPORTED_FILTERS = { + "date": {"type": "direct", "param": "date"}, + "periods": {"type": "direct", "param": "periods"}, + "timeframe": {"type": "direct", "param": "timeframe"}, + "tracking_option_id_1": {"type": "direct", "param": "tracking_option_id1"}, + "tracking_option_id_2": {"type": "direct", "param": "tracking_option_id2"}, + "standard_layout": {"type": "direct", "param": "standard_layout"}, + "payments_only": {"type": "direct", "param": "payments_only"}, + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + """ + Return column names for the balance sheet report. + + Returns: + List[str]: Column names + """ + base_columns = [ + "report_id", + "report_name", + "report_title", + "report_type", + "report_date", + "updated_date_utc", + "section", + "subsection", + "depth", + "row_type", + "row_title", + "account_id", + ] + + # Add generic period columns + period_columns = [f"period_{i}" for i in range(1, self.MAX_PERIODS + 1)] + + return base_columns + period_columns + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch balance sheet report from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit (reports don't paginate, so we use default limit) + parser = SELECTQueryParser( + query, "balance_sheet_report", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Convert date parameters to proper format + if 'date' in api_params: + api_params['date'] = self._convert_date_parameter(api_params['date']) + + try: + # Fetch balance sheet report (single API call - no pagination) + response = api.get_report_balance_sheet( + xero_tenant_id=self.handler.tenant_id, + **api_params + ) + + # Parse report structure to DataFrame + df = self._parse_report_to_dataframe(response) + + except Exception as e: + raise Exception(f"Failed to fetch balance sheet report: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df diff --git a/mindsdb/integrations/handlers/xero_handler/tables/bank_summary_report_table.py b/mindsdb/integrations/handlers/xero_handler/tables/bank_summary_report_table.py new file mode 100644 index 00000000000..bb5641e8a57 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/bank_summary_report_table.py @@ -0,0 +1,136 @@ +from typing import List +from datetime import datetime, timedelta, date +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_report_tables import XeroReportTable +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + + +class BankSummaryReportTable(XeroReportTable): + """Table for Xero Bank Summary Report""" + + # Define which parameters can be pushed to the Xero API + SUPPORTED_FILTERS = { + "from_date": {"type": "direct", "param": "from_date"}, + "to_date": {"type": "direct", "param": "to_date"}, + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + """ + Return column names for the bank summary report. + + Returns: + List[str]: Column names + """ + base_columns = [ + "report_id", + "report_name", + "report_title", + "report_type", + "report_date", + "updated_date_utc", + "section", + "subsection", + "depth", + "row_type", + "row_title", + "account_id", + ] + + # Add generic period columns + period_columns = [f"period_{i}" for i in range(1, self.MAX_PERIODS + 1)] + + return base_columns + period_columns + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch bank summary report from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit + parser = SELECTQueryParser( + query, "bank_summary_report", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Set default dates if not provided + # from_date defaults to beginning of current month + # to_date defaults to end of current month + now = datetime.now() + + if 'from_date' not in api_params: + # First day of current month as date object + api_params['from_date'] = date(now.year, now.month, 1) + else: + api_params['from_date'] = self._convert_date_parameter(api_params['from_date']) + + if 'to_date' not in api_params: + # Last day of current month as date object + # Get first day of next month, then subtract one day + if now.month == 12: + next_month = date(now.year + 1, 1, 1) + else: + next_month = date(now.year, now.month + 1, 1) + api_params['to_date'] = next_month - timedelta(days=1) + else: + api_params['to_date'] = self._convert_date_parameter(api_params['to_date']) + + try: + # Fetch bank summary report + response = api.get_report_bank_summary( + xero_tenant_id=self.handler.tenant_id, + **api_params + ) + + # Parse report structure to DataFrame + df = self._parse_report_to_dataframe(response) + + except Exception as e: + raise Exception(f"Failed to fetch bank summary report: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df diff --git a/mindsdb/integrations/handlers/xero_handler/tables/bank_transactions_table.py b/mindsdb/integrations/handlers/xero_handler/tables/bank_transactions_table.py new file mode 100644 index 00000000000..0e3bbd359e7 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/bank_transactions_table.py @@ -0,0 +1,215 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + + +class BankTransactionsTable(XeroTable): + """Table for Xero Bank Transactions""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "type": {"type": "where", "xero_field": "Type", "value_type": "string"}, + "status": {"type": "where", "xero_field": "Status", "value_type": "string"}, + "date": {"type": "where", "xero_field": "Date", "value_type": "date"}, + "contact_id": {"type": "where", "xero_field": "Contact.ContactID", "value_type": "guid"}, + } + + COLUMN_REMAP = { + "contact_contact_id": "contact_id", + "contact_contact_number": "contact_number", + "contact_contact_persons": "contact_persons", + "contact_contact_groups": "contact_groups", + "bank_account_bank_account_number": "bank_account_number", + "bank_account_bank_account_type": "bank_account_type", + } + + def get_columns(self) -> List[str]: + return [ + "bank_account__class", + "bank_account_account_id", + "bank_account_add_to_watchlist", + "bank_account_code", + "bank_account_currency_code", + "bank_account_description", + "bank_account_enable_payments_to_account", + "bank_account_has_attachments", + "bank_account_name", + "bank_account_number", + "bank_account_reporting_code", + "bank_account_reporting_code_name", + "bank_account_show_in_expense_claims", + "bank_account_status", + "bank_account_system_account", + "bank_account_tax_type", + "bank_account_type", + "bank_account_updated_date_utc", + "bank_account_validation_errors", + "bank_transaction_id", + "contact_account_number", + "contact_accounts_payable_tax_type", + "contact_accounts_receivable_tax_type", + "contact_addresses", + "contact_attachments", + "contact_balances", + "contact_bank_account_details", + "contact_batch_payments", + "contact_branding_theme", + "contact_company_number", + "contact_default_currency", + "contact_discount", + "contact_email_address", + "contact_first_name", + "contact_groups", + "contact_has_attachments", + "contact_has_validation_errors", + "contact_id", + "contact_is_customer", + "contact_is_supplier", + "contact_last_name", + "contact_merged_to_contact_id", + "contact_name", + "contact_number", + "contact_payment_terms", + "contact_persons", + "contact_phones", + "contact_purchases_default_account_code", + "contact_purchases_default_line_amount_type", + "contact_purchases_tracking_categories", + "contact_sales_default_account_code", + "contact_sales_default_line_amount_type", + "contact_sales_tracking_categories", + "contact_status", + "contact_status_attribute_string", + "contact_tax_number", + "contact_tracking_category_name", + "contact_tracking_category_option", + "contact_updated_date_utc", + "contact_validation_errors", + "contact_website", + "contact_xero_network_key", + "currency_code", + "currency_rate", + "date", + "has_attachments", + "is_reconciled", + "line_amount_types", + "line_items", + "overpayment_id", + "prepayment_id", + "reference", + "status", + "status_attribute_string", + "sub_total", + "total", + "total_tax", + "type", + "updated_date_utc", + "url", + "validation_errors" + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit (no default limit) + parser = SELECTQueryParser( + query, "bank_transactions", columns=self.get_columns(), use_default_limit=False + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Implement pagination to fetch all required records + all_data = [] + page = 1 + page_size = 1000 # Xero's maximum page size + records_fetched = 0 + + try: + while result_limit is None or records_fetched < result_limit: + # Calculate how many records to fetch in this page + if result_limit is not None: + records_to_fetch = min(page_size, result_limit - records_fetched) + else: + records_to_fetch = page_size + + # Fetch bank transactions with pagination parameters + response = api.get_bank_transactions( + xero_tenant_id=self.handler.tenant_id, + page=page, + page_size=records_to_fetch, + **api_params + ) + + if not response.bank_transactions: + break # No more data + + all_data.extend(response.bank_transactions) + records_fetched += len(response.bank_transactions) + + # Check pagination metadata to determine if there are more pages + if hasattr(response, 'pagination') and response.pagination: + # If we've reached the last page, stop + if page >= response.pagination.page_count: + break + else: + # Fallback: If we got fewer records than requested, we've reached the end + if len(response.bank_transactions) < records_to_fetch: + break + + page += 1 + + except Exception as e: + raise Exception(f"Failed to fetch bank transactions: {str(e)}") + + # Convert all data to DataFrame + df = self._convert_response_to_dataframe(all_data) + if len(df) > 0: + df.rename(columns=self.COLUMN_REMAP, inplace=True) + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit (in case filters reduced the result set) + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/bank_transfers_table.py b/mindsdb/integrations/handlers/xero_handler/tables/bank_transfers_table.py new file mode 100644 index 00000000000..dad105d2bc2 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/bank_transfers_table.py @@ -0,0 +1,172 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import XeroTable, extract_comparison_conditions +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + + +class BankTransfersTable(XeroTable): + """Table for Xero Bank Transfers""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "type": {"type": "where", "xero_field": "Type", "value_type": "string"}, + "status": {"type": "where", "xero_field": "Status", "value_type": "string"}, + "date": {"type": "where", "param": "date_from", "param_upper": "date_to"}, + "contact_id": {"type": "where", "xero_field": "Contact.ContactID", "value_type": "string"}, + } + + COLUMN_REMAP = { + "contact_contact_id": "contact_id", + "contact_contact_number": "contact_number", + "contact_contact_persons": "contact_persons", + "contact_contact_groups": "contact_groups", + "bank_account_bank_account_number": "bank_account_number", + "bank_account_bank_account_type": "bank_account_type", + } + + def get_columns(self) -> List[str]: + return [ + "bank_account__class", + "bank_account_account_id", + "bank_account_add_to_watchlist", + "bank_account_code", + "bank_account_currency_code", + "bank_account_description", + "bank_account_enable_payments_to_account", + "bank_account_has_attachments", + "bank_account_name", + "bank_account_number", + "bank_account_reporting_code", + "bank_account_reporting_code_name", + "bank_account_show_in_expense_claims", + "bank_account_status", + "bank_account_system_account", + "bank_account_tax_type", + "bank_account_type", + "bank_account_updated_date_utc", + "bank_account_validation_errors", + "bank_transaction_id", + "contact_account_number", + "contact_accounts_payable_tax_type", + "contact_accounts_receivable_tax_type", + "contact_addresses", + "contact_attachments", + "contact_balances", + "contact_bank_account_details", + "contact_batch_payments", + "contact_branding_theme", + "contact_company_number", + "contact_default_currency", + "contact_discount", + "contact_email_address", + "contact_first_name", + "contact_groups", + "contact_has_attachments", + "contact_has_validation_errors", + "contact_id", + "contact_is_customer", + "contact_is_supplier", + "contact_last_name", + "contact_merged_to_contact_id", + "contact_name", + "contact_number", + "contact_payment_terms", + "contact_persons", + "contact_phones", + "contact_purchases_default_account_code", + "contact_purchases_default_line_amount_type", + "contact_purchases_tracking_categories", + "contact_sales_default_account_code", + "contact_sales_default_line_amount_type", + "contact_sales_tracking_categories", + "contact_status", + "contact_status_attribute_string", + "contact_tax_number", + "contact_tracking_category_name", + "contact_tracking_category_option", + "contact_updated_date_utc", + "contact_validation_errors", + "contact_website", + "contact_xero_network_key", + "currency_code", + "currency_rate", + "date", + "has_attachments", + "is_reconciled", + "line_amount_types", + "line_items", + "overpayment_id", + "prepayment_id", + "reference", + "status", + "status_attribute_string", + "sub_total", + "total", + "total_tax", + "type", + "updated_date_utc", + "url", + "validation_errors" + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + try: + # Fetch bank transfers with optimized parameters + bank_transfers = api.get_bank_transfers(xero_tenant_id=self.handler.tenant_id, **api_params) + df = self._convert_response_to_dataframe(bank_transfers.bank_transfers or []) + df.rename(columns=self.COLUMN_REMAP, inplace=True) + except Exception as e: + raise Exception(f"Failed to fetch bank transfers: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + from mindsdb.integrations.utilities.sql_utils import filter_dataframe + df = filter_dataframe(df, remaining_conditions) + + # Parse and execute query + parser = SELECTQueryParser( + query, "bank_transfers", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + from mindsdb.integrations.utilities.sql_utils import sort_dataframe + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/budget_summary_report_table.py b/mindsdb/integrations/handlers/xero_handler/tables/budget_summary_report_table.py new file mode 100644 index 00000000000..1351a8dc6ce --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/budget_summary_report_table.py @@ -0,0 +1,118 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_report_tables import XeroReportTable +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + + +class BudgetSummaryReportTable(XeroReportTable): + """Table for Xero Budget Summary Report""" + + # Define which parameters can be pushed to the Xero API + SUPPORTED_FILTERS = { + "date": {"type": "direct", "param": "date"}, + "periods": {"type": "direct", "param": "periods"}, + "timeframe": {"type": "direct", "param": "timeframe"}, + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + """ + Return column names for the budget summary report. + + Returns: + List[str]: Column names + """ + base_columns = [ + "report_id", + "report_name", + "report_title", + "report_type", + "report_date", + "updated_date_utc", + "section", + "subsection", + "depth", + "row_type", + "row_title", + "account_id", + ] + + # Add generic period columns + period_columns = [f"period_{i}" for i in range(1, self.MAX_PERIODS + 1)] + + return base_columns + period_columns + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch budget summary report from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit + parser = SELECTQueryParser( + query, "budget_summary_report", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Convert date parameters to proper format + if 'date' in api_params: + api_params['date'] = self._convert_date_parameter(api_params['date']) + + try: + # Fetch budget summary report + response = api.get_report_budget_summary( + xero_tenant_id=self.handler.tenant_id, + **api_params + ) + + # Parse report structure to DataFrame + df = self._parse_report_to_dataframe(response) + + except Exception as e: + raise Exception(f"Failed to fetch budget summary report: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df diff --git a/mindsdb/integrations/handlers/xero_handler/tables/budgets_table.py b/mindsdb/integrations/handlers/xero_handler/tables/budgets_table.py new file mode 100644 index 00000000000..7b7483d2051 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/budgets_table.py @@ -0,0 +1,92 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class BudgetsTable(XeroTable): + """Table for Xero Budgets""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "budget_id": {"type": "where", "xero_field": "BudgetID", "value_type": "guid"}, + "type": {"type": "where", "xero_field": "Type", "value_type": "string"}, + "date": {"type": "date", "param": "date_from", "param_upper": "date_to"}, + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + return [ + "budget_id", + "status", + "description", + "tracking", + "budget_lines", + "type", + "updated_date_utc", + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + try: + # Fetch budgets with optimized parameters + budgets = api.get_budgets(xero_tenant_id=self.handler.tenant_id, **api_params) + df = self._convert_response_to_dataframe(budgets.budgets or []) + df.rename(columns=self.COLUMN_REMAP, inplace=True) + except Exception as e: + raise Exception(f"Failed to fetch budgets: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Parse and execute query + parser = SELECTQueryParser( + query, "budgets", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/contact_groups_table.py b/mindsdb/integrations/handlers/xero_handler/tables/contact_groups_table.py new file mode 100644 index 00000000000..ae5a7634bd3 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/contact_groups_table.py @@ -0,0 +1,88 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class ContactGroupsTable(XeroTable): + """Table for Xero Contact Groups""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "contact_group_id": {"type": "where", "xero_field": "ContactGroupID", "value_type": "guid"}, + "name": {"type": "where", "param": "name", "value_type": "string"}, + "status": {"type": "where", "param": "status", "value_type": "string"}, + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + return [ + "contact_group_id", + "name", + "status", + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + try: + # Fetch contact groups with optimized parameters + contact_groups = api.get_contact_groups(xero_tenant_id=self.handler.tenant_id, **api_params) + df = self._convert_response_to_dataframe(contact_groups.contact_groups or []) + df.rename(columns=self.COLUMN_REMAP, inplace=True) + except Exception as e: + raise Exception(f"Failed to fetch contact groups: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Parse and execute query + parser = SELECTQueryParser( + query, "contact_groups", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/contacts_table.py b/mindsdb/integrations/handlers/xero_handler/tables/contacts_table.py new file mode 100644 index 00000000000..7d9b462bbb5 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/contacts_table.py @@ -0,0 +1,148 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class ContactsTable(XeroTable): + """Table for Xero Contacts""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "contact_id": {"type": "id_list", "param": "i_ds"}, + "name": {"type": "direct", "param": "search_term", "value_type": "string"}, + "first_name": {"type": "direct", "param": "search_term", "value_type": "string"}, + "last_name": {"type": "direct", "param": "search_term", "value_type": "string"}, + "email_address": {"type": "direct", "param": "search_term", "value_type": "string"}, + "contact_number": {"type": "direct", "param": "search_term", "value_type": "string"}, + "company_number": {"type": "direct", "param": "search_term", "value_type": "string"}, + "status": {"type": "where", "param": "status", "value_type": "string"}, + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + return [ + "contact_id", + "contact_status", + "name", + "first_name", + "last_name", + "company_number", + "email_address", + "bank_account_details", + "tax_number", + "accounts_receivable_tax_type", + "accounts_payable_tax_type", + "addresses", + "phones", + "updated_date_utc", + "is_supplier", + "is_customer", + "default_currency" + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit (no default limit) + parser = SELECTQueryParser( + query, "contacts", columns=self.get_columns(), use_default_limit=False + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Implement pagination to fetch all required records + all_data = [] + page = 1 + page_size = 1000 # Xero's maximum page size + records_fetched = 0 + + try: + while result_limit is None or records_fetched < result_limit: + # Calculate how many records to fetch in this page + if result_limit is not None: + records_to_fetch = min(page_size, result_limit - records_fetched) + else: + records_to_fetch = page_size + + # Fetch contacts with pagination parameters + response = api.get_contacts( + xero_tenant_id=self.handler.tenant_id, + summary_only=False, + page=page, + page_size=records_to_fetch, + **api_params + ) + + if not response.contacts: + break # No more data + + all_data.extend(response.contacts) + records_fetched += len(response.contacts) + + # Check pagination metadata to determine if there are more pages + if hasattr(response, 'pagination') and response.pagination: + # If we've reached the last page, stop + if page >= response.pagination.page_count: + break + else: + # Fallback: If we got fewer records than requested, we've reached the end + if len(response.contacts) < records_to_fetch: + break + + page += 1 + + except Exception as e: + raise Exception(f"Failed to fetch contacts: {str(e)}") + + # Convert all data to DataFrame + df = self._convert_response_to_dataframe(all_data) + if len(df) > 0: + df.rename(columns=self.COLUMN_REMAP, inplace=True) + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit (in case filters reduced the result set) + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/credit_notes_table.py b/mindsdb/integrations/handlers/xero_handler/tables/credit_notes_table.py new file mode 100644 index 00000000000..b84d0e1eb5b --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/credit_notes_table.py @@ -0,0 +1,169 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class CreditNotesTable(XeroTable): + """Table for Xero Credit Notes""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "credit_note_id": {"type": "where", "xero_field": "CreditNoteID", "value_type": "guid"}, + "credit_note_number": {"type": "where", "xero_field": "CreditNoteNumber", "value_type": "string"}, + "type": {"type": "where", "xero_field": "Type", "value_type": "string"}, + "reference": {"type": "where", "xero_field": "Reference", "value_type": "string"}, + "remaining_credit": {"type": "where", "xero_field": "RemainingCredit", "value_type": "number"}, + "date": {"type": "where", "xero_field": "Date", "value_type": "date"}, + "status": {"type": "where", "xero_field": "Status", "value_type": "string"}, + "line_amount_types": {"type": "where", "xero_field": "LineAmountTypes", "value_type": "string"}, + "sub_total": {"type": "where", "xero_field": "SubTotal", "value_type": "number"}, + "total_tax": {"type": "where", "xero_field": "TotalTax", "value_type": "number"}, + "total": {"type": "where", "xero_field": "Total", "value_type": "number"}, + "updated_date_utc": {"type": "where", "xero_field": "UpdatedDateUTC", "value_type": "date"}, + "currency_code": {"type": "where", "xero_field": "CurrencyCode", "value_type": "string"}, + "fully_paid_on_date": {"type": "where", "xero_field": "FullyPaidOnDate", "value_type": "date"}, + "contact_id": {"type": "where", "xero_field": "Contact.ContactID", "value_type": "guid"}, + } + + COLUMN_REMAP = { + "contact_contact_id": "contact_id", + "contact_contact_persons": "contact_persons", + "contact_contact_groups": "contact_groups", + } + + def get_columns(self) -> List[str]: + return [ + "credit_note_id", + "credit_note_number", + "payments", + "has_errors", + "invoice_addresses", + "type", + "reference", + "remaining_credit", + "allocations", + "has_attachments", + "contact_id", + "contact_name", + "contact_addresses", + "contact_phones", + "contact_groups", + "contact_persons", + "contact_has_validation_errors", + "date_string", + "date", + "status", + "line_amount_types", + "line_items", + "sub_total", + "total_tax", + "total", + "updated_date_utc", + "currency_code", + "fully_paid_on_date" + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit (no default limit) + parser = SELECTQueryParser( + query, "credit_notes", columns=self.get_columns(), use_default_limit=False + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Implement pagination to fetch all required records + all_data = [] + page = 1 + page_size = 1000 # Xero's maximum page size + records_fetched = 0 + + try: + while result_limit is None or records_fetched < result_limit: + # Calculate how many records to fetch in this page + if result_limit is not None: + records_to_fetch = min(page_size, result_limit - records_fetched) + else: + records_to_fetch = page_size + + # Fetch credit notes with pagination parameters + response = api.get_credit_notes( + xero_tenant_id=self.handler.tenant_id, + page=page, + page_size=records_to_fetch, + **api_params + ) + + if not response.credit_notes: + break # No more data + + all_data.extend(response.credit_notes) + records_fetched += len(response.credit_notes) + + # Check pagination metadata to determine if there are more pages + if hasattr(response, 'pagination') and response.pagination: + # If we've reached the last page, stop + if page >= response.pagination.page_count: + break + else: + # Fallback: If we got fewer records than requested, we've reached the end + if len(response.credit_notes) < records_to_fetch: + break + + page += 1 + + except Exception as e: + raise Exception(f"Failed to fetch credit notes: {str(e)}") + + # Convert all data to DataFrame + df = self._convert_response_to_dataframe(all_data) + if len(df) > 0: + df.rename(columns=self.COLUMN_REMAP, inplace=True) + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit (in case filters reduced the result set) + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/executive_summary_report_table.py b/mindsdb/integrations/handlers/xero_handler/tables/executive_summary_report_table.py new file mode 100644 index 00000000000..a8b3d6cedd9 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/executive_summary_report_table.py @@ -0,0 +1,116 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_report_tables import XeroReportTable +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + + +class ExecutiveSummaryReportTable(XeroReportTable): + """Table for Xero Executive Summary Report""" + + # Define which parameters can be pushed to the Xero API + SUPPORTED_FILTERS = { + "date": {"type": "direct", "param": "date"}, + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + """ + Return column names for the executive summary report. + + Returns: + List[str]: Column names + """ + base_columns = [ + "report_id", + "report_name", + "report_title", + "report_type", + "report_date", + "updated_date_utc", + "section", + "subsection", + "depth", + "row_type", + "row_title", + "account_id", + ] + + # Add generic period columns + period_columns = [f"period_{i}" for i in range(1, self.MAX_PERIODS + 1)] + + return base_columns + period_columns + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch executive summary report from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit + parser = SELECTQueryParser( + query, "executive_summary_report", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Convert date parameters to proper format + if 'date' in api_params: + api_params['date'] = self._convert_date_parameter(api_params['date']) + + try: + # Fetch executive summary report + response = api.get_report_executive_summary( + xero_tenant_id=self.handler.tenant_id, + **api_params + ) + + # Parse report structure to DataFrame + df = self._parse_report_to_dataframe(response) + + except Exception as e: + raise Exception(f"Failed to fetch executive summary report: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df diff --git a/mindsdb/integrations/handlers/xero_handler/tables/invoices_table.py b/mindsdb/integrations/handlers/xero_handler/tables/invoices_table.py new file mode 100644 index 00000000000..4d4f28eea4a --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/invoices_table.py @@ -0,0 +1,175 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class InvoicesTable(XeroTable): + """Table for Xero Invoices""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "invoice_id": {"type": "id_list", "param": "i_ds"}, + "invoice_number": {"type": "direct", "param": "search_term", "value_type": "string"}, + "contact_id": {"type": "id_list", "param": "contact_i_ds"}, + "status": {"type": "id_list", "param": "statuses"}, + "contact_name": {"type": "where", "xero_field": "Contact.Name", "value_type": "string"}, + "contact_number": {"type": "where", "xero_field": "Contact.ContactNumber", "value_type": "string"}, + "reference": {"type": "direct", "param": "search_term", "value_type": "string"}, + "date": {"type": "where", "xero_field": "Date", "value_type": "date"}, + "type": {"type": "where", "xero_field": "Type", "value_type": "string"}, + "amount_due": {"type": "where", "xero_field": "AmountDue", "value_type": "number"}, + "amount_paid": {"type": "where", "xero_field": "AmountPaid", "value_type": "number"}, + "due_date": {"type": "where", "xero_field": "DueDate", "value_type": "date"} + } + + COLUMN_REMAP = { + "contact_contact_id": "contact_id", + "contact_contact_persons": "contact_persons", + "contact_contact_groups": "contact_groups", + } + + def get_columns(self) -> List[str]: + return [ + "invoice_id", + "invoice_number", + "reference", + "payments", + "credit_notes", + "type", + "pre_payments", + "over_payments", + "amount_due", + "amount_paid", + "amount_credited", + "currency_rate", + "is_discounted", + "has_attachments", + "invoice_addresses", + "hasErrors", + "invoice_payment_services", + "contact_id", + "contact_name", + "contact_addresses", + "contact_phones", + "contact_groups", + "contact_persons", + "contact_has_validation_errors", + "date", + "date_string", + "due_date", + "due_date_string", + "branding_theme_id", + "status", + "line_amount_types", + "line_items", + "sub_total", + "total_tax", + "total", + "updated_date_utc", + "currency_code" + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit (no default limit) + parser = SELECTQueryParser( + query, "invoices", columns=self.get_columns(), use_default_limit=False + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Implement pagination to fetch all required records + all_data = [] + page = 1 + page_size = 1000 # Xero's maximum page size + records_fetched = 0 + + try: + while result_limit is None or records_fetched < result_limit: + # Calculate how many records to fetch in this page + if result_limit is not None: + records_to_fetch = min(page_size, result_limit - records_fetched) + else: + records_to_fetch = page_size + + # Fetch invoices with pagination parameters + response = api.get_invoices( + xero_tenant_id=self.handler.tenant_id, + page=page, + page_size=records_to_fetch, + **api_params + ) + + if not response.invoices: + break # No more data + + all_data.extend(response.invoices) + records_fetched += len(response.invoices) + + # Check pagination metadata to determine if there are more pages + if hasattr(response, 'pagination') and response.pagination: + # If we've reached the last page, stop + if page >= response.pagination.page_count: + break + else: + # Fallback: If we got fewer records than requested, we've reached the end + if len(response.invoices) < records_to_fetch: + break + + page += 1 + + except Exception as e: + raise Exception(f"Failed to fetch invoices: {str(e)}") + + # Convert all data to DataFrame + df = self._convert_response_to_dataframe(all_data) + if len(df) > 0: + df.rename(columns=self.COLUMN_REMAP, inplace=True) + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit (in case filters reduced the result set) + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/items_table.py b/mindsdb/integrations/handlers/xero_handler/tables/items_table.py new file mode 100644 index 00000000000..ef6193ca31a --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/items_table.py @@ -0,0 +1,101 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class ItemsTable(XeroTable): + """Table for Xero Items""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "item_id": {"type": "where", "xero_field": "ItemID", "value_type": "guid"}, + "code": {"type": "where", "xero_field": "Code", "value_type": "string"}, + "name": {"type": "where", "xero_field": "Name", "value_type": "string"}, + "is_tracked_as_inventory": {"type": "where", "xero_field": "IsTrackedAsInventory", "value_type": "bool"}, + "is_sold": {"type": "where", "xero_field": "IsSold", "value_type": "bool"}, + "is_purchased": {"type": "where", "xero_field": "IsPurchased", "value_type": "bool"} + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + return [ + "item_id", + "code", + "description", + "purchase_description", + "purchase_details", + "updated_date_utc", + "sales_details_unit_price", + "sales_details_account_code", + "sales_details_tax_type", + "name", + "is_tracked_as_inventory", + "is_sold", + "is_purchased", + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + try: + # Fetch items with optimized parameters + items = api.get_items(xero_tenant_id=self.handler.tenant_id, **api_params) + df = self._convert_response_to_dataframe(items.items or []) + df.rename(columns=self.COLUMN_REMAP, inplace=True) + except Exception as e: + raise Exception(f"Failed to fetch items: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Parse and execute query + parser = SELECTQueryParser( + query, "items", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/journals_table.py b/mindsdb/integrations/handlers/xero_handler/tables/journals_table.py new file mode 100644 index 00000000000..0813a7b4977 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/journals_table.py @@ -0,0 +1,91 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class JournalsTable(XeroTable): + """Table for Xero Journals""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "journal_id": {"type": "where", "xero_field": "JournalID", "value_type": "guid"}, + "journal_date": {"type": "where", "xero_field": "Date", "value_type": "date"}, + "journal_number": {"type": "where", "xero_field": "JournalNumber", "value_type": "number"}, + "created_date_utc": {"type": "where", "xero_field": "CreatedDateUTC", "value_type": "date"}, + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + return [ + "journal_id", + "journal_date", + "journal_number", + "created_date_utc", + "journal_lines" + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + try: + # Fetch journals with optimized parameters + journals = api.get_journals(xero_tenant_id=self.handler.tenant_id, **api_params) + df = self._convert_response_to_dataframe(journals.journals or []) + df.rename(columns=self.COLUMN_REMAP, inplace=True) + except Exception as e: + raise Exception(f"Failed to fetch journals: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Parse and execute query + parser = SELECTQueryParser( + query, "journals", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/manual_journals_table.py b/mindsdb/integrations/handlers/xero_handler/tables/manual_journals_table.py new file mode 100644 index 00000000000..d85291d68ae --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/manual_journals_table.py @@ -0,0 +1,136 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class ManualJournalsTable(XeroTable): + """Table for Xero Manual Journals""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "manual_journal_id": {"type": "where", "xero_field": "ManualJournalID", "value_type": "guid"}, + "date": {"type": "where", "xero_field": "Date", "value_type": "date"}, + "status": {"type": "where", "xero_field": "Status", "value_type": "string"}, + "line_amount_types": {"type": "where", "xero_field": "LineAmountTypes", "value_type": "string"}, + "show_on_cash_basis_reports": {"type": "where", "xero_field": "ShowOnCashBasisReports", "value_type": "bool"}, + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + return [ + "manual_journal_id", + "date", + "line_amount_types", + "status", + "narration", + "journal_lines", + "show_on_cash_basis_reports", + "has_attachments", + "updated_date_utc" + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit (no default limit) + parser = SELECTQueryParser( + query, "manual_journals", columns=self.get_columns(), use_default_limit=False + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Implement pagination to fetch all required records + all_data = [] + page = 1 + page_size = 1000 # Xero's maximum page size + records_fetched = 0 + + try: + while result_limit is None or records_fetched < result_limit: + # Calculate how many records to fetch in this page + if result_limit is not None: + records_to_fetch = min(page_size, result_limit - records_fetched) + else: + records_to_fetch = page_size + + # Fetch manual journals with pagination parameters + response = api.get_manual_journals( + xero_tenant_id=self.handler.tenant_id, + page=page, + page_size=records_to_fetch, + **api_params + ) + + if not response.manual_journals: + break # No more data + + all_data.extend(response.manual_journals) + records_fetched += len(response.manual_journals) + + # Check pagination metadata to determine if there are more pages + if hasattr(response, 'pagination') and response.pagination: + # If we've reached the last page, stop + if page >= response.pagination.page_count: + break + else: + # Fallback: If we got fewer records than requested, we've reached the end + if len(response.manual_journals) < records_to_fetch: + break + + page += 1 + + except Exception as e: + raise Exception(f"Failed to fetch manual journals: {str(e)}") + + # Convert all data to DataFrame + df = self._convert_response_to_dataframe(all_data) + if len(df) > 0: + df.rename(columns=self.COLUMN_REMAP, inplace=True) + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit (in case filters reduced the result set) + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/organisations_table.py b/mindsdb/integrations/handlers/xero_handler/tables/organisations_table.py new file mode 100644 index 00000000000..ae940cc5163 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/organisations_table.py @@ -0,0 +1,111 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class OrganisationsTable(XeroTable): + """Table for Xero Organisations""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + return [ + "organisation_id", + "name", + "legal_name", + "pays_tax", + "version", + "organisation_type", + "base_currency", + "country_code", + "is_demo_company", + "organisation_status", + "tax_number", + "financial_year_end_day", + "financial_year_end_month", + "sales_tax_basis", + "sales_tax_period", + "default_sales_tax", + "default_purchases_tax", + "period_lock_date", + "created_date_utc", + "organisation_entity_type", + "timezone", + "short_code", + "edition", + "class", + "addresses", + "phones", + "external_links", + "payment_terms", + "tax_number_name" + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + try: + # Fetch organisations with optimized parameters + organisations = api.get_organisations(xero_tenant_id=self.handler.tenant_id, **api_params) + df = self._convert_response_to_dataframe(organisations.organisations or []) + df.rename(columns=self.COLUMN_REMAP, inplace=True) + except Exception as e: + raise Exception(f"Failed to fetch organisations: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Parse and execute query + parser = SELECTQueryParser( + query, "organisations", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/overpayments_table.py b/mindsdb/integrations/handlers/xero_handler/tables/overpayments_table.py new file mode 100644 index 00000000000..675c55f1bb5 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/overpayments_table.py @@ -0,0 +1,153 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class OverpaymentsTable(XeroTable): + """Table for Xero Overpayments""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "overpayment_id": {"type": "where", "xero_field": "OverpaymentID", "value_type": "guid"}, + "date": {"type": "where", "xero_field": "Date", "value_type": "date"}, + "status": {"type": "where", "xero_field": "Status", "value_type": "string"}, + "line_amount_types": {"type": "where", "xero_field": "LineAmountTypes", "value_type": "string"}, + "sub_total": {"type": "where", "xero_field": "SubTotal", "value_type": "number"}, + "total_tax": {"type": "where", "xero_field": "TotalTax", "value_type": "number"}, + "total": {"type": "where", "xero_field": "Total", "value_type": "number"}, + "currency_code": {"type": "where", "xero_field": "CurrencyCode", "value_type": "string"}, + "type": {"type": "where", "xero_field": "Type", "value_type": "string"}, + "currency_rate": {"type": "where", "xero_field": "CurrencyRate", "value_type": "number"}, + "remaining_credit": {"type": "where", "xero_field": "RemainingCredit", "value_type": "number"}, + "contact_id": {"type": "where", "xero_field": "Contact.ContactID", "value_type": "guid"}, + } + + COLUMN_REMAP = { + "contact_contact_id": "contact_id" + } + + def get_columns(self) -> List[str]: + return [ + "overpayment_id", + "contact_id", + "contact_name", + "date_string", + "date", + "status", + "line_amount_types", + "sub_total", + "total_tax", + "total", + "updated_date_utc", + "currency_code", + "type", + "currency_rate", + "remaining_credit", + "allocations", + "has_attachments" + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit (no default limit) + parser = SELECTQueryParser( + query, "overpayments", columns=self.get_columns(), use_default_limit=False + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Implement pagination to fetch all required records + all_data = [] + page = 1 + page_size = 1000 # Xero's maximum page size + records_fetched = 0 + + try: + while result_limit is None or records_fetched < result_limit: + # Calculate how many records to fetch in this page + if result_limit is not None: + records_to_fetch = min(page_size, result_limit - records_fetched) + else: + records_to_fetch = page_size + + # Fetch overpayments with pagination parameters + response = api.get_overpayments( + xero_tenant_id=self.handler.tenant_id, + page=page, + page_size=records_to_fetch, + **api_params + ) + + if not response.overpayments: + break # No more data + + all_data.extend(response.overpayments) + records_fetched += len(response.overpayments) + + # Check pagination metadata to determine if there are more pages + if hasattr(response, 'pagination') and response.pagination: + # If we've reached the last page, stop + if page >= response.pagination.page_count: + break + else: + # Fallback: If we got fewer records than requested, we've reached the end + if len(response.overpayments) < records_to_fetch: + break + + page += 1 + + except Exception as e: + raise Exception(f"Failed to fetch overpayments: {str(e)}") + + # Convert all data to DataFrame + df = self._convert_response_to_dataframe(all_data) + if len(df) > 0: + df.rename(columns=self.COLUMN_REMAP, inplace=True) + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit (in case filters reduced the result set) + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/payment_services_table.py b/mindsdb/integrations/handlers/xero_handler/tables/payment_services_table.py new file mode 100644 index 00000000000..b7d624004e1 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/payment_services_table.py @@ -0,0 +1,88 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class PaymentServicesTable(XeroTable): + """Table for Xero Payment Services""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + return [ + "payment_service_id", + "payment_service_name", + "payment_service_url", + "pay_now_text", + "payment_service_type", + + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + try: + # Fetch payment services with optimized parameters + payment_services = api.get_payment_services(xero_tenant_id=self.handler.tenant_id, **api_params) + df = self._convert_response_to_dataframe(payment_services.payment_services or []) + df.rename(columns=self.COLUMN_REMAP, inplace=True) + except Exception as e: + raise Exception(f"Failed to fetch payment services: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Parse and execute query + parser = SELECTQueryParser( + query, "payment_services", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/payments_table.py b/mindsdb/integrations/handlers/xero_handler/tables/payments_table.py new file mode 100644 index 00000000000..d4d6f682279 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/payments_table.py @@ -0,0 +1,214 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class PaymentsTable(XeroTable): + """Table for Xero Payments""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "payment_id": {"type": "where", "xero_field": "PaymentID", "value_type": "guid"}, + "status": {"type": "where", "xero_field": "Status", "value_type": "string"}, + "date": {"type": "where", "xero_field": "Date", "value_type": "date"}, + "invoice_id": {"type": "where", "xero_field": "Invoice.InvoiceID", "value_type": "guid"}, + "reference": {"type": "where", "xero_field": "Reference", "value_type": "string"}, + "contact_id": {"type": "where", "xero_field": "Invoice.Contact.ContactID", "value_type": "guid"}, + } + + COLUMN_REMAP = { + # Account fields (top-level) + "account_account_id": "account_id", + "account_code": "account_code", + + # BatchPayment nested fields + "batch_payment_batch_payment_id": "batch_payment_id", + "batch_payment_account_account_id": "batch_payment_account_id", + "batch_payment_account_code": "batch_payment_account_code", + "batch_payment_date_string": "batch_payment_date_string", + "batch_payment_date": "batch_payment_date", + "batch_payment_type": "batch_payment_type", + "batch_payment_status": "batch_payment_status", + "batch_payment_total_amount": "batch_payment_total_amount", + "batch_payment_updated_date_utc": "batch_payment_updated_date_utc", + "batch_payment_is_reconciled": "batch_payment_is_reconciled", + + # Invoice nested fields + "invoice_invoice_id": "invoice_id", + "invoice_invoice_number": "invoice_number", + "invoice_type": "invoice_type", + "invoice_currency_code": "invoice_currency_code", + "invoice_contact_contact_id": "invoice_contact_id", + "invoice_contact_name": "invoice_contact_name", + "invoice_payments": "invoice_payments", + "invoice_credit_notes": "invoice_credit_notes", + "invoice_prepayments": "invoice_prepayments", + "invoice_overpayments": "invoice_overpayments", + "invoice_line_items": "invoice_line_items", + "invoice_invoice_addresses": "invoice_addresses", + "invoice_contact_contact_groups": "invoice_contact_groups", + "invoice_contact_contact_persons": "invoice_contact_persons", + + } + + def get_columns(self) -> List[str]: + return [ + # Core payment fields + "payment_id", + "date", + "bank_amount", + "amount", + "reference", + "currency_rate", + "payment_type", + "status", + "updated_date_utc", + "has_account", + "is_reconciled", + "has_validation_errors", + + # Account fields + "account_id", + "account_code", + + # BatchPayment nested fields + "batch_payment_id", + "batch_payment_account_id", + "batch_payment_account_code", + "batch_payment_date_string", + "batch_payment_date", + "batch_payment_type", + "batch_payment_status", + "batch_payment_total_amount", + "batch_payment_updated_date_utc", + "batch_payment_is_reconciled", + + # Invoice nested fields + "invoice_id", + "invoice_number", + "invoice_type", + "invoice_currency_code", + "invoice_is_discounted", + "invoice_contact_id", + "invoice_contact_name", + "invoice_contact_addresses", + "invoice_contact_phones", + "invoice_contact_groups", + "invoice_contact_persons", + "invoice_contact_has_validation_errors", + + # Invoice list fields (JSON strings) + "invoice_payments", + "invoice_credit_notes", + "invoice_prepayments", + "invoice_overpayments", + "invoice_line_items", + "invoice_addresses", + "invoice_payment_services", + + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit (no default limit) + parser = SELECTQueryParser( + query, "payments", columns=self.get_columns(), use_default_limit=False + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Implement pagination to fetch all required records + all_data = [] + page = 1 + page_size = 1000 # Xero's maximum page size + records_fetched = 0 + + try: + while result_limit is None or records_fetched < result_limit: + # Calculate how many records to fetch in this page + if result_limit is not None: + records_to_fetch = min(page_size, result_limit - records_fetched) + else: + records_to_fetch = page_size + + # Fetch payments with pagination parameters + response = api.get_payments( + xero_tenant_id=self.handler.tenant_id, + page=page, + page_size=records_to_fetch, + **api_params + ) + + if not response.payments: + break # No more data + + all_data.extend(response.payments) + records_fetched += len(response.payments) + + # Check pagination metadata to determine if there are more pages + if hasattr(response, 'pagination') and response.pagination: + # If we've reached the last page, stop + if page >= response.pagination.page_count: + break + else: + # Fallback: If we got fewer records than requested, we've reached the end + if len(response.payments) < records_to_fetch: + break + + page += 1 + + except Exception as e: + raise Exception(f"Failed to fetch payments: {str(e)}") + + # Convert all data to DataFrame + df = self._convert_response_to_dataframe(all_data) + if len(df) > 0: + df.rename(columns=self.COLUMN_REMAP, inplace=True) + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit (in case filters reduced the result set) + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/prepayments_table.py b/mindsdb/integrations/handlers/xero_handler/tables/prepayments_table.py new file mode 100644 index 00000000000..7a00c4d1788 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/prepayments_table.py @@ -0,0 +1,154 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class PrepaymentsTable(XeroTable): + """Table for Xero Prepayments""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "prepayment_id": {"type": "where", "xero_field": "PrepaymentID", "value_type": "guid"}, + "status": {"type": "where", "xero_field": "Status", "value_type": "string"}, + "date": {"type": "where", "xero_field": "Date", "value_type": "date"}, + "contact_id": {"type": "where", "xero_field": "Contact.ContactID", "value_type": "guid"}, + "line_amount_types": {"type": "where", "xero_field": "LineAmountTypes", "value_type": "string"}, + "sub_total": {"type": "where", "xero_field": "SubTotal", "value_type": "number"}, + "total_tax": {"type": "where", "xero_field": "TotalTax", "value_type": "number"}, + "total": {"type": "where", "xero_field": "Total", "value_type": "number"}, + "currency_code": {"type": "where", "xero_field": "CurrencyCode", "value_type": "string"}, + "fully_paid_on_date": {"type": "where", "xero_field": "FullyPaidOnDate", "value_type": "date"}, + "type": {"type": "where", "xero_field": "Type", "value_type": "string"}, + "currency_rate": {"type": "where", "xero_field": "CurrencyRate", "value_type": "number"}, + "remaining_credit": {"type": "where", "xero_field": "RemainingCredit", "value_type": "number"}, + } + + COLUMN_REMAP = { + "contact_contact_id": "contact_id", + } + + def get_columns(self) -> List[str]: + return [ + "prepayment_id", + "contact_id", + "contact_name", + "date", + "status", + "line_amount_types", + "sub_total", + "total_tax", + "total", + "updated_date_utc", + "currency_code", + "fully_paid_on_date", + "type", + "currency_rate", + "remaining_credit", + "allocations", + "has_attachments" + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch accounts from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit (no default limit) + parser = SELECTQueryParser( + query, "prepayments", columns=self.get_columns(), use_default_limit=False + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Implement pagination to fetch all required records + all_data = [] + page = 1 + page_size = 1000 # Xero's maximum page size + records_fetched = 0 + + try: + while result_limit is None or records_fetched < result_limit: + # Calculate how many records to fetch in this page + if result_limit is not None: + records_to_fetch = min(page_size, result_limit - records_fetched) + else: + records_to_fetch = page_size + + # Fetch prepayments with pagination parameters + response = api.get_prepayments( + xero_tenant_id=self.handler.tenant_id, + page=page, + page_size=records_to_fetch, + **api_params + ) + + if not response.prepayments: + break # No more data + + all_data.extend(response.prepayments) + records_fetched += len(response.prepayments) + + # Check pagination metadata to determine if there are more pages + if hasattr(response, 'pagination') and response.pagination: + # If we've reached the last page, stop + if page >= response.pagination.page_count: + break + else: + # Fallback: If we got fewer records than requested, we've reached the end + if len(response.prepayments) < records_to_fetch: + break + + page += 1 + + except Exception as e: + raise Exception(f"Failed to fetch prepayments: {str(e)}") + + # Convert all data to DataFrame + df = self._convert_response_to_dataframe(all_data) + if len(df) > 0: + df.rename(columns=self.COLUMN_REMAP, inplace=True) + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit (in case filters reduced the result set) + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/profit_loss_report_table.py b/mindsdb/integrations/handlers/xero_handler/tables/profit_loss_report_table.py new file mode 100644 index 00000000000..2ae1ed3e562 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/profit_loss_report_table.py @@ -0,0 +1,144 @@ +from typing import List +from datetime import datetime, timedelta, date +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_report_tables import XeroReportTable +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + + +class ProfitLossReportTable(XeroReportTable): + """Table for Xero Profit and Loss Report""" + + # Define which parameters can be pushed to the Xero API + SUPPORTED_FILTERS = { + "from_date": {"type": "direct", "param": "from_date"}, + "to_date": {"type": "direct", "param": "to_date"}, + "periods": {"type": "direct", "param": "periods"}, + "timeframe": {"type": "direct", "param": "timeframe"}, + "tracking_category_id": {"type": "direct", "param": "tracking_category_id"}, + "tracking_option_id": {"type": "direct", "param": "tracking_option_id"}, + "tracking_category_id_2": {"type": "direct", "param": "tracking_category_id2"}, + "tracking_option_id_2": {"type": "direct", "param": "tracking_option_id2"}, + "standard_layout": {"type": "direct", "param": "standard_layout"}, + "payments_only": {"type": "direct", "param": "payments_only"}, + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + """ + Return column names for the profit and loss report. + + Returns: + List[str]: Column names + """ + base_columns = [ + "report_id", + "report_name", + "report_title", + "report_type", + "report_date", + "updated_date_utc", + "section", + "subsection", + "depth", + "row_type", + "row_title", + "account_id", + ] + + # Add generic period columns + period_columns = [f"period_{i}" for i in range(1, self.MAX_PERIODS + 1)] + + return base_columns + period_columns + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch profit and loss report from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit + parser = SELECTQueryParser( + query, "profit_loss_report", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Set default dates if not provided + # from_date defaults to beginning of current month + # to_date defaults to end of current month + now = datetime.now() + + if 'from_date' not in api_params: + # First day of current month as date object + api_params['from_date'] = date(now.year, now.month, 1) + else: + api_params['from_date'] = self._convert_date_parameter(api_params['from_date']) + + if 'to_date' not in api_params: + # Last day of current month as date object + # Get first day of next month, then subtract one day + if now.month == 12: + next_month = date(now.year + 1, 1, 1) + else: + next_month = date(now.year, now.month + 1, 1) + api_params['to_date'] = next_month - timedelta(days=1) + else: + api_params['to_date'] = self._convert_date_parameter(api_params['to_date']) + + try: + # Fetch profit and loss report + response = api.get_report_profit_and_loss( + xero_tenant_id=self.handler.tenant_id, + **api_params + ) + + # Parse report structure to DataFrame + df = self._parse_report_to_dataframe(response) + + except Exception as e: + raise Exception(f"Failed to fetch profit and loss report: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df diff --git a/mindsdb/integrations/handlers/xero_handler/tables/purchase_orders_table.py b/mindsdb/integrations/handlers/xero_handler/tables/purchase_orders_table.py new file mode 100644 index 00000000000..8aae600761a --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/purchase_orders_table.py @@ -0,0 +1,189 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class PurchaseOrdersTable(XeroTable): + """Table for Xero Purchase Orders""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "status": {"type": "direct", "param": "status", "value_type": "string"}, + "date": {"type": "date", "param": "date_from", "param_upper": "date_to"}, + } + + COLUMN_REMAP = { + # Contact nested fields + "contact_contact_id": "contact_id", + "contact_name": "contact_name", + "contact_contact_status": "contact_status", + "contact_addresses": "contact_addresses", + "contact_phones": "contact_phones", + "contact_updated_date_utc": "contact_updated_date_utc", + "contact_contact_groups": "contact_groups", + "contact_default_currency": "contact_default_currency", + "contact_contact_persons": "contact_persons", + "contact_has_validation_errors": "contact_has_validation_errors", + } + + def get_columns(self) -> List[str]: + return [ + # Core purchase order fields + "purchase_order_id", + "purchase_order_number", + "reference", + "status", + "type", + + # Date fields + "date", + "date_string", + "delivery_date", + "delivery_date_string", + "expected_arrival_date", + + # Financial fields + "currency_code", + "currency_rate", + "sub_total", + "total_tax", + "total", + "total_discount", + "line_amount_types", + + # Contact nested fields + "contact_id", + "contact_name", + "contact_status", + "contact_default_currency", + "contact_has_validation_errors", + + # Contact list fields (JSON strings) + "contact_addresses", + "contact_phones", + "contact_groups", + "contact_persons", + + # Delivery information + "delivery_address", + "attention_to", + "telephone", + "delivery_instructions", + "sent_to_contact", + + # Line items (JSON string) + "line_items", + + # Metadata + "branding_theme_id", + "has_attachments", + "has_errors", + "is_discounted", + "updated_date_utc", + "status_attribute_string", + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch purchase orders from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit (no default limit) + parser = SELECTQueryParser( + query, "purchase_orders", columns=self.get_columns(), use_default_limit=False + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Implement pagination to fetch all required records + all_data = [] + page = 1 + page_size = 1000 # Xero's maximum page size + records_fetched = 0 + + try: + while result_limit is None or records_fetched < result_limit: + # Calculate how many records to fetch in this page + if result_limit is not None: + records_to_fetch = min(page_size, result_limit - records_fetched) + else: + records_to_fetch = page_size + + # Fetch purchase orders with pagination parameters + response = api.get_purchase_orders( + xero_tenant_id=self.handler.tenant_id, + page=page, + page_size=records_to_fetch, + **api_params + ) + + if not response.purchase_orders: + break # No more data + + all_data.extend(response.purchase_orders) + records_fetched += len(response.purchase_orders) + + # Check pagination metadata to determine if there are more pages + if hasattr(response, 'pagination') and response.pagination: + # If we've reached the last page, stop + if page >= response.pagination.page_count: + break + else: + # Fallback: If we got fewer records than requested, we've reached the end + if len(response.purchase_orders) < records_to_fetch: + break + + page += 1 + + except Exception as e: + raise Exception(f"Failed to fetch purchase orders: {str(e)}") + + # Convert all data to DataFrame + df = self._convert_response_to_dataframe(all_data) + if len(df) > 0: + df.rename(columns=self.COLUMN_REMAP, inplace=True) + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit (in case filters reduced the result set) + if result_limit: + df = df.head(result_limit) + + return df diff --git a/mindsdb/integrations/handlers/xero_handler/tables/quotes_table.py b/mindsdb/integrations/handlers/xero_handler/tables/quotes_table.py new file mode 100644 index 00000000000..426a99fc9ad --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/quotes_table.py @@ -0,0 +1,107 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + + +class QuotesTable(XeroTable): + """Table for Xero Quotes""" + + # Define which columns can be pushed to the Xero API + SUPPORTED_FILTERS = { + "quote_number": {"type": "direct", "param": "quote_number"}, + "status": {"type": "direct", "param": "status"}, + "date": {"type": "date", "param": "date_from", "param_upper": "date_to"}, + "expiry_date": {"type": "date", "param": "expiry_date_from", "param_upper": "expiry_date_to"}, + "contact_id": {"type": "direct", "param": "contact_id"} + } + + def get_columns(self) -> List[str]: + return [ + "quote_id", + "quote_number", + "reference", + "terms", + "contact", + "line_items", + "date", + "date_string", + "expiry_date", + "expiry_date_string", + "status", + "currency_rate", + "currency_code", + "sub_total", + "total_tax", + "total", + "total_discount", + "title", + "summary", + "branding_theme_id", + "updated_date_utc", + "line_amount_types" + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch quotes from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + try: + # Fetch quotes with optimized parameters + quotes = api.get_quotes(xero_tenant_id=self.handler.tenant_id, **api_params) + df = self._convert_response_to_dataframe(quotes.quotes or []) + except Exception as e: + raise Exception(f"Failed to fetch quotes: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Parse and execute query + parser = SELECTQueryParser( + query, "quotes", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df \ No newline at end of file diff --git a/mindsdb/integrations/handlers/xero_handler/tables/repeating_invoices_table.py b/mindsdb/integrations/handlers/xero_handler/tables/repeating_invoices_table.py new file mode 100644 index 00000000000..85a174116ac --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/repeating_invoices_table.py @@ -0,0 +1,169 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + XeroTable, + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + +class RepeatingInvoicesTable(XeroTable): + """Table for Xero Repeating Invoices""" + + # Define which columns can be pushed to the Xero API + # Note: Repeating invoices only support WHERE clause filtering, no dedicated params + SUPPORTED_FILTERS = { + "repeating_invoice_id": {"type": "where", "xero_field": "RepeatingInvoiceID", "value_type": "guid"}, + "type": {"type": "where", "xero_field": "Type", "value_type": "string"}, + "status": {"type": "where", "xero_field": "Status", "value_type": "string"}, + "reference": {"type": "where", "xero_field": "Reference", "value_type": "string"}, + "approved_for_sending": {"type": "where", "xero_field": "ApprovedForSending", "value_type": "boolean"}, + "line_amount_types": {"type": "where", "xero_field": "LineAmountTypes", "value_type": "string"}, + "total": {"type": "where", "xero_field": "Total", "value_type": "number"}, + "sub_total": {"type": "where", "xero_field": "SubTotal", "value_type": "number"}, + "total_tax": {"type": "where", "xero_field": "TotalTax", "value_type": "number"}, + "currency_code": {"type": "where", "xero_field": "CurrencyCode", "value_type": "string"}, + + # Contact nested fields + "contact_id": {"type": "where", "xero_field": "Contact.ContactID", "value_type": "guid"}, + "contact_name": {"type": "where", "xero_field": "Contact.Name", "value_type": "string"}, + + # Schedule nested fields + "schedule_unit": {"type": "where", "xero_field": "Schedule.Unit", "value_type": "string"}, + "schedule_due_date": {"type": "where", "xero_field": "Schedule.DueDate", "value_type": "number"}, + "schedule_due_date_type": {"type": "where", "xero_field": "Schedule.DueDateType", "value_type": "string"}, + "schedule_next_scheduled_date": {"type": "where", "xero_field": "Schedule.NextScheduledDate", "value_type": "date"}, + + } + + COLUMN_REMAP = { + # Contact nested fields + "contact_contact_id": "contact_id", + "contact_name": "contact_name", + "contact_addresses": "contact_addresses", + "contact_phones": "contact_phones", + "contact_contact_groups": "contact_groups", + "contact_contact_persons": "contact_persons", + "contact_has_validation_errors": "contact_has_validation_errors", + + # Schedule nested fields + "schedule_period": "schedule_period", + "schedule_unit": "schedule_unit", + "schedule_due_date": "schedule_due_date", + "schedule_due_date_type": "schedule_due_date_type", + "schedule_start_date": "schedule_start_date", + "schedule_next_scheduled_date": "schedule_next_scheduled_date", + "schedule_end_date": "schedule_end_date", + } + + def get_columns(self) -> List[str]: + return [ + # Core repeating invoice fields + "repeating_invoice_id", + "id", + "type", + "status", + "reference", + + # Financial fields + "currency_code", + "line_amount_types", + "sub_total", + "total_tax", + "total", + + # Contact nested fields + "contact_id", + "contact_name", + "contact_has_validation_errors", + + # Contact list fields (JSON strings) + "contact_addresses", + "contact_phones", + "contact_groups", + "contact_persons", + + # Schedule nested fields + "schedule_period", + "schedule_unit", + "schedule_due_date", + "schedule_due_date_type", + "schedule_start_date", + "schedule_next_scheduled_date", + "schedule_end_date", + + # Line items (JSON string) + "line_items", + + # Metadata + "branding_theme_id", + "has_attachments", + "approved_for_sending", + "send_copy", + "mark_as_sent", + "include_pdf", + ] + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch repeating invoices from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + try: + # Fetch repeating invoices with WHERE clause if provided + repeating_invoices = api.get_repeating_invoices( + xero_tenant_id=self.handler.tenant_id, + **api_params + ) + df = self._convert_response_to_dataframe(repeating_invoices.repeating_invoices or []) + df.rename(columns=self.COLUMN_REMAP, inplace=True) + except Exception as e: + raise Exception(f"Failed to fetch repeating invoices: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Parse and execute query + parser = SELECTQueryParser( + query, "repeating_invoices", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df diff --git a/mindsdb/integrations/handlers/xero_handler/tables/trial_balance_report_table.py b/mindsdb/integrations/handlers/xero_handler/tables/trial_balance_report_table.py new file mode 100644 index 00000000000..f059051e0b3 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/tables/trial_balance_report_table.py @@ -0,0 +1,117 @@ +from typing import List +import pandas as pd +from mindsdb_sql_parser import ast +from mindsdb.integrations.handlers.xero_handler.xero_report_tables import XeroReportTable +from mindsdb.integrations.handlers.xero_handler.xero_tables import ( + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from xero_python.accounting import AccountingApi + + +class TrialBalanceReportTable(XeroReportTable): + """Table for Xero Trial Balance Report""" + + # Define which parameters can be pushed to the Xero API + SUPPORTED_FILTERS = { + "date": {"type": "direct", "param": "date"}, + "payments_only": {"type": "direct", "param": "payments_only"}, + } + + COLUMN_REMAP = {} + + def get_columns(self) -> List[str]: + """ + Return column names for the trial balance report. + + Returns: + List[str]: Column names + """ + base_columns = [ + "report_id", + "report_name", + "report_title", + "report_type", + "report_date", + "updated_date_utc", + "section", + "subsection", + "depth", + "row_type", + "row_title", + "account_id", + ] + + # Add generic period columns + period_columns = [f"period_{i}" for i in range(1, self.MAX_PERIODS + 1)] + + return base_columns + period_columns + + def select(self, query: ast.Select) -> pd.DataFrame: + """ + Fetch trial balance report from Xero API + + Args: + query: SELECT query + + Returns: + pd.DataFrame: Query results + """ + self.handler.connect() + api = AccountingApi(self.handler.api_client) + + # Parse query to get result_limit + parser = SELECTQueryParser( + query, "trial_balance_report", columns=self.get_columns() + ) + selected_columns, _, order_by_conditions, result_limit = parser.parse_query() + + # Extract and parse WHERE conditions + api_params = {} + remaining_conditions = [] + + if query.where: + conditions = extract_comparison_conditions(query.where) + api_params, remaining_conditions = self._parse_conditions_for_api( + conditions, self.SUPPORTED_FILTERS + ) + + # Convert date parameters to proper format + if 'date' in api_params: + api_params['date'] = self._convert_date_parameter(api_params['date']) + + try: + # Fetch trial balance report + response = api.get_report_trial_balance( + xero_tenant_id=self.handler.tenant_id, + **api_params + ) + + # Parse report structure to DataFrame + df = self._parse_report_to_dataframe(response) + + except Exception as e: + raise Exception(f"Failed to fetch trial balance report: {str(e)}") + + # Apply remaining filters in memory + if remaining_conditions and len(df) > 0: + df = filter_dataframe(df, remaining_conditions) + + # Apply column selection + if len(df) == 0: + df = pd.DataFrame([], columns=selected_columns) + else: + available_columns = [col for col in selected_columns if col in df.columns] + df = df[available_columns] + + # Apply ordering + if order_by_conditions: + df = sort_dataframe(df, order_by_conditions) + + # Apply limit + if result_limit: + df = df.head(result_limit) + + return df diff --git a/mindsdb/integrations/handlers/xero_handler/xero_handler.py b/mindsdb/integrations/handlers/xero_handler/xero_handler.py new file mode 100644 index 00000000000..3be6a14f980 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/xero_handler.py @@ -0,0 +1,597 @@ +import requests +import base64 +import threading +from typing import Optional, Dict, Any +from datetime import datetime, timedelta, timezone +import json + +from mindsdb.integrations.libs.api_handler import APIHandler +from mindsdb.integrations.libs.response import HandlerStatusResponse +from mindsdb.integrations.utilities.handlers.auth_utilities.exceptions import AuthException + +from xero_python.identity import IdentityApi +from xero_python.api_client import ApiClient, Configuration +from xero_python.api_client.configuration import OAuth2Token + +from .tables.accounts_table import AccountsTable +from .tables.bank_transactions_table import BankTransactionsTable +from .tables.budgets_table import BudgetsTable +from .tables.contact_groups_table import ContactGroupsTable +from .tables.contacts_table import ContactsTable +from .tables.quotes_table import QuotesTable +from .tables.credit_notes_table import CreditNotesTable +from .tables.invoices_table import InvoicesTable +from .tables.items_table import ItemsTable +from .tables.journals_table import JournalsTable +from .tables.manual_journals_table import ManualJournalsTable +from .tables.organisations_table import OrganisationsTable +from .tables.overpayments_table import OverpaymentsTable +from .tables.payments_table import PaymentsTable +from .tables.prepayments_table import PrepaymentsTable +from .tables.purchase_orders_table import PurchaseOrdersTable +from .tables.repeating_invoices_table import RepeatingInvoicesTable + +# Report tables +from .tables.balance_sheet_report_table import BalanceSheetReportTable +from .tables.profit_loss_report_table import ProfitLossReportTable +from .tables.trial_balance_report_table import TrialBalanceReportTable +from .tables.bank_summary_report_table import BankSummaryReportTable +from .tables.budget_summary_report_table import BudgetSummaryReportTable +from .tables.executive_summary_report_table import ExecutiveSummaryReportTable + +class XeroHandler(APIHandler): + """ + Xero Handler for MindsDB + + Implements OAuth2 authentication with Xero API and provides read-only access + to accounting data including budgets, contacts, invoices, items, and more. + """ + + name = 'xero' + + # Class-level lock to prevent concurrent token refresh attempts (race condition protection) + _refresh_lock = threading.Lock() + + def __init__(self, name: str, **kwargs): + """ + Initialize the Xero handler + + Args: + name: Handler name + **kwargs: Additional arguments including connection_data and handler_storage + """ + super().__init__(name) + self.connection_data = kwargs.get("connection_data", {}) + self.handler_storage = kwargs.get("handler_storage") + self.kwargs = kwargs + + self.connection = None + self.is_connected = False + self.tenant_id = None + self.api_client = None + + # OAuth2 configuration + self.client_id = self.connection_data.get("client_id") + self.client_secret = self.connection_data.get("client_secret") + self.redirect_uri = self.connection_data.get("redirect_uri") + self.code = self.connection_data.get("code") + + # Register tables + self._register_table("accounts", AccountsTable(self)) + self._register_table("bank_transactions", BankTransactionsTable(self)) + self._register_table("budgets", BudgetsTable(self)) + self._register_table("contact_groups", ContactGroupsTable(self)) + self._register_table("contacts", ContactsTable(self)) + self._register_table("credit_notes", CreditNotesTable(self)) + self._register_table("invoices", InvoicesTable(self)) + self._register_table("items", ItemsTable(self)) + self._register_table("journals", JournalsTable(self)) + self._register_table("manual_journals", ManualJournalsTable(self)) + self._register_table("organisations", OrganisationsTable(self)) + self._register_table("overpayments", OverpaymentsTable(self)) + # The PaymentServicesTable is not currently supported due to limitations in the Xero API + # and/or incomplete implementation in MindsDB. If Xero expands API support for payment services + # or if a future release of MindsDB implements the required functionality, this table registration + # may be enabled. For now, it remains commented out to avoid exposing unsupported features. + # self._register_table("payment_services", PaymentServicesTable(self)) + self._register_table("payments", PaymentsTable(self)) + self._register_table("prepayments", PrepaymentsTable(self)) + self._register_table("purchase_orders", PurchaseOrdersTable(self)) + self._register_table("quotes", QuotesTable(self)) + self._register_table("repeating_invoices", RepeatingInvoicesTable(self)) + + # Register report tables + self._register_table("balance_sheet_report", BalanceSheetReportTable(self)) + self._register_table("profit_loss_report", ProfitLossReportTable(self)) + self._register_table("trial_balance_report", TrialBalanceReportTable(self)) + self._register_table("bank_summary_report", BankSummaryReportTable(self)) + self._register_table("budget_summary_report", BudgetSummaryReportTable(self)) + self._register_table("executive_summary_report", ExecutiveSummaryReportTable(self)) + + def _use_token_injection_path(self) -> bool: + """ + Determine if using token injection (backend) or code flow (direct). + + Returns: + bool: True if access_token or refresh_token provided, False for code flow + """ + return "access_token" in self.connection_data or "refresh_token" in self.connection_data + + def connect(self) -> ApiClient: + """ + Establish connection to Xero API with OAuth2 authentication. + + Supports two authentication paths: + 1. Token Injection (backend): Uses provided access_token/refresh_token + 2. Code Flow (direct): OAuth2 authorization code exchange + + Returns: + ApiClient: The configured Xero API client + """ + if self.is_connected and self.connection is not None: + return self.connection + + try: + # Choose authentication path + if self._use_token_injection_path(): + # Modern path: Token injection from backend + token_data = self._connect_with_token_injection() + else: + # Legacy path: Code exchange flow + token_data = self._connect_with_code_exchange() + + # Set tenant_id if provided or use stored one + self.tenant_id = self.connection_data.get("tenant_id") or token_data.get("tenant_id") + + # Create API client with access token + self._setup_api_client(token_data["access_token"]) + self.is_connected = True + + except AuthException: + raise + except Exception as e: + raise Exception(f"Failed to connect to Xero: {str(e)}") + + return self.connection + + def _connect_with_token_injection(self) -> Dict[str, Any]: + """ + Handle authentication via token injection from backend systems. + + Supports: + - Loading stored tokens from previous refresh operations (most important for rotating refresh tokens!) + - Falling back to connection_data tokens if no stored tokens available + - Automatic refresh if refresh_token and client credentials provided + - Race condition protection: only one thread refreshes tokens at a time + - Grace period of 5 minutes before token expiry + - Token refresh skipped if client credentials not available (use access_token as-is) + + **Rotating Refresh Token Pattern:** + Xero invalidates refresh tokens after each use and returns a new one. This method: + 1. Tries to load previously stored tokens (which contain the latest refresh token) + 2. Falls back to connection_data tokens only for initial setup + 3. Uses a lock to prevent concurrent refresh attempts (critical for rotating tokens) + + Returns: + dict: Token data with access_token, refresh_token, expires_at, tenant_id + """ + # Step 1: Try to load previously stored tokens first + # This is CRITICAL for rotating refresh tokens - stored tokens have the latest refresh token + stored_token_data = self._load_stored_tokens() + + if stored_token_data: + token_data = stored_token_data + else: + # No stored tokens - use provided tokens from connection data + access_token = self.connection_data.get("access_token") + refresh_token = self.connection_data.get("refresh_token") + expires_at = self.connection_data.get("expires_at") + + if not access_token and not refresh_token: + raise ValueError("At least access_token or refresh_token must be provided for token injection") + + # Build initial token data + token_data = { + "access_token": access_token, + "refresh_token": refresh_token, + "expires_at": expires_at, + "tenant_id": self.connection_data.get("tenant_id"), + } + + # Store initial tokens so next connection uses them (important for rotating refresh tokens) + self._store_tokens(token_data) + + # Step 2: Check if token needs refresh with race condition protection + if self._is_token_expired(token_data) and token_data.get("refresh_token"): + if self.client_id and self.client_secret: + # Acquire lock to prevent concurrent token refresh attempts + with self._refresh_lock: + # Double-check pattern: re-check stored tokens after acquiring lock + # Another thread may have already refreshed the token + stored_token_data = self._load_stored_tokens() + if stored_token_data and not self._is_token_expired(stored_token_data): + # Token was refreshed by another thread while we waited for the lock + token_data = stored_token_data + else: + # Proceed with refresh + token_data = self._refresh_tokens(token_data["refresh_token"]) + self._store_tokens(token_data) + + # Update connection_data for current session to use new tokens + self.connection_data["access_token"] = token_data["access_token"] + if "refresh_token" in token_data: + self.connection_data["refresh_token"] = token_data["refresh_token"] + if "expires_at" in token_data: + self.connection_data["expires_at"] = token_data["expires_at"] + else: + # No credentials available for refresh - warn but continue with expired token + # The API call will fail if token is truly invalid + pass + elif not token_data.get("access_token") and token_data.get("refresh_token"): + # No access token but have refresh token - try to get new one + if self.client_id and self.client_secret: + with self._refresh_lock: + # Double-check after acquiring lock + stored_token_data = self._load_stored_tokens() + if stored_token_data and stored_token_data.get("access_token"): + token_data = stored_token_data + else: + token_data = self._refresh_tokens(token_data["refresh_token"]) + self._store_tokens(token_data) + + # Update connection_data for current session + self.connection_data["access_token"] = token_data["access_token"] + if "refresh_token" in token_data: + self.connection_data["refresh_token"] = token_data["refresh_token"] + else: + raise ValueError( + "Cannot refresh token: access_token is missing and client credentials (client_id/client_secret) " + "are not provided. Please provide either a valid access_token or both client credentials." + ) + + return token_data + + def _connect_with_code_exchange(self) -> Dict[str, Any]: + """ + Handle traditional OAuth2 code flow authentication. + + Flow: + 1. Check for stored tokens + 2. Refresh if expired + 3. Exchange code if provided + 4. Raise AuthException if no tokens/code available + + Returns: + dict: Token data with access_token, refresh_token, expires_at, tenant_id + """ + # Try to load existing tokens from storage + token_data = self._load_stored_tokens() + + if token_data: + # Check if token is expired + if self._is_token_expired(token_data): + token_data = self._refresh_tokens(token_data["refresh_token"]) + self._store_tokens(token_data) + return token_data + + if self.code: + # Exchange authorization code for tokens + token_data = self._exchange_code() + self._store_tokens(token_data) + return token_data + + # No tokens and no code - need authorization + auth_url = self._get_auth_url() + raise AuthException( + f"Authorization required. Please visit the following URL to authorize:\n{auth_url}", + auth_url=auth_url, + ) + + def check_connection(self) -> HandlerStatusResponse: + """ + Check if the connection to Xero API is active + + Returns: + HandlerStatusResponse: Status response with connection details + """ + response = HandlerStatusResponse(success=False) + + try: + self.connect() + + # Try to fetch identity information to verify connection + identity_api = IdentityApi(self.api_client) + connections = identity_api.get_connections() + + if len(connections) > 0: + response.success = True + # IMPORTANT: Set copy_storage to persist refreshed tokens between requests + # This ensures that tokens refreshed during this connection are saved for future connections + response.copy_storage = "success" + else: + response.error_message = "No Xero connections found for this user" + except AuthException as e: + # For auth exceptions, return them with redirect URL if available + response.success = False + response.error_message = str(e) + if hasattr(e, 'auth_url') and e.auth_url: + response.redirect_url = e.auth_url + except Exception as e: + response.error_message = f"Connection check failed: {str(e)}" + + return response + + def _setup_api_client(self, access_token: str) -> None: + """ + Setup the Xero API client with the access token + + Args: + access_token: OAuth2 access token + """ + # Create OAuth2Token object with the access token + oauth2_token = OAuth2Token( + client_id=self.client_id, + client_secret=self.client_secret, + ) + + # Update token with the access token + oauth2_token.update_token( + access_token=access_token, + refresh_token=None, + scope=["openid", "profile", "email", "accounting.transactions", "accounting.settings"], + expires_in=1800, # 30 minutes + token_type="Bearer", + ) + + # Create configuration with the OAuth2Token + configuration = Configuration(oauth2_token=oauth2_token) + + # Create ApiClient with dummy token saver/getter (we're read-only) + self.api_client = ApiClient( + configuration=configuration, + oauth2_token_saver=lambda token: None, # No-op saver + oauth2_token_getter=lambda: { + "access_token": oauth2_token.access_token, + "refresh_token": oauth2_token.refresh_token, + "scope": oauth2_token.scope, + "expires_in": oauth2_token.expires_in, + "token_type": oauth2_token.token_type, + }, + ) + + def _get_auth_url(self) -> str: + """ + Generate the Xero OAuth2 authorization URL + + Returns: + str: Authorization URL for user to visit + """ + base_url = "https://login.xero.com/identity/connect/authorize" + params = { + "response_type": "code", + "client_id": self.client_id, + "redirect_uri": self.redirect_uri, + "scope": ( + "openid profile email accounting.transactions accounting.settings offline_access" + ), + "state": "security_token", + } + + query_string = "&".join([f"{k}={v}" for k, v in params.items()]) + return f"{base_url}?{query_string}" + + def _exchange_code(self) -> Dict[str, Any]: + """ + Exchange authorization code for access and refresh tokens + + Returns: + dict: Token data including access_token, refresh_token, expires_at, and tenant_id + + Raises: + Exception: If code exchange fails + """ + token_url = "https://identity.xero.com/connect/token" + + data = { + "grant_type": "authorization_code", + "code": self.code, + "redirect_uri": self.redirect_uri, + "client_id": self.client_id, + "client_secret": self.client_secret, + } + + response = requests.post(token_url, data=data) + response.raise_for_status() + + token_response = response.json() + + # Get tenant ID from identity endpoint + tenant_id = self._get_tenant_id(token_response["access_token"]) + + return { + "access_token": token_response["access_token"], + "refresh_token": token_response["refresh_token"], + "expires_at": datetime.now(timezone.utc) + timedelta(seconds=token_response["expires_in"]), + "tenant_id": tenant_id, + } + + def _refresh_tokens(self, refresh_token: str) -> Dict[str, Any]: + """ + Refresh the access token using the refresh token with Basic Authentication. + + Per Xero documentation, token refresh requires: + - Basic Authentication header with base64(client_id:client_secret) + - POST request body with grant_type and refresh_token + + **CRITICAL: Xero Rotating Refresh Tokens** + - Xero invalidates the refresh token after each use + - The response ALWAYS includes a new refresh_token + - We MUST extract and use this new token, not fall back to the old one + - Using the old token will fail with "Invalid refresh token" on next refresh + + Args: + refresh_token: OAuth2 refresh token + + Returns: + dict: Updated token data with access_token, NEW refresh_token, expires_at, tenant_id + + Raises: + ValueError: If credentials are missing + Exception: If token refresh fails or response doesn't contain new refresh token + """ + token_url = "https://identity.xero.com/connect/token" + + # Validate that client credentials are available + if not self.client_id or not self.client_secret: + raise ValueError( + "Client ID and Client Secret are required to refresh tokens. " + "Please provide these credentials in your connection configuration." + ) + + # Create Basic Authentication header + auth_string = base64.b64encode( + f"{self.client_id}:{self.client_secret}".encode() + ).decode() + + headers = { + "Authorization": f"Basic {auth_string}", + "Content-Type": "application/x-www-form-urlencoded", + } + + data = { + "grant_type": "refresh_token", + "refresh_token": refresh_token, + } + + response = requests.post(token_url, headers=headers, data=data) + response.raise_for_status() + + token_response = response.json() + + # CRITICAL: Xero ALWAYS returns a new refresh_token in the response + # If it's missing, something went wrong + if "refresh_token" not in token_response: + raise Exception( + f"Xero token refresh response did not include a new refresh_token. " + f"This indicates a critical issue with token rotation. Response keys: {list(token_response.keys())}" + ) + + # Preserve tenant_id from connection data or stored tokens + tenant_id = ( + self.connection_data.get("tenant_id") + or (self._load_stored_tokens() or {}).get("tenant_id") + ) + + return { + "access_token": token_response["access_token"], + "refresh_token": token_response["refresh_token"], # Use NEW refresh token, never fallback to old one + "expires_at": datetime.now(timezone.utc) + timedelta(seconds=token_response["expires_in"]), + "tenant_id": tenant_id, + } + + def _get_tenant_id(self, access_token: str) -> str: + """ + Get the tenant ID from Xero's identity endpoint + + Args: + access_token: OAuth2 access token + + Returns: + str: Tenant ID (organization ID) + """ + identity_url = "https://api.xero.com/api.xro/2.0/Connections" + headers = {"Authorization": f"Bearer {access_token}"} + + response = requests.get(identity_url, headers=headers) + response.raise_for_status() + + connections = response.json() + if connections and len(connections) > 0: + return connections[0].get("tenantId") + + raise Exception("No Xero tenants found for this user") + + def _is_token_expired(self, token_data: Dict[str, Any]) -> bool: + """ + Check if the access token is expired or about to expire. + + Tokens are considered expired if they expire within 5 minutes (grace period). + Supports both ISO 8601 string format and Unix timestamps. + + Args: + token_data: Token data dictionary + + Returns: + bool: True if token is expired or expires within 5 minutes + """ + if not token_data or "expires_at" not in token_data: + return True + + expires_at = token_data["expires_at"] + if not expires_at: + return True + + # Parse expires_at to datetime + if isinstance(expires_at, str): + try: + expires_at = datetime.fromisoformat(expires_at) + except (ValueError, TypeError): + # If parsing fails, assume expired + return True + elif isinstance(expires_at, (int, float)): + # Unix timestamp + expires_at = datetime.fromtimestamp(expires_at) + else: + # Unknown format, assume expired + return True + + # Consider token expired if it expires within 5 minutes (grace period) + buffer_time = datetime.now(timezone.utc) + timedelta(minutes=5) + return buffer_time > expires_at + + def _store_tokens(self, token_data: Dict[str, Any]) -> None: + """ + Store tokens securely in handler storage + + Args: + token_data: Token data to store + """ + if not self.handler_storage: + return + + # Convert datetime to string for JSON serialization + stored_data = token_data.copy() + if isinstance(stored_data.get("expires_at"), datetime): + stored_data["expires_at"] = stored_data["expires_at"].isoformat() + + self.handler_storage.encrypted_json_set("xero_tokens", stored_data) + + def _load_stored_tokens(self) -> Optional[Dict[str, Any]]: + """ + Load stored tokens from handler storage + + Returns: + dict: Stored token data or None if not found + """ + if not self.handler_storage: + return None + + try: + token_data = self.handler_storage.encrypted_json_get("xero_tokens") + if token_data and isinstance(token_data.get("expires_at"), str): + token_data["expires_at"] = datetime.fromisoformat(token_data["expires_at"]) + return token_data + except Exception: + return None + + def native_query(self, query: str) -> None: + """ + Execute native query - not supported for Xero + + Args: + query: SQL query + + Raises: + NotImplementedError + """ + raise NotImplementedError("Native queries are not supported for Xero handler") diff --git a/mindsdb/integrations/handlers/xero_handler/xero_report_tables.py b/mindsdb/integrations/handlers/xero_handler/xero_report_tables.py new file mode 100644 index 00000000000..49f385eec87 --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/xero_report_tables.py @@ -0,0 +1,298 @@ +from typing import List, Dict, Any, Optional +import pandas as pd +from mindsdb.integrations.handlers.xero_handler.xero_tables import XeroTable + + +class XeroReportTable(XeroTable): + """ + Base class for Xero Report tables. + + Xero reports have a hierarchical row/cell structure that differs from regular entity endpoints. + This class provides common functionality for parsing report responses into flat DataFrames. + """ + + # Maximum number of period columns to support (for multi-period reports) + MAX_PERIODS = 12 + + def _parse_report_to_dataframe(self, response) -> pd.DataFrame: + """ + Parse Xero report response into a flattened DataFrame. + + Xero reports return a hierarchical structure: + - ReportWithRows.reports[].rows[] (recursive) + - Each row has: row_type, title, cells[] + - Rows can contain nested rows (sections/subsections) + + This method flattens the hierarchy into a single-level table with: + - report_id, report_name, report_date (report metadata) + - section, subsection, depth (hierarchy info) + - row_title, row_type (row data) + - account_id (from cell attributes if available) + - period_1, period_2, ... (cell values mapped to generic columns) + + Args: + response: ReportWithRows response from Xero API + + Returns: + pd.DataFrame: Flattened report data + """ + all_rows = [] + + # Xero API can return multiple reports (though usually just one) + if not hasattr(response, 'reports') or not response.reports: + return pd.DataFrame() + + for report in response.reports: + # Extract report metadata + report_metadata = { + 'report_id': getattr(report, 'report_id', None), + 'report_name': getattr(report, 'report_name', None), + 'report_title': getattr(report, 'report_title', None), + 'report_type': getattr(report, 'report_type', None), + 'report_date': getattr(report, 'report_date', None), + 'updated_date_utc': getattr(report, 'updated_date_utc', None), + } + + # Get column names from header row + column_names = self._extract_column_names(report.rows) + + # Parse all rows recursively + parsed_rows = self._parse_rows_recursive( + report.rows, + column_names=column_names, + parent_section='', + parent_subsection='', + depth=0 + ) + + # Add report metadata to each row + for row_data in parsed_rows: + row_data.update(report_metadata) + + all_rows.extend(parsed_rows) + + # Convert to DataFrame + if not all_rows: + return pd.DataFrame() + + df = pd.DataFrame(all_rows) + + # Ensure period columns exist (even if empty) + for i in range(1, self.MAX_PERIODS + 1): + period_col = f'period_{i}' + if period_col not in df.columns: + df[period_col] = None + + # Reorder columns to put metadata first + metadata_cols = ['report_id', 'report_name', 'report_title', 'report_type', + 'report_date', 'updated_date_utc', 'section', 'subsection', + 'depth', 'row_type', 'row_title', 'account_id'] + period_cols = [f'period_{i}' for i in range(1, self.MAX_PERIODS + 1)] + + # Only include columns that exist + ordered_cols = [col for col in metadata_cols if col in df.columns] + ordered_cols += [col for col in period_cols if col in df.columns] + + # Add any remaining columns + remaining_cols = [col for col in df.columns if col not in ordered_cols] + ordered_cols += remaining_cols + + df = df[ordered_cols] + + return df + + def _extract_column_names(self, rows: List) -> List[str]: + """ + Extract column names from the header row. + + Args: + rows: List of ReportRow objects + + Returns: + List[str]: Column names extracted from header cells + """ + if not rows: + return [] + + # Find the first row with row_type='Header' + for row in rows: + row_type = getattr(row, 'row_type', None) + if row_type and str(row_type).upper() in ['HEADER', 'ROW_TYPE_HEADER']: + cells = getattr(row, 'cells', []) + if cells: + return [self._get_cell_value(cell) for cell in cells] + + # If no header row found, try the first row with cells + for row in rows: + cells = getattr(row, 'cells', []) + if cells: + return [self._get_cell_value(cell) for cell in cells] + + return [] + + def _parse_rows_recursive( + self, + rows: List, + column_names: List[str], + parent_section: str = '', + parent_subsection: str = '', + depth: int = 0 + ) -> List[Dict[str, Any]]: + """ + Recursively parse rows and nested rows into flat records. + + Args: + rows: List of ReportRow objects + column_names: List of column names from header + parent_section: Section title from parent row + parent_subsection: Subsection title from parent row + depth: Current nesting depth + + Returns: + List[Dict]: Flattened row data + """ + parsed_rows = [] + + for row in rows: + row_type = str(getattr(row, 'row_type', 'Row')).upper() + row_title = getattr(row, 'title', '') + cells = getattr(row, 'cells', []) + nested_rows = getattr(row, 'rows', []) + + # Skip header rows (already processed) + if row_type in ['HEADER', 'ROW_TYPE_HEADER']: + continue + + # Handle section rows (contain nested rows) + if row_type in ['SECTION', 'ROW_TYPE_SECTION'] and nested_rows: + # Section row - update context and recurse into nested rows + new_section = row_title if depth == 0 else parent_section + new_subsection = row_title if depth > 0 else parent_subsection + + parsed_rows.extend( + self._parse_rows_recursive( + nested_rows, + column_names=column_names, + parent_section=new_section, + parent_subsection=new_subsection, + depth=depth + 1 + ) + ) + else: + # Data row or summary row - extract cell values + # Initialize row data + actual_row_title = row_title + + # If row has cells, extract data + if cells: + # First cell typically contains the row title/account name + first_cell = cells[0] + first_cell_value = self._get_cell_value(first_cell) + + # Use first cell value as row title if row.title is empty/None + if not actual_row_title and first_cell_value: + actual_row_title = first_cell_value + + row_data = { + 'section': parent_section, + 'subsection': parent_subsection, + 'depth': depth, + 'row_type': row_type, + 'row_title': actual_row_title, + } + + # Extract cell values and map to generic period columns + for i, cell in enumerate(cells): + cell_value = self._get_cell_value(cell) + + if i == 0: + # First column: get account_id from cell attributes if available + account_id = self._get_cell_attribute(cell, 'account') + if account_id: + row_data['account_id'] = account_id + else: + # Subsequent columns: map to period columns + # Period numbers start at 1 (column 0 is the row title) + period_num = i + if period_num <= self.MAX_PERIODS: + row_data[f'period_{period_num}'] = cell_value + + parsed_rows.append(row_data) + + # If this row has nested rows, recurse + if nested_rows: + parsed_rows.extend( + self._parse_rows_recursive( + nested_rows, + column_names=column_names, + parent_section=parent_section, + parent_subsection=row_title, # Current row becomes subsection + depth=depth + 1 + ) + ) + + return parsed_rows + + def _get_cell_value(self, cell) -> Optional[str]: + """ + Extract value from a report cell. + + Args: + cell: ReportCell object + + Returns: + str or None: Cell value + """ + if not cell: + return None + + value = getattr(cell, 'value', None) + return str(value) if value is not None else None + + def _get_cell_attribute(self, cell, attribute_id: str) -> Optional[str]: + """ + Extract a specific attribute from cell attributes. + + Args: + cell: ReportCell object + attribute_id: Attribute ID to extract (e.g., 'account') + + Returns: + str or None: Attribute value + """ + if not cell: + return None + + attributes = getattr(cell, 'attributes', []) + if not attributes: + return None + + for attr in attributes: + attr_id = getattr(attr, 'id', None) + if attr_id == attribute_id: + return getattr(attr, 'value', None) + + return None + + def _convert_date_parameter(self, value: Any) -> str: + """ + Convert date parameter to Xero API format (YYYY-MM-DD). + + Args: + value: Date value (string, datetime, etc.) + + Returns: + str: Formatted date string + """ + if not value: + return None + + # If already a string in YYYY-MM-DD format, return as-is + if isinstance(value, str): + return value + + # If datetime object, format it + if hasattr(value, 'strftime'): + return value.strftime('%Y-%m-%d') + + return str(value) diff --git a/mindsdb/integrations/handlers/xero_handler/xero_tables.py b/mindsdb/integrations/handlers/xero_handler/xero_tables.py new file mode 100644 index 00000000000..f9015d95c2e --- /dev/null +++ b/mindsdb/integrations/handlers/xero_handler/xero_tables.py @@ -0,0 +1,360 @@ +import pandas as pd +import json +import re +from abc import abstractmethod +from typing import List, Dict, Tuple, Any +from enum import Enum +from datetime import datetime, date +from decimal import Decimal +from mindsdb.integrations.libs.api_handler import APITable +from mindsdb_sql_parser import ast +from mindsdb.integrations.utilities.handlers.query_utilities import SELECTQueryParser +from mindsdb.integrations.utilities.sql_utils import ( + extract_comparison_conditions, + filter_dataframe, + sort_dataframe +) +from xero_python.accounting import AccountingApi + + +class XeroTable(APITable): + """ + Base class for Xero API tables with common functionality + """ + + def __init__(self, handler): + """ + Initialize the Xero table + + Args: + handler: The Xero handler instance + """ + super().__init__(handler) + self.handler = handler + + def _to_snake_case(self, name: str) -> str: + """ + Convert PascalCase or camelCase string to snake_case + + Args: + name: String in PascalCase or camelCase + + Returns: + str: String in snake_case + """ + # Insert an underscore before any uppercase letter that follows a lowercase letter or digit + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + # Insert an underscore before any uppercase letter that follows a lowercase letter, digit, or uppercase letter followed by lowercase + s2 = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1) + return s2.lower() + + def insert(self, query: ast.Insert) -> None: + """Insert operations are not supported""" + raise NotImplementedError("Insert operations are not supported for Xero tables") + + def update(self, query: ast.Update) -> None: + """Update operations are not supported""" + raise NotImplementedError("Update operations are not supported for Xero tables") + + def delete(self, query: ast.Delete) -> None: + """Delete operations are not supported""" + raise NotImplementedError("Delete operations are not supported for Xero tables") + + def _json_serialize(self, obj: Any) -> Any: + """ + Convert non-JSON-serializable objects to JSON-serializable types + + Args: + obj: Object to serialize + + Returns: + JSON-serializable representation of the object + """ + if isinstance(obj, Decimal): + return float(obj) + elif isinstance(obj, (datetime, date)): + return obj.isoformat() + elif isinstance(obj, Enum): + return obj.value + elif isinstance(obj, dict): + return {k: self._json_serialize(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [self._json_serialize(item) for item in obj] + return obj + + def _flatten_dict(self, value: Any, prefix: str = "", depth: int = 0, max_depth: int = 3) -> Dict[str, Any]: + """ + Recursively flatten nested dictionaries up to a maximum depth + + Args: + value: The value to flatten (dict, list, or primitive) + prefix: Current key prefix for nested keys + depth: Current recursion depth + max_depth: Maximum depth to recurse (default 3) + + Returns: + Dict[str, Any]: Flattened dictionary with underscore-separated keys + """ + result = {} + + # Handle None or empty values - skip creating columns + if value is None or (isinstance(value, (dict, list)) and not value): + return result + + # Handle Enum at any depth + if isinstance(value, Enum): + return {prefix: value.value} + + # Handle Decimal - convert to float + if isinstance(value, Decimal): + return {prefix: float(value)} + + # Handle datetime/date - convert to ISO string + if isinstance(value, (datetime, date)): + return {prefix: value.isoformat()} + + # Handle dictionaries - recurse if within depth limit + if isinstance(value, dict): + if depth >= max_depth: + # At max depth, convert to JSON string with custom serialization + result[prefix] = json.dumps(self._json_serialize(value)) + else: + for sub_key, sub_value in value.items(): + # Convert sub_key to snake_case + snake_case_key = self._to_snake_case(sub_key) + new_prefix = f"{prefix}_{snake_case_key}" if prefix else snake_case_key + flattened = self._flatten_dict(sub_value, new_prefix, depth + 1, max_depth) + result.update(flattened) + return result + + # Handle lists - convert to JSON string with custom serialization + if isinstance(value, list): + result[prefix] = json.dumps(self._json_serialize(value)) + return result + + # Handle primitive values + result[prefix] = value + return result + + def _convert_response_to_dataframe(self, response_data: list) -> pd.DataFrame: + """ + Convert API response to DataFrame with recursive flattening + + Args: + response_data: List of response objects + + Returns: + pd.DataFrame: Flattened dataframe with nested objects expanded up to 3 levels + """ + if not response_data: + return pd.DataFrame() + + # Convert objects to dictionaries + rows = [] + for item in response_data: + if hasattr(item, "to_dict"): + row = item.to_dict() + elif isinstance(item, dict): + row = item + else: + row = item.__dict__ + + # Recursively flatten the row + parsed_row = {} + for key, value in row.items(): + # Convert the initial key to snake_case as well + snake_case_key = self._to_snake_case(key) + flattened = self._flatten_dict(value, snake_case_key, depth=0, max_depth=3) + parsed_row.update(flattened) + + rows.append(parsed_row) + + df = pd.DataFrame(rows) + return df + + def _map_operator_to_xero(self, sql_op: str) -> str: + """ + Map SQL operator to Xero WHERE clause operator + + Args: + sql_op: SQL operator (=, !=, >, <, >=, <=) + + Returns: + str: Xero operator (== for =, others unchanged) + """ + mapping = { + "=": "==", + "!=": "!=", + ">": ">", + "<": "<", + ">=": ">=", + "<=": "<=", + } + return mapping.get(sql_op.lower(), "==") + + def _format_value_for_xero(self, value: Any, value_type: str) -> str: + """ + Format value for Xero WHERE clause + + Args: + value: The value to format + value_type: Type hint ('string', 'number', 'date', 'guid') + + Returns: + str: Formatted value for Xero WHERE clause + """ + if value_type == "string": + # Escape quotes and wrap in double quotes + escaped_value = str(value).replace('"', '\\"') + return f'"{escaped_value}"' + if value_type == "bool": + return "true" if value else "false" + elif value_type == "number": + return str(value) + elif value_type == "date": + # Convert to Xero date format: DateTime(year, month, day) + # Handle various date formats + if isinstance(value, datetime): + date_obj = value + elif isinstance(value, str): + # Try parsing common date formats + for fmt in ["%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%Y/%m/%d", "%d-%m-%Y", "%d/%m/%Y"]: + try: + date_obj = datetime.strptime(value, fmt) + break + except ValueError: + continue + else: + # If no format matches, try ISO format parse + try: + date_obj = datetime.fromisoformat(str(value).replace('Z', '+00:00')) + except: + raise ValueError(f"Unable to parse date value: {value}") + else: + raise ValueError(f"Unsupported date type: {type(value)}") + + # Format as Xero expects: DateTime(year, month, day) + return f'DateTime({date_obj.year}, {date_obj.month:02d}, {date_obj.day:02d})' + elif value_type == "guid": + return f'Guid("{value}")' + else: + # Default to string + escaped_value = str(value).replace('"', '\\"') + return f'"{escaped_value}"' + + def _parse_conditions_for_api( + self, conditions: List, supported_filters: Dict + ) -> Tuple[Dict, List]: + """ + Parse WHERE conditions into API parameters and remaining conditions + + Args: + conditions: List of [operator, column, value] from extract_comparison_conditions + supported_filters: Dict mapping column names to their API parameter info + + Returns: + Tuple of (api_params dict, remaining_conditions list) + """ + api_params = {} + remaining_conditions = [] + xero_where_clauses = [] + + for op, column, value in conditions: + # Handle BETWEEN operator by converting to >= and <= + if op.lower() == "between": + if not isinstance(value, (tuple, list)) or len(value) != 2: + remaining_conditions.append([op, column, value]) + continue + # Convert BETWEEN to two separate conditions: >= lower_bound AND <= upper_bound + lower_bound, upper_bound = value + # Recursively process the two conditions + lower_conditions, _ = self._parse_conditions_for_api( + [[">=", column, lower_bound]], supported_filters + ) + upper_conditions, _ = self._parse_conditions_for_api( + [["<=", column, upper_bound]], supported_filters + ) + # Merge the conditions into api_params + for key, val in lower_conditions.items(): + if key == "where": + xero_where_clauses.append(val) + else: + api_params[key] = val + for key, val in upper_conditions.items(): + if key == "where": + xero_where_clauses.append(val) + else: + api_params[key] = val + continue + + filter_info = supported_filters.get(column) + + if not filter_info: + # Cannot push down, filter in memory + remaining_conditions.append([op, column, value]) + continue + + filter_type = filter_info.get("type", "direct") + + if filter_type == "id_list": + # For i_ds, contact_i_ds, invoice_numbers, etc. + param_name = filter_info.get("param") + if op == "=": + api_params[param_name] = [value] + elif op == "in": + api_params[param_name] = value if isinstance(value, list) else [value] + else: + remaining_conditions.append([op, column, value]) + + + elif filter_type == "where": + # Build Xero WHERE clause + if op.lower() in ["in", "not in"]: + # Xero does not support IN directly, handle in memory + remaining_conditions.append([op, column, value]) + continue + + xero_op = self._map_operator_to_xero(op) + xero_field = filter_info.get("xero_field", column) + value_type = filter_info.get("value_type", "string") + xero_value = self._format_value_for_xero(value, value_type) + xero_where_clauses.append(f"{xero_field}{xero_op}{xero_value}") + + elif filter_type == "date": + # For date_from, date_to parameters + param_name = filter_info.get("param") + if op in ["=", ">=", ">"]: + api_params[param_name] = value + elif op in ["<=", "<"]: + # Some APIs use date_to for upper bound + date_to_param = filter_info.get("param_upper", None) + if date_to_param: + api_params[date_to_param] = value + else: + remaining_conditions.append([op, column, value]) + else: + remaining_conditions.append([op, column, value]) + + elif filter_type == "direct": + # For status, contact_id, etc. + param_name = filter_info.get("param") + if op == "=": + api_params[param_name] = value + else: + remaining_conditions.append([op, column, value]) + + # Combine WHERE clauses with AND + if xero_where_clauses: + api_params["where"] = " AND ".join(xero_where_clauses) + + return api_params, remaining_conditions + + @abstractmethod + def get_columns(self) -> List[str]: + """Get list of available columns""" + pass + + @abstractmethod + def select(self, query: ast.Select) -> pd.DataFrame: + """Execute SELECT query""" + pass \ No newline at end of file diff --git a/mindsdb/integrations/utilities/handlers/query_utilities/select_query_utilities.py b/mindsdb/integrations/utilities/handlers/query_utilities/select_query_utilities.py index cf68b6e5117..a4cb4c2480a 100644 --- a/mindsdb/integrations/utilities/handlers/query_utilities/select_query_utilities.py +++ b/mindsdb/integrations/utilities/handlers/query_utilities/select_query_utilities.py @@ -1,4 +1,4 @@ -from typing import Text, List, Dict, Tuple +from typing import Text, List, Dict, Tuple, Optional import pandas as pd from mindsdb_sql_parser import ast @@ -20,13 +20,18 @@ class SELECTQueryParser(BaseQueryParser): Name of the table to query. columns : List[Text] List of columns in the table. + use_default_limit : bool, optional + If True, applies a default limit of 1000 when no LIMIT clause is specified. + If False, returns None when no LIMIT clause is specified (fetch all records). + Default is True for backwards compatibility. """ - def __init__(self, query: ast.Select, table: Text, columns: List[Text]): + def __init__(self, query: ast.Select, table: Text, columns: List[Text], use_default_limit: bool = True): super().__init__(query) self.table = table self.columns = columns + self.use_default_limit = use_default_limit - def parse_query(self) -> Tuple[List[Text], List[List[Text]], Dict[Text, List[Text]], int]: + def parse_query(self) -> Tuple[List[Text], List[List[Text]], Dict[Text, List[Text]], Optional[int]]: """ Parses a SQL SELECT statement into its components: SELECT, WHERE, ORDER BY, LIMIT. """ @@ -62,14 +67,16 @@ def parse_order_by_clause(self) -> Dict[Text, List[Text]]: else: return [] - def parse_limit_clause(self) -> int: + def parse_limit_clause(self) -> Optional[int]: """ Parses the LIMIT clause of the query. + Returns the LIMIT value if specified, or a default of 1000 if use_default_limit is True, + or None if use_default_limit is False (fetch all records). """ if self.query.limit: result_limit = self.query.limit.value else: - result_limit = 20 + result_limit = 1000 if self.use_default_limit else None return result_limit diff --git a/mindsdb/integrations/utilities/sql_utils.py b/mindsdb/integrations/utilities/sql_utils.py index 5d76f4ae825..cf97e261e50 100644 --- a/mindsdb/integrations/utilities/sql_utils.py +++ b/mindsdb/integrations/utilities/sql_utils.py @@ -117,6 +117,11 @@ def _extract_comparison_conditions(node: ASTNode, **kwargs): if isinstance(arg1.args[0], ast.Identifier): arg1 = arg1.args[0] + # Handle TypeCast by unwrapping to get the underlying identifier + if isinstance(arg1, ast.TypeCast): + if isinstance(arg1.arg, ast.Identifier): + arg1 = arg1.arg + if not isinstance(arg1, ast.Identifier): # Only support [identifier] =//>=/<=/etc [constant] comparisons. raise NotImplementedError(f"Not implemented arg1: {arg1}") diff --git a/mindsdb/interfaces/database/projects.py b/mindsdb/interfaces/database/projects.py index 81ad63f5965..7b31e685370 100644 --- a/mindsdb/interfaces/database/projects.py +++ b/mindsdb/interfaces/database/projects.py @@ -166,6 +166,9 @@ def get_conditions_to_move(node): ): return [node] + # Return empty list for unsupported conditions (e.g., TypeCast, Function, etc.) + return [] + conditions = get_conditions_to_move(query.where) if conditions: