Initial commit

Hugo Rialan · Hugo Rialan · commit b4e4111cc09f · 2024-03-06T17:48:56.000+01:00
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1 @@
+* @hrialan
diff --git a/.github/workflows/publish-package.yaml b/.github/workflows/publish-package.yaml
@@ -0,0 +1,21 @@
+name: Publish package to GitHub Packages
+on:
+  release:
+    types: [published]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20.x'
+          registry-url: 'https://npm.pkg.github.com'
+          scope: '@hrialan'
+      - run: npm ci
+      - run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+
+.df-credentials.json
+node_modules/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,18 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+    - id: trailing-whitespace
+    - id: end-of-file-fixer
+    - id: fix-byte-order-marker
+    - id: check-merge-conflict
+    - id: check-json
+    - id: check-yaml
+    - id: check-added-large-files
+    - id: no-commit-to-branch
+      args: [--branch, main]
+- repo: https://github.com/devoteamgcloud/pre-commit-dataform
+  rev: v1.0.2
+  hooks:
+    - id: dataform_format
+    
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Devoteam G Cloud
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
@@ -0,0 +1,67 @@
+# Dataform Assertions
+
+This Dataform package provides a set of assertions for testing the data in your warehouse. It includes assertions for data freshness, unique keys, row conditions, and data completeness.
+
+Contributions are welcome! If you have an idea for a new assertion, please open an issue or submit a pull request.
+
+## Contents
+
+- [Installation](#installation)
+- [Usage](#usage)
+- [Available assertions](#available-assertions)
+- [License](#license)
+
+## Installation
+
+Follow the instructions in the [Dataform documentation](https://cloud.google.com/dataform/docs/install-package) to install this package. Here is a quick summary:
+
+1. In the `package.json`file dependencies, add the following line:
+```json
+"dataform-assertions": "https://github.com/devoteamgcloud/dataform-assertions/archive/refs/tags/[RELEASE_VERSION].tar.gz"
+```
+2. Click on `Install packages` in the Dataform web UI or use the `dataform install` CLI command in the terminal.
+3. You are ready to go!
+
+## Usage
+
+Create a js file in the `/definitions` folder of your Dataform project and add the following code with the desired parameters:
+
+```javascript
+const commonAssertions = require("dataform-assertions");
+
+const commonAssertionsResult = commonAssertions({
+  globalAssertionsParams: {
+    // If not provided, the default values will be used
+    "database": "your-database",
+    "schema": "your-schema",
+    "location": "your-location",
+    "tags": ["your-tags"],
+    "disabledInEnvs": ["your-disabled-environments"]
+  },
+  rowConditions: {
+    "your-table": {
+      "your-condition": "your-SQL-condition"
+    }
+  }
+});
+```
+
+You can find a more complete example in [`definitions/example.js`](./definitions/example.js).
+
+
+## Available assertions
+
+This package includes the following types of assertions:
+
+- **Row conditions**: Check if the rows in a table satisfy a given SQL condition.
+- **Unique key conditions**: Check if a given primary key (can be a set of columns) is not duplicated in a table.
+- **Data freshness conditions**: Check if the data in a table is fresh enough given some conditions.
+- **Data completeness conditions**: Check if the data in a column have less than a given percentage of null values.
+
+## Supported warehouses
+
+This package has been tested with BigQuery. It has not been tested with other warehouses.
+
+# License
+
+This project is licensed under the MIT License. See the LICENSE file for details.
diff --git a/dataform.json b/dataform.json
@@ -0,0 +1,10 @@
+{
+    "warehouse": "bigquery",
+    "defaultSchema": "dataform",
+    "assertionSchema": "dataform_assertions",
+    "defaultDatabase": "sandbox-hrialan",
+    "defaultLocation": "EU",
+    "vars":{
+        "env":"dv"
+    }
+}
diff --git a/definitions/example.js b/definitions/example.js
@@ -0,0 +1,81 @@
+const commonAssertions = require("../index");
+
+const commonAssertionsResult = commonAssertions({
+  globalAssertionsParams: {
+    "database": "sandbox-hrialan",
+    "schema": "assertions_" + dataform.projectConfig.vars.env,
+    "location": "EU",
+    "tags": ["assertions"],
+    // Sometimes data quality is not good in some environments,
+    // so we can disable the assertions in those environments.
+    // "disabledInEnvs": ["dv", "qa"]
+  },
+  rowConditions: {
+    "first_table": {
+      "id_not_null": "id IS NOT NULL",
+      "id_strict_positive": "id > 0"
+    },
+    "second_table": {
+      "id_not_null": "id IS NOT NULL"
+    }
+  },
+  uniqueKeyConditions: {
+    "first_table": ["id"],
+    "second_table": ["id"]
+  },
+  dataFreshnessConditions: {
+    "first_table": {
+      "dateColumn": "updated_date",
+      "timeUnit": "DAY",
+      "delayCondition": 1,
+    },
+    "second_table": {
+      "dateColumn": "updated_date",
+      "timeUnit": "MONTH",
+      "delayCondition": 3,
+    }
+  },
+  dataCompletenessConditions: {
+    "first_table": {
+      // Format: "column": allowedPercentageNull
+      "updated_date": 1, // 1% of null values allowed in the updated_date column
+      "id": 20
+    },
+    "second_table": {
+      "id": 30
+    }
+  }
+});
+
+/*
+ * ASSERTIONS AUDIT TABLE EXAMPLE
+ * The following code snippet is used to publish the results of the created assertions in a table for audit purposes.
+ * The result is a table with the following columns:
+ * | assertion_name | assertion_type |
+ * |----------------|----------------|
+ * | id_not_null    | row_condition  |
+ * |       ...      |       ...      |
+ */
+
+let selectClauses = [];
+
+for (const key in commonAssertionsResult) {
+  if (commonAssertionsResult.hasOwnProperty(key)) {
+    const commonAssertionsResultForKey = commonAssertionsResult[key];
+    if (commonAssertionsResultForKey.length > 0) {
+      const selectClause = commonAssertionsResultForKey.map(assertion => {
+        return `SELECT "${assertion.proto.target.name}" AS assertion_name, '${key}' AS assertion_type`;
+      }).join("\n UNION ALL \n");
+
+      selectClauses.push(selectClause);
+    }
+  }
+}
+
+const sqlQuery = selectClauses.join("\n UNION ALL \n");
+
+publish("assertions_audit", {
+  type: "table"
+}).query(
+  (ctx) => sqlQuery
+);
diff --git a/definitions/first_table.sqlx b/definitions/first_table.sqlx
@@ -0,0 +1,11 @@
+config {
+  type: "table"
+}
+
+SELECT
+  1 AS id,
+  CURRENT_DATE() AS updated_date
+UNION ALL
+SELECT
+  2 AS id,
+  CURRENT_DATE() AS updated_date
diff --git a/definitions/second_table.sqlx b/definitions/second_table.sqlx
@@ -0,0 +1,27 @@
+config {
+  type: "table"
+}
+
+SELECT
+  NULL AS id,
+  DATE(1970, 1, 1) AS updated_date
+UNION ALL
+SELECT
+  2 AS id,
+  DATE(1970, 1, 1) AS updated_date
+UNION ALL
+SELECT
+  NULL AS id,
+  DATE(1970, 1, 1) AS updated_date
+UNION ALL
+SELECT
+  NULL AS id,
+  DATE(1970, 1, 1) AS updated_date
+UNION ALL
+SELECT
+  3 AS id,
+  NULL AS updated_date
+UNION ALL
+SELECT
+  NULL AS id,
+  DATE(1970, 1, 1) AS updated_date
diff --git a/includes/data_completeness_assertions.js b/includes/data_completeness_assertions.js
@@ -0,0 +1,52 @@
+/**
+ * data_completeness_assertions.js
+ *
+ * This file contains a function to create data completeness assertions for specific tables and columns in a database.
+ * The assertions are used to check if the percentage of null values in each specified column exceeds an allowed limit.
+ * The conditions for data completeness checks are defined in an object format:
+ * { tableName: { columnName: allowedPercentageNull, ... }, ... }
+ *
+ * The function `createDataCompletenessAssertion` takes in global parameters, a table name, and column conditions to create these assertions.
+ */
+
+/**
+ * @param {Object} globalParams - See index.js for details.
+ * @param {string} tableName - The name of the table to check for data completeness.
+ * @param {Object} columnConditions - An object mapping column names to their allowed percentage of null values. If a value is an object, it should have an `allowedPercentageNull` property.
+ */
+
+const assertions = [];
+
+const createDataCompletenessAssertion = (globalParams, tableName, columnConditions) => {
+
+  for (let columnName in columnConditions) {
+    const allowedPercentageNull = columnConditions[columnName];
+
+    const assertion = assert(`assert_data_completeness_${tableName}_${columnName}`)
+      .database(globalParams.database)
+      .schema(globalParams.schema)
+      .description(`Check data completeness for ${tableName}.${columnName}, allowed percentage of null values: ${allowedPercentageNull}`)
+      .tags("assert-data-completeness")
+      .query(ctx => `SELECT COUNT(*) AS total_rows,
+                        SUM(CASE WHEN ${columnName} IS NULL THEN 1 ELSE 0 END) AS null_count
+                        FROM ${ctx.ref(tableName)}
+                        HAVING SAFE_DIVIDE(null_count, total_rows) > ${allowedPercentageNull / 100} AND null_count > 0 AND total_rows > 0`);
+
+    (globalParams.tags && globalParams.tags.forEach((tag) => assertion.tags(tag)));
+
+    (globalParams.disabledInEnvs && globalParams.disabledInEnvs.includes(dataform.projectConfig.vars.env)) && assertion.disabled();
+
+    assertions.push(assertion);
+  }
+
+};
+
+module.exports = (globalParams, dataCompletenessConditions) => {
+  // Loop through dataCompletenessConditions to create data completeness check assertions.
+  for (let tableName in dataCompletenessConditions) {
+    const columnConditions = dataCompletenessConditions[tableName];
+    createDataCompletenessAssertion(globalParams, tableName, columnConditions);
+  }
+
+  return assertions;
+};
diff --git a/includes/data_freshness_assertions.js b/includes/data_freshness_assertions.js
@@ -0,0 +1,62 @@
+/**
+ * data_freshness_assertions.js
+ *
+ * This file contains a function to create data freshness assertions for specific tables in a database.
+ * The assertions are used to check if the data in each specified table is fresh, based on a specified delay condition and time unit.
+ *
+ * The function `createDataFreshnessAssertion` takes in global parameters, a table name, a delay condition, a time unit, and a date column to create these assertions.
+ */
+
+/**
+ * @param {Object} globalParams - See index.js for details.
+ * @param {string} tableName - The name of the table to check for data freshness.
+ * @param {number} delayCondition - The maximum allowed delay (in units specified by `timeUnit`) for the data to be considered fresh.
+ * @param {string} timeUnit - The unit of time to use for the delay condition. This should be a string that is valid in a SQL `DATE_DIFF` function, such as 'DAY', 'HOUR', etc.
+ * @param {string} dateColumn - The name of the date column to check for data freshness.
+ */
+
+const assertions = [];
+
+const createDataFreshnessAssertion = (globalParams, tableName, delayCondition, timeUnit, dateColumn) => {
+  const assertion = assert(`assert_freshness_${tableName}`)
+    .database(globalParams.database)
+    .schema(globalParams.schema)
+    .description(`Assert that data in ${tableName} is fresh with a delay less than ${delayCondition} ${timeUnit}`)
+    .tags("assert-data-freshness")
+    .query(ctx => `
+                WITH
+                    freshness AS (
+                        SELECT
+                            DATE_DIFF(CURRENT_DATE(), MAX(${dateColumn}), ${timeUnit}) AS delay
+                        FROM
+                            ${ctx.ref(tableName)}
+                    )
+                SELECT
+                    *
+                FROM
+                    freshness
+                WHERE
+                    delay > ${delayCondition}
+            `);
+
+  (globalParams.tags && globalParams.tags.forEach((tag) => assertion.tags(tag)));
+
+  (globalParams.disabledInEnvs && globalParams.disabledInEnvs.includes(dataform.projectConfig.vars.env)) && assertion.disabled();
+
+  assertions.push(assertion);
+};
+
+module.exports = (globalParams, freshnessConditions) => {
+
+  // Loop through freshnessConditions to create assertions.
+  for (let tableName in freshnessConditions) {
+    const {
+      delayCondition,
+      timeUnit,
+      dateColumn
+    } = freshnessConditions[tableName];
+    createDataFreshnessAssertion(globalParams, tableName, delayCondition, timeUnit, dateColumn);
+  }
+
+  return assertions;
+};
diff --git a/includes/row_condition_assertions.js b/includes/row_condition_assertions.js
diff --git a/includes/unique_key_assertions.js b/includes/unique_key_assertions.js
diff --git a/index.js b/index.js
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+`
	`2`	`+.df-credentials.json`
	`3`	`+node_modules/`