Merge pull request #21 from devoteamgcloud/dev-hugo

V2 Release

Author: Hugo Rialan

.gitignore

@@ -1,3 +1,5 @@
 
 .df-credentials.json
 node_modules/
+
+.DS_Store

README.md

@@ -54,8 +54,10 @@ const commonAssertionsResult = commonAssertions({
     "disabledInEnvs": ["dv"] // Check match with 'dataform.projectConfig.vars.env' value
   },
   rowConditions: {
-    "your_table": {
-      "id_not_null": "id IS NOT NULL",
+    "your_schema": {
+      "your_table": {
+        "id_not_null": "id IS NOT NULL",
+      }
     }
   }
 });

dataform.json

@@ -5,6 +5,7 @@
     "defaultDatabase": "sandbox-hrialan",
     "defaultLocation": "EU",
     "vars":{
-        "env":"dv"
+        "env":"dv",
+        "example":""
     }
 }

definitions/example.js

@@ -12,13 +12,15 @@ const commonAssertionsResult = commonAssertions({
     // "disabledInEnvs": ["dv", "qa"]
   },
   config: {
-    "first_table": {
-      "where": "updated_date >= CURRENT_DATE() - 7"
-    },
+    "dataform": {
+      "first_table": {
+        "where": "updated_date >= CURRENT_DATE() - 7"
+      }
+    }
   },
   rowConditions: {
     // Format: "schema": { "table": { "conditionName": "conditionQuery", ... }, ... }
-    "dataform": {
+    ["dataform" + dataform.projectConfig.vars.example]: {
       "first_table": {
         "id_not_null": "id IS NOT NULL",
         "id_strict_positive": "id > 0"
@@ -45,10 +47,7 @@ const commonAssertionsResult = commonAssertions({
         "timeZone": "America/Los_Angeles"
       },
       "second_table": {
-        // If timeUnit is not DAY, WEEK, MONTH, QUARTER, or YEAR, dateColumn should be a TIMESTAMP.
-        // Check here for valid Date time units: https://cloud.google.com/bigquery/docs/reference/standard-sql/date_functions#date_diff
-        // Check here for valid Timestamp time units: https://cloud.google.com/bigquery/docs/reference/standard-sql/timestamp_functions#timestamp_diff
-        "dateColumn": "TIMESTAMP(updated_date)",
+        "dateColumn": "updated_date",
         "timeUnit": "HOUR",
         "delayCondition": 3,
         "timeZone": "-08"

includes/data_completeness_assertions.js

@@ -4,7 +4,7 @@
  * This file contains a function to create data completeness assertions for specific tables and columns in a database.
  * The assertions are used to check if the percentage of null values in each specified column exceeds an allowed limit.
  * The conditions for data completeness checks are defined in an object format:
- * { tableName: { columnName: allowedPercentageNull, ... }, ... }
+ * schemaName : { tableName: { columnName: allowedPercentageNull, ... }, ... }
  *
  * The function `createDataCompletenessAssertion` takes in global parameters, a table name, and column conditions to create these assertions.
  */
@@ -59,7 +59,7 @@ module.exports = (globalParams, config, dataCompletenessConditions) => {
     const tableNames = dataCompletenessConditions[schemaName];
     for (let tableName in tableNames) {
       const columnConditions = tableNames[tableName];
-      const filter = config[tableName]?.where ?? true;
+      const filter = config[schemaName][tableName]?.where ?? true;
       createDataCompletenessAssertion(globalParams, schemaName, tableName, filter, columnConditions);
     }
   }

includes/data_freshness_assertions.js

@@ -20,7 +20,7 @@
 
 const assertions = [];
 
-const createDataFreshnessAssertion = (globalParams, schemaName, tableName, filter, delayCondition, timeUnit, dateColumn) => {
+const createDataFreshnessAssertion = (globalParams, schemaName, tableName, filter, delayCondition, timeUnit, dateColumn, timeZone = "UTC") => {
   const assertion = assert(`assert_freshness_${schemaName}_${tableName}`)
     .database(globalParams.database)
     .schema(globalParams.schema)
@@ -40,7 +40,7 @@ const createDataFreshnessAssertion = (globalParams, schemaName, tableName, filte
                         SELECT
                           ${["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"].includes(timeUnit)
                               ? `DATE_DIFF(CURRENT_DATE("${timeZone}"), MAX(${dateColumn}), ${timeUnit})`
-                              : `TIMESTAMP_DIFF(CURRENT_TIMESTAMP(), MAX(${dateColumn}), ${timeUnit})`} AS delay
+                              : `TIMESTAMP_DIFF(CURRENT_TIMESTAMP(), TIMESTAMP(MAX(${dateColumn}),"${timeZone}"), ${timeUnit})`} AS delay
                         FROM
                             filtering
                     )
@@ -59,7 +59,6 @@ const createDataFreshnessAssertion = (globalParams, schemaName, tableName, filte
   assertions.push(assertion);
 };
 
-
 module.exports = (globalParams, config, freshnessConditions) => {
   // Loop through freshnessConditions to create assertions.
   for (let schemaName in freshnessConditions) {
@@ -68,10 +67,11 @@ module.exports = (globalParams, config, freshnessConditions) => {
       const {
         delayCondition,
         timeUnit,
-        dateColumn
+        dateColumn,
+        timeZone
       } = tableNames[tableName];
-      const filter = config[tableName]?.where ?? true;
-      createDataFreshnessAssertion(globalParams, schemaName, tableName, delayCondition, timeUnit, dateColumn);
+      const filter = config[schemaName][tableName]?.where ?? true;
+      createDataFreshnessAssertion(globalParams, schemaName, tableName, filter, delayCondition, timeUnit, dateColumn, timeZone);
     }
   }
 

includes/referential_integrity_assertions.js

@@ -4,7 +4,7 @@
  * This file contains a function to create referential integrity assertions for specific tables in a database.
  * The assertions are used to check if the foreign key relationships are maintained between tables.
  * The conditions for referential integrity checks are defined in an object format:
- * { parentTable: [{ parentKey, childTable, childKey }, ...], ... }
+ * schemaName : { parentTable: [{ parentKey, childTable, childKey }, ...], ... }
  *
  * The function `createReferentialIntegrityAssertions` takes in global parameters and the referential integrity conditions.
  */
@@ -68,7 +68,7 @@ module.exports = (globalParams, config, referentialIntegrityConditions) => {
     const parentTables = referentialIntegrityConditions[parentSchema];
     for (let parentTable in parentTables) {
       const relationships = parentTables[parentTable];
-      const parentFilter = config[parentTable]?.where ?? true;
+      const parentFilter = config[parentSchema][parentTable]?.where ?? true;
 
       relationships.forEach(({
         parentKey,

includes/row_condition_assertions.js

@@ -4,7 +4,7 @@
  * This file contains a function to create row condition assertions for specific tables in a database.
  * The assertions are used to check if the rows in each specified table meet a certain condition.
  * The conditions for row checks are defined in an object format:
- * { tableName: { conditionName: conditionQuery, ... }, ... }
+ * schemaName : { tableName: { conditionName: conditionQuery, ... }, ... }
  *
  * The function `createRowConditionAssertion` takes in global parameters, a table name, a condition name, and a condition query to create these assertions.
  */
@@ -56,7 +56,7 @@ module.exports = (globalParams, config, rowConditions) => {
     for (let tableName in tableNames) {
       for (let conditionName in tableNames[tableName]) {
         const conditionQuery = tableNames[tableName][conditionName];
-        const filter = config[tableName]?.where ?? true;
+        const filter = config[schemaName][tableName]?.where ?? true;
         createRowConditionAssertion(
           globalParams,
           schemaName,

includes/unique_key_assertions.js

@@ -4,7 +4,7 @@
  * This file contains a function to create unique key assertions for specific tables in a database.
  * The assertions are used to check if the combination of values in specified columns forms a unique key for each row in the table.
  * The conditions for unique key checks are defined in an object format:
- * { tableName: [column1, column2, ...], ... }
+ * schemaName : { tableName: [column1, column2, ...], ... }
  *
  * The function `createUniqueKeyAssertion` takes in global parameters, a table name, and an array of column names to create these assertions.
  */
@@ -57,7 +57,7 @@ module.exports = (globalParams, config, uniqueKeyConditions) => {
     const tableNames = uniqueKeyConditions[schemaName];
     for (let tableName in tableNames) {
       const columns = tableNames[tableName];
-      const filter = config[tableName]?.where ?? true;
+      const filter = config[schemaName][tableName]?.where ?? true;
       createUniqueKeyAssertion(globalParams, schemaName, tableName, filter, columns);
     }
   }

index.js

@@ -45,7 +45,7 @@ module.exports = ({
     const uniqueKeyAssertionsResult = unique_key_assertions(globalAssertionsParams, config, uniqueKeyConditions);
     const dataFreshnessAssertionsResult = data_freshness_assertions(globalAssertionsParams, config, dataFreshnessConditions);
     const dataCompletenessAssertionsResult = data_completeness_assertions(globalAssertionsParams, config, dataCompletenessConditions);
-    const referentialIntegrityAssertionsResult = referential_integrity_assertions(globalAssertionsParams, config, referentialIntegrityConditions); // New assertion
+    const referentialIntegrityAssertionsResult = referential_integrity_assertions(globalAssertionsParams, config, referentialIntegrityConditions);
 
     return {
         rowConditionAssertions: rowConditionAssertionsResult,