fix: cache invalidation for standard and custom objects, remove suppo…

…rt for schemas in standard objects (#2008)
supaglue-labs · Nov 30, 2023 · e3b21cf · e3b21cf
1 parent b460314
commit e3b21cf
Show file tree

Hide file tree

Showing 7 changed files with 29 additions and 51 deletions.
diff --git a/packages/core/destination_writers/postgres_impl.ts b/packages/core/destination_writers/postgres_impl.ts
@@ -30,9 +30,8 @@ import { keysOfSnakecasedSequenceWithTenant } from '../keys/engagement/sequence'
 import { keysOfSnakecasedSequenceStateWithTenant } from '../keys/engagement/sequence_state';
 import { keysOfSnakecasedSequenceStepWithTenant } from '../keys/engagement/sequence_step';
 import { keysOfSnakecasedEngagementUserWithTenant } from '../keys/engagement/user';
-import { logger, omit, schemasAndEntitiesEnabled } from '../lib';
+import { logger, omit } from '../lib';
 import type { WriteCommonObjectRecordsResult, WriteObjectRecordsResult } from './base';
-import { toTransformedPropertiesWithAdditionalFields } from './base';
 import {
   getSnakecasedKeysMapper,
   jsonStringifyWithoutNullChars,
@@ -272,8 +271,6 @@ DO UPDATE SET (${columnsToUpdateStr}) = (${excludedColumnsToUpdateStr})`);
     const table = objectType === 'standard' ? objectName : kCustomObject;
     const qualifiedTable = `"${schema}".${table}`;
     const childLogger = logger.child({ providerName, customerId });
-    // Write `supaglue_mapped_data` for existing Schemas and Entities users. We should write empty object otherwise.
-    const isSchemasOrEntitiesApplication = schemasAndEntitiesEnabled(applicationId);
 
     try {
       await setup();
@@ -287,9 +284,7 @@ DO UPDATE SET (${columnsToUpdateStr}) = (${excludedColumnsToUpdateStr})`);
         _supaglue_last_modified_at: record.metadata.lastModifiedAt,
         _supaglue_is_deleted: record.metadata.isDeleted,
         _supaglue_raw_data: record.rawData,
-        _supaglue_mapped_data: isSchemasOrEntitiesApplication
-          ? toTransformedPropertiesWithAdditionalFields(record.mappedProperties)
-          : {},
+        _supaglue_mapped_data: {},
         ...(objectType === 'custom' ? { _supaglue_object_name: objectName } : {}),
       };
       const columns = Object.keys(mappedRecord);
@@ -353,9 +348,6 @@ DO UPDATE SET (${columnsToUpdateStr}) = (${excludedColumnsToUpdateStr})`,
     const qualifiedTable = `"${schema}".${table}`;
     const tempTable = `"temp_${table}"`;
 
-    // Write `supaglue_mapped_data` for existing Schemas and Entities users. We should write empty object otherwise.
-    const isSchemasOrEntitiesApplication = schemasAndEntitiesEnabled(applicationId);
-
     try {
       await setup();
 
@@ -419,9 +411,7 @@ DO UPDATE SET (${columnsToUpdateStr}) = (${excludedColumnsToUpdateStr})`,
                 _supaglue_last_modified_at: record.lastModifiedAt,
                 _supaglue_is_deleted: record.isDeleted,
                 _supaglue_raw_data: record.rawData,
-                _supaglue_mapped_data: isSchemasOrEntitiesApplication
-                  ? toTransformedPropertiesWithAdditionalFields(record.mappedProperties)
-                  : {},
+                _supaglue_mapped_data: {},
                 _supaglue_object_name: objectName,
               };
 

diff --git a/packages/core/remotes/impl/hubspot/index.ts b/packages/core/remotes/impl/hubspot/index.ts
@@ -1264,7 +1264,7 @@ class HubSpotClient extends AbstractCrmRemoteClient implements MarketingAutomati
     return {
       id: response.data.id,
       objectName: object.name,
-      data: response.data.properties,
+      data: response.data,
       metadata: getMetadataFromRecord(response.data),
     };
   }

diff --git a/packages/core/services/entity_record_service.ts b/packages/core/services/entity_record_service.ts
@@ -182,13 +182,16 @@ function mapObjectToEntityFields(data: EntityRecordData, fieldMappingConfig: Fie
     case 'inherit_all_fields':
       return data;
     case 'defined': {
-      const coreFields = fieldMappingConfig.coreFieldMappings.reduce((acc, { schemaField, mappedField }) => {
-        const value = data[mappedField];
-        if (value) {
-          return { ...acc, [schemaField]: value };
-        }
-        return acc;
-      }, {} as Record<string, unknown>);
+      const coreFields = fieldMappingConfig.coreFieldMappings.reduce(
+        (acc, { schemaField, mappedField }) => {
+          const value = data[mappedField];
+          if (value) {
+            return { ...acc, [schemaField]: value };
+          }
+          return acc;
+        },
+        {} as Record<string, unknown>
+      );
 
       const additionalFields = fieldMappingConfig.additionalFieldMappings.reduce(
         (acc, { schemaField, mappedField }) => {

diff --git a/packages/core/services/object_record_service.ts b/packages/core/services/object_record_service.ts
@@ -150,29 +150,12 @@ export class ObjectRecordService {
     standardOrCustomObject: StandardOrCustomObject,
     recordId: string
   ): Promise<FullObjectRecord> {
-    let fieldMappingConfig = null;
-    if (standardOrCustomObject.type === 'standard') {
-      fieldMappingConfig = await this.#connectionService.getFieldMappingConfig(
-        connection.id,
-        'standard',
-        standardOrCustomObject.name
-      );
-    }
     const remoteClient = await this.#remoteService.getRemoteClient(connection.id);
-    const fields =
-      !fieldMappingConfig || fieldMappingConfig?.type === 'inherit_all_fields'
-        ? (await remoteClient.listProperties(standardOrCustomObject)).map((p) => p.id)
-        : [
-            ...new Set([
-              ...fieldMappingConfig.coreFieldMappings.map((m) => m.mappedField),
-              ...fieldMappingConfig.additionalFieldMappings.map((m) => m.mappedField),
-            ]),
-          ];
+    const fields = (await remoteClient.listProperties(standardOrCustomObject)).map((p) => p.id);
     const record = await remoteClient.getObjectRecord(standardOrCustomObject, recordId, fields);
     return {
       id: recordId,
       objectName: standardOrCustomObject.name,
-      mappedProperties: fieldMappingConfig ? mapObjectToSchema(record.data, fieldMappingConfig) : record.data,
       rawData: record.data,
       metadata: record.metadata,
     };

diff --git a/packages/sync-workflows/activities/sync_object_records.ts b/packages/sync-workflows/activities/sync_object_records.ts
@@ -56,19 +56,16 @@ export function createSyncObjectRecords(
     if (connection.providerName === 'hubspot') {
       switch (objectType) {
         case 'common':
-          associationsToFetch = syncConfig.config.commonObjects?.find(
-            (config) => config.object === object
-          )?.associationsToFetch;
+          associationsToFetch = syncConfig.config.commonObjects?.find((config) => config.object === object)
+            ?.associationsToFetch;
           break;
         case 'standard':
-          associationsToFetch = syncConfig.config.standardObjects?.find(
-            (config) => config.object === object
-          )?.associationsToFetch;
+          associationsToFetch = syncConfig.config.standardObjects?.find((config) => config.object === object)
+            ?.associationsToFetch;
           break;
         case 'custom':
-          associationsToFetch = syncConfig.config.customObjects?.find(
-            (config) => config.object === object
-          )?.associationsToFetch;
+          associationsToFetch = syncConfig.config.customObjects?.find((config) => config.object === object)
+            ?.associationsToFetch;
           break;
         default:
           break;

diff --git a/packages/types/entity_record.ts b/packages/types/entity_record.ts
@@ -1,4 +1,9 @@
-import type { BaseFullRecord, ObjectRecordUpsertData, PropertiesWithAdditionalFields } from './object_record';
+import type {
+  BaseFullRecord,
+  ObjectRecordData,
+  ObjectRecordUpsertData,
+  PropertiesWithAdditionalFields,
+} from './object_record';
 
 /**
  * @deprecated
@@ -40,4 +45,5 @@ export type EntityRecord = {
  */
 export type FullEntityRecord = BaseFullRecord & {
   entity: SimpleEntity;
+  mappedProperties: ObjectRecordData;
 };
diff --git a/packages/types/object_record.ts b/packages/types/object_record.ts
@@ -10,7 +10,7 @@ export type ListedObjectRecordRawDataOnly<D = Record<string, unknown>> = {
 
 export type ListedObjectRecord<
   D extends Record<string, unknown> = Record<string, unknown>,
-  P extends Record<string, unknown> = Record<string, unknown>
+  P extends Record<string, unknown> = Record<string, unknown>,
 > = ListedObjectRecordRawDataOnly<D> & {
   // rawProperties should only have properties
   // - in salesforce, this is easy
@@ -72,7 +72,6 @@ export type ObjectMetadata = {
 
 export type BaseFullRecord = {
   id: string;
-  mappedProperties: ObjectRecordData;
   rawData: Record<string, unknown>;
   metadata: ObjectMetadata;
 };