Sync monorepo state at 4721a746083d39904866cbe387288c8f6f5566a6

Author: uc-build-user

CHANGELOG.md

@@ -1,7 +1,13 @@
 # Changelog
 
+## 1.3.0 - 28-06-2024
+
+- Retry getting access token on EOF type network errors which occur when connection is lost.
+- Breaking change: RetryNetworkErrors option for jsonclient now takes a boolean, and the option is on by default
+
 ## 1.2.0 - 09-04-2024
 
+- Update userstore sample to exercise partial update columns
 - Add methods for creating, retrieving, updating, and deleting ColumnDataTypes
 - Add DataType field to Column that refers to a ColumnDataType
 - Add InputDataType and OutputDataType fields to Transformer that refer to ColumnDataTypes

authz/cache_name_provider.go

@@ -11,26 +11,28 @@ import (
 
 const (
 	// CachePrefix is the prefix for all keys in authz cache
-	CachePrefix                 = "authz"
-	objTypePrefix               = "OBJTYPE"      // Primary key for object type
-	objTypeCollectionKeyString  = "OBJTYPE_COL"  // Global collection for object type
-	edgeTypePrefix              = "EDGETYPE"     // Primary key for edge type
-	edgeTypeCollectionKeyString = "EDGETYPE_COL" // Global collection for edge type
-	objPrefix                   = "OBJ"          // Primary key for object
-	objCollectionKeyString      = "OBJ_COL"      // Global collection for object
-	objEdgeCollection           = "OBJEDGES"     // Per object collection of all in/out edges
-	perObjectEdgesPrefix        = "E"            // Per object collection of source/target edges
-	perObjectPathPrefix         = "P"            // Per object collection containing path for a particular source/target/attribute
-	edgePrefix                  = "EDGE"         // Primary key for edge
-	edgeCollectionKeyString     = "EDGE_COL"     // Global collection for edge
-	orgPrefix                   = "ORG"          // Primary key for organization
-	orgCollectionKeyString      = "ORG_COL"      // Global collection for organizations
-	dependencyPrefix            = "DEP"          // Shared dependency key prefix among all items
-	isModifiedPrefix            = "MOD"          // Shared is modified key prefix among all items
+	CachePrefix                     = "authz"
+	objTypePrefix                   = "OBJTYPE"      // Primary key for object type
+	objTypeCollectionKeyString      = "OBJTYPE_COL"  // Global collection for object type
+	edgeTypePrefix                  = "EDGETYPE"     // Primary key for edge type
+	edgeTypeCollectionKeyString     = "EDGETYPE_COL" // Global collection for edge type
+	objPrefix                       = "OBJ"          // Primary key for object
+	objCollectionKeyString          = "OBJ_COL"      // Global collection for object
+	objEdgeCollection               = "OBJEDGES"     // Per object collection of all in/out edges
+	perObjectEdgesPrefix            = "E"            // Per object collection of source/target edges
+	perObjectPathPrefix             = "P"            // Per object collection containing path for a particular source/target/attribute
+	edgePrefix                      = "EDGE"         // Primary key for edge
+	edgeCollectionKeyString         = "EDGE_COL"     // Global collection for edge
+	edgeCollectionPagesPrefixString = "PAGES"        // Pages making up global collection of edges
+	orgPrefix                       = "ORG"          // Primary key for organization
+	orgCollectionKeyString          = "ORG_COL"      // Global collection for organizations
+	dependencyPrefix                = "DEP"          // Shared dependency key prefix among all items
+	isModifiedPrefix                = "MOD"          // Shared is modified key prefix among all items
 )
 
 // CacheNameProvider is the base implementation of the CacheNameProvider interface
 type CacheNameProvider struct {
+	cache.NoRateLimitKeyNameProvider
 	basePrefix string // Base prefix for all keys TenantID_OrgID
 }
 
@@ -73,6 +75,8 @@ const (
 	DependencyKeyID = "DependencyKeyID"
 	// IsModifiedKeyID is the key value indicating change in last TTL
 	IsModifiedKeyID = "IsModifiedKeyID"
+	// IsModifiedCollectionKeyID is the key value indicating change for global colleciton in last TTL
+	IsModifiedCollectionKeyID = "IsModifiedCollectionKeyID"
 	// ObjectTypeCollectionKeyID is the key for global collection of object types
 	ObjectTypeCollectionKeyID = "ObjTypeCollectionKeyID"
 	// EdgeTypeCollectionKeyID is the key for global collection of edge types
@@ -81,6 +85,10 @@ const (
 	ObjectCollectionKeyID = "ObjCollectionKeyID"
 	// EdgeCollectionKeyID is the key for global collection of edges
 	EdgeCollectionKeyID = "EdgeCollectionKeyID"
+	// EdgeCollectionPagesKeyID is the key for pages making up global collection of edges
+	EdgeCollectionPagesKeyID = "EdgeCollectionPagesKeyID"
+	// EdgeCollectionPageKeyID is the key for each individual page in the global collection of edges
+	EdgeCollectionPageKeyID = "EdgeCollectionPageKeyID"
 	// OrganizationCollectionKeyID is the key for global collection of organizations
 	OrganizationCollectionKeyID = "OrgCollectionKeyID"
 	// AttributePathObjToObjID is the primary key for attribute path
@@ -109,10 +117,13 @@ func (c *CacheNameProvider) GetAllKeyIDs() []string {
 		EdgesObjToObjID,
 		DependencyKeyID,
 		IsModifiedKeyID,
+		IsModifiedCollectionKeyID,
 		ObjectTypeCollectionKeyID,
 		EdgeTypeCollectionKeyID,
 		ObjectCollectionKeyID,
 		EdgeCollectionKeyID,
+		EdgeCollectionPagesKeyID,
+		EdgeCollectionPageKeyID,
 		OrganizationCollectionKeyID,
 		AttributePathObjToObjID,
 	}
@@ -165,6 +176,10 @@ func (c *CacheNameProvider) GetKeyName(id cache.KeyNameID, components []string)
 		return c.objCollectionKey()
 	case EdgeCollectionKeyID:
 		return c.edgeCollectionKey()
+	case EdgeCollectionPagesKeyID:
+		return c.edgeCollectionPagesKey()
+	case EdgeCollectionPageKeyID:
+		return c.edgeCollectionPageKey(components[0], components[1])
 	case OrganizationCollectionKeyID:
 		return c.orgCollectionKey()
 	case ObjEdgesKeyID:
@@ -173,6 +188,8 @@ func (c *CacheNameProvider) GetKeyName(id cache.KeyNameID, components []string)
 		return c.dependencyKey(components[0])
 	case IsModifiedKeyID:
 		return c.isModifiedKey(components[0])
+	case IsModifiedCollectionKeyID:
+		return c.isModifiedCollectionKey(components[0])
 	case EdgeFullKeyID:
 		return c.edgeFullKeyNameFromIDs(components[0], components[1], components[2])
 	case AttributePathObjToObjID:
@@ -271,6 +288,20 @@ func (c *CacheNameProvider) edgeCollectionKey() cache.Key {
 	return cache.Key(fmt.Sprintf("%v_%v", c.basePrefix, edgeCollectionKeyString))
 }
 
+// edgeCollectionModifiedKey returns key name for edge collection
+func (c *CacheNameProvider) isModifiedCollectionKey(colKey string) cache.Key {
+	return cache.Key(fmt.Sprintf("%v_%v", colKey, isModifiedPrefix))
+}
+
+// edgeCollectionKey returns key name for edge collection
+func (c *CacheNameProvider) edgeCollectionPagesKey() cache.Key {
+	return cache.Key(fmt.Sprintf("%v_%v_%v", c.basePrefix, edgeCollectionKeyString, edgeCollectionPagesPrefixString))
+}
+
+func (c *CacheNameProvider) edgeCollectionPageKey(cursor string, limit string) cache.Key {
+	return cache.Key(fmt.Sprintf("%v_%v_%v_%v", c.basePrefix, edgeCollectionKeyString, cursor, limit))
+}
+
 // orgCollectionKey returns key name for edge collection
 func (c *CacheNameProvider) orgCollectionKey() cache.Key {
 	return cache.Key(fmt.Sprintf("%v_%v", c.basePrefix, orgCollectionKeyString))
@@ -291,6 +322,11 @@ func (ot ObjectType) GetGlobalCollectionKey(c cache.KeyNameProvider) cache.Key {
 	return c.GetKeyNameStatic(ObjectTypeCollectionKeyID)
 }
 
+// GetGlobalCollectionPagesKey returns the global collection key name for object type
+func (ot ObjectType) GetGlobalCollectionPagesKey(c cache.KeyNameProvider) cache.Key {
+	return "" // Unused since there is no pagination for object types
+}
+
 // GetSecondaryKeys returns the secondary cache key names for object type
 func (ot ObjectType) GetSecondaryKeys(c cache.KeyNameProvider) []cache.Key {
 	return []cache.Key{c.GetKeyNameWithString(ObjectTypeNameKeyID, ot.TypeName)}
@@ -311,6 +347,11 @@ func (ot ObjectType) GetIsModifiedKey(c cache.KeyNameProvider) cache.Key {
 	return c.GetKeyNameWithID(IsModifiedKeyID, ot.ID)
 }
 
+// GetIsModifiedCollectionKey returns the IsModifiedCollectionKeyID key name for object type
+func (ot ObjectType) GetIsModifiedCollectionKey(c cache.KeyNameProvider) cache.Key {
+	return "" // Unused until we turn one page caching
+}
+
 // GetDependencyKeys returns the list of keys for object type dependencies
 func (ot ObjectType) GetDependencyKeys(c cache.KeyNameProvider) []cache.Key {
 	return []cache.Key{} // ObjectTypes don't depend on anything
@@ -331,6 +372,11 @@ func (et EdgeType) GetGlobalCollectionKey(c cache.KeyNameProvider) cache.Key {
 	return c.GetKeyNameStatic(EdgeTypeCollectionKeyID)
 }
 
+// GetGlobalCollectionPagesKey returns the global collection key name for edge type
+func (et EdgeType) GetGlobalCollectionPagesKey(c cache.KeyNameProvider) cache.Key {
+	return "" // Unused since there is no pagination for edge types
+}
+
 // GetPerItemCollectionKey returns the per item collection key name for edge type
 func (et EdgeType) GetPerItemCollectionKey(c cache.KeyNameProvider) cache.Key {
 	return "" // Unused since there nothing stored per edge type, could store edges of this type in the future
@@ -351,6 +397,11 @@ func (et EdgeType) GetIsModifiedKey(c cache.KeyNameProvider) cache.Key {
 	return c.GetKeyNameWithID(IsModifiedKeyID, et.ID)
 }
 
+// GetIsModifiedCollectionKey returns the IsModifiedCollectionKeyID key name for edge type
+func (et EdgeType) GetIsModifiedCollectionKey(c cache.KeyNameProvider) cache.Key {
+	return "" // Unused until we turn one page caching
+}
+
 // GetDependencyKeys returns the list of keys for edge type dependencies
 func (et EdgeType) GetDependencyKeys(c cache.KeyNameProvider) []cache.Key {
 	// EdgeTypes depend on source/target object types but we don't store that dependency because we currently flush the whole cache on object type delete
@@ -380,6 +431,11 @@ func (o Object) GetGlobalCollectionKey(c cache.KeyNameProvider) cache.Key {
 	return c.GetKeyNameStatic(ObjectCollectionKeyID)
 }
 
+// GetGlobalCollectionPagesKey returns the global collection key name for objects
+func (o Object) GetGlobalCollectionPagesKey(c cache.KeyNameProvider) cache.Key {
+	return "" // Unused since there is no pagination for objects
+}
+
 // GetPerItemCollectionKey returns the per item collection key name for object
 func (o Object) GetPerItemCollectionKey(c cache.KeyNameProvider) cache.Key {
 	return c.GetKeyNameWithID(ObjEdgesKeyID, o.ID)
@@ -395,6 +451,11 @@ func (o Object) GetIsModifiedKey(c cache.KeyNameProvider) cache.Key {
 	return c.GetKeyNameWithID(IsModifiedKeyID, o.ID)
 }
 
+// GetIsModifiedCollectionKey returns the IsModifiedCollectionKeyID key name for object
+func (o Object) GetIsModifiedCollectionKey(c cache.KeyNameProvider) cache.Key {
+	return "" // Unused until we turn one page caching
+}
+
 // GetDependencyKeys returns the list of keys for object dependencies
 func (o Object) GetDependencyKeys(c cache.KeyNameProvider) []cache.Key {
 	// Objects depend on object types but we don't store that dependency because we currently flush the whole cache on object type delete
@@ -416,6 +477,11 @@ func (e Edge) GetGlobalCollectionKey(c cache.KeyNameProvider) cache.Key {
 	return c.GetKeyNameStatic(EdgeCollectionKeyID)
 }
 
+// GetGlobalCollectionPagesKey returns the global collection key name for edge
+func (e Edge) GetGlobalCollectionPagesKey(c cache.KeyNameProvider) cache.Key {
+	return c.GetKeyNameStatic(EdgeCollectionPagesKeyID)
+}
+
 // GetPerItemCollectionKey returns the per item collection key name for edge
 func (e Edge) GetPerItemCollectionKey(c cache.KeyNameProvider) cache.Key {
 	return ""
@@ -431,6 +497,11 @@ func (e Edge) GetIsModifiedKey(c cache.KeyNameProvider) cache.Key {
 	return c.GetKeyNameWithID(IsModifiedKeyID, e.ID)
 }
 
+// GetIsModifiedCollectionKey returns the IsModifiedCollectionKeyID key name for edge
+func (e Edge) GetIsModifiedCollectionKey(c cache.KeyNameProvider) cache.Key {
+	return c.GetKeyNameWithString(IsModifiedCollectionKeyID, string(e.GetGlobalCollectionKey(c)))
+}
+
 // GetDependencyKeys returns the list of keys for edge dependencies
 func (e Edge) GetDependencyKeys(c cache.KeyNameProvider) []cache.Key {
 	// Edges depend on objects and edge types. We don't store edgetype dependency because we currently flush the whole cache on edge type delete
@@ -447,7 +518,7 @@ func (e Edge) TTL(c cache.TTLProvider) time.Duration {
 	return c.TTL(EdgeTTL)
 }
 
-// GetPrimaryKey returns the primary cache key name for edge
+// GetPrimaryKey returns the primary cache key name for path node
 func (e AttributePathNode) GetPrimaryKey(c cache.KeyNameProvider) cache.Key {
 	return "" // Unused since  AttributePathNode is not stored in cache directly
 }
@@ -457,6 +528,11 @@ func (e AttributePathNode) GetGlobalCollectionKey(c cache.KeyNameProvider) cache
 	return ""
 }
 
+// GetGlobalCollectionPagesKey returns the global collection key name for path node
+func (e AttributePathNode) GetGlobalCollectionPagesKey(c cache.KeyNameProvider) cache.Key {
+	return "" // Unused since there is no pagination for path node
+}
+
 // GetPerItemCollectionKey returns the per item collection key name for  path node
 func (e AttributePathNode) GetPerItemCollectionKey(c cache.KeyNameProvider) cache.Key {
 	return ""
@@ -472,6 +548,11 @@ func (e AttributePathNode) GetIsModifiedKey(c cache.KeyNameProvider) cache.Key {
 	return ""
 }
 
+// GetIsModifiedCollectionKey returns the IsModifiedCollectionKeyID key name for attribute path
+func (e AttributePathNode) GetIsModifiedCollectionKey(c cache.KeyNameProvider) cache.Key {
+	return "" // Unused until we turn one page caching
+}
+
 // GetDependencyKeys returns the list of keys for path node dependencies
 func (e AttributePathNode) GetDependencyKeys(c cache.KeyNameProvider) []cache.Key {
 	//  Path node depend on objects and edges.
@@ -501,6 +582,11 @@ func (o Organization) GetGlobalCollectionKey(c cache.KeyNameProvider) cache.Key
 	return c.GetKeyNameStatic(OrganizationCollectionKeyID)
 }
 
+// GetGlobalCollectionPagesKey returns the global collection key name for organization
+func (o Organization) GetGlobalCollectionPagesKey(c cache.KeyNameProvider) cache.Key {
+	return "" // Unused since there is no pagination for organization
+}
+
 // GetPerItemCollectionKey returns the per item collection key name for organization (none)
 func (o Organization) GetPerItemCollectionKey(c cache.KeyNameProvider) cache.Key {
 	return ""
@@ -516,6 +602,11 @@ func (o Organization) GetIsModifiedKey(c cache.KeyNameProvider) cache.Key {
 	return c.GetKeyNameWithID(IsModifiedKeyID, o.ID)
 }
 
+// GetIsModifiedCollectionKey returns the IsModifiedCollectionKeyID key name for organization
+func (o Organization) GetIsModifiedCollectionKey(c cache.KeyNameProvider) cache.Key {
+	return "" // Unused until we turn one page caching
+}
+
 // GetDependencyKeys returns the list of keys for organization dependencies
 func (o Organization) GetDependencyKeys(c cache.KeyNameProvider) []cache.Key {
 	return []cache.Key{}

authz/cache_ttl_provider.go

@@ -1,6 +1,7 @@
 package authz
 
 import (
+	"math/rand"
 	"time"
 
 	"userclouds.com/infra/cache"
@@ -13,11 +14,12 @@ type CacheTTLProvider struct {
 	objTTL      time.Duration
 	edgeTTL     time.Duration
 	orgTTL      time.Duration
+	exprWindow  time.Duration
 }
 
 // NewCacheTTLProvider creates a new Configurablecache.CacheTTLProvider
-func NewCacheTTLProvider(objTypeTTL time.Duration, edgeTypeTTL time.Duration, objTTL time.Duration, edgeTTL time.Duration) *CacheTTLProvider {
-	return &CacheTTLProvider{objTypeTTL: objTypeTTL, edgeTypeTTL: edgeTypeTTL, objTTL: objTTL, edgeTTL: edgeTTL, orgTTL: objTypeTTL}
+func NewCacheTTLProvider(objTypeTTL time.Duration, edgeTypeTTL time.Duration, objTTL time.Duration, edgeTTL time.Duration, exprWindow time.Duration) *CacheTTLProvider {
+	return &CacheTTLProvider{objTypeTTL: objTypeTTL, edgeTypeTTL: edgeTypeTTL, objTTL: objTTL, edgeTTL: edgeTTL, orgTTL: objTypeTTL, exprWindow: exprWindow}
 }
 
 const (
@@ -35,17 +37,22 @@ const (
 
 // TTL returns the TTL for given type
 func (c *CacheTTLProvider) TTL(id cache.KeyTTLID) time.Duration {
+	var shiftTTL time.Duration
+
+	if c.exprWindow != 0 {
+		shiftTTL = time.Duration(rand.Intn(int(c.exprWindow.Nanoseconds())))
+	}
 	switch id {
 	case ObjectTypeTTL:
-		return c.objTypeTTL
+		return c.objTypeTTL + shiftTTL
 	case EdgeTypeTTL:
-		return c.edgeTypeTTL
+		return c.edgeTypeTTL + shiftTTL
 	case ObjectTTL:
-		return c.objTTL
+		return c.objTTL + shiftTTL
 	case EdgeTTL:
-		return c.edgeTTL
+		return c.edgeTTL + shiftTTL
 	case OrganizationTTL:
-		return c.orgTTL
+		return c.orgTTL + shiftTTL
 	}
 	return cache.SkipCacheTTL
 }