Skip to content

Commit f999623

Browse files
sfc-gh-abeamonsfc-gh-abeamon
committed
Add a tenant lookup interface and use it when starting transactions
1 parent 05bbb7c commit f999623

27 files changed

+478
-528
lines changed

‎bindings/c/fdb_c.cpp

+1-2
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,7 @@ extern "C" DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_erro
8282
code == error_code_grv_proxy_memory_limit_exceeded ||
8383
code == error_code_commit_proxy_memory_limit_exceeded ||
8484
code == error_code_batch_transaction_throttled || code == error_code_process_behind ||
85-
code == error_code_tag_throttled || code == error_code_unknown_tenant ||
86-
code == error_code_proxy_tag_throttled;
85+
code == error_code_tag_throttled || code == error_code_proxy_tag_throttled;
8786
}
8887
return false;
8988
}

‎documentation/sphinx/source/api-error-codes.rst

-2
Original file line numberDiff line numberDiff line change
@@ -174,8 +174,6 @@ FoundationDB may return the following error codes from API functions. If you nee
174174
+-----------------------------------------------+-----+--------------------------------------------------------------------------------+
175175
| tenants_disabled | 2136| Tenants have been disabled in the cluster |
176176
+-----------------------------------------------+-----+--------------------------------------------------------------------------------+
177-
| unknown_tenant | 2137| Tenant is not available from this server |
178-
+-----------------------------------------------+-----+--------------------------------------------------------------------------------+
179177
| api_version_unset | 2200| API version is not set |
180178
+-----------------------------------------------+-----+--------------------------------------------------------------------------------+
181179
| api_version_already_set | 2201| API version may be set only once |

‎fdbclient/ClientKnobs.cpp

-2
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,6 @@ void ClientKnobs::initialize(Randomize randomize) {
9292
init( LOCATION_CACHE_EVICTION_SIZE_SIM, 10 ); if( randomize && BUGGIFY ) LOCATION_CACHE_EVICTION_SIZE_SIM = 3;
9393
init( LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD, 60 );
9494
init( LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL, 60 );
95-
init( TENANT_CACHE_EVICTION_SIZE, 100000 );
96-
init( TENANT_CACHE_EVICTION_SIZE_SIM, 10 ); if( randomize && BUGGIFY ) TENANT_CACHE_EVICTION_SIZE_SIM = 3;
9795

9896
init( GET_RANGE_SHARD_LIMIT, 2 );
9997
init( WARM_RANGE_SHARD_LIMIT, 100 );

‎fdbclient/FDBTypes.cpp

+4-4
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,12 @@
2222
#include "fdbclient/Knobs.h"
2323
#include "fdbclient/NativeAPI.actor.h"
2424

25-
KeyRangeRef toPrefixRelativeRange(KeyRangeRef range, KeyRef prefix) {
26-
if (prefix.empty()) {
25+
KeyRangeRef toPrefixRelativeRange(KeyRangeRef range, Optional<KeyRef> prefix) {
26+
if (!prefix.present() || prefix.get().empty()) {
2727
return range;
2828
} else {
29-
KeyRef begin = range.begin.startsWith(prefix) ? range.begin.removePrefix(prefix) : allKeys.begin;
30-
KeyRef end = range.end.startsWith(prefix) ? range.end.removePrefix(prefix) : allKeys.end;
29+
KeyRef begin = range.begin.startsWith(prefix.get()) ? range.begin.removePrefix(prefix.get()) : allKeys.begin;
30+
KeyRef end = range.end.startsWith(prefix.get()) ? range.end.removePrefix(prefix.get()) : allKeys.end;
3131
return KeyRangeRef(begin, end);
3232
}
3333
}

‎fdbclient/NativeAPI.actor.cpp

+229-377
Large diffs are not rendered by default.

‎fdbclient/ServerKnobs.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -488,6 +488,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
488488
init( START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET, 10.0 );
489489
init( START_TRANSACTION_MAX_QUEUE_SIZE, 1e6 );
490490
init( KEY_LOCATION_MAX_QUEUE_SIZE, 1e6 );
491+
init( TENANT_ID_REQUEST_MAX_QUEUE_SIZE, 1e6 );
491492
init( COMMIT_PROXY_LIVENESS_TIMEOUT, 20.0 );
492493

493494
init( COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE, 0.0005 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE = 0.005;

‎fdbclient/include/fdbclient/ClientKnobs.h

-2
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,6 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
9090
int LOCATION_CACHE_EVICTION_SIZE_SIM;
9191
double LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD;
9292
double LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL;
93-
int TENANT_CACHE_EVICTION_SIZE;
94-
int TENANT_CACHE_EVICTION_SIZE_SIM;
9593

9694
int GET_RANGE_SHARD_LIMIT;
9795
int WARM_RANGE_SHARD_LIMIT;

‎fdbclient/include/fdbclient/CommitProxyInterface.h

+41-2
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ struct CommitProxyInterface {
6262
RequestStream<struct ExclusionSafetyCheckRequest> exclusionSafetyCheckReq;
6363
RequestStream<struct GetDDMetricsRequest> getDDMetrics;
6464
PublicRequestStream<struct ExpireIdempotencyIdRequest> expireIdempotencyId;
65+
PublicRequestStream<struct GetTenantIdRequest> getTenantId;
6566

6667
UID id() const { return commit.getEndpoint().token; }
6768
std::string toString() const { return id().shortString(); }
@@ -90,6 +91,7 @@ struct CommitProxyInterface {
9091
getDDMetrics = RequestStream<struct GetDDMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(9));
9192
expireIdempotencyId =
9293
PublicRequestStream<struct ExpireIdempotencyIdRequest>(commit.getEndpoint().getAdjustedEndpoint(10));
94+
getTenantId = PublicRequestStream<struct GetTenantIdRequest>(commit.getEndpoint().getAdjustedEndpoint(11));
9395
}
9496
}
9597

@@ -107,6 +109,7 @@ struct CommitProxyInterface {
107109
streams.push_back(exclusionSafetyCheckReq.getReceiver());
108110
streams.push_back(getDDMetrics.getReceiver());
109111
streams.push_back(expireIdempotencyId.getReceiver());
112+
streams.push_back(getTenantId.getReceiver());
110113
FlowTransport::transport().addEndpoints(streams);
111114
}
112115
};
@@ -361,10 +364,46 @@ struct GetReadVersionRequest : TimedRequest {
361364
}
362365
};
363366

367+
struct GetTenantIdReply {
368+
constexpr static FileIdentifier file_identifier = 11441284;
369+
int64_t tenantId = TenantInfo::INVALID_TENANT;
370+
371+
GetTenantIdReply() {}
372+
GetTenantIdReply(int64_t tenantId) : tenantId(tenantId) {}
373+
374+
template <class Ar>
375+
void serialize(Ar& ar) {
376+
serializer(ar, tenantId);
377+
}
378+
};
379+
380+
struct GetTenantIdRequest {
381+
constexpr static FileIdentifier file_identifier = 11299717;
382+
SpanContext spanContext;
383+
TenantName tenantName;
384+
ReplyPromise<GetTenantIdReply> reply;
385+
386+
// This version is used to specify the minimum metadata version a proxy must have in order to declare that
387+
// a tenant is not present. If the metadata version is lower, the proxy must wait in case the tenant gets
388+
// created. If latestVersion is specified, then the proxy will wait until it is sure that it has received
389+
// updates from other proxies before answering.
390+
Version minTenantVersion;
391+
392+
GetTenantIdRequest() : minTenantVersion(latestVersion) {}
393+
GetTenantIdRequest(SpanContext spanContext, TenantNameRef const& tenantName, Version minTenantVersion)
394+
: spanContext(spanContext), tenantName(tenantName), minTenantVersion(minTenantVersion) {}
395+
396+
bool verify() const { return true; }
397+
398+
template <class Ar>
399+
void serialize(Ar& ar) {
400+
serializer(ar, reply, spanContext, tenantName, minTenantVersion);
401+
}
402+
};
403+
364404
struct GetKeyServerLocationsReply {
365405
constexpr static FileIdentifier file_identifier = 10636023;
366406
Arena arena;
367-
TenantMapEntry tenantEntry;
368407
std::vector<std::pair<KeyRangeRef, std::vector<StorageServerInterface>>> results;
369408

370409
// if any storage servers in results have a TSS pair, that mapping is in here
@@ -379,7 +418,7 @@ struct GetKeyServerLocationsReply {
379418

380419
template <class Ar>
381420
void serialize(Ar& ar) {
382-
serializer(ar, results, resultsTssMapping, tenantEntry, resultsTagMapping, arena);
421+
serializer(ar, results, resultsTssMapping, resultsTagMapping, arena);
383422
}
384423
};
385424

‎fdbclient/include/fdbclient/DatabaseContext.h

+8-15
Original file line numberDiff line numberDiff line change
@@ -203,13 +203,11 @@ struct EndpointFailureInfo {
203203
};
204204

205205
struct KeyRangeLocationInfo {
206-
TenantMapEntry tenantEntry;
207206
KeyRange range;
208207
Reference<LocationInfo> locations;
209208

210209
KeyRangeLocationInfo() {}
211-
KeyRangeLocationInfo(TenantMapEntry tenantEntry, KeyRange range, Reference<LocationInfo> locations)
212-
: tenantEntry(tenantEntry), range(range), locations(locations) {}
210+
KeyRangeLocationInfo(KeyRange range, Reference<LocationInfo> locations) : range(range), locations(locations) {}
213211
};
214212

215213
struct OverlappingChangeFeedsInfo {
@@ -260,22 +258,17 @@ class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAll
260258
return cx;
261259
}
262260

263-
Optional<KeyRangeLocationInfo> getCachedLocation(const Optional<TenantNameRef>& tenant,
261+
Optional<KeyRangeLocationInfo> getCachedLocation(const TenantInfo& tenant,
264262
const KeyRef&,
265263
Reverse isBackward = Reverse::False);
266-
bool getCachedLocations(const Optional<TenantNameRef>& tenant,
264+
bool getCachedLocations(const TenantInfo& tenant,
267265
const KeyRangeRef&,
268266
std::vector<KeyRangeLocationInfo>&,
269267
int limit,
270268
Reverse reverse);
271-
void cacheTenant(const TenantName& tenant, const TenantMapEntry& tenantEntry);
272-
Reference<LocationInfo> setCachedLocation(const Optional<TenantNameRef>& tenant,
273-
const TenantMapEntry& tenantEntry,
274-
const KeyRangeRef&,
275-
const std::vector<struct StorageServerInterface>&);
276-
void invalidateCachedTenant(const TenantNameRef& tenant);
277-
void invalidateCache(const KeyRef& tenantPrefix, const KeyRef& key, Reverse isBackward = Reverse::False);
278-
void invalidateCache(const KeyRef& tenantPrefix, const KeyRangeRef& keys);
269+
Reference<LocationInfo> setCachedLocation(const KeyRangeRef&, const std::vector<struct StorageServerInterface>&);
270+
void invalidateCache(const Optional<KeyRef>& tenantPrefix, const KeyRef& key, Reverse isBackward = Reverse::False);
271+
void invalidateCache(const Optional<KeyRef>& tenantPrefix, const KeyRangeRef& keys);
279272

280273
// Records that `endpoint` is failed on a healthy server.
281274
void setFailedEndpointOnHealthyServer(const Endpoint& endpoint);
@@ -496,10 +489,8 @@ class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAll
496489

497490
// Cache of location information
498491
int locationCacheSize;
499-
int tenantCacheSize;
500492
CoalescedKeyRangeMap<Reference<LocationInfo>> locationCache;
501493
std::unordered_map<Endpoint, EndpointFailureInfo> failedEndpointsOnHealthyServersInfo;
502-
std::unordered_map<TenantName, TenantMapEntry> tenantCache;
503494

504495
std::map<UID, StorageServerInfo*> server_interf;
505496
std::map<UID, BlobWorkerInterface> blobWorker_interf; // blob workers don't change endpoints for the same ID
@@ -562,6 +553,8 @@ class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAll
562553
Counter transactionKeyServerLocationRequests;
563554
Counter transactionKeyServerLocationRequestsCompleted;
564555
Counter transactionStatusRequests;
556+
Counter transactionTenantLookupRequests;
557+
Counter transactionTenantLookupRequestsCompleted;
565558
Counter transactionsTooOld;
566559
Counter transactionsFutureVersions;
567560
Counter transactionsNotCommitted;

‎fdbclient/include/fdbclient/FDBTypes.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -610,7 +610,7 @@ KeyRef keyBetween(const KeyRangeRef& keys);
610610
// Returns a randomKey between keys. If it's impossible, return keys.end.
611611
Key randomKeyBetween(const KeyRangeRef& keys);
612612

613-
KeyRangeRef toPrefixRelativeRange(KeyRangeRef range, KeyRef prefix);
613+
KeyRangeRef toPrefixRelativeRange(KeyRangeRef range, Optional<KeyRef> prefix);
614614

615615
struct KeySelectorRef {
616616
private:

‎fdbclient/include/fdbclient/NativeAPI.actor.h

+44-19
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,44 @@ struct Watch : public ReferenceCounted<Watch>, NonCopyable {
237237
void setWatch(Future<Void> watchFuture);
238238
};
239239

240+
class Tenant : public ReferenceCounted<Tenant> {
241+
public:
242+
Tenant(Future<int64_t> id, Optional<TenantName> name) : idFuture(id), name(name) {}
243+
244+
int64_t id() const {
245+
ASSERT(idFuture.isReady());
246+
return idFuture.get();
247+
}
248+
249+
StringRef prefix() const {
250+
ASSERT(idFuture.isReady());
251+
if (bigEndianId == -1) {
252+
bigEndianId = bigEndian64(idFuture.get());
253+
}
254+
return StringRef(reinterpret_cast<const uint8_t*>(&bigEndianId), TenantAPI::PREFIX_SIZE);
255+
}
256+
257+
std::string description() const {
258+
StringRef nameStr = name.castTo<TenantNameRef>().orDefault("<unspecified>"_sr);
259+
if (idFuture.canGet()) {
260+
return format("%*s (%lld)", nameStr.size(), nameStr.begin(), idFuture.get());
261+
} else {
262+
return format("%*s", nameStr.size(), nameStr.begin());
263+
}
264+
}
265+
266+
Future<int64_t> idFuture;
267+
Optional<TenantName> name;
268+
269+
private:
270+
mutable int64_t bigEndianId = -1;
271+
};
272+
273+
template <>
274+
struct Traceable<Tenant> : std::true_type {
275+
static std::string toString(const Tenant& tenant) { return printable(tenant.description()); }
276+
};
277+
240278
FDB_DECLARE_BOOLEAN_PARAM(AllowInvalidTenantID);
241279

242280
struct TransactionState : ReferenceCounted<TransactionState> {
@@ -280,37 +318,24 @@ struct TransactionState : ReferenceCounted<TransactionState> {
280318

281319
// VERSION_VECTOR changed default values of readVersionObtainedFromGrvProxy
282320
TransactionState(Database cx,
283-
Optional<TenantName> tenant,
321+
Optional<TenantName> tenantName,
284322
TaskPriority taskID,
285323
SpanContext spanContext,
286324
Reference<TransactionLogInfo> trLogInfo);
287325

288326
Reference<TransactionState> cloneAndReset(Reference<TransactionLogInfo> newTrLogInfo, bool generateNewSpan) const;
289-
TenantInfo getTenantInfo(AllowInvalidTenantID allowInvalidId = AllowInvalidTenantID::False);
327+
TenantInfo getTenantInfo(AllowInvalidTenantID allowInvalidTenantId = AllowInvalidTenantID::False);
290328

291-
Optional<TenantName> const& tenant();
329+
Optional<Reference<Tenant>> const& tenant();
292330
bool hasTenant() const;
293331

294-
int64_t tenantId() const { return tenantId_; }
295-
void trySetTenantId(int64_t tenantId) {
296-
if (tenantId_ == TenantInfo::INVALID_TENANT) {
297-
tenantId_ = tenantId;
298-
}
299-
}
300-
301-
Future<Void> handleUnknownTenant();
332+
int64_t tenantId() const { return tenant_.present() ? tenant_.get()->id() : TenantInfo::INVALID_TENANT; }
302333

303334
private:
304-
Optional<TenantName> tenant_;
305-
int64_t tenantId_ = TenantInfo::INVALID_TENANT;
335+
Optional<Reference<Tenant>> tenant_;
306336
bool tenantSet;
307337
};
308338

309-
class Tenant {
310-
Future<int64_t> id;
311-
TenantName name;
312-
};
313-
314339
class Transaction : NonCopyable {
315340
public:
316341
explicit Transaction(Database const& cx, Optional<TenantName> const& tenant = Optional<TenantName>());
@@ -507,7 +532,7 @@ class Transaction : NonCopyable {
507532
return Standalone<VectorRef<KeyRangeRef>>(tr.transaction.write_conflict_ranges, tr.arena);
508533
}
509534

510-
Optional<TenantName> getTenant() { return trState->tenant(); }
535+
Optional<Reference<Tenant>> getTenant() { return trState->tenant(); }
511536

512537
Reference<TransactionState> trState;
513538
std::vector<Reference<Watch>> watches;

‎fdbclient/include/fdbclient/ReadYourWrites.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ class ReadYourWritesTransaction final : NonCopyable,
212212
}
213213
Transaction& getTransaction() { return tr; }
214214

215-
Optional<TenantName> getTenant() { return tr.getTenant(); }
215+
Optional<Reference<Tenant>> getTenant() { return tr.getTenant(); }
216216
TagSet const& getTags() const { return tr.getTags(); }
217217

218218
// used in template functions as returned Future type

‎fdbclient/include/fdbclient/ServerKnobs.h

+1
Original file line numberDiff line numberDiff line change
@@ -403,6 +403,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
403403
double START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET;
404404
int START_TRANSACTION_MAX_QUEUE_SIZE;
405405
int KEY_LOCATION_MAX_QUEUE_SIZE;
406+
int TENANT_ID_REQUEST_MAX_QUEUE_SIZE;
406407
double COMMIT_PROXY_LIVENESS_TIMEOUT;
407408

408409
double COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE;

‎fdbserver/BlobWorker.actor.cpp

+3-3
Original file line numberDiff line numberDiff line change
@@ -3575,14 +3575,14 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
35753575
ASSERT(tenantEntry.get().id == req.tenantInfo.tenantId);
35763576
tenantPrefix = tenantEntry.get().prefix;
35773577
} else {
3578-
CODE_PROBE(true, "Blob worker unknown tenant");
3578+
CODE_PROBE(true, "Blob worker tenant not found");
35793579
// FIXME - better way. Wait on retry here, or just have better model for tenant metadata?
35803580
// Just throw wrong_shard_server and make the client retry and assume we load it later
3581-
TraceEvent(SevDebug, "BlobWorkerRequestUnknownTenant", bwData->id)
3581+
TraceEvent(SevDebug, "BlobWorkerRequestTenantNotFound", bwData->id)
35823582
.suppressFor(5.0)
35833583
.detail("TenantName", req.tenantInfo.name.get())
35843584
.detail("TenantId", req.tenantInfo.tenantId);
3585-
throw unknown_tenant();
3585+
throw tenant_not_found();
35863586
}
35873587
req.keyRange = KeyRangeRef(req.keyRange.begin.withPrefix(tenantPrefix.get(), req.arena),
35883588
req.keyRange.end.withPrefix(tenantPrefix.get(), req.arena));

0 commit comments

Comments
 (0)