From fb27ce1423255ee5ef761bde7721bcb40ae925ea Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 2 Feb 2024 14:09:47 -0700 Subject: [PATCH 01/90] initial work --- source/crud/bulk-write.md | 461 +++++++++ .../client-bulkWrite-delete-options.json | 260 +++++ .../client-bulkWrite-delete-options.yml | 130 +++ .../client-bulkWrite-errors.json | 440 +++++++++ .../client-bulkWrite-errors.yml | 213 +++++ .../client-bulkWrite-mixed-namespaces.json | 323 +++++++ .../client-bulkWrite-mixed-namespaces.yml | 162 ++++ .../client-bulkWrite-options.json | 322 +++++++ .../client-bulkWrite-options.yml | 152 +++ .../client-bulkWrite-ordered.json | 249 +++++ .../client-bulkWrite-ordered.yml | 136 +++ .../client-bulkWrite-results.json | 880 +++++++++++++++++ .../client-bulkWrite-results.yml | 201 ++++ .../client-bulkWrite-update-options.json | 900 ++++++++++++++++++ .../client-bulkWrite-update-options.yml | 288 ++++++ .../client-bulkWrite-serverErrors.json | 430 +++++++++ .../unified/client-bulkWrite-serverErrors.yml | 211 ++++ 17 files changed, 5758 insertions(+) create mode 100644 source/crud/bulk-write.md create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json create mode 100644 source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml create mode 100644 source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json create mode 100644 source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md new file mode 100644 index 0000000000..32e957a679 --- /dev/null +++ b/source/crud/bulk-write.md @@ -0,0 +1,461 @@ +# Bulk Write + +isabeltodo table + +## Abstract + +This specification defines the driver API for the `bulkWrite` server command introduced in MongoDB 8.0. The API defined in this specification allows users to perform insert, update, and delete operations against mixed namespaces in a minimized number of round trips, and to receive detailed results for each operation performed. This API is distinct from the collection-level bulkWrite method defined in the CRUD specification and the deprecated bulk write specification (isabeltodo link). + +## Specification + +### Definitions + +Document sequence: An `OP_MSG` payload type 1 as defined in the `OP_MSG` specification here (isabeltodo link). + +Namespace: A database name and collection name pair. This may be represented as a string (e.g. `"db.coll"`) or a class with a field for each name, e.g. + +``` +class Namespace { + databaseName: String + collectionName: String +} +``` + +`bulkWrite`: This specification discusses both the server `bulkWrite` command and the new `bulkWrite` driver API method. To avoid ambiguity, this specification refers to the server command as `bulkWrite` and the driver method as `MongoClient.bulkWrite`. + +### `bulkWrite` Command and Response Format + +#### Command + +The `bulkWrite` server command has the following format: + +```json +{ + bulkWrite: 1 + ops: + nsInfo: + errorsOnly: + ordered: + bypassDocumentValidation: + comment: + let: +} +``` + +##### `ops` + +The `ops` field is a list of write operation documents. The documents have the following format: + +###### Insert + +```json +{ + insert: + document: +} +``` + +###### Update + +```json +{ + update: + filter: + updateMods: + multi: + upsert: + arrayFilters: + hint: +} +``` + +###### Delete + +```json +{ + delete: + filter: + multi: + hint: + collation: +} +``` + +##### `nsInfo` + +The `nsInfo` field contains the namespaces on which the write operations should be performed. Drivers MUST NOT include duplicate namespaces in this list. The documents in the `nsInfo` field have the following format: + +```json +{ + ns: +} +``` + +##### `errorsOnly` + +This field corresponds to the `verboseResults` option defined on `BulkWriteOptions`. Its value should be specified as the opposite of `verboseResults`, or `true` if `verboseResults` is unspecified. Drivers MUST always define this field. + +#### Response + +The server's response to `bulkWrite` has the following format: + +```json +{ + ok: <0 | 1> + cursor: { + id: Int64 + firstBatch: + ns: + } + nErrors: + nInserted: + nUpserted: + nMatched: + nModified: + nDeleted: +} +``` + +##### Results Cursor + +The response to `bulkWrite` contains a cursor field with the first batch of individual results and a nonzero cursor ID if there are additional results that did not fit in the initial response. Drivers MUST perform `getMore`s until the cursor is exhausted to retrieve all results and errors before returning from `MongoClient.bulkWrite`. If a top-level error occurs before the cursor has been exhausted, drivers MUST send a `killCursors` command to close the cursor. + +### `MongoClient.bulkWrite` Signature + +``` +interface MongoClient { + bulkWrite(models: List, + options: Optional + ) -> BulkWriteResult throws BulkWriteException; +} +``` + +The `WriteModel`, `BulkWriteOptions`, `BulkWriteResult`, and `BulkWriteException` types defined here are similar to the types of the same names in the `MongoCollection.bulkWrite` definition. Statically typed drivers MUST NOT reuse any existing versions of these types and MUST introduce new types for the `MongoClient.bulkWrite` method. If + +#### `WriteModel` + +The `WriteModel` type defines a single write operation to be performed as part of a bulk write. `WriteModel` may be designed as separate classes that implement a `WriteModel` interface, an enum with a variant for each operation (as shown below), or another similar type. + +``` + +enum WriteModel { + InsertOne { + // The document to insert. + document: Document + } + UpdateOne { + // The filter to apply. + filter: Document + + // The update document or pipeline to apply to the selected document. + update: Document | Array + + // isabeltodo + arrayFilters: Option + + // isabeltodo + collation: Option + + // isabeltodo + hint: Optional + + // isabeltodo + upsert: Optional + } + UpdateMany { + // The filter to apply. + filter: Document + + // The update document or pipeline to apply to the selected documents. + update: Document | Document[] + + // isabeltodo + arrayFilters: Option + + // isabeltodo + collation: Option + + // isabeltodo + hint: Optional + + // isabeltodo + upsert: Optional + } + ReplaceOne { + // The filter to apply. + filter: Document + + // The replacement document. + replacement: Document + + // isabeltodo + collation: Option + + // isabeltodo + hint: Optional + + // isabeltodo + upsert: Optional + } + DeleteOne { + // The filter to apply. + filter: Document + + // isabeltodo + collation: Option + + // isabeltodo + hint: Option + } + DeleteMany { + // The filter to apply. + filter: Document, + + // isabeltodo + collation: Option, + + // isabeltodo + hint: Option + } +} +``` + +##### Namespaces + +Each write model supplied to `Client.bulkWrite` in the `models` parameter MUST have a corresponding namespace that defines the collection on which the operation should be performed. Drivers SHOULD design this pairing in whichever way is most ergonomic for its language. For example, drivers may include a required `namespace` field in each `WriteModel` variant, accept a list of `(Namespace, WriteModel)` tuples for `models`, or define the following pair class: + +``` + +class NamespaceWriteModelPair { + // The namespace on which to perform the operation. + namespace: Namespace + + // The write to perform. + model: WriteModel +} +``` + +#### `BulkWriteOptions` + +``` + +class BulkWriteOptions { + // Whether the operations in this bulk write should be executed in the order in which they were specified. If false, writes will continue to be executed if an individual write fails. If true, writes will stop executing if an individual write fails. Defaults to true. + ordered: Optional + + // If true, allows the writes to opt out of document-level validation. Defaults to false. + bypassDocumentValidation: Optional + + // A map of parameter names and values to apply to all operations within the bulk write. Values must be constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an aggregate expression context (e.g. "$$var"). + let: Optional + + // The write concern to use for this bulk write. + // + // NOT REQUIRED. Drivers MUST expose this option if retrieving a handle to a `MongoClient` with a different write concern configured than that of the user's standard URI options is nontrivial. Drivers MAY omit this option if they provide a way to retrieve a lightweight handle to a `MongoClient` with a custom write concern configured, e.g. a `MongoClient.withWriteConcern` method. + writeConcern: Optional + + // Whether detailed results for each successful operation should be included in the returned `BulkWriteResult`. Defaults to false. + verboseResults: Optional +} +``` + +#### `BulkWriteResult` + +``` + +class BulkWriteResult { + // Indicates whether this write result was acknowledged. If not, then all other members of this result will be undefined. + // + // NOT REQUIRED. See here(isabeltodo link) for additional guidance on modeling unacknowledged results. + acknowledged: boolean + + // Indicates whether the results are verbose. This value will be equal to the value of `verboseResults` in the provided `BulkWriteOptions`. If false, the `insertResults`, `updateResults`, and `deleteResults` fields in this result will be undefined. + // + // NOT REQUIRED. See below (link) for other ways to differentiate summary results from verbose results. + hasVerboseResults: boolean + + // The total number of documents inserted across all insert operations. + insertedCount: i64 + + // The total number of documents upserted across all update operations. + upsertedCount: i64 + + // The total number of documents matched across all update operations. + matchedCount: i64 + + // The total number of documents modified across all update operations. + modifiedCount: i64 + + // The total number of documents deleted across all delete operations. + deletedCount: i64 + + // The results of each individual insert operation that was successfully performed. Results can be accessed by their index in the list of write models provided to the call to `MongoClient.bulkWrite`. + insertResults: Map + + // The results of each individual update operation that was successfully performed. Results can be accessed by their index in the list of write models provided to the call to `MongoClient.bulkWrite`. + updateResults: Map + + // The results of each individual delete operation that was successfully performed. Results can be accessed by their index in the list of write models provided to the call to `MongoClient.bulkWrite`. + deleteResults: Map +} + +class InsertOneResult { + // The _id of the inserted document. + insertedId: Bson +} + +class UpdateResult { + // The number of documents that matched the filter. + matchedCount: Int64 + + // The number of documents that were modified. + modifiedCount: Int64 + + // The number of documents that were upserted. + // + // NOT REQUIRED. Drivers may choose to not provide this property so long as it is always possible to infer whether an upsert has taken place. Since the _id field of an upserted document could be null, a null upsertedId may be ambiguous in some drivers. If so, this field can be used to indicate whether an upsert has taken place. + upsertedCount: Int64 + + // The _id field of the upserted document if an upsert occurred. + upsertedId: Optional +} + +class DeleteResult { + // The number of documents that were deleted. + deletedCount: Int64 +} +``` + +##### Summary vs. Verbose Results + +Users MUST be able to discern whether the result returned from `MongoClient.bulkWrite` is a summary result or a verbose result without accessing the value provided for `verboseResults`. Drivers MUST implement this in one of the following ways: + +* Expose the `hasVerboseResults` field in `BulkWriteResult` as defined above. +* Implement the `insertResults`, `updateResults`, and `deleteResults` fields as optional types and document that they will be unset when `verboseResults` is false. +* Introduce separate `SummaryBulkWriteResult` and `VerboseBulkWriteResult` types. `VerboseBulkWriteResult` MUST have all of the required fields defined on `BulkWriteResult` above. `SummaryBulkWriteResult` MUST have all of the required fields defined on `BulkWriteResult` above except `insertResults`, `updateResults`, and `deleteResults`. + +##### Individual Results + +The `InsertOneResult`, `UpdateResult`, and `DeleteResult` classes are the same as or similar to those of the same names defined in the CRUD specification. Drivers MUST redefine these classes if their existing result classes deviate from the definitions in this specification (e.g. if they contain acknowledgement information). Drivers MAY reuse their existing types for these classes if they match the ones defined here exactly. + +#### `BulkWriteException` + +``` + +class BulkWriteException { + // A top-level error that occurred when attempting to communicate with the server or execute the bulk write. This value may not be populated if the exception was thrown due to errors occurrring on individual writes. + error: Optional + + // Write concern errors that occurred while executing the bulk write. This list may have multiple items if more than one round trip was required to execute the bulk write. + writeConcernErrors: List + + // Errors that occurred during the execution of individual write operations. This map will contain at most one entry if the bulk write was ordered. + writeErrors: Map + + // The results of any successful operations that were performed before the error was encountered. + partialResult: Optional +} +``` + +##### Top-level errors + +A top-level error is defined as any error that occurs during a bulk write that is not the result of an individual write operation failing or a write concern error. Possible top-level errors include, but are not limited to, network errors that occur when communicating with the server, command errors returned from the server, and client-side errors (e.g. an oversized insert document was provided). When a top-level error occurs, drivers MUST embed that error within a `BulkWriteException` as the `error` field and MUST include any results and/or non-top-level errors that have been observed so far in the `writeConcernErrors`, `writeErrors`, and `partialResults` fields. + +If a retryable top-level error is encountered and the `bulkWrite` is retryable, drivers MUST perform a retry according to the retryable writes specification. Note that any bulk write that contains a `multi: true` operation (i.e. `UpdateMany` or `DeleteMany`) is not retryable. If the retry fails or if the bulk write or error was not retryable, drivers MUST halt execution for both ordered and unordered bulk writes and immediately throw a `BulkWriteException`. + +##### Write concern errors + +At most one write concern error may be returned per `bulkWrite` command response. If the driver observes a write concern error, it MUST record the error in the `writeConcernErrors` field and MUST continue executing the bulk write for both ordered and unordered bulk writes. If the `writeConcernErrors` field is not empty at the end of execution, drivers MUST throw the `BulkWriteException` and include any results and/or additional errors that were observed. + +##### Individual write errors + +Failures of individual write operations are reported in the cursor of results returned from the server. Drivers MUST iterate the this cursor fully and record all errors in the `writeErrors` map. If an individual write error is encountered during an ordered bulk write, drivers MUST record the error in `writeErrors` and immediately throw a `BulkWriteException`. Otherwise, drivers MUST continue iterating the cursor and execute any further `bulkWrite` batches. + +### Command Batching + +Drivers MUST accept an arbitrary number of operations as input to the `Client.bulkWrite` method. Because the server imposes restrictions on the size of write operations, this means that a single call to `Client.bulkWrite` may require multiple `bulkWrite` commands to be sent to the server. Drivers MUST split bulk writes into separate commands when the user's list of operations exceeds one of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize`, or `maxMessageSizeBytes`. Each of these values can be retrieved from the selected server's `hello` command response. + +#### Number of Writes + +The `ops` array MUST NOT include more than `maxWriteBatchSize` operations. + +#### Document Size + +All entries within the `ops` array MUST be within `maxBsonObjectSize` bytes. Drivers MUST throw a `BulkWriteException` with a top-level client error if an operation exceeds this size. + +#### Total Message Size + +##### Encrypted bulk writes + +When auto-encryption is enabled, drivers MUST NOT provide the `ops` and `nsInfo` fields as document sequences and MUST limit the size of the command according to the auto-encryption size limits defined here (isabeltodo link and update language in link). + +##### Unencrypted bulk writes with document sequences + +When `ops` and `nsInfo` are provided as document sequences, drivers MUST ensure that the combined size of these sequences does not exceed `maxMessageSizeBytes - 16,000`. `maxMessageSizeBytes` defines the maximum size for an entire `OP_MSG`, so 16KB is subtracted from `maxMessageSizeBytes` here to account for the bytes in the rest of the message. + +##### Unencrypted bulk writes without document sequences + +When `ops` and `nsInfo` are provided within the `bulkWrite` command document and auto-encryption is not enabled, drivers MUST ensure that the total size of the `bulkWrite` command document does not exceed `maxBsonObjectSize`. + +### Pseudocode Implementation + +The following pseudocode is a sample implementation of `MongoClient.bulkWrite`. + +``` +executeBulkWrite(client: MongoClient, models: List, options: BulkWriteOptions) { + let exception = empty BulkWriteException + + while !models.isEmpty() { + try { + let bulkWrite = batchSplitAndBuildCommand(models, options) + let (summaryInfo, resultsCursor, writeConcernError) = client.executeOperation(bulkWrite) + + if writeConcernError != null { + exception.writeConcernErrors.add(writeConcernError) + } + + for result in resultsCursor { + if result.isSuccess() { + exception.partialResult.add(result) + } else { // error + exception.writeErrors.add(result) + if options.ordered { + throw exception + } + } + } + } catch error { + exception.error = error + throw exception + } + } + + if !exception.writeConcernErrors.isEmpty() || !exception.writeErrors.isEmpty() { + throw exception + } else { + return exception.partialResult + } +} +``` + +### Q&A + +#### Why are we adding a new bulk write API rather than updating the `MongoCollection.bulkWrite` implementation? + +The new `bulkWrite` command is only available in MongoDB 8.0+, so it cannot function as a drop-in replacement for the existing bulk write implementation that uses the `insert`, `update`, and `delete` commands. Additionally, because the new `bulkWrite` command allows operations against multiple collections and databases, `MongoClient` is a more appropriate place to expose its functionality. + +#### Why can't drivers reuse existing bulk write types? + +This specification introduces several types that are similar to existing types used in the legacy bulk write API. Although these types are similar now, they may diverge in the future with the introduction of new options and features to the `bulkWrite` command. Introducing new types also provides more clarity to users on the existing differences between the legacy and new bulk write APIs. For example, the `verboseResults` option is only available for `MongoClient.bulkWrite`. + +#### Why are bulk write operation results returned in a cursor? + +Returning results via a cursor rather than an array in the `bulkWrite` response allows full individual results and errors to be returned without the risk of the response exceeding the maximum BSON object size. Using a cursor also leaves open the opportunity to add `findAndModify` to the list of supported write operations in the future. + +#### Why was the `verboseResults` option introduced, and why is its default `false`? + +The `bulkWrite` command returns top-level summary result counts and, optionally, individual results for each operation. Compiling the individual results server-side and consuming these results driver-side is less performant than only recording the summary counts. We expect that most users are not interested in the individual results of their operations and that most users will rely on defaults, so `verboseResults` defaults to `false` to improve performance in the common case. + +#### Why must all errors be embedded within a `BulkWriteException`? + +An error may occur at any point during the execution of the `MongoClient.bulkWrite` method, including after some individual writes have already been executed. Embedding errors in a `BulkWriteException` gives users more information about what had happened during their bulk write prior to the error occurring. diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json new file mode 100644 index 0000000000..43ed517538 --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json @@ -0,0 +1,260 @@ +{ + "description": "client bulkWrite delete options", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "client bulk write delete with collation", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "collation": { + "locale": "simple" + } + } + }, + { + "deleteMany": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": { + "$gt": 1 + } + }, + "collation": { + "locale": "simple" + } + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 3, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "ops": [ + { + "delete": 0, + "filter": { + "_id": 1 + }, + "collation": { + "locale": "simple" + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": { + "$gt": 1 + } + }, + "collation": { + "locale": "simple" + }, + "multi": true + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [] + } + ] + }, + { + "description": "client bulk write delete with hint", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "hint": "_id_" + } + }, + { + "deleteMany": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": { + "$gt": 1 + } + }, + "hint": "_id_" + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 3, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "ops": [ + { + "delete": 0, + "filter": { + "_id": 1 + }, + "hint": "_id_", + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": { + "$gt": 1 + } + }, + "hint": "_id_", + "multi": true + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [] + } + ] + } + ] +} diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml new file mode 100644 index 0000000000..c2ed455d34 --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml @@ -0,0 +1,130 @@ +description: "client bulkWrite delete options" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + +tests: + - description: "client bulk write delete with collation" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - deleteOne: + namespace: &namespace + db: *database0Name + coll: *collection0Name + filter: &deleteOneFilter + _id: 1 + collation: &collation + locale: "simple" + - deleteMany: + namespace: *namespace + filter: &deleteManyFilter + _id: { $gt: 1 } + collation: *collation + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 3 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + ops: + - delete: 0 + filter: *deleteOneFilter + collation: *collation + multi: false + - delete: 0 + filter: *deleteManyFilter + collation: *collation + multi: true + outcome: + - databaseName: *database0Name + collectionName: *collection0Name + documents: [] + - description: "client bulk write delete with hint" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - deleteOne: + namespace: &namespace + db: *database0Name + coll: *collection0Name + filter: &deleteOneFilter + _id: 1 + hint: &hint _id_ + - deleteMany: + namespace: *namespace + filter: &deleteManyFilter + _id: { $gt: 1 } + hint: *hint + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 3 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + ops: + - delete: 0 + filter: *deleteOneFilter + hint: *hint + multi: false + - delete: 0 + filter: *deleteManyFilter + hint: *hint + multi: true + outcome: + - databaseName: *database0Name + collectionName: *collection0Name + documents: [] + \ No newline at end of file diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json new file mode 100644 index 0000000000..9232516dd7 --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json @@ -0,0 +1,440 @@ +{ + "description": "client bulkWrite errors", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ], + "uriOptions": { + "retryWrites": false + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 1 + }, + { + "_id": 2, + "x": 2 + }, + { + "_id": 3, + "x": 3 + } + ] + } + ], + "tests": [ + { + "description": "an individual operation fails during an ordered bulkWrite", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + } + } + }, + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + } + } + }, + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 3 + } + } + } + ], + "verboseResults": true + }, + "expectError": { + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 1, + "insertResults": {}, + "updateResults": {}, + "deleteResults": { + "0": { + "deletedCount": 1 + } + } + }, + "writeErrors": { + "1": { + "code": 17276 + } + } + } + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 2, + "x": 2 + }, + { + "_id": 3, + "x": 3 + } + ] + } + ] + }, + { + "description": "an individual operation fails during an unordered bulkWrite", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + } + } + }, + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + } + } + }, + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 3 + } + } + } + ], + "verboseResults": true, + "ordered": false + }, + "expectError": { + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 2, + "insertResults": {}, + "updateResults": {}, + "deleteResults": { + "0": { + "deletedCount": 1 + }, + "2": { + "deletedCount": 1 + } + } + }, + "writeErrors": { + "1": { + "code": 17276 + } + } + } + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 2, + "x": 2 + } + ] + } + ] + }, + { + "description": "detailed results are omitted from error when verboseResults is false", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + } + } + }, + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + } + } + }, + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 3 + } + } + } + ], + "verboseResults": false + }, + "expectError": { + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 1, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + }, + "writeErrors": { + "1": { + "code": 17276 + } + } + } + } + ] + }, + { + "description": "a top-level failure occurs during a bulkWrite", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 8 + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "document": { + "x": 1 + } + } + } + ], + "verboseResults": true + }, + "expectError": { + "errorCode": 8, + "errorLabelsContain": [ + "RetryableWriteError" + ] + } + } + ] + }, + { + "description": "a write concern error occurs during a bulkWrite", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "document": { + "_id": 10 + } + } + } + ], + "verboseResults": true + }, + "expectError": { + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 10 + } + }, + "updateResults": {}, + "deleteResults": {} + }, + "writeConcernErrors": [ + { + "code": 91, + "errmsg": "Replication is being shut down" + } + ] + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml new file mode 100644 index 0000000000..beae51f1d9 --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml @@ -0,0 +1,213 @@ +description: "client bulkWrite errors" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + uriOptions: + retryWrites: false + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - &document1 { _id: 1, x: 1 } + - &document2 { _id: 2, x: 2 } + - &document3 { _id: 3, x: 3 } + +tests: + - description: "an individual operation fails during an ordered bulkWrite" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - deleteOne: + namespace: &namespace + db: *database0Name + coll: *collection0Name + filter: { _id: 1 } + - deleteOne: + namespace: *namespace + filter: + $expr: + $eq: [ "$_id", "$$id2" ] # Attempt to access a nonexistent let var + - deleteOne: + namespace: *namespace + filter: { _id: 3 } + verboseResults: true + expectError: + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 1 + insertResults: {} + updateResults: {} + deleteResults: + 0: + deletedCount: 1 + writeErrors: + 1: + code: 17276 # Use of undefined variable + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - *document2 + - *document3 + - description: "an individual operation fails during an unordered bulkWrite" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - deleteOne: + namespace: *namespace + filter: { _id: 1 } + - deleteOne: + namespace: *namespace + filter: + $expr: + $eq: [ "$_id", "$$id2" ] # Attempt to access a nonexistent let var + - deleteOne: + namespace: *namespace + filter: { _id: 3 } + verboseResults: true + ordered: false + expectError: + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 2 + insertResults: {} + updateResults: {} + deleteResults: + 0: + deletedCount: 1 + 2: + deletedCount: 1 + writeErrors: + 1: + code: 17276 # Use of undefined variable + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - *document2 + - description: "detailed results are omitted from error when verboseResults is false" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - deleteOne: + namespace: *namespace + filter: { _id: 1 } + - deleteOne: + namespace: *namespace + filter: + $expr: + $eq: [ "$_id", "$$id2" ] # Attempt to access a nonexistent let var + - deleteOne: + namespace: *namespace + filter: { _id: 3 } + verboseResults: false + expectError: + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 1 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + writeErrors: + 1: + code: 17276 # Use of undefined variable + - description: "a top-level failure occurs during a bulkWrite" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - bulkWrite + errorCode: &errorCode 8 # UnknownError + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - insertOne: + namespace: *namespace + document: { x: 1 } + verboseResults: true + expectError: + errorCode: *errorCode + errorLabelsContain: [ RetryableWriteError ] + - description: "a write concern error occurs during a bulkWrite" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - bulkWrite + writeConcernError: &writeConcernError + code: 91 + errmsg: "Replication is being shut down" + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - insertOne: + namespace: *namespace + document: { _id: 10 } + verboseResults: true + expectError: + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 10 + updateResults: {} + deleteResults: {} + writeConcernErrors: [ *writeConcernError ] + \ No newline at end of file diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json new file mode 100644 index 0000000000..3bd65278d2 --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json @@ -0,0 +1,323 @@ +{ + "description": "client bulkWrite with mixed namespaces", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "bulkWrite-db0" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "coll1" + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "bulkWrite-db1" + } + }, + { + "collection": { + "id": "collection2", + "database": "database1", + "collectionName": "coll2" + } + } + ], + "initialData": [ + { + "databaseName": "bulkWrite-db0", + "collectionName": "coll0", + "documents": [] + }, + { + "databaseName": "bulkWrite-db0", + "collectionName": "coll1", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + }, + { + "databaseName": "bulkWrite-db1", + "collectionName": "coll2", + "documents": [ + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ], + "tests": [ + { + "description": "client bulkWrite with mixed namespaces", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "bulkWrite-db0", + "coll": "coll0" + }, + "document": { + "_id": 1 + } + } + }, + { + "insertOne": { + "namespace": { + "db": "bulkWrite-db0", + "coll": "coll0" + }, + "document": { + "_id": 2 + } + } + }, + { + "updateOne": { + "namespace": { + "db": "bulkWrite-db0", + "coll": "coll1" + }, + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "deleteOne": { + "namespace": { + "db": "bulkWrite-db1", + "coll": "coll2" + }, + "filter": { + "_id": 3 + } + } + }, + { + "deleteOne": { + "namespace": { + "db": "bulkWrite-db0", + "coll": "coll1" + }, + "filter": { + "_id": 2 + } + } + }, + { + "replaceOne": { + "namespace": { + "db": "bulkWrite-db1", + "coll": "coll2" + }, + "filter": { + "_id": 4 + }, + "replacement": { + "x": 44 + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 2, + "upsertedCount": 0, + "matchedCount": 2, + "modifiedCount": 2, + "deletedCount": 2, + "insertResults": { + "0": { + "insertedId": 1 + }, + "1": { + "insertedId": 2 + } + }, + "updateResults": { + "2": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + }, + "5": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + } + }, + "deleteResults": { + "3": { + "deletedCount": 1 + }, + "4": { + "deletedCount": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "ops": [ + { + "insert": 0, + "document": { + "_id": 1 + } + }, + { + "insert": 0, + "document": { + "_id": 2 + } + }, + { + "update": 1, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "delete": 2, + "filter": { + "_id": 3 + }, + "multi": false + }, + { + "delete": 1, + "filter": { + "_id": 2 + }, + "multi": false + }, + { + "update": 2, + "filter": { + "_id": 4 + }, + "updateMods": { + "x": 44 + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "bulkWrite-db0.coll0" + }, + { + "ns": "bulkWrite-db0.coll1" + }, + { + "ns": "bulkWrite-db1.coll2" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "bulkWrite-db0", + "collectionName": "coll0", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + }, + { + "databaseName": "bulkWrite-db0", + "collectionName": "coll1", + "documents": [ + { + "_id": 1, + "x": 12 + } + ] + }, + { + "databaseName": "bulkWrite-db1", + "collectionName": "coll2", + "documents": [ + { + "_id": 4, + "x": 44 + } + ] + } + ] + } + ] +} diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml new file mode 100644 index 0000000000..36c1cd754c --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml @@ -0,0 +1,162 @@ +description: "client bulkWrite with mixed namespaces" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name bulkWrite-db0 + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + - collection: + id: &collection1 collection1 + database: *database0 + collectionName: &collection1Name coll1 + - database: + id: &database1 database1 + client: *client0 + databaseName: &database1Name bulkWrite-db1 + - collection: + id: &collection2 collection2 + database: *database1 + collectionName: &collection2Name coll2 + +initialData: + - databaseName: *database0Name + collectionName: *collection0Name + documents: [] + - databaseName: *database0Name + collectionName: *collection1Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - databaseName: *database1Name + collectionName: *collection2Name + documents: + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + +tests: + - description: "client bulkWrite with mixed namespaces" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - insertOne: + namespace: + db: *database0Name + coll: *collection0Name + document: &document1 + _id: 1 + - insertOne: + namespace: + db: *database0Name + coll: *collection0Name + document: &document2 + _id: 2 + - updateOne: + namespace: + db: *database0Name + coll: *collection1Name + filter: &filter1 + _id: 1 + update: &update + $inc: { x: 1 } + - deleteOne: + namespace: + db: *database1Name + coll: *collection2Name + filter: &filter2 + _id: 3 + - deleteOne: + namespace: + db: *database0Name + coll: *collection1Name + filter: &filter3 + _id: 2 + - replaceOne: + namespace: + db: *database1Name + coll: *collection2Name + filter: &filter4 + _id: 4 + replacement: &replacement + x: 44 + verboseResults: true + expectResult: + insertedCount: 2 + upsertedCount: 0 + matchedCount: 2 + modifiedCount: 2 + deletedCount: 2 + insertResults: + 0: + insertedId: 1 + 1: + insertedId: 2 + updateResults: + 2: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + 5: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + deleteResults: + 3: + deletedCount: 1 + 4: + deletedCount: 1 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + ops: + - insert: 0 + document: *document1 + - insert: 0 + document: *document2 + - update: 1 + filter: *filter1 + updateMods: *update + multi: false + - delete: 2 + filter: *filter2 + multi: false + - delete: 1 + filter: *filter3 + multi: false + - update: 2 + filter: *filter4 + updateMods: *replacement + multi: false + nsInfo: + - ns: bulkWrite-db0.coll0 + - ns: bulkWrite-db0.coll1 + - ns: bulkWrite-db1.coll2 + outcome: + - databaseName: *database0Name + collectionName: *collection0Name + documents: + - *document1 + - *document2 + - databaseName: *database0Name + collectionName: *collection1Name + documents: + - { _id: 1, x: 12 } + - databaseName: *database1Name + collectionName: *collection2Name + documents: + - { _id: 4, x: 44 } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json new file mode 100644 index 0000000000..f9af9ae5e3 --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json @@ -0,0 +1,322 @@ +{ + "description": "client bulkWrite top-level options", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client bulkWrite comment", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "document": { + "_id": 3, + "x": 33 + } + } + } + ], + "comment": { + "bulk": "write" + }, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 3 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "comment": { + "bulk": "write" + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "client bulkWrite bypassDocumentValidation", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "document": { + "_id": 4, + "x": 44 + } + } + } + ], + "bypassDocumentValidation": true, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 4 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "bypassDocumentValidation": true + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ] + }, + { + "description": "client bulkWrite let", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "updateOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id1" + ] + } + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + } + } + } + ], + "let": { + "id1": 1, + "id2": 2 + }, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 1, + "insertResults": {}, + "updateResults": { + "0": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + } + }, + "deleteResults": { + "1": { + "deletedCount": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "let": { + "id1": 1, + "id2": 2 + } + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "x": 12 + } + ] + } + ] + } + ] +} diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml new file mode 100644 index 0000000000..0eb1fbfb8a --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml @@ -0,0 +1,152 @@ +description: "client bulkWrite top-level options" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - &doc1 { _id: 1, x: 11 } + - &doc2 { _id: 2, x: 22 } + +tests: + - description: "client bulkWrite comment" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - insertOne: + namespace: &namespace + db: *database0Name + coll: *collection0Name + document: &doc3 { _id: 3, x: 33 } + comment: &comment { "bulk": "write" } + verboseResults: true + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 3 + updateResults: {} + deleteResults: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + comment: *comment + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - *doc1 + - *doc2 + - *doc3 + - description: "client bulkWrite bypassDocumentValidation" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - insertOne: + namespace: *namespace + document: &doc4 { _id: 4, x: 44 } + bypassDocumentValidation: true + verboseResults: true + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 4 + updateResults: {} + deleteResults: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + bypassDocumentValidation: true + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - *doc1 + - *doc2 + - *doc4 + - description: "client bulkWrite let" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - updateOne: + namespace: *namespace + filter: + $expr: + $eq: [ "$_id", "$$id1" ] + update: + $inc: { x: 1 } + - deleteOne: + namespace: *namespace + filter: + $expr: + $eq: [ "$_id", "$$id2" ] + let: &let { id1: 1, id2: 2 } + verboseResults: true + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 1 + modifiedCount: 1 + deletedCount: 1 + insertResults: {} + updateResults: + 0: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + deleteResults: + 1: + deletedCount: 1 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + let: *let + outcome: + - databaseName: *database0Name + collectionName: *collection0Name + documents: + - { _id: 1, x: 12 } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json new file mode 100644 index 0000000000..b477942b16 --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json @@ -0,0 +1,249 @@ +{ + "description": "client bulkWrite with ordered option", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "client bulkWrite with ordered: false", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "document": { + "_id": 1 + } + } + } + ], + "verboseResults": true, + "ordered": false + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 1 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "ordered": false + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + }, + { + "description": "client bulkWrite with ordered: true", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "document": { + "_id": 4 + } + } + } + ], + "verboseResults": true, + "ordered": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 4 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "ordered": true + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 4 + } + ] + } + ] + }, + { + "description": "client bulkWrite defaults to ordered: true", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "document": { + "_id": 4 + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 4 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "ordered": true + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 4 + } + ] + } + ] + } + ] +} diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml new file mode 100644 index 0000000000..c2792a4364 --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml @@ -0,0 +1,136 @@ +description: "client bulkWrite with ordered option" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: [] + +tests: + - description: "client bulkWrite with ordered: false" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - insertOne: + namespace: + db: *database0Name + coll: *collection0Name + document: &document { _id: 1 } + verboseResults: true + ordered: false + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 1 + updateResults: {} + deleteResults: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + ordered: false + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - *document + - description: "client bulkWrite with ordered: true" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - insertOne: + namespace: + db: *database0Name + coll: *collection0Name + document: &document { _id: 4 } + verboseResults: true + ordered: true + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 4 + updateResults: {} + deleteResults: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + ordered: true + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - *document + - description: "client bulkWrite defaults to ordered: true" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - insertOne: + namespace: + db: *database0Name + coll: *collection0Name + document: &document { _id: 4 } + verboseResults: true + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 4 + updateResults: {} + deleteResults: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + ordered: true + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - *document diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json new file mode 100644 index 0000000000..5a25454942 --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json @@ -0,0 +1,880 @@ +{ + "description": "client bulkWrite results", + "schemaVersion": "1.18", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 5, + "x": 55 + }, + { + "_id": 6, + "x": 66 + }, + { + "_id": 7, + "x": 77 + } + ] + } + ], + "tests": [ + { + "description": "client bulkWrite with verboseResults: true returns detailed results", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "document": { + "_id": 8, + "x": 88 + } + } + }, + { + "updateOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "updateMany": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$inc": { + "x": 2 + } + } + } + }, + { + "replaceOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 4 + }, + "replacement": { + "x": 44 + }, + "upsert": true + } + }, + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 5 + } + } + }, + { + "deleteMany": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 1, + "matchedCount": 3, + "modifiedCount": 3, + "deletedCount": 3, + "insertResults": { + "0": { + "insertedId": 8 + } + }, + "updateResults": { + "1": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + }, + "2": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": null + }, + "3": { + "matchedCount": 1, + "modifiedCount": 0, + "upsertedId": 4 + } + }, + "deleteResults": { + "4": { + "deletedCount": 1 + }, + "5": { + "deletedCount": 2 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ops": [ + { + "insert": 0, + "document": { + "_id": 8, + "x": 88 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$inc": { + "x": 2 + } + }, + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "x": 44 + }, + "upsert": true, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 5 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 24 + }, + { + "_id": 3, + "x": 35 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 8, + "x": 88 + } + ] + } + ] + }, + { + "description": "client bulkWrite with verboseResults: false omits detailed results", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "document": { + "_id": 8, + "x": 88 + } + } + }, + { + "updateOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "updateMany": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$inc": { + "x": 2 + } + } + } + }, + { + "replaceOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 4 + }, + "replacement": { + "x": 44 + }, + "upsert": true + } + }, + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 5 + } + } + }, + { + "deleteMany": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + } + } + } + ], + "verboseResults": false + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 1, + "matchedCount": 3, + "modifiedCount": 3, + "deletedCount": 3, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 8, + "x": 88 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$inc": { + "x": 2 + } + }, + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "x": 44 + }, + "upsert": true, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 5 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 24 + }, + { + "_id": 3, + "x": 35 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 8, + "x": 88 + } + ] + } + ] + }, + { + "description": "client bulkWrite defaults to verboseResults: false", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "document": { + "_id": 8, + "x": 88 + } + } + }, + { + "updateOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "updateMany": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$inc": { + "x": 2 + } + } + } + }, + { + "replaceOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 4 + }, + "replacement": { + "x": 44 + }, + "upsert": true + } + }, + { + "deleteOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 5 + } + } + }, + { + "deleteMany": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + } + } + } + ] + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 1, + "matchedCount": 3, + "modifiedCount": 3, + "deletedCount": 3, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 8, + "x": 88 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$inc": { + "x": 2 + } + }, + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "x": 44 + }, + "upsert": true, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 5 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 24 + }, + { + "_id": 3, + "x": 35 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 8, + "x": 88 + } + ] + } + ] + } + ] +} diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml new file mode 100644 index 0000000000..84ded30f59 --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml @@ -0,0 +1,201 @@ +description: "client bulkWrite results" + +schemaVersion: "1.18" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 5, x: 55 } + - { _id: 6, x: 66 } + - { _id: 7, x: 77 } + +tests: + - description: "client bulkWrite with verboseResults: true returns detailed results" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: &requests + - insertOne: + namespace: &namespace + db: *database0Name + coll: *collection0Name + document: &insertDoc + { _id: 8, x: 88 } + - updateOne: + namespace: *namespace + filter: &updateOneFilter + _id: 1 + update: &updateOneUpdate + $inc: { x: 1 } + - updateMany: + namespace: *namespace + filter: &updateManyFilter + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + update: &updateManyUpdate + $inc: { x: 2 } + - replaceOne: + namespace: *namespace + filter: &replaceOneFilter + _id: 4 + replacement: &replacementDoc + { x: 44 } + upsert: true + - deleteOne: + namespace: *namespace + filter: &deleteOneFilter + _id: 5 + - deleteMany: + namespace: *namespace + filter: &deleteManyFilter + $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] + verboseResults: true + expectResult: + insertedCount: 1 + upsertedCount: 1 + matchedCount: 3 + modifiedCount: 3 + deletedCount: 3 + insertResults: + 0: + insertedId: 8 + updateResults: + 1: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + 2: + matchedCount: 2 + modifiedCount: 2 + upsertedId: null + 3: + matchedCount: 1 + modifiedCount: 0 + upsertedId: 4 + deleteResults: + 4: + deletedCount: 1 + 5: + deletedCount: 2 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + errorsOnly: false + ops: &ops + - + insert: 0 + document: *insertDoc + - + update: 0 + filter: *updateOneFilter + updateMods: *updateOneUpdate + multi: false + - + update: 0 + filter: *updateManyFilter + updateMods: *updateManyUpdate + multi: true + - + update: 0 + filter: *replaceOneFilter + updateMods: *replacementDoc + upsert: true + multi: false + - + delete: 0 + filter: *deleteOneFilter + multi: false + - + delete: 0 + filter: *deleteManyFilter + multi: true + nsInfo: &nsInfo + - + ns: "crud-tests.coll0" + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: &outcomeDocuments + - { _id: 1, x: 12 } + - { _id: 2, x: 24 } + - { _id: 3, x: 35 } + - { _id: 4, x: 44 } + - { _id: 8, x: 88 } + - description: "client bulkWrite with verboseResults: false omits detailed results" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: *requests + verboseResults: false + expectResult: &summaryResult + insertedCount: 1 + upsertedCount: 1 + matchedCount: 3 + modifiedCount: 3 + deletedCount: 3 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + errorsOnly: true + ops: *ops + nsInfo: *nsInfo + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: *outcomeDocuments + - description: "client bulkWrite defaults to verboseResults: false" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: *requests + expectResult: *summaryResult + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + errorsOnly: true + ops: *ops + nsInfo: *nsInfo + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: *outcomeDocuments diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json new file mode 100644 index 0000000000..ce829d746d --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json @@ -0,0 +1,900 @@ +{ + "description": "client bulkWrite update options", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 2, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 3, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 4, + "array": [ + 1, + 2, + 3 + ] + } + ] + } + ], + "tests": [ + { + "description": "client bulkWrite update with arrayFilters", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "updateOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "update": { + "$set": { + "array.$[i]": 4 + } + }, + "arrayFilters": [ + { + "i": { + "$gte": 2 + } + } + ] + } + }, + { + "updateMany": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$set": { + "array.$[i]": 5 + } + }, + "arrayFilters": [ + { + "i": { + "$gte": 2 + } + } + ] + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 3, + "modifiedCount": 3, + "deletedCount": 0, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$set": { + "array.$[i]": 4 + } + }, + "arrayFilters": [ + { + "i": { + "$gte": 2 + } + } + ], + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$set": { + "array.$[i]": 5 + } + }, + "arrayFilters": [ + { + "i": { + "$gte": 2 + } + } + ], + "multi": true + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "array": [ + 1, + 4, + 4 + ] + }, + { + "_id": 2, + "array": [ + 1, + 5, + 5 + ] + }, + { + "_id": 3, + "array": [ + 1, + 5, + 5 + ] + }, + { + "_id": 4, + "array": [ + 1, + 2, + 3 + ] + } + ] + } + ] + }, + { + "description": "client bulkWrite update with collation", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "updateOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "update": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "collation": { + "locale": "simple" + } + } + }, + { + "updateMany": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$set": { + "array": [ + 1, + 2, + 5 + ] + } + }, + "collation": { + "locale": "simple" + } + } + }, + { + "replaceOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 4 + }, + "replacement": { + "array": [ + 1, + 2, + 6 + ] + }, + "collation": { + "locale": "simple" + } + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 4, + "modifiedCount": 4, + "deletedCount": 0, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "collation": { + "locale": "simple" + }, + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$set": { + "array": [ + 1, + 2, + 5 + ] + } + }, + "collation": { + "locale": "simple" + }, + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "array": [ + 1, + 2, + 6 + ] + }, + "collation": { + "locale": "simple" + }, + "multi": false + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "array": [ + 1, + 2, + 4 + ] + }, + { + "_id": 2, + "array": [ + 1, + 2, + 5 + ] + }, + { + "_id": 3, + "array": [ + 1, + 2, + 5 + ] + }, + { + "_id": 4, + "array": [ + 1, + 2, + 6 + ] + } + ] + } + ] + }, + { + "description": "client bulkWrite update with hint", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "updateOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "update": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "hint": "_id_" + } + }, + { + "updateMany": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$set": { + "array": [ + 1, + 2, + 5 + ] + } + }, + "hint": "_id_" + } + }, + { + "replaceOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 4 + }, + "replacement": { + "array": [ + 1, + 2, + 6 + ] + }, + "hint": "_id_" + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 4, + "modifiedCount": 4, + "deletedCount": 0, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "hint": "_id_", + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$set": { + "array": [ + 1, + 2, + 5 + ] + } + }, + "hint": "_id_", + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "array": [ + 1, + 2, + 6 + ] + }, + "hint": "_id_", + "multi": false + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "array": [ + 1, + 2, + 4 + ] + }, + { + "_id": 2, + "array": [ + 1, + 2, + 5 + ] + }, + { + "_id": 3, + "array": [ + 1, + 2, + 5 + ] + }, + { + "_id": 4, + "array": [ + 1, + 2, + 6 + ] + } + ] + } + ] + }, + { + "description": "client bulkWrite update with upsert", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "updateOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 5 + }, + "update": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "upsert": true + } + }, + { + "replaceOne": { + "namespace": { + "db": "crud-tests", + "coll": "coll0" + }, + "filter": { + "_id": 6 + }, + "replacement": { + "array": [ + 1, + 2, + 6 + ] + }, + "upsert": true + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 2, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1, + "ops": [ + { + "update": 0, + "filter": { + "_id": 5 + }, + "updateMods": { + "$set": { + "array": [ + 1, + 2, + 4 + ] + } + }, + "upsert": true, + "multi": false + }, + { + "update": 0, + "filter": { + "_id": 6 + }, + "updateMods": { + "array": [ + 1, + 2, + 6 + ] + }, + "upsert": true, + "multi": false + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 2, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 3, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 4, + "array": [ + 1, + 2, + 3 + ] + }, + { + "_id": 5, + "array": [ + 1, + 2, + 4 + ] + }, + { + "_id": 6, + "array": [ + 1, + 2, + 6 + ] + } + ] + } + ] + } + ] +} diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml new file mode 100644 index 0000000000..db04e87d4a --- /dev/null +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml @@ -0,0 +1,288 @@ +description: "client bulkWrite update options" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, array: [ 1, 2, 3 ] } + - { _id: 2, array: [ 1, 2, 3 ] } + - { _id: 3, array: [ 1, 2, 3 ] } + - { _id: 4, array: [ 1, 2, 3 ] } + +tests: + - description: "client bulkWrite update with arrayFilters" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - updateOne: + namespace: &namespace + db: *database0Name + coll: *collection0Name + filter: &updateOneFilter + _id: 1 + update: &arrayFiltersUpdateOneMods + $set: + array.$[i]: 4 + arrayFilters: &updateOneArrayFilters + - i: { $gte: 2 } + - updateMany: + namespace: *namespace + filter: &updateManyFilter + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + update: &arrayFiltersUpdateManyMods + $set: + array.$[i]: 5 + arrayFilters: &updateManyArrayFilters + - i: { $gte: 2 } + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 3 + modifiedCount: 3 + deletedCount: 0 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + ops: + - update: 0 + filter: *updateOneFilter + updateMods: *arrayFiltersUpdateOneMods + arrayFilters: *updateOneArrayFilters + multi: false + - update: 0 + filter: *updateManyFilter + updateMods: *arrayFiltersUpdateManyMods + arrayFilters: *updateManyArrayFilters + multi: true + outcome: + - databaseName: *database0Name + collectionName: *collection0Name + documents: + - { _id: 1, array: [ 1, 4, 4 ] } + - { _id: 2, array: [ 1, 5, 5 ] } + - { _id: 3, array: [ 1, 5, 5 ] } + - { _id: 4, array: [ 1, 2, 3 ] } + - description: "client bulkWrite update with collation" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - updateOne: + namespace: *namespace + filter: *updateOneFilter + update: &updateOneMods + $set: + array: [ 1, 2, 4 ] + collation: &collation + locale: "simple" + - updateMany: + namespace: *namespace + filter: *updateManyFilter + update: &updateManyMods + $set: + array: [ 1, 2, 5 ] + collation: *collation + - replaceOne: + namespace: *namespace + filter: &replaceOneFilter + _id: 4 + replacement: &replacement + array: [ 1, 2, 6 ] + collation: *collation + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 4 + modifiedCount: 4 + deletedCount: 0 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + ops: + - update: 0 + filter: *updateOneFilter + updateMods: *updateOneMods + collation: *collation + multi: false + - update: 0 + filter: *updateManyFilter + updateMods: *updateManyMods + collation: *collation + multi: true + - update: 0 + filter: *replaceOneFilter + updateMods: *replacement + collation: *collation + multi: false + outcome: + - databaseName: *database0Name + collectionName: *collection0Name + documents: + - { _id: 1, array: [ 1, 2, 4 ] } + - { _id: 2, array: [ 1, 2, 5 ] } + - { _id: 3, array: [ 1, 2, 5 ] } + - { _id: 4, array: [ 1, 2, 6 ] } + - description: "client bulkWrite update with hint" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - updateOne: + namespace: *namespace + filter: *updateOneFilter + update: *updateOneMods + hint: &hint _id_ + - updateMany: + namespace: *namespace + filter: *updateManyFilter + update: *updateManyMods + hint: *hint + - replaceOne: + namespace: *namespace + filter: *replaceOneFilter + replacement: *replacement + hint: *hint + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 4 + modifiedCount: 4 + deletedCount: 0 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + ops: + - update: 0 + filter: *updateOneFilter + updateMods: *updateOneMods + hint: *hint + multi: false + - update: 0 + filter: *updateManyFilter + updateMods: *updateManyMods + hint: *hint + multi: true + - update: 0 + filter: *replaceOneFilter + updateMods: *replacement + hint: *hint + multi: false + outcome: + - databaseName: *database0Name + collectionName: *collection0Name + documents: + - { _id: 1, array: [ 1, 2, 4 ] } + - { _id: 2, array: [ 1, 2, 5 ] } + - { _id: 3, array: [ 1, 2, 5 ] } + - { _id: 4, array: [ 1, 2, 6 ] } + - description: "client bulkWrite update with upsert" + runOnRequirements: + - minServerVersion: "8.0" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - updateOne: + namespace: *namespace + filter: &updateOneFilter + _id: 5 + update: *updateOneMods + upsert: true + - replaceOne: + namespace: *namespace + filter: &replaceOneFilter + _id: 6 + replacement: *replacement + upsert: true + expectResult: + insertedCount: 0 + upsertedCount: 2 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + ops: + - update: 0 + filter: *updateOneFilter + updateMods: *updateOneMods + upsert: true + multi: false + - update: 0 + filter: *replaceOneFilter + updateMods: *replacement + upsert: true + multi: false + outcome: + - databaseName: *database0Name + collectionName: *collection0Name + documents: + - { _id: 1, array: [ 1, 2, 3 ] } + - { _id: 2, array: [ 1, 2, 3 ] } + - { _id: 3, array: [ 1, 2, 3 ] } + - { _id: 4, array: [ 1, 2, 3 ] } + - { _id: 5, array: [ 1, 2, 4 ] } + - { _id: 6, array: [ 1, 2, 6 ] } diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json new file mode 100644 index 0000000000..98ffa82b45 --- /dev/null +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json @@ -0,0 +1,430 @@ +{ + "description": "client bulkWrite retryable writes", + "schemaVersion": "1.18", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "topologies": [ + "replicaset" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "client bulkWrite with no multi: true operations succeeds after retryable top-level error", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 189, + "errorLabels": [ + "RetryableWriteError" + ] + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "document": { + "_id": 4, + "x": 44 + } + } + }, + { + "updateOne": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "replaceOne": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "filter": { + "_id": 2 + }, + "replacement": { + "x": 222 + } + } + }, + { + "deleteOne": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "filter": { + "_id": 3 + } + } + } + ] + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 2, + "modifiedCount": 2, + "deletedCount": 1, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1 + } + } + }, + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1 + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 222 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ] + }, + { + "description": "client bulkWrite with multi: true operations fails after retryable top-level error", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 189, + "errorLabels": [ + "RetryableWriteError" + ] + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "document": { + "_id": 4, + "x": 44 + } + } + }, + { + "updateOne": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "replaceOne": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "filter": { + "_id": 2 + }, + "replacement": { + "x": 222 + } + } + }, + { + "deleteOne": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "filter": { + "_id": 3 + } + } + } + ] + }, + "expectError": { + "errorCode": 189, + "errorLabelsContain": [ + "RetryableWriteError" + ] + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 2, + "modifiedCount": 2, + "deletedCount": 1, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1 + } + } + } + ] + } + ] + }, + { + "description": "client bulkWrite with multi: true operations fails after retryable writeConcernError", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableWriteError" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "requests": [ + { + "updateMany": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "deleteMany": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "filter": { + "_id": 3 + } + } + } + ] + }, + "expectError": { + "writeConcernErrors": [ + { + "code": 91, + "errmsg": "Replication is being shut down" + } + ], + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 1, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "bulkWrite": 1 + } + } + } + ] + } + ] + } + ] +} diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml new file mode 100644 index 0000000000..edaefb109d --- /dev/null +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml @@ -0,0 +1,211 @@ +description: "client bulkWrite retryable writes" +schemaVersion: "1.18" +runOnRequirements: + - minServerVersion: "8.0" + topologies: [ replicaset ] + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name retryable-writes-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + +tests: + - description: "client bulkWrite with no multi: true operations succeeds after retryable top-level error" + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: [ bulkWrite ] + errorCode: 189 # PrimarySteppedDown + errorLabels: [ RetryableWriteError ] + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - insertOne: + namespace: + db: *database0Name + coll: *collection0Name + document: { _id: 4, x: 44 } + - updateOne: + namespace: + db: *database0Name + coll: *collection0Name + filter: { _id: 1 } + update: + $inc: { x: 1 } + - replaceOne: + namespace: + db: *database0Name + coll: *collection0Name + filter: { _id: 2 } + replacement: { x: 222 } + - deleteOne: + namespace: + db: *database0Name + coll: *collection0Name + filter: { _id: 3 } + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 2 + modifiedCount: 2 + deletedCount: 1 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + - commandStartedEvent: + command: + bulkWrite: 1 + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 12 } + - { _id: 2, x: 222 } + - { _id: 4, x: 44 } + - description: "client bulkWrite with multi: true operations fails after retryable top-level error" + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: [ bulkWrite ] + errorCode: 189 # PrimarySteppedDown + errorLabels: [ RetryableWriteError ] + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - insertOne: + namespace: + db: *database0Name + coll: *collection0Name + document: { _id: 4, x: 44 } + - updateOne: + namespace: + db: *database0Name + coll: *collection0Name + filter: { _id: 1 } + update: + $inc: { x: 1 } + - replaceOne: + namespace: + db: *database0Name + coll: *collection0Name + filter: { _id: 2 } + replacement: { x: 222 } + - deleteOne: + namespace: + db: *database0Name + coll: *collection0Name + filter: { _id: 3 } + expectError: + errorCode: 189 + errorLabelsContain: [ RetryableWriteError ] + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 2 + modifiedCount: 2 + deletedCount: 1 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 + - description: "client bulkWrite with multi: true operations fails after retryable writeConcernError" + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: [ bulkWrite ] + errorLabels: [ RetryableWriteError ] + writeConcernError: &writeConcernError + code: 91 + errmsg: "Replication is being shut down" + - object: *client0 + name: clientBulkWrite + arguments: + requests: + - updateMany: + namespace: + db: *database0Name + coll: *collection0Name + filter: { _id: 1 } + update: + $inc: { x: 1 } + - deleteMany: + namespace: + db: *database0Name + coll: *collection0Name + filter: { _id: 3 } + expectError: + writeConcernErrors: [ *writeConcernError ] + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 1 + modifiedCount: 1 + deletedCount: 1 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + bulkWrite: 1 From 0ede8aec3300627f8e8845c12a1c41f9294954ac Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 14 Feb 2024 16:17:45 -0700 Subject: [PATCH 02/90] retryability --- source/retryable-writes/retryable-writes.rst | 15 +- .../client-bulkWrite-serverErrors.json | 360 ++++++++++++++++-- .../unified/client-bulkWrite-serverErrors.yml | 177 +++++++-- 3 files changed, 490 insertions(+), 62 deletions(-) diff --git a/source/retryable-writes/retryable-writes.rst b/source/retryable-writes/retryable-writes.rst index 9cc13763b9..bff95753c8 100644 --- a/source/retryable-writes/retryable-writes.rst +++ b/source/retryable-writes/retryable-writes.rst @@ -143,14 +143,15 @@ Supported single-statement write operations include ``insertOne()``, ``updateOne()``, ``replaceOne()``, ``deleteOne()``, ``findOneAndDelete()``, ``findOneAndReplace()``, and ``findOneAndUpdate()``. + Supported multi-statement write operations include ``insertMany()`` and -``bulkWrite()``. The ordered option may be ``true`` or ``false``. In the case of -``bulkWrite()``, ``UpdateMany`` or ``DeleteMany`` operations within the -``requests`` parameter may make some write commands ineligible for retryability. -Drivers MUST evaluate eligibility for each write command sent as part of the -``bulkWrite()`` (after order and batch splitting) individually. Drivers MUST NOT -alter existing logic for order and batch splitting in an attempt to maximize -retryability for operations within a bulk write. +``bulkWrite()``. The ordered option may be ``true`` or ``false``. For both the +collection-level and client-level ``bulkWrite()`` methods, a bulk write batch is +only retryable if it does not contain any ``multi: true`` writes (i.e. +``UpdateMany`` and ``DeleteMany``). Drivers MUST evaluate eligibility for each +write command sent as part of the ``bulkWrite()`` (after order and batch splitting) +individually. Drivers MUST NOT alter existing logic for order and batch splitting in +an attempt to maximize retryability for operations within a bulk write. These methods above are defined in the `CRUD`_ specification. diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json index 98ffa82b45..8d3324b594 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json @@ -83,7 +83,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { @@ -163,15 +163,103 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { - "bulkWrite": 1 + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "_id": 2 + }, + "updateMods": { + "x": 222 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ] } } }, { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { - "bulkWrite": 1 + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "_id": 2 + }, + "updateMods": { + "x": 222 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ] } } } @@ -228,7 +316,119 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ + { + "updateMany": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "deleteMany": { + "namespace": { + "db": "retryable-writes-tests", + "coll": "coll0" + }, + "filter": { + "_id": 3 + } + } + } + ] + }, + "expectError": { + "errorCode": 189, + "errorLabelsContain": [ + "RetryableWriteError" + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": true + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ] + } + } + } + ] + } + ] + }, + { + "description": "client bulkWrite with no multi: true operations succeeds after retryable writeConcernError", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableWriteError" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ { "insertOne": { "namespace": { @@ -284,12 +484,6 @@ } ] }, - "expectError": { - "errorCode": 189, - "errorLabelsContain": [ - "RetryableWriteError" - ] - }, "expectResult": { "insertedCount": 1, "upsertedCount": 0, @@ -314,8 +508,103 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { - "bulkWrite": 1 + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "_id": 2 + }, + "updateMods": { + "x": 222 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ] + } + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "_id": 2 + }, + "updateMods": { + "x": 222 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ] } } } @@ -355,7 +644,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "updateMany": { "namespace": { @@ -391,23 +680,7 @@ "code": 91, "errmsg": "Replication is being shut down" } - ], - "expectResult": { - "insertedCount": 0, - "upsertedCount": 0, - "matchedCount": 1, - "modifiedCount": 1, - "deletedCount": 1, - "insertResults": { - "$$unsetOrMatches": {} - }, - "updateResults": { - "$$unsetOrMatches": {} - }, - "deleteResults": { - "$$unsetOrMatches": {} - } - } + ] } } ], @@ -417,8 +690,35 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { - "bulkWrite": 1 + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": true + }, + { + "delete": 0, + "filter": { + "_id": 3 + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ] } } } diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml index edaefb109d..a882a8f6c7 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml @@ -43,7 +43,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - insertOne: namespace: db: *database0Name @@ -83,11 +83,47 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: - bulkWrite: 1 + ops: + - insert: 0 + document: { _id: 4, x: 44 } + - update: 0 + filter: { _id: 1 } + updateMods: + $inc: { x: 1 } + multi: false + - update: 0 + filter: { _id: 2 } + updateMods: { x: 222 } + multi: false + - delete: 0 + filter: { _id: 3 } + multi: false + nsInfo: + - ns: retryable-writes-tests.coll0 - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: - bulkWrite: 1 + ops: + - insert: 0 + document: { _id: 4, x: 44 } + - update: 0 + filter: { _id: 1 } + updateMods: + $inc: { x: 1 } + multi: false + - update: 0 + filter: { _id: 2 } + updateMods: { x: 222 } + multi: false + - delete: 0 + filter: { _id: 3 } + multi: false + nsInfo: + - ns: retryable-writes-tests.coll0 outcome: - collectionName: *collection0Name databaseName: *database0Name @@ -112,11 +148,64 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: + - updateMany: + namespace: + db: *database0Name + coll: *collection0Name + filter: { _id: 1 } + update: + $inc: { x: 1 } + - deleteMany: + namespace: + db: *database0Name + coll: *collection0Name + filter: { _id: 3 } + expectError: + errorCode: 189 + errorLabelsContain: [ RetryableWriteError ] + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + ops: + - update: 0 + filter: { _id: 1 } + updateMods: + $inc: { x: 1 } + multi: true + - delete: 0 + filter: { _id: 3 } + multi: true + nsInfo: + - ns: retryable-writes-tests.coll0 + - description: "client bulkWrite with no multi: true operations succeeds after retryable writeConcernError" + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: [ bulkWrite ] + errorLabels: [ RetryableWriteError ] + writeConcernError: + code: 91 + errmsg: "Replication is being shut down" + - object: *client0 + name: clientBulkWrite + arguments: + models: - insertOne: namespace: db: *database0Name - coll: *collection0Name + coll: *collection0Name document: { _id: 4, x: 44 } - updateOne: namespace: @@ -136,9 +225,6 @@ tests: db: *database0Name coll: *collection0Name filter: { _id: 3 } - expectError: - errorCode: 189 - errorLabelsContain: [ RetryableWriteError ] expectResult: insertedCount: 1 upsertedCount: 0 @@ -155,8 +241,47 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + ops: + - insert: 0 + document: { _id: 4, x: 44 } + - update: 0 + filter: { _id: 1 } + updateMods: + $inc: { x: 1 } + multi: false + - update: 0 + filter: { _id: 2 } + updateMods: { x: 222 } + multi: false + - delete: 0 + filter: { _id: 3 } + multi: false + nsInfo: + - ns: retryable-writes-tests.coll0 + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: - bulkWrite: 1 + ops: + - insert: 0 + document: { _id: 4, x: 44 } + - update: 0 + filter: { _id: 1 } + updateMods: + $inc: { x: 1 } + multi: false + - update: 0 + filter: { _id: 2 } + updateMods: { x: 222 } + multi: false + - delete: 0 + filter: { _id: 3 } + multi: false + nsInfo: + - ns: retryable-writes-tests.coll0 - description: "client bulkWrite with multi: true operations fails after retryable writeConcernError" operations: - object: testRunner @@ -170,13 +295,13 @@ tests: data: failCommands: [ bulkWrite ] errorLabels: [ RetryableWriteError ] - writeConcernError: &writeConcernError + writeConcernError: code: 91 errmsg: "Replication is being shut down" - object: *client0 name: clientBulkWrite arguments: - requests: + models: - updateMany: namespace: db: *database0Name @@ -190,22 +315,24 @@ tests: coll: *collection0Name filter: { _id: 3 } expectError: - writeConcernErrors: [ *writeConcernError ] - expectResult: - insertedCount: 0 - upsertedCount: 0 - matchedCount: 1 - modifiedCount: 1 - deletedCount: 1 - insertResults: - $$unsetOrMatches: {} - updateResults: - $$unsetOrMatches: {} - deleteResults: - $$unsetOrMatches: {} + writeConcernErrors: + - code: 91 + errmsg: "Replication is being shut down" expectEvents: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: - bulkWrite: 1 + ops: + - update: 0 + filter: { _id: 1 } + updateMods: + $inc: { x: 1 } + multi: true + - delete: 0 + filter: { _id: 3 } + multi: true + nsInfo: + - ns: retryable-writes-tests.coll0 From afb1ab6f66f4f508067834e358850190c93c530f Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 14 Feb 2024 16:21:44 -0700 Subject: [PATCH 03/90] changelog --- source/retryable-writes/retryable-writes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/source/retryable-writes/retryable-writes.rst b/source/retryable-writes/retryable-writes.rst index bff95753c8..0a79127e86 100644 --- a/source/retryable-writes/retryable-writes.rst +++ b/source/retryable-writes/retryable-writes.rst @@ -862,6 +862,7 @@ inconsistent with the server and potentially confusing to developers. Changelog ========= +:2024-02-14: Add guidance for client-level ``bulkWrite()`` retryability. :2024-01-16: Do not use ``writeConcernError.code`` in pre-4.4 mongos response to determine retryability. Do not use ``writeErrors[].code`` in pre-4.4 server responses to determine retryability. From e2cb197742fd727bbf33d1bf58ba8e2a39d473d4 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 16 Feb 2024 11:59:52 -0700 Subject: [PATCH 04/90] transactions --- source/crud/bulk-write.md | 22 +- .../client-bulkWrite-delete-options.json | 4 +- .../client-bulkWrite-delete-options.yml | 4 +- .../client-bulkWrite-errors.json | 15 +- .../client-bulkWrite-errors.yml | 15 +- .../client-bulkWrite-mixed-namespaces.json | 2 +- .../client-bulkWrite-mixed-namespaces.yml | 2 +- .../client-bulkWrite-options.json | 6 +- .../client-bulkWrite-options.yml | 6 +- .../client-bulkWrite-ordered.json | 6 +- .../client-bulkWrite-ordered.yml | 6 +- .../client-bulkWrite-results.json | 6 +- .../client-bulkWrite-results.yml | 6 +- .../client-bulkWrite-update-options.json | 8 +- .../client-bulkWrite-update-options.yml | 8 +- .../tests/unified/client-bulkWrite.json | 401 ++++++++++++++++++ .../tests/unified/client-bulkWrite.yml | 177 ++++++++ source/transactions/transactions.md | 1 + 18 files changed, 635 insertions(+), 60 deletions(-) create mode 100644 source/transactions/tests/unified/client-bulkWrite.json create mode 100644 source/transactions/tests/unified/client-bulkWrite.yml diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 32e957a679..e8f7c8d3e2 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -93,7 +93,7 @@ The `nsInfo` field contains the namespaces on which the write operations should ##### `errorsOnly` -This field corresponds to the `verboseResults` option defined on `BulkWriteOptions`. Its value should be specified as the opposite of `verboseResults`, or `true` if `verboseResults` is unspecified. Drivers MUST always define this field. +This field corresponds to the `verboseResults` option defined on `BulkWriteOptions`. Its value should be `false` if `verboseResults` was set to `true`; otherwise, it should be `true`. This field MUST always be defined in the `bulkWrite` command regardless of whether a value for `verboseResults` was specified. #### Response @@ -222,7 +222,11 @@ enum WriteModel { ##### Namespaces -Each write model supplied to `Client.bulkWrite` in the `models` parameter MUST have a corresponding namespace that defines the collection on which the operation should be performed. Drivers SHOULD design this pairing in whichever way is most ergonomic for its language. For example, drivers may include a required `namespace` field in each `WriteModel` variant, accept a list of `(Namespace, WriteModel)` tuples for `models`, or define the following pair class: +Each write model supplied to `Client.bulkWrite` in the `models` parameter MUST have a corresponding namespace that defines the collection on which the operation should be performed. Drivers SHOULD design this pairing in whichever way is most ergonomic for its language. For example, drivers may: + +- include a required `namespace` field in each `WriteModel` variant +- accept a list of `(Namespace, WriteModel)` tuples for `models` +- define the following pair class: ``` @@ -359,9 +363,9 @@ class BulkWriteException { ##### Top-level errors -A top-level error is defined as any error that occurs during a bulk write that is not the result of an individual write operation failing or a write concern error. Possible top-level errors include, but are not limited to, network errors that occur when communicating with the server, command errors returned from the server, and client-side errors (e.g. an oversized insert document was provided). When a top-level error occurs, drivers MUST embed that error within a `BulkWriteException` as the `error` field and MUST include any results and/or non-top-level errors that have been observed so far in the `writeConcernErrors`, `writeErrors`, and `partialResults` fields. +A top-level error is defined as any error that occurs during a bulk write that is not the result of an individual write operation failing or a write concern error. Possible top-level errors include, but are not limited to, network errors that occur when communicating with the server, command errors returned from the server, and client-side errors (e.g. an oversized insert document was provided). If individual operation results or write concern errors have already been observed prior to a top-level error occurring, drivers MUST embed the top-level error within a `BulkWriteException` as the `error` field to retain this information. Otherwise, drivers MAY throw an exception containing only the top-level error. -If a retryable top-level error is encountered and the `bulkWrite` is retryable, drivers MUST perform a retry according to the retryable writes specification. Note that any bulk write that contains a `multi: true` operation (i.e. `UpdateMany` or `DeleteMany`) is not retryable. If the retry fails or if the bulk write or error was not retryable, drivers MUST halt execution for both ordered and unordered bulk writes and immediately throw a `BulkWriteException`. +A top-level MUST halt execution of a bulk write for both ordered and unordered bulk writes. When a top-level error is encountered, drivers MUST NOT attempt to retrieve responses from the cursor or execute any further `bulkWrite` batches. ##### Write concern errors @@ -369,11 +373,11 @@ At most one write concern error may be returned per `bulkWrite` command response ##### Individual write errors -Failures of individual write operations are reported in the cursor of results returned from the server. Drivers MUST iterate the this cursor fully and record all errors in the `writeErrors` map. If an individual write error is encountered during an ordered bulk write, drivers MUST record the error in `writeErrors` and immediately throw a `BulkWriteException`. Otherwise, drivers MUST continue iterating the cursor and execute any further `bulkWrite` batches. +Failures of individual write operations are reported in the cursor of results returned from the server. Drivers MUST iterate the this cursor fully and record all errors in the `writeErrors` map in `BulkWriteException`. If an individual write error is encountered during an ordered bulk write, drivers MUST record the error in `writeErrors` and immediately throw a `BulkWriteException`. Otherwise, drivers MUST continue iterating the cursor and execute any further `bulkWrite` batches. ### Command Batching -Drivers MUST accept an arbitrary number of operations as input to the `Client.bulkWrite` method. Because the server imposes restrictions on the size of write operations, this means that a single call to `Client.bulkWrite` may require multiple `bulkWrite` commands to be sent to the server. Drivers MUST split bulk writes into separate commands when the user's list of operations exceeds one of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize`, or `maxMessageSizeBytes`. Each of these values can be retrieved from the selected server's `hello` command response. +Drivers MUST accept an arbitrary number of operations as input to the `Client.bulkWrite` method. Because the server imposes restrictions on the size of write operations, this means that a single call to `MongoClient.bulkWrite` may require multiple `bulkWrite` commands to be sent to the server. Drivers MUST split bulk writes into separate commands when the user's list of operations exceeds one of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize`, or `maxMessageSizeBytes`. Each of these values can be retrieved from the selected server's `hello` command response. #### Number of Writes @@ -391,7 +395,7 @@ When auto-encryption is enabled, drivers MUST NOT provide the `ops` and `nsInfo` ##### Unencrypted bulk writes with document sequences -When `ops` and `nsInfo` are provided as document sequences, drivers MUST ensure that the combined size of these sequences does not exceed `maxMessageSizeBytes - 16,000`. `maxMessageSizeBytes` defines the maximum size for an entire `OP_MSG`, so 16KB is subtracted from `maxMessageSizeBytes` here to account for the bytes in the rest of the message. +When `ops` and `nsInfo` are provided as document sequences, drivers MUST ensure that the combined size of these sequences does not exceed `maxMessageSizeBytes - 16,000`. `maxMessageSizeBytes` defines the maximum size for an entire `OP_MSG`, so 16KB is subtracted from `maxMessageSizeBytes` in this size limit to account for the bytes in the rest of the message. ##### Unencrypted bulk writes without document sequences @@ -455,7 +459,3 @@ Returning results via a cursor rather than an array in the `bulkWrite` response #### Why was the `verboseResults` option introduced, and why is its default `false`? The `bulkWrite` command returns top-level summary result counts and, optionally, individual results for each operation. Compiling the individual results server-side and consuming these results driver-side is less performant than only recording the summary counts. We expect that most users are not interested in the individual results of their operations and that most users will rely on defaults, so `verboseResults` defaults to `false` to improve performance in the common case. - -#### Why must all errors be embedded within a `BulkWriteException`? - -An error may occur at any point during the execution of the `MongoClient.bulkWrite` method, including after some individual writes have already been executed. Embedding errors in a `BulkWriteException` gives users more information about what had happened during their bulk write prior to the error occurring. diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json index 43ed517538..77a67625fb 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json @@ -58,7 +58,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "deleteOne": { "namespace": { @@ -167,7 +167,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "deleteOne": { "namespace": { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml index c2ed455d34..f69694e2cc 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml @@ -31,7 +31,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - deleteOne: namespace: &namespace db: *database0Name @@ -83,7 +83,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - deleteOne: namespace: &namespace db: *database0Name diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json index 9232516dd7..10ddb7a5d6 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json @@ -61,7 +61,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "deleteOne": { "namespace": { @@ -155,7 +155,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "deleteOne": { "namespace": { @@ -249,7 +249,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "deleteOne": { "namespace": { @@ -343,7 +343,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { @@ -359,10 +359,7 @@ "verboseResults": true }, "expectError": { - "errorCode": 8, - "errorLabelsContain": [ - "RetryableWriteError" - ] + "errorCode": 8 } } ] @@ -396,7 +393,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml index beae51f1d9..fde7a6e141 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml @@ -33,7 +33,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - deleteOne: namespace: &namespace db: *database0Name @@ -76,7 +76,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - deleteOne: namespace: *namespace filter: { _id: 1 } @@ -119,7 +119,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - deleteOne: namespace: *namespace filter: { _id: 1 } @@ -161,18 +161,17 @@ tests: data: failCommands: - bulkWrite - errorCode: &errorCode 8 # UnknownError + errorCode: 8 # UnknownError - object: *client0 name: clientBulkWrite arguments: - requests: + models: - insertOne: namespace: *namespace document: { x: 1 } verboseResults: true expectError: - errorCode: *errorCode - errorLabelsContain: [ RetryableWriteError ] + errorCode: 8 - description: "a write concern error occurs during a bulkWrite" operations: - name: failPoint @@ -192,7 +191,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - insertOne: namespace: *namespace document: { _id: 10 } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json index 3bd65278d2..f408a8969f 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json @@ -94,7 +94,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml index 36c1cd754c..cc7f3108ed 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml @@ -50,7 +50,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - insertOne: namespace: db: *database0Name diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json index f9af9ae5e3..80e5fef102 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json @@ -54,7 +54,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { @@ -139,7 +139,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { @@ -220,7 +220,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "updateOne": { "namespace": { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml index 0eb1fbfb8a..23dacd9077 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml @@ -30,7 +30,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - insertOne: namespace: &namespace db: *database0Name @@ -70,7 +70,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - insertOne: namespace: *namespace document: &doc4 { _id: 4, x: 44 } @@ -108,7 +108,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - updateOne: namespace: *namespace filter: diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json index b477942b16..546a980b2b 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json @@ -45,7 +45,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { @@ -116,7 +116,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { @@ -187,7 +187,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml index c2792a4364..d09f3d4567 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml @@ -28,7 +28,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - insertOne: namespace: db: *database0Name @@ -66,7 +66,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - insertOne: namespace: db: *database0Name @@ -104,7 +104,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - insertOne: namespace: db: *database0Name diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json index 5a25454942..489b581d06 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json @@ -70,7 +70,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { @@ -357,7 +357,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { @@ -623,7 +623,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "insertOne": { "namespace": { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml index 84ded30f59..9874430ce9 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml @@ -34,7 +34,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: &requests + models: &models - insertOne: namespace: &namespace db: *database0Name @@ -150,7 +150,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: *requests + models: *models verboseResults: false expectResult: &summaryResult insertedCount: 1 @@ -184,7 +184,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: *requests + models: *models expectResult: *summaryResult expectEvents: - client: *client0 diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json index ce829d746d..5b442f3005 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json @@ -78,7 +78,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "updateOne": { "namespace": { @@ -274,7 +274,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "updateOne": { "namespace": { @@ -508,7 +508,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "updateOne": { "namespace": { @@ -730,7 +730,7 @@ "object": "client0", "name": "clientBulkWrite", "arguments": { - "requests": [ + "models": [ { "updateOne": { "namespace": { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml index db04e87d4a..d7f6874ac2 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml @@ -32,7 +32,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - updateOne: namespace: &namespace db: *database0Name @@ -97,7 +97,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - updateOne: namespace: *namespace filter: *updateOneFilter @@ -169,7 +169,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - updateOne: namespace: *namespace filter: *updateOneFilter @@ -234,7 +234,7 @@ tests: - object: *client0 name: clientBulkWrite arguments: - requests: + models: - updateOne: namespace: *namespace filter: &updateOneFilter diff --git a/source/transactions/tests/unified/client-bulkWrite.json b/source/transactions/tests/unified/client-bulkWrite.json new file mode 100644 index 0000000000..74e99f77af --- /dev/null +++ b/source/transactions/tests/unified/client-bulkWrite.json @@ -0,0 +1,401 @@ +{ + "description": "client bulkWrite transactions", + "schemaVersion": "1.18", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "databaseName": "transaction-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 5, + "x": 55 + }, + { + "_id": 6, + "x": 66 + }, + { + "_id": 7, + "x": 77 + } + ] + } + ], + "tests": [ + { + "description": "transactional client bulkWrite", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "session": "session0", + "models": [ + { + "insertOne": { + "namespace": { + "db": "transaction-tests", + "coll": "coll0" + }, + "document": { + "_id": 8, + "x": 88 + } + } + }, + { + "updateOne": { + "namespace": { + "db": "transaction-tests", + "coll": "coll0" + }, + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "updateMany": { + "namespace": { + "db": "transaction-tests", + "coll": "coll0" + }, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "update": { + "$inc": { + "x": 2 + } + } + } + }, + { + "replaceOne": { + "namespace": { + "db": "transaction-tests", + "coll": "coll0" + }, + "filter": { + "_id": 4 + }, + "replacement": { + "x": 44 + }, + "upsert": true + } + }, + { + "deleteOne": { + "namespace": { + "db": "transaction-tests", + "coll": "coll0" + }, + "filter": { + "_id": 5 + } + } + }, + { + "deleteMany": { + "namespace": { + "db": "transaction-tests", + "coll": "coll0" + }, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 1, + "matchedCount": 3, + "modifiedCount": 3, + "deletedCount": 3, + "insertResults": { + "0": { + "insertedId": 8 + } + }, + "updateResults": { + "1": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + }, + "2": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": null + }, + "3": { + "matchedCount": 1, + "modifiedCount": 0, + "upsertedId": 4 + } + }, + "deleteResults": { + "4": { + "deletedCount": 1 + }, + "5": { + "deletedCount": 2 + } + } + } + }, + { + "object": "session0", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + }, + "ops": [ + { + "insert": 0, + "document": { + "_id": 8, + "x": 88 + } + }, + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "update": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 1 + } + }, + { + "_id": { + "$lte": 3 + } + } + ] + }, + "updateMods": { + "$inc": { + "x": 2 + } + }, + "multi": true + }, + { + "update": 0, + "filter": { + "_id": 4 + }, + "updateMods": { + "x": 44 + }, + "upsert": true, + "multi": false + }, + { + "delete": 0, + "filter": { + "_id": 5 + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "$and": [ + { + "_id": { + "$gt": 5 + } + }, + { + "_id": { + "$lte": 7 + } + } + ] + }, + "multi": true + } + ], + "nsInfo": [ + { + "ns": "transaction-tests.coll0" + } + ], + "errorsOnly": false, + "ordered": true + } + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction", + "databaseName": "admin", + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 24 + }, + { + "_id": 3, + "x": 35 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 8, + "x": 88 + } + ] + } + ] + } + ] +} diff --git a/source/transactions/tests/unified/client-bulkWrite.yml b/source/transactions/tests/unified/client-bulkWrite.yml new file mode 100644 index 0000000000..de7a80d67c --- /dev/null +++ b/source/transactions/tests/unified/client-bulkWrite.yml @@ -0,0 +1,177 @@ +description: "client bulkWrite transactions" +schemaVersion: "1.18" +runOnRequirements: + - minServerVersion: "8.0" + topologies: + - replicaset + - sharded + - load-balanced + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name transaction-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + - session: + id: &session0 session0 + client: *client0 + +initialData: + - databaseName: *database0Name + collectionName: *collection0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 5, x: 55 } + - { _id: 6, x: 66 } + - { _id: 7, x: 77 } + +tests: + - description: "transactional client bulkWrite" + operations: + - object: *session0 + name: startTransaction + - object: *client0 + name: clientBulkWrite + arguments: + session: *session0 + models: + - insertOne: + namespace: + db: *database0Name + coll: *collection0Name + document: &insertDoc + { _id: 8, x: 88 } + - updateOne: + namespace: + db: *database0Name + coll: *collection0Name + filter: &updateOneFilter + _id: 1 + update: &updateOneUpdate + $inc: { x: 1 } + - updateMany: + namespace: + db: *database0Name + coll: *collection0Name + filter: &updateManyFilter + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + update: &updateManyUpdate + $inc: { x: 2 } + - replaceOne: + namespace: + db: *database0Name + coll: *collection0Name + filter: &replaceOneFilter + _id: 4 + replacement: &replacementDoc + { x: 44 } + upsert: true + - deleteOne: + namespace: + db: *database0Name + coll: *collection0Name + filter: &deleteOneFilter + _id: 5 + - deleteMany: + namespace: + db: *database0Name + coll: *collection0Name + filter: &deleteManyFilter + $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] + verboseResults: true + expectResult: + insertedCount: 1 + upsertedCount: 1 + matchedCount: 3 + modifiedCount: 3 + deletedCount: 3 + insertResults: + 0: + insertedId: 8 + updateResults: + 1: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + 2: + matchedCount: 2 + modifiedCount: 2 + upsertedId: null + 3: + matchedCount: 1 + modifiedCount: 0 + upsertedId: 4 + deleteResults: + 4: + deletedCount: 1 + 5: + deletedCount: 2 + - object: *session0 + name: commitTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: true + autocommit: false + writeConcern: { $$exists: false } + ops: + - insert: 0 + document: *insertDoc + - update: 0 + filter: *updateOneFilter + updateMods: *updateOneUpdate + multi: false + - update: 0 + filter: *updateManyFilter + updateMods: *updateManyUpdate + multi: true + - update: 0 + filter: *replaceOneFilter + updateMods: *replacementDoc + upsert: true + multi: false + - delete: 0 + filter: *deleteOneFilter + multi: false + - delete: 0 + filter: *deleteManyFilter + multi: true + nsInfo: + - ns: "transaction-tests.coll0" + errorsOnly: false + ordered: true + - commandStartedEvent: + commandName: commitTransaction + databaseName: admin + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: { $$exists: false } + autocommit: false + writeConcern: { $$exists: false } + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: &outcomeDocuments + - { _id: 1, x: 12 } + - { _id: 2, x: 24 } + - { _id: 3, x: 35 } + - { _id: 4, x: 44 } + - { _id: 8, x: 88 } diff --git a/source/transactions/transactions.md b/source/transactions/transactions.md index 171f239242..fcd528cefa 100644 --- a/source/transactions/transactions.md +++ b/source/transactions/transactions.md @@ -1009,6 +1009,7 @@ The following commands are allowed inside transactions: 10. geoSearch 11. create 12. createIndexes on an empty collection created in the same transaction or on a non-existing collection +13. bulkWrite ### Why don’t drivers automatically retry commit after a write concern timeout error? From a24c7bd5b9fafcba8f3784fcaa719d4f3a7b1a32 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 20 Feb 2024 13:10:38 -0700 Subject: [PATCH 05/90] unified test runner changes --- .../unified-test-format.md | 53 +++++++++++++++++ .../tests/crud-api-version-1.json | 58 +++++++++++++++++++ .../tests/crud-api-version-1.yml | 26 +++++++++ 3 files changed, 137 insertions(+) diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 3d4b104a92..d43dca11e8 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -923,6 +923,18 @@ The structure of this object is as follows: to have. The test runner MUST assert that the error does not contain any of the specified labels (e.g. using the `hasErrorLabel` method). +- `writeErrors`: Optional document. A map of `writeError`s expected to be present in the error. The `writeErrors` document + contains numeric keys representing the index of the write that failed and `writeError` object values. The test runner MUST + assert that the error contains a `writeError` for each index present in `writeErrors` and MUST assert that the `writeError`s + match as root-level documents according to the rules in [Evaluating Matches](#evaluating-matches). The test runner MUST assert + that the error does not contain any additional `writeError`s. + +- `writeConcernErrors`: Optional array of one or more objects. An ordered list of write concern errors expected to be + present in the error. The test runner MUST assert that each `writeConcernError` in this list matches the + `writeConcernError` present at the same index in the error's list of `writeConcernError`s as a root-level document according + to the rules in [Evaluating Matches](#evaluating-matches). The test runner MUST assert that the error does not contain any + additional `writeConcernErrors`s. +
- `errorResponse`: Optional document. A value corresponding to the expected server response. The test runner MUST assert @@ -1416,6 +1428,45 @@ the resulting change stream might be saved with [operation.saveResultAsEntity](# Test runners MUST NOT iterate the change stream when executing this operation and test files SHOULD NOT specify [operation.expectResult](#operation_expectResult) for this operation. +#### clientBulkWrite + +These considerations only apply to the `MongoClient.bulkWrite` method. See [bulkWrite](#bulkwrite) for special considerations for `MongoCollection.bulkWrite`. + +The `models` parameter for `clientBulkWrite` is documented as a list of WriteModel interfaces. Each WriteModel +implementation (e.g. InsertOneModel) provides important context to the method, but that type information is not easily +expressed in YAML and JSON. To account for this, test files MUST nest each WriteModel object in a single-key object, +where the key identifies the request type (e.g. "insertOne") and its value is an object expressing the parameters, as in +the following example: + +``` +arguments: + models: + - insertOne: + document: { _id: 1, x: 1 } + - replaceOne: + filter: { _id: 2 } + replacement: { x: 2 } + upsert: true + - updateOne: + filter: { _id: 3 } + update: { $set: { x: 3 } } + upsert: true + - updateMany: + filter: { } + update: { $inc: { x: 1 } } + - deleteOne: + filter: { x: 2 } + - deleteMany: + filter: { x: { $gt: 2 } } + ordered: true +``` + +Because the `insertResults`, `updateResults`, and `deleteResults` may be absent or empty in the `BulkWriteResult` returned from a summary-only bulk write, the `clientBulkWrite` operation MUST use the [$$unsetOrMatches](#unsetormatches) operator for assertions on these fields when `verboseResults` is not set to true. This requirement also applies to result objects defined in the `expectedResult` field of [expectedError](#expectederror). + +The `BulkWriteException` thrown by `MongoClient.bulkWrite` contains an optional `error` field that stores a top-level error that occurred during the bulk write. Test runners MUST inspect the contents of this field when making assertions based on the contents of the `errorCode` and `errorContains` fields in [expectedError](#expectederror). + +`BulkWriteException` also contains `writeErrors` and `writeConcernErrors` fields that define the individual write errors and write concern errors that occurred during the bulk write. Unified tests SHOULD use `writeErrors` and `writeConcernErrors` in `expectedError` to assert on the contents of these fields. Test runners MUST NOT inspect the contents of these fields when making assertions based on any other fields defined in `expectedError`. + #### watch This operation SHOULD NOT be used in test files. See [client_createChangeStream](#client_createChangeStream). @@ -1562,6 +1613,8 @@ behavior between drivers that eagerly create a server-side cursor and those that #### bulkWrite +These considerations only apply to the `MongoCollection.bulkWrite` method. See [clientBulkWrite](#clientbulkwrite) for special considerations for `MongoClient.bulkWrite`. + The `requests` parameter for `bulkWrite` is documented as a list of WriteModel interfaces. Each WriteModel implementation (e.g. InsertOneModel) provides important context to the method, but that type information is not easily expressed in YAML and JSON. To account for this, test files MUST nest each WriteModel object in a single-key object, diff --git a/source/versioned-api/tests/crud-api-version-1.json b/source/versioned-api/tests/crud-api-version-1.json index a387d0587e..6fda017065 100644 --- a/source/versioned-api/tests/crud-api-version-1.json +++ b/source/versioned-api/tests/crud-api-version-1.json @@ -426,6 +426,64 @@ } ] }, + { + "description": "client bulkWrite appends declared API version", + "operations": [ + { + "name": "clientBulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": { + "db": "versioned-api-tests", + "coll": "test" + }, + "document": { + "x": 1 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "ops": [ + { + "insert": 0, + "document": { + "x": 1 + } + } + ], + "nsInfo": [ + { + "ns": "versioned-api-tests.test" + } + ], + "apiVersion": "1", + "apiStrict": { + "$$unsetOrMatches": false + }, + "apiDeprecationErrors": true + } + } + } + ] + } + ] + }, { "description": "countDocuments appends declared API version", "operations": [ diff --git a/source/versioned-api/tests/crud-api-version-1.yml b/source/versioned-api/tests/crud-api-version-1.yml index 50135c1458..1373b6dc95 100644 --- a/source/versioned-api/tests/crud-api-version-1.yml +++ b/source/versioned-api/tests/crud-api-version-1.yml @@ -155,6 +155,32 @@ tests: multi: { $$unsetOrMatches: false } upsert: true <<: *expectedApiVersion + + - description: "client bulkWrite appends declared API version" + operations: + - name: clientBulkWrite + object: *client + arguments: + models: + - insertOne: + namespace: + db: *databaseName + coll: *collectionName + document: { x: 1 } + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: *adminDatabaseName + command: + bulkWrite: 1 + ops: + - insert: 0 + document: { x: 1 } + nsInfo: + - { ns: "versioned-api-tests.test" } + <<: *expectedApiVersion - description: "countDocuments appends declared API version" operations: From 5a9899442dbaa09273dc7acbe07227ca741739cb Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Mon, 26 Feb 2024 10:47:16 -0700 Subject: [PATCH 06/90] add schema --- source/unified-test-format/schema-1.20.json | 1082 +++++++++++++++++++ 1 file changed, 1082 insertions(+) create mode 100644 source/unified-test-format/schema-1.20.json diff --git a/source/unified-test-format/schema-1.20.json b/source/unified-test-format/schema-1.20.json new file mode 100644 index 0000000000..b5395a33c9 --- /dev/null +++ b/source/unified-test-format/schema-1.20.json @@ -0,0 +1,1082 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Unified Test Format", + "type": "object", + "additionalProperties": false, + "required": [ + "description", + "schemaVersion", + "tests" + ], + "properties": { + "description": { + "type": "string" + }, + "schemaVersion": { + "$ref": "#/definitions/version" + }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/runOnRequirement" + } + }, + "createEntities": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/entity" + } + }, + "initialData": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/collectionData" + } + }, + "tests": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/test" + } + }, + "_yamlAnchors": { + "type": "object", + "additionalProperties": true + } + }, + "definitions": { + "version": { + "type": "string", + "pattern": "^[0-9]+(\\.[0-9]+){1,2}$" + }, + "runOnRequirement": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "maxServerVersion": { + "$ref": "#/definitions/version" + }, + "minServerVersion": { + "$ref": "#/definitions/version" + }, + "topologies": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "single", + "replicaset", + "sharded", + "sharded-replicaset", + "load-balanced" + ] + } + }, + "serverless": { + "type": "string", + "enum": [ + "require", + "forbid", + "allow" + ] + }, + "serverParameters": { + "type": "object", + "minProperties": 1 + }, + "auth": { + "type": "boolean" + }, + "authMechanism": { + "type": "string" + }, + "csfle": { + "type": "boolean" + } + } + }, + "entity": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "client": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "uriOptions": { + "type": "object" + }, + "useMultipleMongoses": { + "type": "boolean" + }, + "observeEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent", + "poolCreatedEvent", + "poolReadyEvent", + "poolClearedEvent", + "poolClosedEvent", + "connectionCreatedEvent", + "connectionReadyEvent", + "connectionClosedEvent", + "connectionCheckOutStartedEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedOutEvent", + "connectionCheckedInEvent", + "serverDescriptionChangedEvent", + "topologyDescriptionChangedEvent" + ] + } + }, + "ignoreCommandMonitoringEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "storeEventsAsEntities": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/storeEventsAsEntity" + } + }, + "observeLogMessages": { + "type": "object", + "minProperties": 1, + "additionalProperties": false, + "properties": { + "command": { + "$ref": "#/definitions/logSeverityLevel" + }, + "topology": { + "$ref": "#/definitions/logSeverityLevel" + }, + "serverSelection": { + "$ref": "#/definitions/logSeverityLevel" + }, + "connection": { + "$ref": "#/definitions/logSeverityLevel" + } + } + }, + "serverApi": { + "$ref": "#/definitions/serverApi" + }, + "observeSensitiveCommands": { + "type": "boolean" + } + } + }, + "clientEncryption": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "clientEncryptionOpts" + ], + "properties": { + "id": { + "type": "string" + }, + "clientEncryptionOpts": { + "$ref": "#/definitions/clientEncryptionOpts" + } + } + }, + "database": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "client", + "databaseName" + ], + "properties": { + "id": { + "type": "string" + }, + "client": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "databaseOptions": { + "$ref": "#/definitions/collectionOrDatabaseOptions" + } + } + }, + "collection": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "database", + "collectionName" + ], + "properties": { + "id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "collectionName": { + "type": "string" + }, + "collectionOptions": { + "$ref": "#/definitions/collectionOrDatabaseOptions" + } + } + }, + "session": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "client" + ], + "properties": { + "id": { + "type": "string" + }, + "client": { + "type": "string" + }, + "sessionOptions": { + "type": "object" + } + } + }, + "bucket": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "database" + ], + "properties": { + "id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "bucketOptions": { + "type": "object" + } + } + }, + "thread": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + } + } + } + } + }, + "logComponent": { + "type": "string", + "enum": [ + "command", + "topology", + "serverSelection", + "connection" + ] + }, + "logSeverityLevel": { + "type": "string", + "enum": [ + "emergency", + "alert", + "critical", + "error", + "warning", + "notice", + "info", + "debug", + "trace" + ] + }, + "clientEncryptionOpts": { + "type": "object", + "additionalProperties": false, + "required": [ + "keyVaultClient", + "keyVaultNamespace", + "kmsProviders" + ], + "properties": { + "keyVaultClient": { + "type": "string" + }, + "keyVaultNamespace": { + "type": "string" + }, + "kmsProviders": { + "$ref": "#/definitions/kmsProviders" + } + } + }, + "kmsProviders": { + "$defs": { + "stringOrPlaceholder": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "$$placeholder" + ], + "properties": { + "$$placeholder": {} + } + } + ] + } + }, + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^aws(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "accessKeyId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "secretAccessKey": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "sessionToken": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^azure(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "tenantId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "clientId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "clientSecret": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "identityPlatformEndpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^gcp(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "email": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "privateKey": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "endpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^kmip(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "endpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^local(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "key": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + } + } + }, + "storeEventsAsEntity": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "events" + ], + "properties": { + "id": { + "type": "string" + }, + "events": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "PoolCreatedEvent", + "PoolReadyEvent", + "PoolClearedEvent", + "PoolClosedEvent", + "ConnectionCreatedEvent", + "ConnectionReadyEvent", + "ConnectionClosedEvent", + "ConnectionCheckOutStartedEvent", + "ConnectionCheckOutFailedEvent", + "ConnectionCheckedOutEvent", + "ConnectionCheckedInEvent", + "CommandStartedEvent", + "CommandSucceededEvent", + "CommandFailedEvent", + "ServerDescriptionChangedEvent", + "TopologyDescriptionChangedEvent" + ] + } + } + } + }, + "collectionData": { + "type": "object", + "additionalProperties": false, + "required": [ + "collectionName", + "databaseName", + "documents" + ], + "properties": { + "collectionName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "createOptions": { + "type": "object", + "properties": { + "writeConcern": false + } + }, + "documents": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "expectedEventsForClient": { + "type": "object", + "additionalProperties": false, + "required": [ + "client", + "events" + ], + "properties": { + "client": { + "type": "string" + }, + "eventType": { + "type": "string", + "enum": [ + "command", + "cmap", + "sdam" + ] + }, + "events": { + "type": "array" + }, + "ignoreExtraEvents": { + "type": "boolean" + } + }, + "oneOf": [ + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "command" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCommandEvent" + } + } + } + }, + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "cmap" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCmapEvent" + } + } + } + }, + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "sdam" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedSdamEvent" + } + } + } + }, + { + "additionalProperties": false, + "properties": { + "client": { + "type": "string" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCommandEvent" + } + }, + "ignoreExtraEvents": { + "type": "boolean" + } + } + } + ] + }, + "expectedCommandEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "commandStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "object" + }, + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + }, + "commandSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reply": { + "type": "object" + }, + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + }, + "commandFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + } + } + }, + "expectedCmapEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "poolCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "poolReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "poolClearedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "hasServiceId": { + "type": "boolean" + }, + "interruptInUseConnections": { + "type": "boolean" + } + } + }, + "poolClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + } + } + }, + "connectionCheckOutStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckOutFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + } + } + }, + "connectionCheckedOutEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckedInEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + } + } + }, + "expectedSdamEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "serverDescriptionChangedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "previousDescription": { + "$ref": "#/definitions/serverDescription" + }, + "newDescription": { + "$ref": "#/definitions/serverDescription" + } + } + }, + "topologyDescriptionChangedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "serverHeartbeatStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "serverHeartbeatSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "serverHeartbeatFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + } + } + }, + "serverDescription": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "Standalone", + "Mongos", + "PossiblePrimary", + "RSPrimary", + "RSSecondary", + "RSOther", + "RSArbiter", + "RSGhost", + "LoadBalancer", + "Unknown" + ] + } + } + }, + "expectedLogMessagesForClient": { + "type": "object", + "additionalProperties": false, + "required": [ + "client", + "messages" + ], + "properties": { + "client": { + "type": "string" + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedLogMessage" + } + }, + "ignoreExtraMessages": { + "type": "boolean" + }, + "ignoreMessages": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedLogMessage" + } + } + } + }, + "expectedLogMessage": { + "type": "object", + "additionalProperties": false, + "required": [ + "level", + "component", + "data" + ], + "properties": { + "level": { + "$ref": "#/definitions/logSeverityLevel" + }, + "component": { + "$ref": "#/definitions/logComponent" + }, + "data": { + "type": "object" + }, + "failureIsRedacted": { + "type": "boolean" + } + } + }, + "collectionOrDatabaseOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "readConcern": { + "type": "object" + }, + "readPreference": { + "type": "object" + }, + "writeConcern": { + "type": "object" + }, + "timeoutMS": { + "type": "integer" + } + } + }, + "serverApi": { + "type": "object", + "additionalProperties": false, + "required": [ + "version" + ], + "properties": { + "version": { + "type": "string" + }, + "strict": { + "type": "boolean" + }, + "deprecationErrors": { + "type": "boolean" + } + } + }, + "operation": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "object" + ], + "properties": { + "name": { + "type": "string" + }, + "object": { + "type": "string" + }, + "arguments": { + "type": "object" + }, + "ignoreResultAndError": { + "type": "boolean" + }, + "expectError": { + "$ref": "#/definitions/expectedError" + }, + "expectResult": {}, + "saveResultAsEntity": { + "type": "string" + } + }, + "allOf": [ + { + "not": { + "required": [ + "expectError", + "expectResult" + ] + } + }, + { + "not": { + "required": [ + "expectError", + "saveResultAsEntity" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "expectResult" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "expectError" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "saveResultAsEntity" + ] + } + } + ] + }, + "expectedError": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "isError": { + "type": "boolean", + "const": true + }, + "isClientError": { + "type": "boolean" + }, + "isTimeoutError": { + "type": "boolean" + }, + "errorContains": { + "type": "string" + }, + "errorCode": { + "type": "integer" + }, + "errorCodeName": { + "type": "string" + }, + "errorLabelsContain": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "errorLabelsOmit": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "writeErrors": { + "type": "object" + }, + "writeConcernErrors": { + "type": "array", + "items": { + "type": "object" + } + }, + "errorResponse": { + "type": "object" + }, + "expectResult": {} + } + }, + "test": { + "type": "object", + "additionalProperties": false, + "required": [ + "description", + "operations" + ], + "properties": { + "description": { + "type": "string" + }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/runOnRequirement" + } + }, + "skipReason": { + "type": "string" + }, + "operations": { + "type": "array", + "items": { + "$ref": "#/definitions/operation" + } + }, + "expectEvents": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/expectedEventsForClient" + } + }, + "expectLogMessages": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/expectedLogMessagesForClient" + } + }, + "outcome": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/collectionData" + } + } + } + } + } +} From 98fbfdad88db3d9e9e1f6281916d26cd20c34622 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Mon, 26 Feb 2024 14:30:30 -0700 Subject: [PATCH 07/90] add batching prose tests --- source/crud/bulk-write.md | 2 +- source/crud/tests/README.md | 117 ++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 1 deletion(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index e8f7c8d3e2..3dff691aef 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -377,7 +377,7 @@ Failures of individual write operations are reported in the cursor of results re ### Command Batching -Drivers MUST accept an arbitrary number of operations as input to the `Client.bulkWrite` method. Because the server imposes restrictions on the size of write operations, this means that a single call to `MongoClient.bulkWrite` may require multiple `bulkWrite` commands to be sent to the server. Drivers MUST split bulk writes into separate commands when the user's list of operations exceeds one of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize`, or `maxMessageSizeBytes`. Each of these values can be retrieved from the selected server's `hello` command response. +Drivers MUST accept an arbitrary number of operations as input to the `Client.bulkWrite` method. Because the server imposes restrictions on the size of write operations, this means that a single call to `MongoClient.bulkWrite` may require multiple `bulkWrite` commands to be sent to the server. Drivers MUST split bulk writes into separate commands when the user's list of operations exceeds one or more of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize`, and `maxMessageSizeBytes`. Each of these values can be retrieved from the selected server's `hello` command response. #### Number of Writes diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index a6b76af8ec..327279c5d4 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -201,3 +201,120 @@ CommandSucceededEvents. Then, insert an invalid document (e.g. `{x: 1}`) and ass code is `121` (i.e. DocumentValidationFailure), and that its `details` property is accessible. Additionally, assert that a CommandSucceededEvent was observed and that the `writeErrors[0].errInfo` field in the response document matches the WriteError's `details` property. + +### 3. `MongoClient.bulkWrite` handles a `writeModels` input with greater than `maxWriteBatchSize` operations + +Test that `MongoClient.bulkWrite` properly handles `writeModels` inputs containing a number of writes greater than +`maxWriteBatchSize`. + +This test must only be run on 8.0+ servers. + +Construct a `MongoClient` (referred to as `client`) with +[command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe +CommandStartedEvents. Perform a `hello` command using `client` and record the `maxWriteBatchSize` value contained in the +response. Then, construct the following write model (referred to as `writeModel`): + +```json +InsertOne: { + "namespace": "db.coll", + "document": { "a": "b" } +} +``` + +Construct a list of write models (referred to as `models`) with `writeModel` repeated `maxWriteBatchSize + 1` times. +Execute `bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` +with an `insertedCount` value of `maxWriteBatchSize + 1`. + +Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEvent`) were observed for the `bulkWrite` command. +Assert that the length of `firstEvent.command.ops` is `maxWriteBatchSize`. Assert that the length of `secondEvent.command.ops` +is 1. If the driver exposes `operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to +`secondEvent.operationId`. + +### 4. `MongoClient.bulkWrite` handles a `writeModels` input larger than `maxBsonObjectSize` + +Test that `MongoClient.bulkWrite` properly handles a `writeModels` input for which the sum of the models' entries in the +`ops` array exceeds `maxBsonObjectSize`. + +This test must only be run on 8.0+ servers. + +Construct a `MongoClient` (referred to as `client`) with +[command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe +CommandStartedEvents. Perform a `hello` command using `client` and record the following values from the response: +`maxBsonObjectSize` and `maxMessageSizeBytes`. Then, construct the following document (referred to as `document`): + +```json +{ + "a": "b".repeat(maxBsonObjectSize / 2) +} +``` + +Construct the following list of write models (referred to as `models`): + +```json +[ + InsertOne: { + "namespace": "db.coll", + "document": document + }, + InsertOne: { + "namespace": "db.coll", + "document": document + } +] +``` + +Execute `bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` +with an `insertedCount` value of 2. Then make the following assertions based on whether or not the driver uses document +sequences (`OP_MSG` payload type 1) with `MongoClient.bulkWrite`: + +#### With document sequences + +Assert that a single CommandStartedEvent (referred to as `event`) was observed for the `bulkWrite` command. Assert that +the length of `event.command.ops` is 2. + +#### Without document sequences + +Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEvent`) were observed for the `bulkWrite` +command. For both `firstEvent` and `secondEvent`, assert that the length of `command.ops` is 1. If the driver exposes +`operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to `secondEvent.operationId`. + +### 5. `MongoClient.bulkWrite` with document sequences handles a `bulkWrite` message larger than `maxMessageSizeBytes` + +Test that `MongoClient.bulkWrite` properly handles a `writeModels` input for which the sum of the models' entries in the +`ops` array exceeds `maxMessageSizeBytes`. Drivers that do not use document sequences (`OP_MSG` payload type 1) for bulk +writes should not implement this test. + +This test must only be run on 8.0+ servers. + +Construct a `MongoClient` (referred to as `client`) with +[command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe +CommandStartedEvents. Perform a `hello` command using `client` and record the following values from the response: +`maxBsonObjectSize` and `maxMessageSizeBytes`. Then, construct the following document (referred to as `document`): + +```json +{ + "a": "b".repeat(maxBsonObjectSize - 500) +} +``` + +Construct the following write model (referred to as `model`): + +```json +InsertOne: { + "namespace": "db.coll", + "document": document +} +``` + +Use the following calculation to determine the number of inserts that should be provided to `MongoClient.bulkWrite`: +`maxMessageSizeBytes / maxBsonObjectSize + 1` (referred to as `numModels`). This number ensures that the inserts +provided to `MongoClient.bulkWrite` will require multiple `bulkWrite` commands to be sent to the server. + +Construct as list of write models (referred to as `models`) with `model` repeated `numModels` times. Then execute +`bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` with +an `insertedCount` value of `numModels`. + +Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEvent`) were observed. Record the length of +`firstEvent.command.ops` as `firstOpsLen`. Record the length of `secondEvent.command.ops` as `secondOpsLen`. Assert that +`firstOpsLen + secondOpsLen` is equal to `numModels`. If the driver exposes `operationId`s in its CommandStartedEvents, +assert that `firstEvent.operationId` is equal to `secondEvent.operationId`. From 7f59210c56e8b82d0c6ebb28c8c1ab4364803d0d Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 27 Feb 2024 10:41:05 -0700 Subject: [PATCH 08/90] restructure spec --- source/crud/bulk-write.md | 935 ++++++++++++++++++++++++-------------- 1 file changed, 596 insertions(+), 339 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 3dff691aef..ed022d296f 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -1,461 +1,718 @@ # Bulk Write -isabeltodo table +- Status: In Review +- Minimum Server Version: 8.0 ## Abstract -This specification defines the driver API for the `bulkWrite` server command introduced in MongoDB 8.0. The API defined in this specification allows users to perform insert, update, and delete operations against mixed namespaces in a minimized number of round trips, and to receive detailed results for each operation performed. This API is distinct from the collection-level bulkWrite method defined in the CRUD specification and the deprecated bulk write specification (isabeltodo link). +This specification defines the driver API for the `bulkWrite` server command introduced in MongoDB +8.0. The API defined in this specification allows users to perform insert, update, and delete +operations against mixed namespaces in a minimized number of round trips, and to receive detailed +results for each operation performed. This API is distinct from the +[collection-level bulkWrite method](../crud/crud.md#insert-update-replace-delete-and-bulk-writes) +defined in the CRUD specification and the +[deprecated bulk write specification](../driver-bulk-update.rst). ## Specification -### Definitions +### `MongoClient.bulkWrite` Interface -Document sequence: An `OP_MSG` payload type 1 as defined in the `OP_MSG` specification here (isabeltodo link). +**NOTE:** The types specified in this interface are similar to those used for the collection-level +`bulkWrite` method. Statically typed drivers MUST NOT reuse any types they already define for the +`MongoCollection.bulkWrite` interface and MUST introduce new types. If naming conflicts between +types arise, drivers MAY prepend "Client" to the new type names (e.g. `ClientBulkWriteOptions`). -Namespace: A database name and collection name pair. This may be represented as a string (e.g. `"db.coll"`) or a class with a field for each name, e.g. - -``` -class Namespace { - databaseName: String - collectionName: String +```typescript +interface MongoClient { + /** + * Executes a list of mixed write operations. + * + * @throws BulkWriteException + */ + bulkWrite(models: NamespaceWriteModelPair[], options: Optional): BulkWriteResult; } ``` -`bulkWrite`: This specification discusses both the server `bulkWrite` command and the new `bulkWrite` driver API method. To avoid ambiguity, this specification refers to the server command as `bulkWrite` and the driver method as `MongoClient.bulkWrite`. +### Write Models -### `bulkWrite` Command and Response Format +A `WriteModel` defines a single write operation to be performed as part of a bulk write. -#### Command +```typescript +/** Unifying interface for the various write model types. Drivers may also use an enum with + * variants for each write model for this type. + */ +interface WriteModel {} -The `bulkWrite` server command has the following format: - -```json -{ - bulkWrite: 1 - ops: - nsInfo: - errorsOnly: - ordered: - bypassDocumentValidation: - comment: - let: +class InsertOneModel implements WriteModel { + /** + * The document to insert. + */ + document: Document; } -``` - -##### `ops` -The `ops` field is a list of write operation documents. The documents have the following format: +class UpdateOneModel implements WriteModel { + /** + * The filter to apply. + */ + filter: Document; + + /** + * The update document or pipeline to apply to the selected document. + */ + update: (Document | Document[]); + + /** + * A set of filters specifying to which array elements an update should apply. + * + * This option is sent only if the caller explicitly provides a value. + */ + arrayFilters: Optional; + + /** + * Specifies a collation. + * + * This option is sent only if the caller explicitly provides a value. + */ + collation: Optional; + + /** + * The index to use. Specify either the index name as a string or the index key pattern. If + * specified, then the query system will only consider plans using the hinted index. + * + * This option is only sent if the caller explicitly provides a value. + */ + hint: Optional<(String | Document)>; + + /** + * When true, creates a new document if no document matches the query. Defaults to false. + * + * This options is sent only if the caller explicitly provides a value. + */ + upsert: Optional; +} -###### Insert +class UpdateManyModel implements WriteModel { + /** + * The filter to apply. + */ + filter: Document; + + /** + * The update document or pipeline to apply to the selected documents. + */ + update: (Document | Document[]); + + /** + * A set of filters specifying to which array elements an update should apply. + * + * This option is sent only if the caller explicitly provides a value. + */ + arrayFilters: Optional; + + /** + * Specifies a collation. + * + * This option is sent only if the caller explicitly provides a value. + */ + collation: Optional; + + /** + * The index to use. Specify either the index name as a string or the index key pattern. If + * specified, then the query system will only consider plans using the hinted index. + * + * This option is only sent if the caller explicitly provides a value. + */ + hint: Optional<(String | Document)>; + + /** + * When true, creates a new document if no document matches the query. Defaults to false. + * + * This options is sent only if the caller explicitly provides a value. + */ + upsert: Optional; +} -```json -{ - insert: - document: +class ReplaceOneModel implements WriteModel { + /** + * The filter to apply. + */ + filter: Document; + + /** + * The replacement document. + */ + filter: Document; + + /** + * Specifies a collation. + * + * This option is sent only if the caller explicitly provides a value. + */ + collation: Optional; + + /** + * The index to use. Specify either the index name as a string or the index key pattern. If + * specified, then the query system will only consider plans using the hinted index. + * + * This option is only sent if the caller explicitly provides a value. + */ + hint: Optional<(String | Document)>; + + /** + * When true, creates a new document if no document matches the query. Defaults to false. + * + * This options is sent only if the caller explicitly provides a value. + */ + upsert: Optional; } -``` -###### Update +class DeleteOneModel implements WriteModel { + /** + * The filter to apply. + */ + filter: Document; + + /** + * Specifies a collation. + * + * This option is sent only if the caller explicitly provides a value. + */ + collation: Optional; + + /** + * The index to use. Specify either the index name as a string or the index key pattern. If + * specified, then the query system will only consider plans using the hinted index. + * + * This option is only sent if the caller explicitly provides a value. + */ + hint: Optional<(String | Document)>; +} -```json -{ - update: - filter: - updateMods: - multi: - upsert: - arrayFilters: - hint: +class DeleteManyModel implements WriteModel { + /** + * The filter to apply. + */ + filter: Document; + + /** + * Specifies a collation. + * + * This option is sent only if the caller explicitly provides a value. + */ + collation: Optional; + + /** + * The index to use. Specify either the index name as a string or the index key pattern. If + * specified, then the query system will only consider plans using the hinted index. + * + * This option is only sent if the caller explicitly provides a value. + */ + hint: Optional<(String | Document)>; } ``` -###### Delete +Each write model provided to `MongoClient.bulkWrite` in the `models` parameter MUST have a +corresponding namespace that defines the collection on which the operation should be performed. +Drivers SHOULD design this pairing in whichever way is most ergonomic for its language. For +example, drivers may: -```json -{ - delete: - filter: - multi: - hint: - collation: +- Include a required `namespace` field on each `WriteModel` variant +- Accept a list of `(Namespace, WriteModel)` tuples for `models` +- Define the following pair class: + +```typescript +class NamespaceWriteModelPair { + /** + * The namespace on which to perform the write. + */ + namespace: Namespace; + + /** + * The write to perform. + */ + model: WriteModel; } ``` -##### `nsInfo` +Drivers MUST throw an exception if the list provided for `models` is empty. -The `nsInfo` field contains the namespaces on which the write operations should be performed. Drivers MUST NOT include duplicate namespaces in this list. The documents in the `nsInfo` field have the following format: +### Options -```json -{ - ns: +```typescript +class BulkWriteOptions { + /** + * Whether the operations in this bulk write should be executed in the order in which they were + * specified. If false, writes will continue to be executed if an individual write fails. If + * true, writes will stop executing if an individual write fails. + * + * Defaults to false. + */ + ordered: Optional; + + /** + * If true, allows the writes to opt out of document-level validation. + * + * Defaults to false. + * + * This option is only sent if the caller explicitly provides a value. + */ + bypassDocumentValidation: Optional; + + /** + * A map of parameter names and values to apply to all operations within the bulk write. Value + * must be constant or closed expressions that do not reference document fields. Parameters can + * then be accessed as variables in an aggregate expression context (e.g. "$$var"). + * + * This option is only sent if the caller explicitly provides a value. + */ + let: Optional; + + /** + * The write concern to use for this bulk write. + * + * NOT REQUIRED TO IMPLEMENT. Drivers MUST expose this option if retrieving a handle to a + * client with a different write concern configured than that of the user's standard URI + * options is nontrivial. Drivers MAY omit this option if they provide a way to retrieve a + * lightweight handle to a client with a custom write concern configured, e.g. a + * MongoClient.withWriteConcern() method. + */ + writeConcern: Optional; + + /** + * Whether detailed results for each successful operation should be included in the returned + * BulkWriteResult. + * + * Defaults to false. + */ + verboseResults: Optional; } ``` -##### `errorsOnly` +### Result -This field corresponds to the `verboseResults` option defined on `BulkWriteOptions`. Its value should be `false` if `verboseResults` was set to `true`; otherwise, it should be `true`. This field MUST always be defined in the `bulkWrite` command regardless of whether a value for `verboseResults` was specified. +```typescript +class BulkWriteResult { + /** + * Indicates whether this write result was acknowledged. If not, then all other members of this + * result will be undefined. + * + * NOT REQUIRED TO IMPLEMENT. See [here](../crud/crud.md#write-results) for more guidance on + * modeling unacknowledged results. + */ + acknowledged: Boolean; + + /** + * Indicates whether the results are verbose. If false, the insertResults, updateResults, and + * deleteResults fields in this result will be undefined. + * + * NOT REQUIRED TO IMPLEMENT. See below for other ways to differentiate summary results from + * verbose results. + */ + hasVerboseResults: Boolean; + + /** + * The total number of documents inserted across all insert operations. + */ + insertedCount: Int64; + + /** + * The total number of documents upserted across all update operations. + */ + upsertedCount: Int64; + + /** + * The total number of documents matched across all update operations. + */ + matchedCount: Int64; + + /** + * The total number of documents modified across all update operations. + */ + modifiedCount: Int64; + + /** + * The total number of documents deleted across all delete operations. + */ + deletedCount: Int64; + + /** + * The results of each individual insert operation that was successfully performed. Results + * can be accessed by their index in the list of write models provided to bulkWrite. + */ + insertResults: Map; + + /** + * The results of each individual update operation that was successfully performed. Results + * can be accessed by their index in the list of write models provided to bulkWrite. + */ + updateResult: Map; + + /** + * The results of each individual delete operation that was successfully performed. Results + * can be accessed by their index in the list of write models provided to bulkWrite. + */ + deleteResults: Map; +} -#### Response +class InsertOneResult { + /** + * The _id of the inserted document. + */ + insertedId: Any; +} -The server's response to `bulkWrite` has the following format: +class UpdateResult { + /** + * The number of documents that matched the filter. + */ + matchedCount: Int64; + + /** + * The number of documents that were modified. + */ + modifiedCount: Int64; + + /** + * The number of documents that were upserted. + * + * NOT REQUIRED TO IMPLEMENT. Drivers may choose not to provide this property so long as it is + * always possible to discern whether an upsert took place. + */ + upsertedCount: Int64; + + /** + * The _id field of the upserted document if an upsert occurred. + */ + upsertedId: Optional; +} -```json -{ - ok: <0 | 1> - cursor: { - id: Int64 - firstBatch: - ns: - } - nErrors: - nInserted: - nUpserted: - nMatched: - nModified: - nDeleted: +class DeleteResult { + /** + * The number of documents that were deleted. + */ + deletedCount: Int64; } ``` -##### Results Cursor +#### Summary vs. Verbose Results -The response to `bulkWrite` contains a cursor field with the first batch of individual results and a nonzero cursor ID if there are additional results that did not fit in the initial response. Drivers MUST perform `getMore`s until the cursor is exhausted to retrieve all results and errors before returning from `MongoClient.bulkWrite`. If a top-level error occurs before the cursor has been exhausted, drivers MUST send a `killCursors` command to close the cursor. +Users MUST be able to discern whether a `BulkWriteResult` contains summary or verbose results +without inspecting the value provided for `verboseResults` in `BulkWriteOptions`. Drivers MUST +implement this in one of the following ways: -### `MongoClient.bulkWrite` Signature +- Expose the `hasVerboseResults` field in `BulkWriteResult` as defined above. +- Implement the `insertResults`, `upsertResults`, and `deleteResults` fields as optional types and + document that they will be unset when `verboseResults` is false. +- Introduce separate `SummaryBulkWriteResult` and `VerboseBulkWriteResult` types. + `VerboseBulkWriteResult` MUST have all of the required fields defined on `BulkWriteResult` above. + `SummaryBulkWriteResult` MUST have all of the required fields defined on `BulkWriteResult` above + except `insertResults`, `updateResults`, and `deleteResults`. -``` -interface MongoClient { - bulkWrite(models: List, - options: Optional - ) -> BulkWriteResult throws BulkWriteException; -} -``` - -The `WriteModel`, `BulkWriteOptions`, `BulkWriteResult`, and `BulkWriteException` types defined here are similar to the types of the same names in the `MongoCollection.bulkWrite` definition. Statically typed drivers MUST NOT reuse any existing versions of these types and MUST introduce new types for the `MongoClient.bulkWrite` method. If +#### Individual Results -#### `WriteModel` +The `InsertOneResult`, `UpdateResult`, and `DeleteResult` classes are the same as or similar to +types of the same name defined in the [CRUD specification](crud.md). Drivers MUST redefine these +classes if their existing result classes deviate from the definitions in this specification (e.g. +if they contain acknowledgement information, which is not applicable for individual bulk write +operations). Drivers MAY reuse their existing types for these classes if they match the ones +defined here exactly. -The `WriteModel` type defines a single write operation to be performed as part of a bulk write. `WriteModel` may be designed as separate classes that implement a `WriteModel` interface, an enum with a variant for each operation (as shown below), or another similar type. +### Exception -``` - -enum WriteModel { - InsertOne { - // The document to insert. - document: Document - } - UpdateOne { - // The filter to apply. - filter: Document - - // The update document or pipeline to apply to the selected document. - update: Document | Array - - // isabeltodo - arrayFilters: Option - - // isabeltodo - collation: Option - - // isabeltodo - hint: Optional - - // isabeltodo - upsert: Optional - } - UpdateMany { - // The filter to apply. - filter: Document - - // The update document or pipeline to apply to the selected documents. - update: Document | Document[] - - // isabeltodo - arrayFilters: Option - - // isabeltodo - collation: Option - - // isabeltodo - hint: Optional - - // isabeltodo - upsert: Optional - } - ReplaceOne { - // The filter to apply. - filter: Document - - // The replacement document. - replacement: Document - - // isabeltodo - collation: Option - - // isabeltodo - hint: Optional - - // isabeltodo - upsert: Optional - } - DeleteOne { - // The filter to apply. - filter: Document - - // isabeltodo - collation: Option - - // isabeltodo - hint: Option - } - DeleteMany { - // The filter to apply. - filter: Document, - - // isabeltodo - collation: Option, - - // isabeltodo - hint: Option - } +```typescript +class BulkWriteException { + /** + * A top-level error that occurred when attempting to communicate with the server or execute + * the bulk write. This value may not be populated if the exception was thrown due to errors + * occurring on individual writes. + */ + error: Optional; + + /** + * Write concern errors that occurred while executing the bulk write. This list may have + * multiple items if more than one server command was required to execute the bulk write. + */ + writeConcernErrors: WriteConcernError[]; + + /** + * Errors that occurred during the execution of individual write operations. This map will + * contain at most one entry if the bulk write was ordered. + */ + writeErrors: Map; + + /** + * The results of any successful operations that were performed before the error was + * encountered. + */ + partialResult: Optional; } ``` -##### Namespaces +## Building a `bulkWrite` Command -Each write model supplied to `Client.bulkWrite` in the `models` parameter MUST have a corresponding namespace that defines the collection on which the operation should be performed. Drivers SHOULD design this pairing in whichever way is most ergonomic for its language. For example, drivers may: - -- include a required `namespace` field in each `WriteModel` variant -- accept a list of `(Namespace, WriteModel)` tuples for `models` -- define the following pair class: - -``` - -class NamespaceWriteModelPair { - // The namespace on which to perform the operation. - namespace: Namespace +The `bulkWrite` server command has the following format: - // The write to perform. - model: WriteModel +```json +{ + "bulkWrite": 1, + "ops": , + "nsInfo": , + "errorsOnly": , + "ordered": , + "bypassDocumentValidation": , + "comment": , + "let": , + ...additional operation-agnostic fields } ``` -#### `BulkWriteOptions` +Drivers SHOULD use document sequences ([`OP_MSG`](../message/OP_MSG.rst) payload type 1) for the +`ops` and `nsInfo` fields. Drivers MUST NOT use document sequences when auto-encryption is enabled. -``` +### Operations -class BulkWriteOptions { - // Whether the operations in this bulk write should be executed in the order in which they were specified. If false, writes will continue to be executed if an individual write fails. If true, writes will stop executing if an individual write fails. Defaults to true. - ordered: Optional +The `ops` field is a list of write operation documents. The first entry in each document has +the name of the operation (i.e. "insert", "update", or "delete") as its key and the index in the +`nsInfo` array of the namespace on which the operation should be performed as its value. The +documents have the following format: + +#### Insert - // If true, allows the writes to opt out of document-level validation. Defaults to false. - bypassDocumentValidation: Optional +```json +{ + "insert": , + "document": +} +``` - // A map of parameter names and values to apply to all operations within the bulk write. Values must be constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an aggregate expression context (e.g. "$$var"). - let: Optional +Drivers MUST add an `_id` field at the beginning of the insert document if one is not already +present. - // The write concern to use for this bulk write. - // - // NOT REQUIRED. Drivers MUST expose this option if retrieving a handle to a `MongoClient` with a different write concern configured than that of the user's standard URI options is nontrivial. Drivers MAY omit this option if they provide a way to retrieve a lightweight handle to a `MongoClient` with a custom write concern configured, e.g. a `MongoClient.withWriteConcern` method. - writeConcern: Optional +#### Update - // Whether detailed results for each successful operation should be included in the returned `BulkWriteResult`. Defaults to false. - verboseResults: Optional +```json +{ + "update": , + "filter": , + "updateMods": , + "multi": , + "upsert": , + "arrayFilters": , + "hint": } ``` -#### `BulkWriteResult` +#### Delete +```json +{ + "delete": , + "filter": , + "multi": , + "hint": , + "collation": +} ``` -class BulkWriteResult { - // Indicates whether this write result was acknowledged. If not, then all other members of this result will be undefined. - // - // NOT REQUIRED. See here(isabeltodo link) for additional guidance on modeling unacknowledged results. - acknowledged: boolean - - // Indicates whether the results are verbose. This value will be equal to the value of `verboseResults` in the provided `BulkWriteOptions`. If false, the `insertResults`, `updateResults`, and `deleteResults` fields in this result will be undefined. - // - // NOT REQUIRED. See below (link) for other ways to differentiate summary results from verbose results. - hasVerboseResults: boolean +### Namespace Information - // The total number of documents inserted across all insert operations. - insertedCount: i64 +The `nsInfo` field contains the namespaces on which the write operations should be performed. +Drivers MUST NOT include duplicate namespaces in this list. The documents in the `nsInfo` field +have the following format: - // The total number of documents upserted across all update operations. - upsertedCount: i64 +```json +{ + "ns": +} +``` - // The total number of documents matched across all update operations. - matchedCount: i64 +### Toggling Results Verbosity - // The total number of documents modified across all update operations. - modifiedCount: i64 +The `errorsOnly` field indicates whether the results cursor returned in the `bulkWrite` response +should contain only errors. If false, individual results for successful operations will be returned +in addition to errors. - // The total number of documents deleted across all delete operations. - deletedCount: i64 +This field corresponds to the `verboseResults` option defined on `BulkWriteOptions`. Its value +should be provided as `false` if `verboseResults` was set to `true`; otherwise, it should be +provided as `true`. This field MUST always be defined in the `bulkWrite` command regardless of +whether the user specified a value for `verboseResults`. - // The results of each individual insert operation that was successfully performed. Results can be accessed by their index in the list of write models provided to the call to `MongoClient.bulkWrite`. - insertResults: Map +## Command batching - // The results of each individual update operation that was successfully performed. Results can be accessed by their index in the list of write models provided to the call to `MongoClient.bulkWrite`. - updateResults: Map +Drivers MUST accept an arbitrary number of operations as input to the `Client.bulkWrite` method. +Because the server imposes restrictions on the size of write operations, this means that a single +call to `MongoClient.bulkWrite` may require multiple `bulkWrite` commands to be sent to the server. +Drivers MUST split bulk writes into separate commands when the user's list of operations exceeds +one or more of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize`, and `maxMessageSizeBytes`. +Each of these values can be retrieved from the selected server's `hello` command response. - // The results of each individual delete operation that was successfully performed. Results can be accessed by their index in the list of write models provided to the call to `MongoClient.bulkWrite`. - deleteResults: Map -} +### Number of Writes -class InsertOneResult { - // The _id of the inserted document. - insertedId: Bson -} +The `ops` array MUST NOT include more than `maxWriteBatchSize` operations. -class UpdateResult { - // The number of documents that matched the filter. - matchedCount: Int64 +### Document Size - // The number of documents that were modified. - modifiedCount: Int64 +All entries within the `ops` array MUST be within `maxBsonObjectSize` bytes. Drivers MUST throw an +exception if an operation exceeds this size. - // The number of documents that were upserted. - // - // NOT REQUIRED. Drivers may choose to not provide this property so long as it is always possible to infer whether an upsert has taken place. Since the _id field of an upserted document could be null, a null upsertedId may be ambiguous in some drivers. If so, this field can be used to indicate whether an upsert has taken place. - upsertedCount: Int64 +### Total Message Size - // The _id field of the upserted document if an upsert occurred. - upsertedId: Optional -} +#### Encrypted bulk writes -class DeleteResult { - // The number of documents that were deleted. - deletedCount: Int64 -} -``` +When auto-encryption is enabled, drivers MUST NOT provide the `ops` and `nsInfo` fields as document +sequences and MUST limit the size of the command according to the auto-encryption size limits +defined in the +[Client Side Encryption Specification](../client-side-encryption/client-side-encryption.rst). -##### Summary vs. Verbose Results +#### Unencrypted bulk writes with document sequences -Users MUST be able to discern whether the result returned from `MongoClient.bulkWrite` is a summary result or a verbose result without accessing the value provided for `verboseResults`. Drivers MUST implement this in one of the following ways: +When `ops` and `nsInfo` are provided as document sequences, drivers MUST ensure that the combined +size of these sequences does not exceed `maxMessageSizeBytes - 16,000`. `maxMessageSizeBytes` +defines the maximum size for an entire `OP_MSG`, so 16KB is subtracted from `maxMessageSizeBytes` +in this size limit to account for the bytes in the rest of the message. -* Expose the `hasVerboseResults` field in `BulkWriteResult` as defined above. -* Implement the `insertResults`, `updateResults`, and `deleteResults` fields as optional types and document that they will be unset when `verboseResults` is false. -* Introduce separate `SummaryBulkWriteResult` and `VerboseBulkWriteResult` types. `VerboseBulkWriteResult` MUST have all of the required fields defined on `BulkWriteResult` above. `SummaryBulkWriteResult` MUST have all of the required fields defined on `BulkWriteResult` above except `insertResults`, `updateResults`, and `deleteResults`. +#### Unencrypted bulk writes without document sequences -##### Individual Results +When `ops` and `nsInfo` are provided within the `bulkWrite` command document and auto-encryption is +not enabled, drivers MUST ensure that the total size of the `bulkWrite` command document does not +exceed `maxBsonObjectSize`. -The `InsertOneResult`, `UpdateResult`, and `DeleteResult` classes are the same as or similar to those of the same names defined in the CRUD specification. Drivers MUST redefine these classes if their existing result classes deviate from the definitions in this specification (e.g. if they contain acknowledgement information). Drivers MAY reuse their existing types for these classes if they match the ones defined here exactly. +## Handling the `bulkWrite` Server Response -#### `BulkWriteException` +The server's response to `bulkWrite` has the following format: +```json +{ + "ok": <0 | 1>, + "cursor": { + "id": , + "firstBatch": , + "ns": + }, + "nErrors": , + "nInserted": , + "nUpserted": , + "nMatched": , + "nModified": , + "nDeleted": , + ...additional command-agnostic fields +} ``` -class BulkWriteException { - // A top-level error that occurred when attempting to communicate with the server or execute the bulk write. This value may not be populated if the exception was thrown due to errors occurrring on individual writes. - error: Optional +If any operations were successful (i.e. `nErrors` is less than the number of operations that were +sent), drivers MUST record the summary count fields in a `BulkWriteResult` to be returned to the +user or included in a `BulkWriteException`. - // Write concern errors that occurred while executing the bulk write. This list may have multiple items if more than one round trip was required to execute the bulk write. - writeConcernErrors: List +Drivers MUST attempt to consume the contents of the cursor returned in the server's `bulkWrite` +response before returning to the user. This is required regardless of whether the user requested +verbose or summary results, as the results cursor always contains any write errors that occurred. +If the cursor contains a nonzero cursor ID, drivers MUST perform `getMore`s until the cursor has +been exhausted. Drivers MUST use the same session used for the `bulkWrite` command for each +`getMore` call. - // Errors that occurred during the execution of individual write operations. This map will contain at most one entry if the bulk write was ordered. - writeErrors: Map +The documents in the results cursor have the following format: - // The results of any successful operations that were performed before the error was encountered. - partialResult: Optional +```json +{ + "ok": <0 | 1>, + "code": Optional, + "errmsg": Optional, + "n": , + "nMatched": , + "nModified": , + "upsertedId": Optional } ``` -##### Top-level errors - -A top-level error is defined as any error that occurs during a bulk write that is not the result of an individual write operation failing or a write concern error. Possible top-level errors include, but are not limited to, network errors that occur when communicating with the server, command errors returned from the server, and client-side errors (e.g. an oversized insert document was provided). If individual operation results or write concern errors have already been observed prior to a top-level error occurring, drivers MUST embed the top-level error within a `BulkWriteException` as the `error` field to retain this information. Otherwise, drivers MAY throw an exception containing only the top-level error. - -A top-level MUST halt execution of a bulk write for both ordered and unordered bulk writes. When a top-level error is encountered, drivers MUST NOT attempt to retrieve responses from the cursor or execute any further `bulkWrite` batches. +Note that the responses do not contain information about the type of operation that was performed. +Drivers MAY need to maintain the user's list of write models to infer which type of result should +be recorded. -##### Write concern errors +### Handling Insert Results -At most one write concern error may be returned per `bulkWrite` command response. If the driver observes a write concern error, it MUST record the error in the `writeConcernErrors` field and MUST continue executing the bulk write for both ordered and unordered bulk writes. If the `writeConcernErrors` field is not empty at the end of execution, drivers MUST throw the `BulkWriteException` and include any results and/or additional errors that were observed. +Unlike the other result types, `InsertOneResult` contains an `insertedId` field that is generated +driver-side, either by recording the `_id` field present in the user's insert document or creating +and adding one. Drivers MUST only record these `insertedId`s in a `BulkWriteResult` when a +successful response for the insert operation (i.e. `{ "ok": 1, "n": 1 }`) is received in the +results cursor. This ensures that drivers only report `insertedId`s to users when it is confirmed +that the insert succeeded. -##### Individual write errors +## Handling errors -Failures of individual write operations are reported in the cursor of results returned from the server. Drivers MUST iterate the this cursor fully and record all errors in the `writeErrors` map in `BulkWriteException`. If an individual write error is encountered during an ordered bulk write, drivers MUST record the error in `writeErrors` and immediately throw a `BulkWriteException`. Otherwise, drivers MUST continue iterating the cursor and execute any further `bulkWrite` batches. +### Top-level errors -### Command Batching +A top-level error is any error that occurs that is not the result of a single write operation +failing or a write concern error. Examples include network errors that occur when communicating +with the server, command errors returned from the server, client-side errors, and errors that occur +when attempting to perform a `getMore` to retrieve results from the server. -Drivers MUST accept an arbitrary number of operations as input to the `Client.bulkWrite` method. Because the server imposes restrictions on the size of write operations, this means that a single call to `MongoClient.bulkWrite` may require multiple `bulkWrite` commands to be sent to the server. Drivers MUST split bulk writes into separate commands when the user's list of operations exceeds one or more of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize`, and `maxMessageSizeBytes`. Each of these values can be retrieved from the selected server's `hello` command response. - -#### Number of Writes - -The `ops` array MUST NOT include more than `maxWriteBatchSize` operations. +When a top-level error is encountered and individual results and/or errors have already been +observed, drivers MUST embed the top-level error within a `BulkWriteException` as the `error` field +to retain this information. Otherwise, drivers MAY throw an exception containing only the top-level +error. -#### Document Size +Encountering a top-level error MUST halt execution of a bulk write for both ordered and unordered +bulk writes. This means that drivers MUST NOT attempt to retrieve more responses from the cursor or +execute any further `bulkWrite` batches and MUST immediately throw an exception. -All entries within the `ops` array MUST be within `maxBsonObjectSize` bytes. Drivers MUST throw a `BulkWriteException` with a top-level client error if an operation exceeds this size. +### Write concern errors -#### Total Message Size +Write concern errors are recorded in the `writeConcernErrors` field on `BulkWriteException`. When +a write concern error is encountered, it should not terminate execution of the bulk write for +either ordered or unordered bulk writes. However, drivers MUST throw an exception at the end of +execution if any write concern errors were observed. -##### Encrypted bulk writes +### Individual write errors -When auto-encryption is enabled, drivers MUST NOT provide the `ops` and `nsInfo` fields as document sequences and MUST limit the size of the command according to the auto-encryption size limits defined here (isabeltodo link and update language in link). +Individual write errors retrieved from the cursor are recorded in the `writeErrors` field on +`BulkWriteException`. If an individual write error is encountered during an ordered bulk write, +drivers MUST record the error in `writeErrors` and immediately throw the exception. Otherwise, +drivers MUST continue to iterate the results cursor and execute any further `bulkWrite` batches. -##### Unencrypted bulk writes with document sequences +## Test Plan -When `ops` and `nsInfo` are provided as document sequences, drivers MUST ensure that the combined size of these sequences does not exceed `maxMessageSizeBytes - 16,000`. `maxMessageSizeBytes` defines the maximum size for an entire `OP_MSG`, so 16KB is subtracted from `maxMessageSizeBytes` in this size limit to account for the bytes in the rest of the message. +The majority of tests for `MongoClient.bulkWrite` are written in the +[Unified Test Format](../unified-test-format/unified-test-format.md) and reside in the +[CRUD unified tests directory](../crud/tests/unified/). -##### Unencrypted bulk writes without document sequences +Additional prose tests are specified [here](../crud/tests/README.md). These tests require +constructing very large documents to test drivers' implementation of batch splitting, which is not +feasible in the unified test format at the time of writing this specification. -When `ops` and `nsInfo` are provided within the `bulkWrite` command document and auto-encryption is not enabled, drivers MUST ensure that the total size of the `bulkWrite` command document does not exceed `maxBsonObjectSize`. +## Q&A -### Pseudocode Implementation - -The following pseudocode is a sample implementation of `MongoClient.bulkWrite`. - -``` -executeBulkWrite(client: MongoClient, models: List, options: BulkWriteOptions) { - let exception = empty BulkWriteException - - while !models.isEmpty() { - try { - let bulkWrite = batchSplitAndBuildCommand(models, options) - let (summaryInfo, resultsCursor, writeConcernError) = client.executeOperation(bulkWrite) - - if writeConcernError != null { - exception.writeConcernErrors.add(writeConcernError) - } - - for result in resultsCursor { - if result.isSuccess() { - exception.partialResult.add(result) - } else { // error - exception.writeErrors.add(result) - if options.ordered { - throw exception - } - } - } - } catch error { - exception.error = error - throw exception - } - } - - if !exception.writeConcernErrors.isEmpty() || !exception.writeErrors.isEmpty() { - throw exception - } else { - return exception.partialResult - } -} -``` +### Why are we adding a new bulk write API rather than updating the `MongoCollection.bulkWrite` implementation? -### Q&A +The new `bulkWrite` command is only available in MongoDB 8.0+, so it cannot function as a drop-in +replacement for the existing bulk write implementation that uses the `insert`, `update`, and +`delete` commands. Additionally, because the new `bulkWrite` command allows operations against +multiple collections and databases, `MongoClient` is a more appropriate place to expose its +functionality. -#### Why are we adding a new bulk write API rather than updating the `MongoCollection.bulkWrite` implementation? +### Why can't drivers reuse existing bulk write types? -The new `bulkWrite` command is only available in MongoDB 8.0+, so it cannot function as a drop-in replacement for the existing bulk write implementation that uses the `insert`, `update`, and `delete` commands. Additionally, because the new `bulkWrite` command allows operations against multiple collections and databases, `MongoClient` is a more appropriate place to expose its functionality. +This specification introduces several types that are similar to existing types used in the +`MongoCollection.bulkWrite` API. Although these types are similar now, they may diverge in the +future with the introduction of new options and features to the `bulkWrite` command. Introducing +new types also provides more clarity to users on the existing differences between the +collection-level and client-level bulk write APIs. For example, the `verboseResults` option is only +valid for `MongoClient.bulkWrite`. -#### Why can't drivers reuse existing bulk write types? +### Why are bulk write operation results returned in a cursor? -This specification introduces several types that are similar to existing types used in the legacy bulk write API. Although these types are similar now, they may diverge in the future with the introduction of new options and features to the `bulkWrite` command. Introducing new types also provides more clarity to users on the existing differences between the legacy and new bulk write APIs. For example, the `verboseResults` option is only available for `MongoClient.bulkWrite`. +Returning results via a cursor rather than an array in the `bulkWrite` response allows full +individual results and errors to be returned without the risk of the response exceeding the +maximum BSON object size. Using a cursor also leaves open the opportunity to add `findAndModify` to +the list of supported write operations in the future. -#### Why are bulk write operation results returned in a cursor? +### Why was the `verboseResults` option introduced, and why is its default `false`? -Returning results via a cursor rather than an array in the `bulkWrite` response allows full individual results and errors to be returned without the risk of the response exceeding the maximum BSON object size. Using a cursor also leaves open the opportunity to add `findAndModify` to the list of supported write operations in the future. +The `bulkWrite` command returns top-level summary result counts and, optionally, individual results +for each operation. Compiling the individual results server-side and consuming these results +driver-side is less performant than only recording the summary counts. We expect that most users +are not interested in the individual results of their operations and that most users will rely on +defaults, so `verboseResults` defaults to `false` to improve performance in the common case. -#### Why was the `verboseResults` option introduced, and why is its default `false`? +## **Changelog** -The `bulkWrite` command returns top-level summary result counts and, optionally, individual results for each operation. Compiling the individual results server-side and consuming these results driver-side is less performant than only recording the summary counts. We expect that most users are not interested in the individual results of their operations and that most users will rely on defaults, so `verboseResults` defaults to `false` to improve performance in the common case. +- TODO: Bulk write specification created. From dbb2083471e4d9d9f02ffe1fa4406fc93c7fdc4b Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 29 Feb 2024 09:23:35 -0700 Subject: [PATCH 09/90] various cleanup --- source/crud/bulk-write.md | 80 ++++++++++--------- .../tests/unified/client-bulkWrite.json | 34 ++------ .../tests/unified/client-bulkWrite.yml | 28 ++----- source/transactions/transactions.md | 2 + .../unified-test-format.md | 23 ++++-- .../tests/crud-api-version-1.json | 6 +- .../tests/crud-api-version-1.yml | 7 +- 7 files changed, 80 insertions(+), 100 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index ed022d296f..9c86584d8c 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -26,7 +26,7 @@ types arise, drivers MAY prepend "Client" to the new type names (e.g. `ClientBul interface MongoClient { /** * Executes a list of mixed write operations. - * + * * @throws BulkWriteException */ bulkWrite(models: NamespaceWriteModelPair[], options: Optional): BulkWriteResult; @@ -38,7 +38,8 @@ interface MongoClient { A `WriteModel` defines a single write operation to be performed as part of a bulk write. ```typescript -/** Unifying interface for the various write model types. Drivers may also use an enum with +/** + * Unifying interface for the various write model types. Drivers may also use an enum with * variants for each write model for this type. */ interface WriteModel {} @@ -63,14 +64,14 @@ class UpdateOneModel implements WriteModel { /** * A set of filters specifying to which array elements an update should apply. - * + * * This option is sent only if the caller explicitly provides a value. */ arrayFilters: Optional; /** * Specifies a collation. - * + * * This option is sent only if the caller explicitly provides a value. */ collation: Optional; @@ -78,14 +79,14 @@ class UpdateOneModel implements WriteModel { /** * The index to use. Specify either the index name as a string or the index key pattern. If * specified, then the query system will only consider plans using the hinted index. - * + * * This option is only sent if the caller explicitly provides a value. */ hint: Optional<(String | Document)>; /** * When true, creates a new document if no document matches the query. Defaults to false. - * + * * This options is sent only if the caller explicitly provides a value. */ upsert: Optional; @@ -104,14 +105,14 @@ class UpdateManyModel implements WriteModel { /** * A set of filters specifying to which array elements an update should apply. - * + * * This option is sent only if the caller explicitly provides a value. */ arrayFilters: Optional; /** * Specifies a collation. - * + * * This option is sent only if the caller explicitly provides a value. */ collation: Optional; @@ -119,14 +120,14 @@ class UpdateManyModel implements WriteModel { /** * The index to use. Specify either the index name as a string or the index key pattern. If * specified, then the query system will only consider plans using the hinted index. - * + * * This option is only sent if the caller explicitly provides a value. */ hint: Optional<(String | Document)>; /** * When true, creates a new document if no document matches the query. Defaults to false. - * + * * This options is sent only if the caller explicitly provides a value. */ upsert: Optional; @@ -145,7 +146,7 @@ class ReplaceOneModel implements WriteModel { /** * Specifies a collation. - * + * * This option is sent only if the caller explicitly provides a value. */ collation: Optional; @@ -153,14 +154,14 @@ class ReplaceOneModel implements WriteModel { /** * The index to use. Specify either the index name as a string or the index key pattern. If * specified, then the query system will only consider plans using the hinted index. - * + * * This option is only sent if the caller explicitly provides a value. */ hint: Optional<(String | Document)>; /** * When true, creates a new document if no document matches the query. Defaults to false. - * + * * This options is sent only if the caller explicitly provides a value. */ upsert: Optional; @@ -174,7 +175,7 @@ class DeleteOneModel implements WriteModel { /** * Specifies a collation. - * + * * This option is sent only if the caller explicitly provides a value. */ collation: Optional; @@ -182,7 +183,7 @@ class DeleteOneModel implements WriteModel { /** * The index to use. Specify either the index name as a string or the index key pattern. If * specified, then the query system will only consider plans using the hinted index. - * + * * This option is only sent if the caller explicitly provides a value. */ hint: Optional<(String | Document)>; @@ -196,7 +197,7 @@ class DeleteManyModel implements WriteModel { /** * Specifies a collation. - * + * * This option is sent only if the caller explicitly provides a value. */ collation: Optional; @@ -204,7 +205,7 @@ class DeleteManyModel implements WriteModel { /** * The index to use. Specify either the index name as a string or the index key pattern. If * specified, then the query system will only consider plans using the hinted index. - * + * * This option is only sent if the caller explicitly provides a value. */ hint: Optional<(String | Document)>; @@ -244,16 +245,16 @@ class BulkWriteOptions { * Whether the operations in this bulk write should be executed in the order in which they were * specified. If false, writes will continue to be executed if an individual write fails. If * true, writes will stop executing if an individual write fails. - * + * * Defaults to false. */ ordered: Optional; /** * If true, allows the writes to opt out of document-level validation. - * + * * Defaults to false. - * + * * This option is only sent if the caller explicitly provides a value. */ bypassDocumentValidation: Optional; @@ -262,14 +263,14 @@ class BulkWriteOptions { * A map of parameter names and values to apply to all operations within the bulk write. Value * must be constant or closed expressions that do not reference document fields. Parameters can * then be accessed as variables in an aggregate expression context (e.g. "$$var"). - * + * * This option is only sent if the caller explicitly provides a value. */ let: Optional; /** * The write concern to use for this bulk write. - * + * * NOT REQUIRED TO IMPLEMENT. Drivers MUST expose this option if retrieving a handle to a * client with a different write concern configured than that of the user's standard URI * options is nontrivial. Drivers MAY omit this option if they provide a way to retrieve a @@ -281,7 +282,7 @@ class BulkWriteOptions { /** * Whether detailed results for each successful operation should be included in the returned * BulkWriteResult. - * + * * Defaults to false. */ verboseResults: Optional; @@ -295,7 +296,7 @@ class BulkWriteResult { /** * Indicates whether this write result was acknowledged. If not, then all other members of this * result will be undefined. - * + * * NOT REQUIRED TO IMPLEMENT. See [here](../crud/crud.md#write-results) for more guidance on * modeling unacknowledged results. */ @@ -304,7 +305,7 @@ class BulkWriteResult { /** * Indicates whether the results are verbose. If false, the insertResults, updateResults, and * deleteResults fields in this result will be undefined. - * + * * NOT REQUIRED TO IMPLEMENT. See below for other ways to differentiate summary results from * verbose results. */ @@ -374,7 +375,7 @@ class UpdateResult { /** * The number of documents that were upserted. - * + * * NOT REQUIRED TO IMPLEMENT. Drivers may choose not to provide this property so long as it is * always possible to discern whether an upsert took place. */ @@ -469,6 +470,8 @@ The `bulkWrite` server command has the following format: Drivers SHOULD use document sequences ([`OP_MSG`](../message/OP_MSG.rst) payload type 1) for the `ops` and `nsInfo` fields. Drivers MUST NOT use document sequences when auto-encryption is enabled. +The `bulkWrite` command is executed on the "admin" database. + ### Operations The `ops` field is a list of write operation documents. The first entry in each document has @@ -488,6 +491,12 @@ documents have the following format: Drivers MUST add an `_id` field at the beginning of the insert document if one is not already present. +When a user executes a bulk write with an unacknowledged write concern, drivers SHOULD check the +size of the insert document to verify that it does not exceed `maxBsonObjectSize`. For acknowledged +bulk writes, drivers MAY rely on the server to return an error if the document exceeds +`maxBsonObjectSize`. The value for `maxBsonObjectSize` can be retrieved from the selected server's +`hello` response. + #### Update ```json @@ -537,7 +546,7 @@ should be provided as `false` if `verboseResults` was set to `true`; otherwise, provided as `true`. This field MUST always be defined in the `bulkWrite` command regardless of whether the user specified a value for `verboseResults`. -## Command batching +## Command Batching Drivers MUST accept an arbitrary number of operations as input to the `Client.bulkWrite` method. Because the server imposes restrictions on the size of write operations, this means that a single @@ -550,11 +559,6 @@ Each of these values can be retrieved from the selected server's `hello` command The `ops` array MUST NOT include more than `maxWriteBatchSize` operations. -### Document Size - -All entries within the `ops` array MUST be within `maxBsonObjectSize` bytes. Drivers MUST throw an -exception if an operation exceeds this size. - ### Total Message Size #### Encrypted bulk writes @@ -625,7 +629,7 @@ The documents in the results cursor have the following format: ``` Note that the responses do not contain information about the type of operation that was performed. -Drivers MAY need to maintain the user's list of write models to infer which type of result should +Drivers may need to maintain the user's list of write models to infer which type of result should be recorded. ### Handling Insert Results @@ -634,8 +638,8 @@ Unlike the other result types, `InsertOneResult` contains an `insertedId` field driver-side, either by recording the `_id` field present in the user's insert document or creating and adding one. Drivers MUST only record these `insertedId`s in a `BulkWriteResult` when a successful response for the insert operation (i.e. `{ "ok": 1, "n": 1 }`) is received in the -results cursor. This ensures that drivers only report `insertedId`s to users when it is confirmed -that the insert succeeded. +results cursor. This ensures that drivers only report an `insertedId` when it is confirmed that the +insert succeeded. ## Handling errors @@ -676,8 +680,8 @@ The majority of tests for `MongoClient.bulkWrite` are written in the [CRUD unified tests directory](../crud/tests/unified/). Additional prose tests are specified [here](../crud/tests/README.md). These tests require -constructing very large documents to test drivers' implementation of batch splitting, which is not -feasible in the unified test format at the time of writing this specification. +constructing very large documents to test batch splitting, which is not feasible in the unified +test format at the time of writing this specification. ## Q&A @@ -696,7 +700,7 @@ This specification introduces several types that are similar to existing types u future with the introduction of new options and features to the `bulkWrite` command. Introducing new types also provides more clarity to users on the existing differences between the collection-level and client-level bulk write APIs. For example, the `verboseResults` option is only -valid for `MongoClient.bulkWrite`. +available for `MongoClient.bulkWrite`. ### Why are bulk write operation results returned in a cursor? diff --git a/source/transactions/tests/unified/client-bulkWrite.json b/source/transactions/tests/unified/client-bulkWrite.json index 74e99f77af..eab6316797 100644 --- a/source/transactions/tests/unified/client-bulkWrite.json +++ b/source/transactions/tests/unified/client-bulkWrite.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite transactions", - "schemaVersion": "1.18", + "schemaVersion": "1.20", "runOnRequirements": [ { "minServerVersion": "8.0", @@ -89,10 +89,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "transaction-tests", - "coll": "coll0" - }, + "namespace": "transactions-tests.coll0", "document": { "_id": 8, "x": 88 @@ -101,10 +98,7 @@ }, { "updateOne": { - "namespace": { - "db": "transaction-tests", - "coll": "coll0" - }, + "namespace": "transactions-tests.coll0", "filter": { "_id": 1 }, @@ -117,10 +111,7 @@ }, { "updateMany": { - "namespace": { - "db": "transaction-tests", - "coll": "coll0" - }, + "namespace": "transactions-tests.coll0", "filter": { "$and": [ { @@ -144,10 +135,7 @@ }, { "replaceOne": { - "namespace": { - "db": "transaction-tests", - "coll": "coll0" - }, + "namespace": "transactions-tests.coll0", "filter": { "_id": 4 }, @@ -159,10 +147,7 @@ }, { "deleteOne": { - "namespace": { - "db": "transaction-tests", - "coll": "coll0" - }, + "namespace": "transactions-tests.coll0", "filter": { "_id": 5 } @@ -170,10 +155,7 @@ }, { "deleteMany": { - "namespace": { - "db": "transaction-tests", - "coll": "coll0" - }, + "namespace": "transactions-tests.coll0", "filter": { "$and": [ { @@ -337,7 +319,7 @@ ], "nsInfo": [ { - "ns": "transaction-tests.coll0" + "ns": "transactions-tests.coll0" } ], "errorsOnly": false, diff --git a/source/transactions/tests/unified/client-bulkWrite.yml b/source/transactions/tests/unified/client-bulkWrite.yml index de7a80d67c..746129dbed 100644 --- a/source/transactions/tests/unified/client-bulkWrite.yml +++ b/source/transactions/tests/unified/client-bulkWrite.yml @@ -1,5 +1,5 @@ description: "client bulkWrite transactions" -schemaVersion: "1.18" +schemaVersion: "1.20" runOnRequirements: - minServerVersion: "8.0" topologies: @@ -45,46 +45,34 @@ tests: session: *session0 models: - insertOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: &namespace "transactions-tests.coll0" document: &insertDoc { _id: 8, x: 88 } - updateOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: &updateOneFilter _id: 1 update: &updateOneUpdate $inc: { x: 1 } - updateMany: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: &updateManyFilter $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] update: &updateManyUpdate $inc: { x: 2 } - replaceOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: &replaceOneFilter _id: 4 replacement: &replacementDoc { x: 44 } upsert: true - deleteOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: &deleteOneFilter _id: 5 - deleteMany: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: &deleteManyFilter $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] verboseResults: true @@ -153,7 +141,7 @@ tests: filter: *deleteManyFilter multi: true nsInfo: - - ns: "transaction-tests.coll0" + - ns: *namespace errorsOnly: false ordered: true - commandStartedEvent: diff --git a/source/transactions/transactions.md b/source/transactions/transactions.md index fcd528cefa..2176e291f1 100644 --- a/source/transactions/transactions.md +++ b/source/transactions/transactions.md @@ -1073,6 +1073,8 @@ objective of avoiding duplicate commits. ## **Changelog** +- TODO: Add bulkWrite to the list of commands allowed in transactions. + - 2024-02-15: Migrated from reStructuredText to Markdown. - 2023-11-22: Specify that non-transient transaction errors abort the transaction\ diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index d43dca11e8..032052e70d 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -923,11 +923,11 @@ The structure of this object is as follows: to have. The test runner MUST assert that the error does not contain any of the specified labels (e.g. using the `hasErrorLabel` method). -- `writeErrors`: Optional document. A map of `writeError`s expected to be present in the error. The `writeErrors` document +- `writeErrors`: Optional document. The write errors expected to be present in the error. The `writeErrors` document contains numeric keys representing the index of the write that failed and `writeError` object values. The test runner MUST assert that the error contains a `writeError` for each index present in `writeErrors` and MUST assert that the `writeError`s - match as root-level documents according to the rules in [Evaluating Matches](#evaluating-matches). The test runner MUST assert - that the error does not contain any additional `writeError`s. + match as root-level documents according to the rules in [Evaluating Matches](#evaluating-matches). The test runner MUST + assert that the error does not contain any additional `writeError`s. - `writeConcernErrors`: Optional array of one or more objects. An ordered list of write concern errors expected to be present in the error. The test runner MUST assert that each `writeConcernError` in this list matches the @@ -1461,11 +1461,19 @@ arguments: ordered: true ``` -Because the `insertResults`, `updateResults`, and `deleteResults` may be absent or empty in the `BulkWriteResult` returned from a summary-only bulk write, the `clientBulkWrite` operation MUST use the [$$unsetOrMatches](#unsetormatches) operator for assertions on these fields when `verboseResults` is not set to true. This requirement also applies to result objects defined in the `expectedResult` field of [expectedError](#expectederror). +Because the `insertResults`, `updateResults`, and `deleteResults` may be absent or empty in the `BulkWriteResult` returned +from a summary-only bulk write, the `clientBulkWrite` operation MUST use the [$$unsetOrMatches](#unsetormatches) operator +for assertions on these fields when `verboseResults` is not set to true. This requirement also applies to result objects +defined in the `expectedResult` field of [expectedError](#expectederror). -The `BulkWriteException` thrown by `MongoClient.bulkWrite` contains an optional `error` field that stores a top-level error that occurred during the bulk write. Test runners MUST inspect the contents of this field when making assertions based on the contents of the `errorCode` and `errorContains` fields in [expectedError](#expectederror). +The `BulkWriteException` thrown by `MongoClient.bulkWrite` contains an optional `error` field that stores a top-level error +that occurred during the bulk write. Test runners MUST inspect the contents of this field when making assertions based on +the contents of the `errorCode` and `errorContains` fields in [expectedError](#expectederror). -`BulkWriteException` also contains `writeErrors` and `writeConcernErrors` fields that define the individual write errors and write concern errors that occurred during the bulk write. Unified tests SHOULD use `writeErrors` and `writeConcernErrors` in `expectedError` to assert on the contents of these fields. Test runners MUST NOT inspect the contents of these fields when making assertions based on any other fields defined in `expectedError`. +`BulkWriteException` also contains `writeErrors` and `writeConcernErrors` fields that define the individual write errors and +write concern errors that occurred during the bulk write. Unified tests SHOULD use `writeErrors` and `writeConcernErrors` in +`expectedError` to assert on the contents of these fields. Test runners MUST NOT inspect the contents of these fields when +making assertions based on any other fields defined in `expectedError`. #### watch @@ -3426,6 +3434,9 @@ other specs *and* collating spec changes developed in parallel or during the sam ## Changelog +- TODO: **Schema version 1.20.**\ + Add `writeErrors` and `writeConcernErrors` field to `expectedError` for the client-level bulk write API. + - 2024-02-23: Require test runners to gossip cluster time from internal MongoClient to each session entity. - 2024-02-14: Clarify that errors raised from callback operations should always propagate to `withTransaction`. diff --git a/source/versioned-api/tests/crud-api-version-1.json b/source/versioned-api/tests/crud-api-version-1.json index 6fda017065..d08a42835e 100644 --- a/source/versioned-api/tests/crud-api-version-1.json +++ b/source/versioned-api/tests/crud-api-version-1.json @@ -436,10 +436,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "versioned-api-tests", - "coll": "test" - }, + "namespace": "versioned-api-tests.test", "document": { "x": 1 } @@ -456,7 +453,6 @@ { "commandStartedEvent": { "commandName": "bulkWrite", - "databaseName": "admin", "command": { "bulkWrite": 1, "ops": [ diff --git a/source/versioned-api/tests/crud-api-version-1.yml b/source/versioned-api/tests/crud-api-version-1.yml index 1373b6dc95..2929975a14 100644 --- a/source/versioned-api/tests/crud-api-version-1.yml +++ b/source/versioned-api/tests/crud-api-version-1.yml @@ -163,23 +163,20 @@ tests: arguments: models: - insertOne: - namespace: - db: *databaseName - coll: *collectionName + namespace: &namespace "versioned-api-tests.test" document: { x: 1 } expectEvents: - client: *client events: - commandStartedEvent: commandName: bulkWrite - databaseName: *adminDatabaseName command: bulkWrite: 1 ops: - insert: 0 document: { x: 1 } nsInfo: - - { ns: "versioned-api-tests.test" } + - { ns: *namespace } <<: *expectedApiVersion - description: "countDocuments appends declared API version" From bc888f8ca629f1307f0f22013a2a24999202cbc7 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 29 Feb 2024 09:44:51 -0700 Subject: [PATCH 10/90] schema version updates --- .../client-bulkWrite-delete-options.yml | 11 ++-- .../client-bulkWrite-errors.yml | 9 +-- .../client-bulkWrite-mixed-namespaces.yml | 39 +++++------ .../client-bulkWrite-options.yml | 7 +- .../client-bulkWrite-ordered.yml | 15 ++--- .../client-bulkWrite-results.yml | 11 ++-- .../client-bulkWrite-update-options.yml | 7 +- .../unified/client-bulkWrite-serverErrors.yml | 65 +++++++------------ .../tests/unified/client-bulkWrite.yml | 7 +- .../tests/crud-api-version-1.yml | 3 +- 10 files changed, 75 insertions(+), 99 deletions(-) diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml index f69694e2cc..55d7d5365e 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml @@ -23,6 +23,9 @@ initialData: - { _id: 2, x: 22 } - { _id: 3, x: 33 } +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + tests: - description: "client bulk write delete with collation" runOnRequirements: @@ -33,9 +36,7 @@ tests: arguments: models: - deleteOne: - namespace: &namespace - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: &deleteOneFilter _id: 1 collation: &collation @@ -85,9 +86,7 @@ tests: arguments: models: - deleteOne: - namespace: &namespace - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: &deleteOneFilter _id: 1 hint: &hint _id_ diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml index fde7a6e141..809bf5d314 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml @@ -1,6 +1,6 @@ description: "client bulkWrite errors" -schemaVersion: "1.0" +schemaVersion: "1.20" createEntities: - client: @@ -25,6 +25,9 @@ initialData: - &document2 { _id: 2, x: 2 } - &document3 { _id: 3, x: 3 } +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + tests: - description: "an individual operation fails during an ordered bulkWrite" runOnRequirements: @@ -35,9 +38,7 @@ tests: arguments: models: - deleteOne: - namespace: &namespace - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: { _id: 1 } - deleteOne: namespace: *namespace diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml index cc7f3108ed..39e4d3f898 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml @@ -9,7 +9,7 @@ createEntities: - database: id: &database0 database0 client: *client0 - databaseName: &database0Name bulkWrite-db0 + databaseName: &database0Name db0 - collection: id: &collection0 collection0 database: *database0 @@ -21,7 +21,7 @@ createEntities: - database: id: &database1 database1 client: *client0 - databaseName: &database1Name bulkWrite-db1 + databaseName: &database1Name db1 - collection: id: &collection2 collection2 database: *database1 @@ -42,6 +42,11 @@ initialData: - { _id: 3, x: 33 } - { _id: 4, x: 44 } +_yamlAnchors: + db0Coll0Namespace: &db0Coll0Namespace "db0.coll0" + db0Coll1Namespace: &db0Coll1Namespace "db0.coll1" + db1Coll2Namespace: &db1Coll2Namespace "db1.coll2" + tests: - description: "client bulkWrite with mixed namespaces" runOnRequirements: @@ -52,41 +57,29 @@ tests: arguments: models: - insertOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *db0Coll0Namespace document: &document1 _id: 1 - insertOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *db0Coll0Namespace document: &document2 _id: 2 - updateOne: - namespace: - db: *database0Name - coll: *collection1Name + namespace: *db0Coll1Namespace filter: &filter1 _id: 1 update: &update $inc: { x: 1 } - deleteOne: - namespace: - db: *database1Name - coll: *collection2Name + namespace: *db1Coll2Namespace filter: &filter2 _id: 3 - deleteOne: - namespace: - db: *database0Name - coll: *collection1Name + namespace: *db0Coll1Namespace filter: &filter3 _id: 2 - replaceOne: - namespace: - db: *database1Name - coll: *collection2Name + namespace: *db1Coll2Namespace filter: &filter4 _id: 4 replacement: &replacement @@ -143,9 +136,9 @@ tests: updateMods: *replacement multi: false nsInfo: - - ns: bulkWrite-db0.coll0 - - ns: bulkWrite-db0.coll1 - - ns: bulkWrite-db1.coll2 + - ns: *db0Coll0Namespace + - ns: *db0Coll1Namespace + - ns: *db1Coll2Namespace outcome: - databaseName: *database0Name collectionName: *collection0Name diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml index 23dacd9077..3b04d5bc36 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml @@ -22,6 +22,9 @@ initialData: - &doc1 { _id: 1, x: 11 } - &doc2 { _id: 2, x: 22 } +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + tests: - description: "client bulkWrite comment" runOnRequirements: @@ -32,9 +35,7 @@ tests: arguments: models: - insertOne: - namespace: &namespace - db: *database0Name - coll: *collection0Name + namespace: *namespace document: &doc3 { _id: 3, x: 33 } comment: &comment { "bulk": "write" } verboseResults: true diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml index d09f3d4567..793c4d56dc 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml @@ -20,6 +20,9 @@ initialData: databaseName: *database0Name documents: [] +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + tests: - description: "client bulkWrite with ordered: false" runOnRequirements: @@ -30,9 +33,7 @@ tests: arguments: models: - insertOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace document: &document { _id: 1 } verboseResults: true ordered: false @@ -68,9 +69,7 @@ tests: arguments: models: - insertOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace document: &document { _id: 4 } verboseResults: true ordered: true @@ -106,9 +105,7 @@ tests: arguments: models: - insertOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace document: &document { _id: 4 } verboseResults: true expectResult: diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml index 9874430ce9..dc6caa328b 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml @@ -1,6 +1,6 @@ description: "client bulkWrite results" -schemaVersion: "1.18" +schemaVersion: "1.0" createEntities: - client: @@ -26,6 +26,9 @@ initialData: - { _id: 6, x: 66 } - { _id: 7, x: 77 } +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + tests: - description: "client bulkWrite with verboseResults: true returns detailed results" runOnRequirements: @@ -36,9 +39,7 @@ tests: arguments: models: &models - insertOne: - namespace: &namespace - db: *database0Name - coll: *collection0Name + namespace: *namespace document: &insertDoc { _id: 8, x: 88 } - updateOne: @@ -133,7 +134,7 @@ tests: multi: true nsInfo: &nsInfo - - ns: "crud-tests.coll0" + ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml index d7f6874ac2..2aab53aaf1 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml @@ -24,6 +24,9 @@ initialData: - { _id: 3, array: [ 1, 2, 3 ] } - { _id: 4, array: [ 1, 2, 3 ] } +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + tests: - description: "client bulkWrite update with arrayFilters" runOnRequirements: @@ -34,9 +37,7 @@ tests: arguments: models: - updateOne: - namespace: &namespace - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: &updateOneFilter _id: 1 update: &arrayFiltersUpdateOneMods diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml index a882a8f6c7..1b85bc5675 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml @@ -1,5 +1,5 @@ description: "client bulkWrite retryable writes" -schemaVersion: "1.18" +schemaVersion: "1.20" runOnRequirements: - minServerVersion: "8.0" topologies: [ replicaset ] @@ -25,6 +25,9 @@ initialData: - { _id: 2, x: 22 } - { _id: 3, x: 33 } +_yamlAnchors: + namespace: &namespace "retryable-writes-tests.coll0" + tests: - description: "client bulkWrite with no multi: true operations succeeds after retryable top-level error" operations: @@ -45,27 +48,19 @@ tests: arguments: models: - insertOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace document: { _id: 4, x: 44 } - updateOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: { _id: 1 } update: $inc: { x: 1 } - replaceOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: { _id: 2 } replacement: { x: 222 } - deleteOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: { _id: 3 } expectResult: insertedCount: 1 @@ -102,7 +97,7 @@ tests: filter: { _id: 3 } multi: false nsInfo: - - ns: retryable-writes-tests.coll0 + - ns: *namespace - commandStartedEvent: commandName: bulkWrite databaseName: admin @@ -123,7 +118,7 @@ tests: filter: { _id: 3 } multi: false nsInfo: - - ns: retryable-writes-tests.coll0 + - ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name @@ -150,16 +145,12 @@ tests: arguments: models: - updateMany: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: { _id: 1 } update: $inc: { x: 1 } - deleteMany: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: { _id: 3 } expectError: errorCode: 189 @@ -181,7 +172,7 @@ tests: filter: { _id: 3 } multi: true nsInfo: - - ns: retryable-writes-tests.coll0 + - ns: *namespace - description: "client bulkWrite with no multi: true operations succeeds after retryable writeConcernError" operations: - object: testRunner @@ -203,27 +194,19 @@ tests: arguments: models: - insertOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace document: { _id: 4, x: 44 } - updateOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: { _id: 1 } update: $inc: { x: 1 } - replaceOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: { _id: 2 } replacement: { x: 222 } - deleteOne: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: { _id: 3 } expectResult: insertedCount: 1 @@ -260,7 +243,7 @@ tests: filter: { _id: 3 } multi: false nsInfo: - - ns: retryable-writes-tests.coll0 + - ns: *namespace - commandStartedEvent: commandName: bulkWrite databaseName: admin @@ -281,7 +264,7 @@ tests: filter: { _id: 3 } multi: false nsInfo: - - ns: retryable-writes-tests.coll0 + - ns: *namespace - description: "client bulkWrite with multi: true operations fails after retryable writeConcernError" operations: - object: testRunner @@ -303,16 +286,12 @@ tests: arguments: models: - updateMany: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: { _id: 1 } update: $inc: { x: 1 } - deleteMany: - namespace: - db: *database0Name - coll: *collection0Name + namespace: *namespace filter: { _id: 3 } expectError: writeConcernErrors: @@ -335,4 +314,4 @@ tests: filter: { _id: 3 } multi: true nsInfo: - - ns: retryable-writes-tests.coll0 + - ns: *namespace diff --git a/source/transactions/tests/unified/client-bulkWrite.yml b/source/transactions/tests/unified/client-bulkWrite.yml index 746129dbed..6bbfa3248e 100644 --- a/source/transactions/tests/unified/client-bulkWrite.yml +++ b/source/transactions/tests/unified/client-bulkWrite.yml @@ -23,6 +23,9 @@ createEntities: id: &session0 session0 client: *client0 +_yamlAnchors: + namespace: &namespace "transactions-tests.coll0" + initialData: - databaseName: *database0Name collectionName: *collection0Name @@ -35,7 +38,7 @@ initialData: - { _id: 7, x: 77 } tests: - - description: "transactional client bulkWrite" + - description: "client bulkWrite in a transaction" operations: - object: *session0 name: startTransaction @@ -45,7 +48,7 @@ tests: session: *session0 models: - insertOne: - namespace: &namespace "transactions-tests.coll0" + namespace: *namespace document: &insertDoc { _id: 8, x: 88 } - updateOne: diff --git a/source/versioned-api/tests/crud-api-version-1.yml b/source/versioned-api/tests/crud-api-version-1.yml index 2929975a14..572391241b 100644 --- a/source/versioned-api/tests/crud-api-version-1.yml +++ b/source/versioned-api/tests/crud-api-version-1.yml @@ -34,6 +34,7 @@ _yamlAnchors: apiVersion: "1" apiStrict: { $$unsetOrMatches: false } apiDeprecationErrors: true + namespace: &namespace "versioned-api-tests.test" initialData: - collectionName: *collectionName @@ -163,7 +164,7 @@ tests: arguments: models: - insertOne: - namespace: &namespace "versioned-api-tests.test" + namespace: *namespace document: { x: 1 } expectEvents: - client: *client From b425c23ae2dea6fca341bcbe120a583a62517a97 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 29 Feb 2024 10:03:10 -0700 Subject: [PATCH 11/90] move run on requirements to top level --- .../client-bulkWrite-delete-options.yml | 7 ++----- .../new-bulk-write/client-bulkWrite-errors.yml | 5 ++--- .../client-bulkWrite-mixed-namespaces.yml | 5 ++--- .../new-bulk-write/client-bulkWrite-options.yml | 9 ++------- .../new-bulk-write/client-bulkWrite-ordered.yml | 9 ++------- .../new-bulk-write/client-bulkWrite-results.yml | 9 ++------- .../client-bulkWrite-update-options.yml | 11 ++--------- 7 files changed, 14 insertions(+), 41 deletions(-) diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml index 55d7d5365e..e9e07b0e01 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml @@ -1,6 +1,7 @@ description: "client bulkWrite delete options" - schemaVersion: "1.0" +runOnRequirements: + - minServerVersion: "8.0" createEntities: - client: @@ -28,8 +29,6 @@ _yamlAnchors: tests: - description: "client bulk write delete with collation" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -78,8 +77,6 @@ tests: collectionName: *collection0Name documents: [] - description: "client bulk write delete with hint" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml index 809bf5d314..f6b302367d 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml @@ -1,6 +1,7 @@ description: "client bulkWrite errors" - schemaVersion: "1.20" +runOnRequirements: + - minServerVersion: "8.0" createEntities: - client: @@ -30,8 +31,6 @@ _yamlAnchors: tests: - description: "an individual operation fails during an ordered bulkWrite" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml index 39e4d3f898..c363b47c42 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml @@ -1,6 +1,7 @@ description: "client bulkWrite with mixed namespaces" - schemaVersion: "1.0" +runOnRequirements: + - minServerVersion: "8.0" createEntities: - client: @@ -49,8 +50,6 @@ _yamlAnchors: tests: - description: "client bulkWrite with mixed namespaces" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml index 3b04d5bc36..bbb097dd17 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml @@ -1,6 +1,7 @@ description: "client bulkWrite top-level options" - schemaVersion: "1.0" +runOnRequirements: + - minServerVersion: "8.0" createEntities: - client: @@ -27,8 +28,6 @@ _yamlAnchors: tests: - description: "client bulkWrite comment" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -65,8 +64,6 @@ tests: - *doc2 - *doc3 - description: "client bulkWrite bypassDocumentValidation" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -103,8 +100,6 @@ tests: - *doc2 - *doc4 - description: "client bulkWrite let" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml index 793c4d56dc..ec0e7d98cf 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml @@ -1,6 +1,7 @@ description: "client bulkWrite with ordered option" - schemaVersion: "1.0" +runOnRequirements: + - minServerVersion: "8.0" createEntities: - client: @@ -25,8 +26,6 @@ _yamlAnchors: tests: - description: "client bulkWrite with ordered: false" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -61,8 +60,6 @@ tests: documents: - *document - description: "client bulkWrite with ordered: true" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -97,8 +94,6 @@ tests: documents: - *document - description: "client bulkWrite defaults to ordered: true" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml index dc6caa328b..5d049008d1 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml @@ -1,6 +1,7 @@ description: "client bulkWrite results" - schemaVersion: "1.0" +runOnRequirements: + - minServerVersion: "8.0" createEntities: - client: @@ -31,8 +32,6 @@ _yamlAnchors: tests: - description: "client bulkWrite with verboseResults: true returns detailed results" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -145,8 +144,6 @@ tests: - { _id: 4, x: 44 } - { _id: 8, x: 88 } - description: "client bulkWrite with verboseResults: false omits detailed results" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -179,8 +176,6 @@ tests: databaseName: *database0Name documents: *outcomeDocuments - description: "client bulkWrite defaults to verboseResults: false" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml index 2aab53aaf1..6abaf68c1b 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml @@ -1,6 +1,7 @@ description: "client bulkWrite update options" - schemaVersion: "1.0" +runOnRequirements: + - minServerVersion: "8.0" createEntities: - client: @@ -29,8 +30,6 @@ _yamlAnchors: tests: - description: "client bulkWrite update with arrayFilters" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -92,8 +91,6 @@ tests: - { _id: 3, array: [ 1, 5, 5 ] } - { _id: 4, array: [ 1, 2, 3 ] } - description: "client bulkWrite update with collation" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -164,8 +161,6 @@ tests: - { _id: 3, array: [ 1, 2, 5 ] } - { _id: 4, array: [ 1, 2, 6 ] } - description: "client bulkWrite update with hint" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -229,8 +224,6 @@ tests: - { _id: 3, array: [ 1, 2, 5 ] } - { _id: 4, array: [ 1, 2, 6 ] } - description: "client bulkWrite update with upsert" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite From d7294cdeb630592340e3ee4f85ab5871dd74a474 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 29 Feb 2024 12:04:07 -0700 Subject: [PATCH 12/90] always use verbose results --- .../client-bulkWrite-delete-options.json | 74 ++++---- .../client-bulkWrite-delete-options.yml | 24 ++- .../client-bulkWrite-errors.json | 70 ++----- .../client-bulkWrite-mixed-namespaces.json | 67 +++---- .../client-bulkWrite-options.json | 43 ++--- .../client-bulkWrite-ordered.json | 38 ++-- .../client-bulkWrite-results.json | 115 +++--------- .../client-bulkWrite-update-options.json | 176 +++++++++--------- .../client-bulkWrite-update-options.yml | 72 +++++-- .../client-bulkWrite-serverErrors.json | 109 ++++++----- .../unified/client-bulkWrite-serverErrors.yml | 32 +++- .../tests/crud-api-version-1.json | 23 ++- .../tests/crud-api-version-1.yml | 14 +- 13 files changed, 390 insertions(+), 467 deletions(-) diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json index 77a67625fb..f2f1be94e0 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json @@ -1,6 +1,11 @@ { "description": "client bulkWrite delete options", "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "createEntities": [ { "client": { @@ -45,14 +50,12 @@ ] } ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, "tests": [ { "description": "client bulk write delete with collation", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -61,10 +64,7 @@ "models": [ { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 1 }, @@ -75,10 +75,7 @@ }, { "deleteMany": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": { "$gt": 1 @@ -89,7 +86,8 @@ } } } - ] + ], + "verboseResults": true }, "expectResult": { "insertedCount": 0, @@ -97,14 +95,15 @@ "matchedCount": 0, "modifiedCount": 0, "deletedCount": 3, - "insertResults": { - "$$unsetOrMatches": {} - }, - "updateResults": { - "$$unsetOrMatches": {} - }, + "insertResults": {}, + "updateResults": {}, "deleteResults": { - "$$unsetOrMatches": {} + "0": { + "deletedCount": 1 + }, + "1": { + "deletedCount": 2 + } } } } @@ -157,11 +156,6 @@ }, { "description": "client bulk write delete with hint", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -170,10 +164,7 @@ "models": [ { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 1 }, @@ -182,10 +173,7 @@ }, { "deleteMany": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": { "$gt": 1 @@ -194,7 +182,8 @@ "hint": "_id_" } } - ] + ], + "verboseResults": true }, "expectResult": { "insertedCount": 0, @@ -202,14 +191,15 @@ "matchedCount": 0, "modifiedCount": 0, "deletedCount": 3, - "insertResults": { - "$$unsetOrMatches": {} - }, - "updateResults": { - "$$unsetOrMatches": {} - }, + "insertResults": {}, + "updateResults": {}, "deleteResults": { - "$$unsetOrMatches": {} + "0": { + "deletedCount": 1 + }, + "1": { + "deletedCount": 2 + } } } } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml index e9e07b0e01..3e97dc1bf9 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml @@ -45,18 +45,20 @@ tests: filter: &deleteManyFilter _id: { $gt: 1 } collation: *collation + verboseResults: true expectResult: insertedCount: 0 upsertedCount: 0 matchedCount: 0 modifiedCount: 0 deletedCount: 3 - insertResults: - $$unsetOrMatches: {} - updateResults: - $$unsetOrMatches: {} + insertResults: {} + updateResults: {} deleteResults: - $$unsetOrMatches: {} + 0: + deletedCount: 1 + 1: + deletedCount: 2 expectEvents: - client: *client0 events: @@ -92,18 +94,20 @@ tests: filter: &deleteManyFilter _id: { $gt: 1 } hint: *hint + verboseResults: true expectResult: insertedCount: 0 upsertedCount: 0 matchedCount: 0 modifiedCount: 0 deletedCount: 3 - insertResults: - $$unsetOrMatches: {} - updateResults: - $$unsetOrMatches: {} + insertResults: {} + updateResults: {} deleteResults: - $$unsetOrMatches: {} + 0: + deletedCount: 1 + 1: + deletedCount: 2 expectEvents: - client: *client0 events: diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json index 10ddb7a5d6..a50730dcb6 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json @@ -1,6 +1,11 @@ { "description": "client bulkWrite errors", - "schemaVersion": "1.0", + "schemaVersion": "1.20", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "createEntities": [ { "client": { @@ -48,14 +53,12 @@ ] } ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, "tests": [ { "description": "an individual operation fails during an ordered bulkWrite", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -64,10 +67,7 @@ "models": [ { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 1 } @@ -75,10 +75,7 @@ }, { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$expr": { "$eq": [ @@ -91,10 +88,7 @@ }, { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 3 } @@ -158,10 +152,7 @@ "models": [ { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 1 } @@ -169,10 +160,7 @@ }, { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$expr": { "$eq": [ @@ -185,10 +173,7 @@ }, { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 3 } @@ -252,10 +237,7 @@ "models": [ { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 1 } @@ -263,10 +245,7 @@ }, { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$expr": { "$eq": [ @@ -279,10 +258,7 @@ }, { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 3 } @@ -346,10 +322,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "document": { "x": 1 } @@ -396,10 +369,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "document": { "_id": 10 } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json index f408a8969f..6db78d2ac2 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json @@ -1,6 +1,11 @@ { "description": "client bulkWrite with mixed namespaces", "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "createEntities": [ { "client": { @@ -14,7 +19,7 @@ "database": { "id": "database0", "client": "client0", - "databaseName": "bulkWrite-db0" + "databaseName": "db0" } }, { @@ -35,7 +40,7 @@ "database": { "id": "database1", "client": "client0", - "databaseName": "bulkWrite-db1" + "databaseName": "db1" } }, { @@ -48,12 +53,12 @@ ], "initialData": [ { - "databaseName": "bulkWrite-db0", + "databaseName": "db0", "collectionName": "coll0", "documents": [] }, { - "databaseName": "bulkWrite-db0", + "databaseName": "db0", "collectionName": "coll1", "documents": [ { @@ -67,7 +72,7 @@ ] }, { - "databaseName": "bulkWrite-db1", + "databaseName": "db1", "collectionName": "coll2", "documents": [ { @@ -81,14 +86,14 @@ ] } ], + "_yamlAnchors": { + "db0Coll0Namespace": "db0.coll0", + "db0Coll1Namespace": "db0.coll1", + "db1Coll2Namespace": "db1.coll2" + }, "tests": [ { "description": "client bulkWrite with mixed namespaces", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -97,10 +102,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "bulkWrite-db0", - "coll": "coll0" - }, + "namespace": "db0.coll0", "document": { "_id": 1 } @@ -108,10 +110,7 @@ }, { "insertOne": { - "namespace": { - "db": "bulkWrite-db0", - "coll": "coll0" - }, + "namespace": "db0.coll0", "document": { "_id": 2 } @@ -119,10 +118,7 @@ }, { "updateOne": { - "namespace": { - "db": "bulkWrite-db0", - "coll": "coll1" - }, + "namespace": "db0.coll1", "filter": { "_id": 1 }, @@ -135,10 +131,7 @@ }, { "deleteOne": { - "namespace": { - "db": "bulkWrite-db1", - "coll": "coll2" - }, + "namespace": "db1.coll2", "filter": { "_id": 3 } @@ -146,10 +139,7 @@ }, { "deleteOne": { - "namespace": { - "db": "bulkWrite-db0", - "coll": "coll1" - }, + "namespace": "db0.coll1", "filter": { "_id": 2 } @@ -157,10 +147,7 @@ }, { "replaceOne": { - "namespace": { - "db": "bulkWrite-db1", - "coll": "coll2" - }, + "namespace": "db1.coll2", "filter": { "_id": 4 }, @@ -269,13 +256,13 @@ ], "nsInfo": [ { - "ns": "bulkWrite-db0.coll0" + "ns": "db0.coll0" }, { - "ns": "bulkWrite-db0.coll1" + "ns": "db0.coll1" }, { - "ns": "bulkWrite-db1.coll2" + "ns": "db1.coll2" } ] } @@ -286,7 +273,7 @@ ], "outcome": [ { - "databaseName": "bulkWrite-db0", + "databaseName": "db0", "collectionName": "coll0", "documents": [ { @@ -298,7 +285,7 @@ ] }, { - "databaseName": "bulkWrite-db0", + "databaseName": "db0", "collectionName": "coll1", "documents": [ { @@ -308,7 +295,7 @@ ] }, { - "databaseName": "bulkWrite-db1", + "databaseName": "db1", "collectionName": "coll2", "documents": [ { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json index 80e5fef102..8b0ed91a14 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json @@ -1,6 +1,11 @@ { "description": "client bulkWrite top-level options", "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "createEntities": [ { "client": { @@ -41,14 +46,12 @@ ] } ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, "tests": [ { "description": "client bulkWrite comment", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -57,10 +60,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "document": { "_id": 3, "x": 33 @@ -129,11 +129,6 @@ }, { "description": "client bulkWrite bypassDocumentValidation", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -142,10 +137,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "document": { "_id": 4, "x": 44 @@ -210,11 +202,6 @@ }, { "description": "client bulkWrite let", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -223,10 +210,7 @@ "models": [ { "updateOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$expr": { "$eq": [ @@ -244,10 +228,7 @@ }, { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$expr": { "$eq": [ diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json index 546a980b2b..b19815b3bf 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json @@ -1,6 +1,11 @@ { "description": "client bulkWrite with ordered option", "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "createEntities": [ { "client": { @@ -32,14 +37,12 @@ "documents": [] } ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, "tests": [ { "description": "client bulkWrite with ordered: false", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -48,10 +51,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "document": { "_id": 1 } @@ -106,11 +106,6 @@ }, { "description": "client bulkWrite with ordered: true", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -119,10 +114,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "document": { "_id": 4 } @@ -177,11 +169,6 @@ }, { "description": "client bulkWrite defaults to ordered: true", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -190,10 +177,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "document": { "_id": 4 } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json index 489b581d06..43238bc5a6 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json @@ -1,6 +1,11 @@ { "description": "client bulkWrite results", - "schemaVersion": "1.18", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "createEntities": [ { "client": { @@ -57,14 +62,12 @@ ] } ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, "tests": [ { "description": "client bulkWrite with verboseResults: true returns detailed results", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -73,10 +76,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "document": { "_id": 8, "x": 88 @@ -85,10 +85,7 @@ }, { "updateOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 1 }, @@ -101,10 +98,7 @@ }, { "updateMany": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$and": [ { @@ -128,10 +122,7 @@ }, { "replaceOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 4 }, @@ -143,10 +134,7 @@ }, { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 5 } @@ -154,10 +142,7 @@ }, { "deleteMany": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$and": [ { @@ -347,11 +332,6 @@ }, { "description": "client bulkWrite with verboseResults: false omits detailed results", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -360,10 +340,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "document": { "_id": 8, "x": 88 @@ -372,10 +349,7 @@ }, { "updateOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 1 }, @@ -388,10 +362,7 @@ }, { "updateMany": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$and": [ { @@ -415,10 +386,7 @@ }, { "replaceOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 4 }, @@ -430,10 +398,7 @@ }, { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 5 } @@ -441,10 +406,7 @@ }, { "deleteMany": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$and": [ { @@ -613,11 +575,6 @@ }, { "description": "client bulkWrite defaults to verboseResults: false", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -626,10 +583,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "document": { "_id": 8, "x": 88 @@ -638,10 +592,7 @@ }, { "updateOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 1 }, @@ -654,10 +605,7 @@ }, { "updateMany": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$and": [ { @@ -681,10 +629,7 @@ }, { "replaceOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 4 }, @@ -696,10 +641,7 @@ }, { "deleteOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 5 } @@ -707,10 +649,7 @@ }, { "deleteMany": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$and": [ { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json index 5b442f3005..c3519cc3be 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json @@ -1,6 +1,11 @@ { "description": "client bulkWrite update options", "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "createEntities": [ { "client": { @@ -65,14 +70,12 @@ ] } ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, "tests": [ { "description": "client bulkWrite update with arrayFilters", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -81,10 +84,7 @@ "models": [ { "updateOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 1 }, @@ -104,10 +104,7 @@ }, { "updateMany": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$and": [ { @@ -136,7 +133,8 @@ ] } } - ] + ], + "verboseResults": true }, "expectResult": { "insertedCount": 0, @@ -144,15 +142,20 @@ "matchedCount": 3, "modifiedCount": 3, "deletedCount": 0, - "insertResults": { - "$$unsetOrMatches": {} - }, + "insertResults": {}, "updateResults": { - "$$unsetOrMatches": {} + "0": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + }, + "1": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": null + } }, - "deleteResults": { - "$$unsetOrMatches": {} - } + "deleteResults": {} } } ], @@ -264,11 +267,6 @@ }, { "description": "client bulkWrite update with collation", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -277,10 +275,7 @@ "models": [ { "updateOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 1 }, @@ -300,10 +295,7 @@ }, { "updateMany": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$and": [ { @@ -334,10 +326,7 @@ }, { "replaceOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 4 }, @@ -353,7 +342,8 @@ } } } - ] + ], + "verboseResults": true }, "expectResult": { "insertedCount": 0, @@ -361,15 +351,25 @@ "matchedCount": 4, "modifiedCount": 4, "deletedCount": 0, - "insertResults": { - "$$unsetOrMatches": {} - }, + "insertResults": {}, "updateResults": { - "$$unsetOrMatches": {} + "0": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + }, + "1": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": null + }, + "2": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + } }, - "deleteResults": { - "$$unsetOrMatches": {} - } + "deleteResults": {} } } ], @@ -498,11 +498,6 @@ }, { "description": "client bulkWrite update with hint", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -511,10 +506,7 @@ "models": [ { "updateOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 1 }, @@ -532,10 +524,7 @@ }, { "updateMany": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "$and": [ { @@ -564,10 +553,7 @@ }, { "replaceOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 4 }, @@ -581,7 +567,8 @@ "hint": "_id_" } } - ] + ], + "verboseResults": true }, "expectResult": { "insertedCount": 0, @@ -589,15 +576,25 @@ "matchedCount": 4, "modifiedCount": 4, "deletedCount": 0, - "insertResults": { - "$$unsetOrMatches": {} - }, + "insertResults": {}, "updateResults": { - "$$unsetOrMatches": {} + "0": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + }, + "1": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": null + }, + "2": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + } }, - "deleteResults": { - "$$unsetOrMatches": {} - } + "deleteResults": {} } } ], @@ -720,11 +717,6 @@ }, { "description": "client bulkWrite update with upsert", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -733,10 +725,7 @@ "models": [ { "updateOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 5 }, @@ -754,10 +743,7 @@ }, { "replaceOne": { - "namespace": { - "db": "crud-tests", - "coll": "coll0" - }, + "namespace": "crud-tests.coll0", "filter": { "_id": 6 }, @@ -771,7 +757,8 @@ "upsert": true } } - ] + ], + "verboseResults": true }, "expectResult": { "insertedCount": 0, @@ -779,15 +766,20 @@ "matchedCount": 0, "modifiedCount": 0, "deletedCount": 0, - "insertResults": { - "$$unsetOrMatches": {} - }, + "insertResults": {}, "updateResults": { - "$$unsetOrMatches": {} + "0": { + "matchedCount": 1, + "modifiedCount": 0, + "upsertedId": 5 + }, + "1": { + "matchedCount": 1, + "modifiedCount": 0, + "upsertedId": 6 + } }, - "deleteResults": { - "$$unsetOrMatches": {} - } + "deleteResults": {} } } ], diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml index 6abaf68c1b..f08629db5d 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml @@ -53,18 +53,24 @@ tests: array.$[i]: 5 arrayFilters: &updateManyArrayFilters - i: { $gte: 2 } + verboseResults: true expectResult: insertedCount: 0 upsertedCount: 0 matchedCount: 3 modifiedCount: 3 deletedCount: 0 - insertResults: - $$unsetOrMatches: {} + insertResults: {} updateResults: - $$unsetOrMatches: {} - deleteResults: - $$unsetOrMatches: {} + 0: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + 1: + matchedCount: 2 + modifiedCount: 2 + upsertedId: null + deleteResults: {} expectEvents: - client: *client0 events: @@ -118,18 +124,28 @@ tests: replacement: &replacement array: [ 1, 2, 6 ] collation: *collation + verboseResults: true expectResult: insertedCount: 0 upsertedCount: 0 matchedCount: 4 modifiedCount: 4 deletedCount: 0 - insertResults: - $$unsetOrMatches: {} + insertResults: {} updateResults: - $$unsetOrMatches: {} - deleteResults: - $$unsetOrMatches: {} + 0: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + 1: + matchedCount: 2 + modifiedCount: 2 + upsertedId: null + 2: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + deleteResults: {} expectEvents: - client: *client0 events: @@ -181,18 +197,28 @@ tests: filter: *replaceOneFilter replacement: *replacement hint: *hint + verboseResults: true expectResult: insertedCount: 0 upsertedCount: 0 matchedCount: 4 modifiedCount: 4 deletedCount: 0 - insertResults: - $$unsetOrMatches: {} + insertResults: {} updateResults: - $$unsetOrMatches: {} - deleteResults: - $$unsetOrMatches: {} + 0: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + 1: + matchedCount: 2 + modifiedCount: 2 + upsertedId: null + 2: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + deleteResults: {} expectEvents: - client: *client0 events: @@ -241,18 +267,24 @@ tests: _id: 6 replacement: *replacement upsert: true + verboseResults: true expectResult: insertedCount: 0 upsertedCount: 2 matchedCount: 0 modifiedCount: 0 deletedCount: 0 - insertResults: - $$unsetOrMatches: {} + insertResults: {} updateResults: - $$unsetOrMatches: {} - deleteResults: - $$unsetOrMatches: {} + 0: + matchedCount: 1 + modifiedCount: 0 + upsertedId: 5 + 1: + matchedCount: 1 + modifiedCount: 0 + upsertedId: 6 + deleteResults: {} expectEvents: - client: *client0 events: diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json index 8d3324b594..4091d66d61 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite retryable writes", - "schemaVersion": "1.18", + "schemaVersion": "1.20", "runOnRequirements": [ { "minServerVersion": "8.0", @@ -53,6 +53,9 @@ ] } ], + "_yamlAnchors": { + "namespace": "retryable-writes-tests.coll0" + }, "tests": [ { "description": "client bulkWrite with no multi: true operations succeeds after retryable top-level error", @@ -86,10 +89,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "document": { "_id": 4, "x": 44 @@ -98,10 +98,7 @@ }, { "updateOne": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "filter": { "_id": 1 }, @@ -114,10 +111,7 @@ }, { "replaceOne": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "filter": { "_id": 2 }, @@ -128,16 +122,14 @@ }, { "deleteOne": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "filter": { "_id": 3 } } } - ] + ], + "verboseResults": true }, "expectResult": { "insertedCount": 1, @@ -146,13 +138,26 @@ "modifiedCount": 2, "deletedCount": 1, "insertResults": { - "$$unsetOrMatches": {} + "0": { + "insertedId": 4 + } }, "updateResults": { - "$$unsetOrMatches": {} + "1": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + }, + "2": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + } }, "deleteResults": { - "$$unsetOrMatches": {} + "3": { + "deletedCount": 1 + } } } } @@ -319,10 +324,7 @@ "models": [ { "updateMany": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "filter": { "_id": 1 }, @@ -335,10 +337,7 @@ }, { "deleteMany": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "filter": { "_id": 3 } @@ -431,10 +430,7 @@ "models": [ { "insertOne": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "document": { "_id": 4, "x": 44 @@ -443,10 +439,7 @@ }, { "updateOne": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "filter": { "_id": 1 }, @@ -459,10 +452,7 @@ }, { "replaceOne": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "filter": { "_id": 2 }, @@ -473,16 +463,14 @@ }, { "deleteOne": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "filter": { "_id": 3 } } } - ] + ], + "verboseResults": true }, "expectResult": { "insertedCount": 1, @@ -491,13 +479,26 @@ "modifiedCount": 2, "deletedCount": 1, "insertResults": { - "$$unsetOrMatches": {} + "0": { + "insertedId": 4 + } }, "updateResults": { - "$$unsetOrMatches": {} + "1": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + }, + "2": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": null + } }, "deleteResults": { - "$$unsetOrMatches": {} + "3": { + "deletedCount": 1 + } } } } @@ -647,10 +648,7 @@ "models": [ { "updateMany": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "filter": { "_id": 1 }, @@ -663,10 +661,7 @@ }, { "deleteMany": { - "namespace": { - "db": "retryable-writes-tests", - "coll": "coll0" - }, + "namespace": "retryable-writes-tests.coll0", "filter": { "_id": 3 } diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml index 1b85bc5675..874afe4df1 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml @@ -62,6 +62,7 @@ tests: - deleteOne: namespace: *namespace filter: { _id: 3 } + verboseResults: true expectResult: insertedCount: 1 upsertedCount: 0 @@ -69,11 +70,20 @@ tests: modifiedCount: 2 deletedCount: 1 insertResults: - $$unsetOrMatches: {} + 0: + insertedId: 4 updateResults: - $$unsetOrMatches: {} + 1: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + 2: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null deleteResults: - $$unsetOrMatches: {} + 3: + deletedCount: 1 expectEvents: - client: *client0 events: @@ -208,6 +218,7 @@ tests: - deleteOne: namespace: *namespace filter: { _id: 3 } + verboseResults: true expectResult: insertedCount: 1 upsertedCount: 0 @@ -215,11 +226,20 @@ tests: modifiedCount: 2 deletedCount: 1 insertResults: - $$unsetOrMatches: {} + 0: + insertedId: 4 updateResults: - $$unsetOrMatches: {} + 1: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null + 2: + matchedCount: 1 + modifiedCount: 1 + upsertedId: null deleteResults: - $$unsetOrMatches: {} + 3: + deletedCount: 1 expectEvents: - client: *client0 events: diff --git a/source/versioned-api/tests/crud-api-version-1.json b/source/versioned-api/tests/crud-api-version-1.json index d08a42835e..da39bc900c 100644 --- a/source/versioned-api/tests/crud-api-version-1.json +++ b/source/versioned-api/tests/crud-api-version-1.json @@ -50,7 +50,8 @@ }, "apiDeprecationErrors": true } - ] + ], + "namespace": "versioned-api-tests.test" }, "initialData": [ { @@ -438,11 +439,27 @@ "insertOne": { "namespace": "versioned-api-tests.test", "document": { - "x": 1 + "_id": 6, + "x": 6 } } } - ] + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 6 + } + }, + "updateResults": {}, + "deleteResults": {} } } ], diff --git a/source/versioned-api/tests/crud-api-version-1.yml b/source/versioned-api/tests/crud-api-version-1.yml index 572391241b..dd21967902 100644 --- a/source/versioned-api/tests/crud-api-version-1.yml +++ b/source/versioned-api/tests/crud-api-version-1.yml @@ -165,7 +165,19 @@ tests: models: - insertOne: namespace: *namespace - document: { x: 1 } + document: { _id: 6, x: 6 } + verboseResults: true + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 6 + updateResults: {} + deleteResults: {} expectEvents: - client: *client events: From 82799a75a012f2ea2f7593681d4221df37dd5dff Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 29 Feb 2024 12:42:23 -0700 Subject: [PATCH 13/90] clean up yaml anchors --- .../client-bulkWrite-delete-options.json | 6 +- .../client-bulkWrite-delete-options.yml | 27 ++- .../client-bulkWrite-errors.json | 18 +- .../client-bulkWrite-errors.yml | 19 +- .../client-bulkWrite-options.json | 19 +- .../client-bulkWrite-options.yml | 28 +-- .../client-bulkWrite-ordered.json | 22 +- .../client-bulkWrite-ordered.yml | 16 +- .../client-bulkWrite-results.yml | 204 ++++++++++++++---- .../client-bulkWrite-update-options.json | 6 +- .../client-bulkWrite-update-options.yml | 115 +++++----- .../tests/unified/client-bulkWrite.json | 5 +- .../tests/unified/client-bulkWrite.yml | 47 ++-- 13 files changed, 340 insertions(+), 192 deletions(-) diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json index f2f1be94e0..21fa3a8142 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json @@ -51,7 +51,11 @@ } ], "_yamlAnchors": { - "namespace": "crud-tests.coll0" + "namespace": "crud-tests.coll0", + "collation": { + "locale": "simple" + }, + "hint": "_id_" }, "tests": [ { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml index 3e97dc1bf9..edca3d94d3 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml @@ -26,6 +26,8 @@ initialData: _yamlAnchors: namespace: &namespace "crud-tests.coll0" + collation: &collation { "locale": "simple" } + hint: &hint _id_ tests: - description: "client bulk write delete with collation" @@ -36,14 +38,11 @@ tests: models: - deleteOne: namespace: *namespace - filter: &deleteOneFilter - _id: 1 - collation: &collation - locale: "simple" + filter: { _id: 1 } + collation: *collation - deleteMany: namespace: *namespace - filter: &deleteManyFilter - _id: { $gt: 1 } + filter: { _id: { $gt: 1 } } collation: *collation verboseResults: true expectResult: @@ -67,11 +66,11 @@ tests: bulkWrite: 1 ops: - delete: 0 - filter: *deleteOneFilter + filter: { _id: 1 } collation: *collation multi: false - delete: 0 - filter: *deleteManyFilter + filter: { _id: { $gt: 1 } } collation: *collation multi: true outcome: @@ -86,13 +85,11 @@ tests: models: - deleteOne: namespace: *namespace - filter: &deleteOneFilter - _id: 1 - hint: &hint _id_ + filter: { _id: 1 } + hint: *hint - deleteMany: namespace: *namespace - filter: &deleteManyFilter - _id: { $gt: 1 } + filter: { _id: { $gt: 1 } } hint: *hint verboseResults: true expectResult: @@ -116,11 +113,11 @@ tests: bulkWrite: 1 ops: - delete: 0 - filter: *deleteOneFilter + filter: { _id: 1 } hint: *hint multi: false - delete: 0 - filter: *deleteManyFilter + filter: { _id: { $gt: 1 } } hint: *hint multi: true outcome: diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json index a50730dcb6..648b17cc05 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json @@ -40,21 +40,25 @@ "documents": [ { "_id": 1, - "x": 1 + "x": 11 }, { "_id": 2, - "x": 2 + "x": 22 }, { "_id": 3, - "x": 3 + "x": 33 } ] } ], "_yamlAnchors": { - "namespace": "crud-tests.coll0" + "namespace": "crud-tests.coll0", + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } }, "tests": [ { @@ -127,11 +131,11 @@ "documents": [ { "_id": 2, - "x": 2 + "x": 22 }, { "_id": 3, - "x": 3 + "x": 33 } ] } @@ -216,7 +220,7 @@ "documents": [ { "_id": 2, - "x": 2 + "x": 22 } ] } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml index f6b302367d..e0ea872873 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml @@ -22,12 +22,15 @@ initialData: - collectionName: *collection0Name databaseName: *database0Name documents: - - &document1 { _id: 1, x: 1 } - - &document2 { _id: 2, x: 2 } - - &document3 { _id: 3, x: 3 } + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } _yamlAnchors: namespace: &namespace "crud-tests.coll0" + writeConcernError: &writeConcernError + code: 91 + errmsg: "Replication is being shut down" tests: - description: "an individual operation fails during an ordered bulkWrite" @@ -67,8 +70,8 @@ tests: - collectionName: *collection0Name databaseName: *database0Name documents: - - *document2 - - *document3 + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } - description: "an individual operation fails during an unordered bulkWrite" runOnRequirements: - minServerVersion: "8.0" @@ -111,7 +114,7 @@ tests: - collectionName: *collection0Name databaseName: *database0Name documents: - - *document2 + - { _id: 2, x: 22 } - description: "detailed results are omitted from error when verboseResults is false" runOnRequirements: - minServerVersion: "8.0" @@ -185,9 +188,7 @@ tests: data: failCommands: - bulkWrite - writeConcernError: &writeConcernError - code: 91 - errmsg: "Replication is being shut down" + writeConcernError: *writeConcernError - object: *client0 name: clientBulkWrite arguments: diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json index 8b0ed91a14..72f82b773a 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json @@ -47,7 +47,14 @@ } ], "_yamlAnchors": { - "namespace": "crud-tests.coll0" + "namespace": "crud-tests.coll0", + "comment": { + "bulk": "write" + }, + "let": { + "id1": 1, + "id2": 2 + } }, "tests": [ { @@ -139,8 +146,8 @@ "insertOne": { "namespace": "crud-tests.coll0", "document": { - "_id": 4, - "x": 44 + "_id": 3, + "x": 33 } } } @@ -156,7 +163,7 @@ "deletedCount": 0, "insertResults": { "0": { - "insertedId": 4 + "insertedId": 3 } }, "updateResults": {}, @@ -193,8 +200,8 @@ "x": 22 }, { - "_id": 4, - "x": 44 + "_id": 3, + "x": 33 } ] } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml index bbb097dd17..971429d878 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml @@ -20,11 +20,13 @@ initialData: - collectionName: *collection0Name databaseName: *database0Name documents: - - &doc1 { _id: 1, x: 11 } - - &doc2 { _id: 2, x: 22 } + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } _yamlAnchors: namespace: &namespace "crud-tests.coll0" + comment: &comment { "bulk": "write" } + let: &let { id1: 1, id2: 2 } tests: - description: "client bulkWrite comment" @@ -35,8 +37,8 @@ tests: models: - insertOne: namespace: *namespace - document: &doc3 { _id: 3, x: 33 } - comment: &comment { "bulk": "write" } + document: { _id: 3, x: 33 } + comment: *comment verboseResults: true expectResult: insertedCount: 1 @@ -60,9 +62,9 @@ tests: - collectionName: *collection0Name databaseName: *database0Name documents: - - *doc1 - - *doc2 - - *doc3 + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } - description: "client bulkWrite bypassDocumentValidation" operations: - object: *client0 @@ -71,7 +73,7 @@ tests: models: - insertOne: namespace: *namespace - document: &doc4 { _id: 4, x: 44 } + document: { _id: 3, x: 33 } bypassDocumentValidation: true verboseResults: true expectResult: @@ -82,7 +84,7 @@ tests: deletedCount: 0 insertResults: 0: - insertedId: 4 + insertedId: 3 updateResults: {} deleteResults: {} expectEvents: @@ -96,9 +98,9 @@ tests: - collectionName: *collection0Name databaseName: *database0Name documents: - - *doc1 - - *doc2 - - *doc4 + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } - description: "client bulkWrite let" operations: - object: *client0 @@ -117,7 +119,7 @@ tests: filter: $expr: $eq: [ "$_id", "$$id2" ] - let: &let { id1: 1, id2: 2 } + let: *let verboseResults: true expectResult: insertedCount: 0 diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json index b19815b3bf..f4de54318f 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json @@ -53,7 +53,8 @@ "insertOne": { "namespace": "crud-tests.coll0", "document": { - "_id": 1 + "_id": 1, + "x": 11 } } } @@ -98,7 +99,8 @@ "databaseName": "crud-tests", "documents": [ { - "_id": 1 + "_id": 1, + "x": 11 } ] } @@ -116,7 +118,8 @@ "insertOne": { "namespace": "crud-tests.coll0", "document": { - "_id": 4 + "_id": 1, + "x": 11 } } } @@ -132,7 +135,7 @@ "deletedCount": 0, "insertResults": { "0": { - "insertedId": 4 + "insertedId": 1 } }, "updateResults": {}, @@ -161,7 +164,8 @@ "databaseName": "crud-tests", "documents": [ { - "_id": 4 + "_id": 1, + "x": 11 } ] } @@ -179,7 +183,8 @@ "insertOne": { "namespace": "crud-tests.coll0", "document": { - "_id": 4 + "_id": 1, + "x": 11 } } } @@ -194,7 +199,7 @@ "deletedCount": 0, "insertResults": { "0": { - "insertedId": 4 + "insertedId": 1 } }, "updateResults": {}, @@ -223,7 +228,8 @@ "databaseName": "crud-tests", "documents": [ { - "_id": 4 + "_id": 1, + "x": 11 } ] } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml index ec0e7d98cf..2749cc26d1 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml @@ -33,7 +33,7 @@ tests: models: - insertOne: namespace: *namespace - document: &document { _id: 1 } + document: { _id: 1, x: 11 } verboseResults: true ordered: false expectResult: @@ -58,7 +58,7 @@ tests: - collectionName: *collection0Name databaseName: *database0Name documents: - - *document + - { _id: 1, x: 11 } - description: "client bulkWrite with ordered: true" operations: - object: *client0 @@ -67,7 +67,7 @@ tests: models: - insertOne: namespace: *namespace - document: &document { _id: 4 } + document: { _id: 1, x: 11 } verboseResults: true ordered: true expectResult: @@ -78,7 +78,7 @@ tests: deletedCount: 0 insertResults: 0: - insertedId: 4 + insertedId: 1 updateResults: {} deleteResults: {} expectEvents: @@ -92,7 +92,7 @@ tests: - collectionName: *collection0Name databaseName: *database0Name documents: - - *document + - { _id: 1, x: 11 } - description: "client bulkWrite defaults to ordered: true" operations: - object: *client0 @@ -101,7 +101,7 @@ tests: models: - insertOne: namespace: *namespace - document: &document { _id: 4 } + document: { _id: 1, x: 11 } verboseResults: true expectResult: insertedCount: 1 @@ -111,7 +111,7 @@ tests: deletedCount: 0 insertResults: 0: - insertedId: 4 + insertedId: 1 updateResults: {} deleteResults: {} expectEvents: @@ -125,4 +125,4 @@ tests: - collectionName: *collection0Name databaseName: *database0Name documents: - - *document + - { _id: 1, x: 11 } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml index 5d049008d1..b5a570cd9a 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml @@ -36,37 +36,30 @@ tests: - object: *client0 name: clientBulkWrite arguments: - models: &models + models: - insertOne: namespace: *namespace - document: &insertDoc - { _id: 8, x: 88 } + document: { _id: 8, x: 88 } - updateOne: namespace: *namespace - filter: &updateOneFilter - _id: 1 - update: &updateOneUpdate - $inc: { x: 1 } + filter: { _id: 1 } + update: { $inc: { x: 1 } } - updateMany: namespace: *namespace - filter: &updateManyFilter + filter: $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] - update: &updateManyUpdate - $inc: { x: 2 } + update: { $inc: { x: 2 } } - replaceOne: namespace: *namespace - filter: &replaceOneFilter - _id: 4 - replacement: &replacementDoc - { x: 44 } + filter: { _id: 4 } + replacement: { x: 44 } upsert: true - deleteOne: namespace: *namespace - filter: &deleteOneFilter - _id: 5 + filter: { _id: 5 } - deleteMany: namespace: *namespace - filter: &deleteManyFilter + filter: $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] verboseResults: true expectResult: @@ -103,41 +96,43 @@ tests: command: bulkWrite: 1 errorsOnly: false - ops: &ops + ops: - insert: 0 - document: *insertDoc + document: { _id: 8, x: 88 } - update: 0 - filter: *updateOneFilter - updateMods: *updateOneUpdate + filter: { _id: 1 } + updateMods: { $inc: { x: 1 } } multi: false - update: 0 - filter: *updateManyFilter - updateMods: *updateManyUpdate + filter: + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + updateMods: { $inc: { x: 2 } } multi: true - update: 0 - filter: *replaceOneFilter - updateMods: *replacementDoc + filter: { _id: 4 } + updateMods: { x: 44 } upsert: true multi: false - delete: 0 - filter: *deleteOneFilter + filter: { _id: 5 } multi: false - delete: 0 - filter: *deleteManyFilter + filter: + $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] multi: true - nsInfo: &nsInfo + nsInfo: - ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name - documents: &outcomeDocuments + documents: - { _id: 1, x: 12 } - { _id: 2, x: 24 } - { _id: 3, x: 35 } @@ -148,9 +143,33 @@ tests: - object: *client0 name: clientBulkWrite arguments: - models: *models + models: + - insertOne: + namespace: *namespace + document: { _id: 8, x: 88 } + - updateOne: + namespace: *namespace + filter: { _id: 1 } + update: { $inc: { x: 1 } } + - updateMany: + namespace: *namespace + filter: + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + update: { $inc: { x: 2 } } + - replaceOne: + namespace: *namespace + filter: { _id: 4 } + replacement: { x: 44 } + upsert: true + - deleteOne: + namespace: *namespace + filter: { _id: 5 } + - deleteMany: + namespace: *namespace + filter: + $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] verboseResults: false - expectResult: &summaryResult + expectResult: insertedCount: 1 upsertedCount: 1 matchedCount: 3 @@ -169,19 +188,90 @@ tests: command: bulkWrite: 1 errorsOnly: true - ops: *ops - nsInfo: *nsInfo + ops: + - + insert: 0 + document: { _id: 8, x: 88 } + - + update: 0 + filter: { _id: 1 } + updateMods: { $inc: { x: 1 } } + multi: false + - + update: 0 + filter: + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + updateMods: { $inc: { x: 2 } } + multi: true + - + update: 0 + filter: { _id: 4 } + updateMods: { x: 44 } + upsert: true + multi: false + - + delete: 0 + filter: { _id: 5 } + multi: false + - + delete: 0 + filter: + $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] + multi: true + nsInfo: + - + ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name - documents: *outcomeDocuments + documents: + - { _id: 1, x: 12 } + - { _id: 2, x: 24 } + - { _id: 3, x: 35 } + - { _id: 4, x: 44 } + - { _id: 8, x: 88 } - description: "client bulkWrite defaults to verboseResults: false" operations: - object: *client0 name: clientBulkWrite arguments: - models: *models - expectResult: *summaryResult + models: + - insertOne: + namespace: *namespace + document: { _id: 8, x: 88 } + - updateOne: + namespace: *namespace + filter: { _id: 1 } + update: { $inc: { x: 1 } } + - updateMany: + namespace: *namespace + filter: + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + update: { $inc: { x: 2 } } + - replaceOne: + namespace: *namespace + filter: { _id: 4 } + replacement: { x: 44 } + upsert: true + - deleteOne: + namespace: *namespace + filter: { _id: 5 } + - deleteMany: + namespace: *namespace + filter: + $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] + expectResult: + insertedCount: 1 + upsertedCount: 1 + matchedCount: 3 + modifiedCount: 3 + deletedCount: 3 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} expectEvents: - client: *client0 events: @@ -189,9 +279,45 @@ tests: command: bulkWrite: 1 errorsOnly: true - ops: *ops - nsInfo: *nsInfo + ops: + - + insert: 0 + document: { _id: 8, x: 88 } + - + update: 0 + filter: { _id: 1 } + updateMods: { $inc: { x: 1 } } + multi: false + - + update: 0 + filter: + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + updateMods: { $inc: { x: 2 } } + multi: true + - + update: 0 + filter: { _id: 4 } + updateMods: { x: 44 } + upsert: true + multi: false + - + delete: 0 + filter: { _id: 5 } + multi: false + - + delete: 0 + filter: + $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] + multi: true + nsInfo: + - + ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name - documents: *outcomeDocuments + documents: + - { _id: 1, x: 12 } + - { _id: 2, x: 24 } + - { _id: 3, x: 35 } + - { _id: 4, x: 44 } + - { _id: 8, x: 88 } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json index c3519cc3be..bce6550a26 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json @@ -71,7 +71,11 @@ } ], "_yamlAnchors": { - "namespace": "crud-tests.coll0" + "namespace": "crud-tests.coll0", + "collation": { + "locale": "simple" + }, + "hint": "_id_" }, "tests": [ { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml index f08629db5d..fd10b8dc28 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml @@ -27,6 +27,8 @@ initialData: _yamlAnchors: namespace: &namespace "crud-tests.coll0" + collation: &collation { "locale": "simple" } + hint: &hint _id_ tests: - description: "client bulkWrite update with arrayFilters" @@ -37,22 +39,19 @@ tests: models: - updateOne: namespace: *namespace - filter: &updateOneFilter - _id: 1 - update: &arrayFiltersUpdateOneMods + filter: { _id: 1 } + update: $set: array.$[i]: 4 - arrayFilters: &updateOneArrayFilters - - i: { $gte: 2 } + arrayFilters: [ i: { $gte: 2 } ] - updateMany: namespace: *namespace - filter: &updateManyFilter + filter: $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] - update: &arrayFiltersUpdateManyMods + update: $set: array.$[i]: 5 - arrayFilters: &updateManyArrayFilters - - i: { $gte: 2 } + arrayFilters: [ i: { $gte: 2 } ] verboseResults: true expectResult: insertedCount: 0 @@ -79,14 +78,19 @@ tests: bulkWrite: 1 ops: - update: 0 - filter: *updateOneFilter - updateMods: *arrayFiltersUpdateOneMods - arrayFilters: *updateOneArrayFilters + filter: { _id: 1 } + updateMods: + $set: + array.$[i]: 4 + arrayFilters: [ i: { $gte: 2 } ] multi: false - update: 0 - filter: *updateManyFilter - updateMods: *arrayFiltersUpdateManyMods - arrayFilters: *updateManyArrayFilters + filter: + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + updateMods: + $set: + array.$[i]: 5 + arrayFilters: [ i: { $gte: 2 } ] multi: true outcome: - databaseName: *database0Name @@ -104,25 +108,19 @@ tests: models: - updateOne: namespace: *namespace - filter: *updateOneFilter - update: &updateOneMods - $set: - array: [ 1, 2, 4 ] - collation: &collation - locale: "simple" + filter: { _id: 1 } + update: { $set: { array: [ 1, 2, 4 ] } } + collation: *collation - updateMany: namespace: *namespace - filter: *updateManyFilter - update: &updateManyMods - $set: - array: [ 1, 2, 5 ] + filter: + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + update: { $set: { array: [ 1, 2, 5 ] } } collation: *collation - replaceOne: namespace: *namespace - filter: &replaceOneFilter - _id: 4 - replacement: &replacement - array: [ 1, 2, 6 ] + filter: { _id: 4 } + replacement: { array: [ 1, 2, 6 ] } collation: *collation verboseResults: true expectResult: @@ -154,18 +152,19 @@ tests: bulkWrite: 1 ops: - update: 0 - filter: *updateOneFilter - updateMods: *updateOneMods + filter: { _id: 1 } + updateMods: { $set: { array: [ 1, 2, 4 ] } } collation: *collation multi: false - update: 0 - filter: *updateManyFilter - updateMods: *updateManyMods + filter: + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + updateMods: { $set: { array: [ 1, 2, 5 ] } } collation: *collation multi: true - update: 0 - filter: *replaceOneFilter - updateMods: *replacement + filter: { _id: 4 } + updateMods: { array: [ 1, 2, 6 ] } collation: *collation multi: false outcome: @@ -184,18 +183,19 @@ tests: models: - updateOne: namespace: *namespace - filter: *updateOneFilter - update: *updateOneMods - hint: &hint _id_ + filter: { _id: 1 } + update: { $set: { array: [ 1, 2, 4 ] } } + hint: *hint - updateMany: namespace: *namespace - filter: *updateManyFilter - update: *updateManyMods + filter: + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + update: { $set: { array: [ 1, 2, 5 ] } } hint: *hint - replaceOne: namespace: *namespace - filter: *replaceOneFilter - replacement: *replacement + filter: { _id: 4 } + replacement: { array: [ 1, 2, 6 ] } hint: *hint verboseResults: true expectResult: @@ -227,18 +227,19 @@ tests: bulkWrite: 1 ops: - update: 0 - filter: *updateOneFilter - updateMods: *updateOneMods + filter: { _id: 1 } + updateMods: { $set: { array: [ 1, 2, 4 ] } } hint: *hint multi: false - update: 0 - filter: *updateManyFilter - updateMods: *updateManyMods + filter: + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + updateMods: { $set: { array: [ 1, 2, 5 ] } } hint: *hint multi: true - update: 0 - filter: *replaceOneFilter - updateMods: *replacement + filter: { _id: 4 } + updateMods: { array: [ 1, 2, 6 ] } hint: *hint multi: false outcome: @@ -257,15 +258,13 @@ tests: models: - updateOne: namespace: *namespace - filter: &updateOneFilter - _id: 5 - update: *updateOneMods + filter: { _id: 5 } + update: { $set: { array: [ 1, 2, 4 ] } } upsert: true - replaceOne: namespace: *namespace - filter: &replaceOneFilter - _id: 6 - replacement: *replacement + filter: { _id: 6 } + replacement: { array: [ 1, 2, 6 ] } upsert: true verboseResults: true expectResult: @@ -293,13 +292,13 @@ tests: bulkWrite: 1 ops: - update: 0 - filter: *updateOneFilter - updateMods: *updateOneMods + filter: { _id: 5 } + updateMods: { $set: { array: [ 1, 2, 4 ] } } upsert: true multi: false - update: 0 - filter: *replaceOneFilter - updateMods: *replacement + filter: { _id: 6 } + updateMods: { array: [ 1, 2, 6 ] } upsert: true multi: false outcome: diff --git a/source/transactions/tests/unified/client-bulkWrite.json b/source/transactions/tests/unified/client-bulkWrite.json index eab6316797..9ccde1b1bf 100644 --- a/source/transactions/tests/unified/client-bulkWrite.json +++ b/source/transactions/tests/unified/client-bulkWrite.json @@ -41,6 +41,9 @@ } } ], + "_yamlAnchors": { + "namespace": "transactions-tests.coll0" + }, "initialData": [ { "databaseName": "transaction-tests", @@ -75,7 +78,7 @@ ], "tests": [ { - "description": "transactional client bulkWrite", + "description": "client bulkWrite in a transaction", "operations": [ { "object": "session0", diff --git a/source/transactions/tests/unified/client-bulkWrite.yml b/source/transactions/tests/unified/client-bulkWrite.yml index 6bbfa3248e..d7523f31b2 100644 --- a/source/transactions/tests/unified/client-bulkWrite.yml +++ b/source/transactions/tests/unified/client-bulkWrite.yml @@ -49,34 +49,27 @@ tests: models: - insertOne: namespace: *namespace - document: &insertDoc - { _id: 8, x: 88 } + document: { _id: 8, x: 88 } - updateOne: namespace: *namespace - filter: &updateOneFilter - _id: 1 - update: &updateOneUpdate - $inc: { x: 1 } + filter: { _id: 1 } + update: { $inc: { x: 1 } } - updateMany: namespace: *namespace - filter: &updateManyFilter + filter: $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] - update: &updateManyUpdate - $inc: { x: 2 } + update: { $inc: { x: 2 } } - replaceOne: namespace: *namespace - filter: &replaceOneFilter - _id: 4 - replacement: &replacementDoc - { x: 44 } + filter: { _id: 4 } + replacement: { x: 44 } upsert: true - deleteOne: namespace: *namespace - filter: &deleteOneFilter - _id: 5 + filter: { _id: 5 } - deleteMany: namespace: *namespace - filter: &deleteManyFilter + filter: $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] verboseResults: true expectResult: @@ -123,25 +116,27 @@ tests: writeConcern: { $$exists: false } ops: - insert: 0 - document: *insertDoc + document: { _id: 8, x: 88 } - update: 0 - filter: *updateOneFilter - updateMods: *updateOneUpdate + filter: { _id: 1 } + updateMods: { $inc: { x: 1 } } multi: false - update: 0 - filter: *updateManyFilter - updateMods: *updateManyUpdate + filter: + $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] + updateMods: { $inc: { x: 2 } } multi: true - update: 0 - filter: *replaceOneFilter - updateMods: *replacementDoc + filter: { _id: 4 } + updateMods: { x: 44 } upsert: true multi: false - delete: 0 - filter: *deleteOneFilter + filter: { _id: 5 } multi: false - delete: 0 - filter: *deleteManyFilter + filter: + $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] multi: true nsInfo: - ns: *namespace @@ -160,7 +155,7 @@ tests: outcome: - collectionName: *collection0Name databaseName: *database0Name - documents: &outcomeDocuments + documents: - { _id: 1, x: 12 } - { _id: 2, x: 24 } - { _id: 3, x: 35 } From 905e2ee97cd6cbb4c4295a3d9135fc3822172b5c Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 29 Feb 2024 13:29:34 -0700 Subject: [PATCH 14/90] clean up command started events --- .../client-bulkWrite-delete-options.json | 13 +++ .../client-bulkWrite-delete-options.yml | 10 +++ .../client-bulkWrite-options.json | 82 ++++++++++++++++++- .../client-bulkWrite-options.yml | 36 ++++++++ .../client-bulkWrite-ordered.json | 57 ++++++++++++- .../client-bulkWrite-ordered.yml | 24 ++++++ .../client-bulkWrite-results.json | 9 ++ .../client-bulkWrite-results.yml | 72 +++++++--------- .../client-bulkWrite-update-options.json | 36 ++++++++ .../client-bulkWrite-update-options.yml | 24 ++++++ .../client-bulkWrite-serverErrors.json | 18 ++++ .../unified/client-bulkWrite-serverErrors.yml | 18 ++++ .../tests/unified/client-bulkWrite.json | 8 +- .../tests/unified/client-bulkWrite.yml | 6 +- 14 files changed, 358 insertions(+), 55 deletions(-) diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json index 21fa3a8142..8ad03a2003 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json @@ -118,8 +118,12 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "delete": 0, @@ -143,6 +147,11 @@ }, "multi": true } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } ] } } @@ -214,8 +223,12 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "delete": 0, diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml index edca3d94d3..c9c5cee84a 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml @@ -62,8 +62,12 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false + ordered: true ops: - delete: 0 filter: { _id: 1 } @@ -73,6 +77,8 @@ tests: filter: { _id: { $gt: 1 } } collation: *collation multi: true + nsInfo: + - ns: *namespace outcome: - databaseName: *database0Name collectionName: *collection0Name @@ -109,8 +115,12 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false + ordered: true ops: - delete: 0 filter: { _id: 1 } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json index 72f82b773a..d92be7b197 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json @@ -102,11 +102,29 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "comment": { "bulk": "write" - } + }, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] } } } @@ -177,9 +195,27 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, - "bypassDocumentValidation": true + "errorsOnly": false, + "ordered": true, + "bypassDocumentValidation": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] } } } @@ -281,12 +317,52 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "let": { "id1": 1, "id2": 2 - } + }, + "ops": [ + { + "update": 0, + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id1" + ] + } + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "multi": false + }, + { + "delete": 0, + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] } } } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml index 971429d878..6b7e1d8c49 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml @@ -55,9 +55,18 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false + ordered: true comment: *comment + ops: + - insert: 0 + document: { _id: 3, x: 33 } + nsInfo: + - ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name @@ -91,9 +100,18 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false + ordered: true bypassDocumentValidation: true + ops: + - insert: 0 + document: { _id: 3, x: 33 } + nsInfo: + - ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name @@ -140,9 +158,27 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false + ordered: true let: *let + ops: + - update: 0 + filter: + $expr: + $eq: [ "$_id", "$$id1" ] + updateMods: { $inc: { x: 1 } } + multi: false + - delete: 0 + filter: + $expr: + $eq: [ "$_id", "$$id2" ] + multi: false + nsInfo: + - ns: *namespace outcome: - databaseName: *database0Name collectionName: *collection0Name diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json index f4de54318f..5ce983f632 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json @@ -84,9 +84,26 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, - "ordered": false + "errorsOnly": false, + "ordered": false, + "ops": [ + { + "insert": 0, + "document": { + "_id": 1, + "x": 11 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] } } } @@ -149,9 +166,26 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, - "ordered": true + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 1, + "x": 11 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] } } } @@ -213,9 +247,26 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, - "ordered": true + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 1, + "x": 11 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] } } } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml index 2749cc26d1..1e0b912f68 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml @@ -51,9 +51,17 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false ordered: false + ops: + - insert: 0 + document: { _id: 1, x: 11 } + nsInfo: + - ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name @@ -85,9 +93,17 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false ordered: true + ops: + - insert: 0 + document: { _id: 1, x: 11 } + nsInfo: + - ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name @@ -118,9 +134,17 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false ordered: true + ops: + - insert: 0 + document: { _id: 1, x: 11 } + nsInfo: + - ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json index 43238bc5a6..92aa734640 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json @@ -207,9 +207,12 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, "errorsOnly": false, + "ordered": true, "ops": [ { "insert": 0, @@ -450,9 +453,12 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, "errorsOnly": true, + "ordered": true, "ops": [ { "insert": 0, @@ -692,9 +698,12 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, "errorsOnly": true, + "ordered": true, "ops": [ { "insert": 0, diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml index b5a570cd9a..76275f2867 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml @@ -93,42 +93,38 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 errorsOnly: false + ordered: true ops: - - - insert: 0 + - insert: 0 document: { _id: 8, x: 88 } - - - update: 0 + - update: 0 filter: { _id: 1 } updateMods: { $inc: { x: 1 } } multi: false - - - update: 0 + - update: 0 filter: $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] updateMods: { $inc: { x: 2 } } multi: true - - - update: 0 + - update: 0 filter: { _id: 4 } updateMods: { x: 44 } upsert: true multi: false - - - delete: 0 + - delete: 0 filter: { _id: 5 } multi: false - - - delete: 0 + - delete: 0 filter: $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] multi: true nsInfo: - - - ns: *namespace + - ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name @@ -185,42 +181,38 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 errorsOnly: true + ordered: true ops: - - - insert: 0 + - insert: 0 document: { _id: 8, x: 88 } - - - update: 0 + - update: 0 filter: { _id: 1 } updateMods: { $inc: { x: 1 } } multi: false - - - update: 0 + - update: 0 filter: $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] updateMods: { $inc: { x: 2 } } multi: true - - - update: 0 + - update: 0 filter: { _id: 4 } updateMods: { x: 44 } upsert: true multi: false - - - delete: 0 + - delete: 0 filter: { _id: 5 } multi: false - - - delete: 0 + - delete: 0 filter: $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] multi: true nsInfo: - - - ns: *namespace + - ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name @@ -276,42 +268,38 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 errorsOnly: true + ordered: true ops: - - - insert: 0 + - insert: 0 document: { _id: 8, x: 88 } - - - update: 0 + - update: 0 filter: { _id: 1 } updateMods: { $inc: { x: 1 } } multi: false - - - update: 0 + - update: 0 filter: $and: [ { _id: { $gt: 1 } }, { _id: { $lte: 3 } } ] updateMods: { $inc: { x: 2 } } multi: true - - - update: 0 + - update: 0 filter: { _id: 4 } updateMods: { x: 44 } upsert: true multi: false - - - delete: 0 + - delete: 0 filter: { _id: 5 } multi: false - - - delete: 0 + - delete: 0 filter: $and: [ { _id: { $gt: 5 } }, { _id: { $lte: 7 } } ] multi: true nsInfo: - - - ns: *namespace + - ns: *namespace outcome: - collectionName: *collection0Name databaseName: *database0Name diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json index bce6550a26..d163775bd0 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json @@ -169,8 +169,12 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "update": 0, @@ -221,6 +225,11 @@ ], "multi": true } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } ] } } @@ -383,8 +392,12 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "update": 0, @@ -452,6 +465,11 @@ }, "multi": false } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } ] } } @@ -608,8 +626,12 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "update": 0, @@ -671,6 +693,11 @@ "hint": "_id_", "multi": false } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } ] } } @@ -793,8 +820,12 @@ "events": [ { "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "update": 0, @@ -828,6 +859,11 @@ "upsert": true, "multi": false } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } ] } } diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml index fd10b8dc28..4abed1979f 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml +++ b/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml @@ -74,8 +74,12 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false + ordered: true ops: - update: 0 filter: { _id: 1 } @@ -92,6 +96,8 @@ tests: array.$[i]: 5 arrayFilters: [ i: { $gte: 2 } ] multi: true + nsInfo: + - ns: *namespace outcome: - databaseName: *database0Name collectionName: *collection0Name @@ -148,8 +154,12 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false + ordered: true ops: - update: 0 filter: { _id: 1 } @@ -167,6 +177,8 @@ tests: updateMods: { array: [ 1, 2, 6 ] } collation: *collation multi: false + nsInfo: + - ns: *namespace outcome: - databaseName: *database0Name collectionName: *collection0Name @@ -223,8 +235,12 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false + ordered: true ops: - update: 0 filter: { _id: 1 } @@ -242,6 +258,8 @@ tests: updateMods: { array: [ 1, 2, 6 ] } hint: *hint multi: false + nsInfo: + - ns: *namespace outcome: - databaseName: *database0Name collectionName: *collection0Name @@ -288,8 +306,12 @@ tests: - client: *client0 events: - commandStartedEvent: + commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false + ordered: true ops: - update: 0 filter: { _id: 5 } @@ -301,6 +323,8 @@ tests: updateMods: { array: [ 1, 2, 6 ] } upsert: true multi: false + nsInfo: + - ns: *namespace outcome: - databaseName: *database0Name collectionName: *collection0Name diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json index 4091d66d61..1d32f367ee 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json @@ -171,6 +171,9 @@ "commandName": "bulkWrite", "databaseName": "admin", "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "insert": 0, @@ -222,6 +225,9 @@ "commandName": "bulkWrite", "databaseName": "admin", "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "insert": 0, @@ -362,6 +368,9 @@ "commandName": "bulkWrite", "databaseName": "admin", "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, "ops": [ { "update": 0, @@ -512,6 +521,9 @@ "commandName": "bulkWrite", "databaseName": "admin", "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "insert": 0, @@ -563,6 +575,9 @@ "commandName": "bulkWrite", "databaseName": "admin", "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "insert": 0, @@ -688,6 +703,9 @@ "commandName": "bulkWrite", "databaseName": "admin", "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, "ops": [ { "update": 0, diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml index 874afe4df1..59982abb66 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml @@ -91,6 +91,9 @@ tests: commandName: bulkWrite databaseName: admin command: + bulkWrite: 1 + errorsOnly: false + ordered: true ops: - insert: 0 document: { _id: 4, x: 44 } @@ -112,6 +115,9 @@ tests: commandName: bulkWrite databaseName: admin command: + bulkWrite: 1 + errorsOnly: false + ordered: true ops: - insert: 0 document: { _id: 4, x: 44 } @@ -172,6 +178,9 @@ tests: commandName: bulkWrite databaseName: admin command: + bulkWrite: 1 + errorsOnly: true + ordered: true ops: - update: 0 filter: { _id: 1 } @@ -247,6 +256,9 @@ tests: commandName: bulkWrite databaseName: admin command: + bulkWrite: 1 + errorsOnly: false + ordered: true ops: - insert: 0 document: { _id: 4, x: 44 } @@ -268,6 +280,9 @@ tests: commandName: bulkWrite databaseName: admin command: + bulkWrite: 1 + errorsOnly: false + ordered: true ops: - insert: 0 document: { _id: 4, x: 44 } @@ -324,6 +339,9 @@ tests: commandName: bulkWrite databaseName: admin command: + bulkWrite: 1 + errorsOnly: true + ordered: true ops: - update: 0 filter: { _id: 1 } diff --git a/source/transactions/tests/unified/client-bulkWrite.json b/source/transactions/tests/unified/client-bulkWrite.json index 9ccde1b1bf..8936ce3b22 100644 --- a/source/transactions/tests/unified/client-bulkWrite.json +++ b/source/transactions/tests/unified/client-bulkWrite.json @@ -230,7 +230,6 @@ "commandName": "bulkWrite", "databaseName": "admin", "command": { - "bulkWrite": 1, "lsid": { "$$sessionLsid": "session0" }, @@ -240,6 +239,9 @@ "writeConcern": { "$$exists": false }, + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "insert": 0, @@ -324,9 +326,7 @@ { "ns": "transactions-tests.coll0" } - ], - "errorsOnly": false, - "ordered": true + ] } } }, diff --git a/source/transactions/tests/unified/client-bulkWrite.yml b/source/transactions/tests/unified/client-bulkWrite.yml index d7523f31b2..23ae34e777 100644 --- a/source/transactions/tests/unified/client-bulkWrite.yml +++ b/source/transactions/tests/unified/client-bulkWrite.yml @@ -108,12 +108,14 @@ tests: commandName: bulkWrite databaseName: admin command: - bulkWrite: 1 lsid: { $$sessionLsid: *session0 } txnNumber: 1 startTransaction: true autocommit: false writeConcern: { $$exists: false } + bulkWrite: 1 + errorsOnly: false + ordered: true ops: - insert: 0 document: { _id: 8, x: 88 } @@ -140,8 +142,6 @@ tests: multi: true nsInfo: - ns: *namespace - errorsOnly: false - ordered: true - commandStartedEvent: commandName: commitTransaction databaseName: admin From 981e6c3a578a4b5443bb7fdc26cf19b52edff032 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 29 Feb 2024 13:42:19 -0700 Subject: [PATCH 15/90] move tests into crud directory --- .../{new-bulk-write => }/client-bulkWrite-delete-options.json | 0 .../{new-bulk-write => }/client-bulkWrite-delete-options.yml | 0 .../unified/{new-bulk-write => }/client-bulkWrite-errors.json | 3 ++- .../unified/{new-bulk-write => }/client-bulkWrite-errors.yml | 1 + .../client-bulkWrite-mixed-namespaces.json | 0 .../{new-bulk-write => }/client-bulkWrite-mixed-namespaces.yml | 0 .../unified/{new-bulk-write => }/client-bulkWrite-options.json | 0 .../unified/{new-bulk-write => }/client-bulkWrite-options.yml | 0 .../unified/{new-bulk-write => }/client-bulkWrite-ordered.json | 0 .../unified/{new-bulk-write => }/client-bulkWrite-ordered.yml | 0 .../unified/{new-bulk-write => }/client-bulkWrite-results.json | 0 .../unified/{new-bulk-write => }/client-bulkWrite-results.yml | 0 .../{new-bulk-write => }/client-bulkWrite-update-options.json | 0 .../{new-bulk-write => }/client-bulkWrite-update-options.yml | 0 .../tests/unified/client-bulkWrite-serverErrors.json | 3 ++- .../tests/unified/client-bulkWrite-serverErrors.yml | 1 + 16 files changed, 6 insertions(+), 2 deletions(-) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-delete-options.json (100%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-delete-options.yml (100%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-errors.json (99%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-errors.yml (99%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-mixed-namespaces.json (100%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-mixed-namespaces.yml (100%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-options.json (100%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-options.yml (100%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-ordered.json (100%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-ordered.yml (100%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-results.json (100%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-results.yml (100%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-update-options.json (100%) rename source/crud/tests/unified/{new-bulk-write => }/client-bulkWrite-update-options.yml (100%) diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json b/source/crud/tests/unified/client-bulkWrite-delete-options.json similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.json rename to source/crud/tests/unified/client-bulkWrite-delete-options.json diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml b/source/crud/tests/unified/client-bulkWrite-delete-options.yml similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-delete-options.yml rename to source/crud/tests/unified/client-bulkWrite-delete-options.yml diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json b/source/crud/tests/unified/client-bulkWrite-errors.json similarity index 99% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json rename to source/crud/tests/unified/client-bulkWrite-errors.json index 648b17cc05..9719a9d97d 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.json +++ b/source/crud/tests/unified/client-bulkWrite-errors.json @@ -15,7 +15,8 @@ ], "uriOptions": { "retryWrites": false - } + }, + "useMultipleMongoses": false } }, { diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml b/source/crud/tests/unified/client-bulkWrite-errors.yml similarity index 99% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml rename to source/crud/tests/unified/client-bulkWrite-errors.yml index e0ea872873..732e58a0f6 100644 --- a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-errors.yml +++ b/source/crud/tests/unified/client-bulkWrite-errors.yml @@ -9,6 +9,7 @@ createEntities: observeEvents: [ commandStartedEvent ] uriOptions: retryWrites: false + useMultipleMongoses: false - database: id: &database0 database0 client: *client0 diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json b/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.json similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.json rename to source/crud/tests/unified/client-bulkWrite-mixed-namespaces.json diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml b/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.yml similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-mixed-namespaces.yml rename to source/crud/tests/unified/client-bulkWrite-mixed-namespaces.yml diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json b/source/crud/tests/unified/client-bulkWrite-options.json similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.json rename to source/crud/tests/unified/client-bulkWrite-options.json diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml b/source/crud/tests/unified/client-bulkWrite-options.yml similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-options.yml rename to source/crud/tests/unified/client-bulkWrite-options.yml diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json b/source/crud/tests/unified/client-bulkWrite-ordered.json similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.json rename to source/crud/tests/unified/client-bulkWrite-ordered.json diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml b/source/crud/tests/unified/client-bulkWrite-ordered.yml similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-ordered.yml rename to source/crud/tests/unified/client-bulkWrite-ordered.yml diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json b/source/crud/tests/unified/client-bulkWrite-results.json similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.json rename to source/crud/tests/unified/client-bulkWrite-results.json diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml b/source/crud/tests/unified/client-bulkWrite-results.yml similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-results.yml rename to source/crud/tests/unified/client-bulkWrite-results.yml diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json b/source/crud/tests/unified/client-bulkWrite-update-options.json similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.json rename to source/crud/tests/unified/client-bulkWrite-update-options.json diff --git a/source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml b/source/crud/tests/unified/client-bulkWrite-update-options.yml similarity index 100% rename from source/crud/tests/unified/new-bulk-write/client-bulkWrite-update-options.yml rename to source/crud/tests/unified/client-bulkWrite-update-options.yml diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json index 1d32f367ee..caab0b5463 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json @@ -15,7 +15,8 @@ "id": "client0", "observeEvents": [ "commandStartedEvent" - ] + ], + "useMultipleMongoses": false } }, { diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml index 59982abb66..4f3194e829 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml @@ -8,6 +8,7 @@ createEntities: - client: id: &client0 client0 observeEvents: [ commandStartedEvent ] + useMultipleMongoses: false - database: id: &database0 database0 client: *client0 From 03c3bde54cf9e6bac674789fa38570114bd3f834 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 29 Feb 2024 15:51:44 -0700 Subject: [PATCH 16/90] fix typo --- .../tests/unified/client-bulkWrite.json | 18 +++++++++--------- .../tests/unified/client-bulkWrite.yml | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/source/transactions/tests/unified/client-bulkWrite.json b/source/transactions/tests/unified/client-bulkWrite.json index 8936ce3b22..4fbb4e1f2a 100644 --- a/source/transactions/tests/unified/client-bulkWrite.json +++ b/source/transactions/tests/unified/client-bulkWrite.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite transactions", - "schemaVersion": "1.20", + "schemaVersion": "1.0", "runOnRequirements": [ { "minServerVersion": "8.0", @@ -42,7 +42,7 @@ } ], "_yamlAnchors": { - "namespace": "transactions-tests.coll0" + "namespace": "transaction-tests.coll0" }, "initialData": [ { @@ -92,7 +92,7 @@ "models": [ { "insertOne": { - "namespace": "transactions-tests.coll0", + "namespace": "transaction-tests.coll0", "document": { "_id": 8, "x": 88 @@ -101,7 +101,7 @@ }, { "updateOne": { - "namespace": "transactions-tests.coll0", + "namespace": "transaction-tests.coll0", "filter": { "_id": 1 }, @@ -114,7 +114,7 @@ }, { "updateMany": { - "namespace": "transactions-tests.coll0", + "namespace": "transaction-tests.coll0", "filter": { "$and": [ { @@ -138,7 +138,7 @@ }, { "replaceOne": { - "namespace": "transactions-tests.coll0", + "namespace": "transaction-tests.coll0", "filter": { "_id": 4 }, @@ -150,7 +150,7 @@ }, { "deleteOne": { - "namespace": "transactions-tests.coll0", + "namespace": "transaction-tests.coll0", "filter": { "_id": 5 } @@ -158,7 +158,7 @@ }, { "deleteMany": { - "namespace": "transactions-tests.coll0", + "namespace": "transaction-tests.coll0", "filter": { "$and": [ { @@ -324,7 +324,7 @@ ], "nsInfo": [ { - "ns": "transactions-tests.coll0" + "ns": "transaction-tests.coll0" } ] } diff --git a/source/transactions/tests/unified/client-bulkWrite.yml b/source/transactions/tests/unified/client-bulkWrite.yml index 23ae34e777..9688af7da1 100644 --- a/source/transactions/tests/unified/client-bulkWrite.yml +++ b/source/transactions/tests/unified/client-bulkWrite.yml @@ -1,5 +1,5 @@ description: "client bulkWrite transactions" -schemaVersion: "1.20" +schemaVersion: "1.0" runOnRequirements: - minServerVersion: "8.0" topologies: @@ -24,7 +24,7 @@ createEntities: client: *client0 _yamlAnchors: - namespace: &namespace "transactions-tests.coll0" + namespace: &namespace "transaction-tests.coll0" initialData: - databaseName: *database0Name From 01c9ae7bfcc59f9b8975c9afb74906100a8cff73 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 1 Mar 2024 10:26:53 -0700 Subject: [PATCH 17/90] update some language --- source/crud/bulk-write.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 9c86584d8c..1ee500218f 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -535,11 +535,11 @@ have the following format: } ``` -### Toggling Results Verbosity +### `errorsOnly` and `verboseResults` The `errorsOnly` field indicates whether the results cursor returned in the `bulkWrite` response -should contain only errors. If false, individual results for successful operations will be returned -in addition to errors. +should contain only errors and omit individual results. If false, both individual results for +successful operations and errors will be returned. This field corresponds to the `verboseResults` option defined on `BulkWriteOptions`. Its value should be provided as `false` if `verboseResults` was set to `true`; otherwise, it should be @@ -570,10 +570,12 @@ defined in the #### Unencrypted bulk writes with document sequences -When `ops` and `nsInfo` are provided as document sequences, drivers MUST ensure that the combined -size of these sequences does not exceed `maxMessageSizeBytes - 16,000`. `maxMessageSizeBytes` -defines the maximum size for an entire `OP_MSG`, so 16KB is subtracted from `maxMessageSizeBytes` -in this size limit to account for the bytes in the rest of the message. +When `ops` and `nsInfo` are provided as document sequences, drivers MUST ensure that the total size +of the `OP_MSG` built for the `bulkWrite` command does not exceed `maxMessageSizeBytes`. Some +drivers may perform batch-splitting prior to constructing the full `OP_MSG` to be sent to the +server. In this case, drivers MAY use `maxMessageSizeBytes - 16,000` as the upper bound for the +combined number of bytes in `ops` and `nsInfo`. 16KB is subtracted to accommodate for the bytes in +the rest of the message. #### Unencrypted bulk writes without document sequences @@ -605,7 +607,7 @@ The server's response to `bulkWrite` has the following format: If any operations were successful (i.e. `nErrors` is less than the number of operations that were sent), drivers MUST record the summary count fields in a `BulkWriteResult` to be returned to the -user or included in a `BulkWriteException`. +user or embedded in a `BulkWriteException`. Drivers MUST attempt to consume the contents of the cursor returned in the server's `bulkWrite` response before returning to the user. This is required regardless of whether the user requested From fab1deb1c92ac9c746d91cc198c84e837470d350 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 1 Mar 2024 16:22:50 -0700 Subject: [PATCH 18/90] self review changes --- source/crud/bulk-write.md | 31 ++++++------ source/crud/tests/README.md | 8 ++-- .../client-bulkWrite-delete-options.yml | 3 +- .../tests/unified/client-bulkWrite-errors.yml | 1 - .../client-bulkWrite-mixed-namespaces.yml | 48 ++++++++----------- .../unified/client-bulkWrite-options.yml | 2 +- .../unified/client-bulkWrite-ordered.yml | 2 +- .../unified/client-bulkWrite-results.yml | 2 +- .../client-bulkWrite-update-options.yml | 2 +- .../unified/client-bulkWrite-serverErrors.yml | 5 +- .../tests/unified/client-bulkWrite.yml | 2 +- .../tests/crud-api-version-1.yml | 5 +- 12 files changed, 52 insertions(+), 59 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 1ee500218f..0b9b9ad9e0 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -20,7 +20,7 @@ defined in the CRUD specification and the **NOTE:** The types specified in this interface are similar to those used for the collection-level `bulkWrite` method. Statically typed drivers MUST NOT reuse any types they already define for the `MongoCollection.bulkWrite` interface and MUST introduce new types. If naming conflicts between -types arise, drivers MAY prepend "Client" to the new type names (e.g. `ClientBulkWriteOptions`). +types arise, drivers SHOULD prepend "Client" to the new type names (e.g. `ClientBulkWriteOptions`). ```typescript interface MongoClient { @@ -297,8 +297,8 @@ class BulkWriteResult { * Indicates whether this write result was acknowledged. If not, then all other members of this * result will be undefined. * - * NOT REQUIRED TO IMPLEMENT. See [here](../crud/crud.md#write-results) for more guidance on - * modeling unacknowledged results. + * NOT REQUIRED TO IMPLEMENT. See the CRUD specification for more guidance on modeling + * unacknowledged results. */ acknowledged: Boolean; @@ -337,20 +337,17 @@ class BulkWriteResult { deletedCount: Int64; /** - * The results of each individual insert operation that was successfully performed. Results - * can be accessed by their index in the list of write models provided to bulkWrite. + * The results of each individual insert operation that was successfully performed. */ insertResults: Map; /** - * The results of each individual update operation that was successfully performed. Results - * can be accessed by their index in the list of write models provided to bulkWrite. + * The results of each individual update operation that was successfully performed. */ updateResult: Map; /** - * The results of each individual delete operation that was successfully performed. Results - * can be accessed by their index in the list of write models provided to bulkWrite. + * The results of each individual delete operation that was successfully performed. */ deleteResults: Map; } @@ -505,9 +502,9 @@ bulk writes, drivers MAY rely on the server to return an error if the document e "filter": , "updateMods": , "multi": , - "upsert": , - "arrayFilters": , - "hint": + "upsert": Optional, + "arrayFilters": Optional, + "hint": Optional } ``` @@ -518,16 +515,16 @@ bulk writes, drivers MAY rely on the server to return an error if the document e "delete": , "filter": , "multi": , - "hint": , - "collation": + "hint": Optional, + "collation": Optional } ``` ### Namespace Information -The `nsInfo` field contains the namespaces on which the write operations should be performed. -Drivers MUST NOT include duplicate namespaces in this list. The documents in the `nsInfo` field -have the following format: +The `nsInfo` field is an array containing the namespaces on which the write operations should be +performed. Drivers MUST NOT include duplicate namespaces in this list. The documents in the +`nsInfo` array have the following format: ```json { diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 327279c5d4..d8421fb527 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -232,15 +232,15 @@ is 1. If the driver exposes `operationId`s in its CommandStartedEvents, assert t ### 4. `MongoClient.bulkWrite` handles a `writeModels` input larger than `maxBsonObjectSize` -Test that `MongoClient.bulkWrite` properly handles a `writeModels` input for which the sum of the models' entries in the -`ops` array exceeds `maxBsonObjectSize`. +Test that `MongoClient.bulkWrite` properly handles a `writeModels` input which constructs an `ops` array larger than +`maxBsonObjectSize`. This test must only be run on 8.0+ servers. Construct a `MongoClient` (referred to as `client`) with [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe -CommandStartedEvents. Perform a `hello` command using `client` and record the following values from the response: -`maxBsonObjectSize` and `maxMessageSizeBytes`. Then, construct the following document (referred to as `document`): +CommandStartedEvents. Perform a `hello` command using `client` and record the `maxBsonObjectSize` value contained in the +response. Then, construct the following document (referred to as `document`): ```json { diff --git a/source/crud/tests/unified/client-bulkWrite-delete-options.yml b/source/crud/tests/unified/client-bulkWrite-delete-options.yml index c9c5cee84a..db8b9f46d7 100644 --- a/source/crud/tests/unified/client-bulkWrite-delete-options.yml +++ b/source/crud/tests/unified/client-bulkWrite-delete-options.yml @@ -1,5 +1,5 @@ description: "client bulkWrite delete options" -schemaVersion: "1.0" +schemaVersion: "1.1" runOnRequirements: - minServerVersion: "8.0" @@ -134,4 +134,3 @@ tests: - databaseName: *database0Name collectionName: *collection0Name documents: [] - \ No newline at end of file diff --git a/source/crud/tests/unified/client-bulkWrite-errors.yml b/source/crud/tests/unified/client-bulkWrite-errors.yml index 732e58a0f6..46d7683e1a 100644 --- a/source/crud/tests/unified/client-bulkWrite-errors.yml +++ b/source/crud/tests/unified/client-bulkWrite-errors.yml @@ -211,4 +211,3 @@ tests: updateResults: {} deleteResults: {} writeConcernErrors: [ *writeConcernError ] - \ No newline at end of file diff --git a/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.yml b/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.yml index c363b47c42..a34870e053 100644 --- a/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.yml +++ b/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.yml @@ -1,5 +1,5 @@ description: "client bulkWrite with mixed namespaces" -schemaVersion: "1.0" +schemaVersion: "1.1" runOnRequirements: - minServerVersion: "8.0" @@ -57,32 +57,24 @@ tests: models: - insertOne: namespace: *db0Coll0Namespace - document: &document1 - _id: 1 + document: { _id: 1 } - insertOne: namespace: *db0Coll0Namespace - document: &document2 - _id: 2 + document: { _id: 2 } - updateOne: namespace: *db0Coll1Namespace - filter: &filter1 - _id: 1 - update: &update - $inc: { x: 1 } + filter: { _id: 1 } + update: { $inc: { x: 1 } } - deleteOne: namespace: *db1Coll2Namespace - filter: &filter2 - _id: 3 + filter: { _id: 3 } - deleteOne: namespace: *db0Coll1Namespace - filter: &filter3 - _id: 2 + filter: { _id: 2 } - replaceOne: namespace: *db1Coll2Namespace - filter: &filter4 - _id: 4 - replacement: &replacement - x: 44 + filter: { _id: 4 } + replacement: { x: 45 } verboseResults: true expectResult: insertedCount: 2 @@ -117,22 +109,22 @@ tests: bulkWrite: 1 ops: - insert: 0 - document: *document1 + document: { _id: 1 } - insert: 0 - document: *document2 + document: { _id: 2 } - update: 1 - filter: *filter1 - updateMods: *update + filter: { _id: 1 } + updateMods: { $inc: { x: 1 } } multi: false - delete: 2 - filter: *filter2 + filter: { _id: 3 } multi: false - delete: 1 - filter: *filter3 + filter: { _id: 2 } multi: false - update: 2 - filter: *filter4 - updateMods: *replacement + filter: { _id: 4 } + updateMods: { x: 45 } multi: false nsInfo: - ns: *db0Coll0Namespace @@ -142,8 +134,8 @@ tests: - databaseName: *database0Name collectionName: *collection0Name documents: - - *document1 - - *document2 + - { _id: 1 } + - { _id: 2 } - databaseName: *database0Name collectionName: *collection1Name documents: @@ -151,4 +143,4 @@ tests: - databaseName: *database1Name collectionName: *collection2Name documents: - - { _id: 4, x: 44 } + - { _id: 4, x: 45 } diff --git a/source/crud/tests/unified/client-bulkWrite-options.yml b/source/crud/tests/unified/client-bulkWrite-options.yml index 6b7e1d8c49..1ef6e31921 100644 --- a/source/crud/tests/unified/client-bulkWrite-options.yml +++ b/source/crud/tests/unified/client-bulkWrite-options.yml @@ -1,5 +1,5 @@ description: "client bulkWrite top-level options" -schemaVersion: "1.0" +schemaVersion: "1.1" runOnRequirements: - minServerVersion: "8.0" diff --git a/source/crud/tests/unified/client-bulkWrite-ordered.yml b/source/crud/tests/unified/client-bulkWrite-ordered.yml index 1e0b912f68..dc56dcb860 100644 --- a/source/crud/tests/unified/client-bulkWrite-ordered.yml +++ b/source/crud/tests/unified/client-bulkWrite-ordered.yml @@ -1,5 +1,5 @@ description: "client bulkWrite with ordered option" -schemaVersion: "1.0" +schemaVersion: "1.1" runOnRequirements: - minServerVersion: "8.0" diff --git a/source/crud/tests/unified/client-bulkWrite-results.yml b/source/crud/tests/unified/client-bulkWrite-results.yml index 76275f2867..b4731f1930 100644 --- a/source/crud/tests/unified/client-bulkWrite-results.yml +++ b/source/crud/tests/unified/client-bulkWrite-results.yml @@ -1,5 +1,5 @@ description: "client bulkWrite results" -schemaVersion: "1.0" +schemaVersion: "1.1" runOnRequirements: - minServerVersion: "8.0" diff --git a/source/crud/tests/unified/client-bulkWrite-update-options.yml b/source/crud/tests/unified/client-bulkWrite-update-options.yml index 4abed1979f..a04cedea66 100644 --- a/source/crud/tests/unified/client-bulkWrite-update-options.yml +++ b/source/crud/tests/unified/client-bulkWrite-update-options.yml @@ -1,5 +1,5 @@ description: "client bulkWrite update options" -schemaVersion: "1.0" +schemaVersion: "1.1" runOnRequirements: - minServerVersion: "8.0" diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml index 4f3194e829..25a95c24b6 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml @@ -2,7 +2,10 @@ description: "client bulkWrite retryable writes" schemaVersion: "1.20" runOnRequirements: - minServerVersion: "8.0" - topologies: [ replicaset ] + topologies: + - replicaset + - sharded + - load-balanced createEntities: - client: diff --git a/source/transactions/tests/unified/client-bulkWrite.yml b/source/transactions/tests/unified/client-bulkWrite.yml index 9688af7da1..fa8cb535d9 100644 --- a/source/transactions/tests/unified/client-bulkWrite.yml +++ b/source/transactions/tests/unified/client-bulkWrite.yml @@ -1,5 +1,5 @@ description: "client bulkWrite transactions" -schemaVersion: "1.0" +schemaVersion: "1.3" runOnRequirements: - minServerVersion: "8.0" topologies: diff --git a/source/versioned-api/tests/crud-api-version-1.yml b/source/versioned-api/tests/crud-api-version-1.yml index dd21967902..57f1b1c24f 100644 --- a/source/versioned-api/tests/crud-api-version-1.yml +++ b/source/versioned-api/tests/crud-api-version-1.yml @@ -183,11 +183,14 @@ tests: events: - commandStartedEvent: commandName: bulkWrite + databaseName: admin command: bulkWrite: 1 + errorsOnly: false + ordered: true ops: - insert: 0 - document: { x: 1 } + document: { _id: 6, x: 6 } nsInfo: - { ns: *namespace } <<: *expectedApiVersion From 7e1e8979b9b30da390b2e04b951b8b915314d0e0 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Mon, 4 Mar 2024 11:15:34 -0700 Subject: [PATCH 19/90] language updates --- source/crud/bulk-write.md | 17 ++++++++++------- source/crud/tests/README.md | 13 ++++++------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 0b9b9ad9e0..e3abb643a2 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -217,8 +217,8 @@ corresponding namespace that defines the collection on which the operation shoul Drivers SHOULD design this pairing in whichever way is most ergonomic for its language. For example, drivers may: -- Include a required `namespace` field on each `WriteModel` variant -- Accept a list of `(Namespace, WriteModel)` tuples for `models` +- Include a required `namespace` field on each `WriteModel` variant. +- Accept a list of `(Namespace, WriteModel)` tuples for `models`. - Define the following pair class: ```typescript @@ -545,12 +545,14 @@ whether the user specified a value for `verboseResults`. ## Command Batching -Drivers MUST accept an arbitrary number of operations as input to the `Client.bulkWrite` method. +Drivers MUST accept an arbitrary number of operations as input to the `MongoClient.bulkWrite` method. Because the server imposes restrictions on the size of write operations, this means that a single call to `MongoClient.bulkWrite` may require multiple `bulkWrite` commands to be sent to the server. Drivers MUST split bulk writes into separate commands when the user's list of operations exceeds one or more of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize`, and `maxMessageSizeBytes`. -Each of these values can be retrieved from the selected server's `hello` command response. +Each of these values can be retrieved from the selected server's `hello` command response. Drivers +MUST merge results from multiple batches into a single `BulkWriteResult` or `BulkWriteException` +to return from `MongoClient.bulkWrite`. ### Number of Writes @@ -570,8 +572,8 @@ defined in the When `ops` and `nsInfo` are provided as document sequences, drivers MUST ensure that the total size of the `OP_MSG` built for the `bulkWrite` command does not exceed `maxMessageSizeBytes`. Some drivers may perform batch-splitting prior to constructing the full `OP_MSG` to be sent to the -server. In this case, drivers MAY use `maxMessageSizeBytes - 16,000` as the upper bound for the -combined number of bytes in `ops` and `nsInfo`. 16KB is subtracted to accommodate for the bytes in +server. In this case, drivers MAY use `maxMessageSizeBytes - 16,384` as the upper bound for the +combined number of bytes in `ops` and `nsInfo`. 16KiB is subtracted to accommodate for the bytes in the rest of the message. #### Unencrypted bulk writes without document sequences @@ -604,7 +606,8 @@ The server's response to `bulkWrite` has the following format: If any operations were successful (i.e. `nErrors` is less than the number of operations that were sent), drivers MUST record the summary count fields in a `BulkWriteResult` to be returned to the -user or embedded in a `BulkWriteException`. +user or embedded in a `BulkWriteException`. Drivers MUST NOT populate the `partialResult` field in +`BulkWriteException` if no operations were successful. Drivers MUST attempt to consume the contents of the cursor returned in the server's `bulkWrite` response before returning to the user. This is required regardless of whether the user requested diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index d8421fb527..e035a25378 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -280,9 +280,9 @@ command. For both `firstEvent` and `secondEvent`, assert that the length of `com ### 5. `MongoClient.bulkWrite` with document sequences handles a `bulkWrite` message larger than `maxMessageSizeBytes` -Test that `MongoClient.bulkWrite` properly handles a `writeModels` input for which the sum of the models' entries in the -`ops` array exceeds `maxMessageSizeBytes`. Drivers that do not use document sequences (`OP_MSG` payload type 1) for bulk -writes should not implement this test. +Test that `MongoClient.bulkWrite` properly handles a `writeModels` input which constructs an `ops` array larger than +`maxMessageSizeBytes`. Drivers that do not use document sequences (`OP_MSG` payload type 1) for bulk writes should +not implement this test. This test must only be run on 8.0+ servers. @@ -314,7 +314,6 @@ Construct as list of write models (referred to as `models`) with `model` repeate `bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` with an `insertedCount` value of `numModels`. -Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEvent`) were observed. Record the length of -`firstEvent.command.ops` as `firstOpsLen`. Record the length of `secondEvent.command.ops` as `secondOpsLen`. Assert that -`firstOpsLen + secondOpsLen` is equal to `numModels`. If the driver exposes `operationId`s in its CommandStartedEvents, -assert that `firstEvent.operationId` is equal to `secondEvent.operationId`. +Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEvent`) were observed. Assert that the sum +of the lengths of `firstEvent.command.ops` and `secondEvent.command.ops` is equal to `numModels`. If the driver exposes +`operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to `secondEvent.operationId`. From d90750286e71b0a92ccbd6c4eb55bc4ef4ddbf70 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Mon, 4 Mar 2024 13:54:16 -0700 Subject: [PATCH 20/90] minor updates --- .../client-bulkWrite-delete-options.json | 2 +- .../unified/client-bulkWrite-errors.json | 52 ++++++++++++++----- .../tests/unified/client-bulkWrite-errors.yml | 33 ++++++++---- .../client-bulkWrite-mixed-namespaces.json | 8 +-- .../unified/client-bulkWrite-options.json | 2 +- .../unified/client-bulkWrite-ordered.json | 2 +- .../unified/client-bulkWrite-results.json | 2 +- .../client-bulkWrite-update-options.json | 2 +- .../client-bulkWrite-serverErrors.json | 4 +- .../tests/unified/client-bulkWrite.json | 2 +- .../tests/crud-api-version-1.json | 6 ++- 11 files changed, 82 insertions(+), 33 deletions(-) diff --git a/source/crud/tests/unified/client-bulkWrite-delete-options.json b/source/crud/tests/unified/client-bulkWrite-delete-options.json index 8ad03a2003..5bdf2b124a 100644 --- a/source/crud/tests/unified/client-bulkWrite-delete-options.json +++ b/source/crud/tests/unified/client-bulkWrite-delete-options.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite delete options", - "schemaVersion": "1.0", + "schemaVersion": "1.1", "runOnRequirements": [ { "minServerVersion": "8.0" diff --git a/source/crud/tests/unified/client-bulkWrite-errors.json b/source/crud/tests/unified/client-bulkWrite-errors.json index 9719a9d97d..1404fad427 100644 --- a/source/crud/tests/unified/client-bulkWrite-errors.json +++ b/source/crud/tests/unified/client-bulkWrite-errors.json @@ -59,7 +59,8 @@ "writeConcernError": { "code": 91, "errmsg": "Replication is being shut down" - } + }, + "undefinedVarCode": 17276 }, "tests": [ { @@ -144,11 +145,6 @@ }, { "description": "an individual operation fails during an unordered bulkWrite", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -229,11 +225,6 @@ }, { "description": "detailed results are omitted from error when verboseResults is false", - "runOnRequirements": [ - { - "minServerVersion": "8.0" - } - ], "operations": [ { "object": "client0", @@ -302,8 +293,8 @@ "description": "a top-level failure occurs during a bulkWrite", "operations": [ { - "name": "failPoint", "object": "testRunner", + "name": "failPoint", "arguments": { "client": "client0", "failPoint": { @@ -342,6 +333,43 @@ } ] }, + { + "description": "a bulk write with only errors does not report a partial result", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "deleteOne": { + "namespace": "crud-tests.coll0", + "filter": { + "$expr": { + "$eq": [ + "$_id", + "$$id2" + ] + } + } + } + } + ], + "verboseResults": true + }, + "expectError": { + "expectResult": { + "$$unsetOrMatches": {} + }, + "writeErrors": { + "0": { + "code": 17276 + } + } + } + } + ] + }, { "description": "a write concern error occurs during a bulkWrite", "operations": [ diff --git a/source/crud/tests/unified/client-bulkWrite-errors.yml b/source/crud/tests/unified/client-bulkWrite-errors.yml index 46d7683e1a..e05bef220a 100644 --- a/source/crud/tests/unified/client-bulkWrite-errors.yml +++ b/source/crud/tests/unified/client-bulkWrite-errors.yml @@ -32,6 +32,7 @@ _yamlAnchors: writeConcernError: &writeConcernError code: 91 errmsg: "Replication is being shut down" + undefinedVarCode: &undefinedVarCode 17276 # Use of an undefined variable tests: - description: "an individual operation fails during an ordered bulkWrite" @@ -66,7 +67,7 @@ tests: deletedCount: 1 writeErrors: 1: - code: 17276 # Use of undefined variable + code: *undefinedVarCode outcome: - collectionName: *collection0Name databaseName: *database0Name @@ -74,8 +75,6 @@ tests: - { _id: 2, x: 22 } - { _id: 3, x: 33 } - description: "an individual operation fails during an unordered bulkWrite" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -110,15 +109,13 @@ tests: deletedCount: 1 writeErrors: 1: - code: 17276 # Use of undefined variable + code: *undefinedVarCode outcome: - collectionName: *collection0Name databaseName: *database0Name documents: - { _id: 2, x: 22 } - description: "detailed results are omitted from error when verboseResults is false" - runOnRequirements: - - minServerVersion: "8.0" operations: - object: *client0 name: clientBulkWrite @@ -151,11 +148,11 @@ tests: $$unsetOrMatches: {} writeErrors: 1: - code: 17276 # Use of undefined variable + code: *undefinedVarCode - description: "a top-level failure occurs during a bulkWrite" operations: - - name: failPoint - object: testRunner + - object: testRunner + name: failPoint arguments: client: *client0 failPoint: @@ -176,6 +173,24 @@ tests: verboseResults: true expectError: errorCode: 8 + - description: "a bulk write with only errors does not report a partial result" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + models: + - deleteOne: + namespace: *namespace + filter: + $expr: + $eq: [ "$_id", "$$id2" ] # Attempt to access a nonexistent let var + verboseResults: true + expectError: + expectResult: + $$unsetOrMatches: {} # Empty or nonexistent result when no successful writes occurred + writeErrors: + 0: + code: *undefinedVarCode - description: "a write concern error occurs during a bulkWrite" operations: - name: failPoint diff --git a/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.json b/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.json index 6db78d2ac2..33df3257c7 100644 --- a/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.json +++ b/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite with mixed namespaces", - "schemaVersion": "1.0", + "schemaVersion": "1.1", "runOnRequirements": [ { "minServerVersion": "8.0" @@ -152,7 +152,7 @@ "_id": 4 }, "replacement": { - "x": 44 + "x": 45 } } } @@ -249,7 +249,7 @@ "_id": 4 }, "updateMods": { - "x": 44 + "x": 45 }, "multi": false } @@ -300,7 +300,7 @@ "documents": [ { "_id": 4, - "x": 44 + "x": 45 } ] } diff --git a/source/crud/tests/unified/client-bulkWrite-options.json b/source/crud/tests/unified/client-bulkWrite-options.json index d92be7b197..fd1a393009 100644 --- a/source/crud/tests/unified/client-bulkWrite-options.json +++ b/source/crud/tests/unified/client-bulkWrite-options.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite top-level options", - "schemaVersion": "1.0", + "schemaVersion": "1.1", "runOnRequirements": [ { "minServerVersion": "8.0" diff --git a/source/crud/tests/unified/client-bulkWrite-ordered.json b/source/crud/tests/unified/client-bulkWrite-ordered.json index 5ce983f632..a55d6619b5 100644 --- a/source/crud/tests/unified/client-bulkWrite-ordered.json +++ b/source/crud/tests/unified/client-bulkWrite-ordered.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite with ordered option", - "schemaVersion": "1.0", + "schemaVersion": "1.1", "runOnRequirements": [ { "minServerVersion": "8.0" diff --git a/source/crud/tests/unified/client-bulkWrite-results.json b/source/crud/tests/unified/client-bulkWrite-results.json index 92aa734640..726d15ffda 100644 --- a/source/crud/tests/unified/client-bulkWrite-results.json +++ b/source/crud/tests/unified/client-bulkWrite-results.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite results", - "schemaVersion": "1.0", + "schemaVersion": "1.1", "runOnRequirements": [ { "minServerVersion": "8.0" diff --git a/source/crud/tests/unified/client-bulkWrite-update-options.json b/source/crud/tests/unified/client-bulkWrite-update-options.json index d163775bd0..5d53864023 100644 --- a/source/crud/tests/unified/client-bulkWrite-update-options.json +++ b/source/crud/tests/unified/client-bulkWrite-update-options.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite update options", - "schemaVersion": "1.0", + "schemaVersion": "1.1", "runOnRequirements": [ { "minServerVersion": "8.0" diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json index caab0b5463..646adc8b19 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json @@ -5,7 +5,9 @@ { "minServerVersion": "8.0", "topologies": [ - "replicaset" + "replicaset", + "sharded", + "load-balanced" ] } ], diff --git a/source/transactions/tests/unified/client-bulkWrite.json b/source/transactions/tests/unified/client-bulkWrite.json index 4fbb4e1f2a..5b9f5aa781 100644 --- a/source/transactions/tests/unified/client-bulkWrite.json +++ b/source/transactions/tests/unified/client-bulkWrite.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite transactions", - "schemaVersion": "1.0", + "schemaVersion": "1.3", "runOnRequirements": [ { "minServerVersion": "8.0", diff --git a/source/versioned-api/tests/crud-api-version-1.json b/source/versioned-api/tests/crud-api-version-1.json index da39bc900c..d959d25503 100644 --- a/source/versioned-api/tests/crud-api-version-1.json +++ b/source/versioned-api/tests/crud-api-version-1.json @@ -470,13 +470,17 @@ { "commandStartedEvent": { "commandName": "bulkWrite", + "databaseName": "admin", "command": { "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, "ops": [ { "insert": 0, "document": { - "x": 1 + "_id": 6, + "x": 6 } } ], From a4cf43f75fe773254bfe66d673f8b61af9164fa2 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Mon, 11 Mar 2024 16:04:02 -0600 Subject: [PATCH 21/90] review changes --- source/crud/bulk-write.md | 229 +++++++++++++++++++++++--------------- 1 file changed, 137 insertions(+), 92 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index e3abb643a2..97413cde40 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -15,12 +15,15 @@ defined in the CRUD specification and the ## Specification -### `MongoClient.bulkWrite` Interface +> \[!NOTE\] +> +> The `BulkWriteOptions`, `BulkWriteResult`, and `BulkWriteException` types defined in this +> specification are similar to those used for the `MongoCollection.bulkWrite` method. Statically +> typed drivers MUST NOT reuse their existing definitions for these types for the +> `MongoClient.bulkWrite` API and MUST introduce new types. If naming conflicts arise, drivers +> SHOULD prepend "Client" to the new type names (e.g. `ClientBulkWriteOptions`). -**NOTE:** The types specified in this interface are similar to those used for the collection-level -`bulkWrite` method. Statically typed drivers MUST NOT reuse any types they already define for the -`MongoCollection.bulkWrite` interface and MUST introduce new types. If naming conflicts between -types arise, drivers SHOULD prepend "Client" to the new type names (e.g. `ClientBulkWriteOptions`). +### `MongoClient.bulkWrite` Interface ```typescript interface MongoClient { @@ -82,12 +85,13 @@ class UpdateOneModel implements WriteModel { * * This option is only sent if the caller explicitly provides a value. */ - hint: Optional<(String | Document)>; + hint: Optional; /** - * When true, creates a new document if no document matches the query. Defaults to false. + * When true, creates a new document if no document matches the query. * - * This options is sent only if the caller explicitly provides a value. + * This option is only sent if the caller explicitly provides a value. The server's default + * value is false. */ upsert: Optional; } @@ -123,12 +127,13 @@ class UpdateManyModel implements WriteModel { * * This option is only sent if the caller explicitly provides a value. */ - hint: Optional<(String | Document)>; + hint: Optional; /** - * When true, creates a new document if no document matches the query. Defaults to false. + * When true, creates a new document if no document matches the query. * - * This options is sent only if the caller explicitly provides a value. + * This option is only sent if the caller explicitly provides a value. The server's default + * value is false. */ upsert: Optional; } @@ -142,7 +147,7 @@ class ReplaceOneModel implements WriteModel { /** * The replacement document. */ - filter: Document; + replacement: Document; /** * Specifies a collation. @@ -157,12 +162,13 @@ class ReplaceOneModel implements WriteModel { * * This option is only sent if the caller explicitly provides a value. */ - hint: Optional<(String | Document)>; + hint: Optional; /** - * When true, creates a new document if no document matches the query. Defaults to false. + * When true, creates a new document if no document matches the query. * - * This options is sent only if the caller explicitly provides a value. + * This option is only sent if the caller explicitly provides a value. The server's default + * value is false. */ upsert: Optional; } @@ -186,7 +192,7 @@ class DeleteOneModel implements WriteModel { * * This option is only sent if the caller explicitly provides a value. */ - hint: Optional<(String | Document)>; + hint: Optional; } class DeleteManyModel implements WriteModel { @@ -208,16 +214,17 @@ class DeleteManyModel implements WriteModel { * * This option is only sent if the caller explicitly provides a value. */ - hint: Optional<(String | Document)>; + hint: Optional; } ``` Each write model provided to `MongoClient.bulkWrite` in the `models` parameter MUST have a corresponding namespace that defines the collection on which the operation should be performed. -Drivers SHOULD design this pairing in whichever way is most ergonomic for its language. For +Drivers SHOULD design this pairing in whichever way is most idiomatic for its language. For example, drivers may: -- Include a required `namespace` field on each `WriteModel` variant. +- Include a required `namespace` field on each `WriteModel` variant and accept a list of + `WriteModel` objects for the `models` parameter. - Accept a list of `(Namespace, WriteModel)` tuples for `models`. - Define the following pair class: @@ -253,9 +260,8 @@ class BulkWriteOptions { /** * If true, allows the writes to opt out of document-level validation. * - * Defaults to false. - * - * This option is only sent if the caller explicitly provides a value. + * This option is only sent if the caller explicitly provides a value. The server's default + * value is false. */ bypassDocumentValidation: Optional; @@ -270,12 +276,6 @@ class BulkWriteOptions { /** * The write concern to use for this bulk write. - * - * NOT REQUIRED TO IMPLEMENT. Drivers MUST expose this option if retrieving a handle to a - * client with a different write concern configured than that of the user's standard URI - * options is nontrivial. Drivers MAY omit this option if they provide a way to retrieve a - * lightweight handle to a client with a custom write concern configured, e.g. a - * MongoClient.withWriteConcern() method. */ writeConcern: Optional; @@ -283,7 +283,8 @@ class BulkWriteOptions { * Whether detailed results for each successful operation should be included in the returned * BulkWriteResult. * - * Defaults to false. + * Defaults to false. This value corresponds inversely to the errorsOnly field in the bulkWrite + * command. */ verboseResults: Optional; } @@ -297,8 +298,7 @@ class BulkWriteResult { * Indicates whether this write result was acknowledged. If not, then all other members of this * result will be undefined. * - * NOT REQUIRED TO IMPLEMENT. See the CRUD specification for more guidance on modeling - * unacknowledged results. + * NOT REQUIRED TO IMPLEMENT. See below for more guidance on modeling unacknowledged results. */ acknowledged: Boolean; @@ -344,7 +344,7 @@ class BulkWriteResult { /** * The results of each individual update operation that was successfully performed. */ - updateResult: Map; + updateResults: Map; /** * The results of each individual delete operation that was successfully performed. @@ -370,18 +370,14 @@ class UpdateResult { */ modifiedCount: Int64; - /** - * The number of documents that were upserted. - * - * NOT REQUIRED TO IMPLEMENT. Drivers may choose not to provide this property so long as it is - * always possible to discern whether an upsert took place. - */ - upsertedCount: Int64; - /** * The _id field of the upserted document if an upsert occurred. + * + * It MUST be possible to discern between a BSON Null upserted ID value and this field being + * unset. If necessary, drivers MAY add a didUpsert boolean field to differentiate between + * these two cases. */ - upsertedId: Optional; + upsertedId: Optional; } class DeleteResult { @@ -392,21 +388,31 @@ class DeleteResult { } ``` -#### Summary vs. Verbose Results +#### Unacknowledged results + +`BulkWriteResult` has an optional `acknowledged` field to indicate whether the result was +acknowledged. This is not required to implement. Drivers should follow the guidance in the CRUD +specification [here](../crud/crud.md#write-results) to determine how to model unacknowledged +results. + +#### Summary vs. verbose results Users MUST be able to discern whether a `BulkWriteResult` contains summary or verbose results without inspecting the value provided for `verboseResults` in `BulkWriteOptions`. Drivers MUST implement this in one of the following ways: -- Expose the `hasVerboseResults` field in `BulkWriteResult` as defined above. -- Implement the `insertResults`, `upsertResults`, and `deleteResults` fields as optional types and +- Expose the `hasVerboseResults` field in `BulkWriteResult` as defined above. Document that + `insertResults`, `updateResults`, and `deleteResults` will be undefined when `hasVerboseResults` + is false. Raise an error if a user tries to access one of these fields when `hasVerboseResults` + is false. +- Implement the `insertResults`, `updateResults`, and `deleteResults` fields as optional types and document that they will be unset when `verboseResults` is false. - Introduce separate `SummaryBulkWriteResult` and `VerboseBulkWriteResult` types. `VerboseBulkWriteResult` MUST have all of the required fields defined on `BulkWriteResult` above. `SummaryBulkWriteResult` MUST have all of the required fields defined on `BulkWriteResult` above except `insertResults`, `updateResults`, and `deleteResults`. -#### Individual Results +#### Individual results The `InsertOneResult`, `UpdateResult`, and `DeleteResult` classes are the same as or similar to types of the same name defined in the [CRUD specification](crud.md). Drivers MUST redefine these @@ -455,16 +461,16 @@ The `bulkWrite` server command has the following format: "bulkWrite": 1, "ops": , "nsInfo": , - "errorsOnly": , - "ordered": , - "bypassDocumentValidation": , - "comment": , - "let": , + "errorsOnly": Optional, + "ordered": Optional, + "bypassDocumentValidation": Optional, + "comment": Optional, + "let": Optional, ...additional operation-agnostic fields } ``` -Drivers SHOULD use document sequences ([`OP_MSG`](../message/OP_MSG.rst) payload type 1) for the +Drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.rst) payload type 1) for the `ops` and `nsInfo` fields. Drivers MUST NOT use document sequences when auto-encryption is enabled. The `bulkWrite` command is executed on the "admin" database. @@ -485,10 +491,10 @@ documents have the following format: } ``` -Drivers MUST add an `_id` field at the beginning of the insert document if one is not already -present. +If the document to be inserted does not contain an `_id` field, drivers MUST generate a new +[`ObjectId`](../objectid.rst) and add it as the `_id` field at the beginning of the document. -When a user executes a bulk write with an unacknowledged write concern, drivers SHOULD check the +When a user executes a bulk write with an unacknowledged write concern, drivers MUST check the size of the insert document to verify that it does not exceed `maxBsonObjectSize`. For acknowledged bulk writes, drivers MAY rely on the server to return an error if the document exceeds `maxBsonObjectSize`. The value for `maxBsonObjectSize` can be retrieved from the selected server's @@ -501,10 +507,10 @@ bulk writes, drivers MAY rely on the server to return an error if the document e "update": , "filter": , "updateMods": , - "multi": , + "multi": Optional, "upsert": Optional, "arrayFilters": Optional, - "hint": Optional + "hint": Optional } ``` @@ -514,8 +520,8 @@ bulk writes, drivers MAY rely on the server to return an error if the document e { "delete": , "filter": , - "multi": , - "hint": Optional, + "multi": Optional, + "hint": Optional, "collation": Optional } ``` @@ -536,12 +542,42 @@ performed. Drivers MUST NOT include duplicate namespaces in this list. The docum The `errorsOnly` field indicates whether the results cursor returned in the `bulkWrite` response should contain only errors and omit individual results. If false, both individual results for -successful operations and errors will be returned. - -This field corresponds to the `verboseResults` option defined on `BulkWriteOptions`. Its value -should be provided as `false` if `verboseResults` was set to `true`; otherwise, it should be -provided as `true`. This field MUST always be defined in the `bulkWrite` command regardless of -whether the user specified a value for `verboseResults`. +successful operations and errors will be returned. This field is optional and defaults to false on +the server. + +`errorsOnly` corresponds to the `verboseResults` option defined on `BulkWriteOptions`. If the user +specified a value for `verboseResults`, drivers MUST define `errorsOnly` as the opposite of +`verboseResults`. If the user did not specify a value for `verboseResults`, drivers MUST define +`errorsOnly` as `true`. + +### `ordered` + +The `ordered` field defines whether writes should be executed in the order in which they were +specified, and, if an error occurs, whether the server should halt execution of further writes. It +is optional and defaults to true on the server. Drivers MUST explicitly define `ordered` as `true` +in the `bulkWrite` command if a value is not specified in `BulkWriteOptions`. This is required to +avoid inconsistencies between server and driver behavior if the server default changes in the +future. + +### Size Limits + +The server reports a `maxBsonObjectSize` in its `hello` response. This value defines the maximum +size for documents that are inserted into the database. Documents that are sent to the server but +are not intended to be inserted into the database (e.g. command documents) have a size limit of +`maxBsonObjectSize + 16KiB`. When an acknowledged write concern is used, drivers MUST NOT perform +any checks related to these size limits and MUST rely on the server to raise an error if a limit is +exceeded. However, when an unacknowledged write concern is used, drivers MUST raise an error if one +of the following limits is exceeded: + +- The size of a document to be inserted MUST NOT exceed `maxBsonObjectSize`. This applies to the + `document` field of an `InsertOneModel` and the `replacement` field of a `ReplaceOneModel`. +- The size of an entry in the `ops` array MUST NOT exceed `maxBsonObjectSize + 16KiB`. +- The size of the `bulkWrite` command document MUST NOT exceed `maxBsonObjectSize`. The overhead + bytes are omitted here intentionally to accommodate additional fields that mongos appends to the + command document. + +See [SERVER-10643](https://jira.mongodb.org/browse/SERVER-10643) for more details on these size +limits. ## Command Batching @@ -549,38 +585,34 @@ Drivers MUST accept an arbitrary number of operations as input to the `MongoClie Because the server imposes restrictions on the size of write operations, this means that a single call to `MongoClient.bulkWrite` may require multiple `bulkWrite` commands to be sent to the server. Drivers MUST split bulk writes into separate commands when the user's list of operations exceeds -one or more of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize`, and `maxMessageSizeBytes`. -Each of these values can be retrieved from the selected server's `hello` command response. Drivers -MUST merge results from multiple batches into a single `BulkWriteResult` or `BulkWriteException` -to return from `MongoClient.bulkWrite`. +one or more of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize` (for `OP_MSG` payload type +0), and `maxMessageSizeBytes` (for `OP_MSG` payload type 1). Each of these values can be retrieved +from the selected server's `hello` command response. Drivers MUST merge results from multiple batches +into a single `BulkWriteResult` or `BulkWriteException` to return from `MongoClient.bulkWrite`. ### Number of Writes -The `ops` array MUST NOT include more than `maxWriteBatchSize` operations. +`maxWriteBatchSize` defines the total number of writes allowed in one command. Drivers MUST split a +bulk write into multiple commands if the user provides more than `maxWriteBatchSize` operations in +the argument for `models`. ### Total Message Size #### Encrypted bulk writes When auto-encryption is enabled, drivers MUST NOT provide the `ops` and `nsInfo` fields as document -sequences and MUST limit the size of the command according to the auto-encryption size limits -defined in the +sequences and MUST limit the size of each `bulkWrite` command according to the auto-encryption size +limits defined in the [Client Side Encryption Specification](../client-side-encryption/client-side-encryption.rst). -#### Unencrypted bulk writes with document sequences +#### Unencrypted bulk writes When `ops` and `nsInfo` are provided as document sequences, drivers MUST ensure that the total size -of the `OP_MSG` built for the `bulkWrite` command does not exceed `maxMessageSizeBytes`. Some +of the `OP_MSG` built for each `bulkWrite` command does not exceed `maxMessageSizeBytes`. Some drivers may perform batch-splitting prior to constructing the full `OP_MSG` to be sent to the server. In this case, drivers MAY use `maxMessageSizeBytes - 16,384` as the upper bound for the -combined number of bytes in `ops` and `nsInfo`. 16KiB is subtracted to accommodate for the bytes in -the rest of the message. - -#### Unencrypted bulk writes without document sequences - -When `ops` and `nsInfo` are provided within the `bulkWrite` command document and auto-encryption is -not enabled, drivers MUST ensure that the total size of the `bulkWrite` command document does not -exceed `maxBsonObjectSize`. +combined number of bytes in `ops` and `nsInfo`. 16KiB is subtracted as an approximate overhead +allowance to accommodate for the bytes in the rest of the message. ## Handling the `bulkWrite` Server Response @@ -614,25 +646,38 @@ response before returning to the user. This is required regardless of whether th verbose or summary results, as the results cursor always contains any write errors that occurred. If the cursor contains a nonzero cursor ID, drivers MUST perform `getMore`s until the cursor has been exhausted. Drivers MUST use the same session used for the `bulkWrite` command for each -`getMore` call. +`getMore` call. When connected to a load balancer, drivers MUST use the connection used for the +`bulkWrite` command to create the cursor to ensure the same server is targeted. The documents in the results cursor have the following format: ```json { "ok": <0 | 1>, + "idx": Int32, "code": Optional, "errmsg": Optional, "n": , - "nMatched": , - "nModified": , - "upsertedId": Optional + "nModified": Optional, + "upsertedId": Optional } ``` +If an error occurred (i.e. the value for `ok` is 0), the `code` and `errmsg` fields will be +populated with details about the failure. + +If the write succeeded, (i.e. the value for `ok` is 1), `n`, `nModified`, and `upsertedId` will be +populated with the following values based on the type of write: + +| Response Field | Insert | Update | Delete | +| -------------- | ------ | ------ | ------ | +| `n` | The number of documents that were inserted. | The number of documents that matched the filter. | The number of documents that were deleted. | +| `nModified` | Not present. | The number of documents that were modified. | Not present. | +| `upsertedId` | Not present. | The `_id` value for the upserted document. Only present if an upsert took place. | Not present. | + Note that the responses do not contain information about the type of operation that was performed. Drivers may need to maintain the user's list of write models to infer which type of result should -be recorded. +be recorded based on the value of `idx`. ### Handling Insert Results @@ -643,14 +688,14 @@ successful response for the insert operation (i.e. `{ "ok": 1, "n": 1 }`) is rec results cursor. This ensures that drivers only report an `insertedId` when it is confirmed that the insert succeeded. -## Handling errors +## Handling Errors -### Top-level errors +### Top-Level Errors A top-level error is any error that occurs that is not the result of a single write operation failing or a write concern error. Examples include network errors that occur when communicating -with the server, command errors returned from the server, client-side errors, and errors that occur -when attempting to perform a `getMore` to retrieve results from the server. +with the server, command errors (`{ "ok": 0 }`) returned from the server, client-side errors, and +errors that occur when attempting to perform a `getMore` to retrieve results from the server. When a top-level error is encountered and individual results and/or errors have already been observed, drivers MUST embed the top-level error within a `BulkWriteException` as the `error` field @@ -661,14 +706,14 @@ Encountering a top-level error MUST halt execution of a bulk write for both orde bulk writes. This means that drivers MUST NOT attempt to retrieve more responses from the cursor or execute any further `bulkWrite` batches and MUST immediately throw an exception. -### Write concern errors +### Write Concern Errors Write concern errors are recorded in the `writeConcernErrors` field on `BulkWriteException`. When a write concern error is encountered, it should not terminate execution of the bulk write for either ordered or unordered bulk writes. However, drivers MUST throw an exception at the end of execution if any write concern errors were observed. -### Individual write errors +### Individual Write Errors Individual write errors retrieved from the cursor are recorded in the `writeErrors` field on `BulkWriteException`. If an individual write error is encountered during an ordered bulk write, From 13f88a7e3f7c05494bbdc3e8bf826da33491f686 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Mon, 11 Mar 2024 16:07:14 -0600 Subject: [PATCH 22/90] fix ordered --- source/crud/bulk-write.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 97413cde40..f46fd3b374 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -253,7 +253,7 @@ class BulkWriteOptions { * specified. If false, writes will continue to be executed if an individual write fails. If * true, writes will stop executing if an individual write fails. * - * Defaults to false. + * Defaults to true. */ ordered: Optional; @@ -494,12 +494,6 @@ documents have the following format: If the document to be inserted does not contain an `_id` field, drivers MUST generate a new [`ObjectId`](../objectid.rst) and add it as the `_id` field at the beginning of the document. -When a user executes a bulk write with an unacknowledged write concern, drivers MUST check the -size of the insert document to verify that it does not exceed `maxBsonObjectSize`. For acknowledged -bulk writes, drivers MAY rely on the server to return an error if the document exceeds -`maxBsonObjectSize`. The value for `maxBsonObjectSize` can be retrieved from the selected server's -`hello` response. - #### Update ```json From ee27603e00bc207b82697ee41338c5924b2ef7dc Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Mon, 11 Mar 2024 16:17:06 -0600 Subject: [PATCH 23/90] add future work --- source/crud/bulk-write.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index f46fd3b374..479595b72e 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -724,6 +724,16 @@ Additional prose tests are specified [here](../crud/tests/README.md). These test constructing very large documents to test batch splitting, which is not feasible in the unified test format at the time of writing this specification. +## Future Work + +### Support using document sequences in encrypted bulk writes + +Libmongocrypt is currently only capable of encrypting command documents, not wire protocol +messages. This means that all command fields must be embedded within the command document. When +[DRIVERS-2859](https://jira.mongodb.org/browse/DRIVERS-2859) is completed, drivers will be able to +specify `ops` and `nsInfo` as document sequences (`OP_MSG` payload type 1) for encrypted bulk +writes. + ## Q&A ### Why are we adding a new bulk write API rather than updating the `MongoCollection.bulkWrite` implementation? @@ -758,6 +768,10 @@ driver-side is less performant than only recording the summary counts. We expect are not interested in the individual results of their operations and that most users will rely on defaults, so `verboseResults` defaults to `false` to improve performance in the common case. +### Why are document sequences not permitted for encrypted bulk writes? + + + ## **Changelog** - TODO: Bulk write specification created. From 94ed55509151264df307e64898bfdfe5dd3299d7 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 12 Mar 2024 10:55:51 -0600 Subject: [PATCH 24/90] add errinfo --- source/crud/bulk-write.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 479595b72e..214af82390 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -651,14 +651,15 @@ The documents in the results cursor have the following format: "idx": Int32, "code": Optional, "errmsg": Optional, + "errInfo": Optional, "n": , "nModified": Optional, "upsertedId": Optional } ``` -If an error occurred (i.e. the value for `ok` is 0), the `code` and `errmsg` fields will be -populated with details about the failure. +If an error occurred (i.e. the value for `ok` is 0), the `code`, `errmsg`, and optionally +`errInfo` fields will be populated with details about the failure. If the write succeeded, (i.e. the value for `ok` is 1), `n`, `nModified`, and `upsertedId` will be populated with the following values based on the type of write: From b7a9d8645743374ebc45b71a3c26bde3a88a6604 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 12 Mar 2024 15:38:10 -0600 Subject: [PATCH 25/90] remove note escapes --- source/crud/bulk-write.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 214af82390..f680d2cefc 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -15,7 +15,7 @@ defined in the CRUD specification and the ## Specification -> \[!NOTE\] +> [!NOTE] > > The `BulkWriteOptions`, `BulkWriteResult`, and `BulkWriteException` types defined in this > specification are similar to those used for the `MongoCollection.bulkWrite` method. Statically From d557b6ecb92754c4f66a26a93def2aa65f272b42 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 12 Mar 2024 15:53:19 -0600 Subject: [PATCH 26/90] must -> should --- source/unified-test-format/unified-test-format.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 032052e70d..4e860b912f 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -1462,8 +1462,8 @@ arguments: ``` Because the `insertResults`, `updateResults`, and `deleteResults` may be absent or empty in the `BulkWriteResult` returned -from a summary-only bulk write, the `clientBulkWrite` operation MUST use the [$$unsetOrMatches](#unsetormatches) operator -for assertions on these fields when `verboseResults` is not set to true. This requirement also applies to result objects +from a summary-only bulk write, the `clientBulkWrite` operation SHOULD use the [$$unsetOrMatches](#unsetormatches) operator +for assertions on these fields when `verboseResults` is not set to true. This also applies to result objects defined in the `expectedResult` field of [expectedError](#expectederror). The `BulkWriteException` thrown by `MongoClient.bulkWrite` contains an optional `error` field that stores a top-level error From 89c99f67cf8d64aeadddc6877f90b901f9960ab8 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 14 Mar 2024 10:31:59 -0600 Subject: [PATCH 27/90] remove q --- source/crud/bulk-write.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index f680d2cefc..fcac1d8df2 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -769,10 +769,6 @@ driver-side is less performant than only recording the summary counts. We expect are not interested in the individual results of their operations and that most users will rely on defaults, so `verboseResults` defaults to `false` to improve performance in the common case. -### Why are document sequences not permitted for encrypted bulk writes? - - - ## **Changelog** - TODO: Bulk write specification created. From 82aa4f286e9a11571d9d475163bb142c27738bce Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 14 Mar 2024 12:52:32 -0600 Subject: [PATCH 28/90] more review changes --- source/crud/bulk-write.md | 4 +--- source/unified-test-format/unified-test-format.md | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index fcac1d8df2..bd1c726b77 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -566,9 +566,7 @@ of the following limits is exceeded: - The size of a document to be inserted MUST NOT exceed `maxBsonObjectSize`. This applies to the `document` field of an `InsertOneModel` and the `replacement` field of a `ReplaceOneModel`. - The size of an entry in the `ops` array MUST NOT exceed `maxBsonObjectSize + 16KiB`. -- The size of the `bulkWrite` command document MUST NOT exceed `maxBsonObjectSize`. The overhead - bytes are omitted here intentionally to accommodate additional fields that mongos appends to the - command document. +- The size of the `bulkWrite` command document MUST NOT exceed `maxBsonObjectSize + 16KiB`. See [SERVER-10643](https://jira.mongodb.org/browse/SERVER-10643) for more details on these size limits. diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 4e860b912f..64fe175969 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -927,13 +927,14 @@ The structure of this object is as follows: contains numeric keys representing the index of the write that failed and `writeError` object values. The test runner MUST assert that the error contains a `writeError` for each index present in `writeErrors` and MUST assert that the `writeError`s match as root-level documents according to the rules in [Evaluating Matches](#evaluating-matches). The test runner MUST - assert that the error does not contain any additional `writeError`s. + assert that the error does not contain any additional `writeError`s. This field is only intended for use with the + [clientBulkWrite](#clientbulkwrite) operation. - `writeConcernErrors`: Optional array of one or more objects. An ordered list of write concern errors expected to be present in the error. The test runner MUST assert that each `writeConcernError` in this list matches the `writeConcernError` present at the same index in the error's list of `writeConcernError`s as a root-level document according to the rules in [Evaluating Matches](#evaluating-matches). The test runner MUST assert that the error does not contain any - additional `writeConcernErrors`s. + additional `writeConcernErrors`s. This field is only intended for use with the [clientBulkWrite](#clientbulkwrite) operation.
From fb6007e76db08728f3951c423dca4d7d0da909b9 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 15 Mar 2024 10:32:08 -0600 Subject: [PATCH 29/90] remove outdated test, add bypassDocumentValidation language --- source/crud/bulk-write.md | 15 ++++++++++ source/crud/tests/README.md | 55 ++----------------------------------- 2 files changed, 18 insertions(+), 52 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index bd1c726b77..b5a3a750b6 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -767,6 +767,21 @@ driver-side is less performant than only recording the summary counts. We expect are not interested in the individual results of their operations and that most users will rely on defaults, so `verboseResults` defaults to `false` to improve performance in the common case. +### Why should drivers send `bypassDocumentValidation: false` for `bulkWrite`? + +[DRIVERS-450](https://jira.mongodb.org/browse/DRIVERS-450) introduced a requirement that drivers +only send a value for `bypassDocumentValidation` in the `insert` command if it was specified as +true. The original motivation for this change is not documented. Conversely, this specification +requires that drivers send `bypassDocumentValidation` if it is set by the user, regardless of its +value. + +Explicitly defining `bypassDocumentValidation: false` aligns with the server's default to perform +schema validation and thus has no effect. However, checking the value of an option that the user +specified and omitting it from the command document if it matches the server's default creates +unnecessary work for drivers. Always sending the user's specified value also safeguards against the +unlikely event that the server changes the default value for `bypassDocumentValidation` in the +future. + ## **Changelog** - TODO: Bulk write specification created. diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index e035a25378..9cac86c395 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -202,7 +202,7 @@ code is `121` (i.e. DocumentValidationFailure), and that its `details` property a CommandSucceededEvent was observed and that the `writeErrors[0].errInfo` field in the response document matches the WriteError's `details` property. -### 3. `MongoClient.bulkWrite` handles a `writeModels` input with greater than `maxWriteBatchSize` operations +### 3. `MongoClient.bulkWrite` batch splits a `writeModels` input with greater than `maxWriteBatchSize` operations Test that `MongoClient.bulkWrite` properly handles `writeModels` inputs containing a number of writes greater than `maxWriteBatchSize`. @@ -230,59 +230,10 @@ Assert that the length of `firstEvent.command.ops` is `maxWriteBatchSize`. Asser is 1. If the driver exposes `operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to `secondEvent.operationId`. -### 4. `MongoClient.bulkWrite` handles a `writeModels` input larger than `maxBsonObjectSize` +### 4. `MongoClient.bulkWrite` batch splits when an `ops` payload exceeds `maxMessageSizeBytes` Test that `MongoClient.bulkWrite` properly handles a `writeModels` input which constructs an `ops` array larger than -`maxBsonObjectSize`. - -This test must only be run on 8.0+ servers. - -Construct a `MongoClient` (referred to as `client`) with -[command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe -CommandStartedEvents. Perform a `hello` command using `client` and record the `maxBsonObjectSize` value contained in the -response. Then, construct the following document (referred to as `document`): - -```json -{ - "a": "b".repeat(maxBsonObjectSize / 2) -} -``` - -Construct the following list of write models (referred to as `models`): - -```json -[ - InsertOne: { - "namespace": "db.coll", - "document": document - }, - InsertOne: { - "namespace": "db.coll", - "document": document - } -] -``` - -Execute `bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` -with an `insertedCount` value of 2. Then make the following assertions based on whether or not the driver uses document -sequences (`OP_MSG` payload type 1) with `MongoClient.bulkWrite`: - -#### With document sequences - -Assert that a single CommandStartedEvent (referred to as `event`) was observed for the `bulkWrite` command. Assert that -the length of `event.command.ops` is 2. - -#### Without document sequences - -Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEvent`) were observed for the `bulkWrite` -command. For both `firstEvent` and `secondEvent`, assert that the length of `command.ops` is 1. If the driver exposes -`operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to `secondEvent.operationId`. - -### 5. `MongoClient.bulkWrite` with document sequences handles a `bulkWrite` message larger than `maxMessageSizeBytes` - -Test that `MongoClient.bulkWrite` properly handles a `writeModels` input which constructs an `ops` array larger than -`maxMessageSizeBytes`. Drivers that do not use document sequences (`OP_MSG` payload type 1) for bulk writes should -not implement this test. +`maxMessageSizeBytes`. This test must only be run on 8.0+ servers. From c34e4327cd7a5810b63fb0409c51fdd09cc8688b Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 15 Mar 2024 10:33:24 -0600 Subject: [PATCH 30/90] clarify --- source/crud/bulk-write.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index b5a3a750b6..b1f515c1e7 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -772,8 +772,8 @@ defaults, so `verboseResults` defaults to `false` to improve performance in the [DRIVERS-450](https://jira.mongodb.org/browse/DRIVERS-450) introduced a requirement that drivers only send a value for `bypassDocumentValidation` in the `insert` command if it was specified as true. The original motivation for this change is not documented. Conversely, this specification -requires that drivers send `bypassDocumentValidation` if it is set by the user, regardless of its -value. +requires that drivers send `bypassDocumentValidation` in the `bulkWrite` command if it is set by +the user in `BulkWriteOptions`, regardless of its value. Explicitly defining `bypassDocumentValidation: false` aligns with the server's default to perform schema validation and thus has no effect. However, checking the value of an option that the user From 0491ceceefe5ae56ec62e056cfeac70fc5fface7 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 15 Mar 2024 11:54:29 -0600 Subject: [PATCH 31/90] insert -> write --- source/crud/bulk-write.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index b1f515c1e7..6371de0bed 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -770,10 +770,10 @@ defaults, so `verboseResults` defaults to `false` to improve performance in the ### Why should drivers send `bypassDocumentValidation: false` for `bulkWrite`? [DRIVERS-450](https://jira.mongodb.org/browse/DRIVERS-450) introduced a requirement that drivers -only send a value for `bypassDocumentValidation` in the `insert` command if it was specified as -true. The original motivation for this change is not documented. Conversely, this specification -requires that drivers send `bypassDocumentValidation` in the `bulkWrite` command if it is set by -the user in `BulkWriteOptions`, regardless of its value. +only send a value for `bypassDocumentValidation` on write commands if it was specified as true. The +original motivation for this change is not documented. Conversely, this specification requires that +drivers send `bypassDocumentValidation` in the `bulkWrite` command if it is set by the user in +`BulkWriteOptions`, regardless of its value. Explicitly defining `bypassDocumentValidation: false` aligns with the server's default to perform schema validation and thus has no effect. However, checking the value of an option that the user From c4ddf5b0fe70cbc2abf0f7fc340e7f98db7059f4 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 19 Mar 2024 10:59:27 -0600 Subject: [PATCH 32/90] clarify naming --- source/unified-test-format/unified-test-format.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 64fe175969..e1b772aab0 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -1356,7 +1356,10 @@ Entity operations correspond to an API method on a driver object. If [operation. method on that class. Test files SHALL use camelCase when referring to API methods and parameters, even if the defining specifications use -other forms (e.g. snake_case in GridFS). +other forms (e.g. snake_case in GridFS). Test files SHOULD use the exact API method names defined in specifications for +entity test operations. Test files MAY use a different descriptive name if a naming conflict occurs. For example, the +name "clientBulkWrite" is used for the client-level bulk write operation to differentiate it from the collection-level +bulk write operation. This spec does not provide exhaustive documentation for all possible API methods that may appear in a test; however, the following sections discuss all supported entities and their operations in some level of detail. Special handling for From c308b1b457ff8f1cf4c5896facb7047b896d4c11 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 2 Apr 2024 13:58:08 -0600 Subject: [PATCH 33/90] add network errors tests --- .../client-bulkWrite-clientErrors.json | 350 ++++++++++++++++++ .../unified/client-bulkWrite-clientErrors.yml | 172 +++++++++ 2 files changed, 522 insertions(+) create mode 100644 source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.json create mode 100644 source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.yml diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.json new file mode 100644 index 0000000000..64b1988ffd --- /dev/null +++ b/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.json @@ -0,0 +1,350 @@ +{ + "description": "client bulkWrite retryable writes with client errors", + "schemaVersion": "1.20", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ], + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "retryable-writes-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite with one network error succeeds after retry", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "closeConnection": true + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll0", + "document": { + "_id": 4, + "x": 44 + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 4 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ] + }, + { + "description": "client bulkWrite with two network errors fails after retry", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "closeConnection": true + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll0", + "document": { + "_id": 4, + "x": 44 + } + } + } + ], + "verboseResults": true + }, + "expectError": { + "isClientError": true, + "errorLabelsContain": [ + "RetryableWriteError" + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + } + ] +} diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.yml new file mode 100644 index 0000000000..d1b08ec37e --- /dev/null +++ b/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.yml @@ -0,0 +1,172 @@ +description: "client bulkWrite retryable writes with client errors" +schemaVersion: "1.20" +runOnRequirements: + - minServerVersion: "8.0" + topologies: + - replicaset + - sharded + - load-balanced + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + useMultipleMongoses: false + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name retryable-writes-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + +_yamlAnchors: + namespace: &namespace "retryable-writes-tests.coll0" + +tests: + - description: "client bulkWrite with one network error succeeds after retry" + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: [ bulkWrite ] + closeConnection: true + - object: *client0 + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *namespace + document: { _id: 4, x: 44 } + verboseResults: true + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 4 + updateResults: {} + deleteResults: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: false + ordered: true + ops: + - insert: 0 + document: { _id: 4, x: 44 } + nsInfo: + - ns: *namespace + # An implicit session is included with the transaction number: + lsid: { "$$exists": true } + txnNumber: { "$$exists": true } + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: false + ordered: true + ops: + - insert: 0 + document: { _id: 4, x: 44 } + nsInfo: + - ns: *namespace + # An implicit session is included with the transaction number: + lsid: { "$$exists": true } + txnNumber: { "$$exists": true } + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + - description: "client bulkWrite with two network errors fails after retry" + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 2 + data: + failCommands: [ bulkWrite ] + closeConnection: true + - object: *client0 + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *namespace + document: { _id: 4, x: 44 } + verboseResults: true + expectError: + isClientError: true + errorLabelsContain: ["RetryableWriteError"] # Error label added by driver. + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: false + ordered: true + ops: + - insert: 0 + document: { _id: 4, x: 44 } + nsInfo: + - ns: *namespace + # An implicit session is included with the transaction number: + lsid: { "$$exists": true } + txnNumber: { "$$exists": true } + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: false + ordered: true + ops: + - insert: 0 + document: { _id: 4, x: 44 } + nsInfo: + - ns: *namespace + # An implicit session is included with the transaction number: + lsid: { "$$exists": true } + txnNumber: { "$$exists": true } + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } From 48b7cbfb89b02f37e06f55bb8cc28d42f605ce2c Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 2 Apr 2024 14:01:46 -0600 Subject: [PATCH 34/90] add comment option --- source/crud/bulk-write.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 6371de0bed..e8b7a0290e 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -279,6 +279,14 @@ class BulkWriteOptions { */ writeConcern: Optional; + /** + * Enables users to specify an arbitrary comment to help trace the operation through + * the database profiler, currentOp and logs. + * + * This option is only sent if the caller explicitly provides a value. + */ + comment: Optional; + /** * Whether detailed results for each successful operation should be included in the returned * BulkWriteResult. From d5d98f0cc1ad0585ccd13aa0c4d554b59af7e782 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 2 Apr 2024 14:07:32 -0600 Subject: [PATCH 35/90] use exists for upsertedId --- .../client-bulkWrite-mixed-namespaces.json | 8 +++-- .../client-bulkWrite-mixed-namespaces.yml | 4 +-- .../unified/client-bulkWrite-options.json | 4 ++- .../unified/client-bulkWrite-options.yml | 2 +- .../unified/client-bulkWrite-results.json | 8 +++-- .../unified/client-bulkWrite-results.yml | 4 +-- .../client-bulkWrite-update-options.json | 32 ++++++++++++++----- .../client-bulkWrite-update-options.yml | 16 +++++----- .../client-bulkWrite-serverErrors.json | 16 +++++++--- .../unified/client-bulkWrite-serverErrors.yml | 8 ++--- .../tests/unified/client-bulkWrite.json | 8 +++-- .../tests/unified/client-bulkWrite.yml | 4 +-- 12 files changed, 76 insertions(+), 38 deletions(-) diff --git a/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.json b/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.json index 33df3257c7..f90755dc85 100644 --- a/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.json +++ b/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.json @@ -177,12 +177,16 @@ "2": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "5": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } } }, "deleteResults": { diff --git a/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.yml b/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.yml index a34870e053..4e4cb01e16 100644 --- a/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.yml +++ b/source/crud/tests/unified/client-bulkWrite-mixed-namespaces.yml @@ -91,11 +91,11 @@ tests: 2: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } 5: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } deleteResults: 3: deletedCount: 1 diff --git a/source/crud/tests/unified/client-bulkWrite-options.json b/source/crud/tests/unified/client-bulkWrite-options.json index fd1a393009..56e0cb5cf5 100644 --- a/source/crud/tests/unified/client-bulkWrite-options.json +++ b/source/crud/tests/unified/client-bulkWrite-options.json @@ -300,7 +300,9 @@ "0": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } } }, "deleteResults": { diff --git a/source/crud/tests/unified/client-bulkWrite-options.yml b/source/crud/tests/unified/client-bulkWrite-options.yml index 1ef6e31921..9c38835420 100644 --- a/source/crud/tests/unified/client-bulkWrite-options.yml +++ b/source/crud/tests/unified/client-bulkWrite-options.yml @@ -150,7 +150,7 @@ tests: 0: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } deleteResults: 1: deletedCount: 1 diff --git a/source/crud/tests/unified/client-bulkWrite-results.json b/source/crud/tests/unified/client-bulkWrite-results.json index 726d15ffda..97a9e50b21 100644 --- a/source/crud/tests/unified/client-bulkWrite-results.json +++ b/source/crud/tests/unified/client-bulkWrite-results.json @@ -177,12 +177,16 @@ "1": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "2": { "matchedCount": 2, "modifiedCount": 2, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "3": { "matchedCount": 1, diff --git a/source/crud/tests/unified/client-bulkWrite-results.yml b/source/crud/tests/unified/client-bulkWrite-results.yml index b4731f1930..eb001bbb42 100644 --- a/source/crud/tests/unified/client-bulkWrite-results.yml +++ b/source/crud/tests/unified/client-bulkWrite-results.yml @@ -75,11 +75,11 @@ tests: 1: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } 2: matchedCount: 2 modifiedCount: 2 - upsertedId: null + upsertedId: { $$exists: false } 3: matchedCount: 1 modifiedCount: 0 diff --git a/source/crud/tests/unified/client-bulkWrite-update-options.json b/source/crud/tests/unified/client-bulkWrite-update-options.json index 5d53864023..93a2774e5f 100644 --- a/source/crud/tests/unified/client-bulkWrite-update-options.json +++ b/source/crud/tests/unified/client-bulkWrite-update-options.json @@ -151,12 +151,16 @@ "0": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "1": { "matchedCount": 2, "modifiedCount": 2, - "upsertedId": null + "upsertedId": { + "$$exists": false + } } }, "deleteResults": {} @@ -369,17 +373,23 @@ "0": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "1": { "matchedCount": 2, "modifiedCount": 2, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "2": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } } }, "deleteResults": {} @@ -603,17 +613,23 @@ "0": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "1": { "matchedCount": 2, "modifiedCount": 2, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "2": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } } }, "deleteResults": {} diff --git a/source/crud/tests/unified/client-bulkWrite-update-options.yml b/source/crud/tests/unified/client-bulkWrite-update-options.yml index a04cedea66..fe188a490c 100644 --- a/source/crud/tests/unified/client-bulkWrite-update-options.yml +++ b/source/crud/tests/unified/client-bulkWrite-update-options.yml @@ -64,11 +64,11 @@ tests: 0: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } 1: matchedCount: 2 modifiedCount: 2 - upsertedId: null + upsertedId: { $$exists: false } deleteResults: {} expectEvents: - client: *client0 @@ -140,15 +140,15 @@ tests: 0: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } 1: matchedCount: 2 modifiedCount: 2 - upsertedId: null + upsertedId: { $$exists: false } 2: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } deleteResults: {} expectEvents: - client: *client0 @@ -221,15 +221,15 @@ tests: 0: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } 1: matchedCount: 2 modifiedCount: 2 - upsertedId: null + upsertedId: { $$exists: false } 2: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } deleteResults: {} expectEvents: - client: *client0 diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json index 646adc8b19..e38bc0e4c1 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json @@ -149,12 +149,16 @@ "1": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "2": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } } }, "deleteResults": { @@ -499,12 +503,16 @@ "1": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "2": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } } }, "deleteResults": { diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml index 25a95c24b6..cb6950367f 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml @@ -80,11 +80,11 @@ tests: 1: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } 2: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } deleteResults: 3: deletedCount: 1 @@ -245,11 +245,11 @@ tests: 1: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } 2: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } deleteResults: 3: deletedCount: 1 diff --git a/source/transactions/tests/unified/client-bulkWrite.json b/source/transactions/tests/unified/client-bulkWrite.json index 5b9f5aa781..7454c6f360 100644 --- a/source/transactions/tests/unified/client-bulkWrite.json +++ b/source/transactions/tests/unified/client-bulkWrite.json @@ -193,12 +193,16 @@ "1": { "matchedCount": 1, "modifiedCount": 1, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "2": { "matchedCount": 2, "modifiedCount": 2, - "upsertedId": null + "upsertedId": { + "$$exists": false + } }, "3": { "matchedCount": 1, diff --git a/source/transactions/tests/unified/client-bulkWrite.yml b/source/transactions/tests/unified/client-bulkWrite.yml index fa8cb535d9..193a2bd905 100644 --- a/source/transactions/tests/unified/client-bulkWrite.yml +++ b/source/transactions/tests/unified/client-bulkWrite.yml @@ -85,11 +85,11 @@ tests: 1: matchedCount: 1 modifiedCount: 1 - upsertedId: null + upsertedId: { $$exists: false } 2: matchedCount: 2 modifiedCount: 2 - upsertedId: null + upsertedId: { $$exists: false } 3: matchedCount: 1 modifiedCount: 0 From 43a4ffb92eb84f8a8df947ca8b8de64dbef0912e Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 2 Apr 2024 14:19:41 -0600 Subject: [PATCH 36/90] rename write concern error message --- .../crud/tests/unified/client-bulkWrite-errors.json | 8 +++----- .../crud/tests/unified/client-bulkWrite-errors.yml | 13 ++++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/source/crud/tests/unified/client-bulkWrite-errors.json b/source/crud/tests/unified/client-bulkWrite-errors.json index 1404fad427..a33dffd503 100644 --- a/source/crud/tests/unified/client-bulkWrite-errors.json +++ b/source/crud/tests/unified/client-bulkWrite-errors.json @@ -56,10 +56,8 @@ ], "_yamlAnchors": { "namespace": "crud-tests.coll0", - "writeConcernError": { - "code": 91, - "errmsg": "Replication is being shut down" - }, + "writeConcernErrorCode": 91, + "writeConcernErrorMessage": "Replication is being shut down", "undefinedVarCode": 17276 }, "tests": [ @@ -429,7 +427,7 @@ "writeConcernErrors": [ { "code": 91, - "errmsg": "Replication is being shut down" + "message": "Replication is being shut down" } ] } diff --git a/source/crud/tests/unified/client-bulkWrite-errors.yml b/source/crud/tests/unified/client-bulkWrite-errors.yml index e05bef220a..7c587f8242 100644 --- a/source/crud/tests/unified/client-bulkWrite-errors.yml +++ b/source/crud/tests/unified/client-bulkWrite-errors.yml @@ -29,9 +29,8 @@ initialData: _yamlAnchors: namespace: &namespace "crud-tests.coll0" - writeConcernError: &writeConcernError - code: 91 - errmsg: "Replication is being shut down" + writeConcernErrorCode: &writeConcernErrorCode 91 + writeConcernErrorMessage: &writeConcernErrorMessage "Replication is being shut down" undefinedVarCode: &undefinedVarCode 17276 # Use of an undefined variable tests: @@ -204,7 +203,9 @@ tests: data: failCommands: - bulkWrite - writeConcernError: *writeConcernError + writeConcernError: + code: *writeConcernErrorCode + errmsg: *writeConcernErrorMessage - object: *client0 name: clientBulkWrite arguments: @@ -225,4 +226,6 @@ tests: insertedId: 10 updateResults: {} deleteResults: {} - writeConcernErrors: [ *writeConcernError ] + writeConcernErrors: + - code: *writeConcernErrorCode + message: *writeConcernErrorMessage From d1e1a330c39f81eefc71819de8e6046ff326df6b Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 2 Apr 2024 14:41:11 -0600 Subject: [PATCH 37/90] update upserted --- source/crud/bulk-write.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index e8b7a0290e..777ff61628 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -660,7 +660,7 @@ The documents in the results cursor have the following format: "errInfo": Optional, "n": , "nModified": Optional, - "upsertedId": Optional + "upserted": Optional } ``` @@ -674,7 +674,7 @@ populated with the following values based on the type of write: | -------------- | ------ | ------ | ------ | | `n` | The number of documents that were inserted. | The number of documents that matched the filter. | The number of documents that were deleted. | | `nModified` | Not present. | The number of documents that were modified. | Not present. | -| `upsertedId` | Not present. | The `_id` value for the upserted document. Only present if an upsert took place. | Not present. | +| `upserted` | Not present. | A document containing the `_id` value for the upserted document. Only present if an upsert took place. | Not present. | Note that the responses do not contain information about the type of operation that was performed. Drivers may need to maintain the user's list of write models to infer which type of result should From ec6863b62c7c51029f8a6a95c00c5c03294d1a54 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 2 Apr 2024 14:43:22 -0600 Subject: [PATCH 38/90] language --- source/crud/bulk-write.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 777ff61628..8f051f4ca6 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -779,8 +779,8 @@ defaults, so `verboseResults` defaults to `false` to improve performance in the [DRIVERS-450](https://jira.mongodb.org/browse/DRIVERS-450) introduced a requirement that drivers only send a value for `bypassDocumentValidation` on write commands if it was specified as true. The -original motivation for this change is not documented. Conversely, this specification requires that -drivers send `bypassDocumentValidation` in the `bulkWrite` command if it is set by the user in +original motivation for this change is not documented. This specification requires that drivers +send `bypassDocumentValidation` in the `bulkWrite` command if it is set by the user in `BulkWriteOptions`, regardless of its value. Explicitly defining `bypassDocumentValidation: false` aligns with the server's default to perform From 7c8eb73953e4a649029a83902ad23e957f7b5f90 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 3 Apr 2024 13:44:11 -0600 Subject: [PATCH 39/90] test lsid and txnNumber --- .../client-bulkWrite-serverErrors.json | 34 ++++++++++++++++--- .../unified/client-bulkWrite-serverErrors.yml | 10 +++++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json index e38bc0e4c1..fd517adb5f 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json @@ -223,7 +223,13 @@ { "ns": "retryable-writes-tests.coll0" } - ] + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } } } }, @@ -277,7 +283,13 @@ { "ns": "retryable-writes-tests.coll0" } - ] + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } } } } @@ -577,7 +589,13 @@ { "ns": "retryable-writes-tests.coll0" } - ] + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } } } }, @@ -631,7 +649,13 @@ { "ns": "retryable-writes-tests.coll0" } - ] + ], + "lsid": { + "$$exists": true + }, + "txnNumber": { + "$$exists": true + } } } } @@ -699,7 +723,7 @@ "writeConcernErrors": [ { "code": 91, - "errmsg": "Replication is being shut down" + "message": "Replication is being shut down" } ] } diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml index cb6950367f..e5022870c4 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml @@ -115,6 +115,8 @@ tests: multi: false nsInfo: - ns: *namespace + lsid: { $$exists: true } + txnNumber: { $$exists: true } - commandStartedEvent: commandName: bulkWrite databaseName: admin @@ -139,6 +141,8 @@ tests: multi: false nsInfo: - ns: *namespace + lsid: { $$exists: true } + txnNumber: { $$exists: true } outcome: - collectionName: *collection0Name databaseName: *database0Name @@ -280,6 +284,8 @@ tests: multi: false nsInfo: - ns: *namespace + lsid: { $$exists: true } + txnNumber: { $$exists: true } - commandStartedEvent: commandName: bulkWrite databaseName: admin @@ -304,6 +310,8 @@ tests: multi: false nsInfo: - ns: *namespace + lsid: { $$exists: true } + txnNumber: { $$exists: true } - description: "client bulkWrite with multi: true operations fails after retryable writeConcernError" operations: - object: testRunner @@ -335,7 +343,7 @@ tests: expectError: writeConcernErrors: - code: 91 - errmsg: "Replication is being shut down" + message: "Replication is being shut down" expectEvents: - client: *client0 events: From bd384b506bc2c5ecd38bf8a137b94454451c63e9 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 4 Apr 2024 09:43:05 -0600 Subject: [PATCH 40/90] improve size batching length check --- source/crud/tests/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 9cac86c395..6c593fd8fa 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -265,6 +265,7 @@ Construct as list of write models (referred to as `models`) with `model` repeate `bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` with an `insertedCount` value of `numModels`. -Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEvent`) were observed. Assert that the sum -of the lengths of `firstEvent.command.ops` and `secondEvent.command.ops` is equal to `numModels`. If the driver exposes -`operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to `secondEvent.operationId`. +Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEvent`) were observed. Assert that the +length of `firstEvent.command.ops` is `numModels - 1`. Assert that the length of `secondEvent.command.ops` is 1. If +the driver exposes `operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to +`secondEvent.operationId`. From 1633501508e352bcdb8d6ff0e93df5b6fa1f0c13 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 4 Apr 2024 10:21:31 -0600 Subject: [PATCH 41/90] add bypassdocumentvalidation false test --- .../unified/client-bulkWrite-options.json | 91 +++++++++++++++++++ .../unified/client-bulkWrite-options.yml | 45 +++++++++ 2 files changed, 136 insertions(+) diff --git a/source/crud/tests/unified/client-bulkWrite-options.json b/source/crud/tests/unified/client-bulkWrite-options.json index 56e0cb5cf5..e0e1aa225c 100644 --- a/source/crud/tests/unified/client-bulkWrite-options.json +++ b/source/crud/tests/unified/client-bulkWrite-options.json @@ -383,6 +383,97 @@ ] } ] + }, + { + "description": "client bulkWrite bypassDocumentValidation: false is sent", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 3, + "x": 33 + } + } + } + ], + "bypassDocumentValidation": false, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 3 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "bypassDocumentValidation": false, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] } ] } diff --git a/source/crud/tests/unified/client-bulkWrite-options.yml b/source/crud/tests/unified/client-bulkWrite-options.yml index 9c38835420..a82108c08e 100644 --- a/source/crud/tests/unified/client-bulkWrite-options.yml +++ b/source/crud/tests/unified/client-bulkWrite-options.yml @@ -184,3 +184,48 @@ tests: collectionName: *collection0Name documents: - { _id: 1, x: 12 } + - description: "client bulkWrite bypassDocumentValidation: false is sent" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *namespace + document: { _id: 3, x: 33 } + bypassDocumentValidation: false + verboseResults: true + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 3 + updateResults: {} + deleteResults: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: false + ordered: true + bypassDocumentValidation: false + ops: + - insert: 0 + document: { _id: 3, x: 33 } + nsInfo: + - ns: *namespace + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } From 7f3127513088697eeea378c87e60a02969e3bf1c Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 4 Apr 2024 10:33:45 -0600 Subject: [PATCH 42/90] writeconcernerror batch prose test --- source/crud/tests/README.md | 46 +++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 6c593fd8fa..9752d81488 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -212,7 +212,7 @@ This test must only be run on 8.0+ servers. Construct a `MongoClient` (referred to as `client`) with [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe CommandStartedEvents. Perform a `hello` command using `client` and record the `maxWriteBatchSize` value contained in the -response. Then, construct the following write model (referred to as `writeModel`): +response. Then, construct the following write model (referred to as `model`): ```json InsertOne: { @@ -221,7 +221,7 @@ InsertOne: { } ``` -Construct a list of write models (referred to as `models`) with `writeModel` repeated `maxWriteBatchSize + 1` times. +Construct a list of write models (referred to as `models`) with `model` repeated `maxWriteBatchSize + 1` times. Execute `bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` with an `insertedCount` value of `maxWriteBatchSize + 1`. @@ -269,3 +269,45 @@ Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEve length of `firstEvent.command.ops` is `numModels - 1`. Assert that the length of `secondEvent.command.ops` is 1. If the driver exposes `operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to `secondEvent.operationId`. + +### `MongoClient.bulkWrite` collects `writeConcernError`s across batches + +Test that `MongoClient.bulkWrite` properly collects and reports `writeConcernError`s returned in separate batches. + +This test must only be run on 8.0+ servers. + +Construct a `MongoClient` (referred to as `client`) with `retryWrites: false` configured. Perform a `hello` command +using `client` and record the `maxWriteBatchSize` value contained in the response. Then, configure the following +fail point with `client`: + +```json +{ + "configureFailPoint": "failCommand", + "mode": { "times": 2 }, + "data": { + "failCommands": ["bulkWrite"], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } +} +``` + +Construct the following write model (referred to as `model`): + +```json +InsertOne: { + "namespace": "db.coll", + "document": { "a": "b" } +} +``` + +Construct a list of write models (referred to as `models`) with `model` repeated `maxWriteBatchSize + 1` times. +Execute `bulkWrite` on `client` with `models`. Assert that the bulk write fails and returns a `BulkWriteError` +(referred to as `error`). + +Assert that `error.writeConcernErrors` has a length of 2. + +Assert that `error.partialResult` is populated. Assert that `error.partialResult.insertedCount` is equal to +`maxWriteBatchSize + 1`. From 3ab6d00cfa44e9fbea812ae0c2a50f0bb0c8fbf1 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 4 Apr 2024 11:29:11 -0600 Subject: [PATCH 43/90] writeerror batch prose test --- source/crud/tests/README.md | 52 ++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 9752d81488..7b94709455 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -270,7 +270,7 @@ length of `firstEvent.command.ops` is `numModels - 1`. Assert that the length of the driver exposes `operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to `secondEvent.operationId`. -### `MongoClient.bulkWrite` collects `writeConcernError`s across batches +### 5. `MongoClient.bulkWrite` collects `WriteConcernError`s across batches Test that `MongoClient.bulkWrite` properly collects and reports `writeConcernError`s returned in separate batches. @@ -311,3 +311,53 @@ Assert that `error.writeConcernErrors` has a length of 2. Assert that `error.partialResult` is populated. Assert that `error.partialResult.insertedCount` is equal to `maxWriteBatchSize + 1`. + +### 6. `MongoClient.bulkWrite` handles `WriteError`s across batches + +Test that `MongoClient.bulkWrite` handles individual write errors across batches for ordered and unordered bulk +writes. + +This test must only be run on 8.0+ servers. + +Construct a `MongoClient` (referred to as `client`). Perform a `hello` command using `client` and record the +`maxWriteBatchSize` value contained in the response. + +Construct a `MongoCollection` (referred to as `collection`) with the namespace "db.coll" (referred to as `namespace`). +Drop `collection`. Then, construct the following document (referred to as `document`): + +```json +{ + "_id": 1 +} +``` + +Insert `document` into `collection`. + +Create the following write model (referred to as `model`): + +```json +InsertOne { + "namespace": namespace, + "document": document +} +``` + +Construct a list of write models (referred to as `models`) with `model` repeated `maxWriteBatchSize + 1` times. + +#### Unordered + +Test that an unordered bulk write collects `WriteError`s across batches. + +Execute `bulkWrite` on `client` with `models` and `ordered` set to false. Assert that the bulk write fails and returns +a `BulkWriteError` (referred to as `unorderedError`). + +Assert that `unorderedError.writeErrors` has a length of `maxWriteBatchSize + 1`. + +#### Ordered + +Test that an ordered bulk write does not execute further batches when a `WriteError` occurs. + +Execute `bulkWrite` on `client` with `models` and `ordered` set to true. Assert that the bulk write fails and returns +a `BulkWriteError` (referred to as `orderedError`). + +Assert that `orderedError.writeErrors` has a length of 1. From 06cf1e390b2a763d7c2e2d6375d7f723f33776cf Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 5 Apr 2024 12:23:15 -0600 Subject: [PATCH 44/90] command monitoring, cursor tests --- source/crud/tests/README.md | 109 ++++++++++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 5 deletions(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 7b94709455..e4df86e677 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -276,9 +276,10 @@ Test that `MongoClient.bulkWrite` properly collects and reports `writeConcernErr This test must only be run on 8.0+ servers. -Construct a `MongoClient` (referred to as `client`) with `retryWrites: false` configured. Perform a `hello` command -using `client` and record the `maxWriteBatchSize` value contained in the response. Then, configure the following -fail point with `client`: +Construct a `MongoClient` (referred to as `client`) with `retryWrites: false` configured and +[command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe +CommandStartedEvents. Perform a `hello` command using `client` and record the `maxWriteBatchSize` value contained +in the response. Then, configure the following fail point with `client`: ```json { @@ -312,6 +313,8 @@ Assert that `error.writeConcernErrors` has a length of 2. Assert that `error.partialResult` is populated. Assert that `error.partialResult.insertedCount` is equal to `maxWriteBatchSize + 1`. +Assert that two CommandStartedEvents were observed for the `bulkWrite` command. + ### 6. `MongoClient.bulkWrite` handles `WriteError`s across batches Test that `MongoClient.bulkWrite` handles individual write errors across batches for ordered and unordered bulk @@ -319,8 +322,10 @@ writes. This test must only be run on 8.0+ servers. -Construct a `MongoClient` (referred to as `client`). Perform a `hello` command using `client` and record the -`maxWriteBatchSize` value contained in the response. +Construct a `MongoClient` (referred to as `client`) with +[command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe +CommandStartedEvents. Perform a `hello` command using `client` and record the `maxWriteBatchSize` value contained in the +response. Construct a `MongoCollection` (referred to as `collection`) with the namespace "db.coll" (referred to as `namespace`). Drop `collection`. Then, construct the following document (referred to as `document`): @@ -353,6 +358,8 @@ a `BulkWriteError` (referred to as `unorderedError`). Assert that `unorderedError.writeErrors` has a length of `maxWriteBatchSize + 1`. +Assert that two CommandStartedEvents were observed for the `bulkWrite` command. + #### Ordered Test that an ordered bulk write does not execute further batches when a `WriteError` occurs. @@ -361,3 +368,95 @@ Execute `bulkWrite` on `client` with `models` and `ordered` set to true. Assert a `BulkWriteError` (referred to as `orderedError`). Assert that `orderedError.writeErrors` has a length of 1. + +Assert that one CommandStartedEvent was observed for the `bulkWrite` command. + +### 7. `MongoClient.bulkWrite` handles a cursor requiring a `getMore` + +Test that `MongoClient.bulkWrite` properly iterates the results cursor when `getMore` is required. This test creates +a list of operation results that requires cursor iteration by performing inserts that each yield a `DuplicateKeyError` +containing a very large `_id` value. + +This test must only be run on 8.0+ servers. + +Construct a `MongoClient` (referred to as `client`) with +[command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe +CommandStartedEvents. Perform a `hello` command using `client` and record the `maxBsonObjectSize` value from the +response. Then, construct the following document (referred to as `document`): + +```json +{ + "_id": "a".repeat(maxBsonObjectSize - 500) +} +``` + +Construct a `MongoCollection` (referred to as `collection`) with the namespace "db.coll" (referred to as `namespace`). +Drop `collection`. Insert `document` into `collection`. + +Create the following write model (referred to as `model`): + +```json +InsertOne { + "namespace": namespace, + "document": document +} +``` + +Construct a list of write models (referred to as `models`) with `model` repeated 2 times. Execute `bulkWrite` on +`client` with `models` and `ordered` set to false. Assert that the bulk write fails and returns a `BulkWriteError` +(referred to as `error`). + +Assert that the length of `error.writeErrors` is 2. + +Assert that a CommandStartedEvent was observed for the `getMore` command. + + +### 8. `MongoClient` handles a `getMore` error + +Test that `MongoClient.bulkWrite` properly handles a failure that occurs when attempting a `getMore`. + +This test must only be run on 8.0+ servers. + +Construct a `MongoClient` (referred to as `client`) with +[command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe +CommandStartedEvents. Perform a `hello` command using `client` and record the `maxBsonObjectSize` value from the +response. Then, configure the following fail point with `client`: + +```json +{ + "configureFailPoint": "failCommand", + "mode": { "times": 1 }, + "data": { + "failCommands": ["getMore"], + "errorCode": 8 + } +} +``` + +Construct the following document (referred to as `document`): + +```json +{ + "_id": "a".repeat(maxBsonObjectSize - 500) +} +``` + +Construct a `MongoCollection` (referred to as `collection`) with the namespace "db.coll" (referred to as `namespace`). +Drop `collection`. Insert `document` into `collection`. + +Create the following write model (referred to as `model`): + +```json +InsertOne { + "namespace": namespace, + "document": document +} +``` + +Construct a list of write models (referred to as `models`) with `model` repeated 2 times. Execute `bulkWrite` on +`client` with `models` and `ordered` set to false. Assert that the bulk write fails and returns a `BulkWriteError` +(referred to as `bulkWriteError`). + +Assert that the length of `bulkWriteError.writeErrors` is 2. + +Assert that a CommandStartedEvent was observed for the `getMore` command. From 1677d5749f858c83ac19e1b53792e982bd619fc8 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 5 Apr 2024 12:50:43 -0600 Subject: [PATCH 45/90] killcursors --- source/crud/bulk-write.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 8f051f4ca6..8946384f64 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -705,7 +705,9 @@ error. Encountering a top-level error MUST halt execution of a bulk write for both ordered and unordered bulk writes. This means that drivers MUST NOT attempt to retrieve more responses from the cursor or -execute any further `bulkWrite` batches and MUST immediately throw an exception. +execute any further `bulkWrite` batches and MUST immediately throw an exception. If the results +cursor has not been exhausted on the server when a top-level error occurs, drivers MUST send the +`killCursors` command to attempt to close it. ### Write Concern Errors From c74634cd6ae50e0de86a5c33740cf7b34d934fbc Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 5 Apr 2024 12:54:48 -0600 Subject: [PATCH 46/90] add pipeline tests --- .../client-bulkWrite-update-pipeline.json | 257 ++++++++++++++++++ .../client-bulkWrite-update-pipeline.yml | 132 +++++++++ 2 files changed, 389 insertions(+) create mode 100644 source/crud/tests/unified/client-bulkWrite-update-pipeline.json create mode 100644 source/crud/tests/unified/client-bulkWrite-update-pipeline.yml diff --git a/source/crud/tests/unified/client-bulkWrite-update-pipeline.json b/source/crud/tests/unified/client-bulkWrite-update-pipeline.json new file mode 100644 index 0000000000..57b6c9c1ba --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-update-pipeline.json @@ -0,0 +1,257 @@ +{ + "description": "client bulkWrite update pipeline", + "schemaVersion": "1.1", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 1 + }, + { + "_id": 2, + "x": 2 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite updateOne with pipeline", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "update": [ + { + "$addFields": { + "foo": 1 + } + } + ] + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 0, + "insertResults": {}, + "updateResults": { + "0": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedId": { + "$$exists": false + } + } + }, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "update": 0, + "filter": { + "_id": 1 + }, + "updateMods": [ + { + "$addFields": { + "foo": 1 + } + } + ], + "multi": false + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "x": 1, + "foo": 1 + }, + { + "_id": 2, + "x": 2 + } + ] + } + ] + }, + { + "description": "client bulkWrite updateMany with pipeline", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateMany": { + "namespace": "crud-tests.coll0", + "filter": {}, + "update": [ + { + "$addFields": { + "foo": 1 + } + } + ] + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 2, + "modifiedCount": 2, + "deletedCount": 0, + "insertResults": {}, + "updateResults": { + "0": { + "matchedCount": 2, + "modifiedCount": 2, + "upsertedId": { + "$$exists": false + } + } + }, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "ops": [ + { + "update": 0, + "filter": {}, + "updateMods": [ + { + "$addFields": { + "foo": 1 + } + } + ], + "multi": true + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [ + { + "_id": 1, + "x": 1, + "foo": 1 + }, + { + "_id": 2, + "x": 2, + "foo": 1 + } + ] + } + ] + } + ] +} diff --git a/source/crud/tests/unified/client-bulkWrite-update-pipeline.yml b/source/crud/tests/unified/client-bulkWrite-update-pipeline.yml new file mode 100644 index 0000000000..fe0e29a508 --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-update-pipeline.yml @@ -0,0 +1,132 @@ +description: "client bulkWrite update pipeline" +schemaVersion: "1.1" +runOnRequirements: + - minServerVersion: "8.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 1} + - {_id: 2, x: 2} + +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + +tests: + - description: "client bulkWrite updateOne with pipeline" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + models: + - updateOne: + namespace: *namespace + filter: { _id: 1 } + update: + - $addFields: + foo: 1 + verboseResults: true + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 1 + modifiedCount: 1 + deletedCount: 0 + insertResults: {} + updateResults: + 0: + matchedCount: 1 + modifiedCount: 1 + upsertedId: { "$$exists": false } + deleteResults: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: false + ordered: true + ops: + - update: 0 + filter: { _id: 1 } + updateMods: + - $addFields: + foo: 1 + multi: false + nsInfo: + - ns: *namespace + outcome: + - databaseName: *database0Name + collectionName: *collection0Name + documents: + - {_id: 1, x: 1, foo: 1} + - {_id: 2, x: 2 } + + - description: "client bulkWrite updateMany with pipeline" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + models: + - updateMany: + namespace: *namespace + filter: {} + update: + - $addFields: + foo: 1 + verboseResults: true + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 2 + modifiedCount: 2 + deletedCount: 0 + insertResults: {} + updateResults: + 0: + matchedCount: 2 + modifiedCount: 2 + upsertedId: { "$$exists": false } + deleteResults: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: false + ordered: true + ops: + - update: 0 + filter: { } + updateMods: + - $addFields: + foo: 1 + multi: true + nsInfo: + - ns: *namespace + outcome: + - databaseName: *database0Name + collectionName: *collection0Name + documents: + - {_id: 1, x: 1, foo: 1} + - {_id: 2, x: 2, foo: 1} From 7460b423b773d0d224dbe77c1a02a160c7fade8a Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 5 Apr 2024 13:41:23 -0600 Subject: [PATCH 47/90] update vs. replace validation --- source/crud/bulk-write.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 8946384f64..e6fafd9879 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -244,6 +244,19 @@ class NamespaceWriteModelPair { Drivers MUST throw an exception if the list provided for `models` is empty. +#### Update vs. replace document validation + +Update documents provided in `UpdateOne` and `UpdateMany` write models are required only to contain +atomic modifiers (i.e. keys that start with "$"). Drivers MUST throw an error if an update document +is empty or if the document's first key does not start with "$". Drivers MUST rely on the server to +return an error if any other entries in the update document are not atomic modifiers. Drivers are +not required to perform validation on update pipelines. + +Replacement documents provided in `ReplaceOne` write models are required not to contain atomic +modifiers. Drivers MUST throw an error if a replacement document is nonempty and its first key +starts with "$". Drivers MUST rely on the server to return an error if any other entries in the +replacement document are atomic modifiers. + ### Options ```typescript From a83fe6b28bd2a18f7d92a6e3cb9a0a3c619b30f7 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 5 Apr 2024 13:41:55 -0600 Subject: [PATCH 48/90] add validation tests --- .../client-bulkWrite-update-validation.json | 216 ++++++++++++++++++ .../client-bulkWrite-update-validation.yml | 79 +++++++ 2 files changed, 295 insertions(+) create mode 100644 source/crud/tests/unified/client-bulkWrite-update-validation.json create mode 100644 source/crud/tests/unified/client-bulkWrite-update-validation.yml diff --git a/source/crud/tests/unified/client-bulkWrite-update-validation.json b/source/crud/tests/unified/client-bulkWrite-update-validation.json new file mode 100644 index 0000000000..1ac3e8d048 --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-update-validation.json @@ -0,0 +1,216 @@ +{ + "description": "client-bulkWrite-update-validation", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite replaceOne prohibits atomic modifiers", + "operations": [ + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "replacement": { + "$set": { + "x": 22 + } + } + } + } + ] + }, + "expectError": { + "isClientError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "client bulkWrite updateOne requires atomic modifiers", + "operations": [ + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "models": [ + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": 1 + }, + "update": { + "x": 22 + } + } + } + ] + }, + "expectError": { + "isClientError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "client bulkWrite updateMany requires atomic modifiers", + "operations": [ + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "models": [ + { + "updateMany": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": { + "$gt": 1 + } + }, + "update": { + "x": 44 + } + } + } + ] + }, + "expectError": { + "isClientError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + } + ] +} diff --git a/source/crud/tests/unified/client-bulkWrite-update-validation.yml b/source/crud/tests/unified/client-bulkWrite-update-validation.yml new file mode 100644 index 0000000000..f597e0762c --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-update-validation.yml @@ -0,0 +1,79 @@ +description: "client-bulkWrite-update-validation" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +initialData: &initialData + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + +tests: + - description: "client bulkWrite replaceOne prohibits atomic modifiers" + operations: + - name: clientBulkWrite + object: *client0 + arguments: + models: + - replaceOne: + namespace: *namespace + filter: { _id: 1 } + replacement: { $set: { x: 22 } } + expectError: + isClientError: true + expectEvents: + - client: *client0 + events: [] + outcome: *initialData + + - description: "client bulkWrite updateOne requires atomic modifiers" + operations: + - name: clientBulkWrite + object: *client0 + arguments: + models: + - updateOne: + namespace: *namespace + filter: { _id: 1 } + update: { x: 22 } + expectError: + isClientError: true + expectEvents: + - client: *client0 + events: [] + outcome: *initialData + + - description: "client bulkWrite updateMany requires atomic modifiers" + operations: + - name: clientBulkWrite + object: *client0 + arguments: + models: + - updateMany: + namespace: *namespace + filter: { _id: { $gt: 1 } } + update: { x: 44 } + expectError: + isClientError: true + expectEvents: + - client: *client0 + events: [] + outcome: *initialData From 107708638079c5781b786cfdda447596ab520245 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 5 Apr 2024 13:46:11 -0600 Subject: [PATCH 49/90] partial result note --- source/unified-test-format/unified-test-format.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index e1b772aab0..0bef99c1ba 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -1434,7 +1434,8 @@ Test runners MUST NOT iterate the change stream when executing this operation an #### clientBulkWrite -These considerations only apply to the `MongoClient.bulkWrite` method. See [bulkWrite](#bulkwrite) for special considerations for `MongoCollection.bulkWrite`. +These considerations only apply to the `MongoClient.bulkWrite` method. See [bulkWrite](#bulkwrite) for special considerations +for `MongoCollection.bulkWrite`. The `models` parameter for `clientBulkWrite` is documented as a list of WriteModel interfaces. Each WriteModel implementation (e.g. InsertOneModel) provides important context to the method, but that type information is not easily @@ -1479,6 +1480,10 @@ write concern errors that occurred during the bulk write. Unified tests SHOULD u `expectedError` to assert on the contents of these fields. Test runners MUST NOT inspect the contents of these fields when making assertions based on any other fields defined in `expectedError`. +While operations typically raise an error *or* return a result, the `MongoClient.bulkWrite` operation may +report both via the `partialResult` property of a `BulkWriteException`. In this case, the intermediary write result may be +matched with [expectedError_expectResult](#expectedError_expectResult) + #### watch This operation SHOULD NOT be used in test files. See [client_createChangeStream](#client_createChangeStream). From 947004bde804a5c4916c4bbb9612309a40cbc3cd Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 5 Apr 2024 14:07:49 -0600 Subject: [PATCH 50/90] fix failed iteration test --- source/crud/tests/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index e4df86e677..6c3ef702e5 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -457,6 +457,8 @@ Construct a list of write models (referred to as `models`) with `model` repeated `client` with `models` and `ordered` set to false. Assert that the bulk write fails and returns a `BulkWriteError` (referred to as `bulkWriteError`). -Assert that the length of `bulkWriteError.writeErrors` is 2. +Assert that the length of `bulkWriteError.writeErrors` is 1. + +Assert that `bulkWriteError.error` is populated with an error with error code 8. Assert that a CommandStartedEvent was observed for the `getMore` command. From e6d8bc5cc41280d01573fa131ece94813475d652 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 9 Apr 2024 10:09:53 -0600 Subject: [PATCH 51/90] empty models test --- .../tests/unified/client-bulkWrite-errors.json | 16 ++++++++++++++++ .../tests/unified/client-bulkWrite-errors.yml | 9 +++++++++ 2 files changed, 25 insertions(+) diff --git a/source/crud/tests/unified/client-bulkWrite-errors.json b/source/crud/tests/unified/client-bulkWrite-errors.json index a33dffd503..0c38499732 100644 --- a/source/crud/tests/unified/client-bulkWrite-errors.json +++ b/source/crud/tests/unified/client-bulkWrite-errors.json @@ -433,6 +433,22 @@ } } ] + }, + { + "description": "an empty list of write models is a client-side error", + "operations": [ + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "models": [], + "verboseResults": true + }, + "expectError": { + "isClientError": true + } + } + ] } ] } diff --git a/source/crud/tests/unified/client-bulkWrite-errors.yml b/source/crud/tests/unified/client-bulkWrite-errors.yml index 7c587f8242..97ce175602 100644 --- a/source/crud/tests/unified/client-bulkWrite-errors.yml +++ b/source/crud/tests/unified/client-bulkWrite-errors.yml @@ -229,3 +229,12 @@ tests: writeConcernErrors: - code: *writeConcernErrorCode message: *writeConcernErrorMessage + - description: "an empty list of write models is a client-side error" + operations: + - name: clientBulkWrite + object: *client0 + arguments: + models: [] + verboseResults: true + expectError: + isClientError: true From e2effb8e35270dfc02b499463750b6578e9f6916 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 9 Apr 2024 10:22:10 -0600 Subject: [PATCH 52/90] basic write concern test --- .../unified/client-bulkWrite-options.json | 78 +++++++++++++++++++ .../unified/client-bulkWrite-options.yml | 41 +++++++++- 2 files changed, 118 insertions(+), 1 deletion(-) diff --git a/source/crud/tests/unified/client-bulkWrite-options.json b/source/crud/tests/unified/client-bulkWrite-options.json index e0e1aa225c..fcee08079f 100644 --- a/source/crud/tests/unified/client-bulkWrite-options.json +++ b/source/crud/tests/unified/client-bulkWrite-options.json @@ -54,6 +54,9 @@ "let": { "id1": 1, "id2": 2 + }, + "writeConcern": { + "w": "majority" } }, "tests": [ @@ -474,6 +477,81 @@ ] } ] + }, + { + "description": "client bulkWrite writeConcern", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 3, + "x": 33 + } + } + } + ], + "writeConcern": { + "w": "majority" + }, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 3 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "writeConcern": { + "w": "majority" + }, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ] } ] } diff --git a/source/crud/tests/unified/client-bulkWrite-options.yml b/source/crud/tests/unified/client-bulkWrite-options.yml index a82108c08e..283e322387 100644 --- a/source/crud/tests/unified/client-bulkWrite-options.yml +++ b/source/crud/tests/unified/client-bulkWrite-options.yml @@ -25,8 +25,9 @@ initialData: _yamlAnchors: namespace: &namespace "crud-tests.coll0" - comment: &comment { "bulk": "write" } + comment: &comment { bulk: "write" } let: &let { id1: 1, id2: 2 } + writeConcern: &writeConcern { w: "majority" } tests: - description: "client bulkWrite comment" @@ -229,3 +230,41 @@ tests: - { _id: 1, x: 11 } - { _id: 2, x: 22 } - { _id: 3, x: 33 } + - description: "client bulkWrite writeConcern" + operations: + - object: *client0 + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *namespace + document: { _id: 3, x: 33 } + writeConcern: *writeConcern + verboseResults: true + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 3 + updateResults: {} + deleteResults: {} + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: false + ordered: true + writeConcern: *writeConcern + ops: + - insert: 0 + document: { _id: 3, x: 33 } + nsInfo: + - ns: *namespace From fa5395bcdc2b162cab8f96b48af857e1d049d2f7 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 9 Apr 2024 10:46:07 -0600 Subject: [PATCH 53/90] more write concern tests --- .../unified/client-bulkWrite-options.json | 158 ++++++++++++++++++ .../unified/client-bulkWrite-options.yml | 86 +++++++++- 2 files changed, 241 insertions(+), 3 deletions(-) diff --git a/source/crud/tests/unified/client-bulkWrite-options.json b/source/crud/tests/unified/client-bulkWrite-options.json index fcee08079f..a1e6af3bf3 100644 --- a/source/crud/tests/unified/client-bulkWrite-options.json +++ b/source/crud/tests/unified/client-bulkWrite-options.json @@ -15,6 +15,17 @@ ] } }, + { + "client": { + "id": "writeConcernClient", + "uriOptions": { + "w": 1 + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, { "database": { "id": "database0", @@ -552,6 +563,153 @@ ] } ] + }, + { + "description": "client bulkWrite inherits writeConcern from client", + "operations": [ + { + "object": "writeConcernClient", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 3, + "x": 33 + } + } + } + ], + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 3 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "writeConcernClient", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "writeConcern": { + "w": 1 + }, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ] + }, + { + "description": "client bulkWrite writeConcern option overrides client writeConcern", + "operations": [ + { + "object": "writeConcernClient", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 3, + "x": 33 + } + } + } + ], + "writeConcern": { + "w": "majority" + }, + "verboseResults": true + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 3 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + ], + "expectEvents": [ + { + "client": "writeConcernClient", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": false, + "ordered": true, + "writeConcern": { + "w": "majority" + }, + "ops": [ + { + "insert": 0, + "document": { + "_id": 3, + "x": 33 + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + } + ] + } + ] } ] } diff --git a/source/crud/tests/unified/client-bulkWrite-options.yml b/source/crud/tests/unified/client-bulkWrite-options.yml index 283e322387..fdcf788799 100644 --- a/source/crud/tests/unified/client-bulkWrite-options.yml +++ b/source/crud/tests/unified/client-bulkWrite-options.yml @@ -7,6 +7,11 @@ createEntities: - client: id: &client0 client0 observeEvents: [ commandStartedEvent ] + - client: + id: &writeConcernClient writeConcernClient + uriOptions: + &clientWriteConcern { w: 1 } + observeEvents: [ commandStartedEvent ] - database: id: &database0 database0 client: *client0 @@ -27,7 +32,7 @@ _yamlAnchors: namespace: &namespace "crud-tests.coll0" comment: &comment { bulk: "write" } let: &let { id1: 1, id2: 2 } - writeConcern: &writeConcern { w: "majority" } + writeConcern: &majorityWriteConcern { w: "majority" } tests: - description: "client bulkWrite comment" @@ -239,7 +244,7 @@ tests: - insertOne: namespace: *namespace document: { _id: 3, x: 33 } - writeConcern: *writeConcern + writeConcern: *majorityWriteConcern verboseResults: true expectResult: insertedCount: 1 @@ -262,7 +267,82 @@ tests: bulkWrite: 1 errorsOnly: false ordered: true - writeConcern: *writeConcern + writeConcern: *majorityWriteConcern + ops: + - insert: 0 + document: { _id: 3, x: 33 } + nsInfo: + - ns: *namespace + - description: "client bulkWrite inherits writeConcern from client" + operations: + - object: *writeConcernClient + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *namespace + document: { _id: 3, x: 33 } + verboseResults: true + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 3 + updateResults: {} + deleteResults: {} + expectEvents: + - client: *writeConcernClient + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: false + ordered: true + writeConcern: { w: 1 } + ops: + - insert: 0 + document: { _id: 3, x: 33 } + nsInfo: + - ns: *namespace + - description: "client bulkWrite writeConcern option overrides client writeConcern" + operations: + - object: *writeConcernClient + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *namespace + document: { _id: 3, x: 33 } + writeConcern: *majorityWriteConcern + verboseResults: true + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + 0: + insertedId: 3 + updateResults: {} + deleteResults: {} + expectEvents: + - client: *writeConcernClient + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: false + ordered: true + writeConcern: *majorityWriteConcern ops: - insert: 0 document: { _id: 3, x: 33 } From c23106892946b77f0da0fd06e22710d642b156bb Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 9 Apr 2024 10:50:51 -0600 Subject: [PATCH 54/90] add retryWrites:false test --- .../client-bulkWrite-serverErrors.json | 94 +++++++++++++++++++ .../unified/client-bulkWrite-serverErrors.yml | 45 +++++++++ 2 files changed, 139 insertions(+) diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json index fd517adb5f..f9812241f3 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json @@ -21,6 +21,18 @@ "useMultipleMongoses": false } }, + { + "client": { + "id": "clientRetryWritesFalse", + "uriOptions": { + "retryWrites": false + }, + "observeEvents": [ + "commandStartedEvent" + ], + "useMultipleMongoses": false + } + }, { "database": { "id": "database0", @@ -773,6 +785,88 @@ ] } ] + }, + { + "description": "client bulkWrite with retryWrites: false does not retry", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "clientRetryWritesFalse", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 189, + "errorLabels": [ + "RetryableWriteError" + ] + } + } + } + }, + { + "object": "clientRetryWritesFalse", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll0", + "document": { + "_id": 4, + "x": 44 + } + } + } + ] + }, + "expectError": { + "errorCode": 189, + "errorLabelsContain": [ + "RetryableWriteError" + ] + } + } + ], + "expectEvents": [ + { + "client": "clientRetryWritesFalse", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + } + ], + "nsInfo": [ + { + "ns": "retryable-writes-tests.coll0" + } + ] + } + } + } + ] + } + ] } ] } diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml index e5022870c4..d77e491a99 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml @@ -12,6 +12,12 @@ createEntities: id: &client0 client0 observeEvents: [ commandStartedEvent ] useMultipleMongoses: false + - client: + id: &clientRetryWritesFalse clientRetryWritesFalse + uriOptions: + retryWrites: false + observeEvents: [ commandStartedEvent ] + useMultipleMongoses: false - database: id: &database0 database0 client: *client0 @@ -365,3 +371,42 @@ tests: multi: true nsInfo: - ns: *namespace + - description: "client bulkWrite with retryWrites: false does not retry" + operations: + - object: testRunner + name: failPoint + arguments: + client: *clientRetryWritesFalse + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: [ bulkWrite ] + errorCode: 189 # PrimarySteppedDown + errorLabels: [ RetryableWriteError ] + - object: *clientRetryWritesFalse + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *namespace + document: { _id: 4, x: 44 } + expectError: + errorCode: 189 + errorLabelsContain: [ RetryableWriteError ] + expectEvents: + - client: *clientRetryWritesFalse + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: true + ordered: true + ops: + - insert: 0 + document: { _id: 4, x: 44 } + nsInfo: + - ns: *namespace From bef523123d3f07d76f966fb4639837274044e6b6 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 9 Apr 2024 15:44:58 -0600 Subject: [PATCH 55/90] add unacknowledged test --- source/crud/tests/README.md | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 6c3ef702e5..5e3dcb2716 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -462,3 +462,54 @@ Assert that the length of `bulkWriteError.writeErrors` is 1. Assert that `bulkWriteError.error` is populated with an error with error code 8. Assert that a CommandStartedEvent was observed for the `getMore` command. + +### 9. `MongoClient.bulkWrite` returns error for unacknowledged too-large insert + +This test must only be run on 8.0+ servers. + +Construct a `MongoClient` (referred to as `client`). + +Perform a `hello` command using `client` and record the following values from the response: `maxBsonObjectSize`. + +Then, construct the following document (referred to as `document`): + +```json +{ + "a": "b".repeat(maxBsonObjectSize) +} +``` + +#### With insert + +Construct the following write model (referred to as `model`): + +```json +InsertOne: { + "namespace": "db.coll", + "document": document +} +``` + +Construct as list of write models (referred to as `models`) with the one `model`. + +Call `MongoClient.bulkWrite` with `models` and `BulkWriteOptions.writeConcern` set to an unacknowledged write concern. + +Expect a client-side error due the size. + +#### With replace + +Construct the following write model (referred to as `model`): + +```json +ReplaceOne: { + "namespace": "db.coll", + "filter": {}, + "replacement": document +} +``` + +Construct as list of write models (referred to as `models`) with the one `model`. + +Call `MongoClient.bulkWrite` with `models` and `BulkWriteOptions.writeConcern` set to an unacknowledged write concern. + +Expect a client-side error due the size. \ No newline at end of file From 9b82f6fde2cc36918732fc3ec058abddf1363010 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 9 Apr 2024 15:53:04 -0600 Subject: [PATCH 56/90] reduce _id size --- source/crud/tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 5e3dcb2716..94e185f9e8 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -386,7 +386,7 @@ response. Then, construct the following document (referred to as `document`): ```json { - "_id": "a".repeat(maxBsonObjectSize - 500) + "_id": "a".repeat(maxBsonObjectSize / 2) } ``` @@ -437,7 +437,7 @@ Construct the following document (referred to as `document`): ```json { - "_id": "a".repeat(maxBsonObjectSize - 500) + "_id": "a".repeat(maxBsonObjectSize / 2) } ``` From 6b4f623af7bf7bc599c8d062179412bdb5ed862e Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 11 Apr 2024 10:57:15 -0600 Subject: [PATCH 57/90] iteration test updates --- source/crud/tests/README.md | 122 ++++++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 40 deletions(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 94e185f9e8..61637ae58b 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -315,7 +315,7 @@ Assert that `error.partialResult` is populated. Assert that `error.partialResult Assert that two CommandStartedEvents were observed for the `bulkWrite` command. -### 6. `MongoClient.bulkWrite` handles `WriteError`s across batches +### 6. `MongoClient.bulkWrite` handles individual `WriteError`s across batches Test that `MongoClient.bulkWrite` handles individual write errors across batches for ordered and unordered bulk writes. @@ -373,45 +373,86 @@ Assert that one CommandStartedEvent was observed for the `bulkWrite` command. ### 7. `MongoClient.bulkWrite` handles a cursor requiring a `getMore` -Test that `MongoClient.bulkWrite` properly iterates the results cursor when `getMore` is required. This test creates -a list of operation results that requires cursor iteration by performing inserts that each yield a `DuplicateKeyError` -containing a very large `_id` value. +Test that `MongoClient.bulkWrite` properly iterates the results cursor when `getMore` is required. This test must only be run on 8.0+ servers. Construct a `MongoClient` (referred to as `client`) with [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe CommandStartedEvents. Perform a `hello` command using `client` and record the `maxBsonObjectSize` value from the -response. Then, construct the following document (referred to as `document`): +response. + +Construct a `MongoCollection` (referred to as `collection`) with the namespace "db.coll" (referred to as `namespace`). +Drop `collection`. Then create the following list of write models (referred to as `models`): ```json -{ - "_id": "a".repeat(maxBsonObjectSize / 2) -} +UpdateOne { + "namespace": namespace, + "filter": { "_id": "a".repeat(maxBsonObjectSize / 2) }, + "update": { "$set": { "x": 1 } }, + "upsert": true +}, +UpdateOne { + "namespace": namespace, + "filter": { "_id": "b".repeat(maxBsonObjectSize / 2) }, + "update": { "$set": { "x": 1 } }, + "upsert": true +}, ``` +Execute `bulkWrite` on `client` with `models` and `verboseResults` set to true. Assert that the bulk write succeeds and +returns a `BulkWriteResult` (referred to as `result`). + +Assert that `result.upsertedCount` is equal to 2. + +Assert that the length of `result.updateResults` is equal to 2. + +Assert that a CommandStartedEvent was observed for the `getMore` command. + +### 8. `MongoClient.bulkWrite` handles a cursor requiring `getMore` within a transaction + +Test that `MongoClient.bulkWrite` executed within a transaction properly iterates the results cursor when `getMore` is +required. + +This test must only be run on 8.0+ servers. This test must not be run against standalone servers. + +Construct a `MongoClient` (referred to as `client`) with +[command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe +CommandStartedEvents. Perform a `hello` command using `client` and record the `maxBsonObjectSize` value from the +response. + Construct a `MongoCollection` (referred to as `collection`) with the namespace "db.coll" (referred to as `namespace`). -Drop `collection`. Insert `document` into `collection`. +Drop `collection`. -Create the following write model (referred to as `model`): +Start a session on `client` (referred to as `session`). Start a transaction on `session`. + +Create the following list of write models (referred to as `models`): ```json -InsertOne { +UpdateOne { "namespace": namespace, - "document": document -} + "filter": { "_id": "a".repeat(maxBsonObjectSize / 2) }, + "update": { "$set": { "x": 1 } }, + "upsert": true +}, +UpdateOne { + "namespace": namespace, + "filter": { "_id": "b".repeat(maxBsonObjectSize / 2) }, + "update": { "$set": { "x": 1 } }, + "upsert": true +}, ``` -Construct a list of write models (referred to as `models`) with `model` repeated 2 times. Execute `bulkWrite` on -`client` with `models` and `ordered` set to false. Assert that the bulk write fails and returns a `BulkWriteError` -(referred to as `error`). +Execute `bulkWrite` on `client` with `models`, `session`, and `verboseResults` set to true. Assert that the bulk write +succeeds and returns a `BulkWriteResult` (referred to as `result`). -Assert that the length of `error.writeErrors` is 2. +Assert that `result.upsertedCount` is equal to 2. -Assert that a CommandStartedEvent was observed for the `getMore` command. +Assert that the length of `result.updateResults` is equal to 2. +Assert that a CommandStartedEvent was observed for the `getMore` command. -### 8. `MongoClient` handles a `getMore` error +### 9. `MongoClient.bulkWrite` handles a `getMore` error Test that `MongoClient.bulkWrite` properly handles a failure that occurs when attempting a `getMore`. @@ -433,37 +474,38 @@ response. Then, configure the following fail point with `client`: } ``` -Construct the following document (referred to as `document`): - -```json -{ - "_id": "a".repeat(maxBsonObjectSize / 2) -} -``` - Construct a `MongoCollection` (referred to as `collection`) with the namespace "db.coll" (referred to as `namespace`). -Drop `collection`. Insert `document` into `collection`. - -Create the following write model (referred to as `model`): +Drop `collection`. Then create the following list of write models (referred to as `models`): ```json -InsertOne { +UpdateOne { "namespace": namespace, - "document": document -} + "filter": { "_id": "a".repeat(maxBsonObjectSize / 2) }, + "update": { "$set": { "x": 1 } }, + "upsert": true +}, +UpdateOne { + "namespace": namespace, + "filter": { "_id": "b".repeat(maxBsonObjectSize / 2) }, + "update": { "$set": { "x": 1 } }, + "upsert": true +}, ``` -Construct a list of write models (referred to as `models`) with `model` repeated 2 times. Execute `bulkWrite` on -`client` with `models` and `ordered` set to false. Assert that the bulk write fails and returns a `BulkWriteError` -(referred to as `bulkWriteError`). +Execute `bulkWrite` on `client` with `models` and `verboseResults` set to true. Assert that the bulk write fails and returns +a `BulkWriteError` (referred to as `bulkWriteError`). -Assert that the length of `bulkWriteError.writeErrors` is 1. +Assert that `bulkWriteError.error` is populated with an error (referred to as `topLevelError`). Assert that +`topLevelError.errorCode` is equal to 8. -Assert that `bulkWriteError.error` is populated with an error with error code 8. +Assert that `bulkWriteError.partialResult` is populated with a result (referred to as `partialResult`). Assert that +`partialResult.upsertedCount` is equal to 2. Assert that the length of `partialResult.updateResults` is equal to 1. Assert that a CommandStartedEvent was observed for the `getMore` command. -### 9. `MongoClient.bulkWrite` returns error for unacknowledged too-large insert +Assert that a CommandStartedEvent was observed for the `killCursors` command. + +### 10. `MongoClient.bulkWrite` returns error for unacknowledged too-large insert This test must only be run on 8.0+ servers. @@ -512,4 +554,4 @@ Construct as list of write models (referred to as `models`) with the one `model` Call `MongoClient.bulkWrite` with `models` and `BulkWriteOptions.writeConcern` set to an unacknowledged write concern. -Expect a client-side error due the size. \ No newline at end of file +Expect a client-side error due the size. From 1540e57d32655f9dcce04950ee8937117edc491b Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 11 Apr 2024 11:26:51 -0600 Subject: [PATCH 58/90] errorResponse --- source/crud/bulk-write.md | 6 +++++- source/crud/tests/unified/client-bulkWrite-errors.yml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index e6fafd9879..21bcb63460 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -711,6 +711,9 @@ failing or a write concern error. Examples include network errors that occur whe with the server, command errors (`{ "ok": 0 }`) returned from the server, client-side errors, and errors that occur when attempting to perform a `getMore` to retrieve results from the server. +When a top-level error is caused by a command error (i.e. an `{ "ok": 0 }` server response), +drivers MUST provide access to the raw server reply in the error returned to the user. + When a top-level error is encountered and individual results and/or errors have already been observed, drivers MUST embed the top-level error within a `BulkWriteException` as the `error` field to retain this information. Otherwise, drivers MAY throw an exception containing only the top-level @@ -720,7 +723,8 @@ Encountering a top-level error MUST halt execution of a bulk write for both orde bulk writes. This means that drivers MUST NOT attempt to retrieve more responses from the cursor or execute any further `bulkWrite` batches and MUST immediately throw an exception. If the results cursor has not been exhausted on the server when a top-level error occurs, drivers MUST send the -`killCursors` command to attempt to close it. +`killCursors` command to attempt to close it. The result returned from the `killCursors` command +MAY be ignored. ### Write Concern Errors diff --git a/source/crud/tests/unified/client-bulkWrite-errors.yml b/source/crud/tests/unified/client-bulkWrite-errors.yml index 97ce175602..6d50122295 100644 --- a/source/crud/tests/unified/client-bulkWrite-errors.yml +++ b/source/crud/tests/unified/client-bulkWrite-errors.yml @@ -9,7 +9,7 @@ createEntities: observeEvents: [ commandStartedEvent ] uriOptions: retryWrites: false - useMultipleMongoses: false + useMultipleMongoses: false # Target a single mongos with failpoint - database: id: &database0 database0 client: *client0 From aaf6a212b37760255f35069f708f00cb8602db85 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 11 Apr 2024 11:46:01 -0600 Subject: [PATCH 59/90] add files --- .../client-bulkWrite-errorResponse.json | 68 +++++++++++++++++++ .../client-bulkWrite-errorResponse.yml | 37 ++++++++++ 2 files changed, 105 insertions(+) create mode 100644 source/crud/tests/unified/client-bulkWrite-errorResponse.json create mode 100644 source/crud/tests/unified/client-bulkWrite-errorResponse.yml diff --git a/source/crud/tests/unified/client-bulkWrite-errorResponse.json b/source/crud/tests/unified/client-bulkWrite-errorResponse.json new file mode 100644 index 0000000000..edf2339d8a --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-errorResponse.json @@ -0,0 +1,68 @@ +{ + "description": "client bulkWrite errorResponse", + "schemaVersion": "1.12", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false + } + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite operations support errorResponse assertions", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 8 + } + } + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1 + } + } + } + ] + }, + "expectError": { + "errorCode": 8, + "errorResponse": { + "code": 8 + } + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/client-bulkWrite-errorResponse.yml b/source/crud/tests/unified/client-bulkWrite-errorResponse.yml new file mode 100644 index 0000000000..45e53171ec --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-errorResponse.yml @@ -0,0 +1,37 @@ +description: "client bulkWrite errorResponse" +schemaVersion: "1.12" +runOnRequirements: + - minServerVersion: "8.0" + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false # Avoid setting fail points with multiple mongoses + +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + +tests: + - description: "client bulkWrite operations support errorResponse assertions" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ bulkWrite ] + errorCode: &errorCode 8 # UnknownError + - object: *client0 + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *namespace + document: { _id: 1 } + expectError: + errorCode: *errorCode + errorResponse: + code: *errorCode From c5ba1816b3b30f5534915362008afb31b2226614 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 17 Apr 2024 13:45:29 -0600 Subject: [PATCH 60/90] add unacknowledged monitoring test --- .../unacknowledged-client-bulkWrite.json | 186 ++++++++++++++++++ .../unacknowledged-client-bulkWrite.yml | 93 +++++++++ 2 files changed, 279 insertions(+) create mode 100644 source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.json create mode 100644 source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.yml diff --git a/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.json b/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.json new file mode 100644 index 0000000000..d6153f444e --- /dev/null +++ b/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.json @@ -0,0 +1,186 @@ +{ + "description": "unacknowledged-client-bulkWrite", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ], + "uriOptions": { + "w": 0 + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "command-monitoring-tests.test" + }, + "tests": [ + { + "description": "A successful mixed client bulkWrite", + "operations": [ + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "command-monitoring-tests.test", + "document": { + "_id": 4, + "x": 44 + } + } + }, + { + "updateOne": { + "namespace": "command-monitoring-tests.test", + "filter": { + "_id": 3 + }, + "update": { + "$set": { + "x": 333 + } + } + } + } + ] + }, + "expectResult": { + "insertedCount": { + "$$unsetOrMatches": 0 + }, + "upsertedCount": { + "$$unsetOrMatches": 0 + }, + "matchedCount": { + "$$unsetOrMatches": 0 + }, + "modifiedCount": { + "$$unsetOrMatches": 0 + }, + "deletedCount": { + "$$unsetOrMatches": 0 + }, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 4, + "x": 44 + } + }, + { + "update": 0, + "filter": { + "_id": 3 + }, + "updateMods": { + "$set": { + "x": 333 + } + }, + "multi": false + } + ], + "nsInfo": [ + { + "ns": "command-monitoring-tests.test" + } + ] + } + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite", + "reply": { + "ok": 1, + "nInserted": { + "$$exists": false + }, + "nMatched": { + "$$exists": false + }, + "nModified": { + "$$exists": false + }, + "nUpserted": { + "$$exists": false + }, + "nDeleted": { + "$$exists": false + } + } + } + } + ] + } + ] + } + ] +} diff --git a/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.yml b/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.yml new file mode 100644 index 0000000000..4ec16bde8f --- /dev/null +++ b/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.yml @@ -0,0 +1,93 @@ +description: "unacknowledged-client-bulkWrite" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client client + observeEvents: + - commandStartedEvent + - commandSucceededEvent + - commandFailedEvent + uriOptions: + w: 0 + - database: + id: &database database + client: *client + databaseName: &databaseName command-monitoring-tests + - collection: + id: &collection collection + database: *database + collectionName: &collectionName test + +initialData: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + +_yamlAnchors: + namespace: &namespace "command-monitoring-tests.test" + +tests: + - description: 'A successful mixed client bulkWrite' + operations: + - object: *client + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *namespace + document: { _id: 4, x: 44 } + - updateOne: + namespace: *namespace + filter: { _id: 3 } + update: { $set: { x: 333 } } + expectResult: + insertedCount: + $$unsetOrMatches: 0 + upsertedCount: + $$unsetOrMatches: 0 + matchedCount: + $$unsetOrMatches: 0 + modifiedCount: + $$unsetOrMatches: 0 + deletedCount: + $$unsetOrMatches: 0 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + expectEvents: + - + client: *client + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + bulkWrite: 1 + errorsOnly: true + ordered: true + ops: + - insert: 0 + document: { _id: 4, x: 44 } + - update: 0 + filter: { _id: 3 } + updateMods: { $set: { x: 333 } } + multi: false + nsInfo: + - ns: *namespace + - commandSucceededEvent: + commandName: bulkWrite + reply: + ok: 1 + nInserted: { $$exists: false } + nMatched: { $$exists: false } + nModified: { $$exists: false } + nUpserted: { $$exists: false } + nDeleted: { $$exists: false } From 6995d246dfe97f3626879257c68f39b6a0dc00a5 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 17 Apr 2024 13:51:47 -0600 Subject: [PATCH 61/90] add handshake error test --- source/etc/generate-handshakeError-tests.py | 9 +- .../tests/unified/handshakeError.json | 206 ++++++++++++++++++ .../tests/unified/handshakeError.yml | 86 ++++++++ 3 files changed, 300 insertions(+), 1 deletion(-) diff --git a/source/etc/generate-handshakeError-tests.py b/source/etc/generate-handshakeError-tests.py index 0fc3821c8d..a07a9df833 100644 --- a/source/etc/generate-handshakeError-tests.py +++ b/source/etc/generate-handshakeError-tests.py @@ -6,10 +6,16 @@ Operation = namedtuple( 'Operation', ['operation_name', 'command_name', 'object', 'arguments']) +CLIENT_BULK_WRITE_ARGUMENTS = '''models: + - insertOne: + namespace: retryable-writes-handshake-tests.coll + document: { _id: 8, x: 88 }''' + CLIENT_OPERATIONS = [ Operation('listDatabases', 'listDatabases', 'client', ['filter: {}']), Operation('listDatabaseNames', 'listDatabases', 'client', []), - Operation('createChangeStream', 'aggregate', 'client', ['pipeline: []']) + Operation('createChangeStream', 'aggregate', 'client', ['pipeline: []']), + Operation('clientBulkWrite', 'bulkWrite', 'client', [CLIENT_BULK_WRITE_ARGUMENTS]) ] RUN_COMMAND_ARGUMENTS = '''command: { ping: 1 } @@ -107,6 +113,7 @@ 'findOneAndReplace', 'insertMany', 'bulkWrite', + 'clientBulkWrite' ] ] diff --git a/source/retryable-writes/tests/unified/handshakeError.json b/source/retryable-writes/tests/unified/handshakeError.json index df37bd7232..dc20c934dc 100644 --- a/source/retryable-writes/tests/unified/handshakeError.json +++ b/source/retryable-writes/tests/unified/handshakeError.json @@ -53,6 +53,212 @@ } ], "tests": [ + { + "description": "client.clientBulkWrite succeeds after retryable handshake network error", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "ping", + "saslContinue" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "ping", + "command": { + "ping": 1 + } + }, + "expectError": { + "isError": true + } + }, + { + "name": "clientBulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-handshake-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + } + ] + }, + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "ping": 1 + }, + "databaseName": "retryable-writes-handshake-tests" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite succeeds after retryable handshake server error (ShutdownInProgress)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "ping", + "saslContinue" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "ping", + "command": { + "ping": 1 + } + }, + "expectError": { + "isError": true + } + }, + { + "name": "clientBulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-handshake-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckOutStartedEvent": {} + } + ] + }, + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "ping": 1 + }, + "databaseName": "retryable-writes-handshake-tests" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, { "description": "collection.insertOne succeeds after retryable handshake network error", "operations": [ diff --git a/source/retryable-writes/tests/unified/handshakeError.yml b/source/retryable-writes/tests/unified/handshakeError.yml index 9b2774bc77..3bbceab8d6 100644 --- a/source/retryable-writes/tests/unified/handshakeError.yml +++ b/source/retryable-writes/tests/unified/handshakeError.yml @@ -50,6 +50,92 @@ tests: # - Triggers failpoint (second time). # - Tests whether operation successfully retries the handshake and succeeds. + - description: "client.clientBulkWrite succeeds after retryable handshake network error" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: [ping, saslContinue] + closeConnection: true + - name: runCommand + object: *database + arguments: { commandName: ping, command: { ping: 1 } } + expectError: { isError: true } + - name: clientBulkWrite + object: *client + arguments: + models: + - insertOne: + namespace: retryable-writes-handshake-tests.coll + document: { _id: 8, x: 88 } + expectEvents: + - client: *client + eventType: cmap + events: + - { connectionCheckOutStartedEvent: {} } + - { connectionCheckOutStartedEvent: {} } + - { connectionCheckOutStartedEvent: {} } + - { connectionCheckOutStartedEvent: {} } + - client: *client + events: + - commandStartedEvent: + command: { ping: 1 } + databaseName: *databaseName + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: bulkWrite + - commandSucceededEvent: + commandName: bulkWrite + + - description: "client.clientBulkWrite succeeds after retryable handshake server error (ShutdownInProgress)" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: [ping, saslContinue] + closeConnection: true + - name: runCommand + object: *database + arguments: { commandName: ping, command: { ping: 1 } } + expectError: { isError: true } + - name: clientBulkWrite + object: *client + arguments: + models: + - insertOne: + namespace: retryable-writes-handshake-tests.coll + document: { _id: 8, x: 88 } + expectEvents: + - client: *client + eventType: cmap + events: + - { connectionCheckOutStartedEvent: {} } + - { connectionCheckOutStartedEvent: {} } + - { connectionCheckOutStartedEvent: {} } + - { connectionCheckOutStartedEvent: {} } + - client: *client + events: + - commandStartedEvent: + command: { ping: 1 } + databaseName: *databaseName + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: bulkWrite + - commandSucceededEvent: + commandName: bulkWrite + - description: "collection.insertOne succeeds after retryable handshake network error" operations: - name: failPoint From e6f810096b67fe965a7fa6bd653cb3b2a48d7419 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 17 Apr 2024 14:06:59 -0600 Subject: [PATCH 62/90] operation ID server selection test --- .../tests/logging/operation-id.json | 187 ++++++++++++++++++ .../tests/logging/operation-id.yml | 99 ++++++++++ 2 files changed, 286 insertions(+) diff --git a/source/server-selection/tests/logging/operation-id.json b/source/server-selection/tests/logging/operation-id.json index 276e4b8d6d..6cdbcb3f5a 100644 --- a/source/server-selection/tests/logging/operation-id.json +++ b/source/server-selection/tests/logging/operation-id.json @@ -47,6 +47,9 @@ } } ], + "_yamlAnchors": { + "namespace": "logging-tests.server-selection" + }, "tests": [ { "description": "Successful bulkWrite operation: log messages have operationIds", @@ -224,6 +227,190 @@ ] } ] + }, + { + "description": "Successful client bulkWrite operation: log messages have operationIds", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "topologyDescriptionChangedEvent": {} + }, + "count": 2 + } + }, + { + "name": "clientBulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "logging-tests.server-selection", + "document": { + "x": 1 + } + } + } + ] + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "serverSelection", + "data": { + "message": "Server selection started", + "operationId": { + "$$type": [ + "int", + "long" + ] + }, + "operation": "bulkWrite" + } + }, + { + "level": "debug", + "component": "serverSelection", + "data": { + "message": "Server selection succeeded", + "operationId": { + "$$type": [ + "int", + "long" + ] + }, + "operation": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "Failed client bulkWrite operation: log messages have operationIds", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "hello", + "ismaster" + ], + "appName": "loggingClient", + "closeConnection": true + } + } + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverDescriptionChangedEvent": { + "newDescription": { + "type": "Unknown" + } + } + }, + "count": 1 + } + }, + { + "name": "bulkWrite", + "object": "client", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "logging-tests.server-selection", + "document": { + "x": 1 + } + } + } + ] + }, + "expectError": { + "isClientError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "serverSelection", + "data": { + "message": "Server selection started", + "operationId": { + "$$type": [ + "int", + "long" + ] + }, + "operation": "bulkWrite" + } + }, + { + "level": "info", + "component": "serverSelection", + "data": { + "message": "Waiting for suitable server to become available", + "operationId": { + "$$type": [ + "int", + "long" + ] + }, + "operation": "bulkWrite" + } + }, + { + "level": "debug", + "component": "serverSelection", + "data": { + "message": "Server selection failed", + "operationId": { + "$$type": [ + "int", + "long" + ] + }, + "operation": "bulkWrite" + } + } + ] + } + ] } ] } diff --git a/source/server-selection/tests/logging/operation-id.yml b/source/server-selection/tests/logging/operation-id.yml index 430e81a58b..950c3c4343 100644 --- a/source/server-selection/tests/logging/operation-id.yml +++ b/source/server-selection/tests/logging/operation-id.yml @@ -30,6 +30,9 @@ createEntities: - client: id: &failPointClient failPointClient +_yamlAnchors: + namespace: &namespace "logging-tests.server-selection" + tests: - description: "Successful bulkWrite operation: log messages have operationIds" operations: @@ -122,3 +125,99 @@ tests: operationId: { $$type: [int, long] } operation: insert + - description: "Successful client bulkWrite operation: log messages have operationIds" + runOnRequirements: + - minServerVersion: "8.0" # required for bulkWrite command + operations: + # ensure we've discovered the server so it is immediately available + # and no extra "waiting for suitable server" messages are emitted. + # expected topology events reflect initial server discovery and server connect event. + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + topologyDescriptionChangedEvent: {} + count: 2 + - name: clientBulkWrite + object: *client + arguments: + models: + - insertOne: + namespace: *namespace + document: { x: 1 } + expectLogMessages: + - client: *client + messages: + - level: debug + component: serverSelection + data: + message: "Server selection started" + operationId: { $$type: [int, long] } + operation: bulkWrite + - level: debug + component: serverSelection + data: + message: "Server selection succeeded" + operationId: { $$type: [int, long] } + operation: bulkWrite + + - description: "Failed client bulkWrite operation: log messages have operationIds" + runOnRequirements: + - minServerVersion: "8.0" # required for bulkWrite command + operations: + # fail all hello/legacy hello commands for the main client. + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: ["hello", "ismaster"] + appName: *appName + closeConnection: true + # wait until we've marked the server unknown due + # to a failed heartbeat. + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + serverDescriptionChangedEvent: + newDescription: + type: Unknown + count: 1 + - name: bulkWrite + object: *client + arguments: + models: + - insertOne: + namespace: *namespace + document: { x: 1 } + expectError: + isClientError: true # server selection timeout + expectLogMessages: + - client: *client + messages: + - level: debug + component: serverSelection + data: + message: "Server selection started" + operationId: { $$type: [int, long] } + operation: bulkWrite + - level: info + component: serverSelection + data: + message: "Waiting for suitable server to become available" + operationId: { $$type: [int, long] } + operation: bulkWrite + - level: debug + component: serverSelection + data: + message: "Server selection failed" + operationId: { $$type: [int, long] } + operation: bulkWrite + + From c09c1d42d17df48fa6c8b38b156b286bcfb9a53d Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 17 Apr 2024 14:10:15 -0600 Subject: [PATCH 63/90] add pinning test --- .../tests/unified/mongos-pin-auto-tests.py | 5 + .../tests/unified/mongos-pin-auto.json | 279 ++++++++++++++++++ .../tests/unified/mongos-pin-auto.yml | 84 ++++++ 3 files changed, 368 insertions(+) diff --git a/source/transactions/tests/unified/mongos-pin-auto-tests.py b/source/transactions/tests/unified/mongos-pin-auto-tests.py index 99a34b485d..ab1dc72cb4 100644 --- a/source/transactions/tests/unified/mongos-pin-auto-tests.py +++ b/source/transactions/tests/unified/mongos-pin-auto-tests.py @@ -291,6 +291,11 @@ insert: *collection_name documents: - { _id : 1 }'''), + # clientBulkWrite: + 'clientBulkWrite': ('bulkWrite', '*client0', r'''models: + - insertOne: + namespace: database0.collection0 + document: { _id: 8, x: 88 }'''), } # Maps from error_name to error_data. diff --git a/source/transactions/tests/unified/mongos-pin-auto.json b/source/transactions/tests/unified/mongos-pin-auto.json index 93eac8bb77..e3b1e7ccd6 100644 --- a/source/transactions/tests/unified/mongos-pin-auto.json +++ b/source/transactions/tests/unified/mongos-pin-auto.json @@ -2004,6 +2004,99 @@ } ] }, + { + "description": "remain pinned after non-transient Interrupted error on clientBulkWrite bulkWrite", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "targetedFailPoint", + "object": "testRunner", + "arguments": { + "session": "session0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 11601 + } + } + } + }, + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "session": "session0", + "models": [ + { + "insertOne": { + "namespace": "database0.collection0", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "errorLabelsOmit": [ + "TransientTransactionError" + ] + } + }, + { + "object": "testRunner", + "name": "assertSessionPinned", + "arguments": { + "session": "session0" + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + }, { "description": "unpin after transient connection error on insertOne insert", "operations": [ @@ -5175,6 +5268,192 @@ ] } ] + }, + { + "description": "unpin after transient connection error on clientBulkWrite bulkWrite", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "targetedFailPoint", + "object": "testRunner", + "arguments": { + "session": "session0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "closeConnection": true + } + } + } + }, + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "session": "session0", + "models": [ + { + "insertOne": { + "namespace": "database0.collection0", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "errorLabelsContain": [ + "TransientTransactionError" + ] + } + }, + { + "object": "testRunner", + "name": "assertSessionUnpinned", + "arguments": { + "session": "session0" + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + }, + { + "description": "unpin after transient ShutdownInProgress error on clientBulkWrite bulkWrite", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "targetedFailPoint", + "object": "testRunner", + "arguments": { + "session": "session0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorCode": 91 + } + } + } + }, + { + "name": "clientBulkWrite", + "object": "client0", + "arguments": { + "session": "session0", + "models": [ + { + "insertOne": { + "namespace": "database0.collection0", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "errorLabelsContain": [ + "TransientTransactionError" + ] + } + }, + { + "object": "testRunner", + "name": "assertSessionUnpinned", + "arguments": { + "session": "session0" + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] } ] } diff --git a/source/transactions/tests/unified/mongos-pin-auto.yml b/source/transactions/tests/unified/mongos-pin-auto.yml index 7a76347555..086e6a4f01 100644 --- a/source/transactions/tests/unified/mongos-pin-auto.yml +++ b/source/transactions/tests/unified/mongos-pin-auto.yml @@ -676,6 +676,34 @@ tests: - *abortTransaction outcome: *outcome + - description: remain pinned after non-transient Interrupted error on clientBulkWrite bulkWrite + operations: + - *startTransaction + - *initialCommand + - name: targetedFailPoint + object: testRunner + arguments: + session: *session0 + failPoint: + configureFailPoint: failCommand + mode: {times: 1} + data: + failCommands: ["bulkWrite"] + errorCode: 11601 + - name: clientBulkWrite + object: *client0 + arguments: + session: *session0 + models: + - insertOne: + namespace: database0.collection0 + document: { _id: 8, x: 88 } + expectError: + errorLabelsOmit: ["TransientTransactionError"] + - *assertSessionPinned + - *abortTransaction + outcome: *outcome + - description: unpin after transient connection error on insertOne insert operations: - *startTransaction @@ -1614,3 +1642,59 @@ tests: - *abortTransaction outcome: *outcome + - description: unpin after transient connection error on clientBulkWrite bulkWrite + operations: + - *startTransaction + - *initialCommand + - name: targetedFailPoint + object: testRunner + arguments: + session: *session0 + failPoint: + configureFailPoint: failCommand + mode: {times: 1} + data: + failCommands: ["bulkWrite"] + closeConnection: true + - name: clientBulkWrite + object: *client0 + arguments: + session: *session0 + models: + - insertOne: + namespace: database0.collection0 + document: { _id: 8, x: 88 } + expectError: + errorLabelsContain: ["TransientTransactionError"] + - *assertSessionUnpinned + - *abortTransaction + outcome: *outcome + + - description: unpin after transient ShutdownInProgress error on clientBulkWrite bulkWrite + operations: + - *startTransaction + - *initialCommand + - name: targetedFailPoint + object: testRunner + arguments: + session: *session0 + failPoint: + configureFailPoint: failCommand + mode: {times: 1} + data: + failCommands: ["bulkWrite"] + errorCode: 91 + - name: clientBulkWrite + object: *client0 + arguments: + session: *session0 + models: + - insertOne: + namespace: database0.collection0 + document: { _id: 8, x: 88 } + expectError: + errorLabelsContain: ["TransientTransactionError"] + - *assertSessionUnpinned + - *abortTransaction + outcome: *outcome + From fa4b4348efa7975075da145ed34bc3b574779197 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 17 Apr 2024 14:11:08 -0600 Subject: [PATCH 64/90] add write concern transaction test --- .../tests/unified/client-bulkWrite.json | 168 ++++++++++++++++++ .../tests/unified/client-bulkWrite.yml | 82 +++++++++ 2 files changed, 250 insertions(+) diff --git a/source/transactions/tests/unified/client-bulkWrite.json b/source/transactions/tests/unified/client-bulkWrite.json index 7454c6f360..b13702799b 100644 --- a/source/transactions/tests/unified/client-bulkWrite.json +++ b/source/transactions/tests/unified/client-bulkWrite.json @@ -39,6 +39,23 @@ "id": "session0", "client": "client0" } + }, + { + "client": { + "id": "client_with_wmajority", + "uriOptions": { + "w": "majority" + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "session": { + "id": "session_with_wmajority", + "client": "client_with_wmajority" + } } ], "_yamlAnchors": { @@ -385,6 +402,157 @@ ] } ] + }, + { + "description": "client writeConcern ignored for client bulkWrite in transaction", + "operations": [ + { + "object": "session_with_wmajority", + "name": "startTransaction", + "arguments": { + "writeConcern": { + "w": 1 + } + } + }, + { + "object": "client_with_wmajority", + "name": "clientBulkWrite", + "arguments": { + "session": "session_with_wmajority", + "models": [ + { + "insertOne": { + "namespace": "transaction-tests.coll0", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + }, + { + "object": "session_with_wmajority", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client_with_wmajority", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "lsid": { + "$$sessionLsid": "session_with_wmajority" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + }, + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true, + "ops": [ + { + "insert": 0, + "document": { + "_id": 8, + "x": 88 + } + } + ], + "nsInfo": [ + { + "ns": "transaction-tests.coll0" + } + ] + } + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session_with_wmajority" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "w": 1 + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 5, + "x": 55 + }, + { + "_id": 6, + "x": 66 + }, + { + "_id": 7, + "x": 77 + }, + { + "_id": 8, + "x": 88 + } + ] + } + ] } ] } diff --git a/source/transactions/tests/unified/client-bulkWrite.yml b/source/transactions/tests/unified/client-bulkWrite.yml index 193a2bd905..d2234ed304 100644 --- a/source/transactions/tests/unified/client-bulkWrite.yml +++ b/source/transactions/tests/unified/client-bulkWrite.yml @@ -22,6 +22,15 @@ createEntities: - session: id: &session0 session0 client: *client0 + - client: + id: &client_with_wmajority client_with_wmajority + uriOptions: + w: majority + observeEvents: + - commandStartedEvent + - session: + id: &session_with_wmajority session_with_wmajority + client: *client_with_wmajority _yamlAnchors: namespace: &namespace "transaction-tests.coll0" @@ -161,3 +170,76 @@ tests: - { _id: 3, x: 35 } - { _id: 4, x: 44 } - { _id: 8, x: 88 } + - description: 'client writeConcern ignored for client bulkWrite in transaction' + operations: + - object: *session_with_wmajority + name: startTransaction + arguments: + writeConcern: + w: 1 + - object: *client_with_wmajority + name: clientBulkWrite + arguments: + session: *session_with_wmajority + models: + - insertOne: + namespace: *namespace + document: { _id: 8, x: 88 } + expectResult: + insertedCount: 1 + upsertedCount: 0 + matchedCount: 0 + modifiedCount: 0 + deletedCount: 0 + insertResults: + $$unsetOrMatches: {} + updateResults: + $$unsetOrMatches: {} + deleteResults: + $$unsetOrMatches: {} + - object: *session_with_wmajority + name: commitTransaction + expectEvents: + - + client: *client_with_wmajority + events: + - commandStartedEvent: + commandName: bulkWrite + databaseName: admin + command: + lsid: { $$sessionLsid: *session_with_wmajority } + txnNumber: 1 + startTransaction: true + autocommit: false + writeConcern: { $$exists: false } + bulkWrite: 1 + errorsOnly: true + ordered: true + ops: + - insert: 0 + document: { _id: 8, x: 88 } + nsInfo: + - ns: *namespace + - + commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session_with_wmajority } + txnNumber: { $numberLong: '1' } + startTransaction: { $$exists: false } + autocommit: false + writeConcern: + w: 1 + commandName: commitTransaction + databaseName: admin + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 5, x: 55 } + - { _id: 6, x: 66 } + - { _id: 7, x: 77 } + - { _id: 8, x: 88 } From 919c50670c03b801ed927c33e11a46b52918e6c8 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 17 Apr 2024 15:00:27 -0600 Subject: [PATCH 65/90] another transaction test --- .../tests/unified/client-bulkWrite.json | 34 +++++++++++++++++++ .../tests/unified/client-bulkWrite.yml | 17 ++++++++++ 2 files changed, 51 insertions(+) diff --git a/source/transactions/tests/unified/client-bulkWrite.json b/source/transactions/tests/unified/client-bulkWrite.json index b13702799b..f8f1d97169 100644 --- a/source/transactions/tests/unified/client-bulkWrite.json +++ b/source/transactions/tests/unified/client-bulkWrite.json @@ -553,6 +553,40 @@ ] } ] + }, + { + "description": "client bulkWrite with writeConcern in a transaction causes a transaction error", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "session": "session0", + "writeConcern": { + "w": 1 + }, + "models": [ + { + "insertOne": { + "namespace": "transaction-tests.coll0", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "isClientError": true, + "errorContains": "Cannot set write concern after starting a transaction" + } + } + ] } ] } diff --git a/source/transactions/tests/unified/client-bulkWrite.yml b/source/transactions/tests/unified/client-bulkWrite.yml index d2234ed304..eda2babbe7 100644 --- a/source/transactions/tests/unified/client-bulkWrite.yml +++ b/source/transactions/tests/unified/client-bulkWrite.yml @@ -243,3 +243,20 @@ tests: - { _id: 6, x: 66 } - { _id: 7, x: 77 } - { _id: 8, x: 88 } + - description: "client bulkWrite with writeConcern in a transaction causes a transaction error" + operations: + - object: *session0 + name: startTransaction + - object: *client0 + name: clientBulkWrite + arguments: + session: *session0 + writeConcern: + w: 1 + models: + - insertOne: + namespace: *namespace + document: { _id: 8, x: 88 } + expectError: + isClientError: true + errorContains: "Cannot set write concern after starting a transaction" From af0a16dac347b610169b260313e30669f6d1cd55 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 17 Apr 2024 15:05:05 -0600 Subject: [PATCH 66/90] add csot test --- .../tests/README.md | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/source/client-side-operations-timeout/tests/README.md b/source/client-side-operations-timeout/tests/README.md index b4160500f5..be5ee48664 100644 --- a/source/client-side-operations-timeout/tests/README.md +++ b/source/client-side-operations-timeout/tests/README.md @@ -24,7 +24,7 @@ test MUST be unset using `internalClient` after the test has been executed. All MUST be configured with read/write concern `majority`, read preference `primary`, and command monitoring enabled to listen for `command_started` events. -### 1. Multi-batch writes +### 1. Multi-batch inserts This test MUST only run against standalones on server versions 4.4 and higher. The `insertMany` call takes an exceedingly long time on replicasets and sharded clusters. Drivers MAY adjust the timeouts used in this test to allow @@ -598,6 +598,39 @@ Tests in this section MUST only run against replica sets and sharded clusters wi 1. `command_started` and `command_failed` events for an `insert` command. 2. `command_started` and `command_failed` events for an `abortTransaction` command. + +### 11. Multi-batch bulkWrites + +This test MUST only run against standalones on server versions 8.0 and higher. The `bulkWrite` call takes an +exceedingly long time on replicasets and sharded clusters. Drivers MAY adjust the timeouts used in this test to allow +for differing bulk encoding performance. + +1. Using `internalClient`, drop the `db.coll` collection. + +2. Using `internalClient`, set the following fail point: + + ```javascript + { + configureFailPoint: "failCommand", + mode: { + times: 2 + }, + data: { + failCommands: ["bulkWrite"], + blockConnection: true, + blockTimeMS: 1010 + } + } + ``` + +3. Create a new MongoClient (referred to as `client`) with `timeoutMS=2000`. + +4. Using `client`, insert 50 1-megabyte documents in a single `MongoClient.bulkWrite` call. + + - Expect this to fail with a timeout error. + +5. Verify that two `bulkWrite` commands were executed against `db.coll` as part of the `MongoClient.bulkWrite` call. + ## Unit Tests The tests enumerated in this section could not be expressed in either spec or prose format. Drivers SHOULD implement From 72eda1b8d5ecf2d9fe6fc8ca378cd542c00b0cec Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 17 Apr 2024 15:12:00 -0600 Subject: [PATCH 67/90] cleanup --- source/server-selection/tests/logging/operation-id.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/server-selection/tests/logging/operation-id.yml b/source/server-selection/tests/logging/operation-id.yml index 950c3c4343..24e48f9410 100644 --- a/source/server-selection/tests/logging/operation-id.yml +++ b/source/server-selection/tests/logging/operation-id.yml @@ -219,5 +219,3 @@ tests: message: "Server selection failed" operationId: { $$type: [int, long] } operation: bulkWrite - - From f389db6016ede749416eb2ddf0f9ece0b010acb5 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 17 Apr 2024 15:15:48 -0600 Subject: [PATCH 68/90] add q&a --- source/crud/bulk-write.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 21bcb63460..bea6610355 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -809,6 +809,12 @@ unnecessary work for drivers. Always sending the user's specified value also saf unlikely event that the server changes the default value for `bypassDocumentValidation` in the future. +### Why is providing access to the raw server response when a command error occurs required? + +This allows users to access new error fields that the server may add in the future without needing +to upgrade their driver version. See [DRIVERS-2385](https://jira.mongodb.org/browse/DRIVERS-2385) +for more details. + ## **Changelog** - TODO: Bulk write specification created. From 8f54d527095bef5e71673813c1b4d2842b1c0380 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Mon, 22 Apr 2024 09:55:02 -0600 Subject: [PATCH 69/90] nsInfo clarifications and test --- source/crud/bulk-write.md | 29 +++++++++++++++++------- source/crud/tests/README.md | 45 +++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index bea6610355..94c9f2551c 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -594,14 +594,18 @@ limits. ## Command Batching -Drivers MUST accept an arbitrary number of operations as input to the `MongoClient.bulkWrite` method. -Because the server imposes restrictions on the size of write operations, this means that a single -call to `MongoClient.bulkWrite` may require multiple `bulkWrite` commands to be sent to the server. -Drivers MUST split bulk writes into separate commands when the user's list of operations exceeds -one or more of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize` (for `OP_MSG` payload type -0), and `maxMessageSizeBytes` (for `OP_MSG` payload type 1). Each of these values can be retrieved -from the selected server's `hello` command response. Drivers MUST merge results from multiple batches -into a single `BulkWriteResult` or `BulkWriteException` to return from `MongoClient.bulkWrite`. +Drivers MUST accept an arbitrary number of operations as input to the `MongoClient.bulkWrite` +method. Because the server imposes restrictions on the size of write operations, this means that a +single call to `MongoClient.bulkWrite` may require multiple `bulkWrite` commands to be sent to the +server. Drivers MUST split bulk writes into separate commands when the user's list of operations +exceeds one or more of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize` (for `OP_MSG` +payload type 0), and `maxMessageSizeBytes` (for `OP_MSG` payload type 1). Each of these values can +be retrieved from the selected server's `hello` command response. Drivers MUST merge results from +multiple batches into a single `BulkWriteResult` or `BulkWriteException` to return from +`MongoClient.bulkWrite`. + +When constructing the `nsInfo` array for a `bulkWrite` batch, drivers MUST only include the +namespaces that are referenced in the `ops` array for that batch. ### Number of Writes @@ -815,6 +819,15 @@ This allows users to access new error fields that the server may add in the futu to upgrade their driver version. See [DRIVERS-2385](https://jira.mongodb.org/browse/DRIVERS-2385) for more details. +### Why are drivers required to send `nsInfo` as a document sequence? + +`nsInfo` could exceed `maxBsonObjectSize` if a user is doing `maxWriteBatchSize` operations, each +operation is on a unique namespace, and each namespace is near the +[maximum length](https://www.mongodb.com/docs/manual/reference/limits/#mongodb-limit-Restriction-on-Collection-Names) +allowed for namespaces given the values for these limits at the time of writing this specification. +Providing `nsInfo` as a document sequence reduces the likelihood that a driver would need to batch +split a user's bulk write in this scenario. + ## **Changelog** - TODO: Bulk write specification created. diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 61637ae58b..601d456f65 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -555,3 +555,48 @@ Construct as list of write models (referred to as `models`) with the one `model` Call `MongoClient.bulkWrite` with `models` and `BulkWriteOptions.writeConcern` set to an unacknowledged write concern. Expect a client-side error due the size. + +### 11. `MongoClient.bulkWrite` excludes namespaces from other batches + +Test that `MongoClient.bulkWrite` only includes the namespaces used for the operations in a single batch. + +This test must only be run on 8.0+ servers. + +Construct a `MongoClient` (referred to as `client`) with +[command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe +CommandStartedEvents. Perform a `hello` command using `client` and record the `maxWriteBatchSize` value from the +response. + +Create the following write model (referred to as `model`): + +```json +InsertOne { + "namespace": "db.coll", + "document": { "a": "b" } +} +``` + +Create a list of write models containing `model` repeated `maxWriteBatchSize` times (referred to as `models`). + +Append the following write model to `models`: + +```json +InsertOne { + "namespace": "db.coll1", + "document": { "a": "b" } +} +``` + +Execute `bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` +(referred to as `result`). + +Assert that `result.insertedCount` is equal to `maxWriteBatchSize + 1`. + +Assert that two CommandStartedEvents were observed for the `bulkWrite` command (referred to as `firstEvent` and +`secondEvent`). + +Assert that the length of `firstEvent.command.nsInfo` is 1. Assert that the the namespace contained in +`firstEvent.command.nsInfo` is "db.coll". + +Assert that the length of `secondEvent.command.nsInfo` is 1. Assert that the the namespace contained in +`secondEvent.command.nsInfo` is "db.coll1". From ec71c964c5112dc9dfdf37b3f403e76da3416cba Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 23 Apr 2024 10:43:22 -0600 Subject: [PATCH 70/90] retry bulkWrite when getMore fails --- source/crud/tests/README.md | 2 +- source/retryable-writes/retryable-writes.rst | 68 ++++++++++++-------- source/retryable-writes/tests/README.rst | 54 ++++++++++++++++ 3 files changed, 96 insertions(+), 28 deletions(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 601d456f65..3e337c71ba 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -371,7 +371,7 @@ Assert that `orderedError.writeErrors` has a length of 1. Assert that one CommandStartedEvent was observed for the `bulkWrite` command. -### 7. `MongoClient.bulkWrite` handles a cursor requiring a `getMore` +### 7. `MongoClient.bulkWrite` handles a results cursor requiring a `getMore` Test that `MongoClient.bulkWrite` properly iterates the results cursor when `getMore` is required. diff --git a/source/retryable-writes/retryable-writes.rst b/source/retryable-writes/retryable-writes.rst index 0a79127e86..ce9b6b94a9 100644 --- a/source/retryable-writes/retryable-writes.rst +++ b/source/retryable-writes/retryable-writes.rst @@ -246,15 +246,44 @@ The RetryableWriteError label might be added to an error in a variety of ways: returning them to the driver. As new server versions are released, the errors that are labeled with the RetryableWriteError label may change. Drivers MUST NOT add a RetryableWriteError label to any error derived from a 4.4+ server - response (i.e. any error that is not a network error). + response to a write command (i.e. any error that is not a network error). -- When receiving a command result with an error from a pre-4.4 server that - supports retryable writes, the driver MUST add a RetryableWriteError label to - errors that meet the following criteria if the retryWrites option is set to - true on the client performing the relevant operation: +- When receiving a command result with an error from a pre-4.4 server that supports retryable + writes, the driver MUST add a RetryableWriteError label to errors that meet the + `Error Code Criteria`_ if the retryWrites option is set to true on the client performing the + relevant operation. - - a mongod or mongos response with any the following error codes in the - top-level ``code`` field: +- When the driver receives an error response to the ``getMore`` command while attempting to iterate + the results cursor returned from the ``bulkWrite`` command, it MUST add a RetryableWriteError + label to the overall error for the bulk write if the error meets the `Error Code Criteria`_, + the retryWrites option is set to true on the client performing the operation, and the + ``bulkWrite`` is retryable (i.e. it does not contain multi:true operations). If a + RetryableWriteError label is attached, drivers MUST attempt to retry the entire ``bulkWrite`` + command and MUST NOT attempt to retry the ``getMore`` that failed. + +To understand why the driver should only add the RetryableWriteError label to an +error when the retryWrites option is true on the MongoClient performing the +operation, see `Why does the driver only add the RetryableWriteError label to +errors that occur on a MongoClient with retryWrites set to true?`_ + +Note: During a retryable write operation on a sharded cluster, mongos may retry +the operation internally, in which case it will not add a RetryableWriteError +label to any error that occurs after those internal retries to prevent excessive +retrying. + +For more information about error labels, see the `Transactions specification`_. + +.. _Error Handling: ../server-discovery-and-monitoring/server-discovery-and-monitoring.rst#error-handling +.. _Transactions specification: ../transactions/transactions.md#error-labels + +Error Code Criteria +""""""""""""""""""" + +The following errors are eligible for attaching a RetryableWriteError label when evaluating based +on error code: + + - a mongod or mongos response with any of the following error codes in the top-level ``code`` + field: .. list-table:: :header-rows: 1 @@ -289,31 +318,16 @@ The RetryableWriteError label might be added to an error in a variety of ways: - a mongod response with any of the previously listed codes in the ``writeConcernError.code`` field. - Drivers MUST NOT add a RetryableWriteError label based on the following: +Drivers MUST NOT add a RetryableWriteError label based on the following: - any ``writeErrors[].code`` fields in a mongod or mongos response - the ``writeConcernError.code`` field in a mongos response - The criteria for retryable errors is similar to the discussion in the SDAM - spec's section on `Error Handling`_, but includes additional error codes. See - `What do the additional error codes mean?`_ for the reasoning behind these - additional errors. - -To understand why the driver should only add the RetryableWriteError label to an -error when the retryWrites option is true on the MongoClient performing the -operation, see `Why does the driver only add the RetryableWriteError label to -errors that occur on a MongoClient with retryWrites set to true?`_ - -Note: During a retryable write operation on a sharded cluster, mongos may retry -the operation internally, in which case it will not add a RetryableWriteError -label to any error that occurs after those internal retries to prevent excessive -retrying. - -For more information about error labels, see the `Transactions specification`_. - -.. _Error Handling: ../server-discovery-and-monitoring/server-discovery-and-monitoring.rst#error-handling -.. _Transactions specification: ../transactions/transactions.md#error-labels +The criteria for retryable errors is similar to the discussion in the SDAM +spec's section on `Error Handling`_, but includes additional error codes. See +`What do the additional error codes mean?`_ for the reasoning behind these +additional errors. Generating Transaction IDs ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/source/retryable-writes/tests/README.rst b/source/retryable-writes/tests/README.rst index 7c03d23d6f..76dc275896 100644 --- a/source/retryable-writes/tests/README.rst +++ b/source/retryable-writes/tests/README.rst @@ -541,6 +541,60 @@ and sharded clusters. 7. Disable the fail point on ``s0``. +#. Test that a failed ``getMore`` causes a retry of the ``bulkWrite`` command. + + This test must only be run against replica sets and sharded clusters on server versions 8.0+. + + 1. Create a client (referred to as ``client``) with command monitoring enabled. If testing + against a sharded deployment, be sure to connect to only a single mongos. + + 2. Perform a ``hello`` on ``client`` and record the ``maxBsonObjectSize`` value in the response. + + 3. Create a collection (referred to as ``collection``) with the namespace "db.coll". Drop + ``collection``. + + 4. Enable the following failpoint:: + + { + configureFailPoint: "failCommand", + mode: { times: 1 }, + data: { + failCommands: ["getMore"], + errorCode: 6 + } + } + + 5. Create the following list of write models (referred to as ``models``):: + + [ + ReplaceOneModel { + namespace: namespace, + filter: { _id: "a".repeat(maxBsonObjectSize / 2) } + replacement: { x: 1 }, + upsert: true + }, + ReplaceOneModel { + namespace: namespace, + filter: { _id: "b".repeat(maxBsonObjectSize / 2) } + replacement: { x: 1 }, + upsert: true + } + ] + + 6. Execute ``bulkWrite()`` on ``client`` with ``models`` and ``verboseResults`` set to true. + Assert that the bulk write succeeds. + + 7. Assert that the following sequence of command events occurred: + + - CommandStartedEvent for ``bulkWrite`` + - CommandSucceededEvent for ``bulkWrite`` + - CommandStartedEvent for ``getMore`` + - CommandFailedEvent for ``getMore`` + - CommandStartedEvent for ``bulkWrite`` + - CommandSucceededEvent for ``bulkWrite`` + - CommandStartedEvent for ``getMore`` + - CommandSucceededEvent for ``getMore`` + Changelog ========= From 6cde925e5015204373677ce06d69c54c130ed7a0 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 24 Apr 2024 11:45:08 -0600 Subject: [PATCH 71/90] Revert "retry bulkWrite when getMore fails" This reverts commit ec71c964c5112dc9dfdf37b3f403e76da3416cba. --- source/crud/tests/README.md | 2 +- source/retryable-writes/retryable-writes.rst | 68 ++++++++------------ source/retryable-writes/tests/README.rst | 54 ---------------- 3 files changed, 28 insertions(+), 96 deletions(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 3e337c71ba..601d456f65 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -371,7 +371,7 @@ Assert that `orderedError.writeErrors` has a length of 1. Assert that one CommandStartedEvent was observed for the `bulkWrite` command. -### 7. `MongoClient.bulkWrite` handles a results cursor requiring a `getMore` +### 7. `MongoClient.bulkWrite` handles a cursor requiring a `getMore` Test that `MongoClient.bulkWrite` properly iterates the results cursor when `getMore` is required. diff --git a/source/retryable-writes/retryable-writes.rst b/source/retryable-writes/retryable-writes.rst index ce9b6b94a9..0a79127e86 100644 --- a/source/retryable-writes/retryable-writes.rst +++ b/source/retryable-writes/retryable-writes.rst @@ -246,44 +246,15 @@ The RetryableWriteError label might be added to an error in a variety of ways: returning them to the driver. As new server versions are released, the errors that are labeled with the RetryableWriteError label may change. Drivers MUST NOT add a RetryableWriteError label to any error derived from a 4.4+ server - response to a write command (i.e. any error that is not a network error). + response (i.e. any error that is not a network error). -- When receiving a command result with an error from a pre-4.4 server that supports retryable - writes, the driver MUST add a RetryableWriteError label to errors that meet the - `Error Code Criteria`_ if the retryWrites option is set to true on the client performing the - relevant operation. +- When receiving a command result with an error from a pre-4.4 server that + supports retryable writes, the driver MUST add a RetryableWriteError label to + errors that meet the following criteria if the retryWrites option is set to + true on the client performing the relevant operation: -- When the driver receives an error response to the ``getMore`` command while attempting to iterate - the results cursor returned from the ``bulkWrite`` command, it MUST add a RetryableWriteError - label to the overall error for the bulk write if the error meets the `Error Code Criteria`_, - the retryWrites option is set to true on the client performing the operation, and the - ``bulkWrite`` is retryable (i.e. it does not contain multi:true operations). If a - RetryableWriteError label is attached, drivers MUST attempt to retry the entire ``bulkWrite`` - command and MUST NOT attempt to retry the ``getMore`` that failed. - -To understand why the driver should only add the RetryableWriteError label to an -error when the retryWrites option is true on the MongoClient performing the -operation, see `Why does the driver only add the RetryableWriteError label to -errors that occur on a MongoClient with retryWrites set to true?`_ - -Note: During a retryable write operation on a sharded cluster, mongos may retry -the operation internally, in which case it will not add a RetryableWriteError -label to any error that occurs after those internal retries to prevent excessive -retrying. - -For more information about error labels, see the `Transactions specification`_. - -.. _Error Handling: ../server-discovery-and-monitoring/server-discovery-and-monitoring.rst#error-handling -.. _Transactions specification: ../transactions/transactions.md#error-labels - -Error Code Criteria -""""""""""""""""""" - -The following errors are eligible for attaching a RetryableWriteError label when evaluating based -on error code: - - - a mongod or mongos response with any of the following error codes in the top-level ``code`` - field: + - a mongod or mongos response with any the following error codes in the + top-level ``code`` field: .. list-table:: :header-rows: 1 @@ -318,16 +289,31 @@ on error code: - a mongod response with any of the previously listed codes in the ``writeConcernError.code`` field. -Drivers MUST NOT add a RetryableWriteError label based on the following: + Drivers MUST NOT add a RetryableWriteError label based on the following: - any ``writeErrors[].code`` fields in a mongod or mongos response - the ``writeConcernError.code`` field in a mongos response -The criteria for retryable errors is similar to the discussion in the SDAM -spec's section on `Error Handling`_, but includes additional error codes. See -`What do the additional error codes mean?`_ for the reasoning behind these -additional errors. + The criteria for retryable errors is similar to the discussion in the SDAM + spec's section on `Error Handling`_, but includes additional error codes. See + `What do the additional error codes mean?`_ for the reasoning behind these + additional errors. + +To understand why the driver should only add the RetryableWriteError label to an +error when the retryWrites option is true on the MongoClient performing the +operation, see `Why does the driver only add the RetryableWriteError label to +errors that occur on a MongoClient with retryWrites set to true?`_ + +Note: During a retryable write operation on a sharded cluster, mongos may retry +the operation internally, in which case it will not add a RetryableWriteError +label to any error that occurs after those internal retries to prevent excessive +retrying. + +For more information about error labels, see the `Transactions specification`_. + +.. _Error Handling: ../server-discovery-and-monitoring/server-discovery-and-monitoring.rst#error-handling +.. _Transactions specification: ../transactions/transactions.md#error-labels Generating Transaction IDs ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/source/retryable-writes/tests/README.rst b/source/retryable-writes/tests/README.rst index 76dc275896..7c03d23d6f 100644 --- a/source/retryable-writes/tests/README.rst +++ b/source/retryable-writes/tests/README.rst @@ -541,60 +541,6 @@ and sharded clusters. 7. Disable the fail point on ``s0``. -#. Test that a failed ``getMore`` causes a retry of the ``bulkWrite`` command. - - This test must only be run against replica sets and sharded clusters on server versions 8.0+. - - 1. Create a client (referred to as ``client``) with command monitoring enabled. If testing - against a sharded deployment, be sure to connect to only a single mongos. - - 2. Perform a ``hello`` on ``client`` and record the ``maxBsonObjectSize`` value in the response. - - 3. Create a collection (referred to as ``collection``) with the namespace "db.coll". Drop - ``collection``. - - 4. Enable the following failpoint:: - - { - configureFailPoint: "failCommand", - mode: { times: 1 }, - data: { - failCommands: ["getMore"], - errorCode: 6 - } - } - - 5. Create the following list of write models (referred to as ``models``):: - - [ - ReplaceOneModel { - namespace: namespace, - filter: { _id: "a".repeat(maxBsonObjectSize / 2) } - replacement: { x: 1 }, - upsert: true - }, - ReplaceOneModel { - namespace: namespace, - filter: { _id: "b".repeat(maxBsonObjectSize / 2) } - replacement: { x: 1 }, - upsert: true - } - ] - - 6. Execute ``bulkWrite()`` on ``client`` with ``models`` and ``verboseResults`` set to true. - Assert that the bulk write succeeds. - - 7. Assert that the following sequence of command events occurred: - - - CommandStartedEvent for ``bulkWrite`` - - CommandSucceededEvent for ``bulkWrite`` - - CommandStartedEvent for ``getMore`` - - CommandFailedEvent for ``getMore`` - - CommandStartedEvent for ``bulkWrite`` - - CommandSucceededEvent for ``bulkWrite`` - - CommandStartedEvent for ``getMore`` - - CommandSucceededEvent for ``getMore`` - Changelog ========= From 5993389215ac7bcc158d5fa07cac246997ce3bdd Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 24 Apr 2024 14:48:10 -0600 Subject: [PATCH 72/90] add future work section, rewrite size batch splitting rules --- source/crud/bulk-write.md | 76 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 94c9f2551c..a043e1e9f2 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -625,11 +625,57 @@ limits defined in the #### Unencrypted bulk writes When `ops` and `nsInfo` are provided as document sequences, drivers MUST ensure that the total size -of the `OP_MSG` built for each `bulkWrite` command does not exceed `maxMessageSizeBytes`. Some -drivers may perform batch-splitting prior to constructing the full `OP_MSG` to be sent to the -server. In this case, drivers MAY use `maxMessageSizeBytes - 16,384` as the upper bound for the -combined number of bytes in `ops` and `nsInfo`. 16KiB is subtracted as an approximate overhead -allowance to accommodate for the bytes in the rest of the message. +of the `OP_MSG` built for each `bulkWrite` command does not exceed `maxMessageSizeBytes`. + +The upper bound for the size of an `OP_MSG` includes opcode-related bytes (e.g. the `OP_MSG` +header) and operation-agnostic command field bytes (e.g. `txnNumber`, `lsid`). Drivers MUST limit +the combined size of the `bulkWrite`-specific command fields, `ops` document sequence, and `nsInfo` +document sequence to `maxMessageSizeBytes - 1000` to account for this overhead. The following +pseudocode demonstrates how to apply this limit in batch-splitting logic: + +``` +MESSAGE_OVERHEAD_BYTES = 1000 + +bulkWriteCommand = Document { "bulkWrite": 1 } +bulkWriteCommand.appendOptions(bulkWriteOptions) + +maxOpsNsInfoBytes = MESSAGE_OVERHEAD_BYTES - bulkWriteCommand.numBytes() + +while writeModels.hasNext() { + ops = DocumentSequence {} + nsInfo = DocumentSequence {} + loop { + if !writeModels.hasNext() { + break + } + model = writeModels.next() + + modelDoc = writeModel.toOpsDoc() + bytesAdded = modelDoc.numBytes() + + nsInfoDoc = null + if !nsInfo.contains(model.namespace) { + nsInfoDoc = model.namespace.toNsInfoDoc() + bytesAdded += nsInfoDoc.numBytes() + } + + newSize = ops.numBytes() + nsInfo.numBytes() + bytesAdded + if newSize > maxOpsNsInfoBytes { + break + } else { + ops.push(modelDoc) + if nsInfoDoc != null { + nsInfo.push(nsInfoDoc) + } + } + } + + // construct and send OP_MSG +} +``` + +See [this Q&A entry](#how-was-the-op_msg-overhead-allowance-determined) for more details on how the +overhead allowance was determined. ## Handling the `bulkWrite` Server Response @@ -764,6 +810,13 @@ messages. This means that all command fields must be embedded within the command specify `ops` and `nsInfo` as document sequences (`OP_MSG` payload type 1) for encrypted bulk writes. +### Retry `bulkWrite` when `getMore` fails with a retryable error + +When a `getMore` fails with a retryable error when attempting to iterate the results cursor, +drivers could retry the entire `bulkWrite` command to receive a fresh cursor and retry iteration. +This work was omitted to minimize the scope of the initial implementation and testing of the new +bulk write API, but may be revisited in the future. + ## Q&A ### Why are we adding a new bulk write API rather than updating the `MongoCollection.bulkWrite` implementation? @@ -828,6 +881,19 @@ allowed for namespaces given the values for these limits at the time of writing Providing `nsInfo` as a document sequence reduces the likelihood that a driver would need to batch split a user's bulk write in this scenario. +### How was the `OP_MSG` overhead allowance determined? + +The Command Batching [Total Message Size](#total-message-size) section uses a 1000 byte overhead +allowance to approximate the number of non-`bulkWrite`-specific bytes contained in an `OP_MSG` sent +for a `bulkWrite` batch. This number was determined by constructing `OP_MSG`s with various fields +attached to the command, including `startTransaction`, `autocommit`, and `apiVersion`. Additional +room was allocated to allow for future additions to the `OP_MSG` structure or the introduction of +new command-agnostic fields. + +Drivers are required to use this value even if they are capable of determining the exact size of +the message prior to batch-splitting to standardize implementations across drivers and simplify +batch-splitting testing. + ## **Changelog** - TODO: Bulk write specification created. From ccb03ed5bb1c131f5dc695231033bcb99d028b23 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 26 Apr 2024 11:30:20 -0600 Subject: [PATCH 73/90] add ns batch split test --- source/crud/bulk-write.md | 8 +-- source/crud/tests/README.md | 132 ++++++++++++++++++++++++++++++++---- 2 files changed, 121 insertions(+), 19 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index a043e1e9f2..67fe60c7b7 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -629,9 +629,9 @@ of the `OP_MSG` built for each `bulkWrite` command does not exceed `maxMessageSi The upper bound for the size of an `OP_MSG` includes opcode-related bytes (e.g. the `OP_MSG` header) and operation-agnostic command field bytes (e.g. `txnNumber`, `lsid`). Drivers MUST limit -the combined size of the `bulkWrite`-specific command fields, `ops` document sequence, and `nsInfo` -document sequence to `maxMessageSizeBytes - 1000` to account for this overhead. The following -pseudocode demonstrates how to apply this limit in batch-splitting logic: +the combined size of the `bulkWrite` command document (excluding command-agnostic fields), `ops` +document sequence, and `nsInfo` document sequence to `maxMessageSizeBytes - 1000` to account for this +overhead. The following pseudocode demonstrates how to apply this limit in batch-splitting logic: ``` MESSAGE_OVERHEAD_BYTES = 1000 @@ -639,7 +639,7 @@ MESSAGE_OVERHEAD_BYTES = 1000 bulkWriteCommand = Document { "bulkWrite": 1 } bulkWriteCommand.appendOptions(bulkWriteOptions) -maxOpsNsInfoBytes = MESSAGE_OVERHEAD_BYTES - bulkWriteCommand.numBytes() +maxOpsNsInfoBytes = maxMessageSizeBytes - (MESSAGE_OVERHEAD_BYTES + bulkWriteCommand.numBytes()) while writeModels.hasNext() { ops = DocumentSequence {} diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 601d456f65..9e3da97efa 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -555,48 +555,150 @@ Construct as list of write models (referred to as `models`) with the one `model` Call `MongoClient.bulkWrite` with `models` and `BulkWriteOptions.writeConcern` set to an unacknowledged write concern. Expect a client-side error due the size. +`secondEvent.command.nsInfo` is "db.coll1". -### 11. `MongoClient.bulkWrite` excludes namespaces from other batches +### 11. `MongoClient.bulkWrite` batch splits when the addition of a new namespace exceeds the maximum message size -Test that `MongoClient.bulkWrite` only includes the namespaces used for the operations in a single batch. +Test that `MongoClient.bulkWrite` batch splits a bulk write when the addition of a new namespace to `nsInfo` +causes the size of the message to exceed `maxMessageSizeBytes - 1000`. This test must only be run on 8.0+ servers. Construct a `MongoClient` (referred to as `client`) with [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe -CommandStartedEvents. Perform a `hello` command using `client` and record the `maxWriteBatchSize` value from the -response. +CommandStartedEvents. Perform a `hello` command using `client` and record the `maxMessageSizeBytes` value contained +in the response. -Create the following write model (referred to as `model`): +Calculate the following values: + +``` +opsBytes = maxMessageSizeBytes - 1122 +numModels = opsBytes / maxBsonObjectSize +remainderBytes = opsBytes % maxBsonObjectSize +``` + +Construct the following write model (referred to as `firstModel`): ```json InsertOne { "namespace": "db.coll", - "document": { "a": "b" } + "document": { "a": "b".repeat(maxBsonObjectSize - 57) } } ``` -Create a list of write models containing `model` repeated `maxWriteBatchSize` times (referred to as `models`). +Create a list of write models (referred to as `models`) with `firstModel` repeated `numModels` times. -Append the following write model to `models`: +If `remainderBytes` is greater than or equal to 217, add 1 to `numModels` and append the following write model +to `models`: ```json InsertOne { - "namespace": "db.coll1", - "document": { "a": "b" } + "namespace": "db.coll", + "document": { "a": "b".repeat(remainderBytes - 57) } } ``` +Construct the following namespace (referred to as `namespace`): + +``` +"db." + "c".repeat(200) +``` + +Create the following write model (referred to as `secondModel`): + +```json +InsertOne { + namespace: namespace, + document: { "a": "b" } +} +``` + +Append `secondModel` to `models`. + Execute `bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` (referred to as `result`). -Assert that `result.insertedCount` is equal to `maxWriteBatchSize + 1`. +Assert that `result.insertedCount` is equal to `numModels + 1`. Assert that two CommandStartedEvents were observed for the `bulkWrite` command (referred to as `firstEvent` and `secondEvent`). -Assert that the length of `firstEvent.command.nsInfo` is 1. Assert that the the namespace contained in -`firstEvent.command.nsInfo` is "db.coll". +Assert that the length of `firstEvent.command.ops` is equal to `numModels`. Assert that the length of +`firstEvent.command.nsInfo` is equal to 1. Assert that the namespace contained in `firstEvent.command.nsInfo` is +"db.coll". -Assert that the length of `secondEvent.command.nsInfo` is 1. Assert that the the namespace contained in -`secondEvent.command.nsInfo` is "db.coll1". +Assert that the length of `secondEvent.command.ops` is equal to 1. Assert that the length of +`secondEvent.command.nsInfo` is equal to 1. Assert that the namespace contained in `secondEvent.command.nsInfo` is +`namespace`. + +#### Details on size calculations + +This information is not needed to implement this prose test, but is documented for future reference. This test is +designed to work if `maxBsonObjectSize` or `maxMessageSizeBytes` changes, but will need to be updated if a required +field is added to the `bulkWrite` command or the `insert` operation document, or if the overhead `OP_MSG` allowance +is changed in the bulk write specification. + +The command document for the `bulkWrite` has the following structure and size: + +```json +{ + "bulkWrite": 1, + "errorsOnly": true, + "ordered": true +} + +Size: 43 bytes +``` + +All of the write models will create an `ops` document with the following structure and size: + +```json +{ + "insert": <0 | 1>, + "document": { "a": } +} + +Size: 57 bytes + +``` + +The `ops` document for `secondModel` has a string with one character, so it is a total of 58 bytes. + +`secondModel` will create an `nsInfo` document with the following structure and size: + +```json +{ + "ns": "db." +} + +Size: 217 bytes +``` + +`firstModel` (and the remainder model, if added) will create an `nsInfo` document with the following structure +and size: + +```json +{ + "ns": "db.coll" +} + +Size: 21 bytes +``` + +We need to fill up the rest of the message with bytes such that `secondModels`'s `ops` document will fit, +but its `nsInfo` entry won't. The following calculations are used: + +``` +# 1000 is the OP_MSG overhead required in the spec +maxTotalMessageSize = maxMessageSizeBytes - 1000 + +# bulkWrite command + first namespace entry +existingMessageBytes = 43 + 21 + +# Space to fit the second model's ops entry but not its nsInfo entry +secondModelBytes = 58 + +remainingMessageSize = maxTotalMessageSize - existingMessageBytes - secondModelBytes + +# With the actual numbers plugged in +remainingMessageSize = maxMessageSizeBytes - 1122 +``` From 834cca863bd23637b91bd1be83918a8ec9b88754 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 26 Apr 2024 11:44:42 -0600 Subject: [PATCH 74/90] too large test --- source/crud/bulk-write.md | 2 ++ source/crud/tests/README.md | 26 ++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 67fe60c7b7..a6c03c5054 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -677,6 +677,8 @@ while writeModels.hasNext() { See [this Q&A entry](#how-was-the-op_msg-overhead-allowance-determined) for more details on how the overhead allowance was determined. +Drivers MUST return an error if there is not room to add at least one operation to `ops`. + ## Handling the `bulkWrite` Server Response The server's response to `bulkWrite` has the following format: diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 9e3da97efa..83043a3b36 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -566,8 +566,8 @@ This test must only be run on 8.0+ servers. Construct a `MongoClient` (referred to as `client`) with [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe -CommandStartedEvents. Perform a `hello` command using `client` and record the `maxMessageSizeBytes` value contained -in the response. +CommandStartedEvents. Perform a `hello` command using `client` and record the following values from the response: +`maxBsonObjectSize` and `maxMessageSizeBytes`. Calculate the following values: @@ -702,3 +702,25 @@ remainingMessageSize = maxTotalMessageSize - existingMessageBytes - secondModelB # With the actual numbers plugged in remainingMessageSize = maxMessageSizeBytes - 1122 ``` + +### 12. `MongoClient.bulkWrite` returns an error if no operations can be added to `ops` + +Test that `MongoClient.bulkWrite` returns an error if an operation provided exceeds `maxMessageSizeBytes`. + +This test must only be run on 8.0+ servers. This test may be skipped by drivers that are not able to construct +arbitrarily large documents. + +Construct a `MongoClient` (referred to as `client`). Perform a `hello` command using `client` and record the +`maxMessageSizeBytes` value contained in the response. + +Construct the following write model (referred to as `model`): + +```json +InsertOne { + "namespace": "db.coll", + "document": { "a": "b".repeat(maxMessageSizeBytes) } +} +``` + +Execute `bulkWrite` on `client` with `model`. Assert that an error (referred to as `error`) is returned. +Assert that `error` is a client error. From 80b74999248381c97d8ef786331a418e2639614b Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 26 Apr 2024 11:46:25 -0600 Subject: [PATCH 75/90] language --- source/crud/tests/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 83043a3b36..8733940790 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -705,7 +705,8 @@ remainingMessageSize = maxMessageSizeBytes - 1122 ### 12. `MongoClient.bulkWrite` returns an error if no operations can be added to `ops` -Test that `MongoClient.bulkWrite` returns an error if an operation provided exceeds `maxMessageSizeBytes`. +Test that `MongoClient.bulkWrite` returns an error if an operation provided exceeds `maxMessageSizeBytes` such +that an empty `ops` payload would be sent. This test must only be run on 8.0+ servers. This test may be skipped by drivers that are not able to construct arbitrarily large documents. From 2c9702d52b55665541b787a5e738b82a961dc484 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Fri, 26 Apr 2024 11:46:25 -0600 Subject: [PATCH 76/90] language --- source/crud/tests/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 83043a3b36..8733940790 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -705,7 +705,8 @@ remainingMessageSize = maxMessageSizeBytes - 1122 ### 12. `MongoClient.bulkWrite` returns an error if no operations can be added to `ops` -Test that `MongoClient.bulkWrite` returns an error if an operation provided exceeds `maxMessageSizeBytes`. +Test that `MongoClient.bulkWrite` returns an error if an operation provided exceeds `maxMessageSizeBytes` such +that an empty `ops` payload would be sent. This test must only be run on 8.0+ servers. This test may be skipped by drivers that are not able to construct arbitrarily large documents. From 3fdd5436c640b9ff7371dc8efcd25f2ce6d698cb Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 30 Apr 2024 12:39:17 -0600 Subject: [PATCH 77/90] improve CSOT test --- .../tests/README.md | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/source/client-side-operations-timeout/tests/README.md b/source/client-side-operations-timeout/tests/README.md index be5ee48664..ff2e554d23 100644 --- a/source/client-side-operations-timeout/tests/README.md +++ b/source/client-side-operations-timeout/tests/README.md @@ -601,9 +601,7 @@ Tests in this section MUST only run against replica sets and sharded clusters wi ### 11. Multi-batch bulkWrites -This test MUST only run against standalones on server versions 8.0 and higher. The `bulkWrite` call takes an -exceedingly long time on replicasets and sharded clusters. Drivers MAY adjust the timeouts used in this test to allow -for differing bulk encoding performance. +This test MUST only run against server versions 8.0+. 1. Using `internalClient`, drop the `db.coll` collection. @@ -623,13 +621,28 @@ for differing bulk encoding performance. } ``` -3. Create a new MongoClient (referred to as `client`) with `timeoutMS=2000`. +3. Using `internalClient`, perform a `hello` command and record the `maxBsonObjectSize` and `maxMessageSizeBytes` + values in the response. + +4. Create a new MongoClient (referred to as `client`) with `timeoutMS=2000`. + +5. Create a list of write models (referred to as `models`) with the following write model repeated + (`maxMessageSizeBytes / maxBsonObjectSize + 1`) times: + + ```javascript + { + InsertOne { + namespace: "db.coll", + document: { "a": "b".repeat(maxBsonObjectSize - 500) } + } + } + ``` -4. Using `client`, insert 50 1-megabyte documents in a single `MongoClient.bulkWrite` call. +6. Call `bulkWrite` on `client` with `models`. - Expect this to fail with a timeout error. -5. Verify that two `bulkWrite` commands were executed against `db.coll` as part of the `MongoClient.bulkWrite` call. +7. Verify that two `bulkWrite` commands were executed as part of the `MongoClient.bulkWrite` call. ## Unit Tests From 7ef7c8c9a7535d3dafbe09c502f1ab2f733563aa Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 30 Apr 2024 13:25:55 -0600 Subject: [PATCH 78/90] jeremy language suggestions --- source/crud/bulk-write.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index a6c03c5054..383565251d 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -560,9 +560,9 @@ should contain only errors and omit individual results. If false, both individua successful operations and errors will be returned. This field is optional and defaults to false on the server. -`errorsOnly` corresponds to the `verboseResults` option defined on `BulkWriteOptions`. If the user -specified a value for `verboseResults`, drivers MUST define `errorsOnly` as the opposite of -`verboseResults`. If the user did not specify a value for `verboseResults`, drivers MUST define +`errorsOnly` corresponds inversely to the `verboseResults` option defined on `BulkWriteOptions`. If +the user specified a value for `verboseResults`, drivers MUST define `errorsOnly` as the opposite +of `verboseResults`. If the user did not specify a value for `verboseResults`, drivers MUST define `errorsOnly` as `true`. ### `ordered` @@ -641,11 +641,11 @@ bulkWriteCommand.appendOptions(bulkWriteOptions) maxOpsNsInfoBytes = maxMessageSizeBytes - (MESSAGE_OVERHEAD_BYTES + bulkWriteCommand.numBytes()) -while writeModels.hasNext() { +while (writeModels.hasNext()) { ops = DocumentSequence {} nsInfo = DocumentSequence {} - loop { - if !writeModels.hasNext() { + while (true) { + if (!writeModels.hasNext()) { break } model = writeModels.next() @@ -654,17 +654,17 @@ while writeModels.hasNext() { bytesAdded = modelDoc.numBytes() nsInfoDoc = null - if !nsInfo.contains(model.namespace) { + if (!nsInfo.contains(model.namespace)) { nsInfoDoc = model.namespace.toNsInfoDoc() bytesAdded += nsInfoDoc.numBytes() } newSize = ops.numBytes() + nsInfo.numBytes() + bytesAdded - if newSize > maxOpsNsInfoBytes { + if (newSize > maxOpsNsInfoBytes) { break } else { ops.push(modelDoc) - if nsInfoDoc != null { + if (nsInfoDoc != null) { nsInfo.push(nsInfoDoc) } } @@ -709,7 +709,7 @@ user or embedded in a `BulkWriteException`. Drivers MUST NOT populate the `parti Drivers MUST attempt to consume the contents of the cursor returned in the server's `bulkWrite` response before returning to the user. This is required regardless of whether the user requested verbose or summary results, as the results cursor always contains any write errors that occurred. -If the cursor contains a nonzero cursor ID, drivers MUST perform `getMore`s until the cursor has +If the cursor contains a nonzero cursor ID, drivers MUST perform `getMore` until the cursor has been exhausted. Drivers MUST use the same session used for the `bulkWrite` command for each `getMore` call. When connected to a load balancer, drivers MUST use the connection used for the `bulkWrite` command to create the cursor to ensure the same server is targeted. @@ -749,7 +749,7 @@ be recorded based on the value of `idx`. Unlike the other result types, `InsertOneResult` contains an `insertedId` field that is generated driver-side, either by recording the `_id` field present in the user's insert document or creating -and adding one. Drivers MUST only record these `insertedId`s in a `BulkWriteResult` when a +and adding one. Drivers MUST only record these `insertedId` values in a `BulkWriteResult` when a successful response for the insert operation (i.e. `{ "ok": 1, "n": 1 }`) is received in the results cursor. This ensures that drivers only report an `insertedId` when it is confirmed that the insert succeeded. @@ -887,10 +887,10 @@ split a user's bulk write in this scenario. The Command Batching [Total Message Size](#total-message-size) section uses a 1000 byte overhead allowance to approximate the number of non-`bulkWrite`-specific bytes contained in an `OP_MSG` sent -for a `bulkWrite` batch. This number was determined by constructing `OP_MSG`s with various fields -attached to the command, including `startTransaction`, `autocommit`, and `apiVersion`. Additional -room was allocated to allow for future additions to the `OP_MSG` structure or the introduction of -new command-agnostic fields. +for a `bulkWrite` batch. This number was determined by constructing `OP_MSG` messages with various +fields attached to the command, including `startTransaction`, `autocommit`, and `apiVersion`. +Additional room was allocated to allow for future additions to the `OP_MSG` structure or the +introduction of new command-agnostic fields. Drivers are required to use this value even if they are capable of determining the exact size of the message prior to batch-splitting to standardize implementations across drivers and simplify From e133ed353676ede61a9e4d98808ec0d6e7b8d822 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Tue, 30 Apr 2024 13:29:05 -0600 Subject: [PATCH 79/90] add oid to calculation info --- source/crud/tests/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 8733940790..f4d9a04f37 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -655,7 +655,10 @@ All of the write models will create an `ops` document with the following structu ```json { "insert": <0 | 1>, - "document": { "a": } + "document": { + "_id": , + "a": + } } Size: 57 bytes + From b9f39a6085a138c2f808a22810e81e3b3752c072 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 2 May 2024 09:34:57 -0600 Subject: [PATCH 80/90] update batching tests --- source/crud/tests/README.md | 100 +++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 24 deletions(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index f4d9a04f37..ac0f17fa58 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -555,7 +555,6 @@ Construct as list of write models (referred to as `models`) with the one `model` Call `MongoClient.bulkWrite` with `models` and `BulkWriteOptions.writeConcern` set to an unacknowledged write concern. Expect a client-side error due the size. -`secondEvent.command.nsInfo` is "db.coll1". ### 11. `MongoClient.bulkWrite` batch splits when the addition of a new namespace exceeds the maximum message size @@ -564,6 +563,10 @@ causes the size of the message to exceed `maxMessageSizeBytes - 1000`. This test must only be run on 8.0+ servers. +Repeat the following setup for each test case: + +### Setup + Construct a `MongoClient` (referred to as `client`) with [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe CommandStartedEvents. Perform a `hello` command using `client` and record the following values from the response: @@ -598,22 +601,49 @@ InsertOne { } ``` +Then perform the following two tests: + +#### Case 1: No batch-splitting required + +Create the following write model (referred to as `sameNamespaceModel`): + +```json +InsertOne { + "namespace": "db.coll", + "document": { "a": "b" } +} +``` + +Append `sameNamespaceModel` to `models`. + +Execute `bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` +(referred to as `result`). + +Assert that `result.insertedCount` is equal to `numModels + 1`. + +Assert that one CommandStartedEvent was observed for the `bulkWrite` command (referred to as `event`). + +Assert that the length of `event.command.ops` is `numModels + 1`. Assert that the length of `event.command.nsInfo` +is 1. Assert that the namespace contained in `event.command.nsInfo` is "db.coll". + +#### Case 2: Batch-splitting required + Construct the following namespace (referred to as `namespace`): ``` "db." + "c".repeat(200) ``` -Create the following write model (referred to as `secondModel`): +Create the following write model (referred to as `newNamespaceModel`): ```json InsertOne { - namespace: namespace, - document: { "a": "b" } + "namespace": namespace, + "document": { "a": "b" } } ``` -Append `secondModel` to `models`. +Append `newNamespaceModel` to `models`. Execute `bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` (referred to as `result`). @@ -650,7 +680,7 @@ The command document for the `bulkWrite` has the following structure and size: Size: 43 bytes ``` -All of the write models will create an `ops` document with the following structure and size: +Each write model will create an `ops` document with the following structure and size: ```json { @@ -664,52 +694,52 @@ All of the write models will create an `ops` document with the following structu Size: 57 bytes + ``` -The `ops` document for `secondModel` has a string with one character, so it is a total of 58 bytes. +The `ops` document for both `newNamespaceModel` and `sameNamespaceModel` has a string with one character, so +it is a total of 58 bytes. -`secondModel` will create an `nsInfo` document with the following structure and size: +The models using the "db.coll" namespace will create one `nsInfo` document with the following structure and size: ```json { - "ns": "db." + "ns": "db.coll" } -Size: 217 bytes +Size: 21 bytes ``` -`firstModel` (and the remainder model, if added) will create an `nsInfo` document with the following structure -and size: +`newNamespaceModel` will create an `nsInfo` document with the following structure and size: ```json { - "ns": "db.coll" + "ns": "db." } -Size: 21 bytes +Size: 217 bytes ``` -We need to fill up the rest of the message with bytes such that `secondModels`'s `ops` document will fit, -but its `nsInfo` entry won't. The following calculations are used: +We need to fill up the rest of the message with bytes such that another `ops` document will fit, but another +`nsInfo` entry will not. The following calculations are used: ``` # 1000 is the OP_MSG overhead required in the spec -maxTotalMessageSize = maxMessageSizeBytes - 1000 +maxBulkWriteBytes = maxMessageSizeBytes - 1000 # bulkWrite command + first namespace entry existingMessageBytes = 43 + 21 -# Space to fit the second model's ops entry but not its nsInfo entry -secondModelBytes = 58 +# Space to fit the last model's ops entry +lastModelBytes = 58 -remainingMessageSize = maxTotalMessageSize - existingMessageBytes - secondModelBytes +remainingBulkWriteBytes = maxBulkWriteBytes - existingMessageBytes - lastModelBytes # With the actual numbers plugged in -remainingMessageSize = maxMessageSizeBytes - 1122 +remainingBulkWriteBytes = maxMessageSizeBytes - 1122 ``` ### 12. `MongoClient.bulkWrite` returns an error if no operations can be added to `ops` Test that `MongoClient.bulkWrite` returns an error if an operation provided exceeds `maxMessageSizeBytes` such -that an empty `ops` payload would be sent. +that no operations would be sent. This test must only be run on 8.0+ servers. This test may be skipped by drivers that are not able to construct arbitrarily large documents. @@ -717,7 +747,9 @@ arbitrarily large documents. Construct a `MongoClient` (referred to as `client`). Perform a `hello` command using `client` and record the `maxMessageSizeBytes` value contained in the response. -Construct the following write model (referred to as `model`): +#### Case 1: `document` too large + +Construct the following write model (referred to as `largeDocumentModel`): ```json InsertOne { @@ -726,5 +758,25 @@ InsertOne { } ``` -Execute `bulkWrite` on `client` with `model`. Assert that an error (referred to as `error`) is returned. +Execute `bulkWrite` on `client` with `largeDocumentModel`. Assert that an error (referred to as `error`) is returned. +Assert that `error` is a client error. + +#### Case 2: `namespace` too large + +Construct the following namespace (referred to as `namespace`): + +``` +"db." + "c".repeat(maxMessageSizeBytes) +``` + +Construct the following write model (referred to as `largeNamespaceModel`): + +```json +InsertOne { + "namespace": namespace, + "document": { "a": "b" } +} +``` + +Execute `bulkWrite` on `client` with `largeNamespaceModel`. Assert that an error (referred to as `error`) is returned. Assert that `error` is a client error. From fb659afad5c1aa2a42fbcbc173dac7bbf19fc7cf Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 2 May 2024 09:41:25 -0600 Subject: [PATCH 81/90] remove auto-encryption support --- source/crud/bulk-write.md | 32 ++++++++++++-------------------- source/crud/tests/README.md | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 383565251d..db3379e13c 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -492,7 +492,7 @@ The `bulkWrite` server command has the following format: ``` Drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.rst) payload type 1) for the -`ops` and `nsInfo` fields. Drivers MUST NOT use document sequences when auto-encryption is enabled. +`ops` and `nsInfo` fields. The `bulkWrite` command is executed on the "admin" database. @@ -592,6 +592,15 @@ of the following limits is exceeded: See [SERVER-10643](https://jira.mongodb.org/browse/SERVER-10643) for more details on these size limits. +## Auto-Encryption + +If `MongoClient.bulkWrite` is called on a `MongoClient` configured with `AutoEncryptionOpts`, +drivers MUST return an error with the message: "bulkWrite does not currently support automatic +encryption". + +This is expected to be removed once [DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is +implemented. + ## Command Batching Drivers MUST accept an arbitrary number of operations as input to the `MongoClient.bulkWrite` @@ -615,17 +624,8 @@ the argument for `models`. ### Total Message Size -#### Encrypted bulk writes - -When auto-encryption is enabled, drivers MUST NOT provide the `ops` and `nsInfo` fields as document -sequences and MUST limit the size of each `bulkWrite` command according to the auto-encryption size -limits defined in the -[Client Side Encryption Specification](../client-side-encryption/client-side-encryption.rst). - -#### Unencrypted bulk writes - -When `ops` and `nsInfo` are provided as document sequences, drivers MUST ensure that the total size -of the `OP_MSG` built for each `bulkWrite` command does not exceed `maxMessageSizeBytes`. +Drivers MUST ensure that the total size of the `OP_MSG` built for each `bulkWrite` command does not +exceed `maxMessageSizeBytes`. The upper bound for the size of an `OP_MSG` includes opcode-related bytes (e.g. the `OP_MSG` header) and operation-agnostic command field bytes (e.g. `txnNumber`, `lsid`). Drivers MUST limit @@ -804,14 +804,6 @@ test format at the time of writing this specification. ## Future Work -### Support using document sequences in encrypted bulk writes - -Libmongocrypt is currently only capable of encrypting command documents, not wire protocol -messages. This means that all command fields must be embedded within the command document. When -[DRIVERS-2859](https://jira.mongodb.org/browse/DRIVERS-2859) is completed, drivers will be able to -specify `ops` and `nsInfo` as document sequences (`OP_MSG` payload type 1) for encrypted bulk -writes. - ### Retry `bulkWrite` when `getMore` fails with a retryable error When a `getMore` fails with a retryable error when attempting to iterate the results cursor, diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index ac0f17fa58..7836fd6f58 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -780,3 +780,39 @@ InsertOne { Execute `bulkWrite` on `client` with `largeNamespaceModel`. Assert that an error (referred to as `error`) is returned. Assert that `error` is a client error. + +### 13. `MongoClient.bulkWrite` returns an error if auto-encryption is configured + +This test is expected to be removed when +[DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is resolved. + +Test that `MongoClient.bulkWrite` returns an error if the client has auto-encryption configured. + +This test must only be run on 8.0+ servers. + +Construct a `MongoClient` (referred to as `client`) configured with the following `AutoEncryptionOpts`: + +```json +AutoEncryptionOpts { + "keyVaultNamespace": "db.coll", + "kmsProviders": { + "aws": { + "accessKeyId": "foo", + "secretAccessKey": "bar" + } + } +} +``` + +Construct the following write model (referred to as `model`): + +```json +InsertOne { + "namespace": "db.coll", + "document": { "a": "b" } +} +``` + +Execute `bulkWrite` on `client` with `model`. Assert that an error (referred to as `error`) is returned. +Assert that `error` is a client error containing the message: "bulkWrite does not currently support automatic +encryption". From c12cf6c40112f0e16416f597c144aba38345a8b1 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 2 May 2024 09:54:57 -0600 Subject: [PATCH 82/90] server versions --- .../etc/templates/handshakeError.yml.template | 8 ++++++++ .../tests/unified/handshakeError.json | 10 ++++++++++ .../tests/unified/handshakeError.yml | 4 ++++ .../tests/unified/mongos-pin-auto-tests.py | 13 +++++++++++-- .../tests/unified/mongos-pin-auto.json | 15 +++++++++++++++ .../tests/unified/mongos-pin-auto.yml | 6 ++++++ .../versioned-api/tests/crud-api-version-1.json | 5 +++++ source/versioned-api/tests/crud-api-version-1.yml | 2 ++ 8 files changed, 61 insertions(+), 2 deletions(-) diff --git a/source/retryable-writes/tests/etc/templates/handshakeError.yml.template b/source/retryable-writes/tests/etc/templates/handshakeError.yml.template index 3974392a6f..d9037d5b20 100644 --- a/source/retryable-writes/tests/etc/templates/handshakeError.yml.template +++ b/source/retryable-writes/tests/etc/templates/handshakeError.yml.template @@ -51,6 +51,10 @@ tests: # - Tests whether operation successfully retries the handshake and succeeds. {% for operation in operations %} - description: "{{operation.object}}.{{operation.operation_name}} succeeds after retryable handshake network error" + {%- if (operation.operation_name == 'clientBulkWrite') %} + runOnRequirements: + - minServerVersion: "8.0" # `bulkWrite` added to server 8.0 + {%- endif %} operations: - name: failPoint object: testRunner @@ -95,6 +99,10 @@ tests: commandName: {{operation.command_name}} - description: "{{operation.object}}.{{operation.operation_name}} succeeds after retryable handshake server error (ShutdownInProgress)" + {%- if (operation.operation_name == 'clientBulkWrite') %} + runOnRequirements: + - minServerVersion: "8.0" # `bulkWrite` added to server 8.0 + {%- endif %} operations: - name: failPoint object: testRunner diff --git a/source/retryable-writes/tests/unified/handshakeError.json b/source/retryable-writes/tests/unified/handshakeError.json index dc20c934dc..3c46463759 100644 --- a/source/retryable-writes/tests/unified/handshakeError.json +++ b/source/retryable-writes/tests/unified/handshakeError.json @@ -55,6 +55,11 @@ "tests": [ { "description": "client.clientBulkWrite succeeds after retryable handshake network error", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "operations": [ { "name": "failPoint", @@ -158,6 +163,11 @@ }, { "description": "client.clientBulkWrite succeeds after retryable handshake server error (ShutdownInProgress)", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "operations": [ { "name": "failPoint", diff --git a/source/retryable-writes/tests/unified/handshakeError.yml b/source/retryable-writes/tests/unified/handshakeError.yml index 3bbceab8d6..131bbf2e5c 100644 --- a/source/retryable-writes/tests/unified/handshakeError.yml +++ b/source/retryable-writes/tests/unified/handshakeError.yml @@ -51,6 +51,8 @@ tests: # - Tests whether operation successfully retries the handshake and succeeds. - description: "client.clientBulkWrite succeeds after retryable handshake network error" + runOnRequirements: + - minServerVersion: "8.0" # `bulkWrite` added to server 8.0 operations: - name: failPoint object: testRunner @@ -94,6 +96,8 @@ tests: commandName: bulkWrite - description: "client.clientBulkWrite succeeds after retryable handshake server error (ShutdownInProgress)" + runOnRequirements: + - minServerVersion: "8.0" # `bulkWrite` added to server 8.0 operations: - name: failPoint object: testRunner diff --git a/source/transactions/tests/unified/mongos-pin-auto-tests.py b/source/transactions/tests/unified/mongos-pin-auto-tests.py index ab1dc72cb4..ad2aeabd17 100644 --- a/source/transactions/tests/unified/mongos-pin-auto-tests.py +++ b/source/transactions/tests/unified/mongos-pin-auto-tests.py @@ -318,7 +318,11 @@ def create_pin_test(op_name, error_name): error_data = NON_TRANSIENT_ERRORS[error_name] if op_name.startswith('bulkWrite'): op_name = 'bulkWrite' - return TEMPLATE.format(**locals()) + test = TEMPLATE.format(**locals()) + if op_name == 'clientBulkWrite': + test += ' runOnRequirements:\n' + test += ' - minServerVersion: "8.0" # `bulkWrite` added to server 8.0"\n' + return test def create_unpin_test(op_name, error_name): @@ -329,7 +333,12 @@ def create_unpin_test(op_name, error_name): error_data = TRANSIENT_ERRORS[error_name] if op_name.startswith('bulkWrite'): op_name = 'bulkWrite' - return TEMPLATE.format(**locals()) + test = TEMPLATE.format(**locals()) + if op_name == 'clientBulkWrite': + test += ' runOnRequirements:\n' + test += ' - minServerVersion: "8.0" # `bulkWrite` added to server 8.0"\n' + return test + tests = [] diff --git a/source/transactions/tests/unified/mongos-pin-auto.json b/source/transactions/tests/unified/mongos-pin-auto.json index e3b1e7ccd6..27db520401 100644 --- a/source/transactions/tests/unified/mongos-pin-auto.json +++ b/source/transactions/tests/unified/mongos-pin-auto.json @@ -2095,6 +2095,11 @@ } ] } + ], + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } ] }, { @@ -5360,6 +5365,11 @@ } ] } + ], + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } ] }, { @@ -5453,6 +5463,11 @@ } ] } + ], + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } ] } ] diff --git a/source/transactions/tests/unified/mongos-pin-auto.yml b/source/transactions/tests/unified/mongos-pin-auto.yml index 086e6a4f01..a80dd62031 100644 --- a/source/transactions/tests/unified/mongos-pin-auto.yml +++ b/source/transactions/tests/unified/mongos-pin-auto.yml @@ -703,6 +703,8 @@ tests: - *assertSessionPinned - *abortTransaction outcome: *outcome + runOnRequirements: + - minServerVersion: "8.0" # `bulkWrite` added to server 8.0" - description: unpin after transient connection error on insertOne insert operations: @@ -1669,6 +1671,8 @@ tests: - *assertSessionUnpinned - *abortTransaction outcome: *outcome + runOnRequirements: + - minServerVersion: "8.0" # `bulkWrite` added to server 8.0" - description: unpin after transient ShutdownInProgress error on clientBulkWrite bulkWrite operations: @@ -1697,4 +1701,6 @@ tests: - *assertSessionUnpinned - *abortTransaction outcome: *outcome + runOnRequirements: + - minServerVersion: "8.0" # `bulkWrite` added to server 8.0" diff --git a/source/versioned-api/tests/crud-api-version-1.json b/source/versioned-api/tests/crud-api-version-1.json index d959d25503..fe668620f8 100644 --- a/source/versioned-api/tests/crud-api-version-1.json +++ b/source/versioned-api/tests/crud-api-version-1.json @@ -429,6 +429,11 @@ }, { "description": "client bulkWrite appends declared API version", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "operations": [ { "name": "clientBulkWrite", diff --git a/source/versioned-api/tests/crud-api-version-1.yml b/source/versioned-api/tests/crud-api-version-1.yml index 57f1b1c24f..cb9b45e57b 100644 --- a/source/versioned-api/tests/crud-api-version-1.yml +++ b/source/versioned-api/tests/crud-api-version-1.yml @@ -158,6 +158,8 @@ tests: <<: *expectedApiVersion - description: "client bulkWrite appends declared API version" + runOnRequirements: + - minServerVersion: "8.0" # `bulkWrite` added to server 8.0 operations: - name: clientBulkWrite object: *client From a3d1652e7e26c7911a8c2223695c613e53df198f Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Thu, 2 May 2024 10:08:37 -0600 Subject: [PATCH 83/90] update csot test formatting --- source/client-side-operations-timeout/tests/README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/client-side-operations-timeout/tests/README.md b/source/client-side-operations-timeout/tests/README.md index ff2e554d23..22006b6b79 100644 --- a/source/client-side-operations-timeout/tests/README.md +++ b/source/client-side-operations-timeout/tests/README.md @@ -629,12 +629,10 @@ This test MUST only run against server versions 8.0+. 5. Create a list of write models (referred to as `models`) with the following write model repeated (`maxMessageSizeBytes / maxBsonObjectSize + 1`) times: - ```javascript - { - InsertOne { - namespace: "db.coll", - document: { "a": "b".repeat(maxBsonObjectSize - 500) } - } + ```json + InsertOne { + "namespace": "db.coll", + "document": { "a": "b".repeat(maxBsonObjectSize - 500) } } ``` From d23399ce3b59dfb4a93b11930a0c4857f41434db Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 8 May 2024 09:44:54 -0600 Subject: [PATCH 84/90] 1.20 -> 1.21 --- .../tests/unified/client-bulkWrite-clientErrors.json | 2 +- .../tests/unified/client-bulkWrite-clientErrors.yml | 2 +- .../tests/unified/client-bulkWrite-serverErrors.json | 2 +- .../tests/unified/client-bulkWrite-serverErrors.yml | 2 +- .../unified-test-format/{schema-1.20.json => schema-1.21.json} | 0 source/unified-test-format/unified-test-format.md | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename source/unified-test-format/{schema-1.20.json => schema-1.21.json} (100%) diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.json index 64b1988ffd..e2c0fb9c0a 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.json +++ b/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite retryable writes with client errors", - "schemaVersion": "1.20", + "schemaVersion": "1.21", "runOnRequirements": [ { "minServerVersion": "8.0", diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.yml index d1b08ec37e..85696e89db 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.yml +++ b/source/retryable-writes/tests/unified/client-bulkWrite-clientErrors.yml @@ -1,5 +1,5 @@ description: "client bulkWrite retryable writes with client errors" -schemaVersion: "1.20" +schemaVersion: "1.21" runOnRequirements: - minServerVersion: "8.0" topologies: diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json index f9812241f3..4a0b210eb5 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite retryable writes", - "schemaVersion": "1.20", + "schemaVersion": "1.21", "runOnRequirements": [ { "minServerVersion": "8.0", diff --git a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml index d77e491a99..23d2c622ee 100644 --- a/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml +++ b/source/retryable-writes/tests/unified/client-bulkWrite-serverErrors.yml @@ -1,5 +1,5 @@ description: "client bulkWrite retryable writes" -schemaVersion: "1.20" +schemaVersion: "1.21" runOnRequirements: - minServerVersion: "8.0" topologies: diff --git a/source/unified-test-format/schema-1.20.json b/source/unified-test-format/schema-1.21.json similarity index 100% rename from source/unified-test-format/schema-1.20.json rename to source/unified-test-format/schema-1.21.json diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 0bef99c1ba..5f539d0dd4 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -3443,7 +3443,7 @@ other specs *and* collating spec changes developed in parallel or during the sam ## Changelog -- TODO: **Schema version 1.20.**\ +- 2024-05-08: **Schema version 1.21.**\ Add `writeErrors` and `writeConcernErrors` field to `expectedError` for the client-level bulk write API. - 2024-02-23: Require test runners to gossip cluster time from internal MongoClient to each session entity. From 91bec492e156007c900d12895774a3b51458c56d Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 8 May 2024 09:46:53 -0600 Subject: [PATCH 85/90] save files --- source/crud/tests/unified/client-bulkWrite-errors.json | 2 +- source/crud/tests/unified/client-bulkWrite-errors.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/crud/tests/unified/client-bulkWrite-errors.json b/source/crud/tests/unified/client-bulkWrite-errors.json index 0c38499732..9f17f85331 100644 --- a/source/crud/tests/unified/client-bulkWrite-errors.json +++ b/source/crud/tests/unified/client-bulkWrite-errors.json @@ -1,6 +1,6 @@ { "description": "client bulkWrite errors", - "schemaVersion": "1.20", + "schemaVersion": "1.21", "runOnRequirements": [ { "minServerVersion": "8.0" diff --git a/source/crud/tests/unified/client-bulkWrite-errors.yml b/source/crud/tests/unified/client-bulkWrite-errors.yml index 6d50122295..3a420f1429 100644 --- a/source/crud/tests/unified/client-bulkWrite-errors.yml +++ b/source/crud/tests/unified/client-bulkWrite-errors.yml @@ -1,5 +1,5 @@ description: "client bulkWrite errors" -schemaVersion: "1.20" +schemaVersion: "1.21" runOnRequirements: - minServerVersion: "8.0" From cd64072da7ce26d61ceae4dc661843861abae169 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 8 May 2024 14:24:07 -0600 Subject: [PATCH 86/90] fix lint errors --- .../tests/README.md | 5 +- source/crud/bulk-write.md | 404 ++++++++---------- source/crud/tests/README.md | 83 ++-- source/index.md | 1 + source/retryable-writes/retryable-writes.md | 12 +- .../unified-test-format.md | 55 +-- 6 files changed, 260 insertions(+), 300 deletions(-) diff --git a/source/client-side-operations-timeout/tests/README.md b/source/client-side-operations-timeout/tests/README.md index 22006b6b79..a960c2de21 100644 --- a/source/client-side-operations-timeout/tests/README.md +++ b/source/client-side-operations-timeout/tests/README.md @@ -598,7 +598,6 @@ Tests in this section MUST only run against replica sets and sharded clusters wi 1. `command_started` and `command_failed` events for an `insert` command. 2. `command_started` and `command_failed` events for an `abortTransaction` command. - ### 11. Multi-batch bulkWrites This test MUST only run against server versions 8.0+. @@ -621,8 +620,8 @@ This test MUST only run against server versions 8.0+. } ``` -3. Using `internalClient`, perform a `hello` command and record the `maxBsonObjectSize` and `maxMessageSizeBytes` - values in the response. +3. Using `internalClient`, perform a `hello` command and record the `maxBsonObjectSize` and `maxMessageSizeBytes` values + in the response. 4. Create a new MongoClient (referred to as `client`) with `timeoutMS=2000`. diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 9dfd0b23cc..0e047d4de1 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -5,23 +5,19 @@ ## Abstract -This specification defines the driver API for the `bulkWrite` server command introduced in MongoDB -8.0. The API defined in this specification allows users to perform insert, update, and delete -operations against mixed namespaces in a minimized number of round trips, and to receive detailed -results for each operation performed. This API is distinct from the -[collection-level bulkWrite method](../crud/crud.md#insert-update-replace-delete-and-bulk-writes) -defined in the CRUD specification and the -[deprecated bulk write specification](../driver-bulk-update.rst). +This specification defines the driver API for the `bulkWrite` server command introduced in MongoDB 8.0. The API defined +in this specification allows users to perform insert, update, and delete operations against mixed namespaces in a +minimized number of round trips, and to receive detailed results for each operation performed. This API is distinct from +the [collection-level bulkWrite method](../crud/crud.md#insert-update-replace-delete-and-bulk-writes) defined in the +CRUD specification and the [deprecated bulk write specification](../driver-bulk-update.rst). ## Specification > [!NOTE] -> -> The `BulkWriteOptions`, `BulkWriteResult`, and `BulkWriteException` types defined in this -> specification are similar to those used for the `MongoCollection.bulkWrite` method. Statically -> typed drivers MUST NOT reuse their existing definitions for these types for the -> `MongoClient.bulkWrite` API and MUST introduce new types. If naming conflicts arise, drivers -> SHOULD prepend "Client" to the new type names (e.g. `ClientBulkWriteOptions`). +> The `BulkWriteOptions`, `BulkWriteResult`, and `BulkWriteException` types defined in this specification are similar to +> those used for the `MongoCollection.bulkWrite` method. Statically typed drivers MUST NOT reuse their existing +> definitions for these types for the `MongoClient.bulkWrite` API and MUST introduce new types. If naming conflicts +> arise, drivers SHOULD prepend "Client" to the new type names (e.g. `ClientBulkWriteOptions`). ### `MongoClient.bulkWrite` Interface @@ -218,13 +214,12 @@ class DeleteManyModel implements WriteModel { } ``` -Each write model provided to `MongoClient.bulkWrite` in the `models` parameter MUST have a -corresponding namespace that defines the collection on which the operation should be performed. -Drivers SHOULD design this pairing in whichever way is most idiomatic for its language. For -example, drivers may: +Each write model provided to `MongoClient.bulkWrite` in the `models` parameter MUST have a corresponding namespace that +defines the collection on which the operation should be performed. Drivers SHOULD design this pairing in whichever way +is most idiomatic for its language. For example, drivers may: -- Include a required `namespace` field on each `WriteModel` variant and accept a list of - `WriteModel` objects for the `models` parameter. +- Include a required `namespace` field on each `WriteModel` variant and accept a list of `WriteModel` objects for the + `models` parameter. - Accept a list of `(Namespace, WriteModel)` tuples for `models`. - Define the following pair class: @@ -246,16 +241,14 @@ Drivers MUST throw an exception if the list provided for `models` is empty. #### Update vs. replace document validation -Update documents provided in `UpdateOne` and `UpdateMany` write models are required only to contain -atomic modifiers (i.e. keys that start with "$"). Drivers MUST throw an error if an update document -is empty or if the document's first key does not start with "$". Drivers MUST rely on the server to -return an error if any other entries in the update document are not atomic modifiers. Drivers are -not required to perform validation on update pipelines. +Update documents provided in `UpdateOne` and `UpdateMany` write models are required only to contain atomic modifiers +(i.e. keys that start with "$"). Drivers MUST throw an error if an update document is empty or if the document's first +key does not start with "$". Drivers MUST rely on the server to return an error if any other entries in the update +document are not atomic modifiers. Drivers are not required to perform validation on update pipelines. -Replacement documents provided in `ReplaceOne` write models are required not to contain atomic -modifiers. Drivers MUST throw an error if a replacement document is nonempty and its first key -starts with "$". Drivers MUST rely on the server to return an error if any other entries in the -replacement document are atomic modifiers. +Replacement documents provided in `ReplaceOne` write models are required not to contain atomic modifiers. Drivers MUST +throw an error if a replacement document is nonempty and its first key starts with "$". Drivers MUST rely on the server +to return an error if any other entries in the replacement document are atomic modifiers. ### Options @@ -411,36 +404,31 @@ class DeleteResult { #### Unacknowledged results -`BulkWriteResult` has an optional `acknowledged` field to indicate whether the result was -acknowledged. This is not required to implement. Drivers should follow the guidance in the CRUD -specification [here](../crud/crud.md#write-results) to determine how to model unacknowledged -results. +`BulkWriteResult` has an optional `acknowledged` field to indicate whether the result was acknowledged. This is not +required to implement. Drivers should follow the guidance in the CRUD specification +[here](../crud/crud.md#write-results) to determine how to model unacknowledged results. #### Summary vs. verbose results -Users MUST be able to discern whether a `BulkWriteResult` contains summary or verbose results -without inspecting the value provided for `verboseResults` in `BulkWriteOptions`. Drivers MUST -implement this in one of the following ways: - -- Expose the `hasVerboseResults` field in `BulkWriteResult` as defined above. Document that - `insertResults`, `updateResults`, and `deleteResults` will be undefined when `hasVerboseResults` - is false. Raise an error if a user tries to access one of these fields when `hasVerboseResults` - is false. -- Implement the `insertResults`, `updateResults`, and `deleteResults` fields as optional types and - document that they will be unset when `verboseResults` is false. -- Introduce separate `SummaryBulkWriteResult` and `VerboseBulkWriteResult` types. - `VerboseBulkWriteResult` MUST have all of the required fields defined on `BulkWriteResult` above. - `SummaryBulkWriteResult` MUST have all of the required fields defined on `BulkWriteResult` above - except `insertResults`, `updateResults`, and `deleteResults`. +Users MUST be able to discern whether a `BulkWriteResult` contains summary or verbose results without inspecting the +value provided for `verboseResults` in `BulkWriteOptions`. Drivers MUST implement this in one of the following ways: + +- Expose the `hasVerboseResults` field in `BulkWriteResult` as defined above. Document that `insertResults`, + `updateResults`, and `deleteResults` will be undefined when `hasVerboseResults` is false. Raise an error if a user + tries to access one of these fields when `hasVerboseResults` is false. +- Implement the `insertResults`, `updateResults`, and `deleteResults` fields as optional types and document that they + will be unset when `verboseResults` is false. +- Introduce separate `SummaryBulkWriteResult` and `VerboseBulkWriteResult` types. `VerboseBulkWriteResult` MUST have all + of the required fields defined on `BulkWriteResult` above. `SummaryBulkWriteResult` MUST have all of the required + fields defined on `BulkWriteResult` above except `insertResults`, `updateResults`, and `deleteResults`. #### Individual results -The `InsertOneResult`, `UpdateResult`, and `DeleteResult` classes are the same as or similar to -types of the same name defined in the [CRUD specification](crud.md). Drivers MUST redefine these -classes if their existing result classes deviate from the definitions in this specification (e.g. -if they contain acknowledgement information, which is not applicable for individual bulk write -operations). Drivers MAY reuse their existing types for these classes if they match the ones -defined here exactly. +The `InsertOneResult`, `UpdateResult`, and `DeleteResult` classes are the same as or similar to types of the same name +defined in the [CRUD specification](crud.md). Drivers MUST redefine these classes if their existing result classes +deviate from the definitions in this specification (e.g. if they contain acknowledgement information, which is not +applicable for individual bulk write operations). Drivers MAY reuse their existing types for these classes if they match +the ones defined here exactly. ### Exception @@ -491,17 +479,16 @@ The `bulkWrite` server command has the following format: } ``` -Drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.rst) payload type 1) for the -`ops` and `nsInfo` fields. +Drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.rst) payload type 1) for the `ops` and `nsInfo` +fields. The `bulkWrite` command is executed on the "admin" database. ### Operations -The `ops` field is a list of write operation documents. The first entry in each document has -the name of the operation (i.e. "insert", "update", or "delete") as its key and the index in the -`nsInfo` array of the namespace on which the operation should be performed as its value. The -documents have the following format: +The `ops` field is a list of write operation documents. The first entry in each document has the name of the operation +(i.e. "insert", "update", or "delete") as its key and the index in the `nsInfo` array of the namespace on which the +operation should be performed as its value. The documents have the following format: #### Insert @@ -543,9 +530,8 @@ If the document to be inserted does not contain an `_id` field, drivers MUST gen ### Namespace Information -The `nsInfo` field is an array containing the namespaces on which the write operations should be -performed. Drivers MUST NOT include duplicate namespaces in this list. The documents in the -`nsInfo` array have the following format: +The `nsInfo` field is an array containing the namespaces on which the write operations should be performed. Drivers MUST +NOT include duplicate namespaces in this list. The documents in the `nsInfo` array have the following format: ```json { @@ -555,83 +541,73 @@ performed. Drivers MUST NOT include duplicate namespaces in this list. The docum ### `errorsOnly` and `verboseResults` -The `errorsOnly` field indicates whether the results cursor returned in the `bulkWrite` response -should contain only errors and omit individual results. If false, both individual results for -successful operations and errors will be returned. This field is optional and defaults to false on -the server. +The `errorsOnly` field indicates whether the results cursor returned in the `bulkWrite` response should contain only +errors and omit individual results. If false, both individual results for successful operations and errors will be +returned. This field is optional and defaults to false on the server. -`errorsOnly` corresponds inversely to the `verboseResults` option defined on `BulkWriteOptions`. If -the user specified a value for `verboseResults`, drivers MUST define `errorsOnly` as the opposite -of `verboseResults`. If the user did not specify a value for `verboseResults`, drivers MUST define -`errorsOnly` as `true`. +`errorsOnly` corresponds inversely to the `verboseResults` option defined on `BulkWriteOptions`. If the user specified a +value for `verboseResults`, drivers MUST define `errorsOnly` as the opposite of `verboseResults`. If the user did not +specify a value for `verboseResults`, drivers MUST define `errorsOnly` as `true`. ### `ordered` -The `ordered` field defines whether writes should be executed in the order in which they were -specified, and, if an error occurs, whether the server should halt execution of further writes. It -is optional and defaults to true on the server. Drivers MUST explicitly define `ordered` as `true` -in the `bulkWrite` command if a value is not specified in `BulkWriteOptions`. This is required to -avoid inconsistencies between server and driver behavior if the server default changes in the -future. +The `ordered` field defines whether writes should be executed in the order in which they were specified, and, if an +error occurs, whether the server should halt execution of further writes. It is optional and defaults to true on the +server. Drivers MUST explicitly define `ordered` as `true` in the `bulkWrite` command if a value is not specified in +`BulkWriteOptions`. This is required to avoid inconsistencies between server and driver behavior if the server default +changes in the future. ### Size Limits -The server reports a `maxBsonObjectSize` in its `hello` response. This value defines the maximum -size for documents that are inserted into the database. Documents that are sent to the server but -are not intended to be inserted into the database (e.g. command documents) have a size limit of -`maxBsonObjectSize + 16KiB`. When an acknowledged write concern is used, drivers MUST NOT perform -any checks related to these size limits and MUST rely on the server to raise an error if a limit is -exceeded. However, when an unacknowledged write concern is used, drivers MUST raise an error if one -of the following limits is exceeded: +The server reports a `maxBsonObjectSize` in its `hello` response. This value defines the maximum size for documents that +are inserted into the database. Documents that are sent to the server but are not intended to be inserted into the +database (e.g. command documents) have a size limit of `maxBsonObjectSize + 16KiB`. When an acknowledged write concern +is used, drivers MUST NOT perform any checks related to these size limits and MUST rely on the server to raise an error +if a limit is exceeded. However, when an unacknowledged write concern is used, drivers MUST raise an error if one of the +following limits is exceeded: -- The size of a document to be inserted MUST NOT exceed `maxBsonObjectSize`. This applies to the - `document` field of an `InsertOneModel` and the `replacement` field of a `ReplaceOneModel`. +- The size of a document to be inserted MUST NOT exceed `maxBsonObjectSize`. This applies to the `document` field of an + `InsertOneModel` and the `replacement` field of a `ReplaceOneModel`. - The size of an entry in the `ops` array MUST NOT exceed `maxBsonObjectSize + 16KiB`. - The size of the `bulkWrite` command document MUST NOT exceed `maxBsonObjectSize + 16KiB`. -See [SERVER-10643](https://jira.mongodb.org/browse/SERVER-10643) for more details on these size -limits. +See [SERVER-10643](https://jira.mongodb.org/browse/SERVER-10643) for more details on these size limits. ## Auto-Encryption -If `MongoClient.bulkWrite` is called on a `MongoClient` configured with `AutoEncryptionOpts`, -drivers MUST return an error with the message: "bulkWrite does not currently support automatic -encryption". +If `MongoClient.bulkWrite` is called on a `MongoClient` configured with `AutoEncryptionOpts`, drivers MUST return an +error with the message: "bulkWrite does not currently support automatic encryption". -This is expected to be removed once [DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is -implemented. +This is expected to be removed once [DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is implemented. ## Command Batching -Drivers MUST accept an arbitrary number of operations as input to the `MongoClient.bulkWrite` -method. Because the server imposes restrictions on the size of write operations, this means that a -single call to `MongoClient.bulkWrite` may require multiple `bulkWrite` commands to be sent to the -server. Drivers MUST split bulk writes into separate commands when the user's list of operations -exceeds one or more of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize` (for `OP_MSG` -payload type 0), and `maxMessageSizeBytes` (for `OP_MSG` payload type 1). Each of these values can -be retrieved from the selected server's `hello` command response. Drivers MUST merge results from -multiple batches into a single `BulkWriteResult` or `BulkWriteException` to return from -`MongoClient.bulkWrite`. +Drivers MUST accept an arbitrary number of operations as input to the `MongoClient.bulkWrite` method. Because the server +imposes restrictions on the size of write operations, this means that a single call to `MongoClient.bulkWrite` may +require multiple `bulkWrite` commands to be sent to the server. Drivers MUST split bulk writes into separate commands +when the user's list of operations exceeds one or more of these maximums: `maxWriteBatchSize`, `maxBsonObjectSize` (for +`OP_MSG` payload type 0), and `maxMessageSizeBytes` (for `OP_MSG` payload type 1). Each of these values can be retrieved +from the selected server's `hello` command response. Drivers MUST merge results from multiple batches into a single +`BulkWriteResult` or `BulkWriteException` to return from `MongoClient.bulkWrite`. -When constructing the `nsInfo` array for a `bulkWrite` batch, drivers MUST only include the -namespaces that are referenced in the `ops` array for that batch. +When constructing the `nsInfo` array for a `bulkWrite` batch, drivers MUST only include the namespaces that are +referenced in the `ops` array for that batch. ### Number of Writes -`maxWriteBatchSize` defines the total number of writes allowed in one command. Drivers MUST split a -bulk write into multiple commands if the user provides more than `maxWriteBatchSize` operations in -the argument for `models`. +`maxWriteBatchSize` defines the total number of writes allowed in one command. Drivers MUST split a bulk write into +multiple commands if the user provides more than `maxWriteBatchSize` operations in the argument for `models`. ### Total Message Size -Drivers MUST ensure that the total size of the `OP_MSG` built for each `bulkWrite` command does not -exceed `maxMessageSizeBytes`. +Drivers MUST ensure that the total size of the `OP_MSG` built for each `bulkWrite` command does not exceed +`maxMessageSizeBytes`. -The upper bound for the size of an `OP_MSG` includes opcode-related bytes (e.g. the `OP_MSG` -header) and operation-agnostic command field bytes (e.g. `txnNumber`, `lsid`). Drivers MUST limit -the combined size of the `bulkWrite` command document (excluding command-agnostic fields), `ops` -document sequence, and `nsInfo` document sequence to `maxMessageSizeBytes - 1000` to account for this -overhead. The following pseudocode demonstrates how to apply this limit in batch-splitting logic: +The upper bound for the size of an `OP_MSG` includes opcode-related bytes (e.g. the `OP_MSG` header) and +operation-agnostic command field bytes (e.g. `txnNumber`, `lsid`). Drivers MUST limit the combined size of the +`bulkWrite` command document (excluding command-agnostic fields), `ops` document sequence, and `nsInfo` document +sequence to `maxMessageSizeBytes - 1000` to account for this overhead. The following pseudocode demonstrates how to +apply this limit in batch-splitting logic: ``` MESSAGE_OVERHEAD_BYTES = 1000 @@ -674,8 +650,8 @@ while (writeModels.hasNext()) { } ``` -See [this Q&A entry](#how-was-the-op_msg-overhead-allowance-determined) for more details on how the -overhead allowance was determined. +See [this Q&A entry](#how-was-the-op_msg-overhead-allowance-determined) for more details on how the overhead allowance +was determined. Drivers MUST return an error if there is not room to add at least one operation to `ops`. @@ -701,17 +677,15 @@ The server's response to `bulkWrite` has the following format: } ``` -If any operations were successful (i.e. `nErrors` is less than the number of operations that were -sent), drivers MUST record the summary count fields in a `BulkWriteResult` to be returned to the -user or embedded in a `BulkWriteException`. Drivers MUST NOT populate the `partialResult` field in -`BulkWriteException` if no operations were successful. - -Drivers MUST attempt to consume the contents of the cursor returned in the server's `bulkWrite` -response before returning to the user. This is required regardless of whether the user requested -verbose or summary results, as the results cursor always contains any write errors that occurred. -If the cursor contains a nonzero cursor ID, drivers MUST perform `getMore` until the cursor has -been exhausted. Drivers MUST use the same session used for the `bulkWrite` command for each -`getMore` call. When connected to a load balancer, drivers MUST use the connection used for the +If any operations were successful (i.e. `nErrors` is less than the number of operations that were sent), drivers MUST +record the summary count fields in a `BulkWriteResult` to be returned to the user or embedded in a `BulkWriteException`. +Drivers MUST NOT populate the `partialResult` field in `BulkWriteException` if no operations were successful. + +Drivers MUST attempt to consume the contents of the cursor returned in the server's `bulkWrite` response before +returning to the user. This is required regardless of whether the user requested verbose or summary results, as the +results cursor always contains any write errors that occurred. If the cursor contains a nonzero cursor ID, drivers MUST +perform `getMore` until the cursor has been exhausted. Drivers MUST use the same session used for the `bulkWrite` +command for each `getMore` call. When connected to a load balancer, drivers MUST use the connection used for the `bulkWrite` command to create the cursor to ensure the same server is targeted. The documents in the results cursor have the following format: @@ -729,68 +703,63 @@ The documents in the results cursor have the following format: } ``` -If an error occurred (i.e. the value for `ok` is 0), the `code`, `errmsg`, and optionally -`errInfo` fields will be populated with details about the failure. +If an error occurred (i.e. the value for `ok` is 0), the `code`, `errmsg`, and optionally `errInfo` fields will be +populated with details about the failure. -If the write succeeded, (i.e. the value for `ok` is 1), `n`, `nModified`, and `upsertedId` will be -populated with the following values based on the type of write: +If the write succeeded, (i.e. the value for `ok` is 1), `n`, `nModified`, and `upsertedId` will be populated with the +following values based on the type of write: -| Response Field | Insert | Update | Delete | -| -------------- | ------ | ------ | ------ | -| `n` | The number of documents that were inserted. | The number of documents that matched the filter. | The number of documents that were deleted. | -| `nModified` | Not present. | The number of documents that were modified. | Not present. | -| `upserted` | Not present. | A document containing the `_id` value for the upserted document. Only present if an upsert took place. | Not present. | +| Response Field | Insert | Update | Delete | +| -------------- | ------------------------------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------ | +| `n` | The number of documents that were inserted. | The number of documents that matched the filter. | The number of documents that were deleted. | +| `nModified` | Not present. | The number of documents that were modified. | Not present. | +| `upserted` | Not present. | A document containing the `_id` value for the upserted document. Only present if an upsert took place. | Not present. | -Note that the responses do not contain information about the type of operation that was performed. -Drivers may need to maintain the user's list of write models to infer which type of result should -be recorded based on the value of `idx`. +Note that the responses do not contain information about the type of operation that was performed. Drivers may need to +maintain the user's list of write models to infer which type of result should be recorded based on the value of `idx`. ### Handling Insert Results -Unlike the other result types, `InsertOneResult` contains an `insertedId` field that is generated -driver-side, either by recording the `_id` field present in the user's insert document or creating -and adding one. Drivers MUST only record these `insertedId` values in a `BulkWriteResult` when a -successful response for the insert operation (i.e. `{ "ok": 1, "n": 1 }`) is received in the -results cursor. This ensures that drivers only report an `insertedId` when it is confirmed that the -insert succeeded. +Unlike the other result types, `InsertOneResult` contains an `insertedId` field that is generated driver-side, either by +recording the `_id` field present in the user's insert document or creating and adding one. Drivers MUST only record +these `insertedId` values in a `BulkWriteResult` when a successful response for the insert operation (i.e. +`{ "ok": 1, "n": 1 }`) is received in the results cursor. This ensures that drivers only report an `insertedId` when it +is confirmed that the insert succeeded. ## Handling Errors ### Top-Level Errors -A top-level error is any error that occurs that is not the result of a single write operation -failing or a write concern error. Examples include network errors that occur when communicating -with the server, command errors (`{ "ok": 0 }`) returned from the server, client-side errors, and -errors that occur when attempting to perform a `getMore` to retrieve results from the server. +A top-level error is any error that occurs that is not the result of a single write operation failing or a write concern +error. Examples include network errors that occur when communicating with the server, command errors (`{ "ok": 0 }`) +returned from the server, client-side errors, and errors that occur when attempting to perform a `getMore` to retrieve +results from the server. -When a top-level error is caused by a command error (i.e. an `{ "ok": 0 }` server response), -drivers MUST provide access to the raw server reply in the error returned to the user. +When a top-level error is caused by a command error (i.e. an `{ "ok": 0 }` server response), drivers MUST provide access +to the raw server reply in the error returned to the user. -When a top-level error is encountered and individual results and/or errors have already been -observed, drivers MUST embed the top-level error within a `BulkWriteException` as the `error` field -to retain this information. Otherwise, drivers MAY throw an exception containing only the top-level -error. +When a top-level error is encountered and individual results and/or errors have already been observed, drivers MUST +embed the top-level error within a `BulkWriteException` as the `error` field to retain this information. Otherwise, +drivers MAY throw an exception containing only the top-level error. -Encountering a top-level error MUST halt execution of a bulk write for both ordered and unordered -bulk writes. This means that drivers MUST NOT attempt to retrieve more responses from the cursor or -execute any further `bulkWrite` batches and MUST immediately throw an exception. If the results -cursor has not been exhausted on the server when a top-level error occurs, drivers MUST send the -`killCursors` command to attempt to close it. The result returned from the `killCursors` command -MAY be ignored. +Encountering a top-level error MUST halt execution of a bulk write for both ordered and unordered bulk writes. This +means that drivers MUST NOT attempt to retrieve more responses from the cursor or execute any further `bulkWrite` +batches and MUST immediately throw an exception. If the results cursor has not been exhausted on the server when a +top-level error occurs, drivers MUST send the `killCursors` command to attempt to close it. The result returned from the +`killCursors` command MAY be ignored. ### Write Concern Errors -Write concern errors are recorded in the `writeConcernErrors` field on `BulkWriteException`. When -a write concern error is encountered, it should not terminate execution of the bulk write for -either ordered or unordered bulk writes. However, drivers MUST throw an exception at the end of -execution if any write concern errors were observed. +Write concern errors are recorded in the `writeConcernErrors` field on `BulkWriteException`. When a write concern error +is encountered, it should not terminate execution of the bulk write for either ordered or unordered bulk writes. +However, drivers MUST throw an exception at the end of execution if any write concern errors were observed. ### Individual Write Errors -Individual write errors retrieved from the cursor are recorded in the `writeErrors` field on -`BulkWriteException`. If an individual write error is encountered during an ordered bulk write, -drivers MUST record the error in `writeErrors` and immediately throw the exception. Otherwise, -drivers MUST continue to iterate the results cursor and execute any further `bulkWrite` batches. +Individual write errors retrieved from the cursor are recorded in the `writeErrors` field on `BulkWriteException`. If an +individual write error is encountered during an ordered bulk write, drivers MUST record the error in `writeErrors` and +immediately throw the exception. Otherwise, drivers MUST continue to iterate the results cursor and execute any further +`bulkWrite` batches. ## Test Plan @@ -798,95 +767,86 @@ The majority of tests for `MongoClient.bulkWrite` are written in the [Unified Test Format](../unified-test-format/unified-test-format.md) and reside in the [CRUD unified tests directory](../crud/tests/unified/). -Additional prose tests are specified [here](../crud/tests/README.md). These tests require -constructing very large documents to test batch splitting, which is not feasible in the unified -test format at the time of writing this specification. +Additional prose tests are specified [here](../crud/tests/README.md). These tests require constructing very large +documents to test batch splitting, which is not feasible in the unified test format at the time of writing this +specification. ## Future Work ### Retry `bulkWrite` when `getMore` fails with a retryable error -When a `getMore` fails with a retryable error when attempting to iterate the results cursor, -drivers could retry the entire `bulkWrite` command to receive a fresh cursor and retry iteration. -This work was omitted to minimize the scope of the initial implementation and testing of the new -bulk write API, but may be revisited in the future. +When a `getMore` fails with a retryable error when attempting to iterate the results cursor, drivers could retry the +entire `bulkWrite` command to receive a fresh cursor and retry iteration. This work was omitted to minimize the scope of +the initial implementation and testing of the new bulk write API, but may be revisited in the future. ## Q&A ### Why are we adding a new bulk write API rather than updating the `MongoCollection.bulkWrite` implementation? -The new `bulkWrite` command is only available in MongoDB 8.0+, so it cannot function as a drop-in -replacement for the existing bulk write implementation that uses the `insert`, `update`, and -`delete` commands. Additionally, because the new `bulkWrite` command allows operations against -multiple collections and databases, `MongoClient` is a more appropriate place to expose its -functionality. +The new `bulkWrite` command is only available in MongoDB 8.0+, so it cannot function as a drop-in replacement for the +existing bulk write implementation that uses the `insert`, `update`, and `delete` commands. Additionally, because the +new `bulkWrite` command allows operations against multiple collections and databases, `MongoClient` is a more +appropriate place to expose its functionality. ### Why can't drivers reuse existing bulk write types? -This specification introduces several types that are similar to existing types used in the -`MongoCollection.bulkWrite` API. Although these types are similar now, they may diverge in the -future with the introduction of new options and features to the `bulkWrite` command. Introducing -new types also provides more clarity to users on the existing differences between the -collection-level and client-level bulk write APIs. For example, the `verboseResults` option is only -available for `MongoClient.bulkWrite`. +This specification introduces several types that are similar to existing types used in the `MongoCollection.bulkWrite` +API. Although these types are similar now, they may diverge in the future with the introduction of new options and +features to the `bulkWrite` command. Introducing new types also provides more clarity to users on the existing +differences between the collection-level and client-level bulk write APIs. For example, the `verboseResults` option is +only available for `MongoClient.bulkWrite`. ### Why are bulk write operation results returned in a cursor? -Returning results via a cursor rather than an array in the `bulkWrite` response allows full -individual results and errors to be returned without the risk of the response exceeding the -maximum BSON object size. Using a cursor also leaves open the opportunity to add `findAndModify` to -the list of supported write operations in the future. +Returning results via a cursor rather than an array in the `bulkWrite` response allows full individual results and +errors to be returned without the risk of the response exceeding the maximum BSON object size. Using a cursor also +leaves open the opportunity to add `findAndModify` to the list of supported write operations in the future. ### Why was the `verboseResults` option introduced, and why is its default `false`? -The `bulkWrite` command returns top-level summary result counts and, optionally, individual results -for each operation. Compiling the individual results server-side and consuming these results -driver-side is less performant than only recording the summary counts. We expect that most users -are not interested in the individual results of their operations and that most users will rely on -defaults, so `verboseResults` defaults to `false` to improve performance in the common case. +The `bulkWrite` command returns top-level summary result counts and, optionally, individual results for each operation. +Compiling the individual results server-side and consuming these results driver-side is less performant than only +recording the summary counts. We expect that most users are not interested in the individual results of their operations +and that most users will rely on defaults, so `verboseResults` defaults to `false` to improve performance in the common +case. ### Why should drivers send `bypassDocumentValidation: false` for `bulkWrite`? -[DRIVERS-450](https://jira.mongodb.org/browse/DRIVERS-450) introduced a requirement that drivers -only send a value for `bypassDocumentValidation` on write commands if it was specified as true. The -original motivation for this change is not documented. This specification requires that drivers -send `bypassDocumentValidation` in the `bulkWrite` command if it is set by the user in -`BulkWriteOptions`, regardless of its value. +[DRIVERS-450](https://jira.mongodb.org/browse/DRIVERS-450) introduced a requirement that drivers only send a value for +`bypassDocumentValidation` on write commands if it was specified as true. The original motivation for this change is not +documented. This specification requires that drivers send `bypassDocumentValidation` in the `bulkWrite` command if it is +set by the user in `BulkWriteOptions`, regardless of its value. -Explicitly defining `bypassDocumentValidation: false` aligns with the server's default to perform -schema validation and thus has no effect. However, checking the value of an option that the user -specified and omitting it from the command document if it matches the server's default creates -unnecessary work for drivers. Always sending the user's specified value also safeguards against the -unlikely event that the server changes the default value for `bypassDocumentValidation` in the -future. +Explicitly defining `bypassDocumentValidation: false` aligns with the server's default to perform schema validation and +thus has no effect. However, checking the value of an option that the user specified and omitting it from the command +document if it matches the server's default creates unnecessary work for drivers. Always sending the user's specified +value also safeguards against the unlikely event that the server changes the default value for +`bypassDocumentValidation` in the future. ### Why is providing access to the raw server response when a command error occurs required? -This allows users to access new error fields that the server may add in the future without needing -to upgrade their driver version. See [DRIVERS-2385](https://jira.mongodb.org/browse/DRIVERS-2385) -for more details. +This allows users to access new error fields that the server may add in the future without needing to upgrade their +driver version. See [DRIVERS-2385](https://jira.mongodb.org/browse/DRIVERS-2385) for more details. ### Why are drivers required to send `nsInfo` as a document sequence? -`nsInfo` could exceed `maxBsonObjectSize` if a user is doing `maxWriteBatchSize` operations, each -operation is on a unique namespace, and each namespace is near the +`nsInfo` could exceed `maxBsonObjectSize` if a user is doing `maxWriteBatchSize` operations, each operation is on a +unique namespace, and each namespace is near the [maximum length](https://www.mongodb.com/docs/manual/reference/limits/#mongodb-limit-Restriction-on-Collection-Names) -allowed for namespaces given the values for these limits at the time of writing this specification. -Providing `nsInfo` as a document sequence reduces the likelihood that a driver would need to batch -split a user's bulk write in this scenario. +allowed for namespaces given the values for these limits at the time of writing this specification. Providing `nsInfo` +as a document sequence reduces the likelihood that a driver would need to batch split a user's bulk write in this +scenario. ### How was the `OP_MSG` overhead allowance determined? -The Command Batching [Total Message Size](#total-message-size) section uses a 1000 byte overhead -allowance to approximate the number of non-`bulkWrite`-specific bytes contained in an `OP_MSG` sent -for a `bulkWrite` batch. This number was determined by constructing `OP_MSG` messages with various -fields attached to the command, including `startTransaction`, `autocommit`, and `apiVersion`. -Additional room was allocated to allow for future additions to the `OP_MSG` structure or the -introduction of new command-agnostic fields. +The Command Batching [Total Message Size](#total-message-size) section uses a 1000 byte overhead allowance to +approximate the number of non-`bulkWrite`-specific bytes contained in an `OP_MSG` sent for a `bulkWrite` batch. This +number was determined by constructing `OP_MSG` messages with various fields attached to the command, including +`startTransaction`, `autocommit`, and `apiVersion`. Additional room was allocated to allow for future additions to the +`OP_MSG` structure or the introduction of new command-agnostic fields. -Drivers are required to use this value even if they are capable of determining the exact size of -the message prior to batch-splitting to standardize implementations across drivers and simplify -batch-splitting testing. +Drivers are required to use this value even if they are capable of determining the exact size of the message prior to +batch-splitting to standardize implementations across drivers and simplify batch-splitting testing. ## **Changelog** diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index b7b7c8a892..41070d02ef 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -84,14 +84,14 @@ InsertOne: { } ``` -Construct a list of write models (referred to as `models`) with `model` repeated `maxWriteBatchSize + 1` times. -Execute `bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` -with an `insertedCount` value of `maxWriteBatchSize + 1`. +Construct a list of write models (referred to as `models`) with `model` repeated `maxWriteBatchSize + 1` times. Execute +`bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` with an +`insertedCount` value of `maxWriteBatchSize + 1`. -Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEvent`) were observed for the `bulkWrite` command. -Assert that the length of `firstEvent.command.ops` is `maxWriteBatchSize`. Assert that the length of `secondEvent.command.ops` -is 1. If the driver exposes `operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to -`secondEvent.operationId`. +Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEvent`) were observed for the `bulkWrite` +command. Assert that the length of `firstEvent.command.ops` is `maxWriteBatchSize`. Assert that the length of +`secondEvent.command.ops` is 1. If the driver exposes `operationId`s in its CommandStartedEvents, assert that +`firstEvent.operationId` is equal to `secondEvent.operationId`. ### 4. `MongoClient.bulkWrite` batch splits when an `ops` payload exceeds `maxMessageSizeBytes` @@ -125,12 +125,12 @@ Use the following calculation to determine the number of inserts that should be provided to `MongoClient.bulkWrite` will require multiple `bulkWrite` commands to be sent to the server. Construct as list of write models (referred to as `models`) with `model` repeated `numModels` times. Then execute -`bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` with -an `insertedCount` value of `numModels`. +`bulkWrite` on `client` with `models`. Assert that the bulk write succeeds and returns a `BulkWriteResult` with an +`insertedCount` value of `numModels`. Assert that two CommandStartedEvents (referred to as `firstEvent` and `secondEvent`) were observed. Assert that the -length of `firstEvent.command.ops` is `numModels - 1`. Assert that the length of `secondEvent.command.ops` is 1. If -the driver exposes `operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to +length of `firstEvent.command.ops` is `numModels - 1`. Assert that the length of `secondEvent.command.ops` is 1. If the +driver exposes `operationId`s in its CommandStartedEvents, assert that `firstEvent.operationId` is equal to `secondEvent.operationId`. ### 5. `MongoClient.bulkWrite` collects `WriteConcernError`s across batches @@ -141,8 +141,8 @@ This test must only be run on 8.0+ servers. Construct a `MongoClient` (referred to as `client`) with `retryWrites: false` configured and [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.rst) enabled to observe -CommandStartedEvents. Perform a `hello` command using `client` and record the `maxWriteBatchSize` value contained -in the response. Then, configure the following fail point with `client`: +CommandStartedEvents. Perform a `hello` command using `client` and record the `maxWriteBatchSize` value contained in the +response. Then, configure the following fail point with `client`: ```json { @@ -167,9 +167,9 @@ InsertOne: { } ``` -Construct a list of write models (referred to as `models`) with `model` repeated `maxWriteBatchSize + 1` times. -Execute `bulkWrite` on `client` with `models`. Assert that the bulk write fails and returns a `BulkWriteError` -(referred to as `error`). +Construct a list of write models (referred to as `models`) with `model` repeated `maxWriteBatchSize + 1` times. Execute +`bulkWrite` on `client` with `models`. Assert that the bulk write fails and returns a `BulkWriteError` (referred to as +`error`). Assert that `error.writeConcernErrors` has a length of 2. @@ -180,8 +180,7 @@ Assert that two CommandStartedEvents were observed for the `bulkWrite` command. ### 6. `MongoClient.bulkWrite` handles individual `WriteError`s across batches -Test that `MongoClient.bulkWrite` handles individual write errors across batches for ordered and unordered bulk -writes. +Test that `MongoClient.bulkWrite` handles individual write errors across batches for ordered and unordered bulk writes. This test must only be run on 8.0+ servers. @@ -216,8 +215,8 @@ Construct a list of write models (referred to as `models`) with `model` repeated Test that an unordered bulk write collects `WriteError`s across batches. -Execute `bulkWrite` on `client` with `models` and `ordered` set to false. Assert that the bulk write fails and returns -a `BulkWriteError` (referred to as `unorderedError`). +Execute `bulkWrite` on `client` with `models` and `ordered` set to false. Assert that the bulk write fails and returns a +`BulkWriteError` (referred to as `unorderedError`). Assert that `unorderedError.writeErrors` has a length of `maxWriteBatchSize + 1`. @@ -227,8 +226,8 @@ Assert that two CommandStartedEvents were observed for the `bulkWrite` command. Test that an ordered bulk write does not execute further batches when a `WriteError` occurs. -Execute `bulkWrite` on `client` with `models` and `ordered` set to true. Assert that the bulk write fails and returns -a `BulkWriteError` (referred to as `orderedError`). +Execute `bulkWrite` on `client` with `models` and `ordered` set to true. Assert that the bulk write fails and returns a +`BulkWriteError` (referred to as `orderedError`). Assert that `orderedError.writeErrors` has a length of 1. @@ -355,8 +354,8 @@ UpdateOne { }, ``` -Execute `bulkWrite` on `client` with `models` and `verboseResults` set to true. Assert that the bulk write fails and returns -a `BulkWriteError` (referred to as `bulkWriteError`). +Execute `bulkWrite` on `client` with `models` and `verboseResults` set to true. Assert that the bulk write fails and +returns a `BulkWriteError` (referred to as `bulkWriteError`). Assert that `bulkWriteError.error` is populated with an error (referred to as `topLevelError`). Assert that `topLevelError.errorCode` is equal to 8. @@ -421,8 +420,8 @@ Expect a client-side error due the size. ### 11. `MongoClient.bulkWrite` batch splits when the addition of a new namespace exceeds the maximum message size -Test that `MongoClient.bulkWrite` batch splits a bulk write when the addition of a new namespace to `nsInfo` -causes the size of the message to exceed `maxMessageSizeBytes - 1000`. +Test that `MongoClient.bulkWrite` batch splits a bulk write when the addition of a new namespace to `nsInfo` causes the +size of the message to exceed `maxMessageSizeBytes - 1000`. This test must only be run on 8.0+ servers. @@ -454,8 +453,8 @@ InsertOne { Create a list of write models (referred to as `models`) with `firstModel` repeated `numModels` times. -If `remainderBytes` is greater than or equal to 217, add 1 to `numModels` and append the following write model -to `models`: +If `remainderBytes` is greater than or equal to 217, add 1 to `numModels` and append the following write model to +`models`: ```json InsertOne { @@ -486,8 +485,8 @@ Assert that `result.insertedCount` is equal to `numModels + 1`. Assert that one CommandStartedEvent was observed for the `bulkWrite` command (referred to as `event`). -Assert that the length of `event.command.ops` is `numModels + 1`. Assert that the length of `event.command.nsInfo` -is 1. Assert that the namespace contained in `event.command.nsInfo` is "db.coll". +Assert that the length of `event.command.ops` is `numModels + 1`. Assert that the length of `event.command.nsInfo` is 1. +Assert that the namespace contained in `event.command.nsInfo` is "db.coll". #### Case 2: Batch-splitting required @@ -528,8 +527,8 @@ Assert that the length of `secondEvent.command.ops` is equal to 1. Assert that t This information is not needed to implement this prose test, but is documented for future reference. This test is designed to work if `maxBsonObjectSize` or `maxMessageSizeBytes` changes, but will need to be updated if a required -field is added to the `bulkWrite` command or the `insert` operation document, or if the overhead `OP_MSG` allowance -is changed in the bulk write specification. +field is added to the `bulkWrite` command or the `insert` operation document, or if the overhead `OP_MSG` allowance is +changed in the bulk write specification. The command document for the `bulkWrite` has the following structure and size: @@ -557,8 +556,8 @@ Each write model will create an `ops` document with the following structure and Size: 57 bytes + ``` -The `ops` document for both `newNamespaceModel` and `sameNamespaceModel` has a string with one character, so -it is a total of 58 bytes. +The `ops` document for both `newNamespaceModel` and `sameNamespaceModel` has a string with one character, so it is a +total of 58 bytes. The models using the "db.coll" namespace will create one `nsInfo` document with the following structure and size: @@ -580,8 +579,8 @@ Size: 21 bytes Size: 217 bytes ``` -We need to fill up the rest of the message with bytes such that another `ops` document will fit, but another -`nsInfo` entry will not. The following calculations are used: +We need to fill up the rest of the message with bytes such that another `ops` document will fit, but another `nsInfo` +entry will not. The following calculations are used: ``` # 1000 is the OP_MSG overhead required in the spec @@ -601,8 +600,8 @@ remainingBulkWriteBytes = maxMessageSizeBytes - 1122 ### 12. `MongoClient.bulkWrite` returns an error if no operations can be added to `ops` -Test that `MongoClient.bulkWrite` returns an error if an operation provided exceeds `maxMessageSizeBytes` such -that an empty `ops` payload would be sent. +Test that `MongoClient.bulkWrite` returns an error if an operation provided exceeds `maxMessageSizeBytes` such that an +empty `ops` payload would be sent. This test must only be run on 8.0+ servers. This test may be skipped by drivers that are not able to construct arbitrarily large documents. @@ -646,8 +645,7 @@ Assert that `error` is a client error. ### 13. `MongoClient.bulkWrite` returns an error if auto-encryption is configured -This test is expected to be removed when -[DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is resolved. +This test is expected to be removed when [DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is resolved. Test that `MongoClient.bulkWrite` returns an error if the client has auto-encryption configured. @@ -676,6 +674,5 @@ InsertOne { } ``` -Execute `bulkWrite` on `client` with `model`. Assert that an error (referred to as `error`) is returned. -Assert that `error` is a client error containing the message: "bulkWrite does not currently support automatic -encryption". +Execute `bulkWrite` on `client` with `model`. Assert that an error (referred to as `error`) is returned. Assert that +`error` is a client error containing the message: "bulkWrite does not currently support automatic encryption". diff --git a/source/index.md b/source/index.md index 47298cf74c..0f8276b836 100644 --- a/source/index.md +++ b/source/index.md @@ -3,6 +3,7 @@ - [BSON Binary Subtype 6](client-side-encryption/subtype6.md) - [BSON Corpus](bson-corpus/bson-corpus.md) - [BSON Decimal128 Type Handling in Drivers](bson-decimal128/decimal128.md) +- [Bulk Write](crud/bulk-write.md) - [Causal Consistency Specification](causal-consistency/causal-consistency.md) - [Change Streams](change-streams/change-streams.md) - [Client Side Encryption](client-side-encryption/client-side-encryption.md) diff --git a/source/retryable-writes/retryable-writes.md b/source/retryable-writes/retryable-writes.md index f0099bc8f8..3ab56d3128 100644 --- a/source/retryable-writes/retryable-writes.md +++ b/source/retryable-writes/retryable-writes.md @@ -108,12 +108,12 @@ MongoDB 3.6 will support retryability for some, but not all, write operations. Supported single-statement write operations include `insertOne()`, `updateOne()`, `replaceOne()`, `deleteOne()`, `findOneAndDelete()`, `findOneAndReplace()`, and `findOneAndUpdate()`. -Supported multi-statement write operations include `insertMany()` and `bulkWrite()`. The ordered option may be `true` -or `false`. For both the collection-level and client-level `bulkWrite()` methods, a bulk write batch is only retryable -if it does not contain any `multi: true` writes (i.e. `UpdateMany` and `DeleteMany`). Drivers MUST evaluate eligibility -for each write command sent as part of the `bulkWrite()` (after order and batch splitting) individually. Drivers MUST -NOT alter existing logic for order and batch splitting in an attempt to maximize retryability for operations within a -bulk write. +Supported multi-statement write operations include `insertMany()` and `bulkWrite()`. The ordered option may be `true` or +`false`. For both the collection-level and client-level `bulkWrite()` methods, a bulk write batch is only retryable if +it does not contain any `multi: true` writes (i.e. `UpdateMany` and `DeleteMany`). Drivers MUST evaluate eligibility for +each write command sent as part of the `bulkWrite()` (after order and batch splitting) individually. Drivers MUST NOT +alter existing logic for order and batch splitting in an attempt to maximize retryability for operations within a bulk +write. These methods above are defined in the [CRUD](../crud/crud.md) specification. diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 8e377a8538..638eb14ba2 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -927,17 +927,18 @@ The structure of this object is as follows: `hasErrorLabel` method). - `writeErrors`: Optional document. The write errors expected to be present in the error. The `writeErrors` document - contains numeric keys representing the index of the write that failed and `writeError` object values. The test runner MUST - assert that the error contains a `writeError` for each index present in `writeErrors` and MUST assert that the `writeError`s - match as root-level documents according to the rules in [Evaluating Matches](#evaluating-matches). The test runner MUST - assert that the error does not contain any additional `writeError`s. This field is only intended for use with the - [clientBulkWrite](#clientbulkwrite) operation. + contains numeric keys representing the index of the write that failed and `writeError` object values. The test runner + MUST assert that the error contains a `writeError` for each index present in `writeErrors` and MUST assert that the + `writeError`s match as root-level documents according to the rules in [Evaluating Matches](#evaluating-matches). The + test runner MUST assert that the error does not contain any additional `writeError`s. This field is only intended for + use with the [clientBulkWrite](#clientbulkwrite) operation. - `writeConcernErrors`: Optional array of one or more objects. An ordered list of write concern errors expected to be present in the error. The test runner MUST assert that each `writeConcernError` in this list matches the - `writeConcernError` present at the same index in the error's list of `writeConcernError`s as a root-level document according - to the rules in [Evaluating Matches](#evaluating-matches). The test runner MUST assert that the error does not contain any - additional `writeConcernErrors`s. This field is only intended for use with the [clientBulkWrite](#clientbulkwrite) operation. + `writeConcernError` present at the same index in the error's list of `writeConcernError`s as a root-level document + according to the rules in [Evaluating Matches](#evaluating-matches). The test runner MUST assert that the error does + not contain any additional `writeConcernErrors`s. This field is only intended for use with the + [clientBulkWrite](#clientbulkwrite) operation. @@ -1457,8 +1458,8 @@ Test runners MUST NOT iterate the change stream when executing this operation an #### clientBulkWrite -These considerations only apply to the `MongoClient.bulkWrite` method. See [bulkWrite](#bulkwrite) for special considerations -for `MongoCollection.bulkWrite`. +These considerations only apply to the `MongoClient.bulkWrite` method. See [bulkWrite](#bulkwrite) for special +considerations for `MongoCollection.bulkWrite`. The `models` parameter for `clientBulkWrite` is documented as a list of WriteModel interfaces. Each WriteModel implementation (e.g. InsertOneModel) provides important context to the method, but that type information is not easily @@ -1489,23 +1490,23 @@ arguments: ordered: true ``` -Because the `insertResults`, `updateResults`, and `deleteResults` may be absent or empty in the `BulkWriteResult` returned -from a summary-only bulk write, the `clientBulkWrite` operation SHOULD use the [$$unsetOrMatches](#unsetormatches) operator -for assertions on these fields when `verboseResults` is not set to true. This also applies to result objects -defined in the `expectedResult` field of [expectedError](#expectederror). +Because the `insertResults`, `updateResults`, and `deleteResults` may be absent or empty in the `BulkWriteResult` +returned from a summary-only bulk write, the `clientBulkWrite` operation SHOULD use the +[$$unsetOrMatches](#unsetormatches) operator for assertions on these fields when `verboseResults` is not set to true. +This also applies to result objects defined in the `expectedResult` field of [expectedError](#expectederror). -The `BulkWriteException` thrown by `MongoClient.bulkWrite` contains an optional `error` field that stores a top-level error -that occurred during the bulk write. Test runners MUST inspect the contents of this field when making assertions based on -the contents of the `errorCode` and `errorContains` fields in [expectedError](#expectederror). +The `BulkWriteException` thrown by `MongoClient.bulkWrite` contains an optional `error` field that stores a top-level +error that occurred during the bulk write. Test runners MUST inspect the contents of this field when making assertions +based on the contents of the `errorCode` and `errorContains` fields in [expectedError](#expectederror). -`BulkWriteException` also contains `writeErrors` and `writeConcernErrors` fields that define the individual write errors and -write concern errors that occurred during the bulk write. Unified tests SHOULD use `writeErrors` and `writeConcernErrors` in -`expectedError` to assert on the contents of these fields. Test runners MUST NOT inspect the contents of these fields when -making assertions based on any other fields defined in `expectedError`. +`BulkWriteException` also contains `writeErrors` and `writeConcernErrors` fields that define the individual write errors +and write concern errors that occurred during the bulk write. Unified tests SHOULD use `writeErrors` and +`writeConcernErrors` in `expectedError` to assert on the contents of these fields. Test runners MUST NOT inspect the +contents of these fields when making assertions based on any other fields defined in `expectedError`. -While operations typically raise an error *or* return a result, the `MongoClient.bulkWrite` operation may -report both via the `partialResult` property of a `BulkWriteException`. In this case, the intermediary write result may be -matched with [expectedError_expectResult](#expectedError_expectResult) +While operations typically raise an error *or* return a result, the `MongoClient.bulkWrite` operation may report both +via the `partialResult` property of a `BulkWriteException`. In this case, the intermediary write result may be matched +with [expectedError_expectResult](#expectedError_expectResult) #### watch @@ -1653,7 +1654,8 @@ behavior between drivers that eagerly create a server-side cursor and those that #### bulkWrite -These considerations only apply to the `MongoCollection.bulkWrite` method. See [clientBulkWrite](#clientbulkwrite) for special considerations for `MongoClient.bulkWrite`. +These considerations only apply to the `MongoCollection.bulkWrite` method. See [clientBulkWrite](#clientbulkwrite) for +special considerations for `MongoClient.bulkWrite`. The `requests` parameter for `bulkWrite` is documented as a list of WriteModel interfaces. Each WriteModel implementation (e.g. InsertOneModel) provides important context to the method, but that type information is not easily @@ -3488,7 +3490,8 @@ other specs *and* collating spec changes developed in parallel or during the sam ## Changelog - 2024-05-08: **Schema version 1.21.**\ - Add `writeErrors` and `writeConcernErrors` field to `expectedError` for the client-level bulk write API. + Add `writeErrors` and `writeConcernErrors` field to `expectedError` for the + client-level bulk write API. - 2024-04-15: Note that when `directConnection` is set to true test runners should only provide a single seed. From ae0d729bab35c7e4e02189dcff3751136d14990d Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 8 May 2024 14:42:43 -0600 Subject: [PATCH 87/90] fix schema versions --- .../tests/monitoring/unacknowledged-client-bulkWrite.yml | 2 +- .../crud/tests/unified/client-bulkWrite-update-validation.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.yml b/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.yml index 4ec16bde8f..019102b3d7 100644 --- a/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.yml +++ b/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.yml @@ -1,6 +1,6 @@ description: "unacknowledged-client-bulkWrite" -schemaVersion: "1.0" +schemaVersion: "1.1" createEntities: - client: diff --git a/source/crud/tests/unified/client-bulkWrite-update-validation.yml b/source/crud/tests/unified/client-bulkWrite-update-validation.yml index f597e0762c..478554c322 100644 --- a/source/crud/tests/unified/client-bulkWrite-update-validation.yml +++ b/source/crud/tests/unified/client-bulkWrite-update-validation.yml @@ -1,6 +1,6 @@ description: "client-bulkWrite-update-validation" -schemaVersion: "1.0" +schemaVersion: "1.1" createEntities: - client: From 05852adf454202be09efdb7fbdc74338d0a1eea6 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 8 May 2024 14:50:49 -0600 Subject: [PATCH 88/90] json --- .../tests/monitoring/unacknowledged-client-bulkWrite.json | 2 +- .../crud/tests/unified/client-bulkWrite-update-validation.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.json b/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.json index d6153f444e..eb364d424d 100644 --- a/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.json +++ b/source/command-logging-and-monitoring/tests/monitoring/unacknowledged-client-bulkWrite.json @@ -1,6 +1,6 @@ { "description": "unacknowledged-client-bulkWrite", - "schemaVersion": "1.0", + "schemaVersion": "1.1", "createEntities": [ { "client": { diff --git a/source/crud/tests/unified/client-bulkWrite-update-validation.json b/source/crud/tests/unified/client-bulkWrite-update-validation.json index 1ac3e8d048..617e711338 100644 --- a/source/crud/tests/unified/client-bulkWrite-update-validation.json +++ b/source/crud/tests/unified/client-bulkWrite-update-validation.json @@ -1,6 +1,6 @@ { "description": "client-bulkWrite-update-validation", - "schemaVersion": "1.0", + "schemaVersion": "1.1", "createEntities": [ { "client": { From ad57a1e6d63db781fc7e2405d1a23967b5ae4914 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 8 May 2024 17:19:00 -0600 Subject: [PATCH 89/90] bump schema version in makefile --- source/unified-test-format/tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/unified-test-format/tests/Makefile b/source/unified-test-format/tests/Makefile index 5c30f9a66d..a2b79e3f70 100644 --- a/source/unified-test-format/tests/Makefile +++ b/source/unified-test-format/tests/Makefile @@ -1,4 +1,4 @@ -SCHEMA=../schema-1.20.json +SCHEMA=../schema-1.21.json .PHONY: all invalid valid-fail valid-pass atlas-data-lake versioned-api load-balancers gridfs transactions transactions-convenient-api crud collection-management read-write-concern retryable-reads retryable-writes sessions command-logging-and-monitoring client-side-operations-timeout HAS_AJV From 5a6532f91a84ef41c8f50afd7a229de35f0ea226 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Wed, 8 May 2024 17:27:55 -0600 Subject: [PATCH 90/90] merge 1.20 and 1.21 --- source/unified-test-format/schema-1.21.json | 2082 ++++++++++--------- 1 file changed, 1058 insertions(+), 1024 deletions(-) diff --git a/source/unified-test-format/schema-1.21.json b/source/unified-test-format/schema-1.21.json index b5395a33c9..9d22fe6209 100644 --- a/source/unified-test-format/schema-1.21.json +++ b/source/unified-test-format/schema-1.21.json @@ -1,1082 +1,1116 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Unified Test Format", - "type": "object", - "additionalProperties": false, - "required": [ - "description", - "schemaVersion", - "tests" - ], - "properties": { - "description": { - "type": "string" + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Unified Test Format", + "type": "object", + "additionalProperties": false, + "required": [ + "description", + "schemaVersion", + "tests" + ], + "properties": { + "description": { + "type": "string" + }, + "schemaVersion": { + "$ref": "#/definitions/version" + }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/runOnRequirement" + } + }, + "createEntities": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/entity" + } + }, + "initialData": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/collectionData" + } + }, + "tests": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/test" + } + }, + "_yamlAnchors": { + "type": "object", + "additionalProperties": true + } + }, + "definitions": { + "version": { + "type": "string", + "pattern": "^[0-9]+(\\.[0-9]+){1,2}$" + }, + "runOnRequirement": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "maxServerVersion": { + "$ref": "#/definitions/version" }, - "schemaVersion": { - "$ref": "#/definitions/version" + "minServerVersion": { + "$ref": "#/definitions/version" }, - "runOnRequirements": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/runOnRequirement" + "topologies": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "single", + "replicaset", + "sharded", + "sharded-replicaset", + "load-balanced" + ] + } + }, + "serverless": { + "type": "string", + "enum": [ + "require", + "forbid", + "allow" + ] + }, + "serverParameters": { + "type": "object", + "minProperties": 1 + }, + "auth": { + "type": "boolean" + }, + "authMechanism": { + "type": "string" + }, + "csfle": { + "type": "boolean" + } + } + }, + "entity": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "client": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "uriOptions": { + "type": "object" + }, + "useMultipleMongoses": { + "type": "boolean" + }, + "observeEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent", + "poolCreatedEvent", + "poolReadyEvent", + "poolClearedEvent", + "poolClosedEvent", + "connectionCreatedEvent", + "connectionReadyEvent", + "connectionClosedEvent", + "connectionCheckOutStartedEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedOutEvent", + "connectionCheckedInEvent", + "serverDescriptionChangedEvent", + "topologyDescriptionChangedEvent" + ] + } + }, + "ignoreCommandMonitoringEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "storeEventsAsEntities": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/storeEventsAsEntity" + } + }, + "observeLogMessages": { + "type": "object", + "minProperties": 1, + "additionalProperties": false, + "properties": { + "command": { + "$ref": "#/definitions/logSeverityLevel" + }, + "topology": { + "$ref": "#/definitions/logSeverityLevel" + }, + "serverSelection": { + "$ref": "#/definitions/logSeverityLevel" + }, + "connection": { + "$ref": "#/definitions/logSeverityLevel" + } + } + }, + "serverApi": { + "$ref": "#/definitions/serverApi" + }, + "observeSensitiveCommands": { + "type": "boolean" } + } }, - "createEntities": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/entity" + "clientEncryption": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "clientEncryptionOpts" + ], + "properties": { + "id": { + "type": "string" + }, + "clientEncryptionOpts": { + "$ref": "#/definitions/clientEncryptionOpts" } + } }, - "initialData": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/collectionData" + "database": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "client", + "databaseName" + ], + "properties": { + "id": { + "type": "string" + }, + "client": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "databaseOptions": { + "$ref": "#/definitions/collectionOrDatabaseOptions" } + } }, - "tests": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/test" + "collection": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "database", + "collectionName" + ], + "properties": { + "id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "collectionName": { + "type": "string" + }, + "collectionOptions": { + "$ref": "#/definitions/collectionOrDatabaseOptions" + } + } + }, + "session": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "client" + ], + "properties": { + "id": { + "type": "string" + }, + "client": { + "type": "string" + }, + "sessionOptions": { + "type": "object" + } + } + }, + "bucket": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "database" + ], + "properties": { + "id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "bucketOptions": { + "type": "object" } + } }, - "_yamlAnchors": { - "type": "object", - "additionalProperties": true + "thread": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + } + } } + } }, - "definitions": { - "version": { - "type": "string", - "pattern": "^[0-9]+(\\.[0-9]+){1,2}$" - }, - "runOnRequirement": { - "type": "object", - "additionalProperties": false, - "minProperties": 1, - "properties": { - "maxServerVersion": { - "$ref": "#/definitions/version" - }, - "minServerVersion": { - "$ref": "#/definitions/version" - }, - "topologies": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "enum": [ - "single", - "replicaset", - "sharded", - "sharded-replicaset", - "load-balanced" - ] - } - }, - "serverless": { - "type": "string", - "enum": [ - "require", - "forbid", - "allow" - ] - }, - "serverParameters": { - "type": "object", - "minProperties": 1 - }, - "auth": { - "type": "boolean" - }, - "authMechanism": { - "type": "string" - }, - "csfle": { - "type": "boolean" - } + "logComponent": { + "type": "string", + "enum": [ + "command", + "topology", + "serverSelection", + "connection" + ] + }, + "logSeverityLevel": { + "type": "string", + "enum": [ + "emergency", + "alert", + "critical", + "error", + "warning", + "notice", + "info", + "debug", + "trace" + ] + }, + "clientEncryptionOpts": { + "type": "object", + "additionalProperties": false, + "required": [ + "keyVaultClient", + "keyVaultNamespace", + "kmsProviders" + ], + "properties": { + "keyVaultClient": { + "type": "string" + }, + "keyVaultNamespace": { + "type": "string" + }, + "kmsProviders": { + "$ref": "#/definitions/kmsProviders" + } + } + }, + "kmsProviders": { + "$defs": { + "stringOrPlaceholder": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "$$placeholder" + ], + "properties": { + "$$placeholder": {} + } } + ] + } + }, + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^aws(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "accessKeyId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "secretAccessKey": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "sessionToken": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } }, - "entity": { - "type": "object", - "additionalProperties": false, - "maxProperties": 1, - "minProperties": 1, - "properties": { - "client": { - "type": "object", - "additionalProperties": false, - "required": [ - "id" - ], - "properties": { - "id": { - "type": "string" - }, - "uriOptions": { - "type": "object" - }, - "useMultipleMongoses": { - "type": "boolean" - }, - "observeEvents": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "enum": [ - "commandStartedEvent", - "commandSucceededEvent", - "commandFailedEvent", - "poolCreatedEvent", - "poolReadyEvent", - "poolClearedEvent", - "poolClosedEvent", - "connectionCreatedEvent", - "connectionReadyEvent", - "connectionClosedEvent", - "connectionCheckOutStartedEvent", - "connectionCheckOutFailedEvent", - "connectionCheckedOutEvent", - "connectionCheckedInEvent", - "serverDescriptionChangedEvent", - "topologyDescriptionChangedEvent" - ] - } - }, - "ignoreCommandMonitoringEvents": { - "type": "array", - "minItems": 1, - "items": { - "type": "string" - } - }, - "storeEventsAsEntities": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/storeEventsAsEntity" - } - }, - "observeLogMessages": { - "type": "object", - "minProperties": 1, - "additionalProperties": false, - "properties": { - "command": { - "$ref": "#/definitions/logSeverityLevel" - }, - "topology": { - "$ref": "#/definitions/logSeverityLevel" - }, - "serverSelection": { - "$ref": "#/definitions/logSeverityLevel" - }, - "connection": { - "$ref": "#/definitions/logSeverityLevel" - } - } - }, - "serverApi": { - "$ref": "#/definitions/serverApi" - }, - "observeSensitiveCommands": { - "type": "boolean" - } - } - }, - "clientEncryption": { - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "clientEncryptionOpts" - ], - "properties": { - "id": { - "type": "string" - }, - "clientEncryptionOpts": { - "$ref": "#/definitions/clientEncryptionOpts" - } - } - }, - "database": { - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "client", - "databaseName" - ], - "properties": { - "id": { - "type": "string" - }, - "client": { - "type": "string" - }, - "databaseName": { - "type": "string" - }, - "databaseOptions": { - "$ref": "#/definitions/collectionOrDatabaseOptions" - } - } - }, - "collection": { - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "database", - "collectionName" - ], - "properties": { - "id": { - "type": "string" - }, - "database": { - "type": "string" - }, - "collectionName": { - "type": "string" - }, - "collectionOptions": { - "$ref": "#/definitions/collectionOrDatabaseOptions" - } - } - }, - "session": { - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "client" - ], - "properties": { - "id": { - "type": "string" - }, - "client": { - "type": "string" - }, - "sessionOptions": { - "type": "object" - } - } - }, - "bucket": { - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "database" - ], - "properties": { - "id": { - "type": "string" - }, - "database": { - "type": "string" - }, - "bucketOptions": { - "type": "object" - } - } - }, - "thread": { - "type": "object", - "additionalProperties": false, - "required": [ - "id" - ], - "properties": { - "id": { - "type": "string" - } - } - } + "^azure(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "tenantId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "clientId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "clientSecret": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "identityPlatformEndpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" } + } }, - "logComponent": { - "type": "string", - "enum": [ - "command", - "topology", - "serverSelection", - "connection" - ] + "^gcp(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "email": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "privateKey": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "endpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } }, - "logSeverityLevel": { + "^kmip(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "endpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^local(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "key": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + } + } + }, + "storeEventsAsEntity": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "events" + ], + "properties": { + "id": { + "type": "string" + }, + "events": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "enum": [ - "emergency", - "alert", - "critical", - "error", - "warning", - "notice", - "info", - "debug", - "trace" + "PoolCreatedEvent", + "PoolReadyEvent", + "PoolClearedEvent", + "PoolClosedEvent", + "ConnectionCreatedEvent", + "ConnectionReadyEvent", + "ConnectionClosedEvent", + "ConnectionCheckOutStartedEvent", + "ConnectionCheckOutFailedEvent", + "ConnectionCheckedOutEvent", + "ConnectionCheckedInEvent", + "CommandStartedEvent", + "CommandSucceededEvent", + "CommandFailedEvent", + "ServerDescriptionChangedEvent", + "TopologyDescriptionChangedEvent" ] + } + } + } + }, + "collectionData": { + "type": "object", + "additionalProperties": false, + "required": [ + "collectionName", + "databaseName", + "documents" + ], + "properties": { + "collectionName": { + "type": "string" }, - "clientEncryptionOpts": { - "type": "object", - "additionalProperties": false, - "required": [ - "keyVaultClient", - "keyVaultNamespace", - "kmsProviders" - ], - "properties": { - "keyVaultClient": { - "type": "string" - }, - "keyVaultNamespace": { - "type": "string" - }, - "kmsProviders": { - "$ref": "#/definitions/kmsProviders" - } + "databaseName": { + "type": "string" + }, + "createOptions": { + "type": "object", + "properties": { + "writeConcern": false + } + }, + "documents": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "expectedEventsForClient": { + "type": "object", + "additionalProperties": false, + "required": [ + "client", + "events" + ], + "properties": { + "client": { + "type": "string" + }, + "eventType": { + "type": "string", + "enum": [ + "command", + "cmap", + "sdam" + ] + }, + "events": { + "type": "array" + }, + "ignoreExtraEvents": { + "type": "boolean" + } + }, + "oneOf": [ + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "command" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCommandEvent" + } } + } }, - "kmsProviders": { - "$defs": { - "stringOrPlaceholder": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - "additionalProperties": false, - "required": [ - "$$placeholder" - ], - "properties": { - "$$placeholder": {} - } - } - ] - } + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "cmap" }, - "type": "object", - "additionalProperties": false, - "patternProperties": { - "^aws(:[a-zA-Z0-9_]+)?$": { - "type": "object", - "additionalProperties": false, - "properties": { - "accessKeyId": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - }, - "secretAccessKey": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - }, - "sessionToken": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - } - } - }, - "^azure(:[a-zA-Z0-9_]+)?$": { - "type": "object", - "additionalProperties": false, - "properties": { - "tenantId": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - }, - "clientId": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - }, - "clientSecret": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - }, - "identityPlatformEndpoint": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - } - } - }, - "^gcp(:[a-zA-Z0-9_]+)?$": { - "type": "object", - "additionalProperties": false, - "properties": { - "email": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - }, - "privateKey": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - }, - "endpoint": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - } - } - }, - "^kmip(:[a-zA-Z0-9_]+)?$": { - "type": "object", - "additionalProperties": false, - "properties": { - "endpoint": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - } - } - }, - "^local(:[a-zA-Z0-9_]+)?$": { - "type": "object", - "additionalProperties": false, - "properties": { - "key": { - "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" - } - } - } + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCmapEvent" + } } + } }, - "storeEventsAsEntity": { - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "events" - ], - "properties": { - "id": { - "type": "string" - }, - "events": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "enum": [ - "PoolCreatedEvent", - "PoolReadyEvent", - "PoolClearedEvent", - "PoolClosedEvent", - "ConnectionCreatedEvent", - "ConnectionReadyEvent", - "ConnectionClosedEvent", - "ConnectionCheckOutStartedEvent", - "ConnectionCheckOutFailedEvent", - "ConnectionCheckedOutEvent", - "ConnectionCheckedInEvent", - "CommandStartedEvent", - "CommandSucceededEvent", - "CommandFailedEvent", - "ServerDescriptionChangedEvent", - "TopologyDescriptionChangedEvent" - ] - } - } + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "sdam" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedSdamEvent" + } } + } }, - "collectionData": { - "type": "object", - "additionalProperties": false, - "required": [ - "collectionName", - "databaseName", - "documents" - ], - "properties": { - "collectionName": { - "type": "string" - }, - "databaseName": { - "type": "string" - }, - "createOptions": { - "type": "object", - "properties": { - "writeConcern": false - } - }, - "documents": { - "type": "array", - "items": { - "type": "object" - } - } + { + "additionalProperties": false, + "properties": { + "client": { + "type": "string" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCommandEvent" + } + }, + "ignoreExtraEvents": { + "type": "boolean" + } + } + } + ] + }, + "expectedCommandEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "commandStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "object" + }, + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" } + } }, - "expectedEventsForClient": { - "type": "object", - "additionalProperties": false, - "required": [ - "client", - "events" - ], - "properties": { - "client": { - "type": "string" - }, - "eventType": { - "type": "string", - "enum": [ - "command", - "cmap", - "sdam" - ] - }, - "events": { - "type": "array" - }, - "ignoreExtraEvents": { - "type": "boolean" - } + "commandSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reply": { + "type": "object" }, - "oneOf": [ - { - "required": [ - "eventType" - ], - "properties": { - "eventType": { - "const": "command" - }, - "events": { - "type": "array", - "items": { - "$ref": "#/definitions/expectedCommandEvent" - } - } - } - }, - { - "required": [ - "eventType" - ], - "properties": { - "eventType": { - "const": "cmap" - }, - "events": { - "type": "array", - "items": { - "$ref": "#/definitions/expectedCmapEvent" - } - } - } - }, - { - "required": [ - "eventType" - ], - "properties": { - "eventType": { - "const": "sdam" - }, - "events": { - "type": "array", - "items": { - "$ref": "#/definitions/expectedSdamEvent" - } - } - } - }, - { - "additionalProperties": false, - "properties": { - "client": { - "type": "string" - }, - "events": { - "type": "array", - "items": { - "$ref": "#/definitions/expectedCommandEvent" - } - }, - "ignoreExtraEvents": { - "type": "boolean" - } - } - } - ] + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } }, - "expectedCommandEvent": { - "type": "object", - "additionalProperties": false, - "maxProperties": 1, - "minProperties": 1, - "properties": { - "commandStartedEvent": { - "type": "object", - "additionalProperties": false, - "properties": { - "command": { - "type": "object" - }, - "commandName": { - "type": "string" - }, - "databaseName": { - "type": "string" - }, - "hasServiceId": { - "type": "boolean" - }, - "hasServerConnectionId": { - "type": "boolean" - } - } - }, - "commandSucceededEvent": { - "type": "object", - "additionalProperties": false, - "properties": { - "reply": { - "type": "object" - }, - "commandName": { - "type": "string" - }, - "databaseName": { - "type": "string" - }, - "hasServiceId": { - "type": "boolean" - }, - "hasServerConnectionId": { - "type": "boolean" - } - } - }, - "commandFailedEvent": { - "type": "object", - "additionalProperties": false, - "properties": { - "commandName": { - "type": "string" - }, - "databaseName": { - "type": "string" - }, - "hasServiceId": { - "type": "boolean" - }, - "hasServerConnectionId": { - "type": "boolean" - } - } - } + "commandFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" } + } + } + } + }, + "expectedCmapEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "poolCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} }, - "expectedCmapEvent": { - "type": "object", - "additionalProperties": false, - "maxProperties": 1, - "minProperties": 1, - "properties": { - "poolCreatedEvent": { - "type": "object", - "additionalProperties": false, - "properties": {} - }, - "poolReadyEvent": { - "type": "object", - "additionalProperties": false, - "properties": {} - }, - "poolClearedEvent": { - "type": "object", - "additionalProperties": false, - "properties": { - "hasServiceId": { - "type": "boolean" - }, - "interruptInUseConnections": { - "type": "boolean" - } - } - }, - "poolClosedEvent": { - "type": "object", - "additionalProperties": false, - "properties": {} - }, - "connectionCreatedEvent": { - "type": "object", - "additionalProperties": false, - "properties": {} - }, - "connectionReadyEvent": { - "type": "object", - "additionalProperties": false, - "properties": {} - }, - "connectionClosedEvent": { - "type": "object", - "additionalProperties": false, - "properties": { - "reason": { - "type": "string" - } - } - }, - "connectionCheckOutStartedEvent": { - "type": "object", - "additionalProperties": false, - "properties": {} - }, - "connectionCheckOutFailedEvent": { - "type": "object", - "additionalProperties": false, - "properties": { - "reason": { - "type": "string" - } - } - }, - "connectionCheckedOutEvent": { - "type": "object", - "additionalProperties": false, - "properties": {} - }, - "connectionCheckedInEvent": { - "type": "object", - "additionalProperties": false, - "properties": {} - } + "poolReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "poolClearedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "hasServiceId": { + "type": "boolean" + }, + "interruptInUseConnections": { + "type": "boolean" } + } }, - "expectedSdamEvent": { - "type": "object", - "additionalProperties": false, - "maxProperties": 1, - "minProperties": 1, - "properties": { - "serverDescriptionChangedEvent": { - "type": "object", - "additionalProperties": false, - "properties": { - "previousDescription": { - "$ref": "#/definitions/serverDescription" - }, - "newDescription": { - "$ref": "#/definitions/serverDescription" - } - } - }, - "topologyDescriptionChangedEvent": { - "type": "object", - "additionalProperties": false, - "properties": {} - }, - "serverHeartbeatStartedEvent": { - "type": "object", - "additionalProperties": false, - "properties": { - "awaited": { - "type": "boolean" - } - } - }, - "serverHeartbeatSucceededEvent": { - "type": "object", - "additionalProperties": false, - "properties": { - "awaited": { - "type": "boolean" - } - } - }, - "serverHeartbeatFailedEvent": { - "type": "object", - "additionalProperties": false, - "properties": { - "awaited": { - "type": "boolean" - } - } - } + "poolClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" } + } }, - "serverDescription": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "type": "string", - "enum": [ - "Standalone", - "Mongos", - "PossiblePrimary", - "RSPrimary", - "RSSecondary", - "RSOther", - "RSArbiter", - "RSGhost", - "LoadBalancer", - "Unknown" - ] - } + "connectionCheckOutStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckOutFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" } + } }, - "expectedLogMessagesForClient": { - "type": "object", - "additionalProperties": false, - "required": [ - "client", - "messages" - ], - "properties": { - "client": { - "type": "string" - }, - "messages": { - "type": "array", - "items": { - "$ref": "#/definitions/expectedLogMessage" - } - }, - "ignoreExtraMessages": { - "type": "boolean" - }, - "ignoreMessages": { - "type": "array", - "items": { - "$ref": "#/definitions/expectedLogMessage" - } - } + "connectionCheckedOutEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckedInEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + } + } + }, + "expectedSdamEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "serverDescriptionChangedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "previousDescription": { + "$ref": "#/definitions/serverDescription" + }, + "newDescription": { + "$ref": "#/definitions/serverDescription" } + } }, - "expectedLogMessage": { - "type": "object", - "additionalProperties": false, - "required": [ - "level", - "component", - "data" - ], - "properties": { - "level": { - "$ref": "#/definitions/logSeverityLevel" - }, - "component": { - "$ref": "#/definitions/logComponent" - }, - "data": { - "type": "object" - }, - "failureIsRedacted": { - "type": "boolean" - } + "topologyDescriptionChangedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "previousDescription": { + "$ref": "#/definitions/topologyDescription" + }, + "newDescription": { + "$ref": "#/definitions/topologyDescription" } + } }, - "collectionOrDatabaseOptions": { - "type": "object", - "additionalProperties": false, - "properties": { - "readConcern": { - "type": "object" - }, - "readPreference": { - "type": "object" - }, - "writeConcern": { - "type": "object" - }, - "timeoutMS": { - "type": "integer" - } + "serverHeartbeatStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" } + } }, - "serverApi": { - "type": "object", - "additionalProperties": false, - "required": [ - "version" - ], - "properties": { - "version": { - "type": "string" - }, - "strict": { - "type": "boolean" - }, - "deprecationErrors": { - "type": "boolean" - } + "serverHeartbeatSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "serverHeartbeatFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" } + } + }, + "topologyOpeningEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "topologyClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + } + } + }, + "serverDescription": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "Standalone", + "Mongos", + "PossiblePrimary", + "RSPrimary", + "RSSecondary", + "RSOther", + "RSArbiter", + "RSGhost", + "LoadBalancer", + "Unknown" + ] + } + } + }, + "topologyDescription": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "Single", + "Unknown", + "ReplicaSetNoPrimary", + "ReplicaSetWithPrimary", + "Sharded", + "LoadBalanced" + ] + } + } + }, + "expectedLogMessagesForClient": { + "type": "object", + "additionalProperties": false, + "required": [ + "client", + "messages" + ], + "properties": { + "client": { + "type": "string" + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedLogMessage" + } + }, + "ignoreExtraMessages": { + "type": "boolean" + }, + "ignoreMessages": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedLogMessage" + } + } + } + }, + "expectedLogMessage": { + "type": "object", + "additionalProperties": false, + "required": [ + "level", + "component", + "data" + ], + "properties": { + "level": { + "$ref": "#/definitions/logSeverityLevel" + }, + "component": { + "$ref": "#/definitions/logComponent" + }, + "data": { + "type": "object" + }, + "failureIsRedacted": { + "type": "boolean" + } + } + }, + "collectionOrDatabaseOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "readConcern": { + "type": "object" + }, + "readPreference": { + "type": "object" + }, + "writeConcern": { + "type": "object" + }, + "timeoutMS": { + "type": "integer" + } + } + }, + "serverApi": { + "type": "object", + "additionalProperties": false, + "required": [ + "version" + ], + "properties": { + "version": { + "type": "string" + }, + "strict": { + "type": "boolean" + }, + "deprecationErrors": { + "type": "boolean" + } + } + }, + "operation": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "object" + ], + "properties": { + "name": { + "type": "string" }, - "operation": { - "type": "object", - "additionalProperties": false, + "object": { + "type": "string" + }, + "arguments": { + "type": "object" + }, + "ignoreResultAndError": { + "type": "boolean" + }, + "expectError": { + "$ref": "#/definitions/expectedError" + }, + "expectResult": {}, + "saveResultAsEntity": { + "type": "string" + } + }, + "allOf": [ + { + "not": { "required": [ - "name", - "object" - ], - "properties": { - "name": { - "type": "string" - }, - "object": { - "type": "string" - }, - "arguments": { - "type": "object" - }, - "ignoreResultAndError": { - "type": "boolean" - }, - "expectError": { - "$ref": "#/definitions/expectedError" - }, - "expectResult": {}, - "saveResultAsEntity": { - "type": "string" - } - }, - "allOf": [ - { - "not": { - "required": [ - "expectError", - "expectResult" - ] - } - }, - { - "not": { - "required": [ - "expectError", - "saveResultAsEntity" - ] - } - }, - { - "not": { - "required": [ - "ignoreResultAndError", - "expectResult" - ] - } - }, - { - "not": { - "required": [ - "ignoreResultAndError", - "expectError" - ] - } - }, - { - "not": { - "required": [ - "ignoreResultAndError", - "saveResultAsEntity" - ] - } - } + "expectError", + "expectResult" ] + } }, - "expectedError": { - "type": "object", - "additionalProperties": false, - "minProperties": 1, - "properties": { - "isError": { - "type": "boolean", - "const": true - }, - "isClientError": { - "type": "boolean" - }, - "isTimeoutError": { - "type": "boolean" - }, - "errorContains": { - "type": "string" - }, - "errorCode": { - "type": "integer" - }, - "errorCodeName": { - "type": "string" - }, - "errorLabelsContain": { - "type": "array", - "minItems": 1, - "items": { - "type": "string" - } - }, - "errorLabelsOmit": { - "type": "array", - "minItems": 1, - "items": { - "type": "string" - } - }, - "writeErrors": { - "type": "object" - }, - "writeConcernErrors": { - "type": "array", - "items": { - "type": "object" - } - }, - "errorResponse": { - "type": "object" - }, - "expectResult": {} - } + { + "not": { + "required": [ + "expectError", + "saveResultAsEntity" + ] + } }, - "test": { - "type": "object", - "additionalProperties": false, + { + "not": { "required": [ - "description", - "operations" - ], - "properties": { - "description": { - "type": "string" - }, - "runOnRequirements": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/runOnRequirement" - } - }, - "skipReason": { - "type": "string" - }, - "operations": { - "type": "array", - "items": { - "$ref": "#/definitions/operation" - } - }, - "expectEvents": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/expectedEventsForClient" - } - }, - "expectLogMessages": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/expectedLogMessagesForClient" - } - }, - "outcome": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/collectionData" - } - } + "ignoreResultAndError", + "expectResult" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "expectError" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "saveResultAsEntity" + ] + } + } + ] + }, + "expectedError": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "isError": { + "type": "boolean", + "const": true + }, + "isClientError": { + "type": "boolean" + }, + "isTimeoutError": { + "type": "boolean" + }, + "errorContains": { + "type": "string" + }, + "errorCode": { + "type": "integer" + }, + "errorCodeName": { + "type": "string" + }, + "errorLabelsContain": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "errorLabelsOmit": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "writeErrors": { + "type": "object" + }, + "writeConcernErrors": { + "type": "array", + "items": { + "type": "object" } + }, + "errorResponse": { + "type": "object" + }, + "expectResult": {} + } + }, + "test": { + "type": "object", + "additionalProperties": false, + "required": [ + "description", + "operations" + ], + "properties": { + "description": { + "type": "string" + }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/runOnRequirement" + } + }, + "skipReason": { + "type": "string" + }, + "operations": { + "type": "array", + "items": { + "$ref": "#/definitions/operation" + } + }, + "expectEvents": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/expectedEventsForClient" + } + }, + "expectLogMessages": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/expectedLogMessagesForClient" + } + }, + "outcome": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/collectionData" + } } + } } + } }