Support annotation queue params

js/src/client.ts

@@ -1,4 +1,4 @@
 import * as uuid from "uuid";
 
 import { AsyncCaller, AsyncCallerParams } from "./utils/async_caller.js";
 import {
@@ -417,7 +417,7 @@
     // If there is an item on the queue we were unable to pop,
     // just return it as a single batch.
     if (popped.length === 0 && this.items.length > 0) {
       const item = this.items.shift()!;
       popped.push(item);
       poppedSizeBytes += item.size;
       this.sizeBytes -= item.size;
@@ -833,7 +833,7 @@
         if (this._serverInfo === undefined) {
           try {
             this._serverInfo = await this._getServerInfo();
           } catch (e) {
             console.warn(
               `[WARNING]: LangSmith failed to fetch info on supported operations. Falling back to batch operations and default limits.`
             );
@@ -1551,7 +1551,7 @@
     treeFilter?: string;
     isRoot?: boolean;
     dataSourceType?: string;
   }): Promise<any> {
     let projectIds_ = projectIds || [];
     if (projectNames) {
       projectIds_ = [
@@ -1839,7 +1839,7 @@
         `Failed to list shared examples: ${response.status} ${response.statusText}`
       );
     }
     return result.map((example: any) => ({
       ...example,
       _hostUrl: this.getHostUrl(),
     }));
@@ -1976,7 +1976,7 @@
       }
       // projectId querying
       return true;
     } catch (e) {
       return false;
     }
   }
@@ -3316,7 +3316,7 @@
   async _logEvaluationFeedback(
     evaluatorResponse: EvaluationResult | EvaluationResults,
     run?: Run,
     sourceInfo?: { [key: string]: any }
   ): Promise<[results: EvaluationResult[], feedbacks: Feedback[]]> {
     const evalResults: Array<EvaluationResult> =
       this._selectEvalResults(evaluatorResponse);
@@ -3355,7 +3355,7 @@
   public async logEvaluationFeedback(
     evaluatorResponse: EvaluationResult | EvaluationResults,
     run?: Run,
     sourceInfo?: { [key: string]: any }
   ): Promise<EvaluationResult[]> {
     const [results] = await this._logEvaluationFeedback(
       evaluatorResponse,
@@ -3418,18 +3418,60 @@
    * @param options.name - The name of the annotation queue
    * @param options.description - The description of the annotation queue
    * @param options.queueId - The ID of the annotation queue
+   * @param options.defaultDatasetId - The ID of the default dataset for the queue
+   * @param options.defaultDatasetName - The name of the default dataset for the queue
+   * @param options.numReviewersPerItem - Number of reviewers per item
+   * @param options.enableReservations - Whether to enable reservations
+   * @param options.reservationMinutes - Number of minutes for reservations
+   * @param options.rubricItems - List of rubric items for the queue
+   * @param options.rubricInstructions - Instructions for the rubric
    * @returns The created AnnotationQueue object
    */
   public async createAnnotationQueue(options: {
     name: string;
     description?: string;
     queueId?: string;
+    defaultDatasetId?: string;
+    defaultDatasetName?: string;
+    numReviewersPerItem?: number;
+    enableReservations?: boolean;
+    reservationMinutes?: number;
+    rubricItems?: Array<Record<string, any>>;
+    rubricInstructions?: string;
   }): Promise<AnnotationQueue> {
-    const { name, description, queueId } = options;
+    const {
+      name,
+      description,
+      queueId,
+      defaultDatasetId,
+      defaultDatasetName,
+      numReviewersPerItem,
+      enableReservations = true,
+      reservationMinutes,
+      rubricItems,
+      rubricInstructions,
+    } = options;
+
+    let defaultDataset: string | undefined;
+    if (defaultDatasetId) {
+      defaultDataset = assertUuid(defaultDatasetId, "defaultDatasetId");
+    } else if (defaultDatasetName) {
+      const dataset = await this.readDataset({
+        datasetName: defaultDatasetName,
+      });
+      defaultDataset = dataset.id;
+    }
+
     const body = {
       name,
       description,
       id: queueId || uuid.v4(),
+      default_dataset: defaultDataset,
+      num_reviewers_per_item: numReviewersPerItem,
+      enable_reservations: enableReservations,
+      reservation_minutes: reservationMinutes,
+      rubric_items: rubricItems,
+      rubric_instructions: rubricInstructions,
     };
 
     const response = await this.caller.call(
@@ -3805,7 +3847,7 @@
 
   public async createCommit(
     promptIdentifier: string,
     object: any,
     options?: {
       parentCommitHash?: string;
     }
@@ -3856,7 +3898,7 @@
       isPublic?: boolean;
       isArchived?: boolean;
     }
   ): Promise<Record<string, any>> {
     if (!(await this.promptExists(promptIdentifier))) {
       throw new Error("Prompt does not exist, you must create it first.");
     }

python/langsmith/client.py

@@ -4889,6 +4889,13 @@ def create_annotation_queue(
         name: str,
         description: Optional[str] = None,
         queue_id: Optional[ID_TYPE] = None,
+        default_dataset_id: Optional[ID_TYPE] = None,
+        default_dataset_name: Optional[str] = None,
+        num_reviewers_per_item: Optional[int] = None,
+        enable_reservations: Optional[bool] = True,
+        reservation_minutes: Optional[int] = None,
+        rubric_items: Optional[Sequence[Mapping[str, Any]]] = None,
+        rubric_instructions: Optional[str] = None,
     ) -> ls_schemas.AnnotationQueue:
         """Create an annotation queue on the LangSmith API.
 
@@ -4899,15 +4906,40 @@ def create_annotation_queue(
                 The description of the annotation queue.
             queue_id : str or UUID, optional
                 The ID of the annotation queue.
+            default_dataset : str, optional
+                The default dataset for the queue.
+            num_reviewers_per_item : int, optional
+                Number of reviewers per item.
+            enable_reservations : bool, optional
+                Whether to enable reservations. Defaults to True.
+            reservation_minutes : int, optional
+                Number of minutes for reservations.
+            rubric_items : List[Dict[str, Any]], optional
+                List of rubric items for the queue.
+            rubric_instructions : str, optional
+                Instructions for the rubric.
 
         Returns:
             AnnotationQueue
                 The created annotation queue object.
         """
+        if default_dataset_id is not None:
+            default_dataset = _as_uuid(default_dataset_id, "default_dataset_id")
+        elif default_dataset_name is not None:
+            default_dataset = self.read_dataset(dataset_name=default_dataset_name).id
+        else:
+            default_dataset = None
+
         body = {
             "name": name,
             "description": description,
             "id": queue_id or str(uuid.uuid4()),
+            "default_dataset": default_dataset,
+            "num_reviewers_per_item": num_reviewers_per_item,
+            "enable_reservations": enable_reservations,
+            "reservation_minutes": reservation_minutes,
+            "rubric_items": rubric_items,
+            "rubric_instructions": rubric_instructions,
         }
         response = self.request_with_retries(
             "POST",