SF-3578 Add Import Questions from Paratext

The notes are fetched from the Paratext local copy, and the relevant pieces are parsed. The back end replaces the tag numbers with their user-displayed text (for readability).

The front end makes this call when the user clicks the Paratext Import option in the import dialog. It first gathers the available tags and displays them as a dropdown. Once a value is chosen, it filters the notes down to the selection, and feeds them to the same display used for the Transcelerator questions.

The brunt of this is done, but there are a number of edge cases to work through. And tests.

Author: josephmyers

src/SIL.XForge.Scripture/ClientApp/src/app/checking/import-questions-dialog/import-questions-dialog.component.html

@@ -97,6 +97,17 @@ <h2 mat-dialog-title class="dialog-icon-title">
                   <a mat-button [href]="urls.csvImportHelpPage" target="_blank">{{ t("learn_more") }}</a>
                 </mat-card-actions>
               </mat-card>
+              <mat-card>
+                <mat-card-title>{{ t("import_from_paratext") }}</mat-card-title>
+                <mat-card-content>
+                  <p>{{ t("import_from_paratext_description") }}</p>
+                </mat-card-content>
+                <mat-card-actions>
+                  <button mat-flat-button color="primary" (click)="importFromParatext()">
+                    {{ t("import_from_paratext") }}
+                  </button>
+                </mat-card-actions>
+              </mat-card>
             </div>
             <div class="support-message">
               @for (i of helpInstructions | async; track i) {
@@ -112,6 +123,33 @@ <h2 mat-dialog-title class="dialog-icon-title">
           </div>
         }
 
+        @case ("paratext_tag_selection") {
+          <div class="paratext-tag-selection">
+            @if (loadingParatextTags) {
+              <mat-spinner></mat-spinner>
+            } @else if (paratextTagLoadError) {
+              <div class="paratext-tag-selection-message error">{{ t("import_from_paratext_tag_load_error") }}</div>
+            } @else if (paratextTagOptions.length === 0) {
+              <div class="paratext-tag-selection-message">{{ t("import_from_paratext_no_tags_available") }}</div>
+            } @else {
+              <p class="paratext-tag-selection-message">{{ t("import_from_paratext_select_tag_instructions") }}</p>
+              @if (paratextTagConversionError) {
+                <div class="paratext-tag-selection-message error">
+                  {{ t("import_from_paratext_tag_conversion_error") }}
+                </div>
+              }
+              <mat-form-field appearance="fill">
+                <mat-label>{{ t("import_from_paratext_select_tag") }}</mat-label>
+                <mat-select [(ngModel)]="selectedParatextTagId" [ngModelOptions]="{ standalone: true }">
+                  @for (tag of paratextTagOptions; track tag.id) {
+                    <mat-option [value]="tag.id">{{ tag.name }}</mat-option>
+                  }
+                </mat-select>
+              </mat-form-field>
+            }
+          </div>
+        }
+
         @case ("loading") {
           <div class="loading"><mat-spinner></mat-spinner></div>
         }
@@ -247,6 +285,19 @@ <h2 mat-dialog-title class="dialog-icon-title">
   </mat-dialog-content>
   @if (questionSource != null && importCanceled === false) {
     <mat-dialog-actions align="end">
+      @if (status === "paratext_tag_selection") {
+        <button mat-button (click)="cancelParatextTagSelection()">
+          {{ t("cancel") }}
+        </button>
+        <button
+          mat-flat-button
+          color="primary"
+          (click)="confirmParatextTagSelection()"
+          [disabled]="loadingParatextTags"
+        >
+          {{ t("import_from_paratext_next") }}
+        </button>
+      }
       @if (status === "filter" || status === "file_import_errors") {
         <button mat-button mat-dialog-close>
           {{ t("cancel") }}

src/SIL.XForge.Scripture/ClientApp/src/app/checking/import-questions-dialog/import-questions-dialog.component.scss

@@ -131,6 +131,30 @@
   }
 }
 
+.paratext-tag-selection {
+  display: flex;
+  flex-direction: column;
+  gap: 1.5em;
+  align-items: stretch;
+
+  mat-form-field {
+    width: 100%;
+  }
+
+  mat-spinner {
+    align-self: center;
+  }
+}
+
+.paratext-tag-selection-message {
+  margin: 0;
+  text-align: left;
+
+  &.error {
+    color: var(--mdc-theme-error, #b00020);
+  }
+}
+
 // below is to make denser
 :host ::ng-deep {
   mat-form-field.mat-mdc-form-field.mat-form-field-appearance-outline {

src/SIL.XForge.Scripture/ClientApp/src/app/checking/import-questions-dialog/import-questions-dialog.component.ts

@@ -19,6 +19,7 @@ import { MatIcon } from '@angular/material/icon';
 import { MatInput } from '@angular/material/input';
 import { MatProgressBar } from '@angular/material/progress-bar';
 import { MatProgressSpinner } from '@angular/material/progress-spinner';
+import { MatOption, MatSelect } from '@angular/material/select';
 import {
   MatCell,
   MatCellDef,
@@ -45,10 +46,13 @@ import { RealtimeQuery } from 'xforge-common/models/realtime-query';
 import { OnlineStatusService } from 'xforge-common/online-status.service';
 import { RetryingRequest } from 'xforge-common/retrying-request.service';
 import { quietTakeUntilDestroyed } from 'xforge-common/util/rxjs-util';
+import { stripHtml } from 'xforge-common/util/string-util';
 import { objectId } from 'xforge-common/utils';
 import { environment } from '../../../environments/environment';
+import { ParatextProject } from '../../core/models/paratext-project';
 import { QuestionDoc } from '../../core/models/question-doc';
 import { TextsByBookId } from '../../core/models/texts-by-book-id';
+import { ParatextNote, ParatextNoteTag, ParatextService } from '../../core/paratext.service';
 import { SFProjectService } from '../../core/sf-project.service';
 import {
   ScriptureChooserDialogComponent,
@@ -93,7 +97,14 @@ interface DialogListItem {
 }
 
 type DialogErrorState = 'update_transcelerator' | 'file_import_errors' | 'missing_header_row' | 'offline_conversion';
-type DialogStatus = 'initial' | 'no_questions' | 'filter' | 'loading' | 'progress' | DialogErrorState;
+type DialogStatus =
+  | 'initial'
+  | 'no_questions'
+  | 'filter'
+  | 'loading'
+  | 'progress'
+  | 'paratext_tag_selection'
+  | DialogErrorState;
 
 @Component({
   templateUrl: './import-questions-dialog.component.html',
@@ -134,11 +145,13 @@ type DialogStatus = 'initial' | 'no_questions' | 'filter' | 'loading' | 'progres
     MatCheckbox,
     MatProgressBar,
     MatDialogActions,
-    AsyncPipe
+    AsyncPipe,
+    MatSelect,
+    MatOption
   ]
 })
 export class ImportQuestionsDialogComponent implements OnDestroy {
-  questionSource: null | 'transcelerator' | 'csv_file' = null;
+  questionSource: null | 'transcelerator' | 'csv_file' | 'paratext' = null;
 
   questionList: DialogListItem[] = [];
   filteredList: DialogListItem[] = [];
@@ -155,6 +168,15 @@ export class ImportQuestionsDialogComponent implements OnDestroy {
   importCanceled: boolean = false;
   fileExtensions: string = '.csv,.tsv';
 
+  showParatextTagSelector = false;
+  loadingParatextTags = false;
+  paratextTagLoadError = false;
+  paratextTagConversionError = false;
+  paratextTagOptions: ParatextNoteTag[] = [];
+  selectedParatextTagId: number | null = null;
+  private paratextNotes: ParatextNote[] = [];
+  private paratextProjectsPromise?: Promise<ParatextProject[] | undefined>;
+
   @ViewChild('selectAllCheckbox') selectAllCheckbox!: MatCheckbox;
   @ViewChild('dialogContentBody') dialogContentBody!: ElementRef;
 
@@ -177,6 +199,7 @@ export class ImportQuestionsDialogComponent implements OnDestroy {
     private readonly destroyRef: DestroyRef,
     @Inject(MAT_DIALOG_DATA) public readonly data: ImportQuestionsDialogData,
     projectService: SFProjectService,
+    private readonly paratextService: ParatextService,
     private readonly checkingQuestionsService: CheckingQuestionsService,
     private readonly onlineStatusService: OnlineStatusService,
     private readonly dialogRef: MatDialogRef<ImportQuestionsDialogComponent>,
@@ -239,6 +262,9 @@ export class ImportQuestionsDialogComponent implements OnDestroy {
     if (this.importing) {
       return 'progress';
     }
+    if (this.showParatextTagSelector) {
+      return 'paratext_tag_selection';
+    }
     if (this.invalidRows.length !== 0) {
       return 'file_import_errors';
     }
@@ -414,6 +440,7 @@ export class ImportQuestionsDialogComponent implements OnDestroy {
           isArchived: false,
           dateCreated: currentDate,
           dateModified: currentDate,
+          //todo rename
           transceleratorQuestionId: listItem.question.id
         };
         await this.zone.runOutsideAngular(() =>
@@ -466,6 +493,165 @@ export class ImportQuestionsDialogComponent implements OnDestroy {
     this.loading = false;
   }
 
+  async importFromParatext(): Promise<void> {
+    if (this.loadingParatextTags) {
+      return;
+    }
+
+    this.importClicked = false;
+    this.errorState = undefined;
+    this.questionSource = 'paratext';
+    this.questionList = [];
+    this.filteredList = [];
+    this.invalidRows = [];
+    this.loadingParatextTags = true;
+    this.showParatextTagSelector = true;
+    this.paratextTagLoadError = false;
+    this.paratextTagConversionError = false;
+    this.selectedParatextTagId = null;
+    this.paratextNotes = [];
+    this.paratextTagOptions = [];
+
+    try {
+      const paratextId = await this.getParatextProjectId();
+      if (paratextId == null) {
+        this.paratextTagLoadError = true;
+        return;
+      }
+
+      const notes = await this.paratextService.getNotes(paratextId);
+      this.paratextNotes = notes ?? [];
+      this.paratextTagOptions = this.collectParatextTagOptions(this.paratextNotes);
+      if (this.paratextTagOptions.length > 0) {
+        this.selectedParatextTagId = this.paratextTagOptions[0].id;
+      }
+    } catch (error) {
+      this.paratextNotes = [];
+      this.paratextTagOptions = [];
+      this.selectedParatextTagId = null;
+      this.paratextTagLoadError = true;
+    } finally {
+      this.loadingParatextTags = false;
+    }
+  }
+
+  cancelParatextTagSelection(): void {
+    this.showParatextTagSelector = false;
+    this.questionSource = null;
+    this.loadingParatextTags = false;
+    this.paratextTagLoadError = false;
+    this.paratextTagConversionError = false;
+    this.paratextNotes = [];
+    this.paratextTagOptions = [];
+    this.selectedParatextTagId = null;
+    this.importClicked = false;
+  }
+
+  async confirmParatextTagSelection(): Promise<void> {
+    const tagId = this.selectedParatextTagId;
+    if (tagId == null) {
+      return;
+    }
+
+    this.loading = true;
+    this.showParatextTagSelector = false;
+    this.questionList = [];
+    this.filteredList = [];
+    this.invalidRows = [];
+    this.paratextTagConversionError = false;
+
+    try {
+      const questions = this.convertParatextCommentsToQuestions(this.paratextNotes, tagId);
+      await this.setUpQuestionList(questions, true);
+    } catch (error) {
+      this.paratextTagConversionError = true;
+      this.showParatextTagSelector = true;
+      return;
+    } finally {
+      this.loading = false;
+    }
+  }
+
+  private async getParatextProjectId(): Promise<string | undefined> {
+    try {
+      this.paratextProjectsPromise ??= this.paratextService.getProjects();
+      const projects = await this.paratextProjectsPromise;
+      const project = projects?.find(p => p.projectId === this.data.projectId);
+      return project?.paratextId;
+    } catch (error) {
+      console.error('Failed to load Paratext project list', error);
+      this.paratextProjectsPromise = undefined;
+      throw error;
+    }
+  }
+
+  private collectParatextTagOptions(notes: ParatextNote[]): ParatextNoteTag[] {
+    const tagMap = new Map<number, ParatextNoteTag>();
+    for (const note of notes) {
+      for (const comment of note.comments ?? []) {
+        if (comment.tag != null && !tagMap.has(comment.tag.id)) {
+          tagMap.set(comment.tag.id, comment.tag);
+        }
+      }
+    }
+
+    return Array.from(tagMap.values()).sort((a, b) =>
+      a.name.localeCompare(b.name, this.i18n.localeCode, { sensitivity: 'base' })
+    );
+  }
+
+  private convertParatextCommentsToQuestions(notes: ParatextNote[], tagId: number): SourceQuestion[] {
+    const questions: SourceQuestion[] = [];
+
+    for (const note of notes) {
+      const comments = note.comments ?? [];
+      for (let index = 0; index < comments.length; index++) {
+        const comment = comments[index];
+        if (comment.tag == null || comment.tag.id !== tagId) {
+          continue;
+        }
+
+        const verseRef = this.parseVerseReference(note.verseRef);
+        if (verseRef == null) {
+          continue;
+        }
+
+        const questionText = stripHtml(comment.content ?? '').trim();
+        if (questionText.length === 0) {
+          continue;
+        }
+
+        questions.push({
+          id: note.id,
+          verseRef,
+          text: questionText
+        });
+
+        break;
+      }
+    }
+
+    return questions;
+  }
+
+  private parseVerseReference(reference: string | undefined): VerseRef | null {
+    if (reference == null) {
+      return null;
+    }
+
+    const trimmedReference = reference.trim();
+    if (trimmedReference.length === 0) {
+      return null;
+    }
+
+    const parseResult = VerseRef.tryParse(trimmedReference);
+    if (parseResult.success !== true || parseResult.verseRef == null) {
+      return null;
+    }
+
+    return parseResult.verseRef;
+  }
+
   async fileSelected(file: File): Promise<void> {
     this.loading = true;