Plain mongo connector with fixes to schema discovery and no hardcoded read preference

airbyte-integrations/connectors/source-mongodb-v2-plain/README.md

@@ -0,0 +1,69 @@
+# MongoDb Source
+
+## Documentation
+
+This is the repository for the MongoDb source connector in Java.
+For information about how to use this connector within Airbyte, see [User Documentation](https://docs.airbyte.io/integrations/sources/mongodb-v2)
+
+## Local development
+
+#### Building via Gradle
+
+From the Airbyte repository root, run:
+
+```
+./gradlew :airbyte-integrations:connectors:source-mongodb-v2-plain:build
+```
+
+### Locally running the connector docker image
+
+#### Build
+
+Build the connector image via Gradle:
+
+```
+./gradlew :airbyte-integrations:connectors:source-mongodb-v2-plain:buildConnectorImage
+```
+
+Once built, the docker image name and tag on your host will be `airbyte/source-mongodb-v2-plain:dev`.
+the Dockerfile.
+
+## Testing
+
+We use `JUnit` for Java tests.
+
+### Test Configuration
+
+No specific configuration needed for testing Standalone MongoDb instance, MongoDb Test Container is used.
+In order to test the MongoDb Atlas or Replica set, you need to provide configuration parameters.
+
+## Community Contributor
+
+As a community contributor, you will need to have an Atlas cluster to test MongoDb source.
+
+1. Create `secrets/credentials.json` file
+   1. Insert below json to the file with your configuration
+      ```
+      {
+           "cluster_type": "ATLAS_REPLICA_SET"
+           "database": "database_name",
+           "username": "username",
+           "password": "password",
+           "connection_string": "mongodb+srv://cluster0.abcd1.mongodb.net/",
+           "auth_source": "auth_database",
+      }
+      ```
+      where `installation_type` is one of `ATLAS_REPLICA_SET` or `SELF_HOSTED_REPLICA_SET` depending on the location of the target cluster.
+
+## Airbyte Employee
+
+1. Access the `MONGODB_TEST_CREDS` secret on LastPass
+1. Create a file with the contents at `secrets/credentials.json`
+
+#### Acceptance Tests
+
+To run acceptance and custom integration tests:
+
+```
+./gradlew :airbyte-integrations:connectors:source-mongodb-v2-plain:integrationTest
+```

airbyte-integrations/connectors/source-mongodb-v2-plain/acceptance-test-config.yml

@@ -0,0 +1,33 @@
+# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-mongodb-v2-plain:dev
+acceptance_tests:
+  spec:
+    tests:
+      - spec_path: "integration_tests/expected_spec.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "0.2.5"
+        config_path: "secrets/credentials.json"
+        timeout_seconds: 60
+  connection:
+    tests:
+      - config_path: "secrets/credentials.json"
+        status: "succeed"
+        timeout_seconds: 60
+  discovery:
+    tests:
+      - config_path: "secrets/credentials.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "0.2.5"
+        timeout_seconds: 60
+  basic_read:
+    bypass_reason: "Full refresh syncs are not supported on this connector."
+  full_refresh:
+    bypass_reason: "Full refresh syncs are not supported on this connector."
+  incremental:
+    #    bypass_reason: "Incremental syncs are not yet supported by this connector."
+    #    #TODO: remove bypass_reason once a version that supports incremental syncs is released + uncomment the lines below
+    tests:
+      - config_path: "secrets/credentials.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        timeout_seconds: 180

airbyte-integrations/connectors/source-mongodb-v2-plain/build.gradle

@@ -0,0 +1,123 @@
+plugins {
+    id 'airbyte-java-connector'
+}
+
+airbyteJavaConnector {
+    cdkVersionRequired = '0.38.1'
+    features = ['db-sources', 'datastore-mongo']
+    useLocalCdk = false
+}
+
+application {
+    mainClass = 'io.airbyte.integrations.source.mongodb.MongoDbSource'
+    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
+}
+
+configurations {
+    dataGenerator.extendsFrom testImplementation
+    debeziumTest.extendsFrom testImplementation
+}
+
+sourceSets {
+    dataGenerator {
+        kotlin {
+            srcDirs('src/test/generator')
+        }
+    }
+    debeziumTest {
+        kotlin {
+            srcDirs('src/test/debezium')
+        }
+    }
+}
+
+java {
+    compileJava {
+        options.compilerArgs += "-Xlint:-try,-rawtypes"
+    }
+}
+
+dependencies {
+    implementation 'io.debezium:debezium-embedded:2.6.2.Final'
+    implementation 'io.debezium:debezium-connector-mongodb:2.6.2.Final'
+
+    testImplementation 'org.testcontainers:mongodb:1.19.0'
+
+    dataGeneratorImplementation project(':airbyte-integrations:connectors:source-mongodb-v2-plain')
+    dataGeneratorImplementation platform('com.fasterxml.jackson:jackson-bom:2.15.2')
+    dataGeneratorImplementation 'com.fasterxml.jackson.core:jackson-databind'
+    dataGeneratorImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
+
+    dataGeneratorImplementation ('com.github.javafaker:javafaker:1.0.2') { exclude module: 'snakeyaml' }
+    dataGeneratorImplementation 'io.github.oshai:kotlin-logging-jvm:5.1.0'
+    dataGeneratorImplementation 'org.jetbrains.kotlinx:kotlinx-cli-jvm:0.3.5'
+    dataGeneratorImplementation 'org.mongodb:mongodb-driver-sync:4.10.2'
+
+    debeziumTestImplementation 'io.debezium:debezium-embedded:2.6.0.Final'
+    debeziumTestImplementation 'io.debezium:debezium-connector-mongodb:2.6.0.Final'
+    debeziumTestImplementation 'org.jetbrains.kotlinx:kotlinx-cli-jvm:0.3.5'
+    debeziumTestImplementation 'com.github.spotbugs:spotbugs-annotations:4.7.3'
+}
+
+/*
+ * Executes the script that generates test data and inserts it into the provided database/collection.
+ *
+ * To execute this task, use the following command:
+ *
+ * ./gradlew :airbyte-integrations:connectors:source-mongodb-v2-plain:generateTestData -PconnectionString=<connection string> -PdatabaseName=<database name> -PcollectionName=<collection name> -Pusername=<username>
+ *
+ * Optionally, you can provide -PnumberOfDocuments to change the number of generated documents from the default (10,000).
+ */
+tasks.register('generateTestData', JavaExec) {
+    def arguments = []
+
+    if(project.hasProperty('collectionName')) {
+        arguments.addAll(['--collection-name', collectionName])
+    }
+    if(project.hasProperty('connectionString')) {
+        arguments.addAll(['--connection-string', connectionString])
+    }
+    if(project.hasProperty('databaseName')) {
+        arguments.addAll(['--database-name', databaseName])
+    }
+    if (project.hasProperty('numberOfDocuments')) {
+        arguments.addAll(['--number', numberOfDocuments])
+    }
+    if(project.hasProperty('username')) {
+        arguments.addAll(['--username', username])
+    }
+
+    classpath = sourceSets.dataGenerator.runtimeClasspath
+    main 'io.airbyte.integrations.source.mongodb.MongoDbInsertClient'
+    standardInput = System.in
+    args arguments
+}
+
+/**
+ * Executes the Debezium MongoDB Connector test harness.
+ *
+ * To execute this task, use the following command:
+ *
+ * ./gradlew :airbyte-integrations:connectors:source-mongodb-v2-plain:debeziumTest -PconnectionString=<connection string> -PdatabaseName=<database name> -PcollectionName=<collection name> -Pusername=<username>
+ */
+tasks.register('debeziumTest', JavaExec) {
+    def arguments = []
+
+    if(project.hasProperty('collectionName')) {
+        arguments.addAll(['--collection-name', collectionName])
+    }
+    if(project.hasProperty('connectionString')) {
+        arguments.addAll(['--connection-string', connectionString])
+    }
+    if(project.hasProperty('databaseName')) {
+        arguments.addAll(['--database-name', databaseName])
+    }
+    if(project.hasProperty('username')) {
+        arguments.addAll(['--username', username])
+    }
+
+    classpath = sourceSets.debeziumTest.runtimeClasspath
+    main 'io.airbyte.integrations.source.mongodb.DebeziumMongoDbConnectorTest'
+    standardInput = System.in
+    args arguments
+}

airbyte-integrations/connectors/source-mongodb-v2-plain/gradle.properties

@@ -0,0 +1 @@
+JunitMethodExecutionTimeout=2 m

airbyte-integrations/connectors/source-mongodb-v2-plain/integration_tests/acceptance.py

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import pytest
+
+pytest_plugins = ("connector_acceptance_test.plugin",)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """This fixture is a placeholder for external resources that acceptance test might require."""
+    # TODO: setup test dependencies if needed. otherwise remove the TODO comments
+    yield
+    # TODO: clean up test dependencies