Ingest async implementation

CHANGELOG.md

@@ -4,10 +4,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [UNRELEASED]
+
+### Changed
+
+- [BREAKING] All synchronous queued and streaming ingestion APIs now delegate to their asynchronous counterparts
+  internally and block for results.
+- [BREAKING] Streaming client no longer check for blob size and if it exists.
+- [BREAKING] Exceptions thrown the ingest API are now RuntimeExceptions: IngestionServiceException, IngestionClientException.
+### Added
+- The SDK now provides Reactor Core-based asynchronous APIs for all queued and streaming ingestion endpoints,
+  enabling non-blocking operations.
+
+
 ## [6.0.3] - 2025-25-06
 ### Fixed
 - Fixed `ManagedIdentityTokenProvider`
 
+
 ## [6.0.2] - 2025-24-04
 
 ### Fixed
@@ -19,16 +33,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - The SDK now provides Reactor Core-based asynchronous APIs for all query, management, streaming query/ingestion (StreamingClient) endpoints,
 enabling non-blocking operations. You can read more about Reactor Core and [Mono type here](https://projectreactor.io/docs/core/release/api/).
 - `ConnectionStringBuilder` now supports keywords without regards to spaces or case. It now supports `toString()` that prints a canonical connection string, with censored secrets by default.
+
 ### Changed
 - [BREAKING] All synchronous query/management, streaming query/ingestion (StreamingClient) APIs now delegate to their asynchronous counterparts
 internally and block for results.
-- [BREAKING] * Make ManagedStreamingQueuingPolicy internal, expose just a factor
-* Dont allow users to pass raw data size, provide it only if we have it
+- [BREAKING] Make ManagedStreamingQueuingPolicy internal, expose just a factor
+- [BREAKING] Don't allow users to pass raw data size, provide it only if we have it
+
 - [BREAKING] Removing max keep alive from HttpClientPropertiesBuilder.
 ### Fixed
 - Fixed edge cases in query timeouts.
 - Long Queries would time out after 2 minutes. Remove keep alive timeout to fix.
 
+
 ## [6.0.0-ALPHA-01] - 2024-11-27
 ### Added
 - A new policy heuristic for choosing between queuing and streaming in Managed streaming client. A policy can be configured

data/src/main/java/com/microsoft/azure/kusto/data/Client.java

@@ -7,33 +7,150 @@
 import com.microsoft.azure.kusto.data.exceptions.DataServiceException;
 import reactor.core.publisher.Mono;
 
+/**
+ * A client for interacting with Kusto.
+ */
 public interface Client {
 
-    KustoOperationResult executeQuery(String command) throws DataServiceException, DataClientException;
-
-    KustoOperationResult executeQuery(String database, String command) throws DataServiceException, DataClientException;
-
-    KustoOperationResult executeQuery(String database, String command, ClientRequestProperties properties) throws DataServiceException, DataClientException;
-
+    /**
+     * Executes a query against the default database.
+     *
+     * @param query The query command to execute.
+     * @return The result of the query as a {@link KustoOperationResult}.
+     * @throws DataServiceException If there is an error from the service.
+     * @throws DataClientException If there is an error on the client side.
+     */
+    KustoOperationResult executeQuery(String query) throws DataServiceException, DataClientException;
+
+    /**
+     * Executes a query against the specified database.
+     *
+     * @param database The name of the database.
+     * @param query The query to execute.
+     * @return A {@link KustoOperationResult} with the result of the query.
+     * @throws DataServiceException If there is an error from the service.
+     * @throws DataClientException If there is an error on the client side.
+     */
+    KustoOperationResult executeQuery(String database, String query) throws DataServiceException, DataClientException;
+
+    /**
+     * Executes a query against a specified database with additional request properties.
+     *
+     * @param database The name of the database.
+     * @param query The query query to execute.
+     * @param properties Additional request properties.
+     * @return A {@link KustoOperationResult} with the result of the query.
+     * @throws DataServiceException If there is an error from the service.
+     * @throws DataClientException If there is an error on the client side.
+     */
+    KustoOperationResult executeQuery(String database, String query, ClientRequestProperties properties) throws DataServiceException, DataClientException;
+
+    /**
+     * Executes a query against the default database.
+     *
+     * @param command The query to execute.
+     * @return A {@link KustoOperationResult} emitting the result of the query.
+     */
     Mono<KustoOperationResult> executeQueryAsync(String command);
 
+    /**
+     * Executes a query against a specified database.
+     *
+     * @param database The name of the database.
+     * @param command The query command to execute.
+     * @return A {@link KustoOperationResult} with the result of the query.
+     */
     Mono<KustoOperationResult> executeQueryAsync(String database, String command);
 
+    /**
+     * Executes a query command asynchronously against a specified database with additional request properties.
+     *
+     * @param database The name of the database.
+     * @param command The query command to execute.
+     * @param properties Additional request properties.
+     * @return A {@link Mono} emitting the result of the query as a {@link KustoOperationResult}.
+     */
     Mono<KustoOperationResult> executeQueryAsync(String database, String command, ClientRequestProperties properties);
 
+    /**
+     * Executes a management command against the default database.
+     *
+     * @param command The management command to execute.
+     * @return The result of the command as a {@link KustoOperationResult}.
+     * @throws DataServiceException If there is an error from the service.
+     * @throws DataClientException If there is an error on the client side.
+     */
     KustoOperationResult executeMgmt(String command) throws DataServiceException, DataClientException;
 
+    /**
+     * Executes a management command against a specified database.
+     *
+     * @param database The name of the database.
+     * @param command The management command to execute.
+     * @return The result of the command as a {@link KustoOperationResult}.
+     * @throws DataServiceException If there is an error from the service.
+     * @throws DataClientException If there is an error on the client side.
+     */
     KustoOperationResult executeMgmt(String database, String command) throws DataServiceException, DataClientException;
 
+    /**
+     * Executes a management command against a specified database with additional request properties.
+     *
+     * @param database The name of the database.
+     * @param command The management command to execute.
+     * @param properties Additional request properties.
+     * @return The result of the command as a {@link KustoOperationResult}.
+     * @throws DataServiceException If there is an error from the service.
+     * @throws DataClientException If there is an error on the client side.
+     */
     KustoOperationResult executeMgmt(String database, String command, ClientRequestProperties properties) throws DataServiceException, DataClientException;
 
+    /**
+     * Executes a management command asynchronously against the default database.
+     *
+     * @param command The management command to execute.
+     * @return A {@link Mono} emitting the result of the command as a {@link KustoOperationResult}.
+     */
     Mono<KustoOperationResult> executeMgmtAsync(String command);
 
+    /**
+     * Executes a management command asynchronously against a specified database.
+     *
+     * @param database The name of the database.
+     * @param command The management command to execute.
+     * @return A {@link Mono} emitting the result of the command as a {@link KustoOperationResult}.
+     */
     Mono<KustoOperationResult> executeMgmtAsync(String database, String command);
 
+    /**
+     * Executes a management command asynchronously against a specified database with additional request properties.
+     *
+     * @param database The name of the database.
+     * @param command The management command to execute.
+     * @param properties Additional request properties.
+     * @return A {@link Mono} emitting the result of the command as a {@link KustoOperationResult}.
+     */
     Mono<KustoOperationResult> executeMgmtAsync(String database, String command, ClientRequestProperties properties);
 
+    /**
+     * Executes a query command and returns the result as a JSON string.
+     *
+     * @param database The name of the database.
+     * @param command The query command to execute.
+     * @param properties Additional request properties.
+     * @return The result of the query as a JSON string.
+     * @throws DataServiceException If there is an error from the service.
+     * @throws DataClientException If there is an error on the client side.
+     */
     String executeToJsonResult(String database, String command, ClientRequestProperties properties) throws DataServiceException, DataClientException;
 
+    /**
+     * Executes a query command asynchronously and returns the result as a JSON string.
+     *
+     * @param database The name of the database.
+     * @param command The query command to execute.
+     * @param properties Additional request properties.
+     * @return A {@link Mono} emitting the result of the query as a JSON string.
+     */
     Mono<String> executeToJsonResultAsync(String database, String command, ClientRequestProperties properties);
 }

data/src/main/java/com/microsoft/azure/kusto/data/ClientImpl.java

@@ -76,33 +76,33 @@ public ClientImpl(ConnectionStringBuilder csb, HttpClient httpClient) throws URI
     }
 
     @Override
-    public KustoOperationResult executeQuery(String command) {
-        return executeQuery(defaultDatabaseName, command);
+    public KustoOperationResult executeQuery(String query) {
+        return executeQuery(defaultDatabaseName, query);
     }
 
     @Override
-    public KustoOperationResult executeQuery(String database, String command) {
-        return executeQuery(database, command, null);
+    public KustoOperationResult executeQuery(String database, String query) {
+        return executeQuery(database, query, null);
     }
 
     @Override
-    public KustoOperationResult executeQuery(String database, String command, ClientRequestProperties properties) {
-        return executeQueryAsync(database, command, properties).block();
+    public KustoOperationResult executeQuery(String database, String query, ClientRequestProperties properties) {
+        return executeQueryAsync(database, query, properties).block();
     }
 
     @Override
-    public Mono<KustoOperationResult> executeQueryAsync(String command) {
-        return executeQueryAsync(defaultDatabaseName, command);
+    public Mono<KustoOperationResult> executeQueryAsync(String query) {
+        return executeQueryAsync(defaultDatabaseName, query);
     }
 
     @Override
-    public Mono<KustoOperationResult> executeQueryAsync(String database, String command) {
-        return executeQueryAsync(database, command, null);
+    public Mono<KustoOperationResult> executeQueryAsync(String database, String query) {
+        return executeQueryAsync(database, query, null);
     }
 
     @Override
-    public Mono<KustoOperationResult> executeQueryAsync(String database, String command, ClientRequestProperties properties) {
-        return executeAsync(database, command, properties, CommandType.QUERY);
+    public Mono<KustoOperationResult> executeQueryAsync(String database, String query, ClientRequestProperties properties) {
+        return executeAsync(database, query, properties, CommandType.QUERY);
     }
 
     @Override

data/src/main/java/com/microsoft/azure/kusto/data/ExponentialRetry.java

@@ -3,6 +3,7 @@
 import java.lang.invoke.MethodHandles;
 import java.time.Duration;
 import java.util.List;
+import java.util.function.Predicate;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -12,7 +13,7 @@
 import reactor.util.annotation.Nullable;
 import reactor.util.retry.Retry;
 
-public class ExponentialRetry<E1 extends Throwable, E2 extends Throwable> {
+public class ExponentialRetry {
     private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
     private final int maxAttempts;
@@ -37,50 +38,25 @@ public ExponentialRetry(ExponentialRetry other) {
         this.maxJitterSecs = other.maxJitterSecs;
     }
 
-    // Caller should throw only permanent errors, returning null if a retry is needed
-    public <T> T execute(KustoCheckedFunction<Integer, T, E1, E2> function) throws E1, E2 {
-        for (int currentAttempt = 0; currentAttempt < maxAttempts; currentAttempt++) {
-            log.info("execute: Attempt {}", currentAttempt);
-
-            try {
-                T result = function.apply(currentAttempt);
-                if (result != null) {
-                    return result;
-                }
-            } catch (Exception e) {
-                log.error("execute: Error is permanent, stopping", e);
-                throw e;
-            }
-
-            double currentSleepSecs = sleepBaseSecs * (float) Math.pow(2, currentAttempt);
-            double jitterSecs = (float) Math.random() * maxJitterSecs;
-            double sleepMs = (currentSleepSecs + jitterSecs) * 1000;
-
-            log.info("execute: Attempt {} failed, trying again after sleep of {} seconds", currentAttempt, sleepMs / 1000);
-
-            try {
-                Thread.sleep((long) sleepMs);
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-                throw new RuntimeException("execute: Interrupted while sleeping", e);
-            }
-        }
-
-        return null;
-    }
-
     /**
      * Creates a retry mechanism with exponential backoff and jitter.
      *
      * @param retriableErrorClasses A list of error classes that are considered retriable. If null,
      *                              the method does not retry.
+    * @param filter A filter to use. Default is retrying retriable errors.
      * @return A configured {@link Retry} instance
      */
-    public Retry retry(@Nullable List<Class<? extends Throwable>> retriableErrorClasses) {
+    public Retry retry(@Nullable List<Class<? extends Throwable>> retriableErrorClasses,
+            @Nullable Predicate<? super Throwable> filter) {
+        if (retriableErrorClasses != null && filter != null) {
+            throw new IllegalArgumentException("Cannot specify both retriableErrorClasses and filter");
+        }
+
+        Predicate<? super Throwable> filterToUse = filter == null ? throwable -> shouldRetry(throwable, retriableErrorClasses) : filter;
         return Retry.backoff(maxAttempts, Duration.ofSeconds((long) sleepBaseSecs))
                 .maxBackoff(Duration.ofSeconds(30))
                 .jitter(maxJitterSecs)
-                .filter(throwable -> shouldRetry(throwable, retriableErrorClasses))
+                .filter(filterToUse)
                 .doAfterRetry(retrySignal -> {
                     long currentAttempt = retrySignal.totalRetries() + 1;
                     log.info("Attempt {} failed.", currentAttempt);
@@ -100,5 +76,4 @@ private static boolean shouldRetry(Throwable failure, List<Class<? extends Throw
 
         return false;
     }
-
 }

data/src/main/java/com/microsoft/azure/kusto/data/Utils.java

@@ -156,29 +156,29 @@ public static Mono<String> processGzipBody(Flux<ByteBuffer> gzipBody) {
         // to occur in chunks, making it more memory-efficient for large payloads, as it prevents the entire
         // compressed stream from being loaded into memory at once (which for example is required by GZIPInputStream for decompression).
 
-        EmbeddedChannel channel = new EmbeddedChannel(ZlibCodecFactory.newZlibDecoder(ZlibWrapper.GZIP));
+        EmbeddedChannel decoder = new EmbeddedChannel(ZlibCodecFactory.newZlibDecoder(ZlibWrapper.GZIP));
 
         return gzipBody
                 .reduce(new StringBuilder(), (stringBuilder, byteBuffer) -> {
-                    channel.writeInbound(Unpooled.wrappedBuffer(byteBuffer)); // Write chunk to channel for decompression
+                    decoder.writeInbound(Unpooled.wrappedBuffer(byteBuffer)); // Write chunk to channel for decompression
 
-                    ByteBuf decompressedByteBuf = channel.readInbound();
+                    ByteBuf decompressedByteBuf = decoder.readInbound();
                     if (decompressedByteBuf == null) {
                         return stringBuilder;
                     }
 
                     String string = decompressedByteBuf.toString(StandardCharsets.UTF_8);
                     decompressedByteBuf.release();
 
-                    if (!channel.inboundMessages().isEmpty()) {
+                    if (!decoder.inboundMessages().isEmpty()) { // TODO: remove this when we are sure that only one message exists in the channel
                         throw new IllegalStateException("Expected exactly one message in the channel.");
                     }
 
                     stringBuilder.append(string);
                     return stringBuilder;
                 })
                 .map(StringBuilder::toString)
-                .doFinally(ignore -> channel.finishAndReleaseAll());
+                .doFinally(ignore -> decoder.finishAndReleaseAll());
     }
 
     private static Mono<String> processNonGzipBody(Flux<ByteBuffer> gzipBody) {

data/src/main/java/com/microsoft/azure/kusto/data/auth/CloudInfo.java

@@ -68,7 +68,7 @@ public class CloudInfo implements TraceableAttributes, Serializable {
     private final String kustoServiceResourceId;
     private final String firstPartyAuthorityUrl;
     private static final int ATTEMPT_COUNT = 3;
-    private static final Retry RETRY_CONFIG = new ExponentialRetry<>(ATTEMPT_COUNT).retry(null);
+    private static final Retry RETRY_CONFIG = new ExponentialRetry(ATTEMPT_COUNT).retry(null, null);
 
     public CloudInfo(boolean loginMfaRequired, String loginEndpoint, String kustoClientAppId, String kustoClientRedirectUri, String kustoServiceResourceId,
             String firstPartyAuthorityUrl) {

data/src/main/java/com/microsoft/azure/kusto/data/http/HttpClientFactory.java

@@ -60,8 +60,6 @@ public static HttpClient create(HttpClientProperties properties) {
             options.setProxyOptions(properties.getProxy());
         }
 
-        // Todo: Is the per route connection maximum needed anymore?
-
         return HttpClient.createDefault(options);
     }
 }

data/src/test/java/com/microsoft/azure/kusto/data/exceptions/KustoServiceQueryErrorTest.java

@@ -56,7 +56,6 @@ void testFromOneApiErrorArrayWithMultipleExceptionsOneApi() throws JsonProcessin
         ObjectMapper objectMapper = new ObjectMapper();
         ArrayNode jsonExceptions = (ArrayNode) objectMapper.readTree(json).get("OneApiErrors");
 
-
         KustoServiceQueryError error = KustoServiceQueryError.fromOneApiErrorArray(jsonExceptions, true);
 
         assertEquals("Query execution failed with multiple inner exceptions:\n" +