From 71ff580e5e34cc775be30f9fdea232d375a66697 Mon Sep 17 00:00:00 2001 From: David Kjerrumgaard Date: Mon, 4 Mar 2019 15:13:47 -0800 Subject: [PATCH] Added support for Nifi 1.8 --- .gitignore | 7 + nifi-pulsar-client-service-api/pom.xml | 45 ++ .../nifi/pulsar/PulsarClientService.java | 33 + .../pulsar/cache/PulsarClientLRUCache.java | 59 ++ .../cache/PulsarClientLRUCacheTest.java | 120 ++++ nifi-pulsar-client-service-nar/pom.xml | 41 ++ .../src/main/resources/META-INF/LICENSE | 329 ++++++++++ .../src/main/resources/META-INF/NOTICE | 612 ++++++++++++++++++ nifi-pulsar-client-service/pom.xml | 68 ++ .../pulsar/StandardPulsarClientService.java | 275 ++++++++ ...g.apache.nifi.controller.ControllerService | 15 + .../nifi/pulsar/PulsarClientServiceIT.java | 130 ++++ .../org/apache/nifi/pulsar/TestProcessor.java | 45 ++ .../TestStandardPulsarClientService.java | 47 ++ nifi-pulsar-nar/pom.xml | 50 ++ nifi-pulsar-processors/pom.xml | 89 +++ .../AbstractPulsarConsumerProcessor.java | 468 ++++++++++++++ .../AbstractPulsarProducerProcessor.java | 485 ++++++++++++++ .../pulsar/pubsub/ConsumePulsar.java | 184 ++++++ .../pulsar/pubsub/ConsumePulsarRecord.java | 347 ++++++++++ .../pulsar/pubsub/PublishPulsar.java | 147 +++++ .../pulsar/pubsub/PublishPulsarRecord.java | 185 ++++++ .../org.apache.nifi.processor.Processor | 18 + .../pulsar/AbstractPulsarProcessorTest.java | 41 ++ .../pulsar/pubsub/TestConsumePulsar.java | 169 +++++ .../pubsub/TestConsumePulsarRecord.java | 128 ++++ .../pulsar/pubsub/TestPublishPulsar.java | 40 ++ .../pubsub/TestPublishPulsarRecord.java | 107 +++ .../pubsub/async/TestAsyncConsumePulsar.java | 95 +++ .../async/TestAsyncConsumePulsarRecord.java | 145 +++++ .../pubsub/async/TestAsyncPublishPulsar.java | 148 +++++ .../async/TestAsyncPublishPulsarRecord.java | 156 +++++ .../pubsub/mocks/MockPulsarClientService.java | 202 ++++++ .../pulsar/pubsub/mocks/MockRecordParser.java | 108 ++++ .../pulsar/pubsub/mocks/MockRecordWriter.java | 155 +++++ .../pubsub/sync/TestSyncConsumePulsar.java | 122 ++++ .../sync/TestSyncConsumePulsarRecord.java | 144 +++++ .../pubsub/sync/TestSyncPublishPulsar.java | 148 +++++ .../sync/TestSyncPublishPulsarRecord.java | 137 ++++ pom.xml | 41 ++ 40 files changed, 5885 insertions(+) create mode 100644 .gitignore create mode 100644 nifi-pulsar-client-service-api/pom.xml create mode 100644 nifi-pulsar-client-service-api/src/main/java/org/apache/nifi/pulsar/PulsarClientService.java create mode 100644 nifi-pulsar-client-service-api/src/main/java/org/apache/nifi/pulsar/cache/PulsarClientLRUCache.java create mode 100644 nifi-pulsar-client-service-api/src/test/java/org/apache/nifi/pulsar/cache/PulsarClientLRUCacheTest.java create mode 100644 nifi-pulsar-client-service-nar/pom.xml create mode 100644 nifi-pulsar-client-service-nar/src/main/resources/META-INF/LICENSE create mode 100644 nifi-pulsar-client-service-nar/src/main/resources/META-INF/NOTICE create mode 100644 nifi-pulsar-client-service/pom.xml create mode 100644 nifi-pulsar-client-service/src/main/java/org/apache/nifi/pulsar/StandardPulsarClientService.java create mode 100644 nifi-pulsar-client-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService create mode 100644 nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/PulsarClientServiceIT.java create mode 100644 nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/TestProcessor.java create mode 100644 nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/TestStandardPulsarClientService.java create mode 100644 nifi-pulsar-nar/pom.xml create mode 100644 nifi-pulsar-processors/pom.xml create mode 100644 nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/AbstractPulsarConsumerProcessor.java create mode 100644 nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/AbstractPulsarProducerProcessor.java create mode 100644 nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/ConsumePulsar.java create mode 100644 nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/ConsumePulsarRecord.java create mode 100644 nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/PublishPulsar.java create mode 100644 nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/PublishPulsarRecord.java create mode 100644 nifi-pulsar-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/AbstractPulsarProcessorTest.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestConsumePulsar.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestConsumePulsarRecord.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestPublishPulsar.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestPublishPulsarRecord.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncConsumePulsar.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncConsumePulsarRecord.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncPublishPulsar.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncPublishPulsarRecord.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockPulsarClientService.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockRecordParser.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockRecordWriter.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncConsumePulsar.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncConsumePulsarRecord.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncPublishPulsar.java create mode 100644 nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncPublishPulsarRecord.java create mode 100644 pom.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..78b9769 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.project +.settings/ +*/.classpath +*/.gitignore +*/target/ +/target/ +.DS_Store diff --git a/nifi-pulsar-client-service-api/pom.xml b/nifi-pulsar-client-service-api/pom.xml new file mode 100644 index 0000000..d54295b --- /dev/null +++ b/nifi-pulsar-client-service-api/pom.xml @@ -0,0 +1,45 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-pulsar-bundle + 1.8.0 + + + nifi-pulsar-client-service-api + jar + + + + org.apache.nifi + nifi-api + provided + + + org.apache.pulsar + pulsar-client + ${pulsar.version} + + + org.apache.commons + commons-collections4 + 4.2 + + + diff --git a/nifi-pulsar-client-service-api/src/main/java/org/apache/nifi/pulsar/PulsarClientService.java b/nifi-pulsar-client-service-api/src/main/java/org/apache/nifi/pulsar/PulsarClientService.java new file mode 100644 index 0000000..446364e --- /dev/null +++ b/nifi-pulsar-client-service-api/src/main/java/org/apache/nifi/pulsar/PulsarClientService.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.pulsar; + +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.controller.ControllerService; +import org.apache.pulsar.client.api.PulsarClient; + +@Tags({"Pulsar", "client", "pool"}) +@CapabilityDescription("Provides the ability to create Pulsar Producer / Consumer instances on demand, " + + "based on the configuration properties defined.") +public interface PulsarClientService extends ControllerService { + + public PulsarClient getPulsarClient(); + + public String getPulsarBrokerRootURL(); + +} diff --git a/nifi-pulsar-client-service-api/src/main/java/org/apache/nifi/pulsar/cache/PulsarClientLRUCache.java b/nifi-pulsar-client-service-api/src/main/java/org/apache/nifi/pulsar/cache/PulsarClientLRUCache.java new file mode 100644 index 0000000..446cb88 --- /dev/null +++ b/nifi-pulsar-client-service-api/src/main/java/org/apache/nifi/pulsar/cache/PulsarClientLRUCache.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.pulsar.cache; + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.commons.collections4.map.LRUMap; + +public class PulsarClientLRUCache extends LRUMap { + + private static final long serialVersionUID = 730163138087670453L; + private final static float LOAD_FACTOR = 0.75F; + private final static boolean SCAN_UNTIL_REMOVABLE = false; + + public PulsarClientLRUCache(int maxSize) { + this(maxSize, LOAD_FACTOR, SCAN_UNTIL_REMOVABLE); + } + + public PulsarClientLRUCache(int maxSize, float loadFactor, boolean scanUntilRemovable) { + super(maxSize, loadFactor, scanUntilRemovable); + } + + @Override + public void clear() { + this.values().parallelStream().forEach(closable -> { + releaseResources(closable); + }); + super.clear(); + } + + @Override + protected boolean removeLRU(LinkEntry entry) { + releaseResources(entry.getValue()); // release resources held by entry + return true; // actually delete entry + } + + private void releaseResources(V value) { + try { + value.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/nifi-pulsar-client-service-api/src/test/java/org/apache/nifi/pulsar/cache/PulsarClientLRUCacheTest.java b/nifi-pulsar-client-service-api/src/test/java/org/apache/nifi/pulsar/cache/PulsarClientLRUCacheTest.java new file mode 100644 index 0000000..42abe7d --- /dev/null +++ b/nifi-pulsar-client-service-api/src/test/java/org/apache/nifi/pulsar/cache/PulsarClientLRUCacheTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.pulsar.cache; + +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@SuppressWarnings("rawtypes") +public class PulsarClientLRUCacheTest { + + @Mock + private Producer mockedPulsarProducer; + + @Before + public void setUp() throws InterruptedException { + mockedPulsarProducer = mock(Producer.class); + } + + /** + * Make sure the LRUCache functions as a Map + */ + @Test + public void simpleTest() { + PulsarClientLRUCache cache = new PulsarClientLRUCache(10); + + for (Character i='A'; i<='E'; i++){ + cache.put(i.toString(), mockedPulsarProducer); + } + + assertEquals(5, cache.size()); + + for (Character i='A'; i<='E'; i++){ + assertNotNull( cache.get(i.toString())); + } + } + + @Test + public void evictionTest() { + + PulsarClientLRUCache cache = new PulsarClientLRUCache(5); + + for (Character i='A'; i<='Z'; i++){ + cache.put(i.toString(), mockedPulsarProducer); + } + + // Make sure we only have 5 items in the cache + assertEquals(5, cache.size()); + + // Make sure we have the last 5 items added to the cache + for (Character i='V'; i<='Z'; i++){ + assertNotNull(cache.get(i.toString())); + } + } + + @Test + public void evictionLruTest() { + + PulsarClientLRUCache cache = new PulsarClientLRUCache(5); + + final Character A = 'A'; + + // Write 25 items to the cache, and the letter 'A' every other put. + for (Character i='B'; i<='Z'; i++){ + cache.put(i.toString(), mockedPulsarProducer); + cache.put(A.toString(), mockedPulsarProducer); + } + + // Make sure we only have 5 items in the cache + assertEquals(5, cache.size()); + + // Make sure that the letter 'A' is still in the cache due to frequent access + assertNotNull( cache.get(A.toString()) ); + + // Make sure we have the last 4 items added to the cache + for (Character i='W'; i<='Z'; i++){ + assertNotNull( cache.get(i.toString())); + } + } + + @Test + public void clearTest() throws PulsarClientException { + PulsarClientLRUCache cache = new PulsarClientLRUCache(26); + + for (Character i='A'; i<='Z'; i++) { + cache.put(i.toString(), mockedPulsarProducer); + } + + // Make sure we only have all the items in the cache + assertEquals(26, cache.size()); + cache.clear(); + + verify(mockedPulsarProducer, times(26)).close(); + + // Make sure all the items were removed + assertEquals(0, cache.size()); + } +} diff --git a/nifi-pulsar-client-service-nar/pom.xml b/nifi-pulsar-client-service-nar/pom.xml new file mode 100644 index 0000000..af2279e --- /dev/null +++ b/nifi-pulsar-client-service-nar/pom.xml @@ -0,0 +1,41 @@ + + + 4.0.0 + + + org.apache.nifi + nifi-pulsar-bundle + 1.8.0 + + + nifi-pulsar-client-service-nar + nar + + + + org.apache.nifi + nifi-standard-services-api-nar + 1.8.0 + nar + + + org.apache.nifi + nifi-pulsar-client-service-api + 1.8.0 + + + + \ No newline at end of file diff --git a/nifi-pulsar-client-service-nar/src/main/resources/META-INF/LICENSE b/nifi-pulsar-client-service-nar/src/main/resources/META-INF/LICENSE new file mode 100644 index 0000000..9efc253 --- /dev/null +++ b/nifi-pulsar-client-service-nar/src/main/resources/META-INF/LICENSE @@ -0,0 +1,329 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +The binary distribution of this product bundles 'ANTLR 4' which is available + under a "3-clause BSD" license. For details see http://www.antlr.org/license.html + + Copyright (c) 2012 Terence Parr and Sam Harwell + All rights reserved. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + + Neither the name of the author nor the names of its contributors may be used to endorse + or promote products derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles 'ParaNamer' + which is available under a BSD style license. + + Copyright (c) 2006 Paul Hammant & ThoughtWorks Inc + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles 'icu4j' + which is available under the ICU License. + + COPYRIGHT AND PERMISSION NOTICE + + Copyright (c) 1995-2011 International Business Machines Corporation and others + + All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, and/or + sell copies of the Software, and to permit persons to whom the Software is furnished to do so, + provided that the above copyright notice(s) and this permission notice appear in all copies of + the Software and that both the above copyright notice(s) and this permission notice appear in + supporting documentation. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED + IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY + DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, + NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE + OF THIS SOFTWARE. + + Except as contained in this notice, the name of a copyright holder shall not be used in advertising + or otherwise to promote the sale, use or other dealings in this Software without prior written + authorization of the copyright holder. + +The binary distribution of this product bundles 'Scala Library, Scala Parser, Scala Reflect' which is available + under a "3-clause BSD" license. For details see https://github.com/scala/scala/blob/v2.11.7/doc/LICENSE.md) + + This software includes projects with other licenses -- see `doc/LICENSE.md`. + + Copyright (c) 2002-2017 EPFL + Copyright (c) 2011-2017 Lightbend, Inc. + + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the EPFL nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/nifi-pulsar-client-service-nar/src/main/resources/META-INF/NOTICE b/nifi-pulsar-client-service-nar/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000..30597ef --- /dev/null +++ b/nifi-pulsar-client-service-nar/src/main/resources/META-INF/NOTICE @@ -0,0 +1,612 @@ +nifi-pulsar-client-service-api-nar +Copyright 2015-2017 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +****************** +Apache Software License v2 +****************** + +The following binary components are provided under the Apache Software License v2 + + (ASLv2) Apache HttpComponents Client + The following NOTICE information applies: + Apache HttpComponents Client + Copyright 1999-2016 The Apache Software Foundation + + (ASLv2) Jackson JSON processor + The following NOTICE information applies: + # Jackson JSON processor + + Jackson is a high-performance, Free/Open Source JSON processing library. + It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. + It is currently developed by a community of developers, as well as supported + commercially by FasterXML.com. + + ## Licensing + + Jackson core and extension components may licensed under different licenses. + To find the details that apply to this artifact see the accompanying LICENSE file. + For more information, including possible other licensing options, contact + FasterXML.com (http://fasterxml.com). + + ## Credits + + A list of contributors may be found from CREDITS file, which is included + in some artifacts (usually source distributions); but is always available + from the source code management (SCM) system project uses. + + (ASLv2) bytebuffer-collections + The following NOTICE information applies: + bytebuffer-collections + Copyright 2011-2015 Metamarkets Group Inc. + + (ASLv2) Apache Commons IO + The following NOTICE information applies: + Apache Commons IO + Copyright 2002-2016 The Apache Software Foundation + + (ASLv2) Apache Commons Collections + The following NOTICE information applies: + Apache Commons Collections + Copyright 2001-2013 The Apache Software Foundation + + (ASLv2) Apache Commons Lang + The following NOTICE information applies: + Apache Commons Lang + Copyright 2001-2014 The Apache Software Foundation + + This product includes software from the Spring Framework, + under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + + (ASLv2) Apache Commons Codec + The following NOTICE information applies: + Apache Commons Codec + Copyright 2002-2014 The Apache Software Foundation + + src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java + contains test data from http://aspell.net/test/orig/batch0.tab. + Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + + =============================================================================== + + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright: + Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + + (ASLv2) Apache Commons Logging + The following NOTICE information applies: + Apache Commons Logging + Copyright 2003-2013 The Apache Software Foundation + + (ASLv2) Apache Commons DBCP + The following NOTICE information applies: + Apache Commons DBCP + Copyright 2001-2015 The Apache Software Foundation. + + (ASLv2) Apache Commons Pool + The following NOTICE information applies: + Apache Commons Pool + Copyright 1999-2009 The Apache Software Foundation. + + (ASLv2) Apache Curator + The following NOTICE information applies: + Curator Framework + Copyright 2011-2014 The Apache Software Foundation + + Curator Client + Copyright 2011-2014 The Apache Software Foundation + + Curator Recipes + Copyright 2011-2014 The Apache Software Foundation + + Curator X-Discovery + Copyright 2013-2014 The Apache Software Foundation + + (ASLv2) Druid + The following NOTICE information applies: + Druid - a distributed column store. + Copyright 2012-2016 Metamarkets Group Inc. + Copyright 2015-2016 Yahoo! Inc. + Copyright 2015-2016 Imply Data, Inc. + + ------------------------------------------------------------------------------- + + This product contains a modified version of Andrew Duffy's java-alphanum library + * LICENSE: + * https://github.com/amjjd/java-alphanum/blob/5c036e2e492cc7f3b7bcdebd46b8f9e2a87927e5/LICENSE.txt (Apache License, Version 2.0) + * HOMEPAGE: + * https://github.com/amjjd/java-alphanum + + This product contains conjunctive normal form conversion code and a variance aggregator algorithm adapted from Apache Hive + * LICENSE: + * https://github.com/apache/hive/blob/branch-2.0/LICENSE (Apache License, Version 2.0) + * HOMEPAGE: + * https://github.com/apache/hive + + This product contains variable length long deserialization code adapted from Apache Lucene + * LICENSE: + * https://github.com/apache/lucene-solr/blob/master/lucene/LICENSE.txt (Apache License, Version 2.0) + * HOMEPAGE: + * https://github.com/apache/lucene-solr + + This product contains a modified version of Metamarkets java-util library + * LICENSE: + * https://github.com/metamx/java-util/blob/master/LICENSE (Apache License, Version 2.0) + * HOMEPAGE: + * https://github.com/metamx/java-util + * COMMIT TAG: + * https://github.com/metamx/java-util/commit/826021f + + This product contains a modified version of TestNG 6.8.7 + * LICENSE: + * http://testng.org/license/ (Apache License, Version 2.0) + * HOMEPAGE: + * http://testng.org/ + + This product contains a modified version of Metamarkets bytebuffer-collections library + * LICENSE: + * https://github.com/metamx/bytebuffer-collections/blob/master/LICENSE (Apache License, Version 2.0) + * HOMEPAGE: + * https://github.com/metamx/bytebuffer-collections + * COMMIT TAG: + * https://github.com/metamx/bytebuffer-collections/commit/3d1e7c8 + + This product contains SQL query planning code adapted from Apache Calcite + * LICENSE: + * https://github.com/apache/calcite/blob/master/LICENSE (Apache License, Version 2.0) + * HOMEPAGE: + * https://calcite.apache.org/ + + This product contains a modified version of Metamarkets extendedset library + * LICENSE: + * https://github.com/metamx/extendedset/blob/master/LICENSE (Apache License, Version 2.0) + * HOMEPAGE: + * https://github.com/metamx/extendedset + * COMMIT TAG: + * https://github.com/metamx/extendedset/commit/c9d647d + + This product contains a modified version of Alessandro Colantonio's CONCISE + (COmpressed 'N' Composable Integer SEt) library, extending the functionality of + ConciseSet to use IntBuffers. + * (c) 2010 Alessandro Colantonio + * + * + * LICENSE: + * Apache License, Version 2.0 + * HOMEPAGE: + * https://sourceforge.net/projects/concise/ + + This product contains a modified version of The Guava Authors's Closer class from Guava library: + * LICENSE: + * https://github.com/google/guava/blob/c462d69329709f72a17a64cb229d15e76e72199c/COPYING (Apache License, Version 2.0) + * HOMEPAGE: + * https://github.com/google/guava + * COMMIT TAG: + * https://github.com/google/guava/blob/c462d69329709f72a17a64cb229d15e76e72199c + + This product contains code adapted from Apache Hadoop + * LICENSE: + * https://github.com/apache/hadoop/blob/trunk/LICENSE.txt (Apache License, Version 2.0) + * HOMEPAGE: + * http://hadoop.apache.org/ + + (ASLv2) Emitter + The following NOTICE information applies: + emitter + Copyright 2012-2015 Metamarkets Group Inc. + + (ASLv2) Finagle + The following NOTICE information applies: + Copyright 2017, Twitter Inc. + This software contains portions of code from other open-source projects, including: + >>>>>>>>>>>>>> + Apache Thrift + Copyright 2006-2010 The Apache Software Foundation. + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + (ASLv2) Guava + The following NOTICE information applies: + Guava + Copyright 2015 The Guava Authors + + (ASLv2) Google Guice + The following NOTICE information applies: + Google Guice - Core Library + Copyright 2006-2011 Google, Inc. + + (ASLv2) Jackson JSON processor + The following NOTICE information applies: + # Jackson JSON processor + + Jackson is a high-performance, Free/Open Source JSON processing library. + It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. + It is currently developed by a community of developers, as well as supported + commercially by FasterXML.com. + + ## Licensing + + Jackson core and extension components may licensed under different licenses. + To find the details that apply to this artifact see the accompanying LICENSE file. + For more information, including possible other licensing options, contact + FasterXML.com (http://fasterxml.com). + + ## Credits + + A list of contributors may be found from CREDITS file, which is included + in some artifacts (usually source distributions); but is always available + from the source code management (SCM) system project uses. + + (ASLv2) java-util + The following NOTICE information applies: + java-util + Copyright 2011-2017 Metamarkets Group Inc. + + (ASLv2) jdbi + The following NOTICE information applies: + This product includes software developed by Brian McCallister + + (ASLv2) Joda Time + The following NOTICE information applies: + This product includes software developed by + Joda.org (http://www.joda.org/). + + (ASLv2) Joda Convert + The following NOTICE information applies: + Joda Convert + Copyright 2010-present Stephen Colebourne + + This product includes software developed by + Joda.org (http://www.joda.org/). + + + Joda-Convert includes code from Google Guava, which is licensed as follows: + + Copyright (C) 2011 The Guava Authors + + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + + (ASLv2) Apache log4j + The following NOTICE information applies: + Apache log4j + Copyright 2007 The Apache Software Foundation + + (ASLv2) loglady + The following NOTICE information applies: + Copyright (c) 2013 Daniel Lundin + + (ASLv2) The Netty Project + The following NOTICE information applies: + + The Netty Project + ================= + + Please visit the Netty web site for more information: + + * http://netty.io/ + + Copyright 2014 The Netty Project + + The Netty Project licenses this file to you under the Apache License, + version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + Also, please refer to each LICENSE..txt file, which is located in + the 'license' directory of the distribution file, for the license terms of the + components that this product depends on. + + ------------------------------------------------------------------------------- + This product contains the extensions to Java Collections Framework which has + been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + + This product contains a modified version of Robert Harder's Public Domain + Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + + This product contains a modified portion of 'Webbit', an event based + WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + + This product contains a modified portion of 'SLF4J', a simple logging + facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + + This product contains a modified portion of 'Apache Harmony', an open source + Java SE, which can be obtained at: + + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * http://archive.apache.org/dist/harmony/ + + This product contains a modified portion of 'jbzip2', a Java bzip2 compression + and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + + This product contains a modified portion of 'libdivsufsort', a C API library to construct + the suffix array and the Burrows-Wheeler transformed string for any input string of + a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://github.com/y-256/libdivsufsort + + This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + + This product optionally depends on 'JZlib', a re-implementation of zlib in + pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + + This product optionally depends on 'Compress-LZF', a Java library for encoding and + decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + + This product optionally depends on 'lz4', a LZ4 Java compression + and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + + This product optionally depends on 'lzma-java', a LZMA Java compression + and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + + This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression + and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + + This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data + interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/protobuf + + This product optionally depends on 'Bouncy Castle Crypto APIs' to generate + a temporary self-signed X.509 certificate when the JVM does not provide the + equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + + This product optionally depends on 'Snappy', a compression library produced + by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/snappy + + This product optionally depends on 'JBoss Marshalling', an alternative Java + serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://www.jboss.org/jbossmarshalling + + This product optionally depends on 'Caliper', Google's micro- + benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/google/caliper + + This product optionally depends on 'Apache Commons Logging', a logging + framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + + This product optionally depends on 'Apache Log4J', a logging framework, which + can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + + This product optionally depends on 'Aalto XML', an ultra-high performance + non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * http://wiki.fasterxml.com/AaltoHome + + This product contains a modified version of 'HPACK', a Java implementation of + the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + + This product contains a modified portion of 'Apache Commons Lang', a Java library + provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + + This product contains a forked and modified version of Tomcat Native + + * LICENSE: + * ASL2 + * HOMEPAGE: + * http://tomcat.apache.org/native-doc/ + * https://svn.apache.org/repos/asf/tomcat/native/ + + (ASLv2) opencsv (net.sf.opencsv:opencsv:2.3) + + (ASLv2) scala_util + The following NOTICE information applies: + Copyright 2012 Metamarkets Group Inc. + + (ASLv2) server-metrics + The following NOTICE information applies: + server-metrics + Copyright 2011-2015 Metamarkets Group Inc. + + (ASLv2) Apache ZooKeeper + The following NOTICE information applies: + Apache ZooKeeper + Copyright 2009-2012 The Apache Software Foundation + +************************ +Eclipse Public License 1.0 +************************ + +The following binary components are provided under the Eclipse Public License 1.0. See project link for details. + + (EPL 1.0)(MPL 2.0) H2 Database (com.h2database:h2:jar:1.3.176 - http://www.h2database.com/html/license.html) + + +************************ +Mozilla Public License 2.0 +************************ + +The following binary components are provided under the Mozilla Public License 2.0. See project link for details. + + (MPL 2.0) Rhino (org.mozilla:rhino:jar:1.7R5 - https://developer.mozilla.org/en-US/docs/Mozilla/Projects/Rhino/Download_Rhino) + + +************************ +Common Development and Distribution License 1.1 +************************ + +The following binary components are provided under the Common Development and Distribution License 1.1. See project link for details. + + (CDDL 1.1) (GPL2 w/ CPE) jersey-client (org.glassfish.jersey.core:jersey-client:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-common (org.glassfish.jersey.core:jersey-common:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-container-servlet-core (org.glassfish.jersey.containers:jersey-container-servlet-core:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-entity-filtering (org.glassfish.jersey.ext:jersey-entity-filtering:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-hk2 (org.glassfish.jersey.inject:jersey-hk2:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-media-jaxb (org.glassfish.jersey.media:jersey-media-jaxb:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-media-json-jackson (org.glassfish.jersey.media:jersey-media-json-jackson:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-media-multipart (org.glassfish.jersey.media:jersey-media-multipart:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-server (org.glassfish.jersey.core:jersey-server:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) hk2 (org.glassfish.hk2:hk2:jar:2.5.0-b42 - https://javaee.github.io/glassfish/) + (CDDL 1.1) (GPL2 w/ CPE) hk2-api (org.glassfish.hk2:hk2-api:jar:2.5.0-b42 - https://javaee.github.io/glassfish/) + (CDDL 1.1) (GPL2 w/ CPE) hk2-utils (org.glassfish.hk2:hk2-utils:jar:2.5.0-b42 - https://javaee.github.io/glassfish/) + (CDDL 1.1) (GPL2 w/ CPE) hk2-locator (org.glassfish.hk2:hk2-locator:jar:2.5.0-b42 - https://javaee.github.io/glassfish/) + (CDDL 1.1) (GPL2 w/ CPE) hk2-config (org.glassfish.hk2:hk2-config:jar:2.5.0-b42 - https://javaee.github.io/glassfish/) + (CDDL 1.1) (GPL2 w/ CPE) hk2-core (org.glassfish.hk2:hk2-core:jar:2.5.0-b42 - https://javaee.github.io/glassfish/) + (CDDL 1.1) (GPL2 w/ CPE) hk2-runlevel (org.glassfish.hk2:hk2-runlevel:jar:2.5.0-b42 - https://javaee.github.io/glassfish/) + (CDDL 1.1) (GPL2 w/ CPE) config-types (org.glassfish.hk2:config-types:jar:2.5.0-b42 - https://javaee.github.io/glassfish/) + (CDDL 1.1) (GPL2 w/ CPE) class-model (org.glassfish.hk2:class-model:jar:2.5.0-b42 - https://javaee.github.io/glassfish/) + (CDDL 1.1) (GPL2 w/ CPE) asm-all-repackaged (org.glassfish.hk2.external:asm-all-repackaged:jar:2.5.0-b42 - https://javaee.github.io/glassfish/) + (CDDL 1.1) (GPL2 w/ CPE) aopalliance-repackaged (org.glassfish.hk2.external:aopalliance-repackaged:jar:2.5.0-b42 - https://javaee.github.io/glassfish/) + (CDDL 1.1) (GPL2 w/ CPE) javax.inject:1 as OSGi bundle (org.glassfish.hk2.external:javax.inject:jar:2.4.0-b25 - https://hk2.java.net/external/javax.inject) + (CDDL 1.1) (GPL2 w/ CPE) jersey-client (org.glassfish.jersey.core:jersey-client:jar:2.19 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-common (org.glassfish.jersey.core:jersey-common:jar:2.19 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-guava (org.glassfish.jersey.bundles.repackaged:jersey-guava:jar:2.19 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-client (org.glassfish.jersey.core:jersey-client:jar:2.22.1 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-common (org.glassfish.jersey.core:jersey-common:jar:2.22.1 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-media-multipart (org.glassfish.jersey.media:jersey-media-multipart:jar:2.22.1 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-guava (org.glassfish.jersey.bundles.repackaged:jersey-guava:jar:2.22.1 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-client (com.sun.jersey:jersey-client:jar:1.9 - https://jersey.java.net/jersey-client/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-core (com.sun.jersey:jersey-core:jar:1.9 - https://jersey.java.net/jersey-core/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-json (com.sun.jersey:jersey-json:jar:1.9 - https://jersey.java.net/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-server (com.sun.jersey:jersey-server:jar:1.9 - https://jersey.java.net/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-guice (com.sun.jersey.contribs:jersey-guice:jar:1.9 - https://jersey.java.net/) + + + (CDDL 1.1) (GPL2 w/ CPE) Old JAXB Runtime (com.sun.xml.bind:jaxb-impl:jar:2.2.3-1 - http://jaxb.java.net/) + (CDDL 1.1) (GPL2 w/ CPE) Java Architecture For XML Binding (javax.xml.bind:jaxb-api:jar:2.2.2 - https://jaxb.dev.java.net/) + (CDDL 1.1) (GPL2 w/ CPE) MIME Streaming Extension (org.jvnet.mimepull:mimepull:jar:1.9.3 - http://mimepull.java.net) + (CDDL 1.1) (GPL2 w/ CPE) JavaMail API (compat) (javax.mail:mail:jar:1.4.7 - http://kenai.com/projects/javamail/mail) + (CDDL 1.1) (GPL2 w/ CPE) JSP Implementation (org.glassfish.web:javax.servlet.jsp:jar:2.3.2 - http://jsp.java.net) + (CDDL 1.1) (GPL2 w/ CPE) JavaServer Pages (TM) TagLib Implementation (org.glassfish.web:javax.servlet.jsp.jstl:jar:1.2.2 - http://jstl.java.net) + (CDDL 1.1) (GPL2 w/ CPE) Expression Language 3.0 (org.glassfish:javax.el:jar:3.0.0 - http://el-spec.java.net) + (CDDL 1.1) (GPL2 w/ CPE) JavaServer Pages(TM) API (javax.servlet.jsp:javax.servlet.jsp-api:jar:2.3.1 - http://jsp.java.net) + (CDDL 1.1) (GPL2 w/ CPE) Expression Language 3.0 API (javax.el:javax.el-api:jar:3.0.0 - http://uel-spec.java.net) + +***************** +Public Domain +***************** + +The following binary components are provided to the 'Public Domain'. See project link for details. + + (Public Domain) AOP Alliance 1.0 (http://aopalliance.sourceforge.net/) + +The following binary components are provided under the Creative Commons Zero license version 1.0. See project link for details. + + (CC0v1.0) JSR166e for Twitter (com.twitter:jsr166e:jar:1.1.0 - https://github.com/twitter/jsr166e) diff --git a/nifi-pulsar-client-service/pom.xml b/nifi-pulsar-client-service/pom.xml new file mode 100644 index 0000000..381207c --- /dev/null +++ b/nifi-pulsar-client-service/pom.xml @@ -0,0 +1,68 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-pulsar-bundle + 1.8.0 + + + nifi-pulsar-client-service + jar + + + + org.apache.nifi + nifi-pulsar-client-service-api + 1.8.0 + provided + + + org.apache.nifi + nifi-api + provided + + + org.apache.nifi + nifi-processor-utils + 1.8.0 + provided + + + org.apache.nifi + nifi-ssl-context-service-api + provided + + + org.apache.nifi + nifi-mock + 1.8.0 + test + + + org.slf4j + slf4j-simple + test + + + junit + junit + test + + + diff --git a/nifi-pulsar-client-service/src/main/java/org/apache/nifi/pulsar/StandardPulsarClientService.java b/nifi-pulsar-client-service/src/main/java/org/apache/nifi/pulsar/StandardPulsarClientService.java new file mode 100644 index 0000000..921f3ab --- /dev/null +++ b/nifi-pulsar-client-service/src/main/java/org/apache/nifi/pulsar/StandardPulsarClientService.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.pulsar; + +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.nifi.annotation.lifecycle.OnDisabled; +import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.annotation.lifecycle.OnShutdown; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.ssl.SSLContextService; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.PulsarClientException.UnsupportedAuthenticationException; +import org.apache.pulsar.client.impl.auth.AuthenticationTls; + +public class StandardPulsarClientService extends AbstractControllerService implements PulsarClientService { + + public static final PropertyDescriptor PULSAR_SERVICE_URL = new PropertyDescriptor.Builder() + .name("PULSAR_SERVICE_URL") + .displayName("Pulsar Service URL") + .description("URL for the Pulsar cluster, e.g localhost:6650") + .required(true) + .addValidator(StandardValidators.HOSTNAME_PORT_LIST_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .build(); + + public static final PropertyDescriptor CONCURRENT_LOOKUP_REQUESTS = new PropertyDescriptor.Builder() + .name("CONCURRENT_LOOKUP_REQUESTS") + .displayName("Maximum concurrent lookup-requests") + .description("Number of concurrent lookup-requests allowed on each broker-connection.") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .defaultValue("5000") + .build(); + + public static final PropertyDescriptor CONNECTIONS_PER_BROKER = new PropertyDescriptor.Builder() + .name("CONNECTIONS_PER_BROKER") + .displayName("Maximum connects per Pulsar broker") + .description("Sets the max number of connection that the client library will open to a single broker.\n" + + "By default, the connection pool will use a single connection for all the producers and consumers. " + + "Increasing this parameter may improve throughput when using many producers over a high latency connection.") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .defaultValue("1") + .build(); + + public static final PropertyDescriptor IO_THREADS = new PropertyDescriptor.Builder() + .name("IO_THREADS") + .displayName("I/O Threads") + .description("The number of threads to be used for handling connections to brokers.") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .defaultValue("1") + .build(); + + public static final PropertyDescriptor KEEP_ALIVE_INTERVAL = new PropertyDescriptor.Builder() + .name("KEEP_ALIVE_INTERVAL") + .displayName("Keep Alive interval") + .description("The keep alive interval in seconds for each client-broker-connection.") + .required(false) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .defaultValue("30 sec") + .build(); + + public static final PropertyDescriptor LISTENER_THREADS = new PropertyDescriptor.Builder() + .name("LISTENER_THREADS") + .displayName("Listener Threads") + .description("The number of threads to be used for message listeners") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .defaultValue("1") + .build(); + + public static final PropertyDescriptor MAXIMUM_LOOKUP_REQUESTS = new PropertyDescriptor.Builder() + .name("MAXIMUM_LOOKUP_REQUESTS") + .displayName("Maximum lookup requests") + .description("Number of max lookup-requests allowed on each broker-connection. To prevent overload on broker, " + + "it should be greater than the 'Maximum concurrent lookup-requests' property value.") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .defaultValue("50000") + .build(); + + public static final PropertyDescriptor MAXIMUM_REJECTED_REQUESTS = new PropertyDescriptor.Builder() + .name("MAXIMUM_REJECTED_REQUESTS") + .displayName("Maximum rejected requests per connection") + .description("Max number of broker-rejected requests in a certain time-frame after " + + "which current connection will be closed and client creates a new connection that gives " + + "chance to connect a different broker.") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .defaultValue("50") + .build(); + + public static final PropertyDescriptor OPERATION_TIMEOUT = new PropertyDescriptor.Builder() + .name("OPERATION_TIMEOUT") + .displayName("Operation Timeout") + .description("Producer-create, subscribe and unsubscribe operations will be retried until this " + + "interval, after which the operation will be marked as failed.") + .required(false) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .defaultValue("30 sec") + .build(); + + public static final PropertyDescriptor STATS_INTERVAL = new PropertyDescriptor.Builder() + .name("STATS_INTERVAL") + .displayName("Stats interval") + .description("The interval between each stat infomation update. It should be set to at least 1 second.") + .required(false) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("60 sec") + .build(); + + public static final PropertyDescriptor USE_TCP_NO_DELAY = new PropertyDescriptor.Builder() + .name("USE_TCP_NO_DELAY") + .displayName("Use TCP no-delay flag") + .description("Configure whether to use TCP no-delay flag on the connection, to disable Nagle algorithm.\n" + + "No-delay features make sure packets are sent out on the network as soon as possible, and it's critical " + + "to achieve low latency publishes. On the other hand, sending out a huge number of small packets might " + + "limit the overall throughput, so if latency is not a concern, it's advisable to set the useTcpNoDelay " + + "flag to false.") + .required(true) + .allowableValues("true", "false") + .defaultValue("false") + .build(); + + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("SSL_CONTEXT_SERVICE") + .displayName("SSL Context Service") + .description("Specifies the SSL Context Service to use for communicating with Pulsar.") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + private static List properties; + private volatile PulsarClient client; + private boolean secure = false; + private String pulsarBrokerRootUrl; + + static { + final List props = new ArrayList<>(); + props.add(PULSAR_SERVICE_URL); + props.add(CONCURRENT_LOOKUP_REQUESTS); + props.add(CONNECTIONS_PER_BROKER); + props.add(IO_THREADS); + props.add(KEEP_ALIVE_INTERVAL); + props.add(LISTENER_THREADS); + props.add(MAXIMUM_LOOKUP_REQUESTS); + props.add(MAXIMUM_REJECTED_REQUESTS); + props.add(OPERATION_TIMEOUT); + props.add(STATS_INTERVAL); + props.add(USE_TCP_NO_DELAY); + props.add(SSL_CONTEXT_SERVICE); + properties = Collections.unmodifiableList(props); + } + + @Override + protected List getSupportedPropertyDescriptors() { + return properties; + } + + /** + * @param context the configuration context + * @throws InitializationException if unable to connect to the Pulsar Broker + * @throws UnsupportedAuthenticationException if the Broker URL uses a non-supported authentication mechanism + */ + @OnEnabled + public void onEnabled(final ConfigurationContext context) throws InitializationException, UnsupportedAuthenticationException { + try { + client = getClientBuilder(context).build(); + } catch (Exception e) { + throw new InitializationException("Unable to create Pulsar Client", e); + } + } + + @OnDisabled + @OnShutdown + public void cleanup() throws PulsarClientException { + if (client != null) { + client.close(); + } + } + + @Override + public PulsarClient getPulsarClient() { + return client; + } + + @Override + public String getPulsarBrokerRootURL() { + return pulsarBrokerRootUrl; + } + + private void setPulsarBrokerRootURL(String s) { + pulsarBrokerRootUrl = s; + } + + private static String buildPulsarBrokerRootUrl(String uri, boolean tlsEnabled) { + StringBuilder builder = new StringBuilder().append("pulsar"); + + if (tlsEnabled) { + builder.append("+ssl"); + } + + return builder.append("://") + .append(uri) + .toString(); + } + + private ClientBuilder getClientBuilder(ConfigurationContext context) throws UnsupportedAuthenticationException, MalformedURLException { + + ClientBuilder builder = PulsarClient.builder() + .maxConcurrentLookupRequests(context.getProperty(CONCURRENT_LOOKUP_REQUESTS).evaluateAttributeExpressions().asInteger()) + .connectionsPerBroker(context.getProperty(CONNECTIONS_PER_BROKER).evaluateAttributeExpressions().asInteger()) + .ioThreads(context.getProperty(IO_THREADS).evaluateAttributeExpressions().asInteger()) + .keepAliveInterval(context.getProperty(KEEP_ALIVE_INTERVAL).evaluateAttributeExpressions().asTimePeriod(TimeUnit.SECONDS).intValue(), TimeUnit.SECONDS) + .listenerThreads(context.getProperty(LISTENER_THREADS).evaluateAttributeExpressions().asInteger()) + .maxLookupRequests(context.getProperty(MAXIMUM_LOOKUP_REQUESTS).evaluateAttributeExpressions().asInteger()) + .maxNumberOfRejectedRequestPerConnection(context.getProperty(MAXIMUM_REJECTED_REQUESTS).evaluateAttributeExpressions().asInteger()) + .operationTimeout(context.getProperty(OPERATION_TIMEOUT).evaluateAttributeExpressions().asTimePeriod(TimeUnit.SECONDS).intValue(), TimeUnit.SECONDS) + .statsInterval(context.getProperty(STATS_INTERVAL).evaluateAttributeExpressions().asTimePeriod(TimeUnit.SECONDS).intValue(), TimeUnit.SECONDS) + .enableTcpNoDelay(context.getProperty(USE_TCP_NO_DELAY).asBoolean()); + + // Configure TLS + final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); + + if (sslContextService != null && sslContextService.isTrustStoreConfigured() && sslContextService.isKeyStoreConfigured()) { + Map authParams = new HashMap<>(); + authParams.put("tlsCertFile", sslContextService.getTrustStoreFile()); + authParams.put("tlsKeyFile", sslContextService.getKeyStoreFile()); + + builder = builder.authentication(AuthenticationTls.class.getName(), authParams) + .tlsTrustCertsFilePath(sslContextService.getTrustStoreFile()); + secure = true; + } + + setPulsarBrokerRootURL(buildPulsarBrokerRootUrl(context.getProperty(PULSAR_SERVICE_URL).evaluateAttributeExpressions().getValue(), secure)); + builder = builder.serviceUrl(getPulsarBrokerRootURL()); + return builder; + } +} diff --git a/nifi-pulsar-client-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-pulsar-client-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService new file mode 100644 index 0000000..42484b4 --- /dev/null +++ b/nifi-pulsar-client-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.pulsar.StandardPulsarClientService \ No newline at end of file diff --git a/nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/PulsarClientServiceIT.java b/nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/PulsarClientServiceIT.java new file mode 100644 index 0000000..7954560 --- /dev/null +++ b/nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/PulsarClientServiceIT.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.pulsar; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.concurrent.TimeUnit; + +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Should be run only after a Docker container has been launched using the following command: + * + * `docker run -it -p 6650:6650 -p 8080:8080 -v $PWD/data:/pulsar/data \ + * apachepulsar/pulsar bin/pulsar standalone` + * + */ +public class PulsarClientServiceIT { + + private static final String SUBSCRIPTION_NAME = "test-subscription"; + private static final String TOPIC_NAME = "test-topic"; + private static final String MESSAGE = "Test Message"; + + private TestRunner runner; + private PulsarClientService service; + private PulsarClient client; + private Consumer consumer; + private Producer producer; + + @Before + public void before() throws Exception { + runner = TestRunners.newTestRunner(TestProcessor.class); + service = new StandardPulsarClientService(); + runner.addControllerService("test-good", service); + + runner.setProperty(service, StandardPulsarClientService.PULSAR_SERVICE_URL, "localhost:6650"); + runner.enableControllerService(service); + client = service.getPulsarClient(); + + consumer = client.newConsumer() + .subscriptionName(SUBSCRIPTION_NAME) + .topic(TOPIC_NAME) + .subscribe(); + + producer = client.newProducer() + .topic(TOPIC_NAME) + .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS) + .sendTimeout(10, TimeUnit.SECONDS) + .blockIfQueueFull(true) + .create(); + } + + @After + public void after() throws PulsarClientException { + consumer.close(); + producer.close(); + client.close(); + } + + @Test + public void testInit() { + runner.assertValid(service); + assertNotNull(client); + assertNotNull(consumer); + assertNotNull(producer); + } + + @Test + public void testPublishOneMessage() throws Exception { + + producer.send(MESSAGE.getBytes()); + Message msg = consumer.receive(); + consumer.acknowledge(msg); + + assertNotNull(msg); + assertEquals(MESSAGE, new String(msg.getValue())); + } + + @Test + public void testPublish100Messages() throws Exception { + publishMessages(100); + } + + @Test + public void testPublish1000Messages() throws Exception { + publishMessages(1000); + } + + public void publishMessages(int numMessages) throws Exception { + // Send the messages + for (int idx = 0; idx < numMessages; idx++) { + String message = MESSAGE + "-" + idx; + producer.send(message.getBytes()); + } + + // Consume the messages + for (int idx = 0; idx < numMessages; idx++) { + String message = MESSAGE + "-" + idx; + Message msg = consumer.receive(); + consumer.acknowledge(msg); + + assertNotNull(msg); + assertEquals(message, new String(msg.getValue())); + } + } +} \ No newline at end of file diff --git a/nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/TestProcessor.java b/nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/TestProcessor.java new file mode 100644 index 0000000..4f13c3b --- /dev/null +++ b/nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/TestProcessor.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.pulsar; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.exception.ProcessException; + +public class TestProcessor extends AbstractProcessor { + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + } + + @Override + protected List getSupportedPropertyDescriptors() { + List propDescs = new ArrayList<>(); + propDescs.add(new PropertyDescriptor.Builder() + .name("StandardPulsarClientService test processor") + .description("StandardPulsarClientService test processor") + .identifiesControllerService(PulsarClientService.class) + .required(true) + .build()); + return propDescs; + } +} diff --git a/nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/TestStandardPulsarClientService.java b/nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/TestStandardPulsarClientService.java new file mode 100644 index 0000000..74fa79f --- /dev/null +++ b/nifi-pulsar-client-service/src/test/java/org/apache/nifi/pulsar/TestStandardPulsarClientService.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.pulsar; + +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.Test; + +public class TestStandardPulsarClientService { + + @Test + public void testService() throws InitializationException { + final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class); + final PulsarClientService service = new StandardPulsarClientService(); + runner.addControllerService("test-good", service); + + runner.setProperty(service, StandardPulsarClientService.PULSAR_SERVICE_URL, "localhost:6650"); + runner.enableControllerService(service); + + runner.assertValid(service); + } + + @Test + public void invalidServiceUrlTest() throws InitializationException { + final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class); + final PulsarClientService service = new StandardPulsarClientService(); + + runner.addControllerService("test-bad", service); + runner.setProperty(service, StandardPulsarClientService.PULSAR_SERVICE_URL, "malfromed_url"); + runner.assertNotValid(); + } +} diff --git a/nifi-pulsar-nar/pom.xml b/nifi-pulsar-nar/pom.xml new file mode 100644 index 0000000..bdb855b --- /dev/null +++ b/nifi-pulsar-nar/pom.xml @@ -0,0 +1,50 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-pulsar-bundle + 1.8.0 + + + nifi-pulsar-nar + nar + + + + org.apache.nifi + nifi-pulsar-client-service-nar + 1.8.0 + nar + + + + org.apache.nifi + nifi-pulsar-processors + 1.8.0 + + + + org.apache.nifi + nifi-pulsar-client-service + 1.8.0 + + + + + diff --git a/nifi-pulsar-processors/pom.xml b/nifi-pulsar-processors/pom.xml new file mode 100644 index 0000000..602f7ed --- /dev/null +++ b/nifi-pulsar-processors/pom.xml @@ -0,0 +1,89 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-pulsar-bundle + 1.8.0 + + + nifi-pulsar-processors + jar + + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-record-serialization-service-api + + + org.apache.nifi + nifi-record + + + org.apache.nifi + nifi-utils + 1.8.0 + + + org.apache.nifi + nifi-ssl-context-service-api + + + org.apache.nifi + nifi-pulsar-client-service-api + 1.8.0 + provided + + + org.apache.pulsar + pulsar-client + ${pulsar.version} + + + org.apache.nifi + nifi-mock + test + 1.8.0 + + + org.slf4j + slf4j-simple + test + + + junit + junit + test + + + org.apache.commons + commons-lang3 + 3.8.1 + + + org.apache.directory.studio + org.apache.commons.io + 2.4 + + + + diff --git a/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/AbstractPulsarConsumerProcessor.java b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/AbstractPulsarConsumerProcessor.java new file mode 100644 index 0000000..faaa9c4 --- /dev/null +++ b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/AbstractPulsarConsumerProcessor.java @@ -0,0 +1,468 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnStopped; +import org.apache.nifi.annotation.lifecycle.OnUnscheduled; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.pulsar.PulsarClientService; +import org.apache.nifi.pulsar.cache.PulsarClientLRUCache; +import org.apache.nifi.util.StringUtils; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerBuilder; +import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SubscriptionType; + +public abstract class AbstractPulsarConsumerProcessor extends AbstractProcessor { + + static final AllowableValue EXCLUSIVE = new AllowableValue("Exclusive", "Exclusive", "There can be only 1 consumer on the same topic with the same subscription name"); + static final AllowableValue SHARED = new AllowableValue("Shared", "Shared", "Multiple consumer will be able to use the same subscription name and the messages"); + static final AllowableValue FAILOVER = new AllowableValue("Failover", "Failover", "Multiple consumer will be able to use the same subscription name but only 1 consumer " + + "will receive the messages. If that consumer disconnects, one of the other connected consumers will start receiving messages."); + + static final AllowableValue CONSUME = new AllowableValue(ConsumerCryptoFailureAction.CONSUME.name(), "Consume", + "Mark the message as consumed despite being unable to decrypt the contents"); + static final AllowableValue DISCARD = new AllowableValue(ConsumerCryptoFailureAction.DISCARD.name(), "Discard", + "Discard the message and don't perform any addtional processing on the message"); + static final AllowableValue FAIL = new AllowableValue(ConsumerCryptoFailureAction.FAIL.name(), "Fail", + "Report a failure condition, and then route the message contents to the FAILED relationship."); + + public static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("success") + .description("FlowFiles for which all content was consumed from Pulsar.") + .build(); + + public static final PropertyDescriptor PULSAR_CLIENT_SERVICE = new PropertyDescriptor.Builder() + .name("PULSAR_CLIENT_SERVICE") + .displayName("Pulsar Client Service") + .description("Specified the Pulsar Client Service that can be used to create Pulsar connections") + .required(true) + .identifiesControllerService(PulsarClientService.class) + .build(); + + public static final PropertyDescriptor TOPICS = new PropertyDescriptor.Builder() + .name("TOPICS") + .displayName("Topic Names") + .description("Specify the topics this consumer will subscribe on. " + + "You can specify multiple topics in a comma-separated list." + + "E.g topicA, topicB, topicC ") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + + public static final PropertyDescriptor TOPICS_PATTERN = new PropertyDescriptor.Builder() + .name("TOPICS_PATTERN") + .displayName("Topics Pattern") + .description("Alternatively, you can specify a pattern for topics that this consumer " + + "will subscribe on. It accepts a regular expression and will be compiled into " + + "a pattern internally. E.g. \"persistent://my-tenant/ns-abc/pattern-topic-.*\" " + + "would subscribe to any topic whose name started with 'pattern-topic-' that was in " + + "the 'ns-abc' namespace, and belonged to the 'my-tenant' tentant.") + .required(false) + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .build(); + + public static final PropertyDescriptor SUBSCRIPTION_NAME = new PropertyDescriptor.Builder() + .name("SUBSCRIPTION_NAME") + .displayName("Subscription Name") + .description("Specify the subscription name for this consumer.") + .required(true) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + + public static final PropertyDescriptor ASYNC_ENABLED = new PropertyDescriptor.Builder() + .name("ASYNC_ENABLED") + .displayName("Async Enabled") + .description("Control whether the messages will be consumed asyncronously or not. Messages consumed" + + " syncronously will be acknowledged immediately before processing the next message, while" + + " asyncronous messages will be acknowledged after the Pulsar broker responds. \n" + + "Enabling asyncronous message consumption introduces the possibility of duplicate data " + + "consumption in the case where the Processor is stopped before it has time to send an " + + "acknowledgement back to the Broker. In this scenario, the Broker would assume that the " + + "un-acknowledged message was not successuflly processed and re-send it when the Processor restarted.") + .required(true) + .allowableValues("true", "false") + .defaultValue("false") + .build(); + + public static final PropertyDescriptor MAX_ASYNC_REQUESTS = new PropertyDescriptor.Builder() + .name("MAX_ASYNC_REQUESTS") + .displayName("Maximum Async Requests") + .description("The maximum number of outstanding asynchronous consumer requests for this processor. " + + "Each asynchronous call requires memory, so avoid setting this value to high.") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .defaultValue("50") + .build(); + + public static final PropertyDescriptor ACK_TIMEOUT = new PropertyDescriptor.Builder() + .name("ACK_TIMEOUT") + .displayName("Acknowledgment Timeout") + .description("Set the timeout for unacked messages. Messages that are not acknowledged within the " + + "configured timeout will be replayed. This value needs to be greater than 10 seconds.") + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 sec") + .required(false) + .build(); + + public static final PropertyDescriptor CONSUMER_NAME = new PropertyDescriptor.Builder() + .name("CONSUMER_NAME") + .displayName("Consumer Name") + .description("Set the name of the consumer to uniquely identify this client on the Broker") + .required(false) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + + public static final PropertyDescriptor PRIORITY_LEVEL = new PropertyDescriptor.Builder() + .name("PRIORITY_LEVEL") + .displayName("Consumer Priority Level") + .description("Sets priority level for the shared subscription consumers to which broker " + + "gives more priority while dispatching messages. Here, broker follows descending " + + "priorities. (eg: 0=max-priority, 1, 2,..) ") + .required(false) + .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) + .defaultValue("5") + .build(); + + public static final PropertyDescriptor RECEIVER_QUEUE_SIZE = new PropertyDescriptor.Builder() + .name("RECEIVER_QUEUE_SIZE") + .displayName("Consumer Receiver Queue Size") + .description("The consumer receive queue controls how many messages can be accumulated " + + "by the Consumer before the application calls Consumer.receive(). Using a higher " + + "value could potentially increase the consumer throughput at the expense of bigger " + + "memory utilization. \n" + + "Setting the consumer queue size as zero, \n" + + "\t - Decreases the throughput of the consumer, by disabling pre-fetching of messages. \n" + + "\t - Doesn't support Batch-Message: if consumer receives any batch-message then it closes consumer " + + "connection with broker and consumer will not be able receive any further message unless batch-message " + + "in pipeline is removed") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .defaultValue("1000") + .build(); + + public static final PropertyDescriptor SUBSCRIPTION_TYPE = new PropertyDescriptor.Builder() + .name("SUBSCRIPTION_TYPE") + .displayName("Subscription Type") + .description("Select the subscription type to be used when subscribing to the topic.") + .required(true) + .allowableValues(EXCLUSIVE, SHARED, FAILOVER) + .defaultValue(SHARED.getValue()) + .build(); + + public static final PropertyDescriptor MESSAGE_DEMARCATOR = new PropertyDescriptor.Builder() + .name("MESSAGE_DEMARCATOR") + .displayName("Message Demarcator") + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .defaultValue("\n") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .description("Specifies the string (interpreted as UTF-8) to use for demarcating multiple messages consumed from Pulsar within " + + "a single FlowFile. If not specified, the content of the FlowFile will consist of all of the messages consumed from Pulsar " + + "concatenated together. If specified, the contents of the individual Pulsar messages will be separate by this delimiter. " + + "To enter special character such as 'new line' use CTRL+Enter or Shift+Enter, depending on your OS.") + .build(); + + public static final PropertyDescriptor CONSUMER_BATCH_SIZE = new PropertyDescriptor.Builder() + .name("CONSUMER_BATCH_SIZE") + .displayName("Consumer Message Batch Size") + .description("Set the maximum number of messages consumed at a time, and published to a single FlowFile. " + + "default: 1000. If set to a value greater than 1, messages within the FlowFile will be seperated " + + "by the Message Demarcator.") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .defaultValue("1000") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .build(); + + protected static final List PROPERTIES; + protected static final Set RELATIONSHIPS; + + static { + final List properties = new ArrayList<>(); + properties.add(PULSAR_CLIENT_SERVICE); + properties.add(TOPICS); + properties.add(TOPICS_PATTERN); + properties.add(SUBSCRIPTION_NAME); + properties.add(CONSUMER_NAME); + properties.add(ASYNC_ENABLED); + properties.add(MAX_ASYNC_REQUESTS); + properties.add(ACK_TIMEOUT); + properties.add(PRIORITY_LEVEL); + properties.add(RECEIVER_QUEUE_SIZE); + properties.add(SUBSCRIPTION_TYPE); + properties.add(CONSUMER_BATCH_SIZE); + properties.add(MESSAGE_DEMARCATOR); + + PROPERTIES = Collections.unmodifiableList(properties); + + final Set relationships = new HashSet<>(); + relationships.add(REL_SUCCESS); + RELATIONSHIPS = Collections.unmodifiableSet(relationships); + } + + private PulsarClientService pulsarClientService; + private PulsarClientLRUCache> consumers; + private ExecutorService consumerPool; + private ExecutorCompletionService>> consumerService; + private ExecutorService ackPool; + private ExecutorCompletionService ackService; + + @Override + public Set getRelationships() { + return RELATIONSHIPS; + } + + @Override + protected List getSupportedPropertyDescriptors() { + return PROPERTIES; + } + + @Override + protected Collection customValidate(ValidationContext validationContext) { + Set results = new HashSet<>(); + boolean topicsSet = validationContext.getProperty(TOPICS).isSet(); + boolean topicPatternSet = validationContext.getProperty(TOPICS_PATTERN).isSet(); + + if (!topicsSet && !topicPatternSet) { + results.add(new ValidationResult.Builder().valid(false).explanation( + "At least one of the 'Topics' or 'Topic Pattern' properties must be specified.").build()); + } else if (topicsSet && topicPatternSet) { + results.add(new ValidationResult.Builder().valid(false).explanation( + "Only one of the two properties ('Topics' and 'Topic Pattern') can be specified.").build()); + } + + if (validationContext.getProperty(ACK_TIMEOUT).asTimePeriod(TimeUnit.SECONDS) < 10) { + results.add(new ValidationResult.Builder().valid(false).explanation( + "Acknowledgment Timeout needs to be greater than 10 seconds.").build()); + } + + return results; + } + + @OnScheduled + public void init(ProcessContext context) { + if (context.getProperty(ASYNC_ENABLED).isSet() && context.getProperty(ASYNC_ENABLED).asBoolean()) { + setConsumerPool(Executors.newFixedThreadPool(context.getProperty(MAX_ASYNC_REQUESTS).asInteger())); + setConsumerService(new ExecutorCompletionService<>(getConsumerPool())); + setAckPool(Executors.newFixedThreadPool(context.getProperty(MAX_ASYNC_REQUESTS).asInteger() + 1)); + setAckService(new ExecutorCompletionService<>(getAckPool())); + } + + setPulsarClientService(context.getProperty(PULSAR_CLIENT_SERVICE).asControllerService(PulsarClientService.class)); + } + + @OnUnscheduled + public void shutDown(final ProcessContext context) { + /* + * If we are running in asynchronous mode, then we need to stop all of the consumer threads that + * are running in the ConsumerPool. After, we have stopped them, we need to wait a little bit + * to ensure that all of the messages are properly acked, in order to prevent re-processing the + * same messages in the event of a shutdown and restart of the processor since the un-acked + * messages would be replayed on startup. + */ + if (context.getProperty(ASYNC_ENABLED).isSet() && context.getProperty(ASYNC_ENABLED).asBoolean()) { + try { + getConsumerPool().shutdown(); + getAckPool().shutdown(); + + // Allow some time for the acks to be sent back to the Broker. + getConsumerPool().awaitTermination(10, TimeUnit.SECONDS); + getAckPool().awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + getLogger().error("Unable to stop all the Pulsar Consumers", e); + } + } + } + + @OnStopped + public void cleanUp(final ProcessContext context) { + shutDown(context); + getConsumers().clear(); + } + + /** + * Method returns a string that uniquely identifies a consumer by concatenating + * the topic name and subscription properties together. + */ + protected String getConsumerId(final ProcessContext context, FlowFile flowFile) { + if (context == null) { + return null; + } + + StringBuffer sb = new StringBuffer(); + + if (context.getProperty(TOPICS).isSet()) { + sb.append(context.getProperty(TOPICS).evaluateAttributeExpressions(flowFile).getValue()); + } else { + sb.append(context.getProperty(TOPICS_PATTERN).getValue()); + } + + sb.append("-").append(context.getProperty(SUBSCRIPTION_NAME).getValue()); + + if (context.getProperty(CONSUMER_NAME).isSet()) { + sb.append("-").append(context.getProperty(CONSUMER_NAME).getValue()); + } + return sb.toString(); + } + + protected void consumeAsync(final Consumer consumer, ProcessContext context, ProcessSession session) throws PulsarClientException { + try { + final int maxMessages = context.getProperty(CONSUMER_BATCH_SIZE).isSet() ? context.getProperty(CONSUMER_BATCH_SIZE) + .evaluateAttributeExpressions().asInteger() : Integer.MAX_VALUE; + + getConsumerService().submit(() -> { + List> messages = new LinkedList>(); + Message msg = null; + AtomicInteger msgCount = new AtomicInteger(0); + + while (((msg = consumer.receive(0, TimeUnit.SECONDS)) != null) && msgCount.get() < maxMessages) { + messages.add(msg); + msgCount.incrementAndGet(); + } + return messages; + }); + } catch (final RejectedExecutionException ex) { + getLogger().error("Unable to consume any more Pulsar messages", ex); + context.yield(); + } + } + + protected synchronized Consumer getConsumer(ProcessContext context, String topic) throws PulsarClientException { + + /* Avoid creating producers for non-existent topics */ + if (StringUtils.isBlank(topic)) { + return null; + } + + Consumer consumer = getConsumers().get(topic); + + if (consumer != null && consumer.isConnected()) { + return consumer; + } + + // Create a new consumer and validate that it is connected before returning it. + consumer = getConsumerBuilder(context).subscribe(); + if (consumer != null && consumer.isConnected()) { + getConsumers().put(topic, consumer); + } + + return (consumer != null && consumer.isConnected()) ? consumer : null; + } + + protected synchronized ConsumerBuilder getConsumerBuilder(ProcessContext context) throws PulsarClientException { + + ConsumerBuilder builder = (ConsumerBuilder) getPulsarClientService().getPulsarClient().newConsumer(); + + if (context.getProperty(TOPICS).isSet()) { + builder = builder.topic(Arrays.stream(context.getProperty(TOPICS).evaluateAttributeExpressions().getValue().split("[, ]")) + .map(String::trim).toArray(String[]::new)); + } else if (context.getProperty(TOPICS_PATTERN).isSet()) { + builder = builder.topicsPattern(context.getProperty(TOPICS_PATTERN).getValue()); + } + + if (context.getProperty(CONSUMER_NAME).isSet()) { + builder = builder.consumerName(context.getProperty(CONSUMER_NAME).getValue()); + } + + return builder.subscriptionName(context.getProperty(SUBSCRIPTION_NAME).getValue()) + .ackTimeout(context.getProperty(ACK_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(), TimeUnit.MILLISECONDS) + .priorityLevel(context.getProperty(PRIORITY_LEVEL).asInteger()) + .receiverQueueSize(context.getProperty(RECEIVER_QUEUE_SIZE).asInteger()) + .subscriptionType(SubscriptionType.valueOf(context.getProperty(SUBSCRIPTION_TYPE).getValue())); + } + + protected synchronized ExecutorService getConsumerPool() { + return consumerPool; + } + + protected synchronized void setConsumerPool(ExecutorService pool) { + this.consumerPool = pool; + } + + protected synchronized ExecutorCompletionService>> getConsumerService() { + return consumerService; + } + + protected synchronized void setConsumerService(ExecutorCompletionService>> service) { + this.consumerService = service; + } + + protected synchronized ExecutorService getAckPool() { + return ackPool; + } + + protected synchronized void setAckPool(ExecutorService pool) { + this.ackPool = pool; + } + + protected synchronized ExecutorCompletionService getAckService() { + return ackService; + } + + protected synchronized void setAckService(ExecutorCompletionService ackService) { + this.ackService = ackService; + } + + protected synchronized PulsarClientService getPulsarClientService() { + return pulsarClientService; + } + + protected synchronized void setPulsarClientService(PulsarClientService pulsarClientService) { + this.pulsarClientService = pulsarClientService; + } + + protected synchronized PulsarClientLRUCache> getConsumers() { + if (consumers == null) { + consumers = new PulsarClientLRUCache>(20); + } + return consumers; + } + + protected void setConsumers(PulsarClientLRUCache> consumers) { + this.consumers = consumers; + } +} diff --git a/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/AbstractPulsarProducerProcessor.java b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/AbstractPulsarProducerProcessor.java new file mode 100644 index 0000000..1fd31d4 --- /dev/null +++ b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/AbstractPulsarProducerProcessor.java @@ -0,0 +1,485 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnStopped; +import org.apache.nifi.annotation.lifecycle.OnUnscheduled; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.pulsar.PulsarClientService; +import org.apache.nifi.pulsar.cache.PulsarClientLRUCache; +import org.apache.nifi.util.StringUtils; +import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.MessageRoutingMode; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.shade.org.apache.commons.collections.CollectionUtils; + +public abstract class AbstractPulsarProducerProcessor extends AbstractProcessor { + + public static final String MSG_COUNT = "msg.count"; + public static final String TOPIC_NAME = "topic.name"; + + static final AllowableValue COMPRESSION_TYPE_NONE = new AllowableValue("NONE", "None", "No compression"); + static final AllowableValue COMPRESSION_TYPE_LZ4 = new AllowableValue("LZ4", "LZ4", "Compress with LZ4 algorithm."); + static final AllowableValue COMPRESSION_TYPE_ZLIB = new AllowableValue("ZLIB", "ZLIB", "Compress with ZLib algorithm"); + + static final AllowableValue MESSAGE_ROUTING_MODE_CUSTOM_PARTITION = new AllowableValue("CustomPartition", "Custom Partition", "Route messages to a custom partition"); + static final AllowableValue MESSAGE_ROUTING_MODE_ROUND_ROBIN_PARTITION = new AllowableValue("RoundRobinPartition", "Round Robin Partition", "Route messages to all " + + "partitions in a round robin manner"); + static final AllowableValue MESSAGE_ROUTING_MODE_SINGLE_PARTITION = new AllowableValue("SinglePartition", "Single Partition", "Route messages to a single partition"); + + public static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("success") + .description("FlowFiles for which all content was sent to Pulsar.") + .build(); + + public static final Relationship REL_FAILURE = new Relationship.Builder() + .name("failure") + .description("Any FlowFile that cannot be sent to Pulsar will be routed to this Relationship") + .build(); + + public static final PropertyDescriptor PULSAR_CLIENT_SERVICE = new PropertyDescriptor.Builder() + .name("PULSAR_CLIENT_SERVICE") + .displayName("Pulsar Client Service") + .description("Specified the Pulsar Client Service that can be used to create Pulsar connections") + .required(true) + .identifiesControllerService(PulsarClientService.class) + .build(); + + public static final PropertyDescriptor TOPIC = new PropertyDescriptor.Builder() + .name("TOPIC") + .displayName("Topic Name") + .description("The name of the Pulsar Topic.") + .required(true) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .build(); + + public static final PropertyDescriptor ASYNC_ENABLED = new PropertyDescriptor.Builder() + .name("ASYNC_ENABLED") + .displayName("Async Enabled") + .description("Control whether the messages will be sent asyncronously or not. Messages sent" + + " syncronously will be acknowledged immediately before processing the next message, while" + + " asyncronous messages will be acknowledged after the Pulsar broker responds. Running the" + + " processor with async enabled will result in increased the throughput at the risk of potential" + + " duplicate data being sent to the Pulsar broker.") + .required(true) + .allowableValues("true", "false") + .defaultValue("false") + .build(); + + public static final PropertyDescriptor MAX_ASYNC_REQUESTS = new PropertyDescriptor.Builder() + .name("MAX_ASYNC_REQUESTS") + .displayName("Maximum Async Requests") + .description("The maximum number of outstanding asynchronous publish requests for this processor. " + + "Each asynchronous call requires memory, so avoid setting this value to high.") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .defaultValue("50") + .build(); + + public static final PropertyDescriptor BATCHING_ENABLED = new PropertyDescriptor.Builder() + .name("BATCHING_ENABLED") + .displayName("Batching Enabled") + .description("Control whether automatic batching of messages is enabled for the producer. " + + "default: false [No batching] When batching is enabled, multiple calls to " + + "Producer.sendAsync can result in a single batch to be sent to the broker, leading " + + "to better throughput, especially when publishing small messages. If compression is " + + "enabled, messages will be compressed at the batch level, leading to a much better " + + "compression ratio for similar headers or contents. When enabled default batch delay " + + "is set to 10 ms and default batch size is 1000 messages") + .required(true) + .allowableValues("true", "false") + .defaultValue("false") + .build(); + + public static final PropertyDescriptor BATCHING_MAX_MESSAGES = new PropertyDescriptor.Builder() + .name("BATCHING_MAX_MESSAGES") + .displayName("Batching Max Messages") + .description("Set the maximum number of messages permitted in a batch within the Pulsar client. " + + "default: 1000. If set to a value greater than 1, messages will be queued until this " + + "threshold is reached or the batch interval has elapsed, whichever happens first.") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .defaultValue("1000") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .build(); + + public static final PropertyDescriptor BATCH_INTERVAL = new PropertyDescriptor.Builder() + .name("BATCH_INTERVAL") + .displayName("Batch Interval") + .description("Set the time period within which the messages sent will be batched if batch messages are enabled." + + " If set to a non zero value, messages will be queued until this time interval has been reached OR" + + " until the Batching Max Messages threshould has been reached, whichever occurs first.") + .required(false) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .defaultValue("10 ms") + .build(); + + public static final PropertyDescriptor BLOCK_IF_QUEUE_FULL = new PropertyDescriptor.Builder() + .name("BLOCK_IF_QUEUE_FULL") + .displayName("Block if Message Queue Full") + .description("Set whether the processor should block when the outgoing message queue is full. " + + "Default is false. If set to false, send operations will immediately fail with " + + "ProducerQueueIsFullError when there is no space left in pending queue.") + .required(true) + .allowableValues("true", "false") + .defaultValue("false") + .build(); + + public static final PropertyDescriptor COMPRESSION_TYPE = new PropertyDescriptor.Builder() + .name("COMPRESSION_TYPE") + .displayName("Compression Type") + .description("Set the compression type for the producer.") + .required(true) + .allowableValues(COMPRESSION_TYPE_NONE, COMPRESSION_TYPE_LZ4, COMPRESSION_TYPE_ZLIB) + .defaultValue(COMPRESSION_TYPE_NONE.getValue()) + .build(); + + public static final PropertyDescriptor MESSAGE_DEMARCATOR = new PropertyDescriptor.Builder() + .name("MESSAGE_DEMARCATOR") + .displayName("Message Demarcator") + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .description("Specifies the string (interpreted as UTF-8) to use for demarcating multiple messages within " + + "a single FlowFile. If not specified, the entire content of the FlowFile will be used as a single message. If specified, the " + + "contents of the FlowFile will be split on this delimiter and each section sent as a separate Pulsar message. " + + "To enter special character such as 'new line' use CTRL+Enter or Shift+Enter, depending on your OS.") + .build(); + + public static final PropertyDescriptor MESSAGE_ROUTING_MODE = new PropertyDescriptor.Builder() + .name("MESSAGE_ROUTING_MODE") + .displayName("Message Routing Mode") + .description("Set the message routing mode for the producer. This applies only if the destination topic is partitioned") + .required(true) + .allowableValues(MESSAGE_ROUTING_MODE_CUSTOM_PARTITION, MESSAGE_ROUTING_MODE_ROUND_ROBIN_PARTITION, MESSAGE_ROUTING_MODE_SINGLE_PARTITION) + .defaultValue(MESSAGE_ROUTING_MODE_ROUND_ROBIN_PARTITION.getValue()) + .build(); + + public static final PropertyDescriptor PENDING_MAX_MESSAGES = new PropertyDescriptor.Builder() + .name("PENDING_MAX_MESSAGES") + .displayName("Max Pending Messages") + .description("Set the max size of the queue holding the messages pending to receive an " + + "acknowledgment from the broker.") + .required(false) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .defaultValue("1000") + .build(); + + protected static final List PROPERTIES; + protected static final Set RELATIONSHIPS; + + static { + final List properties = new ArrayList<>(); + properties.add(PULSAR_CLIENT_SERVICE); + properties.add(TOPIC); + properties.add(ASYNC_ENABLED); + properties.add(MAX_ASYNC_REQUESTS); + properties.add(BATCHING_ENABLED); + properties.add(BATCHING_MAX_MESSAGES); + properties.add(BATCH_INTERVAL); + properties.add(BLOCK_IF_QUEUE_FULL); + properties.add(COMPRESSION_TYPE); + properties.add(MESSAGE_ROUTING_MODE); + properties.add(MESSAGE_DEMARCATOR); + properties.add(PENDING_MAX_MESSAGES); + PROPERTIES = Collections.unmodifiableList(properties); + + final Set relationships = new HashSet<>(); + relationships.add(REL_SUCCESS); + relationships.add(REL_FAILURE); + RELATIONSHIPS = Collections.unmodifiableSet(relationships); + } + + @Override + public Set getRelationships() { + return RELATIONSHIPS; + } + + @Override + protected List getSupportedPropertyDescriptors() { + return PROPERTIES; + } + + private PulsarClientService pulsarClientService; + private PulsarClientLRUCache> producers; + private ExecutorService publisherPool; + + // Used to sync between onTrigger method and shutdown code block. + protected AtomicBoolean canPublish = new AtomicBoolean(); + + // Used to track whether we are reporting errors back to the user or not. + protected AtomicBoolean trackFailures = new AtomicBoolean(); + + private int maxRequests = 1; + + protected BlockingQueue> workQueue; + protected BlockingQueue> failureQueue; + protected List asyncPublishers; + + @OnScheduled + public void init(ProcessContext context) { + maxRequests = context.getProperty(MAX_ASYNC_REQUESTS).asInteger(); + setPulsarClientService(context.getProperty(PULSAR_CLIENT_SERVICE).asControllerService(PulsarClientService.class)); + + if (context.getProperty(ASYNC_ENABLED).isSet() && context.getProperty(ASYNC_ENABLED).asBoolean()) { + setPublisherPool(Executors.newFixedThreadPool(maxRequests)); + setAsyncPublishers(new LinkedList()); + // Limit the depth of the work queue to 500 per worker, to prevent long shutdown times. + workQueue = new LinkedBlockingQueue>(500 * maxRequests); + + if (context.hasConnection(REL_FAILURE)) { + failureQueue = new LinkedBlockingQueue>(); + trackFailures.set(true); + } else { + trackFailures.set(false); + } + + for (int idx = 0; idx < maxRequests; idx++) { + AsyncPublisher worker = new AsyncPublisher(); + getAsyncPublishers().add(worker); + getPublisherPool().submit(worker); + } + canPublish.set(true); + } + } + + @OnUnscheduled + public void shutDown(final ProcessContext context) { + /* + * If we are running in asynchronous mode, then we need to stop all of the producer threads that + * are running in the PublisherPool. After, we have stopped them, we need to wait a little bit + * to ensure that all of the messages are properly acked, in order to prevent re-processing the + * same messages in the event of a shutdown and restart of the processor since the un-acked + * messages would be replayed on startup. + */ + if (context.getProperty(ASYNC_ENABLED).isSet() && context.getProperty(ASYNC_ENABLED).asBoolean()) { + try { + // Stop accepting incoming work + canPublish.set(false); + + // Halt the background worker threads, allowing them to empty the workQueue + getAsyncPublishers().forEach(publisher->{ + publisher.halt(); + }); + + // Flush all of the pending messages in the producers + getProducers().values().forEach(producer -> { + try { + producer.flush(); + } catch (PulsarClientException e) { + // ignore + } + }); + + // Shutdown the thread pool + getPublisherPool().shutdown(); + getPublisherPool().awaitTermination(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + getLogger().error("Unable to stop all the Pulsar Producers", e); + } + } + } + + @OnStopped + public void cleanUp(final ProcessContext context) { + if (context.getProperty(ASYNC_ENABLED).isSet() && context.getProperty(ASYNC_ENABLED).asBoolean()) { + if (canPublish.get()) { + shutDown(context); + } + workQueue.clear(); + getProducers().clear(); + getAsyncPublishers().clear(); + } + } + + /** + * If the processor is configured to run in asynchronous mode, then we need to periodically + * check the failureList and route those records to the FAILURE relationship, so that the end + * user is aware of the failures and can handle them as they see fit. + */ + protected void handleFailures(ProcessSession session) { + + if (!trackFailures.get() || CollectionUtils.isEmpty(failureQueue)) { + return; + } + + Pair failure = failureQueue.poll(); + + while (failure != null) { + FlowFile flowFile = session.create(); + final byte[] value = (byte[]) failure.getValue(); + flowFile = session.write(flowFile, out -> { + out.write(value); + }); + session.putAttribute(flowFile, TOPIC_NAME, failure.getKey()); + session.transfer(flowFile, REL_FAILURE); + failure = failureQueue.poll(); + } + } + + private synchronized List.AsyncPublisher> getAsyncPublishers() { + return asyncPublishers; + } + + private synchronized void setAsyncPublishers(List.AsyncPublisher> list) { + asyncPublishers = list; + } + + protected synchronized Producer getProducer(ProcessContext context, String topic) { + + /* Avoid creating producers for non-existent topics */ + if (StringUtils.isBlank(topic)) { + return null; + } + + Producer producer = getProducers().get(topic); + + try { + if (producer != null && producer.isConnected()) { + return producer; + } + + producer = getBuilder(context, topic).create(); + + if (producer != null && producer.isConnected()) { + getProducers().put(topic, producer); + } + } catch (PulsarClientException e) { + getLogger().error("Unable to create Pulsar Producer ", e); + producer = null; + } + return (producer != null && producer.isConnected()) ? producer : null; + } + + private synchronized ProducerBuilder getBuilder(ProcessContext context, String topic) { + ProducerBuilder builder = (ProducerBuilder) getPulsarClientService().getPulsarClient().newProducer(); + return builder.topic(topic) + .enableBatching(context.getProperty(BATCHING_ENABLED).asBoolean()) + .batchingMaxMessages(context.getProperty(BATCHING_MAX_MESSAGES).evaluateAttributeExpressions().asInteger()) + .batchingMaxPublishDelay(context.getProperty(BATCH_INTERVAL).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS).intValue(), TimeUnit.MILLISECONDS) + .blockIfQueueFull(context.getProperty(BLOCK_IF_QUEUE_FULL).asBoolean()) + .compressionType(CompressionType.valueOf(context.getProperty(COMPRESSION_TYPE).getValue())) + .maxPendingMessages(context.getProperty(PENDING_MAX_MESSAGES).evaluateAttributeExpressions().asInteger()) + .messageRoutingMode(MessageRoutingMode.valueOf(context.getProperty(MESSAGE_ROUTING_MODE).getValue())); + } + + protected synchronized PulsarClientService getPulsarClientService() { + return pulsarClientService; + } + + protected synchronized void setPulsarClientService(PulsarClientService pulsarClientService) { + this.pulsarClientService = pulsarClientService; + } + + protected synchronized PulsarClientLRUCache> getProducers() { + if (producers == null) { + producers = new PulsarClientLRUCache>(20); + } + return producers; + } + + protected synchronized void setProducers(PulsarClientLRUCache> producers) { + this.producers = producers; + } + + protected synchronized ExecutorService getPublisherPool() { + return publisherPool; + } + + protected synchronized void setPublisherPool(ExecutorService publisherPool) { + this.publisherPool = publisherPool; + } + + private final class AsyncPublisher implements Runnable { + private boolean keepRunning = true; + private boolean completed = false; + + public void halt() { + keepRunning = false; + + // Finish up + completed = workQueue.isEmpty(); + while (!completed) { + process(); + completed = workQueue.isEmpty(); + } + } + + @Override + public void run() { + while (keepRunning) { + process(); + } + } + + private void process() { + try { + Pair item = workQueue.take(); + Producer producer = getProducers().get(item.getLeft()); + + if (!trackFailures.get()) { + // We don't care about failures, so just fire & forget + producer.sendAsync(item.getValue()); + } else if (producer == null || !producer.isConnected()) { + // We cannot get a valid producer, so add the item to the failure queue + failureQueue.put(item); + } else { + try { + // Send the item asynchronously and confirm we get a messageId back from Pulsar. + if (producer.sendAsync(item.getValue()).join() == null) { + // No messageId indicates failure + failureQueue.put(item); + } + } catch (final Throwable t) { + // Any exception during sendAsync() call indicates failure + failureQueue.put(item); + } + } + } catch (InterruptedException e) { + // Ignore these + } + } + } +} \ No newline at end of file diff --git a/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/ConsumePulsar.java b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/ConsumePulsar.java new file mode 100644 index 0000000..fcfff62 --- /dev/null +++ b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/ConsumePulsar.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.SeeAlso; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processors.pulsar.AbstractPulsarConsumerProcessor; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.shade.org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; + +@SeeAlso({PublishPulsar.class, ConsumePulsarRecord.class, PublishPulsarRecord.class}) +@Tags({"Pulsar", "Get", "Ingest", "Ingress", "Topic", "PubSub", "Consume"}) +@CapabilityDescription("Consumes messages from Apache Pulsar. The complementary NiFi processor for sending messages is PublishPulsar.") +@InputRequirement(InputRequirement.Requirement.INPUT_FORBIDDEN) +@WritesAttributes({ + @WritesAttribute(attribute = "message.count", description = "The number of messages received from Pulsar") +}) +public class ConsumePulsar extends AbstractPulsarConsumerProcessor { + + public static final String MSG_COUNT = "message.count"; + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + try { + Consumer consumer = getConsumer(context, getConsumerId(context, session.get())); + + if (consumer == null) { + context.yield(); + return; + } + + if (context.getProperty(ASYNC_ENABLED).asBoolean()) { + consumeAsync(consumer, context, session); + handleAsync(consumer, context, session); + } else { + consume(consumer, context, session); + } + } catch (PulsarClientException e) { + getLogger().error("Unable to consume from Pulsar Topic ", e); + context.yield(); + throw new ProcessException(e); + } + } + + private void handleAsync(final Consumer consumer, ProcessContext context, ProcessSession session) { + try { + Future>> done = getConsumerService().poll(5, TimeUnit.SECONDS); + + if (done != null) { + + final byte[] demarcatorBytes = context.getProperty(MESSAGE_DEMARCATOR).isSet() ? context.getProperty(MESSAGE_DEMARCATOR) + .evaluateAttributeExpressions().getValue().getBytes(StandardCharsets.UTF_8) : null; + + List> messages = done.get(); + + if (CollectionUtils.isNotEmpty(messages)) { + FlowFile flowFile = session.create(); + OutputStream out = session.write(flowFile); + AtomicInteger msgCount = new AtomicInteger(0); + + messages.forEach(msg -> { + try { + out.write(msg.getValue()); + out.write(demarcatorBytes); + msgCount.getAndIncrement(); + } catch (final IOException ioEx) { + session.rollback(); + return; + } + }); + + IOUtils.closeQuietly(out); + + session.putAttribute(flowFile, MSG_COUNT, msgCount.toString()); + session.getProvenanceReporter().receive(flowFile, getPulsarClientService().getPulsarBrokerRootURL() + "/" + consumer.getTopic()); + session.transfer(flowFile, REL_SUCCESS); + session.commit(); + } + // Acknowledge consuming the message + getAckService().submit(new Callable() { + @Override + public Object call() throws Exception { + return consumer.acknowledgeCumulativeAsync(messages.get(messages.size()-1)).get(); + } + }); + } + } catch (InterruptedException | ExecutionException e) { + getLogger().error("Trouble consuming messages ", e); + } + } + + private void consume(Consumer consumer, ProcessContext context, ProcessSession session) throws PulsarClientException { + try { + final int maxMessages = context.getProperty(CONSUMER_BATCH_SIZE).isSet() ? context.getProperty(CONSUMER_BATCH_SIZE) + .evaluateAttributeExpressions().asInteger() : Integer.MAX_VALUE; + + final byte[] demarcatorBytes = context.getProperty(MESSAGE_DEMARCATOR).isSet() ? context.getProperty(MESSAGE_DEMARCATOR) + .evaluateAttributeExpressions().getValue().getBytes(StandardCharsets.UTF_8) : null; + + FlowFile flowFile = session.create(); + OutputStream out = session.write(flowFile); + Message msg = null; + Message lastMsg = null; + AtomicInteger msgCount = new AtomicInteger(0); + AtomicInteger loopCounter = new AtomicInteger(0); + + while (((msg = consumer.receive(0, TimeUnit.SECONDS)) != null) && loopCounter.get() < maxMessages) { + try { + + lastMsg = msg; + loopCounter.incrementAndGet(); + + // Skip empty messages, as they cause NPE's when we write them to the OutputStream + if (msg.getValue() == null || msg.getValue().length < 1) { + continue; + } + out.write(msg.getValue()); + out.write(demarcatorBytes); + msgCount.getAndIncrement(); + + } catch (final IOException ioEx) { + session.rollback(); + return; + } + } + + IOUtils.closeQuietly(out); + + if (lastMsg != null) { + consumer.acknowledgeCumulative(lastMsg); + } + + if (msgCount.get() < 1) { + session.remove(flowFile); + session.commit(); + } else { + session.putAttribute(flowFile, MSG_COUNT, msgCount.toString()); + session.getProvenanceReporter().receive(flowFile, getPulsarClientService().getPulsarBrokerRootURL() + "/" + consumer.getTopic()); + session.transfer(flowFile, REL_SUCCESS); + getLogger().debug("Created {} from {} messages received from Pulsar Server and transferred to 'success'", + new Object[]{flowFile, msgCount.toString()}); + } + + } catch (PulsarClientException e) { + context.yield(); + session.rollback(); + } + } +} \ No newline at end of file diff --git a/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/ConsumePulsarRecord.java b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/ConsumePulsarRecord.java new file mode 100644 index 0000000..f038449 --- /dev/null +++ b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/ConsumePulsarRecord.java @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.io.IOUtils; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.SeeAlso; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.pulsar.AbstractPulsarConsumerProcessor; +import org.apache.nifi.schema.access.SchemaNotFoundException; +import org.apache.nifi.serialization.MalformedRecordException; +import org.apache.nifi.serialization.RecordReader; +import org.apache.nifi.serialization.RecordReaderFactory; +import org.apache.nifi.serialization.RecordSetWriter; +import org.apache.nifi.serialization.RecordSetWriterFactory; +import org.apache.nifi.serialization.WriteResult; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.shade.org.apache.commons.collections.CollectionUtils; + +@CapabilityDescription("Consumes messages from Apache Pulsar. " + + "The complementary NiFi processor for sending messages is PublishPulsarRecord. Please note that, at this time, " + + "the Processor assumes that all records that are retrieved have the same schema. If any of the Pulsar messages " + + "that are pulled but cannot be parsed or written with the configured Record Reader or Record Writer, the contents " + + "of the message will be written to a separate FlowFile, and that FlowFile will be transferred to the 'parse.failure' " + + "relationship. Otherwise, each FlowFile is sent to the 'success' relationship and may contain many individual " + + "messages within the single FlowFile. A 'record.count' attribute is added to indicate how many messages are contained in the " + + "FlowFile. No two Pulsar messages will be placed into the same FlowFile if they have different schemas.") +@Tags({"Pulsar", "Get", "Record", "csv", "avro", "json", "Ingest", "Ingress", "Topic", "PubSub", "Consume"}) +@WritesAttributes({ + @WritesAttribute(attribute = "record.count", description = "The number of records received") +}) +@InputRequirement(InputRequirement.Requirement.INPUT_FORBIDDEN) +@SeeAlso({PublishPulsar.class, ConsumePulsar.class, PublishPulsarRecord.class}) +public class ConsumePulsarRecord extends AbstractPulsarConsumerProcessor { + + public static final String MSG_COUNT = "record.count"; + private static final String RECORD_SEPARATOR = "\n"; + + public static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder() + .name("Record Reader") + .displayName("Record Reader") + .description("The Record Reader to use for incoming FlowFiles") + .identifiesControllerService(RecordReaderFactory.class) + .required(true) + .build(); + + public static final PropertyDescriptor RECORD_WRITER = new PropertyDescriptor.Builder() + .name("Record Writer") + .displayName("Record Writer") + .description("The Record Writer to use in order to serialize the data before sending to Pulsar") + .identifiesControllerService(RecordSetWriterFactory.class) + .required(true) + .build(); + + public static final PropertyDescriptor MAX_WAIT_TIME = new PropertyDescriptor.Builder() + .name("Max Wait Time") + .description("The maximum amount of time allowed for a Pulsar consumer to poll a subscription for data " + + ", zero means there is no limit. Max time less than 1 second will be equal to zero.") + .defaultValue("2 seconds") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .build(); + + public static final Relationship REL_PARSE_FAILURE = new Relationship.Builder() + .name("parse_failure") + .description("FlowFiles for which the content cannot be parsed.") + .build(); + + private static final List PROPERTIES; + private static final Set RELATIONSHIPS; + + static { + final List properties = new ArrayList<>(); + properties.add(RECORD_READER); + properties.add(RECORD_WRITER); + properties.add(MAX_WAIT_TIME); + properties.addAll(AbstractPulsarConsumerProcessor.PROPERTIES); + PROPERTIES = Collections.unmodifiableList(properties); + + final Set relationships = new HashSet<>(); + relationships.add(REL_SUCCESS); + relationships.add(REL_PARSE_FAILURE); + RELATIONSHIPS = Collections.unmodifiableSet(relationships); + } + + @Override + public Set getRelationships() { + return RELATIONSHIPS; + } + + @Override + protected List getSupportedPropertyDescriptors() { + return PROPERTIES; + } + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER) + .asControllerService(RecordReaderFactory.class); + + final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER) + .asControllerService(RecordSetWriterFactory.class); + + final int maxMessages = context.getProperty(CONSUMER_BATCH_SIZE).isSet() ? context.getProperty(CONSUMER_BATCH_SIZE) + .evaluateAttributeExpressions().asInteger() : Integer.MAX_VALUE; + + final byte[] demarcator = context.getProperty(MESSAGE_DEMARCATOR).isSet() ? context.getProperty(MESSAGE_DEMARCATOR) + .evaluateAttributeExpressions().getValue().getBytes() : RECORD_SEPARATOR.getBytes(); + + try { + Consumer consumer = getConsumer(context, getConsumerId(context, session.get())); + + if (consumer == null) { /* If we aren't connected to Pulsar, then just yield */ + context.yield(); + return; + } + + if (context.getProperty(ASYNC_ENABLED).isSet() && context.getProperty(ASYNC_ENABLED).asBoolean()) { + consumeAsync(consumer, context, session); + handleAsync(context, session, consumer, readerFactory, writerFactory, demarcator); + } else { + consumeMessages(session, consumer, getMessages(consumer, maxMessages), readerFactory, writerFactory, demarcator); + } + } catch (PulsarClientException e) { + getLogger().error("Unable to consume from Pulsar Topic ", e); + context.yield(); + throw new ProcessException(e); + } + } + + /** + * Retrieve a batch of up to maxMessages for processing. + * + * @param consumer - The Pulsar consumer. + * @param maxMessages - The maximum number of messages to consume from Pulsar. + * @return A List of Messages + * @throws PulsarClientException in the event we cannot communicate with the Pulsar broker. + */ + private List> getMessages(final Consumer consumer, int maxMessages) throws PulsarClientException { + List> messages = new LinkedList>(); + Message msg = null; + AtomicInteger msgCount = new AtomicInteger(0); + + while (((msg = consumer.receive(0, TimeUnit.SECONDS)) != null) && msgCount.get() < maxMessages) { + messages.add(msg); + msgCount.incrementAndGet(); + } + return messages; + } + + /** + * Perform the actual processing of the messages, by parsing the messages and writing them out to a FlowFile. + * All of the messages passed in shall be routed to either SUCCESS or PARSE_FAILURE, allowing us to acknowledge + * the receipt of the messages to Pulsar, so they are not re-sent. + * + * @param session - The current ProcessSession. + * @param consumer - The Pulsar consumer. + * @param messages - A list of messages. + * @param readerFactory - The factory used to read the messages. + * @param writerFactory - The factory used to write the messages. + * @throws PulsarClientException if there is an issue communicating with Apache Pulsar. + */ + private void consumeMessages(ProcessSession session, final Consumer consumer, final List> messages, + final RecordReaderFactory readerFactory, RecordSetWriterFactory writerFactory, final byte[] demarcator) throws PulsarClientException { + + if (CollectionUtils.isEmpty(messages)) { + return; + } + + RecordSchema schema = getSchema(readerFactory, messages.get(0)); + final BlockingQueue> parseFailures = new LinkedBlockingQueue>(); + FlowFile flowFile = session.create(); + OutputStream rawOut = session.write(flowFile); + final RecordSetWriter writer = getRecordWriter(writerFactory, schema, rawOut); + + // We were unable to determine the schema, therefore we cannot parse the messages + if (schema == null || writer == null) { + parseFailures.addAll(messages); + + // We aren't going to write any records to the FlowFile, so remove it and close the associated output stream + session.remove(flowFile); + IOUtils.closeQuietly(rawOut); + getLogger().error("Unable create a record writer to consume from the Pulsar topic"); + } else { + try { + writer.beginRecordSet(); + messages.forEach(msg ->{ + final InputStream in = new ByteArrayInputStream(msg.getValue()); + try { + RecordReader r = readerFactory.createRecordReader(Collections.emptyMap(), in, getLogger()); + for (Record record = r.nextRecord(); record != null; record = r.nextRecord()) { + writer.write(record); + } + } catch (MalformedRecordException | IOException | SchemaNotFoundException e) { + parseFailures.add(msg); + } + }); + + WriteResult result = writer.finishRecordSet(); + IOUtils.closeQuietly(writer); + IOUtils.closeQuietly(rawOut); + + if (result != WriteResult.EMPTY) { + session.putAllAttributes(flowFile, result.getAttributes()); + session.putAttribute(flowFile, MSG_COUNT, result.getRecordCount() + ""); + session.getProvenanceReporter().receive(flowFile, getPulsarClientService().getPulsarBrokerRootURL() + "/" + consumer.getTopic()); + session.transfer(flowFile, REL_SUCCESS); + } else { + // We were able to parse the records, but unable to write them to the FlowFile + session.rollback(); + } + + } catch (IOException e) { + getLogger().error("Unable to consume from Pulsar topic ", e); + } + } + + handleFailures(session, parseFailures, demarcator); + consumer.acknowledgeCumulative(messages.get(messages.size()-1)); + } + + private void handleFailures(ProcessSession session, BlockingQueue> parseFailures, byte[] demarcator) { + + if (CollectionUtils.isEmpty(parseFailures)) { + return; + } + + FlowFile flowFile = session.create(); + OutputStream rawOut = session.write(flowFile); + + try { + for (int idx = 0; idx < parseFailures.size(); idx++) { + Message msg = parseFailures.poll(0, TimeUnit.MILLISECONDS); + if (msg.getValue() != null && msg.getValue().length > 0) { + rawOut.write(msg.getValue()); + if (idx < parseFailures.size() - 2) { + rawOut.write(demarcator); + } + } + } + IOUtils.closeQuietly(rawOut); + session.transfer(flowFile, REL_PARSE_FAILURE); + } catch (IOException | InterruptedException e) { + getLogger().error("Unable to route failures", e); + } + } + + /** + * Pull messages off of the CompletableFuture's held in the consumerService and process them in a batch. + * @param demarcator - The bytes used to demarcate the individual messages. + */ + protected void handleAsync(ProcessContext context, ProcessSession session, final Consumer consumer, + final RecordReaderFactory readerFactory, RecordSetWriterFactory writerFactory, byte[] demarcator) throws PulsarClientException { + + final Integer queryTimeout = context.getProperty(MAX_WAIT_TIME).evaluateAttributeExpressions().asTimePeriod(TimeUnit.SECONDS).intValue(); + + try { + Future>> done = null; + do { + done = getConsumerService().poll(queryTimeout, TimeUnit.SECONDS); + + if (done != null) { + List> messages = done.get(); + if (CollectionUtils.isNotEmpty(messages)) { + consumeMessages(session, consumer, messages, readerFactory, writerFactory, demarcator); + } + } + } while (done != null); + + } catch (InterruptedException | ExecutionException e) { + getLogger().error("Trouble consuming messages ", e); + } + } + + private RecordSchema getSchema(RecordReaderFactory readerFactory, Message msg) { + RecordSchema schema = null; + InputStream in = null; + + try { + in = new ByteArrayInputStream(msg.getValue()); + schema = readerFactory.createRecordReader(Collections.emptyMap(), in, getLogger()).getSchema(); + } catch (MalformedRecordException | IOException | SchemaNotFoundException e) { + return null; + } finally { + IOUtils.closeQuietly(in); + } + + return schema; + } + + private RecordSetWriter getRecordWriter(RecordSetWriterFactory writerFactory, RecordSchema srcSchema, OutputStream out) { + try { + RecordSchema writeSchema = writerFactory.getSchema(Collections.emptyMap(), srcSchema); + return writerFactory.createWriter(getLogger(), writeSchema, out); + } catch (SchemaNotFoundException | IOException e) { + return null; + } + } +} \ No newline at end of file diff --git a/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/PublishPulsar.java b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/PublishPulsar.java new file mode 100644 index 0000000..8735802 --- /dev/null +++ b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/PublishPulsar.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.TriggerWhenEmpty; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.SeeAlso; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processors.pulsar.AbstractPulsarProducerProcessor; +import org.apache.nifi.stream.io.util.StreamDemarcator; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; + +@SeeAlso({ConsumePulsar.class, ConsumePulsarRecord.class, PublishPulsarRecord.class}) +@Tags({"Apache", "Pulsar", "Put", "Send", "Message", "PubSub"}) +@CapabilityDescription("Sends the contents of a FlowFile as a message to Apache Pulsar using the Pulsar Producer API." + + "The messages to send may be individual FlowFiles or may be delimited, using a user-specified delimiter, such as " + + "a new-line. The complementary NiFi processor for fetching messages is ConsumePulsar.") +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@WritesAttribute(attribute = "msg.count", description = "The number of messages that were sent to Pulsar for this FlowFile. This attribute is added only to " + + "This attribute is added only to FlowFiles that are routed to success.") +@TriggerWhenEmpty +public class PublishPulsar extends AbstractPulsarProducerProcessor { + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + + handleFailures(session); + + final FlowFile flowFile = session.get(); + if (flowFile == null) { + return; + } + + final String topic = context.getProperty(TOPIC).evaluateAttributeExpressions(flowFile).getValue(); + final Producer producer = getProducer(context, topic); + + /* If we are unable to create a producer, then we know we won't be able + * to send the message successfully, so go ahead and route to failure now. + */ + if (producer == null) { + getLogger().error("Unable to publish to topic {}", new Object[] {topic}); + session.transfer(flowFile, REL_FAILURE); + + if (context.getProperty(ASYNC_ENABLED).asBoolean()) { + // If we are running in asynchronous mode, then slow down the processor to prevent data loss + context.yield(); + } + return; + } + + final byte[] demarcatorBytes = context.getProperty(MESSAGE_DEMARCATOR).isSet() ? context.getProperty(MESSAGE_DEMARCATOR) + .evaluateAttributeExpressions(flowFile).getValue().getBytes(StandardCharsets.UTF_8) : null; + + if (!context.getProperty(ASYNC_ENABLED).asBoolean()) { + try { + send(producer, session, flowFile, demarcatorBytes); + } catch (final PulsarClientException e) { + getLogger().error("Failed to connect to Pulsar Server due to {}", new Object[]{e}); + session.transfer(flowFile, REL_FAILURE); + } + } else if (canPublish.get()) { + byte[] messageContent; + + try (final InputStream in = session.read(flowFile); + final StreamDemarcator demarcator = new StreamDemarcator(in, demarcatorBytes, Integer.MAX_VALUE)) { + while ((messageContent = demarcator.nextToken()) != null) { + workQueue.put(Pair.of(topic, messageContent)); + } + demarcator.close(); + session.transfer(flowFile, REL_SUCCESS); + } catch (Throwable t) { + getLogger().error("Unable to process session due to ", t); + session.transfer(flowFile, REL_FAILURE); + } + } + } + + /** + * Sends the FlowFile content using the demarcator. + */ + private void send(Producer producer, ProcessSession session, FlowFile flowFile, byte[] demarcatorBytes) throws PulsarClientException { + AtomicInteger successCounter = new AtomicInteger(0); + AtomicInteger failureCounter = new AtomicInteger(0); + byte[] messageContent; + + try (final InputStream in = session.read(flowFile); final StreamDemarcator demarcator = new StreamDemarcator(in, demarcatorBytes, Integer.MAX_VALUE)) { + while ((messageContent = demarcator.nextToken()) != null) { + if (producer.send(messageContent) != null) { + successCounter.incrementAndGet(); + } else { + failureCounter.incrementAndGet(); + break; // Quit sending messages if we encounter a failure. + } + } + } catch (final IOException ioEx) { + getLogger().error("Unable to publish message to Pulsar broker " + getPulsarClientService().getPulsarBrokerRootURL(), ioEx); + session.transfer(flowFile, REL_FAILURE); + return; + } + + /* + * Record the number of messages that were sent to Apache Pulsar. + */ + if (successCounter.intValue() > 0) { + session.adjustCounter("Messages Sent", successCounter.get(), true); + session.getProvenanceReporter().send(flowFile, getPulsarClientService().getPulsarBrokerRootURL() + "/" + producer.getTopic(), + "Sent " + successCounter.get() + " messages"); + } + + /* If we had any failures then route the entire FlowFile to Failure. + * The user will have to take care when re-trying this message to avoid + * sending duplicate messages. + */ + if (failureCounter.intValue() == 0) { + session.transfer(flowFile, REL_SUCCESS); + } else { + session.transfer(flowFile, REL_FAILURE); + } + } +} diff --git a/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/PublishPulsarRecord.java b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/PublishPulsarRecord.java new file mode 100644 index 0000000..f209b3f --- /dev/null +++ b/nifi-pulsar-processors/src/main/java/org/apache/nifi/processors/pulsar/pubsub/PublishPulsarRecord.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.TriggerWhenEmpty; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.SeeAlso; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processors.pulsar.AbstractPulsarProducerProcessor; +import org.apache.nifi.schema.access.SchemaNotFoundException; +import org.apache.nifi.serialization.MalformedRecordException; +import org.apache.nifi.serialization.RecordReader; +import org.apache.nifi.serialization.RecordReaderFactory; +import org.apache.nifi.serialization.RecordSetWriter; +import org.apache.nifi.serialization.RecordSetWriterFactory; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.nifi.serialization.record.RecordSet; +import org.apache.pulsar.client.api.Producer; + +@Tags({"Apache", "Pulsar", "Record", "csv", "json", "avro", "logs", "Put", "Send", "Message", "PubSub", "1.0"}) +@CapabilityDescription("Sends the contents of a FlowFile as individual records to Apache Pulsar using the Pulsar 1.x client API. " + + "The contents of the FlowFile are expected to be record-oriented data that can be read by the configured Record Reader. " + + "The complementary NiFi processor for fetching messages is ConsumePulsarRecord.") +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@WritesAttribute(attribute = "msg.count", description = "The number of messages that were sent to Pulsar for this FlowFile. This attribute is added only to " + + "FlowFiles that are routed to success.") +@SeeAlso({PublishPulsar.class, ConsumePulsar.class, ConsumePulsarRecord.class}) +@TriggerWhenEmpty +public class PublishPulsarRecord extends AbstractPulsarProducerProcessor { + + public static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder() + .name("RECORD_READER") + .displayName("Record Reader") + .description("The Record Reader to use for incoming FlowFiles") + .identifiesControllerService(RecordReaderFactory.class) + .required(true) + .build(); + + public static final PropertyDescriptor RECORD_WRITER = new PropertyDescriptor.Builder() + .name("RECORD_WRITER") + .displayName("Record Writer") + .description("The Record Writer to use in order to serialize the data before sending to Pulsar") + .identifiesControllerService(RecordSetWriterFactory.class) + .required(true) + .build(); + + private static final List PROPERTIES; + + static { + final List properties = new ArrayList<>(); + properties.add(RECORD_READER); + properties.add(RECORD_WRITER); + properties.addAll(AbstractPulsarProducerProcessor.PROPERTIES); + PROPERTIES = Collections.unmodifiableList(properties); + } + + @Override + protected List getSupportedPropertyDescriptors() { + return PROPERTIES; + } + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + + handleFailures(session); + + final FlowFile flowFile = session.get(); + if (flowFile == null) { + return; + } + + final String topic = context.getProperty(TOPIC).evaluateAttributeExpressions(flowFile).getValue(); + final Producer producer = getProducer(context, topic); + + /* If we are unable to create a producer, then we know we won't be able + * to send the message successfully, so go ahead and route to failure now. + */ + if (producer == null) { + getLogger().error("Unable to publish to topic {}", new Object[] {topic}); + session.transfer(flowFile, REL_FAILURE); + + if (context.getProperty(ASYNC_ENABLED).asBoolean()) { + // If we are running in asynchronous mode, then slow down the processor to prevent data loss + context.yield(); + } + return; + } + + final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER) + .asControllerService(RecordReaderFactory.class); + + final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER) + .asControllerService(RecordSetWriterFactory.class); + + final Map attributes = flowFile.getAttributes(); + final AtomicLong messagesSent = new AtomicLong(0L); + final InputStream in = session.read(flowFile); + + try { + final RecordReader reader = readerFactory.createRecordReader(attributes, in, getLogger()); + final RecordSet recordSet = reader.createRecordSet(); + final RecordSchema schema = writerFactory.getSchema(attributes, recordSet.getSchema()); + final boolean asyncFlag = (context.getProperty(ASYNC_ENABLED).isSet() && context.getProperty(ASYNC_ENABLED).asBoolean()); + + try { + messagesSent.addAndGet(send(producer, writerFactory, schema, reader, topic, asyncFlag)); + session.putAttribute(flowFile, MSG_COUNT, messagesSent.get() + ""); + session.putAttribute(flowFile, TOPIC_NAME, topic); + session.adjustCounter("Messages Sent", messagesSent.get(), true); + session.getProvenanceReporter().send(flowFile, getPulsarClientService().getPulsarBrokerRootURL(), "Sent " + messagesSent.get() + " records"); + session.transfer(flowFile, REL_SUCCESS); + } catch (InterruptedException e) { + session.transfer(flowFile, REL_FAILURE); + } + + } catch (final SchemaNotFoundException | MalformedRecordException | IOException e) { + session.transfer(flowFile, REL_FAILURE); + } finally { + try { + in.close(); + } catch (final IOException ioEx) { /* Ignore */ } + } + } + + private int send(final Producer producer, final RecordSetWriterFactory writerFactory, final RecordSchema schema, final RecordReader reader, + String topic, boolean asyncFlag) throws IOException, SchemaNotFoundException, InterruptedException { + + final RecordSet recordSet = reader.createRecordSet(); + final ByteArrayOutputStream baos = new ByteArrayOutputStream(1024); + Record record; + int recordCount = 0; + + try { + while ((record = recordSet.next()) != null) { + recordCount++; + baos.reset(); + + try (final RecordSetWriter writer = writerFactory.createWriter(getLogger(), schema, baos)) { + writer.write(record); + writer.flush(); + } + if (asyncFlag) { + workQueue.put(Pair.of(topic, baos.toByteArray())); + } else { + producer.send(baos.toByteArray()); + } + } + return recordCount; + } finally { + reader.close(); + } + } +} diff --git a/nifi-pulsar-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-pulsar-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor new file mode 100644 index 0000000..ebde2cb --- /dev/null +++ b/nifi-pulsar-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.processors.pulsar.pubsub.ConsumePulsar +org.apache.nifi.processors.pulsar.pubsub.PublishPulsar +org.apache.nifi.processors.pulsar.pubsub.ConsumePulsarRecord +org.apache.nifi.processors.pulsar.pubsub.PublishPulsarRecord \ No newline at end of file diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/AbstractPulsarProcessorTest.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/AbstractPulsarProcessorTest.java new file mode 100644 index 0000000..b023f20 --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/AbstractPulsarProcessorTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar; + +import org.apache.nifi.processors.pulsar.pubsub.mocks.MockPulsarClientService; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.TestRunner; +import org.junit.After; + +public abstract class AbstractPulsarProcessorTest { + + protected TestRunner runner; + + protected MockPulsarClientService mockClientService; + + protected void addPulsarClientService() throws InitializationException { + mockClientService = new MockPulsarClientService(); + runner.addControllerService("Pulsar Client Service", mockClientService); + runner.enableControllerService(mockClientService); + runner.setProperty(AbstractPulsarConsumerProcessor.PULSAR_CLIENT_SERVICE, "Pulsar Client Service"); + } + + @After + public final void validate() { + org.mockito.Mockito.validateMockitoUsage(); + } +} diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestConsumePulsar.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestConsumePulsar.java new file mode 100644 index 0000000..daea5ba --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestConsumePulsar.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub; + +import org.apache.nifi.processors.pulsar.AbstractPulsarProcessorTest; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunners; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import static org.junit.Assert.assertEquals; + +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class TestConsumePulsar extends AbstractPulsarProcessorTest { + + @Mock + protected Message mockMessage; + + @Rule public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Before + public void init() throws InitializationException { + runner = TestRunners.newTestRunner(ConsumePulsar.class); + mockMessage = mock(Message.class); + addPulsarClientService(); + } + + @Test + public void singleSyncMessageTest() throws PulsarClientException { + this.sendMessages("Mocked Message", "foo", "bar", false, 1); + } + + @Test + public void multipleSyncMessagesTest() throws PulsarClientException { + this.batchMessages("Mocked Message", "foo", "bar", false, 40); + } + + @Test + public void singleAsyncMessageTest() throws PulsarClientException { + this.sendMessages("Mocked Message", "foo", "bar", true, 1); + } + + @Test + public void multipleAsyncMessagesTest() throws PulsarClientException { + this.sendMessages("Mocked Message", "foo", "bar", true, 40); + } + + /* + * Verify that the consumer gets closed. + */ + @Test + public void onStoppedTest() throws NoSuchMethodException, SecurityException, PulsarClientException { + when(mockMessage.getValue()).thenReturn("Mocked Message".getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsar.TOPICS, "foo"); + runner.setProperty(ConsumePulsar.SUBSCRIPTION_NAME, "bar"); + runner.run(10, true); + runner.assertAllFlowFilesTransferred(ConsumePulsar.REL_SUCCESS); + + runner.assertQueueEmpty(); + + // Verify that the receive method on the consumer was called 10 times + int batchSize = Integer.parseInt(ConsumePulsar.CONSUMER_BATCH_SIZE.getDefaultValue()); + verify(mockClientService.getMockConsumer(), atLeast(10 * batchSize)).receive(0, TimeUnit.SECONDS); + + // Verify that each message was acknowledged + verify(mockClientService.getMockConsumer(), times(10)).acknowledgeCumulative(mockMessage); + + // Verify that the consumer was closed + verify(mockClientService.getMockConsumer(), times(1)).close(); + + } + + protected void batchMessages(String msg, String topic, String sub, boolean async, int batchSize) throws PulsarClientException { + when(mockMessage.getValue()).thenReturn(msg.getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsar.ASYNC_ENABLED, Boolean.toString(async)); + runner.setProperty(ConsumePulsar.TOPICS, topic); + runner.setProperty(ConsumePulsar.SUBSCRIPTION_NAME, sub); + runner.setProperty(ConsumePulsar.CONSUMER_BATCH_SIZE, batchSize + ""); + runner.setProperty(ConsumePulsar.MESSAGE_DEMARCATOR, "\n"); + runner.run(1, true); + + runner.assertAllFlowFilesTransferred(ConsumePulsar.REL_SUCCESS); + + List flowFiles = runner.getFlowFilesForRelationship(ConsumePulsar.REL_SUCCESS); + assertEquals(1, flowFiles.size()); + + flowFiles.get(0).assertAttributeEquals(ConsumePulsar.MSG_COUNT, batchSize + ""); + + StringBuffer sb = new StringBuffer(); + for (int idx = 0; idx < batchSize; idx++) { + sb.append(msg); + sb.append("\n"); + } + + flowFiles.get(0).assertContentEquals(sb.toString()); + + // Verify that every message was acknowledged + if (async) { + verify(mockClientService.getMockConsumer(), times(batchSize)).receive(); + verify(mockClientService.getMockConsumer(), times(batchSize)).acknowledgeAsync(mockMessage); + } else { + verify(mockClientService.getMockConsumer(), times(batchSize + 1)).receive(0, TimeUnit.SECONDS); + verify(mockClientService.getMockConsumer(), times(1)).acknowledgeCumulative(mockMessage); + } + } + + protected void sendMessages(String msg, String topic, String sub, boolean async, int iterations) throws PulsarClientException { + + when(mockMessage.getValue()).thenReturn(msg.getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsar.ASYNC_ENABLED, Boolean.toString(async)); + runner.setProperty(ConsumePulsar.TOPICS, topic); + runner.setProperty(ConsumePulsar.SUBSCRIPTION_NAME, sub); + runner.setProperty(ConsumePulsar.CONSUMER_BATCH_SIZE, 1 + ""); + runner.run(iterations, true); + + runner.assertAllFlowFilesTransferred(ConsumePulsar.REL_SUCCESS); + + List flowFiles = runner.getFlowFilesForRelationship(ConsumePulsar.REL_SUCCESS); + assertEquals(iterations, flowFiles.size()); + + for (MockFlowFile ff : flowFiles) { + ff.assertContentEquals(msg + ConsumePulsar.MESSAGE_DEMARCATOR.getDefaultValue()); + } + + verify(mockClientService.getMockConsumer(), times(iterations * 2)).receive(0, TimeUnit.SECONDS); + + // Verify that every message was acknowledged + if (async) { + verify(mockClientService.getMockConsumer(), times(iterations)).acknowledgeCumulativeAsync(mockMessage); + } else { + verify(mockClientService.getMockConsumer(), times(iterations)).acknowledgeCumulative(mockMessage); + } + } +} diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestConsumePulsarRecord.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestConsumePulsarRecord.java new file mode 100644 index 0000000..6bcf5fd --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestConsumePulsarRecord.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub; + +import static org.apache.nifi.processors.pulsar.pubsub.ConsumePulsarRecord.RECORD_READER; +import static org.apache.nifi.processors.pulsar.pubsub.ConsumePulsarRecord.RECORD_WRITER; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.nifi.processors.pulsar.AbstractPulsarConsumerProcessor; +import org.apache.nifi.processors.pulsar.AbstractPulsarProcessorTest; +import org.apache.nifi.processors.pulsar.pubsub.mocks.MockRecordParser; +import org.apache.nifi.processors.pulsar.pubsub.mocks.MockRecordWriter; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.serialization.RecordSetWriterFactory; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunners; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +public class TestConsumePulsarRecord extends AbstractPulsarProcessorTest { + + protected static String BAD_MSG = "Malformed message"; + protected static String MOCKED_MSG = "Mocked Message, 1"; + protected static String DEFAULT_TOPIC = "foo"; + protected static String DEFAULT_SUB = "bar"; + + @Mock + protected Message mockMessage; + + @Before + public void setup() throws InitializationException { + + mockMessage = mock(Message.class); + + runner = TestRunners.newTestRunner(ConsumePulsarRecord.class); + + final String readerId = "record-reader"; + final MockRecordParser readerService = new MockRecordParser(); + readerService.addSchemaField("name", RecordFieldType.STRING); + readerService.addSchemaField("age", RecordFieldType.INT); + runner.addControllerService(readerId, readerService); + runner.enableControllerService(readerService); + + final String writerId = "record-writer"; + final RecordSetWriterFactory writerService = new MockRecordWriter("name, age"); + runner.addControllerService(writerId, writerService); + runner.enableControllerService(writerService); + + runner.setProperty(RECORD_READER, readerId); + runner.setProperty(RECORD_WRITER, writerId); + addPulsarClientService(); + } + + @Test + public void validatePropertiesValidation() throws Exception { + // Initially the processor won't be properly configured + runner.assertNotValid(); + + runner.setProperty(AbstractPulsarConsumerProcessor.TOPICS, "my-topic"); + runner.setProperty(AbstractPulsarConsumerProcessor.SUBSCRIPTION_NAME, "my-sub"); + runner.assertValid(); + } + + protected List sendMessages(String msg, boolean async, int iterations) throws PulsarClientException { + return sendMessages(msg, DEFAULT_TOPIC, DEFAULT_SUB, async, iterations, 1); + } + + protected List sendMessages(String msg, boolean async, int iterations, int batchSize) throws PulsarClientException { + return sendMessages(msg, DEFAULT_TOPIC, DEFAULT_SUB, async, iterations, batchSize); + } + + protected List sendMessages(String msg, String topic, String sub, boolean async, int iterations) throws PulsarClientException { + return sendMessages(msg, topic, sub, async, iterations, 1); + } + + protected List sendMessages(String msg, String topic, String sub, boolean async, int iterations, int batchSize) throws PulsarClientException { + when(mockMessage.getValue()).thenReturn(msg.getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsarRecord.ASYNC_ENABLED, Boolean.toString(async)); + runner.setProperty(ConsumePulsarRecord.TOPICS, topic); + runner.setProperty(ConsumePulsarRecord.SUBSCRIPTION_NAME, sub); + runner.setProperty(ConsumePulsarRecord.CONSUMER_BATCH_SIZE, batchSize + ""); + + if (async) { + runner.setProperty(ConsumePulsarRecord.MAX_WAIT_TIME, "5 sec"); + } else { + runner.setProperty(ConsumePulsarRecord.MAX_WAIT_TIME, "0 sec"); + } + + runner.run(iterations, true); + runner.assertAllFlowFilesTransferred(ConsumePulsarRecord.REL_SUCCESS); + + List flowFiles = runner.getFlowFilesForRelationship(ConsumePulsarRecord.REL_SUCCESS); + assertEquals(iterations, flowFiles.size()); + + verify(mockClientService.getMockConsumer(), times(iterations * (batchSize+1))).receive(0, TimeUnit.SECONDS); + verify(mockClientService.getMockConsumer(), times(iterations)).acknowledgeCumulative(mockMessage); + + return flowFiles; + } +} diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestPublishPulsar.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestPublishPulsar.java new file mode 100644 index 0000000..c90a4b8 --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestPublishPulsar.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub; + +import org.apache.nifi.processors.pulsar.AbstractPulsarProcessorTest; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.TestRunners; + +import org.apache.pulsar.client.api.Producer; + +import org.junit.Before; +import org.mockito.Mock; +import static org.mockito.Mockito.mock; + +public class TestPublishPulsar extends AbstractPulsarProcessorTest { + + @Mock + protected Producer mockProducer; + + @Before + public void init() throws InitializationException { + mockProducer = mock(Producer.class); + runner = TestRunners.newTestRunner(PublishPulsar.class); + addPulsarClientService(); + } +} diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestPublishPulsarRecord.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestPublishPulsarRecord.java new file mode 100644 index 0000000..e645f8a --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/TestPublishPulsarRecord.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub; + +import static org.apache.nifi.processors.pulsar.pubsub.PublishPulsarRecord.RECORD_READER; +import static org.apache.nifi.processors.pulsar.pubsub.PublishPulsarRecord.RECORD_WRITER; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.nifi.processors.pulsar.AbstractPulsarProcessorTest; +import org.apache.nifi.processors.pulsar.AbstractPulsarProducerProcessor; +import org.apache.nifi.processors.pulsar.pubsub.mocks.MockRecordParser; +import org.apache.nifi.processors.pulsar.pubsub.mocks.MockRecordWriter; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.serialization.RecordSetWriterFactory; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.util.TestRunners; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Before; +import org.junit.Test; + +public class TestPublishPulsarRecord extends AbstractPulsarProcessorTest { + + protected static final String TOPIC_NAME = "unit-test"; + + @Before + public void setup() throws InitializationException { + + runner = TestRunners.newTestRunner(PublishPulsarRecord.class); + + final String readerId = "record-reader"; + final MockRecordParser readerService = new MockRecordParser(); + readerService.addSchemaField("name", RecordFieldType.STRING); + readerService.addSchemaField("age", RecordFieldType.INT); + runner.addControllerService(readerId, readerService); + runner.enableControllerService(readerService); + + final String writerId = "record-writer"; + final RecordSetWriterFactory writerService = new MockRecordWriter("name, age"); + runner.addControllerService(writerId, writerService); + runner.enableControllerService(writerService); + + runner.setProperty(RECORD_READER, readerId); + runner.setProperty(RECORD_WRITER, writerId); + + addPulsarClientService(); + } + + @Test + public void propertyValidationTest() throws Exception { + // Initially the processor won't be properly configured + runner.assertNotValid(); + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, TOPIC_NAME); + runner.assertValid(); + } + + @Test + public void invalidTopicTest() throws UnsupportedEncodingException, PulsarClientException { + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, "${topic}"); + + final String content = "some content"; + Map attributes = new HashMap(); + attributes.put(AbstractPulsarProducerProcessor.TOPIC.getName(), ""); + + runner.enqueue(content.getBytes("UTF-8"), attributes ); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsarRecord.REL_FAILURE); + + // Confirm that no Producer as created + verify(mockClientService.getMockProducerBuilder(), times(0)).topic(anyString()); + } + + @Test + public void dynamicTopicTest() throws UnsupportedEncodingException, PulsarClientException { + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, "${topic}"); + + final String content = "Mary Jane, 32"; + Map attributes = new HashMap(); + attributes.put("topic", TOPIC_NAME); + + runner.enqueue(content.getBytes("UTF-8"), attributes ); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsarRecord.REL_SUCCESS); + + // Verify that we sent the data to topic-b. + verify(mockClientService.getMockProducerBuilder(), times(1)).topic(TOPIC_NAME); + } +} diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncConsumePulsar.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncConsumePulsar.java new file mode 100644 index 0000000..56f0ffc --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncConsumePulsar.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub.async; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; + +import org.apache.nifi.processors.pulsar.pubsub.ConsumePulsar; +import org.apache.nifi.processors.pulsar.pubsub.TestConsumePulsar; +import org.apache.nifi.util.MockFlowFile; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Test; + +public class TestAsyncConsumePulsar extends TestConsumePulsar { + + @Test + public void pulsarClientExceptionTest() throws PulsarClientException { + when(mockClientService.getMockConsumer().receive()).thenThrow(PulsarClientException.class); + + runner.setProperty(ConsumePulsar.TOPICS, "foo"); + runner.setProperty(ConsumePulsar.SUBSCRIPTION_NAME, "bar"); + runner.setProperty(ConsumePulsar.ASYNC_ENABLED, Boolean.toString(true)); + runner.run(); + runner.assertAllFlowFilesTransferred(ConsumePulsar.REL_SUCCESS); + + // Make sure no Flowfiles were generated + List flowFiles = runner.getFlowFilesForRelationship(ConsumePulsar.REL_SUCCESS); + assertEquals(0, flowFiles.size()); + + verify(mockClientService.getMockConsumer(), times(0)).acknowledge(mockMessage); + } + + @Test + public void emptyMessageTest() { + when(mockMessage.getValue()).thenReturn("".getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsar.TOPICS, "foo"); + runner.setProperty(ConsumePulsar.SUBSCRIPTION_NAME, "bar"); + runner.setProperty(ConsumePulsar.ASYNC_ENABLED, Boolean.toString(true)); + runner.run(); + runner.assertAllFlowFilesTransferred(ConsumePulsar.REL_SUCCESS); + + // Make sure no Flowfiles were generated + List flowFiles = runner.getFlowFilesForRelationship(ConsumePulsar.REL_SUCCESS); + assertEquals(1, flowFiles.size()); + } + + @Test + public void singleMessageTest() throws PulsarClientException { + this.sendMessages("Mocked Message", "foo", "bar", true, 1); + } + + @Test + public void multipleMessagesTest() throws PulsarClientException { + this.sendMessages("Mocked Message", "foo", "bar", true, 40); + } + + /* + * Verify that the consumer gets closed. + */ + @Test + public void onStoppedTest() throws NoSuchMethodException, SecurityException, PulsarClientException { + when(mockMessage.getValue()).thenReturn("Mocked Message".getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsar.TOPICS, "foo"); + runner.setProperty(ConsumePulsar.SUBSCRIPTION_NAME, "bar"); + runner.run(10, true); + runner.assertAllFlowFilesTransferred(ConsumePulsar.REL_SUCCESS); + + runner.assertQueueEmpty(); + + // Verify that the consumer was closed + verify(mockClientService.getMockConsumer(), times(1)).close(); + } +} diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncConsumePulsarRecord.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncConsumePulsarRecord.java new file mode 100644 index 0000000..6aec89f --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncConsumePulsarRecord.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub.async; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; + +import org.apache.nifi.processors.pulsar.pubsub.ConsumePulsarRecord; +import org.apache.nifi.processors.pulsar.pubsub.TestConsumePulsarRecord; +import org.apache.nifi.util.MockFlowFile; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Test; + +public class TestAsyncConsumePulsarRecord extends TestConsumePulsarRecord { + + @Test + public void emptyMessageTest() throws PulsarClientException { + when(mockMessage.getValue()).thenReturn("".getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsarRecord.TOPICS, DEFAULT_TOPIC); + runner.setProperty(ConsumePulsarRecord.SUBSCRIPTION_NAME, DEFAULT_SUB); + runner.setProperty(ConsumePulsarRecord.CONSUMER_BATCH_SIZE, 1 + ""); + runner.setProperty(ConsumePulsarRecord.ASYNC_ENABLED, Boolean.toString(true)); + runner.run(); + runner.assertAllFlowFilesTransferred(ConsumePulsarRecord.REL_PARSE_FAILURE); + + verify(mockClientService.getMockConsumer(), times(1)).acknowledgeCumulative(mockMessage); + } + + @Test + public void singleMalformedMessageTest() throws PulsarClientException { + when(mockMessage.getValue()).thenReturn(BAD_MSG.getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsarRecord.TOPICS, DEFAULT_TOPIC); + runner.setProperty(ConsumePulsarRecord.SUBSCRIPTION_NAME, DEFAULT_SUB); + runner.setProperty(ConsumePulsarRecord.CONSUMER_BATCH_SIZE, 1 + ""); + runner.setProperty(ConsumePulsarRecord.ASYNC_ENABLED, Boolean.toString(true)); + runner.run(); + runner.assertAllFlowFilesTransferred(ConsumePulsarRecord.REL_PARSE_FAILURE); + + verify(mockClientService.getMockConsumer(), times(1)).acknowledgeCumulative(mockMessage); + } + + /* + * Send a single message containing a single record + */ + @Test + public void singleMessageTest() throws PulsarClientException { + this.sendMessages(MOCKED_MSG, true, 1); + } + + /* + * Send a single message with multiple records + */ + @Test + public void singleMessageMultiRecordsTest() throws PulsarClientException { + StringBuffer input = new StringBuffer(1024); + StringBuffer expected = new StringBuffer(1024); + + for (int idx = 0; idx < 50; idx++) { + input.append("Justin Thyme, " + idx).append("\n"); + expected.append("\"Justin Thyme\",\"" + idx + "\"").append("\n"); + } + + List results = this.sendMessages(input.toString(), false, 1); + + String flowFileContents = new String(runner.getContentAsByteArray(results.get(0))); + assertEquals(expected.toString(), flowFileContents); + } + + /* + * Send a single message with multiple records, + * some of them good and some malformed + */ + @Test + public void singleMessageWithGoodAndBadRecordsTest() throws PulsarClientException { + StringBuffer input = new StringBuffer(1024); + StringBuffer expected = new StringBuffer(1024); + + for (int idx = 0; idx < 10; idx++) { + if (idx % 2 == 0) { + input.append("Justin Thyme, " + idx).append("\n"); + expected.append("\"Justin Thyme\",\"" + idx + "\"").append("\n"); + } else { + input.append(BAD_MSG).append("\n"); + } + } + + when(mockMessage.getValue()).thenReturn(input.toString().getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsarRecord.ASYNC_ENABLED, Boolean.toString(false)); + runner.setProperty(ConsumePulsarRecord.TOPICS, DEFAULT_TOPIC); + runner.setProperty(ConsumePulsarRecord.SUBSCRIPTION_NAME, DEFAULT_SUB); + runner.setProperty(ConsumePulsarRecord.CONSUMER_BATCH_SIZE, 1 + ""); + runner.run(1, true); + + List successFlowFiles = runner.getFlowFilesForRelationship(ConsumePulsarRecord.REL_SUCCESS); + assertEquals(1, successFlowFiles.size()); + + List failureFlowFiles = runner.getFlowFilesForRelationship(ConsumePulsarRecord.REL_PARSE_FAILURE); + assertEquals(1, failureFlowFiles.size()); + } + + /* + * Send multiple messages with Multiple records each + */ + @Test + public void multipleMultiRecordsTest() throws PulsarClientException { + StringBuffer input = new StringBuffer(1024); + StringBuffer expected = new StringBuffer(1024); + + for (int idx = 0; idx < 50; idx++) { + input.append("Justin Thyme, " + idx).append("\n"); + expected.append("\"Justin Thyme\",\"" + idx + "\"").append("\n"); + } + + List results = this.sendMessages(input.toString(), false, 50, 100); + assertEquals(50, results.size()); + + String flowFileContents = new String(runner.getContentAsByteArray(results.get(0))); + assertTrue(flowFileContents.startsWith(expected.toString(), 0)); + } +} \ No newline at end of file diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncPublishPulsar.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncPublishPulsar.java new file mode 100644 index 0000000..15dd031 --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncPublishPulsar.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub.async; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.nifi.processors.pulsar.pubsub.PublishPulsar; +import org.apache.nifi.processors.pulsar.pubsub.PublishPulsarRecord; +import org.apache.nifi.processors.pulsar.pubsub.TestPublishPulsar; +import org.apache.nifi.util.MockFlowFile; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Test; + +public class TestAsyncPublishPulsar extends TestPublishPulsar { + + @Test + public void singleFlowFileTest() throws UnsupportedEncodingException, PulsarClientException, InterruptedException { + when(mockClientService.getMockProducer().getTopic()).thenReturn("my-topic"); + + runner.setProperty(PublishPulsar.TOPIC, "my-topic"); + runner.setProperty(PublishPulsar.ASYNC_ENABLED, Boolean.TRUE.toString()); + + final String content = "some content"; + runner.enqueue(content.getBytes("UTF-8")); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsar.REL_SUCCESS); + + // Verify that we sent the data to my-topic. + verify(mockClientService.getMockProducerBuilder(), times(1)).topic("my-topic"); + + // Verify that the send method on the producer was called with the expected content + verify(mockClientService.getMockProducer(), times(1)).sendAsync(content.getBytes()); + } + + @Test + public void demarcatedFlowFileTest() throws UnsupportedEncodingException, PulsarClientException { + final String content = "some content"; + final String demarcator = "\n"; + when(mockClientService.getMockProducer().getTopic()).thenReturn("my-topic"); + + runner.setProperty(PublishPulsar.TOPIC, "my-topic"); + runner.setProperty(PublishPulsar.MESSAGE_DEMARCATOR, demarcator); + runner.setProperty(PublishPulsar.ASYNC_ENABLED, Boolean.TRUE.toString()); + + final StringBuffer sb = new StringBuffer(); + + for (int idx = 0; idx < 20; idx++) { + sb.append(content).append(demarcator); + } + + runner.enqueue(sb.toString().getBytes("UTF-8")); + runner.run(10, true, true); + runner.assertAllFlowFilesTransferred(PublishPulsar.REL_SUCCESS); + verify(mockClientService.getMockProducer(), times(20)).sendAsync(content.getBytes()); + } + + @Test + public void pulsarClientExceptionTest() throws UnsupportedEncodingException { + + when(mockClientService.getMockProducer().sendAsync(any(byte[].class))).thenThrow(PulsarClientException.class); + when(mockClientService.getMockProducer().getTopic()).thenReturn("my-topic"); + + runner.setProperty(PublishPulsar.TOPIC, "my-topic"); + runner.setProperty(PublishPulsar.ASYNC_ENABLED, Boolean.TRUE.toString()); + runner.addConnection(PublishPulsarRecord.REL_FAILURE); + + final String content = "some content"; + runner.enqueue(content.getBytes("UTF-8")); + runner.run(5000, false, true); + List success = runner.getFlowFilesForRelationship("success"); + List failures = runner.getFlowFilesForRelationship("failure"); + + assertNotNull(success); + assertEquals(1, success.size()); + + assertNotNull(failures); + assertEquals(1, failures.size()); + } + + @Test + public void multipleFlowFilesTest() throws UnsupportedEncodingException, PulsarClientException { + when(mockClientService.getMockProducer().getTopic()).thenReturn("my-async-topic"); + + runner.setProperty(PublishPulsar.TOPIC, "my-async-topic"); + runner.setProperty(PublishPulsar.ASYNC_ENABLED, Boolean.TRUE.toString()); + + final String content = "some content"; + + for (int idx = 0; idx < 20; idx++) { + runner.enqueue(content.getBytes("UTF-8")); + } + + runner.run(20, true, true); + runner.assertAllFlowFilesTransferred(PublishPulsar.REL_SUCCESS); + + // Verify that the send method on the producer was called with the expected content + verify(mockClientService.getMockProducer(), times(20)).sendAsync(content.getBytes()); + } + + @Test + public final void multipleTopicsTest() throws UnsupportedEncodingException { + when(mockProducer.getTopic()).thenReturn("topic-a").thenReturn("topic-b"); + mockClientService.setMockProducer(mockProducer); + + runner.setProperty(PublishPulsar.TOPIC, "${topic}"); + + final String contentA = "topic A content"; + Map attributesA = new HashMap(); + attributesA.put("topic", "topic-a"); + runner.enqueue(contentA.getBytes("UTF-8"), attributesA); + + final String contentB = "topic B content"; + Map attributesB = new HashMap(); + attributesB.put("topic", "topic-b"); + runner.enqueue(contentB.getBytes("UTF-8"), attributesB); + + runner.run(2, true, true); + runner.assertAllFlowFilesTransferred(PublishPulsar.REL_SUCCESS); + + // Verify that we sent the data to topic-b. + verify(mockClientService.getMockProducerBuilder(), times(1)).topic("topic-a"); + verify(mockClientService.getMockProducerBuilder(), times(1)).topic("topic-b"); + } +} diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncPublishPulsarRecord.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncPublishPulsarRecord.java new file mode 100644 index 0000000..f62c805 --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/async/TestAsyncPublishPulsarRecord.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub.async; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; + +import org.apache.nifi.processors.pulsar.AbstractPulsarProducerProcessor; +import org.apache.nifi.processors.pulsar.pubsub.PublishPulsarRecord; +import org.apache.nifi.processors.pulsar.pubsub.TestPublishPulsarRecord; +import org.apache.nifi.util.MockFlowFile; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Test; +import org.mockito.ArgumentMatcher; +import org.mockito.Matchers; + +public class TestAsyncPublishPulsarRecord extends TestPublishPulsarRecord { + + @Test + public void pulsarClientExceptionTest() throws PulsarClientException { + when(mockClientService.getMockProducer().sendAsync(Matchers.argThat(new ArgumentMatcher() { + @Override + public boolean matches(Object argument) { + return true; + } + }))).thenThrow(PulsarClientException.class); + + final String content = "Mary Jane, 32"; + + runner.enqueue(content); + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, TOPIC_NAME); + runner.setProperty(AbstractPulsarProducerProcessor.ASYNC_ENABLED, Boolean.TRUE.toString()); + runner.addConnection(PublishPulsarRecord.REL_FAILURE); + /* We have to wait for the record to be processed asynchronously and eventually throw the + * exception. When the exception is caught the record is then added to the failure queue + * and another iteration of the onTrigger() method is required to 'handle' the exception properly + * by routing it to the FAILURE relationship. + * + * During a parallel build, this may take 100s of invocations of the onTrigger() method to complete + * this cycle. Therefore, we set the number of iterations below to some very large number to ensure + * that this cycle does complete on these builds + */ + runner.run(5000, false, true); + + verify(mockClientService.getMockProducer(), times(1)).sendAsync("\"Mary Jane\",\"32\"\n".getBytes()); + + List results = runner.getFlowFilesForRelationship(PublishPulsarRecord.REL_FAILURE); + assertEquals(1, results.size()); + + String flowFileContents = new String(runner.getContentAsByteArray(results.get(0))); + assertEquals("\"Mary Jane\",\"32\"\n", flowFileContents); + } + + // Malformed content test, using "some content" + @Test + public void malformedContentTest() throws PulsarClientException { + final String content = "invalid content"; + + runner.enqueue(content); + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, TOPIC_NAME); + runner.setProperty(AbstractPulsarProducerProcessor.ASYNC_ENABLED, Boolean.TRUE.toString()); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsarRecord.REL_FAILURE); + + verify(mockClientService.getMockProducer(), times(0)).send(content.getBytes()); + } + + @Test + public void testSingleRecordSuccess() throws PulsarClientException { + + final String content = "Mary Jane, 32"; + + runner.enqueue(content); + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, TOPIC_NAME); + runner.setProperty(AbstractPulsarProducerProcessor.ASYNC_ENABLED, Boolean.TRUE.toString()); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsarRecord.REL_SUCCESS); + + List results = runner.getFlowFilesForRelationship(PublishPulsarRecord.REL_SUCCESS); + MockFlowFile result = results.get(0); + + result.assertAttributeEquals(PublishPulsarRecord.MSG_COUNT, "1"); + result.assertAttributeEquals(PublishPulsarRecord.TOPIC_NAME, TOPIC_NAME); + + verify(mockClientService.getMockProducer(), times(1)).sendAsync("\"Mary Jane\",\"32\"\n".getBytes()); + } + + @Test + public void testMultipleRecordSuccess() throws PulsarClientException { + StringBuilder sb = new StringBuilder().append("Mary Jane, 32").append("\n") + .append("John Doe, 35").append("\n") + .append("Busta Move, 26").append("\n"); + + runner.enqueue(sb.toString()); + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, TOPIC_NAME); + runner.setProperty(AbstractPulsarProducerProcessor.ASYNC_ENABLED, Boolean.TRUE.toString()); + runner.run(1, true, true); + runner.assertAllFlowFilesTransferred(PublishPulsarRecord.REL_SUCCESS); + + List results = runner.getFlowFilesForRelationship(PublishPulsarRecord.REL_SUCCESS); + assertEquals(1, results.size()); + + MockFlowFile result = results.get(0); + + result.assertContentEquals(sb.toString()); + result.assertAttributeEquals(PublishPulsarRecord.MSG_COUNT, "3"); + result.assertAttributeEquals(PublishPulsarRecord.TOPIC_NAME, TOPIC_NAME); + + verify(mockClientService.getMockProducer(), times(1)).sendAsync("\"Mary Jane\",\"32\"\n".getBytes()); + verify(mockClientService.getMockProducer(), times(1)).sendAsync("\"John Doe\",\"35\"\n".getBytes()); + verify(mockClientService.getMockProducer(), times(1)).sendAsync("\"Busta Move\",\"26\"\n".getBytes()); + } + + @Test + public void testBulkRecordSuccess() throws PulsarClientException { + StringBuilder sb = new StringBuilder(); + + for (int idx = 0; idx < 98634; idx++) { + sb.append("Mary Jane, 32").append("\n"); + } + + runner.enqueue(sb.toString()); + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, TOPIC_NAME); + runner.setProperty(AbstractPulsarProducerProcessor.ASYNC_ENABLED, Boolean.TRUE.toString()); + runner.run(1, true, true); + runner.assertAllFlowFilesTransferred(PublishPulsarRecord.REL_SUCCESS); + + List results = runner.getFlowFilesForRelationship(PublishPulsarRecord.REL_SUCCESS); + assertEquals(1, results.size()); + + MockFlowFile result = results.get(0); + result.assertContentEquals(sb.toString()); + result.assertAttributeEquals(PublishPulsarRecord.MSG_COUNT, "98634"); + result.assertAttributeEquals(PublishPulsarRecord.TOPIC_NAME, TOPIC_NAME); + + verify(mockClientService.getMockProducer(), times(98634)).sendAsync("\"Mary Jane\",\"32\"\n".getBytes()); + } +} diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockPulsarClientService.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockPulsarClientService.java new file mode 100644 index 0000000..af093ac --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockPulsarClientService.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub.mocks; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.pulsar.PulsarClientService; +import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerBuilder; +import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageRoutingMode; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.mockito.ArgumentMatcher; +import org.mockito.Matchers; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doAnswer; + +public class MockPulsarClientService extends AbstractControllerService implements PulsarClientService { + + @Mock + PulsarClient mockClient = mock(PulsarClient.class); + + @Mock + ProducerBuilder mockProducerBuilder = mock(ProducerBuilder.class); + + @Mock + ConsumerBuilder mockConsumerBuilder = mock(ConsumerBuilder.class); + + @Mock + Producer mockProducer = mock(Producer.class); + + @Mock + Consumer mockConsumer = mock(Consumer.class); + + @Mock + TypedMessageBuilder mockTypedMessageBuilder = mock(TypedMessageBuilder.class); + + @Mock + protected Message mockMessage; + + @Mock + MessageId mockMessageId = mock(MessageId.class); + + CompletableFuture future; + + public MockPulsarClientService() { + when(mockClient.newProducer()).thenReturn((ProducerBuilder) mockProducerBuilder); + when(mockClient.newConsumer()).thenReturn((ConsumerBuilder) mockConsumerBuilder); + + when(mockProducerBuilder.topic(anyString())).thenReturn(mockProducerBuilder); + when(mockProducerBuilder.enableBatching(anyBoolean())).thenReturn(mockProducerBuilder); + when(mockProducerBuilder.batchingMaxMessages(anyInt())).thenReturn(mockProducerBuilder); + when(mockProducerBuilder.batchingMaxPublishDelay(anyLong(), any(TimeUnit.class))).thenReturn(mockProducerBuilder); + when(mockProducerBuilder.blockIfQueueFull(anyBoolean())).thenReturn(mockProducerBuilder); + when(mockProducerBuilder.compressionType(any(CompressionType.class))).thenReturn(mockProducerBuilder); + when(mockProducerBuilder.maxPendingMessages(anyInt())).thenReturn(mockProducerBuilder); + when(mockProducerBuilder.messageRoutingMode(any(MessageRoutingMode.class))).thenReturn(mockProducerBuilder); + + when(mockConsumerBuilder.topic(any(String[].class))).thenReturn(mockConsumerBuilder); + when(mockConsumerBuilder.topic(anyString())).thenReturn(mockConsumerBuilder); + when(mockConsumerBuilder.subscriptionName(anyString())).thenReturn(mockConsumerBuilder); + when(mockConsumerBuilder.ackTimeout(anyLong(), any(TimeUnit.class))).thenReturn(mockConsumerBuilder); + when(mockConsumerBuilder.consumerName(anyString())).thenReturn(mockConsumerBuilder); + when(mockConsumerBuilder.cryptoFailureAction(any(ConsumerCryptoFailureAction.class))).thenReturn(mockConsumerBuilder); + when(mockConsumerBuilder.priorityLevel(anyInt())).thenReturn(mockConsumerBuilder); + when(mockConsumerBuilder.receiverQueueSize(anyInt())).thenReturn(mockConsumerBuilder); + when(mockConsumerBuilder.subscriptionType(any(SubscriptionType.class))).thenReturn(mockConsumerBuilder); + + try { + when(mockConsumerBuilder.subscribe()).thenReturn(mockConsumer); + when(mockConsumer.isConnected()).thenReturn(true); + when(mockConsumer.receive()).thenReturn(mockMessage); + doAnswer(new Answer>() { + public Message answer(InvocationOnMock invocation) { + return mockMessage; + } + }).when(mockConsumer).receive(0, TimeUnit.SECONDS); + + when(mockProducerBuilder.create()).thenReturn(mockProducer); + defineDefaultProducerBehavior(); + } catch (PulsarClientException e) { + e.printStackTrace(); + } + } + + public void setMockMessage(Message msg) { + this.mockMessage = msg; + + // Configure the consumer behavior + try { + when(mockConsumer.receive()).thenReturn(mockMessage); + } catch (PulsarClientException e) { + e.printStackTrace(); + } + + CompletableFuture> future = CompletableFuture.supplyAsync(() -> { + return mockMessage; + }); + + when(mockConsumer.receiveAsync()).thenReturn(future); + } + + public Producer getMockProducer() { + return mockProducer; + } + + public void setMockProducer(Producer mockProducer) { + this.mockProducer = mockProducer; + defineDefaultProducerBehavior(); + } + + private void defineDefaultProducerBehavior() { + try { + when(mockProducer.send(Matchers.argThat(new ArgumentMatcher() { + @Override + public boolean matches(Object argument) { + return true; + } + }))).thenReturn(mockMessageId); + + future = CompletableFuture.supplyAsync(() -> { + return mock(MessageId.class); + }); + + when(mockProducer.sendAsync(Matchers.argThat(new ArgumentMatcher() { + @Override + public boolean matches(Object argument) { + return true; + } + }))).thenReturn(future); + + when(mockProducer.isConnected()).thenReturn(true); + when(mockProducer.newMessage()).thenReturn(mockTypedMessageBuilder); + when(mockTypedMessageBuilder.value((T) any(byte[].class))).thenReturn(mockTypedMessageBuilder); + when(mockTypedMessageBuilder.sendAsync()).thenReturn(future); + + } catch (PulsarClientException e) { + e.printStackTrace(); + } + } + + public Consumer getMockConsumer() { + return mockConsumer; + } + + public ProducerBuilder getMockProducerBuilder() { + return mockProducerBuilder; + } + + public ConsumerBuilder getMockConsumerBuilder() { + return mockConsumerBuilder; + } + + public TypedMessageBuilder getMockTypedMessageBuilder() { + return mockTypedMessageBuilder; + } + + @Override + public PulsarClient getPulsarClient() { + return mockClient; + } + + @Override + public String getPulsarBrokerRootURL() { + return "pulsar://mocked:6650"; + } +} \ No newline at end of file diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockRecordParser.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockRecordParser.java new file mode 100644 index 0000000..a4d0544 --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockRecordParser.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub.mocks; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.schema.access.SchemaNotFoundException; +import org.apache.nifi.serialization.MalformedRecordException; +import org.apache.nifi.serialization.RecordReader; +import org.apache.nifi.serialization.RecordReaderFactory; +import org.apache.nifi.serialization.SchemaValidationException; +import org.apache.nifi.serialization.SimpleRecordSchema; +import org.apache.nifi.serialization.record.MapRecord; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordField; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.serialization.record.RecordSchema; + +public class MockRecordParser extends AbstractControllerService implements RecordReaderFactory { + private final List records = new ArrayList<>(); + private final List fields = new ArrayList<>(); + private final int failAfterN; + + public MockRecordParser() { + this(-1); + } + + public MockRecordParser(final int failAfterN) { + this.failAfterN = failAfterN; + } + + public void addSchemaField(final String fieldName, final RecordFieldType type) { + fields.add(new RecordField(fieldName, type.getDataType())); + } + + public void addRecord(Object... values) { + records.add(values); + } + + @Override + public RecordReader createRecordReader(Map variables, InputStream in, ComponentLog logger) throws IOException, SchemaNotFoundException { + final BufferedReader reader = new BufferedReader(new InputStreamReader(in)); + + return new RecordReader() { + private int recordCount = 0; + + @Override + public void close() throws IOException { + } + + @Override + public Record nextRecord(boolean coerceTypes, boolean dropUnknown) throws IOException, MalformedRecordException, SchemaValidationException { + if (failAfterN >= recordCount) { + throw new MalformedRecordException("Intentional Unit Test Exception because " + recordCount + " records have been read"); + } + final String line = reader.readLine(); + if (line == null) { + return null; + } + + recordCount++; + + try { + final String[] values = line.split(","); + final Map valueMap = new HashMap<>(); + int i = 0; + for (final RecordField field : fields) { + final String fieldName = field.getFieldName(); + valueMap.put(fieldName, values[i++].trim()); + } + + return new MapRecord(new SimpleRecordSchema(fields), valueMap); + + } catch (final Exception ex) { + throw new MalformedRecordException(ex.getMessage()); + } + } + + @Override + public RecordSchema getSchema() { + return new SimpleRecordSchema(fields); + } + }; + } +} \ No newline at end of file diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockRecordWriter.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockRecordWriter.java new file mode 100644 index 0000000..a1aa4e0 --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/mocks/MockRecordWriter.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.pulsar.pubsub.mocks; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; +import java.util.Map; + +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.schema.access.SchemaNotFoundException; +import org.apache.nifi.serialization.RecordSetWriter; +import org.apache.nifi.serialization.RecordSetWriterFactory; +import org.apache.nifi.serialization.WriteResult; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.nifi.serialization.record.RecordSet; + +public class MockRecordWriter extends AbstractControllerService implements RecordSetWriterFactory { + private final String header; + private final int failAfterN; + private final boolean quoteValues; + + public MockRecordWriter(final String header) { + this(header, true, -1); + } + + public MockRecordWriter(final String header, final boolean quoteValues) { + this(header, quoteValues, -1); + } + + public MockRecordWriter(final String header, final boolean quoteValues, final int failAfterN) { + this.header = header; + this.quoteValues = quoteValues; + this.failAfterN = failAfterN; + } + + @Override + public RecordSchema getSchema(Map variables, RecordSchema readSchema) throws SchemaNotFoundException, IOException { + return null; + } + + @Override + public RecordSetWriter createWriter(final ComponentLog logger, final RecordSchema schema, final OutputStream out) { + return new RecordSetWriter() { + + private int recordCount = 0; + + @Override + public void flush() throws IOException { + out.flush(); + } + + @Override + public WriteResult write(final RecordSet rs) throws IOException { + out.write(header.getBytes()); + out.write("\n".getBytes()); + + final int numCols = rs.getSchema().getFieldCount(); + Record record = null; + while ((record = rs.next()) != null) { + if (++recordCount > failAfterN && failAfterN > -1) { + throw new IOException("Unit Test intentionally throwing IOException after " + failAfterN + " records were written"); + } + + int i = 0; + for (final String fieldName : record.getSchema().getFieldNames()) { + final String val = record.getAsString(fieldName); + if (quoteValues) { + out.write("\"".getBytes()); + if (val != null) { + out.write(val.getBytes()); + } + out.write("\"".getBytes()); + } else if (val != null) { + out.write(val.getBytes()); + } + + if (i++ < numCols - 1) { + out.write(",".getBytes()); + } + } + // out.write("\n".getBytes()); + } + + return WriteResult.of(recordCount, Collections.emptyMap()); + } + + @Override + public String getMimeType() { + return "text/plain"; + } + + @Override + public WriteResult write(Record record) throws IOException { + + if (++recordCount > failAfterN && failAfterN > -1) { + throw new IOException("Unit Test intentionally throwing IOException after " + failAfterN + " records were written"); + } + + int i = 0; + final int numCols = 2; + + for (final String fieldName : record.getSchema().getFieldNames()) { + final String val = record.getAsString(fieldName); + if (quoteValues) { + out.write("\"".getBytes()); + if (val != null) { + out.write(val.getBytes()); + } + out.write("\"".getBytes()); + } else if (val != null) { + out.write(val.getBytes()); + } + + if (i++ < numCols - 1) { + out.write(",".getBytes()); + } + } + out.write("\n".getBytes()); + + return WriteResult.of(1, Collections.emptyMap()); + } + + @Override + public void close() throws IOException { + } + + @Override + public void beginRecordSet() throws IOException { + } + + @Override + public WriteResult finishRecordSet() throws IOException { + return (recordCount > 0) ? WriteResult.of(1, Collections.emptyMap()) : WriteResult.EMPTY; + } + }; + } +} \ No newline at end of file diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncConsumePulsar.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncConsumePulsar.java new file mode 100644 index 0000000..df7c0c8 --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncConsumePulsar.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub.sync; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.nifi.processors.pulsar.pubsub.ConsumePulsar; +import org.apache.nifi.processors.pulsar.pubsub.TestConsumePulsar; +import org.apache.nifi.util.MockFlowFile; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Test; + +public class TestSyncConsumePulsar extends TestConsumePulsar { + + @Test + public void nullMessageTest() throws PulsarClientException { + when(mockClientService.getMockConsumer().receive(0, TimeUnit.SECONDS)).thenReturn(mockMessage).thenReturn(null); + when(mockMessage.getData()).thenReturn(null); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsar.TOPICS, "foo"); + runner.setProperty(ConsumePulsar.SUBSCRIPTION_NAME, "bar"); + runner.run(); + runner.assertAllFlowFilesTransferred(ConsumePulsar.REL_SUCCESS); + + // Make sure no Flowfiles were generated + List flowFiles = runner.getFlowFilesForRelationship(ConsumePulsar.REL_SUCCESS); + assertEquals(0, flowFiles.size()); + + verify(mockClientService.getMockConsumer(), atLeast(1)).acknowledgeCumulative(mockMessage); + } + + @Test + public void pulsarClientExceptionTest() throws PulsarClientException { + when(mockClientService.getMockConsumer().receive()).thenThrow(PulsarClientException.class); + + runner.setProperty(ConsumePulsar.TOPICS, "foo"); + runner.setProperty(ConsumePulsar.SUBSCRIPTION_NAME, "bar"); + runner.run(); + runner.assertAllFlowFilesTransferred(ConsumePulsar.REL_SUCCESS); + + // Make sure no Flowfiles were generated + List flowFiles = runner.getFlowFilesForRelationship(ConsumePulsar.REL_SUCCESS); + assertEquals(0, flowFiles.size()); + + verify(mockClientService.getMockConsumer(), times(0)).acknowledge(mockMessage); + } + + @Test + public void emptyMessageTest() throws PulsarClientException { + when(mockClientService.getMockConsumer().receive(0, TimeUnit.SECONDS)).thenReturn(mockMessage).thenReturn(null); + when(mockMessage.getData()).thenReturn("".getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsar.TOPICS, "foo"); + runner.setProperty(ConsumePulsar.SUBSCRIPTION_NAME, "bar"); + runner.run(); + runner.assertAllFlowFilesTransferred(ConsumePulsar.REL_SUCCESS); + + // Make sure no Flowfiles were generated + List flowFiles = runner.getFlowFilesForRelationship(ConsumePulsar.REL_SUCCESS); + assertEquals(0, flowFiles.size()); + + verify(mockClientService.getMockConsumer(), atLeast(1)).acknowledgeCumulative(mockMessage); + } + + @Test + public void singleMessageTest() throws PulsarClientException { + this.sendMessages("Mocked Message", "foo", "bar", false, 1); + } + + @Test + public void multipleMessagesTest() throws PulsarClientException { + this.sendMessages("Mocked Message", "foo", "bar", false, 40); + } + + @Test + public final void batchMessageTest() throws PulsarClientException { + this.batchMessages("Mocked Message", "foo", "bar", false, 400); + } + + /* + * Verify that the consumer gets closed. + */ + @Test + public void onStoppedTest() throws PulsarClientException { + when(mockMessage.getData()).thenReturn("Mocked Message".getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsar.TOPICS, "foo"); + runner.setProperty(ConsumePulsar.SUBSCRIPTION_NAME, "bar"); + runner.setProperty(ConsumePulsar.CONSUMER_BATCH_SIZE, 1 + ""); + runner.run(1, true); + runner.assertAllFlowFilesTransferred(ConsumePulsar.REL_SUCCESS); + + runner.assertQueueEmpty(); + + // Verify that the consumer was closed + verify(mockClientService.getMockConsumer(), times(1)).close(); + } +} diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncConsumePulsarRecord.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncConsumePulsarRecord.java new file mode 100644 index 0000000..304b062 --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncConsumePulsarRecord.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub.sync; + +import org.apache.nifi.processors.pulsar.pubsub.ConsumePulsarRecord; +import org.apache.nifi.processors.pulsar.pubsub.TestConsumePulsarRecord; +import org.apache.nifi.util.MockFlowFile; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; + +public class TestSyncConsumePulsarRecord extends TestConsumePulsarRecord { + + @Test + public void emptyMessageTest() throws PulsarClientException { + when(mockMessage.getValue()).thenReturn("".getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsarRecord.TOPICS, DEFAULT_TOPIC); + runner.setProperty(ConsumePulsarRecord.SUBSCRIPTION_NAME, DEFAULT_SUB); + runner.setProperty(ConsumePulsarRecord.CONSUMER_BATCH_SIZE, 1 + ""); + runner.run(); + runner.assertAllFlowFilesTransferred(ConsumePulsarRecord.REL_PARSE_FAILURE); + + verify(mockClientService.getMockConsumer(), times(1)).acknowledgeCumulative(mockMessage); + } + + @Test + public void singleMalformedMessageTest() throws PulsarClientException { + when(mockMessage.getValue()).thenReturn(BAD_MSG.getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsarRecord.TOPICS, DEFAULT_TOPIC); + runner.setProperty(ConsumePulsarRecord.SUBSCRIPTION_NAME, DEFAULT_SUB); + runner.setProperty(ConsumePulsarRecord.CONSUMER_BATCH_SIZE, 1 + ""); + runner.run(); + runner.assertAllFlowFilesTransferred(ConsumePulsarRecord.REL_PARSE_FAILURE); + + verify(mockClientService.getMockConsumer(), times(1)).acknowledgeCumulative(mockMessage); + } + + /* + * Send a single message containing a single record + */ + @Test + public void singleMessageTest() throws PulsarClientException { + this.sendMessages(MOCKED_MSG, false, 1); + } + + /* + * Send a single message with multiple records + */ + @Test + public void singleMessageMultiRecordsTest() throws PulsarClientException { + StringBuffer input = new StringBuffer(1024); + StringBuffer expected = new StringBuffer(1024); + + for (int idx = 0; idx < 50; idx++) { + input.append("Justin Thyme, " + idx).append("\n"); + expected.append("\"Justin Thyme\",\"" + idx + "\"").append("\n"); + } + + List results = this.sendMessages(input.toString(), false, 1); + + String flowFileContents = new String(runner.getContentAsByteArray(results.get(0))); + assertEquals(expected.toString(), flowFileContents); + } + + /* + * Send a single message with multiple records, + * some of them good and some malformed + */ + @Test + public void singleMessageWithGoodAndBadRecordsTest() throws PulsarClientException { + StringBuffer input = new StringBuffer(1024); + StringBuffer expected = new StringBuffer(1024); + + for (int idx = 0; idx < 10; idx++) { + if (idx % 2 == 0) { + input.append("Justin Thyme, " + idx).append("\n"); + expected.append("\"Justin Thyme\",\"" + idx + "\"").append("\n"); + } else { + input.append(BAD_MSG).append("\n"); + } + } + + when(mockMessage.getValue()).thenReturn(input.toString().getBytes()); + mockClientService.setMockMessage(mockMessage); + + runner.setProperty(ConsumePulsarRecord.ASYNC_ENABLED, Boolean.toString(false)); + runner.setProperty(ConsumePulsarRecord.TOPICS, DEFAULT_TOPIC); + runner.setProperty(ConsumePulsarRecord.SUBSCRIPTION_NAME, DEFAULT_SUB); + runner.setProperty(ConsumePulsarRecord.CONSUMER_BATCH_SIZE, 1 + ""); + runner.setProperty(ConsumePulsarRecord.MAX_WAIT_TIME, "0 sec"); + runner.run(1, true); + + List successFlowFiles = runner.getFlowFilesForRelationship(ConsumePulsarRecord.REL_SUCCESS); + assertEquals(1, successFlowFiles.size()); + + List failureFlowFiles = runner.getFlowFilesForRelationship(ConsumePulsarRecord.REL_PARSE_FAILURE); + assertEquals(1, failureFlowFiles.size()); + } + + /* + * Send multiple messages with Multiple records each + */ + @Test + public void multipleMultiRecordsTest() throws PulsarClientException { + StringBuffer input = new StringBuffer(1024); + StringBuffer expected = new StringBuffer(1024); + + for (int idx = 0; idx < 50; idx++) { + input.append("Justin Thyme, " + idx).append("\n"); + expected.append("\"Justin Thyme\",\"" + idx + "\"").append("\n"); + } + + List results = this.sendMessages(input.toString(), false, 50, 1); + assertEquals(50, results.size()); + + String flowFileContents = new String(runner.getContentAsByteArray(results.get(0))); + assertEquals(expected.toString(), flowFileContents); + } +} diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncPublishPulsar.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncPublishPulsar.java new file mode 100644 index 0000000..201672c --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncPublishPulsar.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub.sync; + +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.nifi.processors.pulsar.pubsub.PublishPulsar; +import org.apache.nifi.processors.pulsar.pubsub.TestPublishPulsar; +import org.apache.nifi.util.MockFlowFile; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Test; +import org.mockito.ArgumentMatcher; +import org.mockito.Matchers; + +public class TestSyncPublishPulsar extends TestPublishPulsar { + + @Test + public void pulsarClientExceptionTest() throws PulsarClientException, UnsupportedEncodingException { + when(mockClientService.getMockProducer().send(Matchers.argThat(new ArgumentMatcher() { + @Override + public boolean matches(Object argument) { + return true; + } + }))).thenThrow(PulsarClientException.class); + + mockClientService.setMockProducer(mockProducer); + + runner.setProperty(PublishPulsar.TOPIC, "my-topic"); + + final String content = "some content"; + runner.enqueue(content.getBytes("UTF-8")); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsar.REL_FAILURE); + } + + @Test + public void invalidTopicTest() throws UnsupportedEncodingException, PulsarClientException { + runner.setProperty(PublishPulsar.TOPIC, "${topic}"); + + final String content = "some content"; + Map attributes = new HashMap(); + attributes.put("topic", null); + + runner.enqueue(content.getBytes("UTF-8"), attributes ); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsar.REL_FAILURE); + + // Confirm that no Producer as created + verify(mockClientService.getMockProducerBuilder(), times(0)).topic(anyString()); + } + + @Test + public void dynamicTopicTest() throws UnsupportedEncodingException, PulsarClientException { + when(mockProducer.getTopic()).thenReturn("topic-b"); + mockClientService.setMockProducer(mockProducer); + + runner.setProperty(PublishPulsar.TOPIC, "${topic}"); + + final String content = "some content"; + Map attributes = new HashMap(); + attributes.put("topic", "topic-b"); + + runner.enqueue(content.getBytes("UTF-8"), attributes ); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsar.REL_SUCCESS); + + // Verify that we sent the data to topic-b. + verify(mockClientService.getMockProducerBuilder(), times(1)).topic("topic-b"); + } + + @Test + public void singleFlowFileTest() throws UnsupportedEncodingException, PulsarClientException { + when(mockClientService.getMockProducer().getTopic()).thenReturn("my-topic"); + + runner.setProperty(PublishPulsar.TOPIC, "my-topic"); + + final String content = "some content"; + runner.enqueue(content.getBytes("UTF-8")); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsar.REL_SUCCESS); + + final MockFlowFile outFile = runner.getFlowFilesForRelationship(PublishPulsar.REL_SUCCESS).get(0); + outFile.assertContentEquals(content); + + // Verify that we sent the data to my-topic. + verify(mockClientService.getMockProducerBuilder(), times(1)).topic("my-topic"); + + // Verify that the send method on the producer was called with the expected content + verify(mockClientService.getMockProducer(), times(1)).send(content.getBytes()); + } + + @Test + public void multipleFlowFilesTest() throws UnsupportedEncodingException, PulsarClientException { + when(mockClientService.getMockProducer().getTopic()).thenReturn("my-topic"); + + runner.setProperty(PublishPulsar.TOPIC, "my-topic"); + final String content = "some content"; + for (int idx = 0; idx < 20; idx++) { + runner.enqueue(content.getBytes("UTF-8")); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsar.REL_SUCCESS); + } + // Verify that the send method on the producer was called with the expected content + verify(mockClientService.getMockProducer(), times(20)).send(content.getBytes()); + } + + @Test + public void demarcatedFlowFileTest() throws UnsupportedEncodingException, PulsarClientException { + final String content = "some content"; + final String demarcator = "\n"; + when(mockClientService.getMockProducer().getTopic()).thenReturn("my-topic"); + + runner.setProperty(PublishPulsar.TOPIC, "my-topic"); + runner.setProperty(PublishPulsar.MESSAGE_DEMARCATOR, demarcator); + + final StringBuffer sb = new StringBuffer(); + + for (int idx = 0; idx < 20; idx++) { + sb.append(content).append(demarcator); + } + + runner.enqueue(sb.toString().getBytes("UTF-8")); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsar.REL_SUCCESS); + verify(mockClientService.getMockProducer(), times(20)).send(content.getBytes()); + } +} diff --git a/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncPublishPulsarRecord.java b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncPublishPulsarRecord.java new file mode 100644 index 0000000..dd6850b --- /dev/null +++ b/nifi-pulsar-processors/src/test/java/org/apache/nifi/processors/pulsar/pubsub/sync/TestSyncPublishPulsarRecord.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.pulsar.pubsub.sync; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; + +import org.apache.nifi.processors.pulsar.AbstractPulsarProducerProcessor; +import org.apache.nifi.processors.pulsar.pubsub.PublishPulsarRecord; +import org.apache.nifi.processors.pulsar.pubsub.TestPublishPulsarRecord; +import org.apache.nifi.util.MockFlowFile; +import org.apache.pulsar.client.api.PulsarClientException; +import org.junit.Test; +import org.mockito.ArgumentMatcher; +import org.mockito.Matchers; + +public class TestSyncPublishPulsarRecord extends TestPublishPulsarRecord { + + @Test + public void pulsarClientExceptionTest() throws PulsarClientException { + when(mockClientService.getMockProducer().send(Matchers.argThat(new ArgumentMatcher() { + @Override + public boolean matches(Object argument) { + return true; + } + }))).thenThrow(PulsarClientException.class); + + final String content = "Mary Jane, 32"; + + runner.enqueue(content); + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, TOPIC_NAME); + runner.run(); + + verify(mockClientService.getMockProducer(), times(1)).send("\"Mary Jane\",\"32\"\n".getBytes()); + runner.assertAllFlowFilesTransferred(PublishPulsarRecord.REL_FAILURE); + + List results = runner.getFlowFilesForRelationship(PublishPulsarRecord.REL_FAILURE); + assertEquals(1, results.size()); + + String flowFileContents = new String(runner.getContentAsByteArray(results.get(0))); + assertEquals(content, flowFileContents); + } + + // Malformed content test, using "some content" + @Test + public void malformedContentTest() throws PulsarClientException { + final String content = "invalid content"; + + runner.enqueue(content); + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, TOPIC_NAME); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsarRecord.REL_FAILURE); + + verify(mockClientService.getMockProducer(), times(0)).send(content.getBytes()); + } + + @Test + public void testSingleRecordSuccess() throws PulsarClientException { + final String content = "Mary Jane, 32"; + + runner.enqueue(content); + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, TOPIC_NAME); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsarRecord.REL_SUCCESS); + + List results = runner.getFlowFilesForRelationship(PublishPulsarRecord.REL_SUCCESS); + MockFlowFile result = results.get(0); + + result.assertAttributeEquals(PublishPulsarRecord.MSG_COUNT, "1"); + verify(mockClientService.getMockProducer(), times(1)).send("\"Mary Jane\",\"32\"\n".getBytes()); + + } + + @Test + public void testMultipleRecordSuccess() throws PulsarClientException { + StringBuilder sb = new StringBuilder().append("Mary Jane, 32").append("\n") + .append("John Doe, 35").append("\n") + .append("Busta Move, 26").append("\n"); + + runner.enqueue(sb.toString()); + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, TOPIC_NAME); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsarRecord.REL_SUCCESS); + + List results = runner.getFlowFilesForRelationship(PublishPulsarRecord.REL_SUCCESS); + assertEquals(1, results.size()); + + MockFlowFile result = results.get(0); + result.assertContentEquals(sb.toString()); + result.assertAttributeEquals(PublishPulsarRecord.MSG_COUNT, "3"); + + verify(mockClientService.getMockProducer(), times(1)).send("\"Mary Jane\",\"32\"\n".getBytes()); + verify(mockClientService.getMockProducer(), times(1)).send("\"John Doe\",\"35\"\n".getBytes()); + verify(mockClientService.getMockProducer(), times(1)).send("\"Busta Move\",\"26\"\n".getBytes()); + } + + @Test + public void testBulkRecordSuccess() throws PulsarClientException { + StringBuilder sb = new StringBuilder(); + + for (int idx = 0; idx < 1000; idx++) { + sb.append("Mary Jane, 32").append("\n"); + } + + runner.enqueue(sb.toString()); + runner.setProperty(AbstractPulsarProducerProcessor.TOPIC, TOPIC_NAME); + runner.run(); + runner.assertAllFlowFilesTransferred(PublishPulsarRecord.REL_SUCCESS); + + List results = runner.getFlowFilesForRelationship(PublishPulsarRecord.REL_SUCCESS); + assertEquals(1, results.size()); + + MockFlowFile result = results.get(0); + result.assertContentEquals(sb.toString()); + result.assertAttributeEquals(PublishPulsarRecord.MSG_COUNT, "1000"); + + verify(mockClientService.getMockProducer(), times(1000)).send("\"Mary Jane\",\"32\"\n".getBytes()); + } +} diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..e379c6b --- /dev/null +++ b/pom.xml @@ -0,0 +1,41 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-nar-bundles + 1.8.0 + + + nifi-pulsar-bundle + pom + + + nifi-pulsar-processors + nifi-pulsar-nar + nifi-pulsar-client-service-api + nifi-pulsar-client-service + nifi-pulsar-client-service-nar + + + + 2.3.0 + + + A bundle of processors that can store and retrieve data from Apache Pulsar +