From 61db38c785ea1fd7975bd160d7a8417fddd92663 Mon Sep 17 00:00:00 2001 From: ahshahid Date: Thu, 21 Nov 2019 15:47:49 -0800 Subject: [PATCH 01/27] Fix for SNAP-3241. The closing of RowRecorder resultset was not releasing the read lock on table. fix other test failures --- .../hadoop/mapreduce/RowRecordReader.java | 2 +- .../mapred/hive/GFXDHiveInputFormatTest.java | 4 +- .../hadoop/TransactionHDFSTableTest.java | 14 +++++++ .../gemfirexd/jdbc/ConnectionURLTest.java | 6 ++- .../gemfirexd/jdbc/GfxdCallbacksTest.java | 10 +++++ .../com/pivotal/gemfirexd/jdbc/JsonTest.java | 7 ++++ .../jdbc/LangScripts_OrderbyTest.java | 1 + .../jdbc/LangScripts_ReopenScanTest.java | 13 +++++-- .../pivotal/gemfirexd/jdbc/OrdersTest.java | 4 ++ .../gemfirexd/jdbc/PkAsRegionKeyTest.java | 8 +++- .../jdbc/transactions/LocalIndexTxTest.java | 9 +++++ .../jdbc/transactions/TransactionTest.java | 39 ++++++++++++++++--- .../snapshot/SnapshotTransactionTest.java | 36 ++++++++++++++++- 13 files changed, 135 insertions(+), 18 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/hadoop/mapreduce/RowRecordReader.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/hadoop/mapreduce/RowRecordReader.java index 1bb2ee297..6351e4cf9 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/hadoop/mapreduce/RowRecordReader.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/hadoop/mapreduce/RowRecordReader.java @@ -214,7 +214,7 @@ public void close() throws IOException { if (!isClosed) { GfxdConnectionWrapper.restoreContextStack(es, rs); try { - rs.lightWeightClose(); + rs.close(); } catch (SQLException e) { logger.warn("Error while trying to free reader resources", e); } diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/internal/engine/hadoop/mapred/hive/GFXDHiveInputFormatTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/internal/engine/hadoop/mapred/hive/GFXDHiveInputFormatTest.java index f14fd1341..70db12721 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/internal/engine/hadoop/mapred/hive/GFXDHiveInputFormatTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/internal/engine/hadoop/mapred/hive/GFXDHiveInputFormatTest.java @@ -106,8 +106,10 @@ public void testHiveInputFormat() throws Exception { while (rr.next(key, value)) { assertEquals(count++, value.getRowAsResultSet().getInt("col1")); } - assertEquals(20, count); + rr.close(); + + st.execute("drop table if exists app.mytab1 "); TestUtil.shutDown(); } diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/internal/hadoop/TransactionHDFSTableTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/internal/hadoop/TransactionHDFSTableTest.java index 3c4ac69d6..6c331cc24 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/internal/hadoop/TransactionHDFSTableTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/internal/hadoop/TransactionHDFSTableTest.java @@ -97,6 +97,7 @@ public void testRRTransactionalInsertUpdateDeleteWithoutEvictionCriteriaPKbasedO // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); @@ -162,6 +163,8 @@ public void testRRTransactionalInsertUpdateDeleteWithoutEvictionCriteriaTableSca // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); @@ -224,6 +227,7 @@ public void testRCTransactionalInsertDeleteWithoutEvictionCriteriaPKbasedOperati // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); @@ -287,6 +291,7 @@ public void testRCTransactionalInsertDeleteWithoutEvictionCriteriaTableScan() th // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); @@ -404,6 +409,7 @@ public void testRRTransactionalInsertUpdateDeleteWithEvictionCriteriaPKbasedOper // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); @@ -522,6 +528,7 @@ public void testRRTransactionalInsertUpdateDeleteWithEvictionCriteriaTableScan() // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); @@ -572,6 +579,7 @@ public void testRRTransactionalInsertUpdateDeleteWithoutEvictionCriteriaUniqueIn // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); @@ -621,6 +629,7 @@ public void testRRTransactionalInsertUpdateDeleteWithEvictionCriteriaUniqueIndex // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); @@ -670,6 +679,7 @@ public void testRRTransactionalInsertUpdateDeleteWithEvictionCriteriaUniqueIndex // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); @@ -728,6 +738,7 @@ public void testRRTransactionalInsertUpdateDeleteWithEvictionCriteriaUniqueIndex // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); // not sure why it is getting region destroyed exception conn.close(); @@ -787,6 +798,7 @@ public void testRCTransactionalInsertUpdateDeleteReadPKbasedOperation() throws E // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); @@ -994,6 +1006,7 @@ public void testRCTransactionalInsertDeleteWithoutEvictionCriteria() throws Exce // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); @@ -1081,6 +1094,7 @@ public void testRCTransactionalInsertDeleteWithoutEvictionCriteriaCount() // Close connection, resultset etc... st.execute("drop table tran.t1"); st.execute("drop hdfsstore myhdfs"); + st.execute("drop schema tran restrict"); st.close(); conn.commit(); conn.close(); diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/ConnectionURLTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/ConnectionURLTest.java index 1c68b89ca..7eee79a51 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/ConnectionURLTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/ConnectionURLTest.java @@ -96,7 +96,7 @@ public void testConnectionURLWithoutName() throws SQLException { s.execute("insert into t1 (c1, c2) values (10, 'YYYY')"); s.execute("insert into t1 (c1, c2) values (20, 'YYYY')"); s.executeQuery("select * from t1 where t1.c1=10"); - + s.execute("drop table t1"); conn.close(); } @@ -164,7 +164,8 @@ public void testMultiConnectionURLWithoutName() throws SQLException { s = conn.createStatement(); s.executeQuery("select * from t1 where t1.c1=10"); - return; + s.execute("drop table t1"); + conn.close(); } /** @@ -210,6 +211,7 @@ public void testMultiConnectionURLWithName() throws SQLException { rs = s.executeQuery("select id from sys.members"); assertTrue(rs.next()); assertFalse(rs.next()); + s.execute("drop table t1"); conn.close(); // check exception without dbName using a subsubprotocol diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/GfxdCallbacksTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/GfxdCallbacksTest.java index 833ee1cc9..64b19bb68 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/GfxdCallbacksTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/GfxdCallbacksTest.java @@ -115,6 +115,8 @@ public void testListenerAddition() throws SQLException { addListener("ID1", "EMP", "TESTTABLE", "com.pivotal.gemfirexd.jdbc.GfxdTestCacheListener", "initInfoStrForListener",null); s.execute("insert into EMP.TESTTABLE values (1, 2, 3)"); + s.execute("drop table emp.testtable"); + s.execute("drop schema emp restrict"); } public void testMultipleListenerAdditionAndRemoval() throws SQLException { @@ -132,6 +134,8 @@ public void testMultipleListenerAdditionAndRemoval() throws SQLException { removeListener("ID1", "EMP", "TESTTABLE"); s.execute("insert into EMP.TESTTABLE values (5, 6, 7)"); s.execute("update EMP.TESTTABLE set thirdid=10 where id=1"); + s.execute("drop table emp.testtable"); + s.execute("drop schema emp restrict"); } public void testWriterAddition() throws SQLException { @@ -144,6 +148,8 @@ public void testWriterAddition() throws SQLException { addWriter("EMP", "TESTTABLE", "com.pivotal.gemfirexd.jdbc.GfxdTestCacheWriter", "initInfoStrForWriter", ""); s.execute("insert into EMP.TESTTABLE values (1, 2, 3)"); + s.execute("drop table emp.testtable"); + s.execute("drop schema emp restrict"); } public void testMultipleWriterAddition() throws SQLException { @@ -158,6 +164,8 @@ public void testMultipleWriterAddition() throws SQLException { addWriter("EMP", "TESTTABLE", "com.pivotal.gemfirexd.jdbc.GfxdTestCacheWriter", "initInfoStrForWriter", null); s.execute("insert into EMP.TESTTABLE values (1, 2, 3)"); + s.execute("drop table emp.testtable"); + s.execute("drop schema emp restrict"); } public void testWriterRemoval() throws SQLException { @@ -173,5 +181,7 @@ public void testWriterRemoval() throws SQLException { s.execute("update EMP.TESTTABLE set secondid=1 where id=1"); removeWriter("EMP", "TESTTABLE"); s.execute("insert into EMP.TESTTABLE values (5, 6, 7)"); + s.execute("drop table emp.testtable"); + s.execute("drop schema emp restrict"); } } diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/JsonTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/JsonTest.java index 65a302fab..9da75cb79 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/JsonTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/JsonTest.java @@ -174,6 +174,8 @@ public static void simpleJSONOps (Connection conn, boolean isPartitioned, boolea } i++; } + stmt.execute("drop view view1"); + stmt.execute("drop table t1"); } public void testJSON() throws Exception { @@ -269,6 +271,7 @@ public void testJSON_stream() throws Exception { i++; } assertEquals(2, i); + stmt.execute("drop table t1"); } public void testBug51374_1() throws Exception { @@ -322,6 +325,8 @@ public void testBug51374_1() throws Exception { } i++; } + stmt.execute("drop table t1"); + stmt.execute("drop table t2"); } finally { new File("T1.dat").delete(); } @@ -367,6 +372,8 @@ public void testBug51374_2() throws Exception { System.out.println("output=" + output); i++; } + stmt.execute("drop table t1"); + stmt.execute("drop table t2"); } finally { new File("T1.dat").delete(); } diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java index 025f115b6..5ea663d26 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java @@ -487,6 +487,7 @@ public void testLangScript_OrderbyTestNoPartitioning() throws Exception { "select t1.* from t1, t2 where t1.c1=t2.t2c1 order by t2c1", new String[][] { {"3","a"},{"4","c"} } }, { "drop table t1", null }, + { "drop table t2", null }, { "create table person (name varchar(10), age int)"+ getOffHeapSuffix(), null }, { "insert into person values ('John', 10)", null }, { "insert into person values ('John', 30)", null }, diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_ReopenScanTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_ReopenScanTest.java index 7a55ae94d..1de41e99e 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_ReopenScanTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_ReopenScanTest.java @@ -109,7 +109,10 @@ public void testLangScript_ReopenScanTestNoPartitioning() throws Exception { "insert into n values 1,2,3", null }, { "insert into x values 1,2,3", null }, { "select * from x where x in (select x from n)", new String[][] { {"1"},{"2"},{"3"} } }, - { "drop table n", null } + { "drop table n", null }, + { "drop table x", null }, + { "drop table y", null }, + { "drop table z", null } }; // Do not use partitioning as default, use replicate @@ -181,9 +184,11 @@ public void testLangScript_ReopenScanWithPartitioning() throws Exception { "insert into n values 1,2,3", null }, { "insert into x values 1,2,3", null }, { "select * from x where x in (select x from n)", new String[][] { {"1"},{"2"},{"3"} } }, - { "drop table n", null } - - }; + { "drop table n", null }, + { "drop table x", null }, + { "drop table y", null }, + { "drop table z", null } + }; Connection conn = TestUtil.getConnection(); Statement stmt = conn.createStatement(); diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/OrdersTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/OrdersTest.java index b93aba5ae..5620b7e4a 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/OrdersTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/OrdersTest.java @@ -67,6 +67,9 @@ public void testQuery() throws Exception { rs = q.executeQuery(); while (rs.next()); // consume ResultSet } + Statement st = conn.createStatement(); + st.execute("drop table orders"); + st.close(); } @@ -87,6 +90,7 @@ public void loadTestData(Connection conn) throws SQLException { Statement s = conn.createStatement(); // We create a table... + s.execute("drop table if exits orders"); s.execute("create table orders" + "(id int not null , cust_name varchar(200), vol int, " + "security_id varchar(10), num int, addr varchar(100))"); diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/PkAsRegionKeyTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/PkAsRegionKeyTest.java index 78cfce4a7..21cfddf53 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/PkAsRegionKeyTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/PkAsRegionKeyTest.java @@ -64,6 +64,7 @@ public void testPkAsRegionKeySingleColumn() throws SQLException { assertFalse("Expected a result set containing one row " + "but there is another with " + rs.getInt(1), rs.next()); + s.execute("drop table t1"); } /** @@ -102,6 +103,7 @@ public void testPkAsRegionKeyMultiColumn() throws SQLException { assertFalse("Expected a result set conatining one row " + "but there is another with " + rs.getInt(1), rs.next()); + s.execute("drop table t1"); } /** @@ -455,6 +457,7 @@ public void testQueryWithOutPk() throws SQLException { searchKey+=10; } + s.execute("drop table broker_tickets"); } /** @@ -552,6 +555,7 @@ public void testRegionEntryAsRowLocation() throws SQLException { searchKey += 10; } + s.execute("drop table broker_tickets"); } /** @@ -670,8 +674,8 @@ public void testSingleVmIndexScans () throws Exception { "broker_tickets"); while (rs.next()) { assertEquals("Result should match ", 15, rs.getInt(1)); - } - + } + s.execute("drop table broker_tickets"); } } diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/LocalIndexTxTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/LocalIndexTxTest.java index 9594c13b8..4352dc0ab 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/LocalIndexTxTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/LocalIndexTxTest.java @@ -202,6 +202,7 @@ public double overrideDerbyOptimizerIndexUsageCostForSortedIndexScan( conn.commit(); } finally { + st.execute("drop table t1"); GemFireXDQueryObserverHolder.clearInstance(); } } @@ -277,6 +278,7 @@ public double overrideDerbyOptimizerIndexUsageCostForSortedIndexScan( assertFalse(rs.next()); } finally { + st.execute("drop table t1"); GemFireXDQueryObserverHolder.clearInstance(); } } @@ -340,6 +342,7 @@ public double overrideDerbyOptimizerIndexUsageCostForSortedIndexScan( conn.commit(); } finally { + st.execute("drop table t1"); GemFireXDQueryObserverHolder.clearInstance(); } } @@ -412,6 +415,7 @@ public double overrideDerbyOptimizerIndexUsageCostForSortedIndexScan( conn.commit(); } finally { + st.execute("drop table t1"); GemFireXDQueryObserverHolder.clearInstance(); } } @@ -492,6 +496,7 @@ public double overrideDerbyOptimizerIndexUsageCostForSortedIndexScan( // 22 rows should be deleted some txnl and some committed assertEquals(22,st.executeUpdate("delete from t1 where c2 = 8 or c2 > 34")); conn.commit(); + st.execute("drop table t1"); } finally { GemFireXDQueryObserverHolder.clearInstance(); @@ -556,6 +561,7 @@ public double overrideDerbyOptimizerIndexUsageCostForSortedIndexScan( assertEquals(14,st.executeUpdate("update t1 set c2 = 8 " + " where c5 >= 1 and c5 < 21 ")); conn.commit(); + st.execute("drop table t1"); } finally { GemFireXDQueryObserverHolder.clearInstance(); @@ -621,6 +627,7 @@ public double overrideDerbyOptimizerIndexUsageCostForSortedIndexScan( " where c5 >= 1 and c5 < 21 ")); conn.commit(); + st.execute("drop table t1"); } finally { GemFireXDQueryObserverHolder.clearInstance(); @@ -683,6 +690,7 @@ public double overrideDerbyOptimizerIndexUsageCostForSortedIndexScan( " where c5 >= 18 and c5 < 26 ")); conn.commit(); + st.execute("drop table t1"); } finally { GemFireXDQueryObserverHolder.clearInstance(); } @@ -764,5 +772,6 @@ private void txInsertUpdateDeleteComboForBatching(boolean isPR) // 22 rows should be deleted some txnl and some committed assertEquals(22, st.executeUpdate("delete from t1 where c2 = 8 or c2 > 34")); conn.commit(); + st.execute("drop table t1"); } } diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/TransactionTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/TransactionTest.java index 877472a63..35af282a1 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/TransactionTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/TransactionTest.java @@ -389,6 +389,8 @@ public void run() { rs.close(); st2.close(); conn.commit(); + st.execute("drop table tran.t1"); + st.execute("drop schema tran restrict"); conn.close(); } @@ -603,8 +605,10 @@ public void testTransactionalKeyBasedUpdates() throws Exception { this.doOffHeapValidations(); assertEquals("Numbers of rows in resultset should be one", 1, numRows); rs.close(); - st.close(); conn.commit(); + st.execute("drop table test.t1"); + st.execute("drop schema test restrict"); + st.close(); conn.close(); } @@ -631,8 +635,11 @@ public void testTruncate() throws Exception { numRows++; } assertEquals("ResultSet should have zero rows", 0, numRows); - st.close(); conn.commit(); + st.execute("drop table test.t1"); + st.execute("drop schema test restrict"); + st.close(); + } /** @@ -686,6 +693,9 @@ public void testNewGFETransactionNotCreatedForPKBasedOp() throws Exception { assertNull(GfxdConnectionHolder.getHolder().getConnectionID(tid)); */ conn.commit(); + st.execute("drop table trade.securities"); + st.execute("drop schema trade restrict"); + } /** @@ -756,8 +766,11 @@ public void testTransactionsAndIndexMaintenance() throws Exception { numRows++; } assertEquals("Should return 1000 rows ", rows, numRows); - st.close(); conn.commit(); + st.execute("drop table test.t1"); + st.execute("drop schema test restrict"); + st.close(); + } /** @@ -833,6 +846,8 @@ public void testTransactionalDeletes() throws Exception { assertFalse(rs.next()); conn.commit(); rs.close(); + st.execute("drop table tran.t1"); + st.execute("drop schema tran restrict"); st.close(); conn.close(); } @@ -872,7 +887,8 @@ public void testTransactionalDeleteWithLocalIndexes() throws Exception { } finally { GemFireXDQueryObserverHolder.removeObserver(checkIndex); } - + st.execute("drop table tran.t1"); + st.execute("drop schema tran restrict"); st.close(); conn.close(); } @@ -906,6 +922,8 @@ public void testTransactionsAndUniqueIndexes() throws Exception { } } conn.rollback(); + st.execute("drop table tran.t1"); + st.execute("drop schema tran restrict"); conn.close(); /* final CheckIndexOperations checkIndex = new CheckIndexOperations( @@ -1018,6 +1036,8 @@ public void testTransactionalUpdate() throws Exception { st.close(); psUpdate.close(); ps.close(); + st.execute("drop table trade.securities"); + st.execute("drop schema trade restrict"); conn.close(); } @@ -1918,8 +1938,10 @@ public void testTxnUpdateGettingLostBug2_43222_44096() throws Exception { + ", " + rs.getObject(6) + ", " + rs.getObject(7)); } assertEquals(1, sel_cnt); - conn.commit(); + stmt.execute("drop table txn.customer"); + stmt.execute("drop schema txn restrict"); + } public void testTxnUpdateGettingLostBug_43222() throws Exception { @@ -1991,8 +2013,9 @@ public void testTxnUpdateGettingLostBug_43222() throws Exception { } assertEquals(1, sel_cnt); - conn.commit(); + stmt.execute("drop table txn.customer"); + stmt.execute("drop schema txn restrict"); } public void testDeleteLockBeforeReferenceKeyCheck() throws Exception { @@ -2093,6 +2116,8 @@ public void testWrongConsecutiveUpdates_42647() throws SQLException { assertTrue(rs.next()); assertEquals(-100.0, rs.getDouble(6)); assertFalse(rs.next()); + stmt.execute("drop table txn.customer"); + stmt.execute("drop schema txn restrict"); } class ObserverForBug43152 extends GemFireXDQueryObserverAdapter { @@ -2255,6 +2280,8 @@ public void testTransactionsAndUniqueIndexes_43152() throws Exception { GemFireXDQueryObserverHolder.putInstance(observer); conn.commit(); assertFalse(observer.wasMethodCalled()); + st.execute("drop table tran.t1"); + st.execute("drop schema tran restrict"); } public void testPkNonPkUpdates() throws Exception { diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/snapshot/SnapshotTransactionTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/snapshot/SnapshotTransactionTest.java index c5c078c40..daaca4579 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/snapshot/SnapshotTransactionTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/snapshot/SnapshotTransactionTest.java @@ -131,6 +131,7 @@ public void testBatchInsertAutoCommitWithConflict() throws Exception { Region r = Misc.getRegionForTable("TRAN.T1", true); assert (r.size() == 10); + st.execute("drop table tran.t1"); } //auto commit is disabled. @@ -185,6 +186,7 @@ public void run() { assertEquals("ResultSet should contain 0 rows ", 0, numRows); rs.close(); + st.execute("drop table tran.t1"); st.close(); conn.close(); } @@ -217,7 +219,7 @@ public void _testAutoCommitWithIndexSnapshot() throws Exception { //conn.commit(); // Close connection, resultset etc... - + st.execute("drop table t1"); st.close(); conn.close(); } @@ -267,6 +269,7 @@ public void testDeleteCommitSnapshot() throws Exception { // Close connection, resultset etc... rs.close(); + st.execute("drop table t1"); st.close(); conn.close(); } @@ -298,6 +301,7 @@ public void _testAutoCommitWithIndex() throws Exception { // Close connection, resultset etc... rs.close(); + st.execute("drop table t1"); st.close(); conn.close(); } @@ -999,6 +1003,7 @@ public void run() { // Close connection, resultset etc... rs.close(); + st.execute("drop table t1"); st.close(); //conn.commit(); conn.close(); @@ -1074,6 +1079,7 @@ public void testReadSnapshotOnReplicatedTable() throws Exception { // Close connection, resultset etc... rs.close(); + st.execute("drop table t1"); st.close(); conn.commit(); conn.close(); @@ -1154,6 +1160,7 @@ public void testReadSnapshotOnPartitionedTable() throws Exception { // Close connection, resultset etc... rs.close(); + st.execute("drop table t1"); st.close(); //conn.commit(); conn.close(); @@ -1210,6 +1217,7 @@ public void testSnapshotAgainstUpdateOperations() throws Exception { assertEquals("C3 should be 20 ", 20, c2); } assertEquals("ResultSet should contain two rows ", 2, numRows); + st.execute("drop table t1"); //assert that old value is returned } @@ -1249,6 +1257,7 @@ public void testSnapshotAgainstDeleteOperations() throws Exception { numRows++; } assertEquals("ResultSet should contain two row ", 2, numRows); + st.execute("drop table t1"); //assert that old value is returned } @@ -1272,6 +1281,7 @@ public void testSnapshotAgainstPrepStmt() throws Exception { //PreparedStatement pst = conn.prepareStatement(sql); //pst.executeQuery(); GemFireCacheImpl.getInstance().getCacheTransactionManager().commit(); + st.execute("drop table tran.t1"); } public void _testSnapshotAgainstMultipleTable() throws Exception { @@ -1439,6 +1449,8 @@ public void _testSnapshotAgainstMultipleTable() throws Exception { // Close connection, resultset etc... rs.close(); + st.execute("drop table if exits t1"); + st.execute("drop table if exists t2"); st.close(); //conn.commit(); conn.close(); @@ -1506,7 +1518,17 @@ public void run() { public void testCommitWithConflicts() throws Exception { Connection conn = getConnection(); Statement st = conn.createStatement(); - st.execute("create schema tran"); + try { + st.execute("create schema tran"); + } catch (SQLException sqle) { + // ignore if schema already exists + try { + st.close(); + } catch (Exception e) { + //ignore + } + st = conn.createStatement(); + } st.execute("Create table tran.t1 (c1 int not null , c2 int not null, " + "primary key(c1)) replicate persistent enable concurrency checks" +getSuffix()); conn.commit(); @@ -1573,6 +1595,8 @@ public void run() { txMgrImpl.commit(); rs.close(); + st2.execute("drop table tran.t1"); + st2.execute("drop schema tran restrict"); st2.close(); conn.commit(); conn.close(); @@ -1637,6 +1661,7 @@ public void run() { } assertEquals("ResultSet should contain 10 rows ", 10, numRows); Misc.getGemFireCache().getCacheTransactionManager().commit(); + st.execute("drop table t1"); conn.commit(); } @@ -1696,6 +1721,7 @@ public void run() { Misc.getGemFireCache().getCacheTransactionManager().commit(); conn.commit(); rs.close(); + st.execute("drop table t1"); st.close(); conn.close(); } @@ -1764,6 +1790,7 @@ public void testSnapshotInsertRollback() throws Exception { // Close connection, resultset etc... rs.close(); + st.execute("drop table t1"); st.close(); conn.commit(); conn.close(); @@ -1861,6 +1888,7 @@ public void testSnapshotUpdateRollback() throws Exception { // Close connection, resultset etc... rs.close(); + st.execute("drop table t1"); st.close(); conn.commit(); conn.close(); @@ -1908,6 +1936,7 @@ public void testDeleteRollback() throws Exception { // Close connection, resultset etc... rs.close(); + st.execute("drop table t1"); st.close(); conn.commit(); conn.close(); @@ -2177,6 +2206,7 @@ public void _testReadSnapshotOnPartitionedTableInConcurrency() throws Exception assert(versionAfterExecutingUpdate == (versionAfterExecutingThreadWithNewTx+1)); conn.commit(); rs.close(); + st.execute("drop table t1"); st.close(); conn.close(); } @@ -2353,6 +2383,8 @@ public void run() { System.out.println(numRows1); // The count should be 21 after releasing the lock and re-initializing snapshot map assert (numRows1 == 26); + st.execute("drop table t2"); + st.execute("drop table t1"); } } From cea9e3a9dbe96814e6e72dc88a339a5b970e183b Mon Sep 17 00:00:00 2001 From: ahshahid Date: Thu, 21 Nov 2019 16:33:19 -0800 Subject: [PATCH 02/27] corrected test failures --- .../pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java | 1 + .../test/java/com/pivotal/gemfirexd/jdbc/OrdersTest.java | 2 +- .../gemfirexd/jdbc/transactions/TransactionTest.java | 9 ++++----- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java index 5ea663d26..2cffec3cd 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java @@ -1019,6 +1019,7 @@ public void testLangScript_OrderbyWithPartitioning() throws Exception { "select t1.* from t1, t2 where t1.c1=t2.t2c1 order by t2c1", new String[][] { {"3","a"},{"4","c"} } }, { "drop table t1", null }, + { "drop table t2", null }, { "create table person (name varchar(10), age int)"+ getOffHeapSuffix(), null }, { "insert into person values ('John', 10)", null }, { "insert into person values ('John', 30)", null }, diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/OrdersTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/OrdersTest.java index 5620b7e4a..74b80523d 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/OrdersTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/OrdersTest.java @@ -90,7 +90,7 @@ public void loadTestData(Connection conn) throws SQLException { Statement s = conn.createStatement(); // We create a table... - s.execute("drop table if exits orders"); + s.execute("drop table if exists orders"); s.execute("create table orders" + "(id int not null , cust_name varchar(200), vol int, " + "security_id varchar(10), num int, addr varchar(100))"); diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/TransactionTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/TransactionTest.java index 35af282a1..0e1211afa 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/TransactionTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/transactions/TransactionTest.java @@ -385,12 +385,11 @@ public void run() { assertEquals("Second columns should be 10 , ", 10, rs.getInt(2)); } assertEquals("ResultSet should have two rows ", 1, numRow); - rs.close(); - st2.close(); conn.commit(); - st.execute("drop table tran.t1"); - st.execute("drop schema tran restrict"); + st2.execute("drop table tran.t1"); + st2.execute("drop schema tran restrict"); + st2.close(); conn.close(); } @@ -1033,11 +1032,11 @@ public void testTransactionalUpdate() throws Exception { conn.commit(); rs.close(); this.doOffHeapValidations(); - st.close(); psUpdate.close(); ps.close(); st.execute("drop table trade.securities"); st.execute("drop schema trade restrict"); + st.close(); conn.close(); } From bcdab1f140b327b041db830a9f46bb4f4d6a4b98 Mon Sep 17 00:00:00 2001 From: ahshahid Date: Thu, 21 Nov 2019 16:51:42 -0800 Subject: [PATCH 03/27] corrected test failures --- .../com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java index 2cffec3cd..00b10c907 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java @@ -469,6 +469,7 @@ public void testLangScript_OrderbyTestNoPartitioning() throws Exception {"a","e"},{"b",null},{"g","c"},{null,"d"} } }, { "drop table t1", null }, { "drop table t2", null }, + { "drop table t3", null }, { "create table t1 (c1 int, c2 varchar(10))"+ getOffHeapSuffix(), null }, { "create table t2 (t2c1 int)"+ getOffHeapSuffix(), null }, { "insert into t1 values (3, 'a'), (4, 'c'), (2, 'b'), (1, 'c')", null }, @@ -1001,6 +1002,7 @@ public void testLangScript_OrderbyWithPartitioning() throws Exception {"a","e"},{"b",null},{"g","c"},{null,"d"} } }, { "drop table t1", null }, { "drop table t2", null }, + { "drop table t3", null }, { "create table t1 (c1 int, c2 varchar(10))"+ getOffHeapSuffix(), null }, { "create table t2 (t2c1 int)"+ getOffHeapSuffix(), null }, { "insert into t1 values (3, 'a'), (4, 'c'), (2, 'b'), (1, 'c')", null }, From 5ba8eea89975178caa399417bf0fa93d2587ace4 Mon Sep 17 00:00:00 2001 From: paresh-p11 <43569032+paresh-p11@users.noreply.github.com> Date: Fri, 22 Nov 2019 18:23:14 +0530 Subject: [PATCH 04/27] SNAP-3238 (#532) SNAP-3238 * Fixing logic to choose member object. --- .../internal/engine/db/FabricDatabase.java | 3 +- .../PersistentStateInRecoveryMode.java | 259 ++++++++++++++++-- .../RecoveredMetadataRequestMessage.java | 9 +- .../internal/engine/store/RowFormatter.java | 19 +- 4 files changed, 244 insertions(+), 46 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java index c39dccf92..95657a232 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java @@ -1196,8 +1196,7 @@ private List postCreateDDLReplay(final EmbedConnecti } else { final DDLConflatable conflatable = (DDLConflatable) qVal; String schemaForTable = conflatable.getSchemaForTableNoThrow(); - if (!this.memStore.getGemFireCache().isSnappyRecoveryMode() && - this.memStore.restrictedDDLStmtQueue() && + if (!recoveryMode && this.memStore.restrictedDDLStmtQueue() && !(!disallowMetastoreOnLocator && schemaForTable != null && Misc.isSnappyHiveMetaTable(schemaForTable))) { continue; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java index 12b39d3ba..4dc320a8d 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java @@ -20,20 +20,19 @@ import com.gemstone.gemfire.DataSerializable; import com.gemstone.gemfire.DataSerializer; -import com.gemstone.gemfire.LogWriter; import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember; import com.gemstone.gemfire.internal.cache.AbstractDiskRegion; import com.gemstone.gemfire.internal.cache.DiskInitFile; import com.gemstone.gemfire.internal.cache.DiskStoreImpl; import com.gemstone.gemfire.internal.cache.GemFireCacheImpl; import com.gemstone.gemfire.internal.cache.RegionEntry; -import com.gemstone.gemfire.internal.cache.persistence.DiskStoreID; import com.gemstone.gemfire.internal.cache.persistence.PRPersistentConfig; -import com.gemstone.gemfire.internal.cache.versions.RegionVersionHolder; import com.gemstone.gemfire.internal.cache.versions.RegionVersionVector; import com.pivotal.gemfirexd.internal.engine.GfxdConstants; import com.pivotal.gemfirexd.internal.engine.Misc; import com.pivotal.gemfirexd.internal.engine.ddl.DDLConflatable; +import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils; +import com.pivotal.gemfirexd.internal.iapi.services.sanity.SanityManager; import java.io.DataInput; import java.io.DataOutput; @@ -154,6 +153,14 @@ public String toString() { sb.append(e); sb.append("\n"); } + sb.append("prToNumBuckets:"); + for (Map.Entry e : prToNumBuckets.entrySet()) { + sb.append("\nbucket: " + e.getKey() + " ::: numBuckets: " + e.getValue()); + } + sb.append("\n"); + sb.append("replicatedRegions\n"); + sb.append(replicatedRegions); + sb.append("\n"); sb.append("Catalog Objects\n"); for (Object obj : this.catalogObjects) { /// since like catalogtableobjects, other types also // has implicit tostring conversion, pattern matching isn't required ? @@ -206,6 +213,23 @@ public RecoveryModePersistentView( public RecoveryModePersistentView() { } + public RecoveryModePersistentView(RecoveryModePersistentView view) { + this.member = view.getMember(); + this.regionPath = view.getRegionPath(); + this.diskStoreName = view.getDiskStoreName(); + this.rvv = view.getRvv().getCloneForTransmission(); + this.mostRecentEntryModifiedTime = view.getMostRecentEntryModifiedTime(); + this.latestOplogTime = view.getLatestOplogTime(); + } + + public long getMostRecentEntryModifiedTime() { + return mostRecentEntryModifiedTime; + } + + public long getLatestOplogTime() { + return latestOplogTime; + } + public RegionVersionVector getRvv() { return this.rvv; } @@ -250,63 +274,244 @@ public String toString() { this.member + ": region: " + this.regionPath + + ": latestOplogTime:" + + this.latestOplogTime + + ": mostRecentEntryModifiedTime:" + + this.mostRecentEntryModifiedTime + + ": rvv:" + + this.rvv + ": diskStoreName:" + this.diskStoreName; return sb; } + void log(String str) { + if(GemFireXDUtils.TraceRecoveryMode) { + SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_RECOVERY_MODE, str); + } + } + // todo: add test to check if compareTo method is commutative and transitive @Override public int compareTo(RecoveryModePersistentView other) { // They should be called for the same region assert this.regionPath.equals(other.regionPath); - if (mostRecentEntryModifiedTime == other.mostRecentEntryModifiedTime && latestOplogTime == other.latestOplogTime && Objects.equals(regionPath, other.regionPath) && Objects.equals(diskStoreName, other.diskStoreName) && rvv.sameAs(other.rvv) && member.equals(other.member)) { + SanityManager.DEBUG_PRINT("info", "Comparing same object. Doesn't make sense to do this. Please check."); return 0; } - if (this.rvv.logicallySameAs(other.rvv)) { + if (this.rvv.logicallySameAs(other.rvv) || + (!this.rvv.dominates(other.rvv) && !other.rvv.dominates(this.rvv))) { + log("RVV based approach is not usable."); if (this.latestOplogTime <= other.latestOplogTime) { + SanityManager.DEBUG_PRINT("info", " 1. this.latestOplogTime <= other.latestOplogTime"); + log("Deciding on basis of Oplog file time: return -1"); return -1; } - if (this.mostRecentEntryModifiedTime < other.mostRecentEntryModifiedTime) { // replacing "<=" with "<" as former makes it non-commutative + if (this.mostRecentEntryModifiedTime < other.mostRecentEntryModifiedTime) { + // replacing "<=" with "<" as former makes it non-commutative + log("Deciding on basis of Modified entry time: return -1"); return -1; } + log("Oplog file time AND Modified entry time are greater for LHS: return 1"); return 1; } else { + log("Use RVV based approach."); if (this.rvv.dominates(other.rvv)) { + log("RVV of LHS dominates RHS: return 1"); return 1; - } else if (other.rvv.dominates(this.rvv)) { + } else { + log("RVV of RHS dominates LHS: return -1"); return -1; } + } + } + } + + public static class RecoveryModePersistentViewPair + implements Comparable { + + private RecoveryModePersistentView rowView = null; + private RecoveryModePersistentView colView = null; + + public RecoveryModePersistentView getRowView() { + return rowView; + } + + public RecoveryModePersistentView getColView() { + return colView; + } + + public RecoveryModePersistentViewPair(RecoveryModePersistentView rowView, + RecoveryModePersistentView colView) { + this.rowView = rowView; + this.colView = colView; + } + + @Override + public String toString() { + String theString = "RecoveryModePersistentViewPair\nROW VIEW: " + + this.getRowView() + + "\nCOL VIEW: " + + this.getColView(); + return theString; + } + + @Override + public int compareTo(RecoveryModePersistentViewPair other) { + // They should be called for the same region + log("Comparing :::\nthis view: " + this + "\n other view: " + other); + + RecoveryModePersistentView thisRowView = this.getRowView(); + RecoveryModePersistentView thisColView = this.getColView(); + RecoveryModePersistentView otherRowView = other.getRowView(); + RecoveryModePersistentView otherColView = other.getColView(); + + if ((thisRowView.mostRecentEntryModifiedTime == otherRowView.mostRecentEntryModifiedTime && + thisRowView.latestOplogTime == otherRowView.latestOplogTime && + Objects.equals(thisRowView.regionPath, otherRowView.regionPath) && + Objects.equals(thisRowView.diskStoreName, otherRowView.diskStoreName) && + thisRowView.rvv.sameAs(otherRowView.rvv) && + thisRowView.member.equals(otherRowView.member)) + && + (thisColView.mostRecentEntryModifiedTime == otherColView.mostRecentEntryModifiedTime && + thisColView.latestOplogTime == otherColView.latestOplogTime && + Objects.equals(thisColView.regionPath, otherColView.regionPath) && + Objects.equals(thisColView.diskStoreName, otherColView.diskStoreName) && + thisColView.rvv.sameAs(otherColView.rvv) && + thisColView.member.equals(otherColView.member) || + (thisColView == null && otherColView == null)) + ) { + SanityManager.DEBUG_PRINT("info", "Comparing same object. Doesn't make sense to do this. Please check."); + return 0; + } + + int compareValue = getCompareValue(thisRowView, thisColView, otherRowView, otherColView); + return compareValue; + } - Map versionHolderOne - = this.rvv.getMemberToVersion(); - Map versionHolderTwo - = other.rvv.getMemberToVersion(); - - for (Map.Entry e1 : versionHolderOne.entrySet()) { - for (Map.Entry e2 : versionHolderTwo.entrySet()) { - if (e1.getKey().equals(e2.getKey())) { - RegionVersionHolder rvh1 = e1.getValue(); - RegionVersionHolder rvh2 = e2.getValue(); - if (rvh1.dominates(rvh2)) { - return 1; - } else if (rvh2.dominates(rvh1)) { - return -1; - } - // comprise ; because both has some set of changes that other doesn't - // take sum of all versions from each entry in the map and see which is bigger - // sum ???? don't know - } + void log(String str) { + if(GemFireXDUtils.TraceRecoveryMode) { + SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_RECOVERY_MODE, str); + } + } + + private int getCompareValue( + RecoveryModePersistentView thisRowView, + RecoveryModePersistentView thisColView, + RecoveryModePersistentView otherRowView, + RecoveryModePersistentView otherColView) { + + if (thisColView == null || otherColView == null) { + // row tables as col regs are absent + int retVal = (new RecoveryModePersistentView(thisRowView).compareTo(otherRowView)); + log("Row table: return value = " + retVal); + int retVal1 = retVal; + return retVal1; + } else { + // col tables + boolean rowAndColRVVLogicallySame = + thisRowView.rvv.logicallySameAs(otherRowView.rvv) && + (thisColView.rvv.logicallySameAs(otherColView.rvv)); + boolean rowAndColLatestOplogTimeSame = + thisColView.latestOplogTime == otherColView.latestOplogTime && + thisRowView.latestOplogTime == otherRowView.latestOplogTime; + boolean otherRowDominates = otherRowView.rvv.dominates(thisRowView.rvv); + boolean otherColDominates = otherColView.rvv.dominates(thisColView.rvv); + boolean thisRowDominates = thisRowView.rvv.dominates(otherRowView.rvv); + boolean thisColDominates = thisColView.rvv.dominates(otherColView.rvv); + +// t, t | t, t +// t, t | f, f +// f, f | t, t +// f, f | f, f +// t, f | f, t +// f, t | t, f + boolean isRVVNotUsable = + ((thisRowDominates == otherRowDominates && + otherColDominates == thisColDominates) || + (thisRowDominates != otherRowDominates && + otherColDominates != thisColDominates)); + + log("thisRowView Dominates otherRowView: " + thisRowView.rvv.dominates(otherRowView.rvv)); + log("otherRowView Dominates thisRowView: " + otherRowView.rvv.dominates(thisRowView.rvv)); + log("thisColView Dominates otherColView: " + thisColView.rvv.dominates(otherColView.rvv)); + log("otherColView Dominates thisColView: " + otherColView.rvv.dominates(thisColView.rvv)); + + if (isRVVNotUsable) { + log("RVV based approach is not usable."); + if (rowAndColLatestOplogTimeSame) { + int retVal = compareTime(thisColView.getMostRecentEntryModifiedTime(), + otherColView.getMostRecentEntryModifiedTime(), + thisRowView.getMostRecentEntryModifiedTime(), + otherRowView.getMostRecentEntryModifiedTime()); + log("Decide on basis of MostRecentEntryModifiedTime: return value = " + retVal); + return retVal; + } else { + int retVal = compareTime(thisColView.getLatestOplogTime(), + otherColView.getLatestOplogTime(), + thisRowView.getLatestOplogTime(), + otherRowView.getLatestOplogTime()); + log("Decide on basis of LatestOplogTime: return value = " + retVal); + return retVal; + } + } else { +// NOTES : At this point there can't be a case when all 4 combinations are false or all are true. +// Also cannot be a case when row combinations are true and col combinations are false(and vice versa). + +// Also there cannot be a case when there is conflict. +// eg: (thisrowdominates)true & (otherrowdominates)false for row AND +// (thiscoldominates)false & (othercoldominates)true for col + + + if (thisColDominates == otherColDominates) { // true == true , false == false + // Decide on basis of row rvv +// t, t | f, t +// t, t | t, f +// f, f | t, f +// f, f | f, t + return thisRowDominates ? 1 : -1; + } else if (thisRowDominates == otherRowDominates) { // true == true , false == false +// t, t | f, t +// t, t | t, f +// f, f | t, f +// f, f | f, t + // Decide on basis of col rvv + return thisColDominates ? 1 : -1; + } else { +// t, f | t, f +// f, t | f, t + // conflicting case - not sure when will this happen? + return thisColDominates && thisRowDominates ? 1 : -1; } + } } - return 1; + } + + private int compareTime(Long thisColViewTime, Long otherColViewTime, Long thisRowViewTime, Long otherRowViewTime) { + // | equal col, thisRowGreater + // | thisColgreater, equalrow + // 1 ---| thisColGreater, thisRowGreater + // | thisColGreater, otherRowGreater + + // | equal col, otherRowGreater + // | otherColGreater, equal row + // -1 ---| otherColGreater, thisRowGreater + // | otherColGreater, otherRowGreater + if ((thisColViewTime > otherColViewTime && + thisRowViewTime >= otherRowViewTime) || + (thisColViewTime == otherColViewTime && + thisRowViewTime > otherRowViewTime)) { + return 1; + } else { + return -1; + } } } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/RecoveredMetadataRequestMessage.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/RecoveredMetadataRequestMessage.java index d9b581a9e..d662d1b70 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/RecoveredMetadataRequestMessage.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/RecoveredMetadataRequestMessage.java @@ -2,6 +2,7 @@ import com.gemstone.gemfire.cache.execute.ResultCollector; import com.gemstone.gemfire.distributed.DistributedMember; +import com.pivotal.gemfirexd.internal.engine.GfxdConstants; import com.pivotal.gemfirexd.internal.engine.Misc; import com.pivotal.gemfirexd.internal.engine.distributed.GfxdDistributionAdvisor; import com.pivotal.gemfirexd.internal.engine.distributed.RecoveryModeResultHolder; @@ -42,7 +43,11 @@ public RecoveredMetadataRequestMessage() { @Override protected void execute() throws Exception { GemFireXDUtils.waitForNodeInitialization(); - sendPersistentStateMsg(Misc.getMemStore().getPersistentStateMsg()); + PersistentStateInRecoveryMode psrm = Misc.getMemStore().getPersistentStateMsg(); + if (GemFireXDUtils.TraceRecoveryMode) { + SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_RECOVERY_MODE, psrm.toString()); + } + sendPersistentStateMsg(psrm); this.lastResultSent = true; } @@ -82,7 +87,7 @@ public void sendList(ArrayList arrayList, Integer partSize, ListType type .RecoveryModePersistentView>)arrayChunk))); } else { // last chunk of the list this.lastResult(new RecoveryModeResultHolder.PersistentStateInRMAllRegionViews((ArrayList)arrayChunk)); + .RecoveryModePersistentView>)arrayChunk)); } } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RowFormatter.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RowFormatter.java index b55b33d3c..9d6c22e2b 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RowFormatter.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RowFormatter.java @@ -108,7 +108,7 @@ public final class RowFormatter implements Serializable { * normally not used because the offset then is negative of next variable * column offset, but when serializing columns back to back with embedded * widths (rather than offsets) this is used to denote a null column. - * + * * public static final int OFFSET_IS_NULL = -1; */ @@ -1039,8 +1039,6 @@ public RowFormatter(final ColumnDescriptorList cdl, final String schemaName, this.metadata = null; this.isTableFormatter = false; this.isPrimaryKeyFormatter = false; -// SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_ROW_FORMATTER, -// "RowFormatter#cqi#column[] = " + ArrayUtils.toString(this.columns)); } /** @@ -1105,8 +1103,6 @@ public RowFormatter(final ColumnDescriptor[] cdl, final int size, this.isTableFormatter = true; } this.isPrimaryKeyFormatter = false; -// SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_ROW_FORMATTER, -// "RowFormatter#const1#column[] = " + ArrayUtils.toString(this.columns)); } /** @@ -1165,8 +1161,6 @@ private RowFormatter(final ColumnDescriptorList cdl, final int size, if (isTableFormatter && container != null) { container.hasLobs = hasLobs(); } -// SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_ROW_FORMATTER, -// "RowFormatter#const2#column[] = " + ArrayUtils.toString(this.columns)); } /** @@ -1225,8 +1219,6 @@ private RowFormatter(final ColumnDescriptorList cdl, final int size, this.metadata = null; this.isTableFormatter = false; this.isPrimaryKeyFormatter = false; -// SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_ROW_FORMATTER, -// "RowFormatter#const3#column[] = " + ArrayUtils.toString(this.columns)); } /** @@ -1287,8 +1279,6 @@ private RowFormatter(final ColumnDescriptorList cdl, final int size, this.metadata = getMetaData(schemaName, tableName, schemaVersion); this.isTableFormatter = false; this.isPrimaryKeyFormatter = isPrimaryKeyFormatter; -// SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_ROW_FORMATTER, -// "RowFormatter#const4#column[] = " + ArrayUtils.toString(this.columns)); } /** @@ -1924,7 +1914,7 @@ public final void writeVersion(final DataOutput out) throws IOException { /** * Read the schema version from the start of byte array. */ - public static int readVersion(final byte[] bytes) { + static int readVersion(final byte[] bytes) { // version is written at the start of byte array final int schemaVersion = readCompactInt(bytes, 0); // special token TOKEN_RECOVERY_VERSION is used for upgrade from old Oplogs @@ -7136,8 +7126,7 @@ private int[] initNumVarWidthCols(final ColumnDescriptorList cdl, if (hasUnknown || maxDataLength > Integer.MAX_VALUE) { maxDataLength = Integer.MAX_VALUE; } - // todo change is ok? - calcNumOffsetBytesToUse((int)maxDataLength, result[NUM_VAR_WIDTH_COLS_POS], + calcNumOffsetBytesToUse((int)maxDataLength, result[VAR_DATA_OFFSET_POS], result); return result; } @@ -7168,7 +7157,7 @@ private int[] initNumVarWidthCols(final ColumnDescriptor[] cdl, if (hasUnknown || maxDataLength > Integer.MAX_VALUE) { maxDataLength = Integer.MAX_VALUE; } - calcNumOffsetBytesToUse((int)maxDataLength, result[NUM_VAR_WIDTH_COLS_POS], + calcNumOffsetBytesToUse((int)maxDataLength, result[VAR_DATA_OFFSET_POS], result); return result; } From f80af17332f236ec2832af327a646a6e2f81af4a Mon Sep 17 00:00:00 2001 From: ahshahid Date: Fri, 22 Nov 2019 12:10:55 -0800 Subject: [PATCH 05/27] fixing test failure --- .../gemfirexd/jdbc/LangScripts_OrderbyTest.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java index 00b10c907..4bc0b1f44 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java @@ -269,7 +269,9 @@ public void testLangScript_OrderbyTestNoPartitioning() throws Exception { "select c1 as x, c2 as y from bug2769 group by bug2769.c1, bug2769.c2 order by -(c1 + c2)", new String[][] { {"3","3"},{"3","2"},{"1","2"},{"1","1"} } }, { "drop table obt", null }, { "drop table obt2", null }, - { "create table t (a int, b int, c int)"+ getOffHeapSuffix(), null }, + { "drop table missed", null }, + { "drop table ut", null }, + { "create table t (a int, b int, c int)"+ getOffHeapSuffix(), null }, { "insert into t values (1, 2, null), (2, 3, null), (3, 0, null), (1, 3, null)", null }, { "select * from t order by a", new String[][] { {"1","3",null},{"1","2",null},{"2","3",null},{"3","0",null} } }, @@ -470,7 +472,8 @@ public void testLangScript_OrderbyTestNoPartitioning() throws Exception { "drop table t1", null }, { "drop table t2", null }, { "drop table t3", null }, - { "create table t1 (c1 int, c2 varchar(10))"+ getOffHeapSuffix(), null }, + { "drop table d", null }, + { "create table t1 (c1 int, c2 varchar(10))"+ getOffHeapSuffix(), null }, { "create table t2 (t2c1 int)"+ getOffHeapSuffix(), null }, { "insert into t1 values (3, 'a'), (4, 'c'), (2, 'b'), (1, 'c')", null }, { "insert into t2 values (4), (3)", null }, @@ -802,7 +805,9 @@ public void testLangScript_OrderbyWithPartitioning() throws Exception { "select c1 as x, c2 as y from bug2769 group by bug2769.c1, bug2769.c2 order by -(c1 + c2)", new String[][] { {"3","3"},{"3","2"},{"1","2"},{"1","1"} } }, { "drop table obt", null }, { "drop table obt2", null }, - { "create table t (a int, b int, c int)"+ getOffHeapSuffix(), null }, + { "drop table missed", null }, + { "drop table ut", null }, + { "create table t (a int, b int, c int)"+ getOffHeapSuffix(), null }, { "insert into t values (1, 2, null), (2, 3, null), (3, 0, null), (1, 3, null)", null }, { "select * from t order by a", new String[][] { {"1","3",null},{"1","2",null},{"2","3",null},{"3","0",null} } }, @@ -1003,6 +1008,7 @@ public void testLangScript_OrderbyWithPartitioning() throws Exception { "drop table t1", null }, { "drop table t2", null }, { "drop table t3", null }, + { "drop table d", null }, { "create table t1 (c1 int, c2 varchar(10))"+ getOffHeapSuffix(), null }, { "create table t2 (t2c1 int)"+ getOffHeapSuffix(), null }, { "insert into t1 values (3, 'a'), (4, 'c'), (2, 'b'), (1, 'c')", null }, From 097c9ea7676844f0a687adb92313fc07d6626bcc Mon Sep 17 00:00:00 2001 From: paresh-p11 <43569032+paresh-p11@users.noreply.github.com> Date: Sat, 23 Nov 2019 09:21:24 +0530 Subject: [PATCH 06/27] SNAP-3167 (#533) SNAP-3167 * Adding new property "recovery-state-chunk-size" to handle PersistentStateInRecoveryMode objects transfer in recovery mode. NOTE: part of a work around. Better approach will be implemented in next revision. * Fix debug flag: TraceRecoveryMode --- .../internal/cache/CacheServerLauncher.java | 7 ++++++ .../internal/cache/GemFireCacheImpl.java | 9 ++++++++ .../RecoveryModeResultCollector.java | 3 --- .../PersistentStateInRecoveryMode.java | 7 ++++-- .../distributed/utils/GemFireXDUtils.java | 1 + .../RecoveredMetadataRequestMessage.java | 22 ++++++++++++------- .../internal/engine/store/GemFireStore.java | 7 ++++++ .../tools/internal/GfxdServerLauncher.java | 6 +++++ 8 files changed, 49 insertions(+), 13 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/CacheServerLauncher.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/CacheServerLauncher.java index dc88ea5ef..0b7a494d0 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/CacheServerLauncher.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/CacheServerLauncher.java @@ -266,6 +266,7 @@ protected void restoreStdOut( ) { protected static final String PROPERTIES = "properties"; public static final String REBALANCE = "rebalance"; public static final String RECOVER = "recover"; + public static final String RECOVERY_STATE_CHUNK_SIZE = "recovery-state-chunk-size"; public static final String SERVER_PORT = "server-port"; public static final String SERVER_BIND_ADDRESS = "server-bind-address"; public static final String DISABLE_DEFAULT_SERVER = "disable-default-server"; @@ -337,6 +338,9 @@ else if (arg.startsWith("-lock-memory")) { else if (arg.startsWith("-rebalance")) { options.put(REBALANCE, Boolean.TRUE); } + else if (arg.startsWith("-recovery-state-chunk-size")) { + options.put(RECOVERY_STATE_CHUNK_SIZE, arg.substring(arg.indexOf("=") + 1)); + } else if (arg.startsWith("-recover")) { options.put(RECOVER, Boolean.TRUE); } @@ -469,6 +473,9 @@ else if (arg.startsWith("-dir=")) { else if (arg.startsWith("-rebalance")) { options.put(REBALANCE, Boolean.TRUE); } + else if (arg.startsWith("-recovery-state-chunk-size")) { + options.put(RECOVERY_STATE_CHUNK_SIZE, arg.substring(arg.indexOf("=") + 1)); + } else if (arg.startsWith("-recover")) { options.put(RECOVER, "true"); } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java index 209d72cbd..4fb985521 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java @@ -540,6 +540,8 @@ public class GemFireCacheImpl implements InternalCache, ClientCache, HasCachePer /** indicates whether the snappy system has been booted in recovery mode */ private static boolean snappyRecoverMode; + private static int recoveryStateChunkSize; + private final CacheConfig cacheConfig; // Stores the properties used to initialize declarables. @@ -6123,6 +6125,13 @@ public final boolean isSnappyRecoveryMode() { return snappyRecoverMode; } + public void setRecoveryStateChunkSize(int size) { + recoveryStateChunkSize = size; + } + public int getRecoveryStateChunkSize() { + return recoveryStateChunkSize; + } + public final boolean isGFXDSystem() { return gfxdSystem; } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/RecoveryModeResultCollector.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/RecoveryModeResultCollector.java index 135b39a41..c028f7eb8 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/RecoveryModeResultCollector.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/RecoveryModeResultCollector.java @@ -49,9 +49,6 @@ public GfxdResultCollectorHelper getStreamingHelper() { } public ArrayList getResult() throws FunctionException { - if (GemFireXDUtils.TraceRecoveryMode) { - throw new AssertionError("unexpected Throwable "); - } return this; } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java index 4dc320a8d..b4ca32284 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java @@ -295,6 +295,8 @@ void log(String str) { @Override public int compareTo(RecoveryModePersistentView other) { // They should be called for the same region + log("Comparing RecoveryModePersistentViews :::\n" + + "this view: " + this + "\n other view: " + other); assert this.regionPath.equals(other.regionPath); if (mostRecentEntryModifiedTime == other.mostRecentEntryModifiedTime && latestOplogTime == other.latestOplogTime && @@ -302,7 +304,7 @@ public int compareTo(RecoveryModePersistentView other) { Objects.equals(diskStoreName, other.diskStoreName) && rvv.sameAs(other.rvv) && member.equals(other.member)) { - SanityManager.DEBUG_PRINT("info", "Comparing same object. Doesn't make sense to do this. Please check."); + log("Comparing same object. Doesn't make sense to do this. Please check."); return 0; } if (this.rvv.logicallySameAs(other.rvv) || @@ -365,7 +367,8 @@ public String toString() { @Override public int compareTo(RecoveryModePersistentViewPair other) { // They should be called for the same region - log("Comparing :::\nthis view: " + this + "\n other view: " + other); + log("Comparing RecoveryModePersistentViewPairs :::\n" + + "this view: " + this + "\n other view: " + other); RecoveryModePersistentView thisRowView = this.getRowView(); RecoveryModePersistentView thisColView = this.getColView(); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/utils/GemFireXDUtils.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/utils/GemFireXDUtils.java index 49505eecb..89af3522c 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/utils/GemFireXDUtils.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/utils/GemFireXDUtils.java @@ -3360,6 +3360,7 @@ public static void initFlags() { TraceProcedureExecution = (TraceQuery || DistributionManager.VERBOSE || SanityManager.TRACE_ON(GfxdConstants.TRACE_PROCEDURE_EXEC)) && !SanityManager.TRACE_OFF(GfxdConstants.TRACE_PROCEDURE_EXEC); + TraceRecoveryMode = SanityManager.TRACE_ON(GfxdConstants.TRACE_RECOVERY_MODE); TraceStatementMatching = SanityManager .TRACE_ON(GfxdConstants.TRACE_STATEMENT_MATCHING); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/RecoveredMetadataRequestMessage.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/RecoveredMetadataRequestMessage.java index d662d1b70..584b6712a 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/RecoveredMetadataRequestMessage.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/RecoveredMetadataRequestMessage.java @@ -28,11 +28,6 @@ private enum ListType { REGION_VIEWS } - - private final Integer CATALOG_OBJECTS_SUBLIST_SIZE = 25; - private final Integer OTHER_DDLS_SUBLIST_SIZE = 100; - private final Integer REGION_VIEWS_SUBLIST_SIZE = 30; - /** * Default constructor for deserialization. Not to be invoked directly. */ @@ -64,11 +59,22 @@ public Set getMembers() { } private void sendPersistentStateMsg(PersistentStateInRecoveryMode persistentStateMsg) { + + int ChunkSizeConstant = Misc.getGemFireCache().getRecoveryStateChunkSize() != 0 + ? Misc.getGemFireCache().getRecoveryStateChunkSize() : 30; + + if (GemFireXDUtils.TraceRecoveryMode) { + SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_RECOVERY_MODE,"ChunkSizeConstant:" + ChunkSizeConstant); + } + int CatalogObjectsSublistSize = ChunkSizeConstant*2; + int OtherDDLSSublistSize = ChunkSizeConstant * 4; + int regionViewsSublistSize = ChunkSizeConstant; + RecoveryModeResultHolder.PersistentStateInRMMetadata resultHolderMetadata = new RecoveryModeResultHolder.PersistentStateInRMMetadata(persistentStateMsg.getMember(), persistentStateMsg.getPrToNumBuckets(), persistentStateMsg.getReplicatedRegions(), persistentStateMsg.isServer()); this.sendResult(resultHolderMetadata); - sendList(persistentStateMsg.getCatalogObjects(), CATALOG_OBJECTS_SUBLIST_SIZE, ListType.CATALOG_OBJECTS); - sendList(persistentStateMsg.getOtherDDLs(), OTHER_DDLS_SUBLIST_SIZE, ListType.OTHER_DDLS); - sendList(persistentStateMsg.getAllRegionViews(), REGION_VIEWS_SUBLIST_SIZE, ListType.REGION_VIEWS); + sendList(persistentStateMsg.getCatalogObjects(), CatalogObjectsSublistSize, ListType.CATALOG_OBJECTS); + sendList(persistentStateMsg.getOtherDDLs(), OtherDDLSSublistSize, ListType.OTHER_DDLS); + sendList(persistentStateMsg.getAllRegionViews(), regionViewsSublistSize, ListType.REGION_VIEWS); } public void sendList(ArrayList arrayList, Integer partSize, ListType type) { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java index 7c50eedc7..bd8c20f84 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java @@ -1121,6 +1121,13 @@ else if (!hostData) { this.gemFireCache.getLogger().info( "GemFire Cache has come up in recovery mode."); this.gemFireCache.setRecoverMode(true); + + if (props.containsKey(GfxdConstants.SNAPPY_PREFIX + + CacheServerLauncher.RECOVERY_STATE_CHUNK_SIZE)) { + this.gemFireCache.setRecoveryStateChunkSize(Integer.parseInt(props. + getProperty(GfxdConstants.SNAPPY_PREFIX + + CacheServerLauncher.RECOVERY_STATE_CHUNK_SIZE))); + } } } } catch (CacheExistsException ex) { diff --git a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/tools/internal/GfxdServerLauncher.java b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/tools/internal/GfxdServerLauncher.java index 5a80e3986..bcdf94ee5 100644 --- a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/tools/internal/GfxdServerLauncher.java +++ b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/tools/internal/GfxdServerLauncher.java @@ -265,6 +265,12 @@ public static Properties processProperties(Properties props, if (options != null && options.containsKey(CacheServerLauncher.RECOVER)) { props.put(GfxdConstants.SNAPPY_PREFIX + CacheServerLauncher.RECOVER, "true"); } + if (options != null && + options.containsKey(CacheServerLauncher.RECOVERY_STATE_CHUNK_SIZE)) { + props.put(GfxdConstants.SNAPPY_PREFIX + + CacheServerLauncher.RECOVERY_STATE_CHUNK_SIZE, + options.getOrDefault(CacheServerLauncher.RECOVERY_STATE_CHUNK_SIZE, 30)); + } return props; } From 75ee3656dcddc490bc03b586ce199051753f080e Mon Sep 17 00:00:00 2001 From: ahshahid Date: Fri, 22 Nov 2019 21:31:51 -0800 Subject: [PATCH 07/27] some more fix.. --- .../gemfirexd/jdbc/LangScripts_OrderbyTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java index 4bc0b1f44..4e0dedede 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/jdbc/LangScripts_OrderbyTest.java @@ -271,6 +271,8 @@ public void testLangScript_OrderbyTestNoPartitioning() throws Exception { "drop table obt2", null }, { "drop table missed", null }, { "drop table ut", null }, + { "drop table tab1", null }, + { "drop table bug2769", null }, { "create table t (a int, b int, c int)"+ getOffHeapSuffix(), null }, { "insert into t values (1, 2, null), (2, 3, null), (3, 0, null), (1, 3, null)", null }, { "select * from t order by a", new String[][] { @@ -473,6 +475,11 @@ public void testLangScript_OrderbyTestNoPartitioning() throws Exception { "drop table t2", null }, { "drop table t3", null }, { "drop table d", null }, + { "drop table DERBY147", null }, + { "drop table d2459_A2", null }, + { "drop table d2459_A1", null }, + { "drop table d2459_B2", null }, + { "drop table d2459_B1", null }, { "create table t1 (c1 int, c2 varchar(10))"+ getOffHeapSuffix(), null }, { "create table t2 (t2c1 int)"+ getOffHeapSuffix(), null }, { "insert into t1 values (3, 'a'), (4, 'c'), (2, 'b'), (1, 'c')", null }, @@ -807,6 +814,8 @@ public void testLangScript_OrderbyWithPartitioning() throws Exception { "drop table obt2", null }, { "drop table missed", null }, { "drop table ut", null }, + { "drop table tab1", null }, + { "drop table bug2769", null }, { "create table t (a int, b int, c int)"+ getOffHeapSuffix(), null }, { "insert into t values (1, 2, null), (2, 3, null), (3, 0, null), (1, 3, null)", null }, { "select * from t order by a", new String[][] { @@ -1009,6 +1018,11 @@ public void testLangScript_OrderbyWithPartitioning() throws Exception { "drop table t2", null }, { "drop table t3", null }, { "drop table d", null }, + { "drop table DERBY147", null }, + { "drop table d2459_A2", null }, + { "drop table d2459_A1", null }, + { "drop table d2459_B2", null }, + { "drop table d2459_B1", null }, { "create table t1 (c1 int, c2 varchar(10))"+ getOffHeapSuffix(), null }, { "create table t2 (t2c1 int)"+ getOffHeapSuffix(), null }, { "insert into t1 values (3, 'a'), (4, 'c'), (2, 'b'), (1, 'c')", null }, From 77acfc6494961f22b5d05c6683179b18cd178d62 Mon Sep 17 00:00:00 2001 From: kneeraj Date: Sat, 23 Nov 2019 15:50:03 +0530 Subject: [PATCH 08/27] store side changes for interpreter support. (#526) * store side changes for new snappy-scala interpreter and 'exec scala' sql support. * Changed the internal global metadata region from type to so complex metadata objects can also be stored. * Considers tab for completion only when there is no chraracter after that. --- .../internal/shared/StringPrintWriter.java | 6 +- .../thrift/internal/ClientService.java | 11 +- .../distributed/SnappyResultHolder.java | 93 +++++++++--- .../execution/LeadNodeExecutionObject.java | 1 + .../execution/SQLLeadNodeExecutionObject.java | 5 + .../message/LeadNodeExecutorMsg.java | 24 ++- .../sql/execute/SnappySelectResultSet.java | 2 +- .../internal/engine/store/GemFireStore.java | 4 +- .../internal/impl/sql/GenericStatement.java | 11 +- .../snappy/CallbackFactoryProvider.java | 5 +- .../internal/snappy/ClusterCallbacks.java | 2 + .../internal/snappy/InterpreterExecute.java | 24 +++ .../internal/impl/tools/ij/Main.java | 107 ++++++++------ .../impl/tools/ij/StatementFinder.java | 67 ++++++++- .../internal/impl/tools/ij/utilMain.java | 137 ++++++++++++++---- .../internal/tools/JDBCDisplayUtil.java | 34 ++++- .../gemfirexd/tools/GfxdUtilLauncher.java | 15 ++ 17 files changed, 432 insertions(+), 116 deletions(-) create mode 100644 gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/InterpreterExecute.java diff --git a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/StringPrintWriter.java b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/StringPrintWriter.java index bc248b519..1fc6b5dcb 100644 --- a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/StringPrintWriter.java +++ b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/StringPrintWriter.java @@ -29,7 +29,7 @@ */ public class StringPrintWriter extends PrintWriter { - private final StringBuilder sb; + private StringBuilder sb; private final static Writer dummyLock = new StringWriter(); @@ -223,6 +223,10 @@ public void flush() { // nothing to be done } + public void reset() { + this.sb.setLength(0); + } + @Override public void close() { // nothing to be done diff --git a/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientService.java b/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientService.java index 404e98469..ec9b1d0a7 100644 --- a/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientService.java +++ b/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientService.java @@ -318,10 +318,13 @@ private ClientService(HostAddress hostAddr, OpenConnectionArgs connArgs) throws SnappyException { ClientConfiguration config = ClientConfiguration.getInstance(); - ClientSharedUtils.getLogger(getClass()).info("Starting client on '" + hostName + - "' with ID='" + hostId + "' Source-Revision=" + - config.getSourceRevision()); - + String intpModeProperty = System.getProperty("LAUNCHER_INTERPRETER_MODE"); + boolean isInterpreterMode = intpModeProperty != null && intpModeProperty.equals("true"); + if (!isInterpreterMode) { + ClientSharedUtils.getLogger(getClass()).info("Starting client on '" + hostName + + "' with ID='" + hostId + "' Source-Revision=" + + config.getSourceRevision()); + } this.isClosed = true; this.currentHostConnection = null; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/SnappyResultHolder.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/SnappyResultHolder.java index dc23e1a46..fb66ab156 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/SnappyResultHolder.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/SnappyResultHolder.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.sql.SQLWarning; import java.sql.Types; +import java.util.ArrayList; import java.util.Iterator; import com.gemstone.gemfire.DataSerializer; @@ -61,12 +62,18 @@ public final class SnappyResultHolder extends GfxdDataSerializable { private DataTypeDescriptor[] dtds; private boolean hasMetadata; private boolean isUpdateOrDeleteOrPut; + private boolean interpreterExecution = false; + private String[] intpOutput; public SnappyResultHolder(SparkSQLExecute exec, Boolean isUpdateOrDeleteOrPut) { this.exec = exec; this.isUpdateOrDeleteOrPut = isUpdateOrDeleteOrPut; } + public SnappyResultHolder(String[] interpreterOutputStrs) { + this.intpOutput = interpreterOutputStrs; + this.interpreterExecution = true; + } /** for deserialization */ public SnappyResultHolder() { } @@ -105,16 +112,34 @@ public boolean hasMetadata() { @Override public void toData(final DataOutput out) throws IOException { - this.exec.serializeRows(out, this.hasMetadata); + InternalDataSerializer.writeBoolean(interpreterExecution, out); + if (!interpreterExecution) { + this.exec.serializeRows(out, this.hasMetadata); + } else { + DataSerializer.writeStringArray(intpOutput, out); + } } @Override public void fromData(DataInput in) throws IOException, ClassNotFoundException { - final int numBytes = InternalDataSerializer.readArrayLength(in); - if (numBytes > 0) { - final byte[] rawData = DataSerializer.readByteArray(in, numBytes); - Version v = InternalDataSerializer.getVersionForDataStreamOrNull(in); - fromSerializedData(rawData, numBytes, v); + this.interpreterExecution = DataSerializer.readBoolean(in); + if (!this.interpreterExecution) { + final int numBytes = InternalDataSerializer.readArrayLength(in); + if (numBytes > 0) { + final byte[] rawData = DataSerializer.readByteArray(in, numBytes); + Version v = InternalDataSerializer.getVersionForDataStreamOrNull(in); + fromSerializedData(rawData, numBytes, v); + } + } else { + this.intpOutput = DataSerializer.readStringArray(in); + // just initialize the types assuming one col of type string + colNames = new String[] { "C0" }; + nullability = new boolean[] { true }; + this.dataTypes = new Object[1]; + dtds = new DataTypeDescriptor[ ] { + // Randomly chosen 10000. The lines should not be bigger than this. + DataTypeDescriptor.getBuiltInDataTypeDescriptor(Types.VARCHAR, true, 10000) }; + this.colTypes = new int[] { StoredFormatIds.SQL_VARCHAR_ID }; } } @@ -161,22 +186,24 @@ public final void fromSerializedData(final byte[] rawData, } private void makeTemplateDVDArr() { - if (this.isUpdateOrDeleteOrPut) { - DataValueDescriptor[] dvds = new DataValueDescriptor[1]; - dvds[0] = new SQLInteger(); - dtds = new DataTypeDescriptor[1]; - dtds[0] = DataTypeDescriptor.getBuiltInDataTypeDescriptor(Types.INTEGER, false); - this.templateDVDRow = dvds; - } else { - dtds = new DataTypeDescriptor[colTypes.length]; - DataValueDescriptor[] dvds = new DataValueDescriptor[colTypes.length]; - for (int i = 0; i < colTypes.length; i++) { - int typeId = colTypes[i]; - DataValueDescriptor dvd = getNewNullDVD(typeId, i, dtds, - precisions[i], scales[i]); - dvds[i] = dvd; + if (!this.interpreterExecution) { + if (this.isUpdateOrDeleteOrPut) { + DataValueDescriptor[] dvds = new DataValueDescriptor[1]; + dvds[0] = new SQLInteger(); + dtds = new DataTypeDescriptor[1]; + dtds[0] = DataTypeDescriptor.getBuiltInDataTypeDescriptor(Types.INTEGER, false); + this.templateDVDRow = dvds; + } else { + dtds = new DataTypeDescriptor[colTypes.length]; + DataValueDescriptor[] dvds = new DataValueDescriptor[colTypes.length]; + for (int i = 0; i < colTypes.length; i++) { + int typeId = colTypes[i]; + DataValueDescriptor dvd = getNewNullDVD(typeId, i, dtds, + precisions[i], scales[i]); + dvds[i] = dvd; + } + this.templateDVDRow = dvds; } - this.templateDVDRow = dvds; } } @@ -194,7 +221,10 @@ public void prepareSend(LeadNodeExecutorMsg msg, LeadNodeExecutionObject execObj this.exec.packRows(msg, this, execObject); } - public ExecRow getNextRow() throws IOException, ClassNotFoundException, StandardException { + public ExecRow getNextRow() { + if (this.interpreterExecution) { + return getInterpreterNextRow(); + } final ByteArrayDataInput in = this.dis; if (in != null) { Iterator execRows = this.execRows; @@ -220,6 +250,25 @@ public ExecRow getNextRow() throws IOException, ClassNotFoundException, Standard return null; } + private ExecRow getInterpreterNextRow() { + if (this.intpOutput != null) { + if (this.execRows == null) { + ArrayList list = new ArrayList<>(); + for (int i=0; i< this.intpOutput.length; i++) { + ValueRow vr = new ValueRow(1); + vr.setColumn(1, new SQLVarchar(this.intpOutput[i])); + list.add(vr); + } + this.execRows = list.iterator(); + } + if (this.execRows.hasNext()) { + ValueRow vr = this.execRows.next(); + return vr; + } + } + return null; + } + private DataValueDescriptor getNewNullDVD( int storeType, int colNum, DataTypeDescriptor[] dtds, int precision, int scale) { return getNewNullDVD(storeType, colNum, dtds, precision, scale, nullability[colNum]); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/LeadNodeExecutionObject.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/LeadNodeExecutionObject.java index 2c86f8778..0bf9f8c8d 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/LeadNodeExecutionObject.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/LeadNodeExecutionObject.java @@ -19,4 +19,5 @@ public Version[] getSerializationVersions() { return null; } public abstract String getExceptionString(); + public String getSql() { return null; } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SQLLeadNodeExecutionObject.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SQLLeadNodeExecutionObject.java index ebcca0831..3d2adbba6 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SQLLeadNodeExecutionObject.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SQLLeadNodeExecutionObject.java @@ -204,4 +204,9 @@ public byte getGfxdID() { public String getExceptionString() { return this.sql; } + + @Override + public String getSql() { + return this.sql; + } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/LeadNodeExecutorMsg.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/LeadNodeExecutorMsg.java index 75330bf86..4a02b2058 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/LeadNodeExecutorMsg.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/LeadNodeExecutorMsg.java @@ -23,6 +23,7 @@ import java.sql.SQLException; import java.util.Arrays; import java.util.Set; +import java.util.regex.Matcher; import java.util.regex.Pattern; import com.gemstone.gemfire.DataSerializer; @@ -52,6 +53,7 @@ import com.pivotal.gemfirexd.internal.impl.sql.GenericParameterValueSet; import com.pivotal.gemfirexd.internal.shared.common.reference.SQLState; import com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider; +import com.pivotal.gemfirexd.internal.snappy.InterpreterExecute; import com.pivotal.gemfirexd.internal.snappy.LeadNodeExecutionContext; import com.pivotal.gemfirexd.internal.snappy.SparkSQLExecute; import org.apache.log4j.Logger; @@ -106,10 +108,12 @@ public boolean optimizeForWrite() { @Override protected void execute() throws Exception { + ClassLoader origLoader = Thread.currentThread().getContextClassLoader(); CallbackFactoryProvider.getClusterCallbacks().setLeadClassLoader(); - try { + try { + if (interpreterExecution()) return ; Logger logger = null; if (GemFireXDUtils.TraceQuery) { logger = Logger.getLogger(getClass()); @@ -140,6 +144,24 @@ protected void execute() throws Exception { } } + private boolean interpreterExecution() { + String sql = this.execObject.getSql(); + if (sql != null) { + if (sql.startsWith("exec") || sql.startsWith("EXEC")) { + String user = ctx.getUserName() != null ? ctx.getUserName().toLowerCase() : ctx.getUserName(); + InternalDistributedMember member = this.getSenderForReply(); + final Version v = member.getVersionObject(); + InterpreterExecute intpexec = + CallbackFactoryProvider.getClusterCallbacks().getInterpreterExecution(sql, v, ctx.getConnId()); + String[] results = intpexec.execute(user, this.ctx.getAuthToken()); + SnappyResultHolder srh = new SnappyResultHolder(results); + this.lastResult(srh); + return true; + } + } + return false; + } + private static class SparkExceptionWrapper extends Exception { private static final long serialVersionUID = -4668836542769295434L; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/SnappySelectResultSet.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/SnappySelectResultSet.java index 78531f6e1..3ecaac8a2 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/SnappySelectResultSet.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/SnappySelectResultSet.java @@ -239,7 +239,7 @@ public ResultDescription makeResultDescription() { new ResultColumnDescriptor[colTypes.length], null); for (int i = 0; i < colNames.length; i++) { - String tableName = tableNames[i]; + String tableName = tableNames != null ? tableNames[i] : ""; String schema = ""; String table = ""; if (!tableName.isEmpty()) { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java index bd8c20f84..bee96245e 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java @@ -393,7 +393,7 @@ public final class GemFireStore implements AccessFactory, ModuleControl, public static final ThreadLocal externalCatalogInitThread = new ThreadLocal<>(); - private Region snappyMetadataCmdRgn; + private Region snappyMetadataCmdRgn; /** ************************************************************************* @@ -3093,7 +3093,7 @@ public void setMetadataCmdRgn(Region gcr) { this.snappyMetadataCmdRgn = gcr; } - public Region getMetadataCmdRgn() { + public Region getMetadataCmdRgn() { return this.snappyMetadataCmdRgn; } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/GenericStatement.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/GenericStatement.java index 9b0799dee..6bb9ed659 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/GenericStatement.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/GenericStatement.java @@ -170,8 +170,10 @@ public class GenericStatement private static final Pattern NON_ROUTED_QUERY = Pattern.compile(ROUTED_QUERY_PREFIX + "\\{?\\s*(CALL|EXECUTE)\\s+", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); - - private static ExecutionEngineArbiter engineArbiter = new ExecutionEngineArbiter(); + private static final Pattern GRANT_REVOKE_INTP_PATTERN = + Pattern.compile("^\\s*(GRANT|REVOKE)\\s+(PRIVILEGE)\\s+(EXEC)\\s+(SCALA)\\s+", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + private static ExecutionEngineArbiter engineArbiter = new ExecutionEngineArbiter(); // GemStone changes END /** * Constructor for a Statement given the text of the statement in a String @@ -238,7 +240,10 @@ private GenericPreparedStatement getPreparedStatementForSnappy(boolean commitNes StatementContext statementContext, LanguageConnectionContext lcc, boolean isDDL, boolean checkCancellation, boolean isUpdateOrDeleteOrPut, Throwable cause) throws StandardException { GenericPreparedStatement gps = preparedStmt; - GeneratedClass ac = new SnappyActivationClass(lcc, !isDDL, isPreparedStatement() && !isDDL, + String source = getSource(); + boolean isGrantRevokeIntp = source != null ? GRANT_REVOKE_INTP_PATTERN.matcher(source).find() : false; + GeneratedClass ac = new SnappyActivationClass(lcc, !(isDDL || isGrantRevokeIntp), + isPreparedStatement() && !isDDL && !isGrantRevokeIntp, isUpdateOrDeleteOrPut); gps.setActivationClass(ac); gps.incrementVersionCounter(); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java index 2db1bdb08..579f0db12 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java @@ -61,12 +61,15 @@ public SparkSQLExecute getSQLExecute(String sql, String schema, } @Override + public InterpreterExecute getInterpreterExecution(String sql, Version v, Long connId) { + return null; + } + public SparkSQLExecute getSampleInsertExecute(String baseTable, LeadNodeExecutionContext ctx, Version v, List dvdRows, byte[] serializedDVDs) { return null; } - @Override public void exportData(Long connId, String exportUri, String formatType, String tableNames, Boolean ignoreError){ } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java index 6a9e375a9..67ca7aa54 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java @@ -44,6 +44,8 @@ public interface ClusterCallbacks { SparkSQLExecute getSQLExecute(String sql, String schema, LeadNodeExecutionContext ctx, Version v, boolean isPreparedStatement, boolean isPreparedPhase, ParameterValueSet pvs); + InterpreterExecute getInterpreterExecution(String sql, Version v, Long connId); + SparkSQLExecute getSampleInsertExecute(String baseTable, LeadNodeExecutionContext ctx, Version v, List dvdRows, byte[] serializedDVDs); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/InterpreterExecute.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/InterpreterExecute.java new file mode 100644 index 000000000..7f5433fa0 --- /dev/null +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/InterpreterExecute.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.pivotal.gemfirexd.internal.snappy; + +public interface InterpreterExecute { + + String[] execute(String user, String authToken); + +} diff --git a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/Main.java b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/Main.java index 7f1fed849..fe532f403 100644 --- a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/Main.java +++ b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/Main.java @@ -246,49 +246,70 @@ public void run() { } }); // simple string completion for builtin ij commands - reader.addCompleter(new StringsCompleter(new String[]{ - "PROTOCOL", "protocol", "DRIVER", "driver", - "CONNECT CLIENT", "connect client", - "CONNECT PEER", "connect peer", - "SET CONNECTION", "set connection", - "SHOW CONNECTIONS", "show connections", - "AUTOCOMMIT ON;", "AUTOCOMMIT OFF;", - "autocommit on;", "autocommit off;", - "DISCONNECT", "DISCONNECT CURRENT;", "DISCONNECT ALL;", - "disconnect", "disconnect current;", "disconnect all;", - "SHOW SCHEMAS", "show schemas", - "SHOW TABLES", "show tables", - "SHOW VIEWS", "show views", - "SHOW PROCEDURES IN", "show procedures in", - "SHOW FUNCTIONS", "show functions", - "SHOW INDEXES IN", "SHOW INDEXES FROM", - "show indexes in", "show indexes from", - "SHOW IMPORTEDKEYS IN", "SHOW IMPORTEDKEYS FROM", - "show importedkeys in", "show importedkeys from", - "SHOW MEMBERS;", "show members;", - "DESCRIBE", "describe", "COMMIT;", "commit;", - "ROLLBACK;", "rollback;", "PREPARE", "prepare", - "EXECUTE", "execute", "REMOVE", "remove", "RUN", "run", - "ELAPSEDTIME ON;", "ELAPSEDTIME OFF;", - "elapsedtime on;", "elapsedtime off;", - "MAXIMUMDISPLAYWIDTH", "maximumdisplaywidth", - "MAXIMUMLINEWIDTH", "maximumlinewidth", - "PAGING ON;", "paging on;", "PAGING OFF;", "paging off;", - "ASYNC", "async", "WAIT FOR", "wait for", - "GET", "GET CURSOR", "GET SCROLL INSENSITIVE CURSOR", - "get", "get cursor", "get scroll insensitive cursor", - "NEXT", "next", "FIRST", "first", "LAST", "last", - "PREVIOUS", "previous", "ABSOLUTE", "absolute", - "RELATIVE", "relative", "AFTER LAST", "after last", - "BEFORE FIRST", "before first", "CLOSE", "close", - "GETCURRENTROWNUMBER", "getcurrentrownumber", - "LOCALIZEDDISPLAY ON;", "LOCALIZEDDISPLAY OFF;", - "localizeddisplay on;", "localizeddisplay off;", - "EXIT;", "exit;", "QUIT;", "quit;", "HELP;", "help;", - "EXPLAIN ", "explain ", - "EXPLAINMODE ON;", "explainmode on;", - "EXPLAINMODE OFF;", "explainmode off;", - })); + String intpModeProperty = System.getProperty("LAUNCHER_INTERPRETER_MODE"); + boolean isInterpreterMode = intpModeProperty != null + && intpModeProperty.equals("true") ? true : false; + if (!isInterpreterMode) { + reader.addCompleter(new StringsCompleter(new String[]{ + "PROTOCOL", "protocol", "DRIVER", "driver", + "CONNECT CLIENT", "connect client", + "CONNECT PEER", "connect peer", + "SET CONNECTION", "set connection", + "SHOW CONNECTIONS", "show connections", + "AUTOCOMMIT ON;", "AUTOCOMMIT OFF;", + "autocommit on;", "autocommit off;", + "DISCONNECT", "DISCONNECT CURRENT;", "DISCONNECT ALL;", + "disconnect", "disconnect current;", "disconnect all;", + "SHOW SCHEMAS", "show schemas", + "SHOW TABLES", "show tables", + "SHOW VIEWS", "show views", + "SHOW PROCEDURES IN", "show procedures in", + "SHOW FUNCTIONS", "show functions", + "SHOW INDEXES IN", "SHOW INDEXES FROM", + "show indexes in", "show indexes from", + "SHOW IMPORTEDKEYS IN", "SHOW IMPORTEDKEYS FROM", + "show importedkeys in", "show importedkeys from", + "SHOW MEMBERS;", "show members;", + "DESCRIBE", "describe", "COMMIT;", "commit;", + "ROLLBACK;", "rollback;", "PREPARE", "prepare", + "EXECUTE", "execute", "REMOVE", "remove", "RUN", "run", + "ELAPSEDTIME ON;", "ELAPSEDTIME OFF;", + "elapsedtime on;", "elapsedtime off;", + "MAXIMUMDISPLAYWIDTH", "maximumdisplaywidth", + "MAXIMUMLINEWIDTH", "maximumlinewidth", + "PAGING ON;", "paging on;", "PAGING OFF;", "paging off;", + "ASYNC", "async", "WAIT FOR", "wait for", + "GET", "GET CURSOR", "GET SCROLL INSENSITIVE CURSOR", + "get", "get cursor", "get scroll insensitive cursor", + "NEXT", "next", "FIRST", "first", "LAST", "last", + "PREVIOUS", "previous", "ABSOLUTE", "absolute", + "RELATIVE", "relative", "AFTER LAST", "after last", + "BEFORE FIRST", "before first", "CLOSE", "close", + "GETCURRENTROWNUMBER", "getcurrentrownumber", + "LOCALIZEDDISPLAY ON;", "LOCALIZEDDISPLAY OFF;", + "localizeddisplay on;", "localizeddisplay off;", + "EXIT;", "exit;", "QUIT;", "quit;", "HELP;", "help;", + "EXPLAIN ", "explain ", + "EXPLAINMODE ON;", "explainmode on;", + "EXPLAINMODE OFF;", "explainmode off;" + })); + } else { + reader.addCompleter(new StringsCompleter(new String[]{ + // Adding the scala keywords + "abstract", "case class", "case", "catch", "class", "def", "do", + "else", "extends", "false", "final", "finally", "flatmap", "for", + "forSome", "forEach", "if", "implicit", "import", "lazy", + "match", "map", "new", "null", "object", "override", "package", + "private", "protected", "return", "sealed", "super", "this", "throw", + "trait", "true", "try", "type", "val", "var", "while", "with", "yield", + "snappy", "sc", + + // Adding the commands + ":help", ":reset", ":replay", ":imports", ":implicits", + ":javap", ":require", ":save", ":silent", ":sh", ":type", + ":warnings", ":elapsedtime on", ":quit", ":run", ":maximumdisplaywidth", ":maximumlinewidth" + })); + } // set the default max display width to terminal width int termWidth; if ((maxDisplayWidth == null || maxDisplayWidth.length() == 0) diff --git a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/StatementFinder.java b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/StatementFinder.java index b6fb9673a..3cc9806ee 100644 --- a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/StatementFinder.java +++ b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/StatementFinder.java @@ -42,12 +42,14 @@ Licensed to the Apache Software Foundation (ASF) under one or more import com.pivotal.gemfirexd.internal.iapi.tools.i18n.LocalizedInput; import com.pivotal.gemfirexd.internal.iapi.tools.i18n.LocalizedOutput; +import com.pivotal.gemfirexd.internal.tools.JDBCDisplayUtil; +import com.pivotal.gemfirexd.tools.GfxdUtilLauncher; import com.pivotal.gemfirexd.tools.internal.MiscTools; import jline.console.ConsoleReader; import java.io.IOException; import java.io.Reader; -import java.util.Iterator; +import java.util.ArrayList; import java.util.Map; /** @@ -142,6 +144,9 @@ public StatementFinder(LocalizedInput s, LocalizedOutput promptDest, String basePath, Map params) { // GemStone changes BEGIN this.consoleReader = (ConsoleReader)s.getConsoleReader(); + if (JDBCDisplayUtil.INTERPRETER_MODE) { + this.consoleReader.setCopyPasteDetection(true); + } this.basePath = basePath; this.params = params; // GemStone changes END @@ -172,6 +177,9 @@ public void ReInit(LocalizedInput s) { } // GemStone changes BEGIN this.consoleReader = (ConsoleReader)s.getConsoleReader(); + if (JDBCDisplayUtil.INTERPRETER_MODE) { + this.consoleReader.setCopyPasteDetection(true); + } // GemStone changes END source = s; state = IN_STATEMENT; @@ -189,6 +197,27 @@ public void close() throws IOException { source.close(); } + public static String preProcessIntpLine(String origline) { + if (origline != null) { + String trimmedOrignalLC = origline.trim().toLowerCase(); + for (String cmd : commandsToStripColonPrefix) { + String colonCommand = ":" + cmd; + if (trimmedOrignalLC.startsWith(colonCommand) && (cmd.equals("run") || + cmd.equals("maximumlinewidth") || cmd.equals("maximumdisplaywidth"))) { + return trimmedOrignalLC.substring(1); + } + if (!trimmedOrignalLC.isEmpty() && colonCommand.startsWith(trimmedOrignalLC)) { + return cmd; + } + } + + } + return origline; + } + + // User will give as :command but these will be executed locally + public final static String[] commandsToStripColonPrefix = + new String[] {"elapsedtime on", "quit", "run", "maximumdisplaywidth", "maximumlinewidth"}; /** get the next statement in the input stream. Returns it, dropping its closing semicolon if it has one. If there is @@ -197,25 +226,53 @@ public void close() throws IOException { @return the next statement in the input stream. */ // GemStone changes BEGIN + String cachedPrompt = null; + public String nextStatement(final ConnectionEnv connEnv, final LocalizedOutput out) { /* (original code) public String nextStatement() { */ + String prompt = null; + if (JDBCDisplayUtil.lastWasIncomplete) { + prompt = utilMain.getIncompletePrompt(); + JDBCDisplayUtil.lastWasIncomplete = false; + } else { + prompt = cachedPrompt; + if (prompt == null && connEnv != null) { + prompt = connEnv.getPrompt(true); + cachedPrompt = prompt; + } + } + if (JDBCDisplayUtil.INTERPRETER_MODE && GfxdUtilLauncher.connectStr != null && doPrompt) { + String connStr = GfxdUtilLauncher.connectStr; + GfxdUtilLauncher.connectStr = null; + return connStr; + } + ArrayList initialFiles = GfxdUtilLauncher.initialRunFiles; + if (JDBCDisplayUtil.INTERPRETER_MODE && initialFiles != null && !initialFiles.isEmpty() && doPrompt) { + String runfile = initialFiles.get(0); + initialFiles.remove(runfile); + if (initialFiles.isEmpty()) { + GfxdUtilLauncher.initialRunFiles = null; + } + System.out.println("\n\n***** Running input file " + runfile + " *****\n"); + return "run '" + runfile + "'"; + } if (this.consoleReader != null) { this.statement.setLength(0); if (this.state == END_OF_INPUT) { return null; } - String prompt = null; - if (connEnv != null) { - prompt = connEnv.getPrompt(true); - } + String line = null; OUTER: for (;;) { try { line = this.consoleReader.readLine(prompt); + if (JDBCDisplayUtil.INTERPRETER_MODE) { + return line; + } } catch (java.io.IOException ioe) { throw ijException.iOException(ioe); } diff --git a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/utilMain.java b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/utilMain.java index 9f65ce9b0..f6859fb98 100644 --- a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/utilMain.java +++ b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/utilMain.java @@ -50,14 +50,10 @@ Licensed to the Apache Software Foundation (ASF) under one or more import com.pivotal.gemfirexd.internal.shared.common.StopWatch; import com.pivotal.gemfirexd.internal.shared.common.sanity.SanityManager; import com.pivotal.gemfirexd.internal.tools.JDBCDisplayUtil; +import com.pivotal.gemfirexd.tools.GfxdUtilLauncher; import jline.console.ConsoleReader; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Stack; -import java.util.Hashtable; -import java.util.Properties; +import java.util.*; import java.io.File; import java.io.InputStream; import java.io.FileInputStream; @@ -129,8 +125,21 @@ public class utilMain implements java.security.PrivilegedAction { static String basePrompt = "gfxd"; + private static String incompleteprompt = null; + public static String getIncompletePrompt() { + if (incompleteprompt != null) return incompleteprompt; + StringBuilder sb = new StringBuilder(); + int basePromptLength = basePrompt.length(); + for (int i=0; i= 3 && prompt.trim().length() <= 10) { + if (prompt != null && prompt.trim().length() >= 3 && prompt.trim().length() <= 15) { basePrompt = prompt.trim(); } } @@ -227,6 +236,12 @@ public void initFromEnvironment() } + private static String INTERPRETER_PREFIX = "exec scala "; + private String getInterpreterPrefixCmdString(String command) { + return INTERPRETER_PREFIX + command; + } + + private boolean isInterpreterMode = false; /** * run ij over the specified input, sending output to the * specified output. Any prior input and output will be lost. @@ -238,6 +253,10 @@ public void initFromEnvironment() public void go(LocalizedInput[] in, LocalizedOutput out, Properties connAttributeDefaults) throws ijFatalException { + String intpModeProperty = System.getProperty("LAUNCHER_INTERPRETER_MODE"); + isInterpreterMode = intpModeProperty != null && intpModeProperty.equals("true"); + if (isInterpreterMode) JDBCDisplayUtil.INTERPRETER_MODE = true; + this.out = out; this.connAttributeDefaults = connAttributeDefaults; @@ -270,9 +289,11 @@ public void go(LocalizedInput[] in, LocalizedOutput out, version = "?"; } */ - out.println(convertGfxdMessageToSnappy( - langUtil.getTextMessage("IJ_IjVers30C199", GemFireVersion.getProductVersion() + " " + - GemFireVersion.getProductReleaseStage()))); + if (!isInterpreterMode) { + out.println(convertGfxdMessageToSnappy( + langUtil.getTextMessage("IJ_IjVers30C199", GemFireVersion.getProductVersion() + " " + + GemFireVersion.getProductReleaseStage()))); + } // GemStone changes END for (int i = connEnv.length - 1; i >= 0; i--) { // print out any initial warnings... Connection c = connEnv[i].getConnection(); @@ -310,6 +331,15 @@ public void go(LocalizedInput[] in, LocalizedOutput out, cleanupGo(in); } + private boolean isLocallyAllowedCommands(String command) { + for (String cmd : StatementFinder.commandsToStripColonPrefix) { + if (command != null) { + String lcCommand = command.toLowerCase(); + if (lcCommand.startsWith(cmd)) return true; + } + } + return false; + } /** * Support to run a script. Performs minimal setup * to set the passed in connection into the existing @@ -390,9 +420,12 @@ private int runScriptGuts() { command = commandGrabber[currCE].nextStatement(); */ // GemStone changes END - // if there is no next statement, // pop back to the top saved grabber. + String originalCommand = command; + if (JDBCDisplayUtil.INTERPRETER_MODE && !JDBCDisplayUtil.INITIAL_CMD_IN_PROGRESS) { + command = StatementFinder.preProcessIntpLine(command); + } while (command == null && ! oldGrabbers.empty()) { // close the old input file if not System.in if (fileInput) commandGrabber[currCE].close(); @@ -406,7 +439,12 @@ private int runScriptGuts() { command = commandGrabber[currCE].nextStatement(); */ // GemStone changes END + originalCommand = command; + if (isInterpreterMode && !JDBCDisplayUtil.INITIAL_CMD_IN_PROGRESS) { + command = StatementFinder.preProcessIntpLine(command); + } } + if (isInterpreterMode && command != null && command.isEmpty()) continue; // if there are no grabbers left, // we are done. @@ -416,7 +454,7 @@ private int runScriptGuts() { else { boolean elapsedTimeOn = ijParser.getElapsedTimeState(); long beginTime = 0; - long endTime; + long endTime = 0; if (fileInput) { out.println(command+";"); @@ -439,10 +477,27 @@ private int runScriptGuts() { beginTime = System.currentTimeMillis(); } - ijResult result = ijParser.ijStatement(); - endTime = displayResult(out,result,connEnv[currCE].getConnection(), - beginTime /* GemStoneAddition */, true); - + boolean gotException = true; + ijResult result = ijParser.ijStatement(); + // If this is interpreter mode then if it parses successfully then it should be just + // the allowed commands in INTERPRETER MODE + if (this.isInterpreterMode && !JDBCDisplayUtil.INITIAL_CMD_IN_PROGRESS) { + if (!isLocallyAllowedCommands(command) || !originalCommand.startsWith(":")) { + // throw dummy Parse Exception so that it is carried to actual interpreter + throw new ParseException(); + } + } + ArrayList initialFiles = GfxdUtilLauncher.initialRunFiles; + + if (isInterpreterMode && (initialFiles == null || initialFiles.isEmpty())) { + JDBCDisplayUtil.INITIAL_CMD_IN_PROGRESS = false; + GfxdUtilLauncher.initialRunFiles = null; + } + if (this.isInterpreterMode && !JDBCDisplayUtil.BEFORE_CONNECT) { + endTime = displayResult(out, result, connEnv[currCE].getConnection(), + beginTime /* GemStoneAddition */, true); + } + JDBCDisplayUtil.BEFORE_CONNECT = false; // if something went wrong, an SQLException or ijException was thrown. // we can keep going to the next statement on those (see catches below). // ijParseException means we try the SQL parser. @@ -484,10 +539,23 @@ private int runScriptGuts() { // unless it is considered fatal. handleSQLException(out,e); // GemStone changes BEGIN - if (!"08001".equals(e.getSQLState()) - && !"08004".equals(e.getSQLState())) { - printConnectUsage(command, out); + if (!JDBCDisplayUtil.INTERPRETER_MODE) { + if (!"08001".equals(e.getSQLState()) + && !"08004".equals(e.getSQLState())) { + printConnectUsage(command, out); + } } + String sqlstate = e.getSQLState(); + if (isInterpreterMode && ("40XD0".equalsIgnoreCase(sqlstate) || + "08004".equalsIgnoreCase(sqlstate))) { + System.out.println("\nNot connected to cluster. Exiting..."); + System.exit(1); + } + // Actually exit for anything when the state is before connect + if (JDBCDisplayUtil.BEFORE_CONNECT) { + System.out.println("\nNot connected to cluster. Exiting..." + + e.getSQLState() + " " + e.getMessage()); + } // GemStone changes END } catch (ijException e) { scriptErrorCount++; @@ -746,7 +814,9 @@ private boolean doCatch(String command) { boolean reportRunNum = false; String firstToken = ClientSharedUtils.getStatementToken(command, 0); // final String c = command.trim().toLowerCase(); - if (firstToken.equalsIgnoreCase("set") || firstToken.equalsIgnoreCase("elapsed")) { + if (firstToken != null && ( + firstToken.equalsIgnoreCase("set") || ( + !this.isInterpreterMode && firstToken.equalsIgnoreCase("elapsed")))) { repeatCommand = 1; } else if (repeatCommand > 1) { reportRunNum = true; @@ -763,6 +833,11 @@ private boolean doCatch(String command) { beginTime = System.currentTimeMillis(); } + // Here is where the command is sent to server + // so just prepend the intp prefix if in interpreter mode + if (isInterpreterMode) { + command = getInterpreterPrefixCmdString(command); + } ijResult result = ijParser.executeImmediate(command); if (ijParser.getExplainMode() && command.startsWith("explain ")) { redirected = RedirectedLocalizedOutput.getNewInstance(); @@ -772,10 +847,12 @@ private boolean doCatch(String command) { } // GemStone changes BEGIN boolean displayCount = false; - if (firstToken.equalsIgnoreCase("insert") || firstToken.equalsIgnoreCase("update") - || firstToken.equalsIgnoreCase("delete") || firstToken.equalsIgnoreCase("put") ) { - displayCount = true; - } + if (!this.isInterpreterMode) { + if (firstToken.equalsIgnoreCase("insert") || firstToken.equalsIgnoreCase("update") + || firstToken.equalsIgnoreCase("delete") || firstToken.equalsIgnoreCase("put")) { + displayCount = true; + } + } // GemStone changes END endTime = displayResult(redirected == null ? out : redirected,result,connEnv[currCE].getConnection(), @@ -808,6 +885,9 @@ private boolean doCatch(String command) { // SQL exception occurred in ij's actions; print and continue // unless it is considered fatal. handleSQLException(out,e); + if ("40XD0".equalsIgnoreCase(e.getSQLState()) && JDBCDisplayUtil.INTERPRETER_MODE) { + System.exit(1); + } } catch (ijException i) { // GemStone changes BEGIN out.println(langUtil.getTextMessage("IJ_IjErro0_5", @@ -968,7 +1048,7 @@ static void doPrompt(boolean newStatement, LocalizedOutput out, String tag) { if (newStatement) { // GemStone changes BEGIN - out.print(basePrompt + (tag == null ? "" : tag) + "> "); + out.print(basePrompt + (tag == null ? "" : tag) + "> "); /* (original code) out.print("ij"+(tag==null?"":tag)+"> "); */ @@ -983,7 +1063,12 @@ static void doPrompt(boolean newStatement, LocalizedOutput out, String tag) // GemStone changes BEGIN static String getPrompt(boolean newStatement, String tag) { if (newStatement) { - return basePrompt + tag + "> "; + if (JDBCDisplayUtil.lastWasIncomplete) { + JDBCDisplayUtil.lastWasIncomplete = false; + return getIncompletePrompt(); + } else { + return basePrompt + tag + "> "; + } } return "> "; } diff --git a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/tools/JDBCDisplayUtil.java b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/tools/JDBCDisplayUtil.java index f749a485e..e979d4864 100644 --- a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/tools/JDBCDisplayUtil.java +++ b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/tools/JDBCDisplayUtil.java @@ -83,7 +83,10 @@ public class JDBCDisplayUtil { static private int maxWidth = 128; static public boolean showSelectCount = false; static public boolean showSelectRows = true; //GemStone Addition - + static public boolean INTERPRETER_MODE = false; + static public boolean BEFORE_CONNECT = true; + static public boolean INITIAL_CMD_IN_PROGRESS = true; + static public boolean lastWasIncomplete = false; static { // initialize the locale support functions to default value of JVM LocalizedResource.getInstance(); @@ -434,8 +437,11 @@ static public void DisplayResults(PrintWriter out, ResultSet rs, Connection conn } try { // GemStone changes END - int len = indent_DisplayBanner(out,rsmd, indentLevel, displayColumns, - displayColumnWidths); + int len = 0; + if (!INTERPRETER_MODE) { + len = indent_DisplayBanner(out, rsmd, indentLevel, displayColumns, + displayColumnWidths); + } // When displaying rows, keep going past errors // unless/until the maximum # of errors is reached. @@ -473,7 +479,8 @@ static public void DisplayResults(PrintWriter out, ResultSet rs, Connection conn } } } - if (showSelectCount == true) { + boolean showSelectOutput = showSelectCount && !INTERPRETER_MODE; + if (showSelectOutput == true) { if (numberOfRowsSelected == 1) { out.println(); indentedPrintLine(out, indentLevel, @@ -485,6 +492,7 @@ static public void DisplayResults(PrintWriter out, ResultSet rs, Connection conn LocalizedResource.getNumber(numberOfRowsSelected))); } } + if (INTERPRETER_MODE && !lastWasIncomplete) out.println(); DisplayNestedResults(out, nestedResults, conn, indentLevel, reader /* GemStoneAddition */, timer /* GemStoneAddition */); @@ -782,6 +790,15 @@ else if (s.length() > w) { String[] row = null; if (currentResults != null) { row = currentResults.removeFirst(); + if (INTERPRETER_MODE) { + if (row != null && row[0].equalsIgnoreCase("___INCOMPLETE___")) { + lastWasIncomplete = true; + if (currentResults.size() == 0) { + currentResults = null; + } + return 0; + } + } if (currentResults.size() == 0) { currentResults = null; } @@ -1108,8 +1125,11 @@ static public void DisplayResults(PrintStream out, ResultSet rs, Connection conn if(displayColumnWidths == null) displayColumnWidths = getColumnDisplayWidths(rsmd, displayColumns, false); - int len = indent_DisplayBanner(out,rsmd, indentLevel, displayColumns, - displayColumnWidths); + int len = 0; + if (!INTERPRETER_MODE) { + len = indent_DisplayBanner(out, rsmd, indentLevel, displayColumns, + displayColumnWidths); + } // When displaying rows, keep going past errors // unless/until the maximum # of errors is reached. @@ -1135,7 +1155,7 @@ static public void DisplayResults(PrintStream out, ResultSet rs, Connection conn ShowSQLException(out, e); } } - if (showSelectCount == true) { + if (showSelectCount == true && !INTERPRETER_MODE) { if (numberOfRowsSelected == 1) { out.println(); indentedPrintLine( out, indentLevel, "1 row selected"); diff --git a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/tools/GfxdUtilLauncher.java b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/tools/GfxdUtilLauncher.java index 07ccd3230..353ba3275 100644 --- a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/tools/GfxdUtilLauncher.java +++ b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/tools/GfxdUtilLauncher.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.Map; import java.util.TreeSet; @@ -192,6 +193,20 @@ else if (args.length == 2 && GET_CANONICAL_PATH_ARG.equals(args[0])) { } } + public static ArrayList initialRunFiles = new ArrayList<>(); + + public static String connectStr = null; + + public void setInitialCommands(String connectionString, String commaSeparatedFiles) { + connectStr = connectionString; + if (commaSeparatedFiles != null) { + String[] farr = commaSeparatedFiles.split(","); + for (String f : farr) { + initialRunFiles.add(f); + } + } + } + public static ConsoleReader getConsoleReader() { // don't try to do paging if we have no associated console if (System.console() != null) { From 4927560e6fed037cbf8fb226f4ede5e4732662c6 Mon Sep 17 00:00:00 2001 From: Neeraj Kumar Date: Sat, 23 Nov 2019 16:16:13 +0530 Subject: [PATCH 09/27] Minor fix to run scala files. --- .../gemfirexd/internal/impl/tools/ij/StatementFinder.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/StatementFinder.java b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/StatementFinder.java index 3cc9806ee..3362d4692 100644 --- a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/StatementFinder.java +++ b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/StatementFinder.java @@ -144,8 +144,8 @@ public StatementFinder(LocalizedInput s, LocalizedOutput promptDest, String basePath, Map params) { // GemStone changes BEGIN this.consoleReader = (ConsoleReader)s.getConsoleReader(); - if (JDBCDisplayUtil.INTERPRETER_MODE) { - this.consoleReader.setCopyPasteDetection(true); + if (JDBCDisplayUtil.INTERPRETER_MODE && this.consoleReader != null) { + this.consoleReader.setCopyPasteDetection(true); } this.basePath = basePath; this.params = params; @@ -177,7 +177,7 @@ public void ReInit(LocalizedInput s) { } // GemStone changes BEGIN this.consoleReader = (ConsoleReader)s.getConsoleReader(); - if (JDBCDisplayUtil.INTERPRETER_MODE) { + if (JDBCDisplayUtil.INTERPRETER_MODE && this.consoleReader != null) { this.consoleReader.setCopyPasteDetection(true); } // GemStone changes END From f1bc528b27502081947889ee5d0f4956636e7e8b Mon Sep 17 00:00:00 2001 From: Subodh Chiwate <45190931+schiwate@users.noreply.github.com> Date: Thu, 28 Nov 2019 11:47:58 +0530 Subject: [PATCH 10/27] Snap 3223 Adding class com.snappydata.jdbc.ClientDriver (#534) Allowing to executing a query using driver class com.snappydata.jdbc.ClientDriver. --- .../com/snappydata/jdbc/ClientDriver.java | 26 +++++++++++++++++++ .../sanctionedDataSerializables.txt | 4 +-- 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 gemfirexd/client/src/main/java/com/snappydata/jdbc/ClientDriver.java diff --git a/gemfirexd/client/src/main/java/com/snappydata/jdbc/ClientDriver.java b/gemfirexd/client/src/main/java/com/snappydata/jdbc/ClientDriver.java new file mode 100644 index 000000000..e1cff2070 --- /dev/null +++ b/gemfirexd/client/src/main/java/com/snappydata/jdbc/ClientDriver.java @@ -0,0 +1,26 @@ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package com.snappydata.jdbc; + +/** + * This allows using com.snappydata.jdbc.ClientDriver instead of + * {@link io.snappydata.jdbc.ClientDriver} + */ +public class ClientDriver extends io.snappydata.jdbc.ClientDriver { +} diff --git a/gemfirexd/tools/src/test/resources/codeAnalysis/sanctionedDataSerializables.txt b/gemfirexd/tools/src/test/resources/codeAnalysis/sanctionedDataSerializables.txt index 977127d5d..e02eb0a19 100644 --- a/gemfirexd/tools/src/test/resources/codeAnalysis/sanctionedDataSerializables.txt +++ b/gemfirexd/tools/src/test/resources/codeAnalysis/sanctionedDataSerializables.txt @@ -181,8 +181,8 @@ fromData,279,2a2bb70035b20036b200378099000a12381239b800342bb9003a01003d1c931010b toData,460,2a2bb700c62ab4000b4db20036b200378099002d1238bb002e59b7002f12c7b600312ab40059b600c812c9b600312ab40003b80049b60031b60033b800342ab4001299000c14000db8005ea7000409422b2ab40003b900ca02002ab4003dc6000b2ab4003d2bb600cb2ab40007c700ee03360514000d37062ab4000f14000d9499000c2ab40019b6005c37062ab40053c600232ab40053b6009a360515052bb8008c2ab400532bb600cc2a1505b50051a7000915052bb8008c2ab4000f14000d9499002db200269a0014160614000d949a000bbb002759b70028bf2ab400191606b600cd2ab40019150585b600ceb20036b200378099002a1238bb002e59b7002f12cfb600312ab40098b6004b12d0b6003115050860b6004bb60033b80034b2006fb200d17e9900322ab40053c6002b2ab40053b600d21270bb002e59b7002f12d3b600312ab40053b600d4b80072b60031b60033b80034a700122ab400072bb800d5b800d62bb800d52a59b4009921099499000a21b8005ea700040961b500992ab4001299001f2ab4009909949a00162ab4006109949a000d2a59b400610a61b500612cc600262c2ab400172ab40015b900d703002c2ab400172ab400152ab40015b60068b900d80400b1 com/pivotal/gemfirexd/internal/engine/distributed/SnappyResultHolder,2 -fromData,30,2bb800113d1c9e00172b1cb800124e2bb800133a042a2d1c1904b60014b1 -toData,15,2ab400022b2ab4000fb900100300b1 +fromData,123,2a2bb80016b60004b500022ab400029a00232bb800173d1c9e00172b1cb800184e2bb800193a042a2d1c1904b6001aa7004b2a2bb8001bb500062a04bd001c5903121d53b500092a04bc0459030454b5000a2a04bd001eb5000e2a04bd001f5903100c04112710b8002153b500222a04bc0a590310554fb5000bb1 +toData,44,2ab40002b800122bb800132ab400029a00142ab400032b2ab40011b900140300a7000b2ab400062bb80015b1 com/pivotal/gemfirexd/internal/engine/distributed/StatementCloseExecutorMessage,2 fromData,22,2a2bb700212a2bb80022b500052a2bb80022b50006b1 From 4636546a4467c1cdde9dc0a3f0c15c9d42c146bc Mon Sep 17 00:00:00 2001 From: kneeraj Date: Thu, 28 Nov 2019 17:31:12 +0530 Subject: [PATCH 11/27] Store side support for returning df content in resultset if 'exec scala' (#537) specifies in option. --- .../execution/LeadNodeExecutionObject.java | 3 ++- .../execution/SQLLeadNodeExecutionObject.java | 5 +++-- .../SampleInsertExecutionObject.java | 4 ++-- .../message/LeadNodeExecutorMsg.java | 19 ++++++++++++++----- .../snappy/CallbackFactoryProvider.java | 2 +- .../internal/snappy/ClusterCallbacks.java | 2 +- .../internal/snappy/InterpreterExecute.java | 2 +- 7 files changed, 24 insertions(+), 13 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/LeadNodeExecutionObject.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/LeadNodeExecutionObject.java index 0bf9f8c8d..60876fc21 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/LeadNodeExecutionObject.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/LeadNodeExecutionObject.java @@ -7,7 +7,8 @@ import com.pivotal.gemfirexd.internal.snappy.SparkSQLExecute; public abstract class LeadNodeExecutionObject implements GfxdSerializable { - public abstract SparkSQLExecute getSparkSQlExecute(Version v, LeadNodeExecutionContext ctx) throws Exception; + public abstract SparkSQLExecute getSparkSQlExecute(Version v, + LeadNodeExecutionContext ctx, Object dfObject) throws Exception; public abstract boolean isUpdateOrDeleteOrPut(); public abstract void reset(); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SQLLeadNodeExecutionObject.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SQLLeadNodeExecutionObject.java index 3d2adbba6..f320e71b0 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SQLLeadNodeExecutionObject.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SQLLeadNodeExecutionObject.java @@ -57,11 +57,12 @@ public SQLLeadNodeExecutionObject(String sql, String schema, */ public SQLLeadNodeExecutionObject() {} - public SparkSQLExecute getSparkSQlExecute(Version v, LeadNodeExecutionContext ctx) throws Exception { + public SparkSQLExecute getSparkSQlExecute(Version v, + LeadNodeExecutionContext ctx, Object dfObject) throws Exception { if (isPreparedStatement() && !isPreparedPhase()) { getParams(); } - return CallbackFactoryProvider.getClusterCallbacks().getSQLExecute( + return CallbackFactoryProvider.getClusterCallbacks().getSQLExecute(dfObject, sql, schema, ctx, v, this.isPreparedStatement(), this.isPreparedPhase(), this.pvs); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SampleInsertExecutionObject.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SampleInsertExecutionObject.java index ec1ef1824..af01ce8f7 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SampleInsertExecutionObject.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/execution/SampleInsertExecutionObject.java @@ -34,8 +34,8 @@ public SampleInsertExecutionObject(String baseTableName, List Date: Fri, 29 Nov 2019 23:59:49 +0530 Subject: [PATCH 12/27] [SNAP-3235] (#538) [SNAP-3235] * Giving EntryModifiedTime higher preference than OplogFileTime while choosing between two servers(for a bucket) in recovery mode. --- .../internal/engine/db/FabricDatabase.java | 16 +-- .../PersistentStateInRecoveryMode.java | 102 ++++++++++-------- 2 files changed, 65 insertions(+), 53 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java index 95657a232..6cd73b3b8 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java @@ -1685,8 +1685,8 @@ private void preparePersistentStatesMsg( PersistentStateInRecoveryMode pmsg = new PersistentStateInRecoveryMode(allEntries, otherExtractedDDLs); for (DiskStoreImpl ds : diskStores) { - if (logger.infoEnabled()) { - logger.info("preparePersistentStatesMsg: disk store = " + ds.getName()); + if (logger.fineEnabled()) { + logger.fine("preparePersistentStatesMsg: disk store = " + ds.getName()); } long latestOplogTime = ds.getLatestModifiedTime(); Map drs = ds.getAllDiskRegions(); @@ -1697,18 +1697,18 @@ private void preparePersistentStatesMsg( Map.Entry elem = iter.next(); AbstractDiskRegion adr = elem.getValue(); if (adr.getRecoveredEntryMap() == null) { - if (logger.infoEnabled()) { - logger.info("preparePersistentStatesMsg: adr = " + adr.getFullPath() + " continuing as map is null"); + if (logger.fineEnabled()) { + logger.fine("preparePersistentStatesMsg: adr = " + adr.getFullPath() + " continuing as map is null"); } continue; } if (adr.getRecoveredEntryMap().size() == 0) { - if (logger.infoEnabled()) { - logger.info("preparePersistentStatesMsg: adr = " + adr.getFullPath() + " continuing as size of map is 0"); + if (logger.fineEnabled()) { + logger.fine("preparePersistentStatesMsg: adr = " + adr.getFullPath() + " continuing as size of map is 0"); } } else { - if (logger.infoEnabled()) { - logger.info("preparePersistentStatesMsg: adr = " + adr.getFullPath() + " size = " + adr.getRecoveredEntryMap().size()); + if (logger.fineEnabled()) { + logger.fine("preparePersistentStatesMsg: adr = " + adr.getFullPath() + " size = " + adr.getRecoveredEntryMap().size()); } } long mostRecentModifiedTime = 0; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java index b4ca32284..3f1534fea 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/PersistentStateInRecoveryMode.java @@ -309,18 +309,24 @@ public int compareTo(RecoveryModePersistentView other) { } if (this.rvv.logicallySameAs(other.rvv) || (!this.rvv.dominates(other.rvv) && !other.rvv.dominates(this.rvv))) { + log("Both dominate each other?" + this.rvv.logicallySameAs(other.rvv)); + log("Both don't dominate each other?" + + (!this.rvv.dominates(other.rvv) && !other.rvv.dominates(this.rvv))); log("RVV based approach is not usable."); - if (this.latestOplogTime <= other.latestOplogTime) { - SanityManager.DEBUG_PRINT("info", " 1. this.latestOplogTime <= other.latestOplogTime"); - log("Deciding on basis of Oplog file time: return -1"); - return -1; - } + if (this.mostRecentEntryModifiedTime < other.mostRecentEntryModifiedTime) { // replacing "<=" with "<" as former makes it non-commutative log("Deciding on basis of Modified entry time: return -1"); return -1; + } else if (this.mostRecentEntryModifiedTime > other.mostRecentEntryModifiedTime) { + // replacing "<=" with "<" as former makes it non-commutative + log("Deciding on basis of Modified entry time: return 1"); + return 1; + } else if (this.latestOplogTime <= other.latestOplogTime) { + log("Deciding on basis of Oplog file time: return -1"); + return -1; } - log("Oplog file time AND Modified entry time are greater for LHS: return 1"); + log("Deciding on basis of Oplog file time: return 1"); return 1; } else { log("Use RVV based approach."); @@ -390,7 +396,7 @@ public int compareTo(RecoveryModePersistentViewPair other) { thisColView.member.equals(otherColView.member) || (thisColView == null && otherColView == null)) ) { - SanityManager.DEBUG_PRINT("info", "Comparing same object. Doesn't make sense to do this. Please check."); + log("Comparing same object. Doesn't make sense to do this. Please check debug logs."); return 0; } @@ -414,54 +420,53 @@ private int getCompareValue( // row tables as col regs are absent int retVal = (new RecoveryModePersistentView(thisRowView).compareTo(otherRowView)); log("Row table: return value = " + retVal); - int retVal1 = retVal; - return retVal1; + return retVal; } else { - // col tables - boolean rowAndColRVVLogicallySame = - thisRowView.rvv.logicallySameAs(otherRowView.rvv) && - (thisColView.rvv.logicallySameAs(otherColView.rvv)); - boolean rowAndColLatestOplogTimeSame = - thisColView.latestOplogTime == otherColView.latestOplogTime && - thisRowView.latestOplogTime == otherRowView.latestOplogTime; + // column tables boolean otherRowDominates = otherRowView.rvv.dominates(thisRowView.rvv); boolean otherColDominates = otherColView.rvv.dominates(thisColView.rvv); boolean thisRowDominates = thisRowView.rvv.dominates(otherRowView.rvv); boolean thisColDominates = thisColView.rvv.dominates(otherColView.rvv); + log("thisRowView Dominates otherRowView: " + thisRowView.rvv.dominates(otherRowView.rvv)); + log("otherRowView Dominates thisRowView: " + otherRowView.rvv.dominates(thisRowView.rvv)); + log("thisColView Dominates otherColView: " + thisColView.rvv.dominates(otherColView.rvv)); + log("otherColView Dominates thisColView: " + otherColView.rvv.dominates(thisColView.rvv)); -// t, t | t, t -// t, t | f, f -// f, f | t, t -// f, f | f, f -// t, f | f, t -// f, t | t, f +// Convention: (thisRowDominates, otherRowDominates) | (thisColDominates, otherColDominates) +// ROW | COLUMN +// -------------- +// (t, t) | (t, t) +// (t, t) | (f, f) +// (f, f) | (t, t) +// (f, f) | (f, f) +// (t, f) | (f, t) +// (f, t) | (t, f) boolean isRVVNotUsable = ((thisRowDominates == otherRowDominates && otherColDominates == thisColDominates) || (thisRowDominates != otherRowDominates && otherColDominates != thisColDominates)); - log("thisRowView Dominates otherRowView: " + thisRowView.rvv.dominates(otherRowView.rvv)); - log("otherRowView Dominates thisRowView: " + otherRowView.rvv.dominates(thisRowView.rvv)); - log("thisColView Dominates otherColView: " + thisColView.rvv.dominates(otherColView.rvv)); - log("otherColView Dominates thisColView: " + otherColView.rvv.dominates(thisColView.rvv)); + boolean rowAndColLatestEntryModifiedTimeIsSame = + thisColView.mostRecentEntryModifiedTime == otherColView.mostRecentEntryModifiedTime && + thisRowView.mostRecentEntryModifiedTime == otherRowView.mostRecentEntryModifiedTime; if (isRVVNotUsable) { log("RVV based approach is not usable."); - if (rowAndColLatestOplogTimeSame) { - int retVal = compareTime(thisColView.getMostRecentEntryModifiedTime(), - otherColView.getMostRecentEntryModifiedTime(), - thisRowView.getMostRecentEntryModifiedTime(), - otherRowView.getMostRecentEntryModifiedTime()); - log("Decide on basis of MostRecentEntryModifiedTime: return value = " + retVal); - return retVal; - } else { + if (rowAndColLatestEntryModifiedTimeIsSame) { int retVal = compareTime(thisColView.getLatestOplogTime(), otherColView.getLatestOplogTime(), thisRowView.getLatestOplogTime(), otherRowView.getLatestOplogTime()); log("Decide on basis of LatestOplogTime: return value = " + retVal); return retVal; + } else { + int retVal = compareTime(thisColView.getMostRecentEntryModifiedTime(), + otherColView.getMostRecentEntryModifiedTime(), + thisRowView.getMostRecentEntryModifiedTime(), + otherRowView.getMostRecentEntryModifiedTime()); + log("Decide on basis of MostRecentEntryModifiedTime: return value = " + retVal); + return retVal; } } else { // NOTES : At this point there can't be a case when all 4 combinations are false or all are true. @@ -473,23 +478,30 @@ private int getCompareValue( if (thisColDominates == otherColDominates) { // true == true , false == false + // todo: when both cases are false, it is better to ask user for a preference + // todo: than relying on row buckets. // Decide on basis of row rvv -// t, t | f, t -// t, t | t, f -// f, f | t, f -// f, f | f, t +// ROW | COLUMN +// ------------- +// (f, t)| (t, t) +// (t, f)| (t, t) +// (t, f)| (f, f) +// (f, t)| (f, f) return thisRowDominates ? 1 : -1; } else if (thisRowDominates == otherRowDominates) { // true == true , false == false -// t, t | f, t -// t, t | t, f -// f, f | t, f -// f, f | f, t +// ROW | COLUMN +// ------------- +// (t, t) | (f, t) +// (t, t) | (t, f) +// (f, f) | (t, f) +// (f, f) | (f, t) // Decide on basis of col rvv return thisColDominates ? 1 : -1; } else { -// t, f | t, f -// f, t | f, t - // conflicting case - not sure when will this happen? +// ROW | COLUMN +// ------------- +// (t, f) | (t, f) +// (f, t) | (f, t) return thisColDominates && thisRowDominates ? 1 : -1; } From e5f876dde081981c673ecdec1e4e9941f0660585 Mon Sep 17 00:00:00 2001 From: snappy-sachin Date: Sat, 30 Nov 2019 01:14:42 +0530 Subject: [PATCH 13/27] Fixes for SNAP-3147: (#535) - Adding start time of ClusterStatistics instance creation and its getter method. --- .../gemfirexd/internal/engine/ui/ClusterStatistics.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/ClusterStatistics.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/ClusterStatistics.java index 68bbea446..93bc8640a 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/ClusterStatistics.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/ClusterStatistics.java @@ -38,6 +38,7 @@ private static class SingletonHelper { private static final ClusterStatistics INSTANCE = new ClusterStatistics(); } + private long startTime = System.currentTimeMillis(); private int totalCPUCores = 0; private final CircularFifoBuffer timeLine = @@ -185,6 +186,10 @@ public void updateClusterStatistics(Map memberStatsMap } + public long getStartTime() { + return startTime; + } + public int getTotalCPUCores() { return totalCPUCores; } From fac9de13e81f371d033178d6ff90fff63b3d41f7 Mon Sep 17 00:00:00 2001 From: Piyush Bisen <42795748+bisenpiyush@users.noreply.github.com> Date: Wed, 4 Dec 2019 16:40:35 +0530 Subject: [PATCH 14/27] SDENT-75 (#531) Added information for BOOLEAN datatype in "getTypeInfo" SQL in metadata.properties --- .../gemfirexd/internal/impl/jdbc/metadata.properties | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gemfirexd/prebuild/src/main/resources/com/pivotal/gemfirexd/internal/impl/jdbc/metadata.properties b/gemfirexd/prebuild/src/main/resources/com/pivotal/gemfirexd/internal/impl/jdbc/metadata.properties index 751a67e46..ac4360420 100644 --- a/gemfirexd/prebuild/src/main/resources/com/pivotal/gemfirexd/internal/impl/jdbc/metadata.properties +++ b/gemfirexd/prebuild/src/main/resources/com/pivotal/gemfirexd/internal/impl/jdbc/metadata.properties @@ -1163,6 +1163,9 @@ getTypeInfo=\ CAST (NULL AS INTEGER),CAST (NULL AS INTEGER),CAST (NULL AS INTEGER)), \ ('STRUCT',2002,2147483647,CAST (NULL AS CHAR),CAST (NULL AS CHAR),CAST (NULL AS CHAR), \ 1,FALSE,2,CAST (NULL AS BOOLEAN),FALSE,FALSE, \ + CAST (NULL AS INTEGER),CAST (NULL AS INTEGER),CAST (NULL AS INTEGER)), \ + ('BOOLEAN',-7,1,'''','''','''', \ + TRUE,FALSE,TRUE,CAST (NULL AS BOOLEAN),FALSE,FALSE, \ CAST (NULL AS INTEGER),CAST (NULL AS INTEGER),CAST (NULL AS INTEGER)) \ ) AS TYPEINFO(T1,T2,T3,T4,T5,T6,T7,T8,T9,T10,T11,T12,T14,T15,T18) @@ -1281,6 +1284,9 @@ getTypeInfo2_odbc=\ CAST (NULL AS INTEGER),CAST (NULL AS INTEGER),CAST (NULL AS INTEGER)), \ ('STRUCT',2002,2147483647,CAST (NULL AS CHAR),CAST (NULL AS CHAR),CAST (NULL AS CHAR), \ 1,FALSE,2,CAST (NULL AS BOOLEAN),FALSE,FALSE, \ + CAST (NULL AS INTEGER),CAST (NULL AS INTEGER),CAST (NULL AS INTEGER)), \ + ('BOOLEAN',-7,1,'''','''','''', \ + TRUE,FALSE,TRUE,CAST (NULL AS BOOLEAN),FALSE,FALSE, \ CAST (NULL AS INTEGER),CAST (NULL AS INTEGER),CAST (NULL AS INTEGER)) \ ) AS TYPEINFO(T1,T2,T3,T4,T5,T6,T7,T8,T9,T10,T11,T12,T14,T15,T18) \ WHERE T2 = ? From 74684ad52e98c1e5c9013deeb9fb60b9bae992ed Mon Sep 17 00:00:00 2001 From: Piyush Bisen <42795748+bisenpiyush@users.noreply.github.com> Date: Thu, 5 Dec 2019 18:02:20 +0530 Subject: [PATCH 15/27] Snap 3176 (#539) Added code to suppress the unwanted log messages --- .../internal/iapi/services/context/ContextManager.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/iapi/services/context/ContextManager.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/iapi/services/context/ContextManager.java index 2bf964970..6ee51f767 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/iapi/services/context/ContextManager.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/iapi/services/context/ContextManager.java @@ -716,7 +716,13 @@ else if (finder != null) { */ private void flushErrorString() { - errorStream.print(errorStringBuilder.get().toString()); + //changes to fix SNAP-3176 + String stmtText = errorStringBuilder.get().toString(); + if (!stmtText.contains("SET @@session.sql_mode=ANSI_QUOTES") && + !stmtText.contains("SELECT version FROM v$instance") && + !stmtText.contains("SELECT @@version")) { + errorStream.print(stmtText); + } errorStream.flush(); errorStringBuilder.reset(); } From e23a235db26b3badd589d396b26dba2aecd07dc1 Mon Sep 17 00:00:00 2001 From: suranjan kumar Date: Fri, 6 Dec 2019 15:19:47 +0530 Subject: [PATCH 16/27] [SNAP 2495] Support for bulk recording of version (#536) * Do bulk record versions RVV for snapshot being a CopyOnWriteHashMap is a perf bottleneck when it comes to recording versions. The decision was taken to make read operations faster. Have added bulk record versions to make sure bulk operations record their versions at one go rather than one by one. --- .../internal/cache/DistributedRegion.java | 1 + .../gemfire/internal/cache/TXState.java | 34 +++++---- .../internal/cache/TXStateInterface.java | 2 +- .../gemfire/internal/cache/TXStateProxy.java | 2 +- .../cache/versions/RegionVersionVector.java | 70 +++++++++++++++++-- 5 files changed, 87 insertions(+), 22 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DistributedRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DistributedRegion.java index 895e8d3dc..bbdd8e3e4 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DistributedRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DistributedRegion.java @@ -4935,4 +4935,5 @@ static int calcMemSize(Object value) { return 0; } } + } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXState.java index ff69fddab..3d0c77e74 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXState.java @@ -46,11 +46,7 @@ import com.gemstone.gemfire.internal.cache.locks.NonReentrantLock; import com.gemstone.gemfire.internal.cache.tier.sockets.ClientProxyMembershipID; import com.gemstone.gemfire.internal.cache.tier.sockets.VersionedObjectList; -import com.gemstone.gemfire.internal.cache.versions.RegionVersionHolder; -import com.gemstone.gemfire.internal.cache.versions.RegionVersionVector; -import com.gemstone.gemfire.internal.cache.versions.VersionSource; -import com.gemstone.gemfire.internal.cache.versions.VersionStamp; -import com.gemstone.gemfire.internal.cache.versions.VersionTag; +import com.gemstone.gemfire.internal.cache.versions.*; import com.gemstone.gemfire.internal.concurrent.ConcurrentTHashSet; import com.gemstone.gemfire.internal.concurrent.CustomEntryConcurrentHashMap; import com.gemstone.gemfire.internal.concurrent.MapCallback; @@ -1146,18 +1142,27 @@ private void publishRecordedVersions() { if (isSnapshot() || cache.snapshotEnabledForTest()) { // first take a lock at cache level so that we don't go into deadlock or sort array before // This is for tx RC, for snapshot just record all the versions from the queue - //TODO: this is performance issue: Need to make the lock granular at region level. - // also write a different recordVersion which will record without making clone - cache.acquireWriteLockOnSnapshotRvv(); try { + Map> s = new HashMap(); for (VersionInformation vi : queue) { if (TXStateProxy.LOG_FINE) { logger.info(LocalizedStrings.DEBUG, "Recording version " + vi + " from snapshot to " + - "region."); + "region."); + } + + if (vi.region.isSnapshotEnabledRegion()) { + Map versionV = s.get(vi.region); + if (versionV == null) { + versionV = vi.region.getVersionVector().getMemToVSnapshotCopy(); + s.put(vi.region, versionV); + } + vi.region.getVersionVector(). + recordVersionForSnapshotWithoutPublish((VersionSource) vi.member, vi.version, versionV); } - ((LocalRegion)vi.region).getVersionVector(). - recordVersionForSnapshot((VersionSource)vi.member, vi.version, null); + } + for (Map.Entry> entry : s.entrySet()) { + entry.getKey().getVersionVector().recordAllVersion(entry.getValue()); } } finally { cache.releaseWriteLockOnSnapshotRvv(); @@ -4270,8 +4275,9 @@ public boolean isSnapshot() { } @Override - public void recordVersionForSnapshot(Object member, long version, Region region) { + public void recordVersionForSnapshot(Object member, long version, LocalRegion region) { queue.add(new VersionInformation(member, version, region)); + Boolean wasPresent = writeRegions.putIfAbsent(region, true); if (wasPresent == null) { if (region instanceof BucketRegion) { @@ -4284,8 +4290,8 @@ public void recordVersionForSnapshot(Object member, long version, Region region) class VersionInformation { Object member; long version; - Region region; - public VersionInformation(Object member, long version, Region reg){ + LocalRegion region; + public VersionInformation(Object member, long version, LocalRegion reg){ this.member = member; this.version = version; this.region = reg; diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateInterface.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateInterface.java index 14d8ba42c..fca5a5157 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateInterface.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateInterface.java @@ -177,5 +177,5 @@ public Object lockEntry(RegionEntry entry, Object key, Object callbackArg, public boolean isSnapshot(); - public void recordVersionForSnapshot(Object member, long version, Region region); + public void recordVersionForSnapshot(Object member, long version, LocalRegion region); } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateProxy.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateProxy.java index 9cab5f490..9d955746b 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateProxy.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateProxy.java @@ -4792,7 +4792,7 @@ public boolean isSnapshot() { } @Override - public void recordVersionForSnapshot(Object member, long version, Region region) { + public void recordVersionForSnapshot(Object member, long version, LocalRegion region) { final TXState localState = getTXStateForRead(); localState.recordVersionForSnapshot(member, version, region); } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionVector.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionVector.java index a5ce08768..8b2ec5985 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionVector.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionVector.java @@ -19,12 +19,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.Collections; -import java.util.ConcurrentModificationException; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -784,7 +779,9 @@ public void recordVersionForSnapshot(T member, long version, EntryEventImpl even } holder.recordVersion(version, logger); + memberToVersionSnapshot.put(holder.id, holder); + forPrinting = memberToVersionSnapshot; } @@ -800,6 +797,67 @@ public void recordVersionForSnapshot(T member, long version, EntryEventImpl even } } + public Map> getMemToVSnapshotCopy() { + Map> m = new HashMap(); + for (Map.Entry> entry : memberToVersionSnapshot.entrySet()) { + m.put(entry.getKey(), entry.getValue().clone()); + } + return m; + } + + public Map> recordVersionForSnapshotWithoutPublish(T member, long version, + Map> m) { + LogWriterI18n logger = getLoggerI18n(); + T mbr = member; + GemFireCacheImpl cache = GemFireCacheImpl.getInstance(); + if (cache != null && !cache.snapshotEnabled()) { + return null; + } + RegionVersionHolder holder; + //Find the version holder object + synchronized (memberToVersionSnapshot) { + holder = m.get(mbr); + if (holder == null) { + mbr = getCanonicalId(mbr); + holder = new RegionVersionHolder(mbr); + } + holder.recordVersion(version, logger); + + // instead of putting the holder save it + // and do a putAll + m.put(holder.id, holder); + } + + if (logger!= null && logger.fineEnabled()) { + + logger.fine("Recorded version: " + version + " for member " + member + " in the snapshot region : " + + " the snapshot is " + m + + " it contains version after recording " + + m.get(member).contains(version)); + } + return m; + + } + + public void recordAllVersion(Map> versions) { + LogWriterI18n logger = getLoggerI18n(); + + GemFireCacheImpl cache = GemFireCacheImpl.getInstance(); + if (cache != null && !cache.snapshotEnabled()) { + return; + } + RegionVersionHolder holder; + Map> forPrinting; + //Find the version holder object + synchronized (memberToVersionSnapshot) { + memberToVersionSnapshot.putAll(versions); + forPrinting = memberToVersionSnapshot; + } + if (logger!= null && logger.fineEnabled()) { + + logger.fine("Recorded version: " + versions + " After record:" + forPrinting); + } + } /** * Records a received region-version. These are transmitted in VersionTags * in messages between peers and from servers to clients. In general you From a4105e7a0a11f2487634055b847f22a7cfd9aa6e Mon Sep 17 00:00:00 2001 From: paresh-p11 <43569032+paresh-p11@users.noreply.github.com> Date: Sat, 7 Dec 2019 13:24:36 +0530 Subject: [PATCH 17/27] SNAP-3243:Removing procedure GENERATE_LOAD_SCRIPTS (#541) * A separate procedure is not required to generate load script. Load script will be generated on every EXPORT_DATA call in a separate directory. --- .../ddl/catalog/GfxdSystemProcedures.java | 20 -------------- .../GetLeadNodeInfoAsStringMessage.java | 26 +++++++------------ .../impl/sql/catalog/GfxdDataDictionary.java | 6 ----- .../snappy/CallbackFactoryProvider.java | 4 --- .../internal/snappy/ClusterCallbacks.java | 5 ++-- 5 files changed, 11 insertions(+), 50 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java index 1e436db5e..70b53df72 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java @@ -1610,26 +1610,6 @@ public static void EXPORT_DDLS(String exportUri) throws SQLException { } } - public static void GENERATE_LOAD_SCRIPTS() throws SQLException { - try { - if (GemFireXDUtils.TraceSysProcedures) { - SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_SYS_PROCEDURES, - "Executing GENERATE_LOAD_SCRIPTS"); - } - Long connectionId = Misc.getLanguageConnectionContext().getConnectionId(); - GfxdListResultCollector collector = new GfxdListResultCollector(); - GetLeadNodeInfoAsStringMessage msg = new GetLeadNodeInfoAsStringMessage( - collector, GetLeadNodeInfoAsStringMessage.DataReqType.GENERATE_LOAD_SCRIPTS, connectionId); - msg.executeFunction(); - if (GemFireXDUtils.TraceSysProcedures) { - SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_SYS_PROCEDURES, - "GENERATE_LOAD_SCRIPTS successful."); - } - } catch(StandardException e) { - throw PublicAPI.wrapStandardException(e); - } - } - /** * Create or drop reservoir region for sampler. Note that the creat and drop operation * are intentionally combined in single procedure here to make conflation of create and diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoAsStringMessage.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoAsStringMessage.java index 30c306ca9..627e341e9 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoAsStringMessage.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoAsStringMessage.java @@ -41,9 +41,10 @@ public class GetLeadNodeInfoAsStringMessage extends MemberExecutorMessage rc, DataReqType reqType, Long connID, Object... args) { + public GetLeadNodeInfoAsStringMessage(final ResultCollector rc, + DataReqType reqType, Long connID, Object... args) { super(rc, null, false, true); this.requestType = reqType; this.additionalArgs = args; @@ -99,13 +100,6 @@ protected void execute() throws Exception { } result = exportDDLs(); break; - case GENERATE_LOAD_SCRIPTS: - if (GemFireXDUtils.TraceQuery) { - SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_QUERYDISTRIB, - "GetLeadNodeInfoAsStringMessage - case GENERATE_LOAD_SCRIPTS"); - } - result = generateLoadScripts(); - break; default: throw new IllegalArgumentException("GetLeadNodeInfoAsStringMessage:" + " Unknown data request type: " + this.requestType); @@ -119,21 +113,19 @@ protected void execute() throws Exception { private String exportData() { - com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider.getClusterCallbacks().exportData(connID, - additionalArgs[0].toString(), additionalArgs[1].toString(), additionalArgs[2].toString(), Boolean.parseBoolean(additionalArgs[3].toString())); + com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider + .getClusterCallbacks().exportData(connID, additionalArgs[0].toString(), + additionalArgs[1].toString(), additionalArgs[2].toString(), + Boolean.parseBoolean(additionalArgs[3].toString())); return "Data recovered"; } private String exportDDLs() { - com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider.getClusterCallbacks().exportDDLs(connID, additionalArgs[0].toString()); + com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider + .getClusterCallbacks().exportDDLs(connID, additionalArgs[0].toString()); return "DDLs recovered."; } - private String generateLoadScripts() { - com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider.getClusterCallbacks().generateLoadScripts(connID); - return "load scripts generated"; - } - private String handleGetJarsRequest() { URLClassLoader ul = CallbackFactoryProvider.getStoreCallbacks().getLeadClassLoader(); URL[] allJarUris = ul.getURLs(); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java index 835d98e23..98edaee23 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java @@ -1836,12 +1836,6 @@ private void createGfxdSystemProcedures(TransactionController tc, tc, GFXD_SYS_PROC_CLASSNAME, false); } - { - String[] arg_name = new String[] {}; - TypeDescriptor[] arg_types = new TypeDescriptor[] {}; - super.createSystemProcedureOrFunction("GENERATE_LOAD_SCRIPTS", sysUUID, arg_name, arg_types, 0, 0, RoutineAliasInfo.READS_SQL_DATA, null, newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, false); - } - { // GET_BUCKET_TO_SERVER_MAPPING String[] arg_names = new String[] { "FQTN", "BKT_TO_SERVER_MAPPING" }; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java index a5fc4bf63..d8b15c5d4 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java @@ -77,10 +77,6 @@ public void exportData(Long connId, String exportUri, String formatType, String public void exportDDLs(Long connId, String exportUri) { } - @Override - public void generateLoadScripts(Long connId) { - } - @Override public Object readDataType(ByteArrayDataInput in) { return null; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java index 72d095a77..8a3601e64 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java @@ -49,12 +49,11 @@ SparkSQLExecute getSQLExecute(Object dfObject, String sql, String schema, LeadNo SparkSQLExecute getSampleInsertExecute(String baseTable, LeadNodeExecutionContext ctx, Version v, List dvdRows, byte[] serializedDVDs); - void exportData(Long connId, String exportUri, String formatType, String tableNames, Boolean ignoreError); + void exportData(Long connId, String exportUri, String formatType, String tableNames, + Boolean ignoreError); void exportDDLs(Long connId, String exportUri); - void generateLoadScripts(Long connId); - Object readDataType(ByteArrayDataInput in); /** From 961000f5264bf14de27400d579b4278c9ab80c67 Mon Sep 17 00:00:00 2001 From: hemanthmeka <36498621+hemanthmeka@users.noreply.github.com> Date: Sat, 7 Dec 2019 17:26:38 +0530 Subject: [PATCH 18/27] skipping oplog recovery in recovery mode (#542) --- .../com/gemstone/gemfire/internal/cache/PersistentOplogSet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PersistentOplogSet.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PersistentOplogSet.java index 5b74ebf49..c79360258 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PersistentOplogSet.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PersistentOplogSet.java @@ -729,7 +729,7 @@ private long recoverOplogs(long byteCount, boolean initialRecovery) { if (!parent.isOffline()) { // schedule GFXD index recovery first parent.scheduleIndexRecovery(oplogSet, false); - if(recoverValues() && !recoverValuesSync()) { + if(!parent.getCache().isSnappyRecoveryMode() && recoverValues() && !recoverValuesSync()) { //TODO DAN - should we defer compaction until after //value recovery is complete? Or at least until after //value recovery for a given oplog is complete? From 11a25a6f5f418f7312b26f369621efdb13c1af3d Mon Sep 17 00:00:00 2001 From: Piyush Bisen <42795748+bisenpiyush@users.noreply.github.com> Date: Wed, 11 Dec 2019 11:27:17 +0530 Subject: [PATCH 19/27] SDENT-72 (#540) As a fix of SDENT-72, upgraded the OpenSSL from 1.0.2h to 1.1.1 --- native/build.gradle | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/native/build.gradle b/native/build.gradle index 43c18041f..9f4b5028d 100644 --- a/native/build.gradle +++ b/native/build.gradle @@ -48,14 +48,15 @@ if ((rootProject.name.contains('native') && !rootProject.hasProperty('skipNative String thriftVersion = '1.0.0-1' String boostVersion = '1.59.0' String mpirVersion = '2.7.2' - String opensslVersion = '1.0.2h' + String opensslVersion = '1.1.1' String unixodbcVersion = '2.3.4' String googletestVersion = '1.7.0' + String newThriftVersion = '1.0.0-2' + thriftVersion = newThriftVersion String oldReleasesDir = "https://github.com/SnappyDataInc/thrift/releases/download/${thriftVersion}" - + String releasesDir = oldReleasesDir - String newThriftVersion = '1.0.0-2' String newBoostVersion = '1.65.1' String newUnixodbcVersion = '2.3.4' String newGoogletestVersion = '1.8.0' @@ -83,31 +84,29 @@ if ((rootProject.name.contains('native') && !rootProject.hasProperty('skipNative } if (flavour.startsWith('ubuntu14') || flavour.startsWith('linuxmint17') || flavour.startsWith('elementary0.3')) { - dependencies += [ "thrift-${thriftVersion}-ubuntu14.04.tar.bz2", + dependencies += [ "thrift-${newThriftVersion}-ubuntu14.04.tar.bz2", "boost-${boostVersion}-ubuntu14.04.tar.bz2", "unixODBC-${unixodbcVersion}-ubuntu14.04.tar.bz2", "googletest-${googletestVersion}-ubuntu14.04.tar.bz2" ] } else if (flavour.startsWith('ubuntu16') || flavour.startsWith('ubuntu17') || flavour.startsWith('ubuntu18') || flavour.startsWith('linuxmint18') || flavour.startsWith('linuxmint19') || flavour.startsWith('elementary0.4') || flavour.startsWith('elementary5')) { - thriftVersion = newThriftVersion boostVersion = newBoostVersion unixodbcVersion = newUnixodbcVersion googletestVersion = newGoogletestVersion releasesDir = newReleasesDir - dependencies += [ "thrift-${thriftVersion}-ubuntu16.04.tar.bz2", + dependencies += [ "thrift-${newThriftVersion}-ubuntu16.04.tar.bz2", "boost-${boostVersion}-ubuntu16.04.tar.bz2", "unixODBC-${unixodbcVersion}-ubuntu16.04.tar.bz2", "googletest-${googletestVersion}-ubuntu16.04.tar.bz2" ] } else if (flavour.startsWith('redhat6') || flavour.startsWith('redhat7') || flavour.startsWith('fedora') || flavour.startsWith('centos6') || flavour.startsWith('centos7')) { - thriftVersion = newThriftVersion boostVersion = newBoostVersion unixodbcVersion = newUnixodbcVersion googletestVersion = newGoogletestVersion releasesDir = newReleasesDir - dependencies += [ "thrift-${thriftVersion}-rhel6.tar.bz2", + dependencies += [ "thrift-${newThriftVersion}-rhel6.tar.bz2", "boost-${boostVersion}-rhel6.tar.bz2", "unixODBC-${unixodbcVersion}-rhel6.tar.bz2", "googletest-${googletestVersion}-rhel6.tar.bz2" ] @@ -115,7 +114,7 @@ if ((rootProject.name.contains('native') && !rootProject.hasProperty('skipNative throw new GradleException("Unhandled linux flavour ${flavour}. Use -PlinuxFlavour to force one") } } else { - dependencies += [ "thrift-${thriftVersion}-${osName}.tar.bz2", + dependencies += [ "thrift-${newThriftVersion}-${osName}.tar.bz2", "boost-${boostVersion}-${osName}.tar.bz2", "googletest-${googletestVersion}-${osName}.tar.bz2" ] } @@ -221,10 +220,10 @@ if ((rootProject.name.contains('native') && !rootProject.hasProperty('skipNative } } thrift { - headers.srcDir "${distDir}/thrift-${thriftVersion}/include" + headers.srcDir "${distDir}/thrift-${newThriftVersion}/include" binaries.withType(StaticLibraryBinary) { staticLibraryFile = file(getLibPathForPlatform('thrift', 'thrift', - thriftVersion, true, false, targetPlatform)) + newThriftVersion, true, false, targetPlatform)) } } openssl { From 10cbfafe93666c148c165c2981f30a968569b2db Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 12 Dec 2019 19:41:30 +0530 Subject: [PATCH 20/27] fixing native build warnings and eclipse build --- native/.cproject | 8 ++++---- native/.settings/language.settings.xml | 4 ++-- native/src/snappyclient/cpp/Row.cpp | 2 +- .../src/snappyclient/cpp/impl/ClientService.cpp | 17 +++++++++++------ .../snappyclient/cpp/impl/ControlConnection.h | 6 ++++-- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/native/.cproject b/native/.cproject index 2669cbefb..4a2150de6 100644 --- a/native/.cproject +++ b/native/.cproject @@ -19,7 +19,7 @@ - + @@ -44,7 +44,7 @@ @@ -81,7 +81,7 @@ - + @@ -165,7 +165,7 @@ - + diff --git a/native/.settings/language.settings.xml b/native/.settings/language.settings.xml index f43c01d7c..4fcb1b687 100644 --- a/native/.settings/language.settings.xml +++ b/native/.settings/language.settings.xml @@ -5,7 +5,7 @@ - + @@ -16,7 +16,7 @@ - + diff --git a/native/src/snappyclient/cpp/Row.cpp b/native/src/snappyclient/cpp/Row.cpp index b43919002..f350bba14 100644 --- a/native/src/snappyclient/cpp/Row.cpp +++ b/native/src/snappyclient/cpp/Row.cpp @@ -120,7 +120,7 @@ namespace { #pragma warning(push) #pragma warning(disable: 4018) #endif - && result <= std::numeric_limits::max()) { + && result <= static_cast(std::numeric_limits::max())) { #ifdef _WINDOWS #pragma warning(pop) #endif diff --git a/native/src/snappyclient/cpp/impl/ClientService.cpp b/native/src/snappyclient/cpp/impl/ClientService.cpp index ac4f15d86..caeb0570f 100644 --- a/native/src/snappyclient/cpp/impl/ClientService.cpp +++ b/native/src/snappyclient/cpp/impl/ClientService.cpp @@ -359,11 +359,12 @@ void ClientService::setPendingTransactionAttrs( ClientService::ClientService(const std::string& host, const int port, thrift::OpenConnectionArgs& connArgs) : // default for load-balance is false - m_connArgs(initConnectionArgs(connArgs)), m_loadBalance(false), m_reqdServerType( - thrift::ServerType::THRIFT_SNAPPY_CP), m_useFramedTransport(false), m_serverGroups(), m_transport(), m_client( - createDummyProtocol()), m_connHosts(0), m_connId(0), m_token(), m_isOpen( - false), m_pendingTXAttrs(), m_hasPendingTXAttrs(false), m_isolationLevel( - IsolationLevel::NONE), m_lock(), m_loadBalanceInitialized(false) { + m_connArgs(initConnectionArgs(connArgs)), m_loadBalance(false), + m_loadBalanceInitialized(false), m_reqdServerType(thrift::ServerType::THRIFT_SNAPPY_CP), + m_useFramedTransport(false), m_serverGroups(), m_transport(), + m_client(createDummyProtocol()), m_connHosts(0), m_connId(0), m_token(), + m_isOpen(false), m_pendingTXAttrs(), m_hasPendingTXAttrs(false), + m_isolationLevel(IsolationLevel::NONE), m_lock() { std::map& props = connArgs.properties; std::map::iterator propValue; @@ -1863,7 +1864,11 @@ bool ClientService::handleException(const char* op, bool tryFailover, std::set& failedServers, const TException& te) { if (!m_isOpen) { - newSnappyExceptionForConnectionClose(op, m_currentHostAddr); + if (createNewConnection) { + newSnappyExceptionForConnectionClose(op, m_currentHostAddr); + } else { + throwSQLExceptionForNodeFailure(op, te); + } } if (!m_loadBalance || m_isolationLevel != IsolationLevel::NONE) { tryFailover = false; diff --git a/native/src/snappyclient/cpp/impl/ControlConnection.h b/native/src/snappyclient/cpp/impl/ControlConnection.h index 990a161e8..df8fc1583 100644 --- a/native/src/snappyclient/cpp/impl/ControlConnection.h +++ b/native/src/snappyclient/cpp/impl/ControlConnection.h @@ -93,11 +93,13 @@ namespace io { static std::vector > s_allConnections; /** Global lock for {@link allConnections} */ static boost::mutex s_allConnsLock; + /*********Member functions**************/ ControlConnection() : - m_serverGroups(std::set()) { + m_snappyServerType(thrift::ServerType::THRIFT_SNAPPY_CP), + m_serverGroups(std::set()), m_framedTransport(false) { } - ; + ControlConnection(ClientService * const &service); void failoverToAvailableHost( From dcc866aed44058456178db430c5e0f76755154bc Mon Sep 17 00:00:00 2001 From: kneeraj Date: Thu, 12 Dec 2019 22:52:22 +0530 Subject: [PATCH 21/27] Repl fix (#543) * Augmented a message to get information from lead node to expect and object type as result. Added an enum to get class bytes of a class created with repl on lead but to be fetched on remote executors. * Fixes the display issue with show connections. --- .../internal/engine/GfxdDataSerializable.java | 2 +- .../ddl/catalog/GfxdSystemProcedures.java | 14 +++--- ...ngMessage.java => GetLeadNodeInfoMsg.java} | 49 ++++++++++++++----- .../message/LeadNodeExecutorMsg.java | 4 +- .../internal/impl/tools/ij/utilMain.java | 2 +- 5 files changed, 49 insertions(+), 22 deletions(-) rename gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/{GetLeadNodeInfoAsStringMessage.java => GetLeadNodeInfoMsg.java} (78%) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdDataSerializable.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdDataSerializable.java index cc4c6473d..ec18c1e8d 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdDataSerializable.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdDataSerializable.java @@ -279,7 +279,7 @@ public static synchronized boolean initTypes() { DSFIDFactory.registerGemFireXDClass(PROJECTION_ROW, () -> new ProjectionRow()); DSFIDFactory.registerGemFireXDClass(LEAD_NODE_DATA_MSG, - () -> new GetLeadNodeInfoAsStringMessage()); + () -> new GetLeadNodeInfoMsg()); DSFIDFactory.registerGemFireXDClass(LEAD_DISK_STATE_MSG, () -> new RecoveredMetadataRequestMessage()); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java index 70b53df72..6ca77c531 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java @@ -69,7 +69,7 @@ import com.pivotal.gemfirexd.internal.engine.distributed.GfxdMessage; import com.pivotal.gemfirexd.internal.engine.distributed.QueryCancelFunction; import com.pivotal.gemfirexd.internal.engine.distributed.QueryCancelFunction.QueryCancelFunctionArgs; -import com.pivotal.gemfirexd.internal.engine.distributed.message.GetLeadNodeInfoAsStringMessage; +import com.pivotal.gemfirexd.internal.engine.distributed.message.GetLeadNodeInfoMsg; import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils; import com.pivotal.gemfirexd.internal.engine.distributed.utils.SecurityUtils; import com.pivotal.gemfirexd.internal.engine.jdbc.GemFireXDRuntimeException; @@ -1539,8 +1539,8 @@ public static void GET_DEPLOYED_JARS(String[] jarStrings) throws SQLException { } GfxdListResultCollector collector = new GfxdListResultCollector(); // ConnectionId is not being used for GET_DEPLOYED_JARS; hence passing dummy value(0L) - GetLeadNodeInfoAsStringMessage msg = new GetLeadNodeInfoAsStringMessage( - collector, GetLeadNodeInfoAsStringMessage.DataReqType.GET_JARS, 0L, (Object[])null); + GetLeadNodeInfoMsg msg = new GetLeadNodeInfoMsg( + collector, GetLeadNodeInfoMsg.DataReqType.GET_JARS, 0L, (Object[])null); msg.executeFunction(); ArrayList result = collector.getResult(); String resJarStrings = (String)result.get(0); @@ -1570,8 +1570,8 @@ public static void EXPORT_DATA(String exportUri, String formatType, String table } Long connectionId = Misc.getLanguageConnectionContext().getConnectionId(); GfxdListResultCollector collector = new GfxdListResultCollector(); - GetLeadNodeInfoAsStringMessage msg = new GetLeadNodeInfoAsStringMessage( - collector, GetLeadNodeInfoAsStringMessage.DataReqType.EXPORT_DATA, connectionId, + GetLeadNodeInfoMsg msg = new GetLeadNodeInfoMsg( + collector, GetLeadNodeInfoMsg.DataReqType.EXPORT_DATA, connectionId, exportUri, formatType, tableNames, ignoreError); msg.executeFunction(); @@ -1598,8 +1598,8 @@ public static void EXPORT_DDLS(String exportUri) throws SQLException { } Long connectionId = Misc.getLanguageConnectionContext().getConnectionId(); GfxdListResultCollector collector = new GfxdListResultCollector(); - GetLeadNodeInfoAsStringMessage msg = new GetLeadNodeInfoAsStringMessage( - collector, GetLeadNodeInfoAsStringMessage.DataReqType.EXPORT_DDLS, connectionId, exportUri); + GetLeadNodeInfoMsg msg = new GetLeadNodeInfoMsg( + collector, GetLeadNodeInfoMsg.DataReqType.EXPORT_DDLS, connectionId, exportUri); msg.executeFunction(); if (GemFireXDUtils.TraceSysProcedures) { SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_SYS_PROCEDURES, diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoAsStringMessage.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoMsg.java similarity index 78% rename from gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoAsStringMessage.java rename to gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoMsg.java index 627e341e9..294daeced 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoAsStringMessage.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoMsg.java @@ -27,23 +27,25 @@ import com.pivotal.gemfirexd.internal.iapi.error.StandardException; import com.pivotal.gemfirexd.internal.shared.common.sanity.SanityManager; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; +import java.io.*; import java.net.URL; import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.SQLException; import java.util.Set; -public class GetLeadNodeInfoAsStringMessage extends MemberExecutorMessage { +public class GetLeadNodeInfoMsg extends MemberExecutorMessage { private Object[] additionalArgs; private DataReqType requestType; private Long connID; - public enum DataReqType {GET_JARS, EXPORT_DATA, EXPORT_DDLS} - public GetLeadNodeInfoAsStringMessage(final ResultCollector rc, + public enum DataReqType {GET_JARS, EXPORT_DATA, EXPORT_DDLS, GET_CLASS_BYTES} + + public GetLeadNodeInfoMsg(final ResultCollector rc, DataReqType reqType, Long connID, Object... args) { super(rc, null, false, true); this.requestType = reqType; @@ -51,7 +53,15 @@ public GetLeadNodeInfoAsStringMessage(final ResultCollector rc, this.connID = connID; } - public GetLeadNodeInfoAsStringMessage() { + public GetLeadNodeInfoMsg(final ResultCollector rc, DataReqType reqType, Long connID, String filePath) { + super(rc, null, false, true); + this.requestType = reqType; + this.additionalArgs = new Object[1]; + this.additionalArgs[0] = filePath; + this.connID = connID; + } + + public GetLeadNodeInfoMsg() { super(true); } @@ -78,10 +88,10 @@ public boolean optimizeForWrite() { protected void execute() throws Exception { if (GemFireXDUtils.TraceQuery) { SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_QUERYDISTRIB, - "GetLeadNodeInfoAsStringMessage.execute: "); + "GetLeadNodeInfoMsg.execute: "); } try { - String result = null; + Object result = null; switch (this.requestType) { case GET_JARS: result = handleGetJarsRequest(); @@ -89,19 +99,22 @@ protected void execute() throws Exception { case EXPORT_DATA: if (GemFireXDUtils.TraceQuery) { SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_QUERYDISTRIB, - "GetLeadNodeInfoAsStringMessage - case EXPORT_DATA"); + "GetLeadNodeInfoMsg - case EXPORT_DATA"); } result = exportData(); break; case EXPORT_DDLS: if (GemFireXDUtils.TraceQuery) { SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_QUERYDISTRIB, - "GetLeadNodeInfoAsStringMessage - case EXPORT_DDLS"); + "GetLeadNodeInfoMsg - case EXPORT_DDLS"); } result = exportDDLs(); break; + case GET_CLASS_BYTES: + result = getClassBytes(); + break; default: - throw new IllegalArgumentException("GetLeadNodeInfoAsStringMessage:" + + throw new IllegalArgumentException("GetLeadNodeInfoMsg:" + " Unknown data request type: " + this.requestType); } @@ -111,6 +124,18 @@ protected void execute() throws Exception { } } + private byte[] getClassBytes() throws IOException { + String filePath = (String)this.additionalArgs[0]; + Path p = Paths.get(filePath); + if (!Files.exists(p)) { + throw new FileNotFoundException(filePath); + } + File file = new File(filePath); + FileInputStream fip = new FileInputStream(file); + byte fileContent[] = new byte[(int)file.length()]; + fip.read(fileContent); + return fileContent; + } private String exportData() { com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/LeadNodeExecutorMsg.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/LeadNodeExecutorMsg.java index de0e457b9..e439f4e24 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/LeadNodeExecutorMsg.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/LeadNodeExecutorMsg.java @@ -70,6 +70,8 @@ public final class LeadNodeExecutorMsg extends MemberExecutorMessage { private static final Pattern PARSE_EXCEPTION = Pattern.compile( "(Pars[a-zA-Z]*Exception)|(Pars[a-zA-Z]*Error)"); + private static final Pattern EXEC_COMMAND = Pattern.compile( + "\\s*EXEC\\s+SCALA\\s+.*", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); public LeadNodeExecutorMsg(LeadNodeExecutionContext ctx, GfxdResultCollector rc, LeadNodeExecutionObject execObject) { @@ -147,7 +149,7 @@ protected void execute() throws Exception { private boolean interpreterExecution() throws Exception { String sql = this.execObject.getSql(); if (sql != null) { - if (sql.startsWith("exec") || sql.startsWith("EXEC")) { + if (EXEC_COMMAND.matcher(sql).matches()) { String user = ctx.getUserName() != null ? ctx.getUserName().toLowerCase() : ctx.getUserName(); InternalDistributedMember member = this.getSenderForReply(); final Version v = member.getVersionObject(); diff --git a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/utilMain.java b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/utilMain.java index f6859fb98..52b2e82f2 100644 --- a/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/utilMain.java +++ b/gemfirexd/tools/src/main/java/com/pivotal/gemfirexd/internal/impl/tools/ij/utilMain.java @@ -493,7 +493,7 @@ private int runScriptGuts() { JDBCDisplayUtil.INITIAL_CMD_IN_PROGRESS = false; GfxdUtilLauncher.initialRunFiles = null; } - if (this.isInterpreterMode && !JDBCDisplayUtil.BEFORE_CONNECT) { + if (!this.isInterpreterMode || (this.isInterpreterMode && !JDBCDisplayUtil.BEFORE_CONNECT)) { endTime = displayResult(out, result, connEnv[currCE].getConnection(), beginTime /* GemStoneAddition */, true); } From 04960ba61ca53d1d8ca3160992b62a3ae04369a0 Mon Sep 17 00:00:00 2001 From: Neeraj Kumar Date: Fri, 13 Dec 2019 17:36:06 +0530 Subject: [PATCH 22/27] Closing the file input stream in finally block. --- .../engine/distributed/message/GetLeadNodeInfoMsg.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoMsg.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoMsg.java index 294daeced..02b39867d 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoMsg.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoMsg.java @@ -132,9 +132,13 @@ private byte[] getClassBytes() throws IOException { } File file = new File(filePath); FileInputStream fip = new FileInputStream(file); - byte fileContent[] = new byte[(int)file.length()]; - fip.read(fileContent); - return fileContent; + try { + byte fileContent[] = new byte[(int) file.length()]; + fip.read(fileContent); + return fileContent; + } finally { + if (fip != null) fip.close(); + } } private String exportData() { From 83fd36d8510f3f918403ba2999ebf19a6ffa3e75 Mon Sep 17 00:00:00 2001 From: kneeraj Date: Thu, 19 Dec 2019 17:05:28 +0530 Subject: [PATCH 23/27] Ext table grant (#545) * Support for external table authorized access. Grant revoke can be done on external tables as well. Grant ALL* only can be specified though for external tables. --- .../snappydata/jdbc/TomcatConnectionPool.java | 5 ++-- .../ddl/catalog/GfxdSystemProcedures.java | 21 +++++++++++++++ .../message/GetLeadNodeInfoMsg.java | 27 ++++++++++++++++++- .../internal/impl/sql/GenericStatement.java | 6 ++++- .../impl/sql/catalog/GfxdDataDictionary.java | 13 +++++++++ .../snappy/CallbackFactoryProvider.java | 5 ++++ .../internal/snappy/ClusterCallbacks.java | 2 ++ 7 files changed, 74 insertions(+), 5 deletions(-) diff --git a/gemfirexd/client/src/main/java/io/snappydata/jdbc/TomcatConnectionPool.java b/gemfirexd/client/src/main/java/io/snappydata/jdbc/TomcatConnectionPool.java index 4fd6b08ae..3ec8bda2e 100644 --- a/gemfirexd/client/src/main/java/io/snappydata/jdbc/TomcatConnectionPool.java +++ b/gemfirexd/client/src/main/java/io/snappydata/jdbc/TomcatConnectionPool.java @@ -28,7 +28,6 @@ import java.util.stream.Collectors; import com.pivotal.gemfirexd.Attribute; -import com.sun.xml.internal.fastinfoset.stax.events.Util; import org.apache.tomcat.jdbc.pool.DataSource; import org.apache.tomcat.jdbc.pool.PoolProperties; import org.apache.tomcat.jdbc.pool.interceptor.StatementFinalizer; @@ -162,12 +161,12 @@ private PoolProperties getPoolProperties(Properties prop) { poolProperties.setDriverClassName(driverClassName); String username = prop.getProperty(PoolProps.USER.key); - if (!Util.isEmptyString(username)) { + if (username != null && !username.isEmpty()) { poolProperties.setUsername(username); } String password = prop.getProperty(PoolProps.PASSWORD.key); - if (!Util.isEmptyString(password)) { + if (password != null && !password.isEmpty()) { poolProperties.setPassword(password); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java index 6ca77c531..60407ce19 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java @@ -1550,6 +1550,27 @@ public static void GET_DEPLOYED_JARS(String[] jarStrings) throws SQLException { } } + public static void CHECK_AUTHZ_ON_EXT_TABLES(String currentUser, + String allTableString, String[] resultDetails) throws SQLException { + try { + if (GemFireXDUtils.TraceSysProcedures) { + SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_SYS_PROCEDURES, + "executing CHECK_AUTHZ_ON_EXT_TABLES"); + } + GfxdListResultCollector collector = new GfxdListResultCollector(); + // ConnectionId is not being used for GET_DEPLOYED_JARS; hence passing dummy value(0L) + GetLeadNodeInfoMsg msg = new GetLeadNodeInfoMsg( + collector, GetLeadNodeInfoMsg.DataReqType.CHECK_EXT_TABLE_PERMISSION, + 0L, new Object[] {currentUser, allTableString}); + msg.executeFunction(); + ArrayList result = collector.getResult(); + String res = (String)result.get(0); + resultDetails[0] = res; + } catch (StandardException se) { + throw PublicAPI.wrapStandardException(se); + } + } + /** * A recovery mode procedure which allows the user to export the specified(all) tables/views * in the specified format at the specified location. diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoMsg.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoMsg.java index 02b39867d..f630c9331 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoMsg.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GetLeadNodeInfoMsg.java @@ -21,11 +21,13 @@ import com.gemstone.gemfire.cache.execute.ResultCollector; import com.gemstone.gemfire.distributed.DistributedMember; import com.gemstone.gemfire.internal.snappy.CallbackFactoryProvider; +import com.gemstone.gemfire.internal.snappy.StoreCallbacks; import com.pivotal.gemfirexd.internal.engine.GfxdConstants; import com.pivotal.gemfirexd.internal.engine.Misc; import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils; import com.pivotal.gemfirexd.internal.iapi.error.StandardException; import com.pivotal.gemfirexd.internal.shared.common.sanity.SanityManager; +import com.pivotal.gemfirexd.internal.snappy.ClusterCallbacks; import java.io.*; import java.net.URL; @@ -43,7 +45,7 @@ public class GetLeadNodeInfoMsg extends MemberExecutorMessage { private Long connID; - public enum DataReqType {GET_JARS, EXPORT_DATA, EXPORT_DDLS, GET_CLASS_BYTES} + public enum DataReqType {GET_JARS, EXPORT_DATA, EXPORT_DDLS, GET_CLASS_BYTES, CHECK_EXT_TABLE_PERMISSION} public GetLeadNodeInfoMsg(final ResultCollector rc, DataReqType reqType, Long connID, Object... args) { @@ -113,6 +115,9 @@ protected void execute() throws Exception { case GET_CLASS_BYTES: result = getClassBytes(); break; + case CHECK_EXT_TABLE_PERMISSION: + result = checkExternalTableAuthorization(); + break; default: throw new IllegalArgumentException("GetLeadNodeInfoMsg:" + " Unknown data request type: " + this.requestType); @@ -141,6 +146,26 @@ private byte[] getClassBytes() throws IOException { } } + private String checkExternalTableAuthorization() throws IOException { + if (!Misc.isSecurityEnabled()) return null; + boolean checkAuthOfExternalTables = Boolean.parseBoolean( + System.getProperty("CHECK_EXTERNAL_TABLE_AUTHZ")); + if (!checkAuthOfExternalTables) return null; + + String user = (String)this.additionalArgs[0]; + String allTableStr = (String)this.additionalArgs[1]; + String[] allTable = allTableStr.split(","); + ClusterCallbacks ccb = com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider.getClusterCallbacks(); + String result = null; + for (String t : allTable) { + if (!ccb.isUserAuthorizedForExternalTable(user, t)) { + result = t; + break; + } + } + return result; + } + private String exportData() { com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider .getClusterCallbacks().exportData(connID, additionalArgs[0].toString(), diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/GenericStatement.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/GenericStatement.java index 6bb9ed659..8b1c7095f 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/GenericStatement.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/GenericStatement.java @@ -173,6 +173,9 @@ public class GenericStatement private static final Pattern GRANT_REVOKE_INTP_PATTERN = Pattern.compile("^\\s*(GRANT|REVOKE)\\s+(PRIVILEGE)\\s+(EXEC)\\s+(SCALA)\\s+", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + private static final Pattern GRANT_ON_EXT_TABLE_PATTERN = + Pattern.compile("^\\s*(GRANT|REVOKE)\\s+(ALL)\\s+(ON)", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL); private static ExecutionEngineArbiter engineArbiter = new ExecutionEngineArbiter(); // GemStone changes END /** @@ -241,7 +244,8 @@ private GenericPreparedStatement getPreparedStatementForSnappy(boolean commitNes boolean checkCancellation, boolean isUpdateOrDeleteOrPut, Throwable cause) throws StandardException { GenericPreparedStatement gps = preparedStmt; String source = getSource(); - boolean isGrantRevokeIntp = source != null ? GRANT_REVOKE_INTP_PATTERN.matcher(source).find() : false; + boolean isGrantRevokeIntp = source != null ? (GRANT_REVOKE_INTP_PATTERN.matcher(source).find() + || GRANT_ON_EXT_TABLE_PATTERN.matcher(source).find()) : false; GeneratedClass ac = new SnappyActivationClass(lcc, !(isDDL || isGrantRevokeIntp), isPreparedStatement() && !isDDL && !isGrantRevokeIntp, isUpdateOrDeleteOrPut); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java index 98edaee23..fd42b4d8c 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java @@ -1814,6 +1814,19 @@ private void createGfxdSystemProcedures(TransactionController tc, newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, false); } + { + // GET_JARS -- Smart Connectors will pull all the jars + String[] arg_names = new String[] { "USER", "ALLTABLES", "AUTHZRESULT"}; + TypeDescriptor[] arg_types = new TypeDescriptor[] { + DataTypeDescriptor.getCatalogType(Types.VARCHAR), + DataTypeDescriptor.getCatalogType(Types.VARCHAR), + DataTypeDescriptor.getCatalogType(Types.VARCHAR) + }; + super.createSystemProcedureOrFunction("CHECK_AUTHZ_ON_EXT_TABLES", + sysUUID, arg_names, arg_types, 1, 0, RoutineAliasInfo.READS_SQL_DATA, null, + newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, false); + } + { String[] arg_names = new String[] {"PATH_URI", "FORMAT_TYPE", "TABLES", "IGNORE_ERROR"}; TypeDescriptor[] arg_types = new TypeDescriptor[] { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java index d8b15c5d4..c2635d0f1 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java @@ -65,6 +65,11 @@ public InterpreterExecute getInterpreterExecution(String sql, Version v, Long co return null; } + @Override + public boolean isUserAuthorizedForExternalTable(String user, String table) { + return false; + } + public SparkSQLExecute getSampleInsertExecute(String baseTable, LeadNodeExecutionContext ctx, Version v, List dvdRows, byte[] serializedDVDs) { return null; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java index 8a3601e64..5752356ba 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java @@ -46,6 +46,8 @@ SparkSQLExecute getSQLExecute(Object dfObject, String sql, String schema, LeadNo InterpreterExecute getInterpreterExecution(String sql, Version v, Long connId); + boolean isUserAuthorizedForExternalTable(String user, String table); + SparkSQLExecute getSampleInsertExecute(String baseTable, LeadNodeExecutionContext ctx, Version v, List dvdRows, byte[] serializedDVDs); From d7bee9d50fe50d7224b7db83cbc7c5bb3708f761 Mon Sep 17 00:00:00 2001 From: Amogh Shetkar Date: Thu, 2 Jan 2020 13:40:29 +0530 Subject: [PATCH 24/27] * Version update 1.6.5 --- build.gradle | 2 +- tests/sql/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 37b872c25..5edcf84f7 100644 --- a/build.gradle +++ b/build.gradle @@ -130,7 +130,7 @@ allprojects { PRODUCT_MAJOR = '1' PRODUCT_MINOR = '6' - PRODUCT_MAINT = '4' + PRODUCT_MAINT = '5' PRODUCT_CLASSIFIER = '' PRODUCT_RELEASE_STAGE = '' PRODUCT_VERSION = "${PRODUCT_MAJOR}.${PRODUCT_MINOR}.${PRODUCT_MAINT}${PRODUCT_CLASSIFIER}" diff --git a/tests/sql/build.gradle b/tests/sql/build.gradle index 534215e85..ffbd285ce 100644 --- a/tests/sql/build.gradle +++ b/tests/sql/build.gradle @@ -92,7 +92,7 @@ dependencies { // apply default manifest if (rootProject.hasProperty('enablePublish')) { - createdBy = 'SnappyData Build Team' + createdBy = vendorName } shadowJar { zip64 = true From 28784c8b4ffa3a2c4be54d28a29998d509613a55 Mon Sep 17 00:00:00 2001 From: Piyush Bisen <42795748+bisenpiyush@users.noreply.github.com> Date: Thu, 9 Jan 2020 18:45:00 +0530 Subject: [PATCH 25/27] SDENT-78 (#544) Added support for SSL Co-authored-by: Sumedh Wale --- .../src/snappyclient/cpp/ClientAttribute.cpp | 3 + .../snappyclient/cpp/impl/ClientService.cpp | 85 +++++++++-- .../src/snappyclient/cpp/impl/ClientService.h | 26 +++- .../cpp/impl/ControlConnection.cpp | 35 +++-- .../snappyclient/cpp/impl/ControlConnection.h | 13 +- .../snappyclient/cpp/impl/SSLParameters.cpp | 143 ++++++++++++++++++ .../src/snappyclient/cpp/impl/SSLParameters.h | 89 +++++++++++ .../snappyclient/headers/ClientAttribute.h | 3 + 8 files changed, 358 insertions(+), 39 deletions(-) create mode 100644 native/src/snappyclient/cpp/impl/SSLParameters.cpp create mode 100644 native/src/snappyclient/cpp/impl/SSLParameters.h diff --git a/native/src/snappyclient/cpp/ClientAttribute.cpp b/native/src/snappyclient/cpp/ClientAttribute.cpp index 56f529bef..3eb30124e 100644 --- a/native/src/snappyclient/cpp/ClientAttribute.cpp +++ b/native/src/snappyclient/cpp/ClientAttribute.cpp @@ -77,5 +77,8 @@ const std::string ClientAttribute::THRIFT_USE_BINARY_PROTOCOL = ClientAttribute::addToHashSet("binary-protocol"); const std::string ClientAttribute::THRIFT_USE_FRAMED_TRANSPORT = ClientAttribute::addToHashSet("framed-transport"); +const std::string ClientAttribute::AQP_ERROR = ClientAttribute::addToHashSet("spark.sql.aqp.error"); +const std::string ClientAttribute::AQP_CONFIDENCE = ClientAttribute::addToHashSet("spark.sql.aqp.confidence"); +const std::string ClientAttribute::AQP_BEHAVIOR = ClientAttribute::addToHashSet("spark.sql.aqp.behavior"); } } } diff --git a/native/src/snappyclient/cpp/impl/ClientService.cpp b/native/src/snappyclient/cpp/impl/ClientService.cpp index caeb0570f..2bfb69489 100644 --- a/native/src/snappyclient/cpp/impl/ClientService.cpp +++ b/native/src/snappyclient/cpp/impl/ClientService.cpp @@ -381,7 +381,7 @@ ClientService::ClientService(const std::string& host, const int port, //default for load-balance is true on locators and false on servers // so tentatively set as true and adjust using the ControlConnection if (hasLoadBalance) { - m_loadBalance = (boost::iequals("true", propValue->second)); + m_loadBalance = (boost::iequals(propValue->second, "true")); props.erase(propValue); m_loadBalanceInitialized = true; } @@ -404,15 +404,43 @@ ClientService::ClientService(const std::string& host, const int port, props.erase(propValue); } + // read the AQP properties + if ((propValue = props.find(ClientAttribute::AQP_ERROR)) != props.end()) { + try { + double errorVal = boost::lexical_cast(propValue->second); + if (errorVal < 0.0 || errorVal > 1.0) { + throw std::invalid_argument(":Invalid AQP Error value:"); + } + props.erase(propValue); + } catch (const boost::bad_lexical_cast& ex) { + props.erase(propValue); + throw ex; + } + } + if ((propValue = props.find(ClientAttribute::AQP_CONFIDENCE)) + != props.end()) { + try { + double errorVal = boost::lexical_cast(propValue->second); + if (errorVal < 0.0 || errorVal > 1.0) { + throw std::invalid_argument(":Invalid AQP Confidence value:"); + } + props.erase(propValue); + } catch (const boost::bad_lexical_cast& ex) { + throw ex; + } + } + if ((propValue = props.find(ClientAttribute::AQP_BEHAVIOR)) != props.end()) { + if (propValue->second.empty()) { + throw std::invalid_argument(":Invalid AQP Behavior value:"); + } + props.erase(propValue); + } // now check for the protocol details like SSL etc // and reqd snappyServerType bool binaryProtocol = false; bool framedTransport = false; bool useSSL = false; - //SSLSocketParameters sslParams = null; - std::map::iterator propValue; - std::map& props = connArgs.properties; if ((propValue = props.find(ClientAttribute::THRIFT_USE_BINARY_PROTOCOL)) != props.end()) { binaryProtocol = boost::iequals(propValue->second, "true"); @@ -430,8 +458,7 @@ ClientService::ClientService(const std::string& host, const int port, if ((propValue = props.find(ClientAttribute::SSL_PROPERTIES)) != props.end()) { useSSL = true; - // TODO: SW: SSL params support - //sslParams = Utils::getSSLParameters(propValue->second); + InternalUtils::splitCSV(propValue->second, m_sslParams); props.erase(propValue); } m_reqdServerType = getServerType(true, binaryProtocol, useSSL); @@ -487,7 +514,7 @@ void ClientService::openConnection(thrift::HostAddress& hostAddr, if (m_loadBalance) { // at this point query the control service for preferred server controlConn->getPreferredServer(hostAddr, te, failedServers, - this->m_serverGroups, false); + this->m_serverGroups, this, false); } m_currentHostAddr = hostAddr; @@ -619,7 +646,7 @@ protocol::TProtocol* ClientService::createDummyProtocol() { protocol::TProtocol* ClientService::createProtocol( thrift::HostAddress& hostAddr, const thrift::ServerType::type serverType, - bool useFramedTransport, //const SSLSocketParameters& sslParams, + bool useFramedTransport, boost::shared_ptr& returnTransport) { bool useBinaryProtocol; bool useSSL; @@ -667,9 +694,7 @@ protocol::TProtocol* ClientService::createProtocol( boost::shared_ptr socket; if (useSSL) { - TSSLSocketFactory sslSocketFactory; - sslSocketFactory.authenticate(false); - socket = sslSocketFactory.createSocket(hostAddr.hostName, hostAddr.port); + socket = createSSLSocket(hostAddr.hostName, hostAddr.port); } else { socket.reset(new TSocket(hostAddr.hostName, hostAddr.port)); } @@ -1885,3 +1910,41 @@ bool ClientService::handleException(const char* op, bool tryFailover, updateFailedServersForCurrent(failedServers, true, te); return true; } + +boost::shared_ptr ClientService::createSSLSocket( + const std::string& host, int port) { + SSLSocketFactory sslSocketFactory(m_sslParams); + + std::string sslProperty = getSSLPropertyName(SSLProperty::CLIENTAUTH); + std::string propVal = getSSLPropertyValue(sslProperty); + if (!propVal.empty() && boost::iequals(propVal, "true")) { + sslSocketFactory.authenticate(true); + sslProperty = getSSLPropertyName(SSLProperty::KEYSTORE); + propVal = getSSLPropertyValue(sslProperty); + if (!propVal.empty()) { + sslSocketFactory.loadPrivateKey(propVal.c_str()); + } + sslProperty = getSSLPropertyName(SSLProperty::CERTIFICATE); + propVal = getSSLPropertyValue(sslProperty); + if (!propVal.empty()) { + sslSocketFactory.loadCertificate(propVal.c_str()); + } + } else { + sslSocketFactory.authenticate(false); + } + + sslProperty = this->getSSLPropertyName(SSLProperty::TRUSTSTORE); + propVal = this->getSSLPropertyValue(sslProperty); + if (!propVal.empty()) { + sslSocketFactory.loadTrustedCertificates(propVal.c_str()); + } + sslProperty = getSSLPropertyName(SSLProperty::CIPHERSUITES); + propVal = getSSLPropertyValue(sslProperty); + if (!propVal.empty()) { + sslSocketFactory.ciphers(propVal); + } else { + sslSocketFactory.ciphers("ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH"); + } + + return sslSocketFactory.createSocket(host, port); +} diff --git a/native/src/snappyclient/cpp/impl/ClientService.h b/native/src/snappyclient/cpp/impl/ClientService.h index b05c27b31..0221d904f 100644 --- a/native/src/snappyclient/cpp/impl/ClientService.h +++ b/native/src/snappyclient/cpp/impl/ClientService.h @@ -40,11 +40,13 @@ #include #include +#include #include "../thrift/SnappyDataService.h" +#include "SSLParameters.h" using namespace apache::thrift; - +using namespace apache::thrift::transport; namespace apache { namespace thrift { namespace transport { @@ -127,13 +129,21 @@ namespace io { static protocol::TProtocol* createDummyProtocol(); - static protocol::TProtocol* createProtocol( + protocol::TProtocol* createProtocol( thrift::HostAddress& hostAddr, const thrift::ServerType::type serverType, const bool useFramedTransport, - //const SSLSocketParameters& sslParams, boost::shared_ptr& returnTransport); + inline std::string getSSLPropertyName(SSLProperty sslProperty) { + return m_sslParams.getSSLPropertyName(sslProperty); + } + + inline std::string getSSLPropertyValue( + const std::string& propertyName) const { + return m_sslParams.getSSLPropertyValue(propertyName); + } + void updateFailedServersForCurrent( std::set& failedServers, bool checkAllFailed, const std::exception& failure); @@ -217,7 +227,7 @@ namespace io { static std::string s_hostId; static boost::mutex s_globalLock; static bool s_initialized; - + SSLParameters m_sslParams; /** * Global initialization that is done only once. * The s_globalLock must be held in the invocation. @@ -238,6 +248,10 @@ namespace io { static thrift::ServerType::type getServerType(bool isServer, bool useBinaryProtocol, bool useSSL); + inline thrift::ServerType::type getServerType() const noexcept { + return m_reqdServerType; + } + inline bool isOpen() const noexcept { return m_isOpen; } @@ -264,6 +278,9 @@ namespace io { return m_isolationLevel; } + boost::shared_ptr createSSLSocket( + const std::string& host, int port); + void execute(thrift::StatementResult& result, const std::string& sql, const std::map& outputParams, @@ -397,7 +414,6 @@ namespace io { inline bool isFrameTransport() const noexcept { return m_useFramedTransport; } - }; } /* namespace impl */ diff --git a/native/src/snappyclient/cpp/impl/ControlConnection.cpp b/native/src/snappyclient/cpp/impl/ControlConnection.cpp index f75813b50..d3257b84e 100644 --- a/native/src/snappyclient/cpp/impl/ControlConnection.cpp +++ b/native/src/snappyclient/cpp/impl/ControlConnection.cpp @@ -61,14 +61,14 @@ using namespace io::snappydata::thrift; std::vector > ControlConnection::s_allConnections; boost::mutex ControlConnection::s_allConnsLock; -ControlConnection::ControlConnection(ClientService * const &service) : +ControlConnection::ControlConnection(ClientService* service) : m_serverGroups(service->getServerGrps()) { m_locators = service->getLocators(); m_framedTransport = service->isFrameTransport(); - m_snappyServerType = service->getServerType(true, false, false); + m_snappyServerType = service->getServerType(); m_controlHost = service->getCurrentHostAddress(); boost::assign::insert(m_snappyServerTypeSet)( - service->getServerType(true, false, false)); + service->getServerType()); std::copy(m_locators.begin(), m_locators.end(), std::inserter(m_controlHostSet, m_controlHostSet.end())); m_controlLocator = nullptr; @@ -76,7 +76,7 @@ ControlConnection::ControlConnection(ClientService * const &service) : const boost::optional ControlConnection::getOrCreateControlConnection( const std::vector& hostAddrs, - ClientService * const &service, const std::exception& failure) { + ClientService* service, const std::exception& failure) { // loop through all ControlConnections since size of this global list is // expected to be in single digit (total number of distributed systems) @@ -97,7 +97,7 @@ const boost::optional ControlConnection::getOrCreateControlC if (result == _locators.end()) { continue; } else { - auto serviceServerType = service->getServerType(true, false, false); // TODO: need to discuss with sumedh about this getServerType method + auto serviceServerType = service->getServerType(); auto contrConnServerType = controlConn->m_snappyServerType; if (contrConnServerType == serviceServerType) { return *controlConn; @@ -125,10 +125,10 @@ const boost::optional ControlConnection::getOrCreateControlC if (allConnSize == 0) { // first attempt of creating connection // if we reached here, then need to create a new ControlConnection - std::unique_ptr controlService( + std::unique_ptr controlConn( new ControlConnection(service)); thrift::HostAddress preferredServer; - controlService->getPreferredServer(preferredServer, failure, true); + controlConn->getPreferredServer(preferredServer, failure, service ,true); // check again if new control host already exist index = static_cast(s_allConnections.size()); while (--index >= 0) { @@ -142,7 +142,7 @@ const boost::optional ControlConnection::getOrCreateControlC return *controlConn; } } - s_allConnections.push_back(std::move(controlService)); + s_allConnections.push_back(std::move(controlConn)); return *s_allConnections.back(); } else { thrift::SnappyException ex; @@ -165,19 +165,20 @@ void ControlConnection::getLocatorPreferredServer( void ControlConnection::getPreferredServer( thrift::HostAddress& preferredServer, const std::exception& failure, - bool forFailover) { + ClientService* service, bool forFailover) { std::set failedServers; std::set serverGroups; return getPreferredServer(preferredServer, failure, failedServers, - serverGroups, forFailover); + serverGroups, service, forFailover); } void ControlConnection::getPreferredServer( thrift::HostAddress& preferredServer, const std::exception& failure, std::set& failedServers, - std::set& serverGroups, bool forFailover) { + std::set& serverGroups, ClientService* service, + bool forFailover) { if (m_controlLocator == nullptr) { - failoverToAvailableHost(failedServers, false, failure); + failoverToAvailableHost(failedServers, false, failure, service); forFailover = true; } boost::lock_guard localGuard(m_lock); @@ -237,7 +238,7 @@ void ControlConnection::getPreferredServer( failedServers.insert(m_controlHost); } m_controlLocator->getOutputProtocol()->getTransport()->close(); - failoverToAvailableHost(failedServers, true, tex); + failoverToAvailableHost(failedServers, true, tex, service); } catch (std::exception &ex) { throw unexpectedError(ex, m_controlHost); } @@ -273,7 +274,8 @@ void ControlConnection::searchRandomServer( void ControlConnection::failoverToAvailableHost( std::set& failedServers, - bool checkFailedControlHosts, const std::exception& failure) { + bool checkFailedControlHosts, const std::exception& failure, + ClientService* service) { boost::lock_guard localGuard(m_lock); for (auto iterator = m_controlHostSet.begin(); iterator != m_controlHostSet.end(); ++iterator) { @@ -300,9 +302,7 @@ void ControlConnection::failoverToAvailableHost( || m_snappyServerType == thrift::ServerType::THRIFT_SNAPPY_BP_SSL || m_snappyServerType == thrift::ServerType::THRIFT_SNAPPY_CP_SSL) { - TSSLSocketFactory sslSocketFactory; - tTransport = sslSocketFactory.createSocket(controlAddr.hostName, - controlAddr.port); + tTransport = service->createSSLSocket(controlAddr.hostName, controlAddr.port); } else if (m_snappyServerType == thrift::ServerType::THRIFT_LOCATOR_BP || m_snappyServerType == thrift::ServerType::THRIFT_LOCATOR_CP || m_snappyServerType == thrift::ServerType::THRIFT_SNAPPY_BP @@ -460,7 +460,6 @@ void ControlConnection::getConnectedHost(thrift::HostAddress& hostAddr, connectedHost = host; } } - } void ControlConnection::close(bool clearGlobal) { diff --git a/native/src/snappyclient/cpp/impl/ControlConnection.h b/native/src/snappyclient/cpp/impl/ControlConnection.h index df8fc1583..e60cc98ac 100644 --- a/native/src/snappyclient/cpp/impl/ControlConnection.h +++ b/native/src/snappyclient/cpp/impl/ControlConnection.h @@ -100,11 +100,12 @@ namespace io { m_serverGroups(std::set()), m_framedTransport(false) { } - ControlConnection(ClientService * const &service); + ControlConnection(ClientService* service); void failoverToAvailableHost( std::set& failedServers, - bool checkFailedControlHosts, const std::exception& failure); + bool checkFailedControlHosts, const std::exception& failure, + ClientService* service); void refreshAllHosts( const std::vector& allHosts); @@ -121,18 +122,20 @@ namespace io { std::set serverGroups); void getPreferredServer(thrift::HostAddress& preferredServer, - const std::exception& failure, bool forFailover = false); + const std::exception& failure, ClientService* service, + bool forFailover = false); public: static const boost::optional getOrCreateControlConnection( const std::vector& hostAddrs, - ClientService * const &service, const std::exception& failure); + ClientService* service, const std::exception& failure); void getPreferredServer(thrift::HostAddress& preferredServer, const std::exception& failure, std::set& failedServers, - std::set& serverGroups, bool forFailover = false); + std::set& serverGroups, ClientService* service, + bool forFailover = false); void searchRandomServer( const std::set& skipServers, diff --git a/native/src/snappyclient/cpp/impl/SSLParameters.cpp b/native/src/snappyclient/cpp/impl/SSLParameters.cpp new file mode 100644 index 000000000..8652d91c1 --- /dev/null +++ b/native/src/snappyclient/cpp/impl/SSLParameters.cpp @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +#include "SSLParameters.h" + +#include + +using namespace io::snappydata; +using namespace io::snappydata::client::impl; +using namespace io::snappydata::thrift; + +const std::set SSLParameters::s_sslProperties { + "protocol", "cipher-suites", "client-auth", "enabled-protocols", "keystore", + "keystore-password", "certificate", "certificate-password", "truststore", + "truststore-password" }; + +void SSLParameters::operator()(const std::string& str) { + size_t spos; + if ((spos = str.find('=')) != std::string::npos) { + std::string propertyValue = str.substr(spos + 1); + std::string propertyName = str.substr(0, spos); + setSSLProperty(propertyName, propertyValue); + return; + } +} + +void SSLParameters::setSSLProperty(std::string &propertyName, + std::string& value) { + auto itr = s_sslProperties.find(propertyName); + if (itr != s_sslProperties.end()) { + auto mapItr = m_sslPropValMap.find(propertyName); + if (mapItr == m_sslPropValMap.end()) { + m_sslPropValMap.insert( + std::pair(propertyName, value)); + } + } else { + throw std::invalid_argument(":Unknown SSL Property:" + propertyName); + } +} + +std::string SSLParameters::getSSLPropertyName(SSLProperty sslProperty) { + m_currentProperty = sslProperty; + switch (sslProperty) { + case SSLProperty::PROTOCOL: + return "protocol"; + case SSLProperty::CIPHERSUITES: + return "cipher-suites"; + case SSLProperty::CLIENTAUTH: + return "client-auth"; + case SSLProperty::KEYSTORE: + return "keystore"; + case SSLProperty::KEYSTOREPASSWORD: + return "keystore-password"; + case SSLProperty::CERTIFICATE: + return "certificate"; + case SSLProperty::CERTIFICATEPASSWORD: + return "certificate-password"; + case SSLProperty::TRUSTSTORE: + return "truststore"; + case SSLProperty::TRUSTSTOREPASSWORD: + return "truststore-password"; + default: + throw std::invalid_argument( + ":Unknown SSL Property enum: " + + std::to_string(static_cast(sslProperty))); + } +} + +std::string SSLParameters::getSSLPropertyValue( + const std::string &propertyName) const { + auto itr = s_sslProperties.find(propertyName); + if (itr != s_sslProperties.end()) { + auto itr = m_sslPropValMap.find(propertyName); + if (itr != m_sslPropValMap.end()) { + return itr->second; + } else { + return ""; + } + } else { + throw std::invalid_argument(":Unknown SSL Property: " + propertyName); + } +} + +SSLSocketFactory::SSLSocketFactory(SSLParameters& params) : + TSSLSocketFactory(getProtocol(params)), m_params(params) { + overrideDefaultPasswordCallback(); // use getPassword override +} + +SSLProtocol SSLSocketFactory::getProtocol(const SSLParameters& params) { + std::string propVal = params.getSSLPropertyValue("protocol"); + if (!propVal.empty()) { + if (boost::iequals(propVal, "SSLTLS")) { + return SSLProtocol::SSLTLS; + } else if (boost::iequals(propVal, "SSL3")) { + return SSLProtocol::SSLv3; + } else if (boost::iequals(propVal, "TLS1.0")) { + return SSLProtocol::TLSv1_0; + } else if (boost::iequals(propVal, "TLS1.1")) { + return SSLProtocol::TLSv1_1; + } else if (boost::iequals(propVal, "TLS1.2")) { + return SSLProtocol::TLSv1_2; + } else { + throw std::invalid_argument(":Unknown SSL protocol: " + propVal); + } + } else { + return SSLProtocol::SSLTLS; + } +} + +void SSLSocketFactory::getPassword(std::string& password, int size) { + /* TODO: the password fields should be fetched from UI for ODBC */ + + SSLProperty sslProperty = m_params.m_currentProperty; + std::string name = m_params.getSSLPropertyName(sslProperty); + switch (sslProperty) { + case SSLProperty::KEYSTORE: + name = m_params.getSSLPropertyName(SSLProperty::KEYSTOREPASSWORD); + break; + case SSLProperty::CERTIFICATE: + name = m_params.getSSLPropertyName(SSLProperty::CERTIFICATEPASSWORD); + break; + case SSLProperty::TRUSTSTORE: + name = m_params.getSSLPropertyName(SSLProperty::TRUSTSTOREPASSWORD); + break; + default: + throw std::invalid_argument(":Expected password SSL Property: " + name); + } + password = m_params.getSSLPropertyValue(name); +} diff --git a/native/src/snappyclient/cpp/impl/SSLParameters.h b/native/src/snappyclient/cpp/impl/SSLParameters.h new file mode 100644 index 000000000..073e366f1 --- /dev/null +++ b/native/src/snappyclient/cpp/impl/SSLParameters.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +#ifndef SSLPARAMETERS_H_ +#define SSLPARAMETERS_H_ + +#include "ClientBase.h" +#include "thrift/transport/TSSLSocket.h" + +using namespace apache::thrift::transport; + +namespace io { +namespace snappydata { +namespace client { +namespace impl { + +enum class SSLProperty { + PROTOCOL, + CIPHERSUITES, + CLIENTAUTH, + KEYSTORE, + KEYSTOREPASSWORD, + CERTIFICATE, + CERTIFICATEPASSWORD, + TRUSTSTORE, + TRUSTSTOREPASSWORD +}; + +class SSLSocketFactory; + +class SSLParameters { +private: + static const std::set s_sslProperties; + std::map m_sslPropValMap; + SSLProperty m_currentProperty; + + friend class SSLSocketFactory; + +public: + SSLParameters() : + m_sslPropValMap(), m_currentProperty(SSLProperty::PROTOCOL) { + } + + void setSSLProperty(std::string &propertyName, std::string& value); + std::string getSSLPropertyName(SSLProperty sslProperty); + std::string getSSLPropertyValue(const std::string &propertyName) const; + void operator()(const std::string& str); +}; + +class SSLSocketFactory : public TSSLSocketFactory { +public: + /** + * Constructor + * + * @param params SSL parameters to use. + */ + SSLSocketFactory(SSLParameters& params); + virtual ~SSLSocketFactory() { + } + +protected: + virtual void getPassword(std::string& password, int size); + +private: + SSLParameters& m_params; + + static SSLProtocol getProtocol(const SSLParameters& params); +}; + +} /* namespace impl */ +} /* namespace client */ +} /* namespace snappydata */ +} /* namespace io */ + +#endif /* SSLPARAMETERS_H_ */ diff --git a/native/src/snappyclient/headers/ClientAttribute.h b/native/src/snappyclient/headers/ClientAttribute.h index c4c142cf4..fb636f435 100644 --- a/native/src/snappyclient/headers/ClientAttribute.h +++ b/native/src/snappyclient/headers/ClientAttribute.h @@ -93,6 +93,9 @@ class ClientAttribute { static const std::string ROUTE_QUERY; static const std::string THRIFT_USE_BINARY_PROTOCOL; static const std::string THRIFT_USE_FRAMED_TRANSPORT; + static const std::string AQP_ERROR; + static const std::string AQP_CONFIDENCE; + static const std::string AQP_BEHAVIOR; static const int DEFAULT_LOGIN_TIMEOUT = 0; static const int DEFAULT_SINGLE_HOP_MAX_CONN_PER_SERVER = 5; From cc9cf575a409dea5da8c34adc48b3ca1ed467322 Mon Sep 17 00:00:00 2001 From: Piyush Bisen <42795748+bisenpiyush@users.noreply.github.com> Date: Thu, 9 Jan 2020 18:49:25 +0530 Subject: [PATCH 26/27] Converting catalog,schema and table name into uppercase (#546) Changes to make metadata queries case-insensitive for calalog,schema and table name but retains the old behaviour for pure GemFireXD cluster --- .../engine/store/RegionEntryUtils.java | 3 ++- .../impl/jdbc/EmbedDatabaseMetaData.java | 4 +++- .../pivotal/gemfirexd/ClientServerDUnit.java | 22 +++++++++---------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java index bd3896715..15ed0c38b 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java @@ -1549,7 +1549,8 @@ public boolean isAdmin() { @Override public boolean isSnappyStore() { - return Misc.getMemStoreBooting().isSnappyStore(); + GemFireStore store = Misc.getMemStoreBootingNoThrow(); + return store != null && store.isSnappyStore(); } @Override diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/jdbc/EmbedDatabaseMetaData.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/jdbc/EmbedDatabaseMetaData.java index 0d1b7e428..b60e673e1 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/jdbc/EmbedDatabaseMetaData.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/jdbc/EmbedDatabaseMetaData.java @@ -53,6 +53,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more import java.io.IOException; import java.io.InputStream; +import com.gemstone.gemfire.internal.cache.GemFireCacheImpl; import com.pivotal.gemfirexd.internal.engine.Misc; import com.pivotal.gemfirexd.internal.engine.diag.SysVTIs; import com.pivotal.gemfirexd.internal.engine.locks.GfxdLockSet; @@ -3853,7 +3854,8 @@ private PreparedStatement prepareSPS(String spsName, } static final protected String swapNull(String s) { - return (s == null ? "%" : s); + boolean isSnappyStore = GemFireCacheImpl.getInternalProductCallbacks().isSnappyStore(); + return (s == null ? "%" : (isSnappyStore ? StringUtil.SQLToUpperCase(s) : s)); } /** diff --git a/gemfirexd/tools/src/dunit/java/com/pivotal/gemfirexd/ClientServerDUnit.java b/gemfirexd/tools/src/dunit/java/com/pivotal/gemfirexd/ClientServerDUnit.java index 7f9f054e9..db61eac41 100644 --- a/gemfirexd/tools/src/dunit/java/com/pivotal/gemfirexd/ClientServerDUnit.java +++ b/gemfirexd/tools/src/dunit/java/com/pivotal/gemfirexd/ClientServerDUnit.java @@ -1950,7 +1950,7 @@ public Object call() throws Exception { // check new connections opened on locator and servers assertNumConnections(-1, 0, locator); - assertNumConnections(1, -1, 1, 2); + assertNumConnections(1, -2, 1, 2); // Create a table Statement stmt = conn.createStatement(); @@ -1981,7 +1981,7 @@ public Object call() throws Exception { assertFalse(rs.next()); assertNumConnections(-1, 0, locator); - assertNumConnections(1, -1, 1, 2); + assertNumConnections(1, -2, 1, 2); // now open another connection with server1 URL final Connection conn2 = TestUtil.getNetConnection(localHost.getCanonicalHostName(), @@ -1989,7 +1989,7 @@ public Object call() throws Exception { // check new connection opened on servers successfully load-balanced assertNumConnections(-1, 0, locator); - assertNumConnections(-3, -1, 1, 2); + assertNumConnections(-3, -2, 1, 2); stmt = conn2.createStatement(); rs = stmt.executeQuery("select * from TESTTABLE"); @@ -2006,14 +2006,14 @@ public Object call() throws Exception { assertFalse(rs.next()); assertNumConnections(-1, 0, locator); - assertNumConnections(-3, -1, 1, 2); + assertNumConnections(-3, -2, 1, 2); // now a third connection Connection conn3 = TestUtil.getNetConnection(localHost.getCanonicalHostName(), netPort, null, new Properties()); assertNumConnections(-1, 0, locator); - assertNumConnections(-4, -1, 1, 2); + assertNumConnections(-4, -2, 1, 2); // add expected exception for server connection failure addExpectedException(null, new Object[] { java.net.ConnectException.class, @@ -2043,7 +2043,7 @@ public Object call() throws Exception { // check connections opened on second server assertNumConnections(-1, 0, locator); - assertNumConnections(-4, -2, 2); + assertNumConnections(-4, -3, 2); removeExpectedException(null, new Object[] { java.net.ConnectException.class, DisconnectException.class, @@ -2079,7 +2079,7 @@ public Object call() throws Exception { // check connection opened on second server assertNumConnections(-1, 0, locator); - assertNumConnections(-5, -2, 2); + assertNumConnections(-5, -3, 2); // now drop the table and close the connections stmt = conn.createStatement(); @@ -2088,22 +2088,22 @@ public Object call() throws Exception { conn.close(); assertNumConnections(-1, 0, locator); - assertNumConnections(-5, -3, 2); + assertNumConnections(-5, -4, 2); conn2.close(); assertNumConnections(-1, 0, locator); - assertNumConnections(-5, -3, 2); + assertNumConnections(-5, -4, 2); conn3.close(); assertNumConnections(-1, 0, locator); - assertNumConnections(-5, -4, 2); + assertNumConnections(-5, -5, 2); conn4.close(); assertNumConnections(-1, 0, locator); - assertNumConnections(-5, -5, 2); + assertNumConnections(-5, -6, 2); } /** test for exception when there is no data member available */ From bf7d28d6b53e6839351c56f7ac27f165225841b4 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Sun, 19 Jan 2020 03:44:56 +0530 Subject: [PATCH 27/27] Fixing compilation for g++ <= 4.8.x --- .../cpp/thrift/snappydata_struct_ColumnValue.cpp | 2 +- .../headers/snappydata_struct_ColumnValue.h | 14 +++++++------- .../overrides/snappydata_struct_ColumnValue.cpp | 2 +- .../overrides/snappydata_struct_ColumnValue.h | 14 +++++++------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/native/src/snappyclient/cpp/thrift/snappydata_struct_ColumnValue.cpp b/native/src/snappyclient/cpp/thrift/snappydata_struct_ColumnValue.cpp index ca837b364..469fb3668 100644 --- a/native/src/snappyclient/cpp/thrift/snappydata_struct_ColumnValue.cpp +++ b/native/src/snappyclient/cpp/thrift/snappydata_struct_ColumnValue.cpp @@ -370,7 +370,7 @@ namespace { } const std::pair ColumnValue:: -s_typeMap[io::snappydata::VariadicSize::size] = { +s_typeMap[io::snappydata::VariadicCount::count] = { std::make_pair(SnappyType::BOOLEAN, "bool_val"), std::make_pair(SnappyType::TINYINT, "byte_val"), std::make_pair(SnappyType::SMALLINT, "i16_val"), diff --git a/native/src/snappyclient/headers/snappydata_struct_ColumnValue.h b/native/src/snappyclient/headers/snappydata_struct_ColumnValue.h index 7e9403f0c..f8b927412 100644 --- a/native/src/snappyclient/headers/snappydata_struct_ColumnValue.h +++ b/native/src/snappyclient/headers/snappydata_struct_ColumnValue.h @@ -69,14 +69,14 @@ namespace io { namespace snappydata { -template -struct VariadicSize { - static constexpr int size = sizeof...(T); +template +struct VariadicCount { + static constexpr int count = 1; }; -template -struct VariadicSize > { - static constexpr int size = sizeof...(T); +template +struct VariadicCount > { + static constexpr int count = 1 + sizeof...(TN); }; namespace thrift { @@ -229,7 +229,7 @@ class ColumnValue { UnionType m_val; static const std::pair - s_typeMap[VariadicSize::size]; + s_typeMap[VariadicCount::count]; friend struct std::hash; diff --git a/native/src/snappyclient/overrides/snappydata_struct_ColumnValue.cpp b/native/src/snappyclient/overrides/snappydata_struct_ColumnValue.cpp index ca837b364..469fb3668 100644 --- a/native/src/snappyclient/overrides/snappydata_struct_ColumnValue.cpp +++ b/native/src/snappyclient/overrides/snappydata_struct_ColumnValue.cpp @@ -370,7 +370,7 @@ namespace { } const std::pair ColumnValue:: -s_typeMap[io::snappydata::VariadicSize::size] = { +s_typeMap[io::snappydata::VariadicCount::count] = { std::make_pair(SnappyType::BOOLEAN, "bool_val"), std::make_pair(SnappyType::TINYINT, "byte_val"), std::make_pair(SnappyType::SMALLINT, "i16_val"), diff --git a/native/src/snappyclient/overrides/snappydata_struct_ColumnValue.h b/native/src/snappyclient/overrides/snappydata_struct_ColumnValue.h index 7e9403f0c..f8b927412 100644 --- a/native/src/snappyclient/overrides/snappydata_struct_ColumnValue.h +++ b/native/src/snappyclient/overrides/snappydata_struct_ColumnValue.h @@ -69,14 +69,14 @@ namespace io { namespace snappydata { -template -struct VariadicSize { - static constexpr int size = sizeof...(T); +template +struct VariadicCount { + static constexpr int count = 1; }; -template -struct VariadicSize > { - static constexpr int size = sizeof...(T); +template +struct VariadicCount > { + static constexpr int count = 1 + sizeof...(TN); }; namespace thrift { @@ -229,7 +229,7 @@ class ColumnValue { UnionType m_val; static const std::pair - s_typeMap[VariadicSize::size]; + s_typeMap[VariadicCount::count]; friend struct std::hash;