Skip to content

Commit 2fedc86

Browse files
committed
HIVE-28988: Handle materialized view commit txn during hive acid repl
* Details: * Commit txn of alter mv command was getting populated by write event infos. * This was causing a repl failure on load. * This is now fixed. * Testing: * Added itest.
1 parent ba1b83d commit 2fedc86

File tree

2 files changed

+30
-1
lines changed

2 files changed

+30
-1
lines changed

itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3197,10 +3197,13 @@ public void testMaterializedViewOnAcidTableReplication() throws Throwable {
31973197
.run("insert into t1 values (1)")
31983198
.run("insert into t1 values (2)")
31993199
.run("create materialized view mat_view as select * from t1")
3200+
.run("create materialized view mat_view_txn TBLPROPERTIES ('transactional'='true') as select * from t1")
32003201
.run("show tables")
3201-
.verifyResults(new String[]{"t1", "mat_view"})
3202+
.verifyResults(new String[]{"t1", "mat_view", "mat_view_txn"})
32023203
.run("select * from mat_view")
32033204
.verifyResults(new String[]{"1", "2"})
3205+
.run("select * from mat_view_txn")
3206+
.verifyResults(new String[]{"1", "2"})
32043207
.dump(primaryDbName);
32053208

32063209
//confirm materialized-view not replicated
@@ -3217,6 +3220,9 @@ public void testMaterializedViewOnAcidTableReplication() throws Throwable {
32173220
.run("alter materialized view mat_view rebuild")
32183221
.run("select * from mat_view")
32193222
.verifyResults(new String[]{"1", "2", "3"})
3223+
.run("alter materialized view mat_view_txn rebuild")
3224+
.run("select * from mat_view_txn")
3225+
.verifyResults(new String[]{"1", "2", "3"})
32203226
.dump(primaryDbName);
32213227

32223228
//confirm materialized-view not replicated
@@ -3232,6 +3238,9 @@ public void testMaterializedViewOnAcidTableReplication() throws Throwable {
32323238
.run("alter materialized view mat_view disable rewrite")
32333239
.run("select * from mat_view")
32343240
.verifyResults(new String[]{"1", "2", "3"})
3241+
.run("alter materialized view mat_view_txn disable rewrite")
3242+
.run("select * from mat_view_txn")
3243+
.verifyResults(new String[]{"1", "2", "3"})
32353244
.dump(primaryDbName);
32363245

32373246
//confirm materialized-view not replicated

ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CommitTxnHandler.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
2828
import org.apache.hadoop.hive.metastore.api.WriteEventInfo;
2929
import org.apache.hadoop.hive.metastore.messaging.CommitTxnMessage;
30+
import org.apache.hadoop.hive.metastore.messaging.MessageBuilder;
3031
import org.apache.hadoop.hive.metastore.utils.StringUtils;
3132
import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
3233
import org.apache.hadoop.hive.ql.metadata.HiveFatalException;
@@ -43,6 +44,7 @@
4344
import java.io.IOException;
4445
import java.util.ArrayList;
4546
import java.util.List;
47+
import java.util.stream.Collectors;
4648

4749
class CommitTxnHandler extends AbstractEventHandler<CommitTxnMessage> {
4850

@@ -128,6 +130,20 @@ private List<WriteEventInfo> getAllWriteEventInfo(Context withinContext) throws
128130
})));
129131
}
130132

133+
private List<WriteEventInfo> getAllWriteEventInfoExceptMV(List<WriteEventInfo> writeEventInfoList) {
134+
return writeEventInfoList.stream().filter(writeEventInfo -> {
135+
try {
136+
if (writeEventInfo.getTableObj() != null) {
137+
Table table = new Table((org.apache.hadoop.hive.metastore.api.Table) MessageBuilder.getTObj(writeEventInfo.getTableObj(), org.apache.hadoop.hive.metastore.api.Table.class));
138+
return !table.isMaterializedView();
139+
}
140+
return true;
141+
} catch (Exception e) {
142+
throw new RuntimeException(e);
143+
}
144+
}).collect(Collectors.toList());
145+
}
146+
131147
@Override
132148
public void handle(Context withinContext) throws Exception {
133149
if (!ReplUtils.includeAcidTableInDump(withinContext.hiveConf)) {
@@ -168,6 +184,10 @@ public void handle(Context withinContext) throws Exception {
168184
}
169185
}
170186

187+
// Filtering out all write event infos related to materialized view
188+
if (writeEventInfoList != null) {
189+
writeEventInfoList = getAllWriteEventInfoExceptMV(writeEventInfoList);
190+
}
171191
int numEntry = (writeEventInfoList != null ? writeEventInfoList.size() : 0);
172192
if (numEntry != 0) {
173193
eventMessage.addWriteEventInfo(writeEventInfoList);

0 commit comments

Comments
 (0)