Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[branch-2.0](binlog) Support drop view binlog (#39781) #43408

Closed
wants to merge 1 commit into from
+52 −37
Closed

[branch-2.0](binlog) Support drop view binlog (#39781) #43408

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -935,7 +935,8 @@ public void processDropMaterializedView(DropMaterializedViewStmt dropMaterialize
// Step3: log drop mv operation
EditLog editLog = Env.getCurrentEnv().getEditLog();
editLog.logDropRollup(
new DropInfo(db.getId(), olapTable.getId(), olapTable.getName(), mvIndexId, false, 0));
new DropInfo(db.getId(), olapTable.getId(), olapTable.getName(), mvIndexId, false, false, 0));
deleteIndexList.add(mvIndexId);
LOG.info("finished drop materialized view [{}] in table [{}]", mvName, olapTable.getName());
} catch (MetaNotFoundException e) {
if (dropMaterializedViewStmt.isIfExists()) {
Expand Down
31 changes: 13 additions & 18 deletions fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2054,29 +2054,24 @@ private Status allTabletCommitted(boolean isReplay) {
}

private Status dropAllNonRestoredTableAndPartitions(Database db) {
Set<String> restoredViews = jobInfo.newBackupObjects.views.stream()
.map(view -> view.name).collect(Collectors.toSet());

try {
for (Table table : db.getTables()) {
long tableId = table.getId();
String tableName = table.getName();
TableType tableType = table.getType();
if (tableType == TableType.OLAP) {
BackupOlapTableInfo backupTableInfo = jobInfo.backupOlapTableObjects.get(tableName);
if (tableType == TableType.OLAP && backupTableInfo != null) {
// drop the non restored partitions.
dropNonRestoredPartitions(db, (OlapTable) table, backupTableInfo);
} else if (isCleanTables) {
// otherwise drop the entire table.
LOG.info("drop non restored table {}, table id: {}. {}", tableName, tableId, this);
boolean isForceDrop = false; // move this table into recyclebin.
env.getInternalCatalog().dropTableWithoutCheck(db, table, isForceDrop);
}
} else if (tableType == TableType.VIEW && isCleanTables && !restoredViews.contains(tableName)) {
LOG.info("drop non restored view {}, table id: {}. {}", tableName, tableId, this);
boolean isForceDrop = false; // move this view into recyclebin.
env.getInternalCatalog().dropTableWithoutCheck(db, table, isForceDrop);
BackupOlapTableInfo backupTableInfo = jobInfo.backupOlapTableObjects.get(tableName);
if (tableType != TableType.OLAP && tableType != TableType.ODBC && tableType != TableType.VIEW) {
continue;
}
if (tableType == TableType.OLAP && backupTableInfo != null) {
// drop the non restored partitions.
dropNonRestoredPartitions(db, (OlapTable) table, backupTableInfo);
} else if (isCleanTables) {
// otherwise drop the entire table.
LOG.info("drop non restored table {}({}). {}", tableName, tableId, this);
boolean isView = false;
boolean isForceDrop = false; // move this table into recyclebin.
env.getInternalCatalog().dropTableWithoutCheck(db, table, isView, isForceDrop);
Comment on lines +2062 to +2074
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@wyxxxcat This patch is staled, please update.

}
}
return Status.OK;
Expand Down
7 changes: 6 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/binlog/DropTableRecord.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ public class DropTableRecord {
private long tableId;
@SerializedName(value = "tableName")
private String tableName;
@SerializedName(value = "isView")
private boolean isView = false;
@SerializedName(value = "rawSql")
private String rawSql;

Expand All @@ -39,7 +41,10 @@ public DropTableRecord(long commitSeq, DropInfo info) {
this.dbId = info.getDbId();
this.tableId = info.getTableId();
this.tableName = info.getTableName();
this.rawSql = String.format("DROP TABLE IF EXISTS `%s`", this.tableName);
this.isView = info.isView();
this.rawSql = info.isView()
? String.format("DROP VIEW IF EXISTS `%s`", this.tableName)
: String.format("DROP TABLE IF EXISTS `%s`", this.tableName);
}

public long getCommitSeq() {
Expand Down
20 changes: 13 additions & 7 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -916,23 +916,26 @@ public void dropTable(DropTableStmt stmt) throws DdlException {
}
}

dropTableInternal(db, table, stmt.isForceDrop());
dropTableInternal(db, table, stmt.isView(), stmt.isForceDrop(), watch, costTimes);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

only pick the necessary parameters.

} catch (UserException e) {
throw new DdlException(e.getMessage(), e.getMysqlErrorCode());
} finally {
db.writeUnlock();
}
LOG.info("finished dropping table: {} from db: {}, is force: {}", tableName, dbName, stmt.isForceDrop());
watch.stop();
costTimes.put("6:total", watch.getTime());
LOG.info("finished dropping table: {} from db: {}, is view: {}, is force: {}, cost: {}",
tableName, dbName, stmt.isView(), stmt.isForceDrop(), costTimes);
Comment on lines +925 to +928
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should merge this conflict manually, eg:

LOG.info("finished dropping table: {} from db: {}, is force: {}, is view: {}", tableName, dbName, stmt.isForceDrop(), stmt.isView());

}

// drop table without any check.
public void dropTableWithoutCheck(Database db, Table table, boolean forceDrop) throws DdlException {
public void dropTableWithoutCheck(Database db, Table table, boolean isView, boolean forceDrop) throws DdlException {
if (!db.writeLockIfExist()) {
return;
}
try {
LOG.info("drop table {} without check, force: {}", table.getQualifiedName(), forceDrop);
dropTableInternal(db, table, forceDrop);
dropTableInternal(db, table, isView, forceDrop, null, null);
} catch (Exception e) {
LOG.warn("drop table without check", e);
throw e;
Expand All @@ -942,7 +945,8 @@ public void dropTableWithoutCheck(Database db, Table table, boolean forceDrop) t
}

// Drop a table, the db lock must hold.
private void dropTableInternal(Database db, Table table, boolean forceDrop) throws DdlException {
private void dropTableInternal(Database db, Table table, boolean isView, boolean forceDrop,
StopWatch watch, Map<String, Long> costTimes) throws DdlException {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove StopWatch watch, Map<String, Long> costTimes

table.writeLock();
String tableName = table.getName();
long recycleTime = 0;
Expand All @@ -955,7 +959,9 @@ private void dropTableInternal(Database db, Table table, boolean forceDrop) thro
table.writeUnlock();
}

DropInfo info = new DropInfo(db.getId(), table.getId(), tableName, -1L, forceDrop, recycleTime);
Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentEnv().getCurrentCatalog().getId(),
db.getId(), table.getId());
DropInfo info = new DropInfo(db.getId(), table.getId(), tableName, -1L, isView, forceDrop, recycleTime);
Env.getCurrentEnv().getEditLog().logDropTable(info);
Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentEnv().getCurrentCatalog().getId(),
db.getId(), table.getId());
Expand Down Expand Up @@ -2732,7 +2738,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
try {
dropTable(db, tableId, true, false, 0L);
if (hadLogEditCreateTable) {
DropInfo info = new DropInfo(db.getId(), tableId, olapTable.getName(), -1L, true, 0L);
DropInfo info = new DropInfo(db.getId(), tableId, olapTable.getName(), -1L, false, true, 0L);
Env.getCurrentEnv().getEditLog().logDropTable(info);
}
} catch (Exception ex) {
Expand Down
16 changes: 12 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ public class DropInfo implements Writable {
private String tableName; // not used in equals and hashCode
@SerializedName(value = "indexId")
private long indexId;
@SerializedName(value = "isView")
private boolean isView = false;
@SerializedName(value = "forceDrop")
private boolean forceDrop = false;
@SerializedName(value = "recycleTime")
Expand All @@ -46,11 +48,13 @@ public class DropInfo implements Writable {
public DropInfo() {
}

public DropInfo(long dbId, long tableId, String tableName, long indexId, boolean forceDrop, long recycleTime) {
public DropInfo(long dbId, long tableId, String tableName, long indexId, boolean isView, boolean forceDrop,
long recycleTime) {
this.dbId = dbId;
this.tableId = tableId;
this.tableName = tableName;
this.indexId = indexId;
this.isView = isView;
this.forceDrop = forceDrop;
this.recycleTime = recycleTime;
}
Expand All @@ -71,12 +75,16 @@ public long getIndexId() {
return this.indexId;
}

public boolean isView() {
return this.isView;
}

public boolean isForceDrop() {
return forceDrop;
return this.forceDrop;
}

public Long getRecycleTime() {
return recycleTime;
return this.recycleTime;
}

@Override
Expand Down Expand Up @@ -119,7 +127,7 @@ public boolean equals(Object obj) {
DropInfo info = (DropInfo) obj;

return (dbId == info.dbId) && (tableId == info.tableId) && (indexId == info.indexId)
&& (forceDrop == info.forceDrop) && (recycleTime == info.recycleTime);
&& (isView == info.isView) && (forceDrop == info.forceDrop) && (recycleTime == info.recycleTime);
}

public String toJson() {
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) {
for (long indexId : batchDropInfo.getIndexIdSet()) {
env.getMaterializedViewHandler().replayDropRollup(
new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(),
batchDropInfo.getTableName(), indexId, false, 0),
batchDropInfo.getTableName(), indexId, false, false, 0),
env);
}
break;
Expand Down
10 changes: 5 additions & 5 deletions fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public void testDropInfoSerialization() throws Exception {
DropInfo info1 = new DropInfo();
info1.write(dos);

DropInfo info2 = new DropInfo(1, 2, "t2", -1, true, 0);
DropInfo info2 = new DropInfo(1, 2, "t2", -1, false, true, 0);
info2.write(dos);

dos.flush();
Expand All @@ -65,10 +65,10 @@ public void testDropInfoSerialization() throws Exception {

Assert.assertEquals(rInfo2, rInfo2);
Assert.assertNotEquals(rInfo2, this);
Assert.assertNotEquals(info2, new DropInfo(0, 2, "t2", -1L, true, 0));
Assert.assertNotEquals(info2, new DropInfo(1, 0, "t0", -1L, true, 0));
Assert.assertNotEquals(info2, new DropInfo(1, 2, "t2", -1L, false, 0));
Assert.assertEquals(info2, new DropInfo(1, 2, "t2", -1L, true, 0));
Assert.assertNotEquals(info2, new DropInfo(0, 2, "t2", -1L, false, true, 0));
Assert.assertNotEquals(info2, new DropInfo(1, 0, "t0", -1L, false, true, 0));
Assert.assertNotEquals(info2, new DropInfo(1, 2, "t2", -1L, false, false, 0));
Assert.assertEquals(info2, new DropInfo(1, 2, "t2", -1L, false, true, 0));

// 3. delete files
dis.close();
Expand Down
Loading