Skip to content

Commit a778272

Browse files
dveedenlance6716
dveeden
and
lance6716
authored
replication: Support GTID tag in PreviousGTIDsEvent (#952)
* replication: Support GTID tag in PreviousGTIDsEvent Issue: ref #845 The `PreviousGTIDsEvent` / `PREVIOUS_GTIDS_LOG_EVENT` has changed to work with tagged GTIDs. First the `uuidCount` has changed, it encodes the GTID format. Here format 1 is tagged and format 0 is untagged. Then each entry may have a tag. If there is a tag then the uuid itself isn't printed but the tag is appended to the last entry. Examples: `896e7882-18fe-11ef-ab88-22222d34d411:1-3` regular format, compatible with both formats `896e7882-18fe-11ef-ab88-22222d34d411:1-4:aaaa:1` tagged format. Combination of - `896e7882-18fe-11ef-ab88-22222d34d411:1-4` - `896e7882-18fe-11ef-ab88-22222d34d411:aaaa:1` `896e7882-18fe-11ef-ab88-22222d34d411:1-4:aaaa:1:abc:1-3:bbbbb:1:bbbbbb:1:x:1,896e7882-18fe-11ef-ab88-22222d34d412:1-2` Combination of: ``` 896e7882-18fe-11ef-ab88-22222d34d411:1-4 :aaaa:1 🔤1-3 :bbbbb:1 :bbbbbb:1 ❌1, 896e7882-18fe-11ef-ab88-22222d34d412:1-2 ``` Please also see: `mysqlbinlog --read-from-remote-server --hexdump $binlogfile` to see how MySQL encodes/decodes this. See also: - https://dev.mysql.com/doc/refman/8.4/en/replication-gtids-concepts.html * Add test * Add more tests * Update replication/event.go Co-authored-by: lance6716 <[email protected]> * Update based on review * add my suggestions Signed-off-by: lance6716 <[email protected]> * add test for long tag Signed-off-by: lance6716 <[email protected]> --------- Signed-off-by: lance6716 <[email protected]> Co-authored-by: lance6716 <[email protected]>
1 parent 6bb9e67 commit a778272

File tree

2 files changed

+121
-10
lines changed

2 files changed

+121
-10
lines changed

‎replication/event.go

+67-10
Original file line numberDiff line numberDiff line change
@@ -229,34 +229,91 @@ type PreviousGTIDsEvent struct {
229229
GTIDSets string
230230
}
231231

232+
type GtidFormat int
233+
234+
const (
235+
GtidFormatClassic = iota
236+
GtidFormatTagged
237+
)
238+
239+
// Decode the number of sids (source identifiers) and if it is using
240+
// tagged GTIDs or classic (non-tagged) GTIDs.
241+
//
242+
// Note that each gtid tag increases the sidno here, so a single UUID
243+
// might turn up multiple times if there are multipl tags.
244+
//
245+
// see also:
246+
// decode_nsids_format in mysql/mysql-server
247+
// https://github.com/mysql/mysql-server/blob/61a3a1d8ef15512396b4c2af46e922a19bf2b174/sql/rpl_gtid_set.cc#L1363-L1378
248+
func decodeSid(data []byte) (format GtidFormat, sidnr uint64) {
249+
if data[7] == 1 {
250+
format = GtidFormatTagged
251+
}
252+
253+
if format == GtidFormatTagged {
254+
masked := make([]byte, 8)
255+
copy(masked, data[1:7])
256+
sidnr = binary.LittleEndian.Uint64(masked)
257+
return
258+
}
259+
sidnr = binary.LittleEndian.Uint64(data[:8])
260+
return
261+
}
262+
232263
func (e *PreviousGTIDsEvent) Decode(data []byte) error {
233264
pos := 0
234-
uuidCount := binary.LittleEndian.Uint16(data[pos : pos+8])
265+
266+
format, uuidCount := decodeSid(data)
235267
pos += 8
236268

237269
previousGTIDSets := make([]string, uuidCount)
238-
for i := range previousGTIDSets {
270+
271+
currentSetnr := 0
272+
var buf strings.Builder
273+
for range previousGTIDSets {
239274
uuid := e.decodeUuid(data[pos : pos+16])
240275
pos += 16
276+
var tag string
277+
if format == GtidFormatTagged {
278+
tagLength := int(data[pos]) / 2
279+
pos += 1
280+
if tagLength > 0 { // 0 == no tag, >0 == tag
281+
tag = string(data[pos : pos+tagLength])
282+
pos += tagLength
283+
}
284+
}
285+
286+
if len(tag) > 0 {
287+
buf.WriteString(":")
288+
buf.WriteString(tag)
289+
} else {
290+
if currentSetnr != 0 {
291+
buf.WriteString(",")
292+
}
293+
buf.WriteString(uuid)
294+
currentSetnr += 1
295+
}
296+
241297
sliceCount := binary.LittleEndian.Uint16(data[pos : pos+8])
242298
pos += 8
243-
intervals := make([]string, sliceCount)
244-
for i := range intervals {
299+
for range sliceCount {
300+
buf.WriteString(":")
301+
245302
start := e.decodeInterval(data[pos : pos+8])
246303
pos += 8
247304
stop := e.decodeInterval(data[pos : pos+8])
248305
pos += 8
249-
interval := ""
250306
if stop == start+1 {
251-
interval = fmt.Sprintf("%d", start)
307+
fmt.Fprintf(&buf, "%d", start)
252308
} else {
253-
interval = fmt.Sprintf("%d-%d", start, stop-1)
309+
fmt.Fprintf(&buf, "%d-%d", start, stop-1)
254310
}
255-
intervals[i] = interval
256311
}
257-
previousGTIDSets[i] = fmt.Sprintf("%s:%s", uuid, strings.Join(intervals, ":"))
312+
if len(tag) == 0 {
313+
currentSetnr += 1
314+
}
258315
}
259-
e.GTIDSets = strings.Join(previousGTIDSets, ",")
316+
e.GTIDSets = buf.String()
260317
return nil
261318
}
262319

‎replication/event_test.go

+54
Original file line numberDiff line numberDiff line change
@@ -140,3 +140,57 @@ func TestIntVarEvent(t *testing.T) {
140140
require.Equal(t, INSERT_ID, ev.Type)
141141
require.Equal(t, uint64(23), ev.Value)
142142
}
143+
144+
func TestDecodeSid(t *testing.T) {
145+
testcases := []struct {
146+
input []byte
147+
gtidFormat GtidFormat
148+
uuidCount uint64
149+
}{
150+
{[]byte{1, 2, 0, 0, 0, 0, 0, 1}, GtidFormatTagged, 2},
151+
{[]byte{1, 1, 0, 0, 0, 0, 0, 1}, GtidFormatTagged, 1},
152+
{[]byte{1, 0, 0, 0, 0, 0, 0, 1}, GtidFormatTagged, 0},
153+
{[]byte{1, 0, 0, 0, 0, 0, 0, 0}, GtidFormatClassic, 1},
154+
}
155+
156+
for _, tc := range testcases {
157+
format, uuidCount := decodeSid(tc.input)
158+
assert.Equal(t, tc.gtidFormat, format)
159+
assert.Equal(t, tc.uuidCount, uuidCount)
160+
}
161+
}
162+
163+
func TestPreviousGTIDEvent(t *testing.T) {
164+
testcases := []struct {
165+
input []byte
166+
GTIDSets string
167+
}{
168+
{
169+
[]byte{0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
170+
"",
171+
},
172+
{
173+
[]byte{0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x89, 0x6e, 0x78, 0x82, 0x18, 0xfe, 0x11, 0xef, 0xab, 0x88, 0x22, 0x22, 0x2d, 0x34, 0xd4, 0x11, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
174+
"896e7882-18fe-11ef-ab88-22222d34d411:1-3",
175+
},
176+
{
177+
[]byte{0x1, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x89, 0x6e, 0x78, 0x82, 0x18, 0xfe, 0x11, 0xef, 0xab, 0x88, 0x22, 0x22, 0x2d, 0x34, 0xd4, 0x11, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x89, 0x6e, 0x78, 0x82, 0x18, 0xfe, 0x11, 0xef, 0xab, 0x88, 0x22, 0x22, 0x2d, 0x34, 0xd4, 0x11, 0x8, 0x61, 0x61, 0x61, 0x61, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
178+
"896e7882-18fe-11ef-ab88-22222d34d411:1-4:aaaa:1",
179+
},
180+
{
181+
[]byte{0x1, 0x7, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x89, 0x6e, 0x78, 0x82, 0x18, 0xfe, 0x11, 0xef, 0xab, 0x88, 0x22, 0x22, 0x2d, 0x34, 0xd4, 0x11, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x89, 0x6e, 0x78, 0x82, 0x18, 0xfe, 0x11, 0xef, 0xab, 0x88, 0x22, 0x22, 0x2d, 0x34, 0xd4, 0x11, 0x8, 0x61, 0x61, 0x61, 0x61, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x89, 0x6e, 0x78, 0x82, 0x18, 0xfe, 0x11, 0xef, 0xab, 0x88, 0x22, 0x22, 0x2d, 0x34, 0xd4, 0x11, 0x6, 0x61, 0x62, 0x63, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x89, 0x6e, 0x78, 0x82, 0x18, 0xfe, 0x11, 0xef, 0xab, 0x88, 0x22, 0x22, 0x2d, 0x34, 0xd4, 0x11, 0xa, 0x62, 0x62, 0x62, 0x62, 0x62, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x89, 0x6e, 0x78, 0x82, 0x18, 0xfe, 0x11, 0xef, 0xab, 0x88, 0x22, 0x22, 0x2d, 0x34, 0xd4, 0x11, 0xc, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x89, 0x6e, 0x78, 0x82, 0x18, 0xfe, 0x11, 0xef, 0xab, 0x88, 0x22, 0x22, 0x2d, 0x34, 0xd4, 0x11, 0x2, 0x78, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x89, 0x6e, 0x78, 0x82, 0x18, 0xfe, 0x11, 0xef, 0xab, 0x88, 0x22, 0x22, 0x2d, 0x34, 0xd4, 0x12, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
182+
"896e7882-18fe-11ef-ab88-22222d34d411:1-4:aaaa:1:abc:1-3:bbbbb:1:bbbbbb:1:x:1,896e7882-18fe-11ef-ab88-22222d34d412:1-2",
183+
},
184+
{
185+
[]byte{0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x2f, 0x20, 0xcc, 0xbc, 0x4c, 0x11, 0xef, 0xa1, 0xd0, 0x02, 0x42, 0xac, 0x11, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x2f, 0x20, 0xcc, 0xbc, 0x4c, 0x11, 0xef, 0xa1, 0xd0, 0x02, 0x42, 0xac, 0x11, 0x00, 0x02, 0x06, 0x61, 0x61, 0x61, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x2f, 0x20, 0xcc, 0xbc, 0x4c, 0x11, 0xef, 0xa1, 0xd0, 0x02, 0x42, 0xac, 0x11, 0x00, 0x02, 0x28, 0x74, 0x61, 0x67, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x2f, 0x20, 0xcc, 0xbc, 0x4c, 0x11, 0xef, 0xa1, 0xd0, 0x02, 0x42, 0xac, 0x11, 0x00, 0x02, 0x40, 0x74, 0x61, 0x67, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
186+
"042f20cc-bc4c-11ef-a1d0-0242ac110002:1-7:aaa:1:tag45678901234567890:1:tag45678901234567890123456789012:1",
187+
},
188+
}
189+
190+
for _, tc := range testcases {
191+
e := PreviousGTIDsEvent{}
192+
err := e.Decode(tc.input)
193+
require.NoError(t, err)
194+
require.Equal(t, tc.GTIDSets, e.GTIDSets)
195+
}
196+
}

0 commit comments

Comments
 (0)