Skip to content

Commit c47e953

Browse files
slugaliskjbpratt
slugalisk
authored and
jbpratt
committed
parse messages with entities
1 parent ea0197d commit c47e953

File tree

7 files changed

+126
-73
lines changed

7 files changed

+126
-73
lines changed

‎connection.go‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,11 @@ type EventDataIn struct {
5151

5252
type EventDataOut struct {
5353
*SimplifiedUser
54-
Targetuserid Userid `json:"-"`
55-
Timestamp int64 `json:"timestamp"`
56-
Data string `json:"data,omitempty"`
57-
Extradata string `json:"extradata,omitempty"`
54+
Targetuserid Userid `json:"-"`
55+
Timestamp int64 `json:"timestamp"`
56+
Data string `json:"data,omitempty"`
57+
Extradata string `json:"extradata,omitempty"`
58+
Entities *Entities `json:"entities,omitempty"`
5859
}
5960

6061
type BanIn struct {
@@ -83,10 +84,11 @@ type PrivmsgIn struct {
8384
type PrivmsgOut struct {
8485
message
8586
targetuid Userid
86-
Messageid int64 `json:"messageid"`
87-
Timestamp int64 `json:"timestamp"`
88-
Nick string `json:"nick,omitempty"`
89-
Data string `json:"data,omitempty"`
87+
Messageid int64 `json:"messageid"`
88+
Timestamp int64 `json:"timestamp"`
89+
Nick string `json:"nick,omitempty"`
90+
Data string `json:"data,omitempty"`
91+
Entities *Entities `json:"entities,omitempty"`
9092
}
9193

9294
// Create a new connection using the specified socket and router.
@@ -370,6 +372,7 @@ func (c *Connection) OnBroadcast(data []byte) {
370372

371373
out := c.getEventDataOut()
372374
out.Data = msg
375+
out.Entities = entities.Extract(msg)
373376
c.Broadcast("BROADCAST", out)
374377
}
375378

@@ -453,6 +456,7 @@ func (c *Connection) OnMsg(data []byte) {
453456

454457
out := c.getEventDataOut()
455458
out.Data = msg
459+
out.Entities = entities.Extract(msg)
456460
c.Broadcast("MSG", out)
457461
}
458462

@@ -495,6 +499,7 @@ func (c *Connection) OnPrivmsg(data []byte) {
495499
Data: msg,
496500
Messageid: 1337, // no saving in db means ids do not matter
497501
Timestamp: unixMilliTime(),
502+
Entities: entities.Extract(msg),
498503
}
499504

500505
pout.message.data, _ = Marshal(pout)

‎entities.go‎

Lines changed: 102 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,95 @@ import (
44
"encoding/json"
55
"fmt"
66
"io/ioutil"
7+
"log"
78
"net/http"
89
"regexp"
10+
"time"
911

1012
parser "github.com/MemeLabs/chat-parser"
13+
"mvdan.cc/xurls/v2"
1114
)
1215

13-
func extractEntities(parserCtx *parser.ParserContext, urls *regexp.Regexp, msg string) *Entities {
16+
var entities *EntityExtractor
17+
18+
func init() {
19+
var err error
20+
entities, err = NewEntityExtractor()
21+
if err != nil {
22+
log.Fatal(err)
23+
}
24+
25+
go entities.scheduleEmoteSync()
26+
}
27+
28+
func loadEmoteManifest() ([]string, error) {
29+
resp, err := http.Get("https://chat.strims.gg/emote-manifest.json")
30+
if err != nil {
31+
return nil, fmt.Errorf("failed to get emotes: %w", err)
32+
}
33+
defer resp.Body.Close()
34+
manifest := struct {
35+
Emotes []struct {
36+
Name string `json:"name"`
37+
} `json:"emotes"`
38+
}{}
39+
if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil {
40+
return nil, fmt.Errorf("failed to parse emotes manifest: %w", err)
41+
}
42+
43+
emotes := make([]string, len(manifest.Emotes))
44+
for i, e := range manifest.Emotes {
45+
emotes[i] = e.Name
46+
}
47+
return emotes, nil
48+
}
49+
50+
func NewEntityExtractor() (*EntityExtractor, error) {
51+
emotes, err := loadEmoteManifest()
52+
if err != nil {
53+
return nil, err
54+
}
55+
56+
return &EntityExtractor{
57+
parserCtx: parser.NewParserContext(parser.ParserContextValues{
58+
Emotes: emotes,
59+
Nicks: []string{},
60+
Tags: []string{"nsfw", "weeb", "nsfl", "loud"},
61+
EmoteModifiers: []string{"mirror", "flip", "rain", "snow", "rustle", "worth", "love", "spin", "wide", "lag", "hyper"},
62+
}),
63+
urls: xurls.Relaxed(),
64+
}, nil
65+
}
66+
67+
type EntityExtractor struct {
68+
parserCtx *parser.ParserContext
69+
urls *regexp.Regexp
70+
}
71+
72+
func (x *EntityExtractor) scheduleEmoteSync() {
73+
for range time.NewTicker(time.Minute).C {
74+
emotes, err := loadEmoteManifest()
75+
if err != nil {
76+
log.Println("failed to update emotes: %v", err)
77+
continue
78+
}
79+
x.parserCtx.Emotes.Replace(parser.RunesFromStrings(emotes))
80+
}
81+
}
82+
83+
func (x *EntityExtractor) AddNick(emote string) {
84+
x.parserCtx.Nicks.Insert([]rune(emote))
85+
}
86+
87+
func (x *EntityExtractor) RemoveNick(emote string) {
88+
x.parserCtx.Nicks.Remove([]rune(emote))
89+
}
90+
91+
func (x *EntityExtractor) Extract(msg string) *Entities {
1492
e := &Entities{}
15-
addEntitiesFromSpan(e, parser.NewParser(parserCtx, parser.NewLexer(msg)).ParseMessage())
93+
addEntitiesFromSpan(e, parser.NewParser(x.parserCtx, parser.NewLexer(msg)).ParseMessage())
1694

17-
for _, b := range urls.FindAllStringIndex(msg, -1) {
95+
for _, b := range x.urls.FindAllStringIndex(msg, -1) {
1896
e.Links = append(e.Links, &Link{
1997
URL: msg[b[0]:b[1]],
2098
Bounds: [2]int{b[0], b[1]},
@@ -35,17 +113,22 @@ func addEntitiesFromSpan(e *Entities, span *parser.Span) {
35113
Bounds: [2]int{span.Pos(), span.End()},
36114
})
37115
case parser.SpanGreentext:
38-
e.Greentexts = append(e.Greentexts, &Greentext{
116+
e.Greentext = &Generic{
39117
Bounds: [2]int{span.Pos(), span.End()},
40-
})
118+
}
119+
case parser.SpanMe:
120+
e.Me = &Generic{
121+
Bounds: [2]int{span.Pos(), span.End()},
122+
}
41123
}
42124

43125
for _, ni := range span.Nodes {
44126
switch n := ni.(type) {
45127
case *parser.Emote:
46128
e.Emotes = append(e.Emotes, &Emote{
47-
Name: n.Name,
48-
Bounds: [2]int{n.Pos(), n.End()},
129+
Name: n.Name,
130+
Modifiers: n.Modifiers,
131+
Bounds: [2]int{n.Pos(), n.End()},
49132
})
50133
case *parser.Nick:
51134
e.Nicks = append(e.Nicks, &Nick{
@@ -69,8 +152,9 @@ type Link struct {
69152
}
70153

71154
type Emote struct {
72-
Name string `json:"name,omitempty"`
73-
Bounds [2]int `json:"bounds,omitempty"`
155+
Name string `json:"name,omitempty"`
156+
Modifiers []string `json:"modifiers,omitempty"`
157+
Bounds [2]int `json:"bounds,omitempty"`
74158
}
75159

76160
type Nick struct {
@@ -91,18 +175,19 @@ type Spoiler struct {
91175
Bounds [2]int `json:"bounds,omitempty"`
92176
}
93177

94-
type Greentext struct {
178+
type Generic struct {
95179
Bounds [2]int `json:"bounds,omitempty"`
96180
}
97181

98182
type Entities struct {
99-
Links []*Link `json:"links,omitempty"`
100-
Emotes []*Emote `json:"emotes,omitempty"`
101-
Nicks []*Nick `json:"nicks,omitempty"`
102-
Tags []*Tag `json:"tags,omitempty"`
103-
Codes []*Code `json:"codes,omitempty"`
104-
Spoilers []*Spoiler `json:"spoilers,omitempty"`
105-
Greentexts []*Greentext `json:"greentexts,omitempty"`
183+
Links []*Link `json:"links,omitempty"`
184+
Emotes []*Emote `json:"emotes,omitempty"`
185+
Nicks []*Nick `json:"nicks,omitempty"`
186+
Tags []*Tag `json:"tags,omitempty"`
187+
Codes []*Code `json:"codes,omitempty"`
188+
Spoilers []*Spoiler `json:"spoilers,omitempty"`
189+
Greentext *Generic `json:"greentext,omitempty"`
190+
Me *Generic `json:"me,omitempty"`
106191
}
107192

108193
func getEmotes() ([]string, error) {

‎entities_test.go‎

Lines changed: 0 additions & 43 deletions
This file was deleted.

‎go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@ module github.com/MemeLabs/chat-backend
33
go 1.14
44

55
require (
6-
github.com/MemeLabs/chat-parser v0.0.0-20200409233434-3633664f904e
6+
github.com/MemeLabs/chat-parser v1.0.0
77
github.com/davecgh/go-spew v1.1.1
88
github.com/dgrijalva/jwt-go v3.2.0+incompatible
9-
github.com/gorilla/websocket v1.4.1
9+
github.com/gorilla/websocket v1.4.2
1010
github.com/jmoiron/sqlx v1.2.0
1111
github.com/mattn/go-sqlite3 v2.0.3+incompatible
1212
github.com/msbranco/goconfig v0.0.0-20160629072055-3189001257ce

‎go.sum‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
github.com/MemeLabs/chat-parser v0.0.0-20200409233434-3633664f904e h1:xIJQG9gd7QQav8ZYhnxudWPHnH3NOafIqmJzO8nwTKc=
2-
github.com/MemeLabs/chat-parser v0.0.0-20200409233434-3633664f904e/go.mod h1:sJ6/lyctAYBGJrxNS649FT/ujrUTTvgznHnfAZOqDvA=
1+
github.com/MemeLabs/chat-parser v0.0.0-20200413045719-58ef6695ba1d h1:3x55bZXuXxxCytsbG83YAixdAU7krEHImc4iFP0C4kM=
2+
github.com/MemeLabs/chat-parser v0.0.0-20200413045719-58ef6695ba1d/go.mod h1:sJ6/lyctAYBGJrxNS649FT/ujrUTTvgznHnfAZOqDvA=
3+
github.com/MemeLabs/chat-parser v1.0.0 h1:MmvRlmKpg5HwnNQs+oZu5u4O2dDnMj+xJnm5+h4UP1U=
4+
github.com/MemeLabs/chat-parser v1.0.0/go.mod h1:sJ6/lyctAYBGJrxNS649FT/ujrUTTvgznHnfAZOqDvA=
35
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
46
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
57
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
@@ -10,6 +12,8 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
1012
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
1113
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
1214
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
15+
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
16+
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
1317
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
1418
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
1519
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=

‎namescache.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ func (nc *namesCache) add(user *User) *User {
7878
}
7979
user.simplified = su
8080
nc.users[user.id] = user
81+
entities.AddNick(user.nick)
8182
}
8283

8384
nc.updateNames()
@@ -94,6 +95,7 @@ func (nc *namesCache) disconnect(user *User) {
9495
conncount := atomic.AddInt32(&u.connections, -1)
9596
if conncount <= 0 {
9697
delete(nc.users, user.id)
98+
entities.RemoveNick(u.nick)
9799
}
98100
}
99101

‎viewerstate.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ func (v *ViewerStateStore) updatePublicState(state *ViewerState) {
155155
return
156156
}
157157

158-
if ok && prev.Equals(state) {
158+
if prev.Equals(state) {
159159
return
160160
}
161161

0 commit comments

Comments
 (0)