Skip to content

Commit

Permalink
Merge pull request #112 from PIGfaces/master
Browse files Browse the repository at this point in the history
perf(static-cmp): Static resource matching optimization
  • Loading branch information
Qianlitp authored Jul 7, 2022
2 parents dbf7064 + 3fa47a8 commit 5bd29ce
Show file tree
Hide file tree
Showing 15 changed files with 170 additions and 110 deletions.
12 changes: 1 addition & 11 deletions cmd/crawlergo/flag.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,22 +98,12 @@ func SetOutputMode() *cli.StringFlag {

func SetOutputJSON() *cli.StringFlag {
return &cli.StringFlag{
Name: "output-dir",
Name: "output-json",
Usage: "write output to a json file.Such as result_www_test_com.json",
Destination: &outputJsonPath,
}
}

func SetIgcognitoContext() *cli.BoolFlag {
return &cli.BoolFlag{
Name: "incognito-context",
Aliases: []string{"i"},
Value: true,
Usage: "whether the browser is launched in incognito mode.",
Destination: &taskConfig.IncognitoContext,
}
}

func SetMaxTabCount() *cli.IntFlag {
return &cli.IntFlag{
Name: "max-tab-count",
Expand Down
6 changes: 4 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,16 @@ go 1.16
replace git.apache.org/thrift.git => github.com/apache/thrift v0.13.0

require (
github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4
github.com/chromedp/chromedp v0.5.2
github.com/chromedp/cdproto v0.0.0-20220629234738-4cfc9cdeeb92
github.com/chromedp/chromedp v0.8.2
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/deckarep/golang-set v1.7.1
github.com/gogf/gf v1.16.6
github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 // indirect
github.com/panjf2000/ants/v2 v2.2.2
github.com/pkg/errors v0.8.1
github.com/sirupsen/logrus v1.4.2
github.com/urfave/cli/v2 v2.0.0
golang.org/x/net v0.0.0-20210520170846-37e1c6afe023
golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b // indirect
)
22 changes: 22 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,15 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4 h1:QD3KxSJ59L2lxG6MXBjNHxiQO2RmxTQ3XcK+wO44WOg=
github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g=
github.com/chromedp/cdproto v0.0.0-20220515234810-83d799542a04/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0=
github.com/chromedp/cdproto v0.0.0-20220629234738-4cfc9cdeeb92 h1:0kiAQSLWZDt4wsmcICou3C6in/OJ58FCqvXcB8Ax1Dk=
github.com/chromedp/cdproto v0.0.0-20220629234738-4cfc9cdeeb92/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0=
github.com/chromedp/chromedp v0.5.2 h1:W8xBXQuUnd2dZK0SN/lyVwsQM7KgW+kY5HGnntms194=
github.com/chromedp/chromedp v0.5.2/go.mod h1:rsTo/xRo23KZZwFmWk2Ui79rBaVRRATCjLzNQlOFSiA=
github.com/chromedp/chromedp v0.8.2 h1:EYSsSqWuKYwyHZEJpU00kOGOMz5DE0qDVckelzauMFA=
github.com/chromedp/chromedp v0.8.2/go.mod h1:vpbCNtfYeOUo2q5reuwX6ZmPpbHRf5PZfAqNR2ObB+g=
github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
github.com/clbanning/mxj v1.8.5-0.20200714211355-ff02cfb8ea28 h1:LdXxtjzvZYhhUaonAaAKArG3pyC67kGL3YY+6hGG8G4=
github.com/clbanning/mxj v1.8.5-0.20200714211355-ff02cfb8ea28/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
Expand All @@ -22,10 +29,16 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfC
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA=
github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
github.com/gogf/gf v1.16.6 h1:Yp5YfwnGz41d1tiVqxcWXiPXyuzjTb7ax4SnPSXxDE8=
github.com/gogf/gf v1.16.6/go.mod h1:4LoHfEBl2jbVmZpVx+qk2La3zWr1V315FtF2PVZuyQ8=
github.com/gomodule/redigo v1.8.5 h1:nRAxCa+SVsyjSBrtZmG/cqb6VbTmuRzpg/PoTFlpumc=
Expand All @@ -36,12 +49,16 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grokify/html-strip-tags-go v0.0.0-20190921062105-daaa06bf1aaf h1:wIOAyJMMen0ELGiFzlmqxdcV1yGbkyHBAB6PolcNbLA=
github.com/grokify/html-strip-tags-go v0.0.0-20190921062105-daaa06bf1aaf/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 h1:V0an7KRw92wmJysvFvtqtKMAPmvS5O0jtB0nYo6t+gs=
github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
Expand All @@ -50,6 +67,7 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/panjf2000/ants/v2 v2.2.2 h1:TWzusBjq/IflXhy+/S6u5wmMLCBdJnB9tPIx9Zmhvok=
github.com/panjf2000/ants/v2 v2.2.2/go.mod h1:1GFm8bV8nyCQvU5K4WvBCTG1/YBFOD2VzjffD8fV55A=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
Expand Down Expand Up @@ -84,8 +102,12 @@ golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b h1:2n253B2r0pYSmEV+UNCQoPfU/FiaizQEK5Gu4Bq4JE8=
golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
Expand Down
49 changes: 36 additions & 13 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
package config

import "time"
import (
"time"

mapset "github.com/deckarep/golang-set"
)

const (
DefaultUA = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.0 Safari/537.36"
Expand Down Expand Up @@ -68,19 +72,25 @@ const (
MULTIPART = "multipart/form-data"
)

var StaticSuffix = []string{
"png", "gif", "jpg", "mp4", "mp3", "mng", "pct", "bmp", "jpeg", "pst", "psp", "ttf",
"tif", "tiff", "ai", "drw", "wma", "ogg", "wav", "ra", "aac", "mid", "au", "aiff",
"dxf", "eps", "ps", "svg", "3gp", "asf", "asx", "avi", "mov", "mpg", "qt", "rm",
"wmv", "m4a", "bin", "xls", "xlsx", "ppt", "pptx", "doc", "docx", "odt", "ods", "odg",
"odp", "exe", "zip", "rar", "tar", "gz", "iso", "rss", "pdf", "txt", "dll", "ico",
"gz2", "apk", "crt", "woff", "map", "woff2", "webp", "less", "dmg", "bz2", "otf", "swf",
"flv", "mpeg", "dat", "xsl", "csv", "cab", "exif", "wps", "m4v", "rmvb",
}
var (
StaticSuffix = []string{
"png", "gif", "jpg", "mp4", "mp3", "mng", "pct", "bmp", "jpeg", "pst", "psp", "ttf",
"tif", "tiff", "ai", "drw", "wma", "ogg", "wav", "ra", "aac", "mid", "au", "aiff",
"dxf", "eps", "ps", "svg", "3gp", "asf", "asx", "avi", "mov", "mpg", "qt", "rm",
"wmv", "m4a", "bin", "xls", "xlsx", "ppt", "pptx", "doc", "docx", "odt", "ods", "odg",
"odp", "exe", "zip", "rar", "tar", "gz", "iso", "rss", "pdf", "txt", "dll", "ico",
"gz2", "apk", "crt", "woff", "map", "woff2", "webp", "less", "dmg", "bz2", "otf", "swf",
"flv", "mpeg", "dat", "xsl", "csv", "cab", "exif", "wps", "m4v", "rmvb",
}
StaticSuffixSet mapset.Set
)

var ScriptSuffix = []string{
"php", "asp", "jsp", "asa",
}
var (
ScriptSuffix = []string{
"php", "asp", "jsp", "asa",
}
ScriptSuffixSet mapset.Set
)

var DefaultIgnoreKeywords = []string{"logout", "quit", "exit"}
var AllowedFormName = []string{"default", "mail", "code", "phone", "username", "password", "qq", "id_card", "url", "date", "number"}
Expand Down Expand Up @@ -129,3 +139,16 @@ var InputTextMap = map[string]map[string]interface{}{
"value": "10",
},
}

func init() {
StaticSuffixSet = initSet(StaticSuffix)
ScriptSuffixSet = initSet(ScriptSuffix)
}

func initSet(suffixs []string) mapset.Set {
set := mapset.NewSet()
for _, s := range suffixs {
set.Add(s)
}
return set
}
16 changes: 16 additions & 0 deletions pkg/config/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package config_test

import (
"testing"

"github.com/Qianlitp/crawlergo/pkg/config"
"github.com/stretchr/testify/assert"
)

func TestStaticSuffix(t *testing.T) {
assert.Equal(t, true, config.StaticSuffixSet.Contains("png"))
assert.Equal(t, false, config.StaticSuffixSet.Contains("demo"))

assert.Equal(t, true, config.ScriptSuffixSet.Contains("asp"))
assert.Equal(t, false, config.ScriptSuffixSet.Contains("demo"))
}
10 changes: 3 additions & 7 deletions pkg/engine/browser.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,22 +21,18 @@ type Browser struct {
lock sync.Mutex
}

func init() {

}

func InitBrowser(chromiumPath string, incognito bool, extraHeaders map[string]interface{}, proxy string, noHeadless bool) *Browser {
func InitBrowser(chromiumPath string, extraHeaders map[string]interface{}, proxy string, noHeadless bool) *Browser {
var bro Browser
opts := append(chromedp.DefaultExecAllocatorOptions[:],

// 执行路径
chromedp.ExecPath(chromiumPath),
// 无头模式
chromedp.Flag("headless", !noHeadless),
// https://github.com/chromedp/chromedp/issues/997#issuecomment-1030596050
// incognito mode not used
// 禁用GPU,不显示GUI
chromedp.Flag("disable-gpu", true),
// 隐身模式启动
chromedp.Flag("incognito", incognito),
// 取消沙盒模式
chromedp.Flag("no-sandbox", true),
// 忽略证书错误
Expand Down
13 changes: 6 additions & 7 deletions pkg/engine/intercept_request.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,12 @@ func (tab *Tab) InterceptRequest(v *fetch.EventRequestPaused) {
tab.HandleHostBinding(&req)

// 静态资源 全部阻断
for _, suffix := range config.StaticSuffix {
if strings.HasSuffix(strings.ToLower(url.Path), suffix) {
_ = fetch.FailRequest(v.RequestID, network.ErrorReasonBlockedByClient).Do(ctx)
req.Source = config.FromStaticRes
tab.AddResultRequest(req)
return
}
// https://github.com/Qianlitp/crawlergo/issues/106
if config.StaticSuffixSet.Contains(url.FileExt()) {
_ = fetch.FailRequest(v.RequestID, network.ErrorReasonBlockedByClient).Do(ctx)
req.Source = config.FromStaticRes
tab.AddResultRequest(req)
return
}

// 处理导航请求
Expand Down
55 changes: 11 additions & 44 deletions pkg/engine/tab.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package engine
import (
"context"
"encoding/json"
"errors"
"fmt"
"regexp"
"strings"
Expand Down Expand Up @@ -34,7 +35,6 @@ type Tab struct {
NavNetworkID string
PageCharset string
PageBindings map[string]interface{}
NavDone chan int
FoundRedirection bool
DocBodyNodeId cdp.NodeID
config TabConfig
Expand Down Expand Up @@ -82,7 +82,6 @@ func NewTab(browser *Browser, navigateReq model2.Request, config TabConfig) *Tab
}
tab.NavigateReq = navigateReq
tab.config = config
tab.NavDone = make(chan int)
tab.DocBodyNodeId = 0

// 设置请求拦截监听
Expand Down Expand Up @@ -159,33 +158,6 @@ func NewTab(browser *Browser, navigateReq model2.Request, config TabConfig) *Tab
return &tab
}

/**
*/
func waitNavigateDone(ctx context.Context) error {
ch := make(chan struct{})
lCtx, lCancel := context.WithCancel(ctx)
tCtx, cancel := context.WithTimeout(ctx, config.DomContentLoadedTimeout)
defer cancel()
chromedp.ListenTarget(lCtx, func(ev interface{}) {
if _, ok := ev.(*page.EventDomContentEventFired); ok {
lCancel()
close(ch)
} else if _, ok := ev.(*page.EventLoadEventFired); ok {
lCancel()
close(ch)
}
})
select {
case <-ch:
return nil
case <-ctx.Done():
return ctx.Err()
case <-tCtx.Done():
return tCtx.Err()
}
}

func (tab *Tab) Start() {
logger.Logger.Info("Crawling " + tab.NavigateReq.Method + " " + tab.NavigateReq.URL.String())
defer tab.Cancel()
Expand All @@ -212,30 +184,25 @@ func (tab *Tab) Start() {
}),
network.SetExtraHTTPHeaders(tab.ExtraHeaders),
// 执行导航
//chromedp.Navigate(tab.NavigateReq.URL.String()),
chromedp.ActionFunc(func(ctx context.Context) error {
_, _, _, err := page.Navigate(tab.NavigateReq.URL.String()).Do(ctx)
if err != nil {
return err
}
return waitNavigateDone(ctx)
}),
chromedp.Navigate(tab.NavigateReq.URL.String()),
}),
); err != nil {
if err.Error() == "context canceled" {
if errors.Is(err, context.Canceled) {
logger.Logger.Debug("Crawling Canceled")
return
}
logger.Logger.Warn("navigate timeout ", tab.NavigateReq.URL.String())
}

go func() {
// 等待所有协程任务结束
waitDone := func() <-chan struct{} {
tab.WG.Wait()
tab.NavDone <- 1
}()
ch := make(chan struct{})
defer close(ch)
return ch
}

select {
case <-tab.NavDone:
case <-waitDone():
logger.Logger.Debug("all navigation tasks done.")
case <-time.After(tab.config.DomContentLoadedTimeout + time.Second*10):
logger.Logger.Warn("navigation tasks TIMEOUT.")
Expand Down Expand Up @@ -405,7 +372,7 @@ func (tab *Tab) DetectCharset() {
var ok bool
var getCharsetRegex = regexp.MustCompile("charset=(.+)$")
err := chromedp.AttributeValue(`meta[http-equiv=Content-Type]`, "content", &content, &ok, chromedp.ByQuery).Do(tCtx)
if err != nil || ok != true {
if err != nil || !ok {
return
}
if strings.Contains(content, "charset=") {
Expand Down
35 changes: 35 additions & 0 deletions pkg/engine/tab_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package engine_test

import (
"sync"
"testing"
"time"
)

func TestChannel(t *testing.T) {
wg := sync.WaitGroup{}

for range "..." {
wg.Add(1)
go func() {
defer wg.Done()
t.Log("=====> go func")
time.Sleep(time.Second * 5)
t.Log("go func done! <<<<========")
}()
}

waitDone := func() <-chan struct{} {
wg.Wait()
ch := make(chan struct{})
defer close(ch)
return ch
}

select {
case <-waitDone():
t.Log("all goroutine done")
case <-time.After(time.Second * 10):
t.Error("timeout")
}
}
Loading

0 comments on commit 5bd29ce

Please sign in to comment.