add transfer checking

.github/workflows/check-failed-biz.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: ^1.15
+          go-version: ^1.16
 
       - name: Get dependencies
         run: |

.github/workflows/check-transfer.yml

@@ -0,0 +1,29 @@
+name: check transfer
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *" # Runs at 00:00 UTC every day.
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+
+      - name: Run
+        id: check
+        run: |
+          go run ./check-transfer
+        env:
+          GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/deal-with-issues.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: ^1.15
+          go-version: ^1.16
 
       - name: Get dependencies
         run: |

.github/workflows/merge-bot.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: ^1.15
+          go-version: ^1.16
 
       - name: Get dependencies
         run: |

check-failed-biz/main.go

@@ -75,20 +75,14 @@ LABEL:
 	os.Exit(0)
 }
 
-type bizInfo struct {
-	Name        string `csv:"name"`
-	BizID       string `csv:"bizid"`
-	Description string `csv:"description"`
-}
-
-func getList() []*bizInfo {
+func getList() []*common.BizInfo {
 
 	body := common.Fetch("https://github.com/hellodword/wechat-feeds/raw/main/list.csv")
 	if !common.WithUTF8Bom(body) {
 		panic("list.csv not utf8 bom")
 	}
 
-	var bis []*bizInfo
+	var bis []*common.BizInfo
 	err := gocsv.Unmarshal(bytes.NewReader(common.TrimUTF8Bom(body)), &bis)
 	if err != nil {
 		panic(err)
@@ -97,16 +91,11 @@ func getList() []*bizInfo {
 	return bis
 }
 
-type bizDetail struct {
-	Name  string `csv:"name" json:"name"`
-	BizID string `csv:"bizid" json:"bizid"`
-}
-
-func getDetails() []*bizDetail {
+func getDetails() []*common.BizDetail {
 
 	body := common.Fetch("https://github.com/hellodword/wechat-feeds/raw/feeds/details.json")
 
-	var bds []*bizDetail
+	var bds []*common.BizDetail
 	err := json.Unmarshal(body, &bds)
 	if err != nil {
 		panic(err)

check-transfer/main.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/google/go-github/v33/github"
+	"github.com/hellodword/wechat-feeds/common"
+	"github.com/mmcdole/gofeed"
+	"math/rand"
+	"net/url"
+	"os"
+	"time"
+)
+
+const (
+	Owner = "hellodword"
+	Repo  = "wechat-feeds"
+
+	IssueID = 2387
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+func main() {
+
+	details := getDetails()
+	if len(details) == 0 {
+		os.Exit(0)
+	}
+
+	ctx := context.Background()
+	clientWithToken, client := common.MakeClients(ctx, os.Getenv("GITHUB_ACCESS_TOKEN"))
+	_, _ = clientWithToken, client
+	client = clientWithToken // for private test
+
+	_, _, _ = clientWithToken.Issues.AddLabelsToIssue(ctx, Owner, Repo,
+		IssueID,
+		[]string{string(common.LabelCheck)})
+
+	buf := bytes.NewBuffer(nil)
+	buf.WriteString(`| name | bizid | new | reason |
+| ---  | ----- | --- | ------ |`)
+	for i := range details {
+		buf.WriteString("\n")
+		buf.WriteString("| ")
+		buf.WriteString(details[i].Name)
+		buf.WriteString(" | ")
+		buf.WriteString(fmt.Sprintf(`[%s](https://github.com/hellodword/wechat-feeds/raw/feeds/%s.xml)`,
+			details[i].BizID, details[i].BizID))
+
+		newBiz, reason := checkTransfer(details[i].BizID)
+		buf.WriteString(" | ")
+		buf.WriteString(newBiz)
+		buf.WriteString(" | ")
+		buf.WriteString(reason)
+		buf.WriteString(" | ")
+
+	}
+
+	fmt.Println(buf.String())
+
+	_, _, err := clientWithToken.Issues.CreateComment(ctx, Owner, Repo,
+		IssueID,
+		&github.IssueComment{
+			Body: github.String(buf.String()),
+		})
+	if err != nil {
+		panic("Issues.CreateComment") // token privacy
+	}
+
+	os.Exit(0)
+}
+
+func checkTransfer(bizid string) (newBiz, reason string) {
+	fp := gofeed.NewParser()
+	feed, err := fp.ParseString(string(common.Fetch(fmt.Sprintf("https://github.com/hellodword/wechat-feeds/raw/feeds/%s.xml", url.QueryEscape(bizid)))))
+	if err != nil || feed == nil || len(feed.Items) == 0 {
+		return
+	}
+
+	i := rand.Intn(len(feed.Items)) // 原文也失效 https://mp.weixin.qq.com/s/6kgFxZ6nm9dLRZV66-9RXA
+
+	articleInfo, _ := common.FetchWX(feed.Items[i].Link)
+
+	newBiz = articleInfo.BizID
+	reason = articleInfo.FailReason
+
+	if newBiz == "" && articleInfo.TransferLink != "" { // 原文有问题，直接从链接取 bizid
+		// http://mp.weixin.qq.com/s?__biz=Mzk0MDIwNTQxNw==&mid=2247505407&idx=1&sn=62616bd14bfe1eb8f570442ffddefcd0#rd
+		newBiz = common.MatchBizID(articleInfo.TransferLink)
+	}
+
+	return
+}
+
+func getDetails() []*common.BizDetail {
+
+	body := common.Fetch("https://github.com/hellodword/wechat-feeds/raw/feeds/details.json")
+
+	var bds []*common.BizDetail
+	err := json.Unmarshal(body, &bds)
+	if err != nil {
+		panic(err)
+	}
+
+	// pick out details without head_img
+
+	var ret []*common.BizDetail
+	for i := range bds {
+		if bds[i].HeadIMG == "" {
+			ret = append(ret, bds[i])
+		}
+	}
+
+	return ret
+}

common/match.go

@@ -0,0 +1,104 @@
+package common
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+func MatchTransferTargetLink(s string) string {
+	r := regexp.MustCompile(`transferTargetLink = '(https?://mp\.weixin\.qq\.com/s[^\s\r\n]+)'`).FindStringSubmatch(s)
+	if len(r) < 2 {
+		return ""
+	} else {
+		return r[1]
+	}
+}
+
+func MatchBizID(s string) string {
+
+	// https://mp.weixin.qq.com/s/1I33XLA5uK1Iljvn3-XVDg
+	// https://mp.weixin.qq.com/s/etTO4fTRwyvSUuh2qJlIaw
+
+	r := regexp.MustCompile(`((var biz = [" =|]*")|(var appuin = [" =|]*")|(__biz=))([a-zA-Z\d/+=]+)`).FindStringSubmatch(s)
+	if len(r) == 6 {
+		return r[5]
+	} else {
+		return ""
+	}
+}
+
+func MatchName(s string) string {
+
+	// 图文 https://mp.weixin.qq.com/s/g0H8YxjN5kUR3Kx9cPgNlA
+
+	r := regexp.MustCompile(`(var nickname = "([^\n]+)";)|(d\.nick_name = getXmlValue\('nick_name.DATA'\) \|\| '([^\n]+)';)|(<strong class="account_nickname_inner js_go_profile">([^\n]+)</strong>)`).FindStringSubmatch(s)
+	if len(r) != 7 {
+		return ""
+	}
+
+	if r[2] != "" {
+		return r[2]
+	}
+	if r[4] != "" {
+		return r[4]
+	}
+	if r[6] != "" {
+		return r[6]
+	}
+
+	return ""
+
+}
+
+type WXArticle struct {
+	Name        string
+	BizID       string
+	Description string
+
+	FailReason   string
+	TransferLink string
+}
+
+func FetchWX(u string) (article WXArticle, err error) {
+	fmt.Println("link", u)
+
+	body := Fetch(u)
+
+	s := string(body)
+
+	if article.FailReason == "" {
+		if strings.Index(s, `此帐号已自主注销，内容无法查看`) != -1 {
+			article.FailReason = `此帐号已自主注销，内容无法查看`
+			err = errors.New(article.FailReason)
+			return
+		} else if strings.Index(s, `该公众号已迁移`) != -1 {
+			article.FailReason = `该公众号已迁移`
+		}
+	}
+
+	link := MatchTransferTargetLink(s)
+	if link != "" {
+		fmt.Println("transfer link", link)
+		newArticle, err := FetchWX(link)
+		newArticle.TransferLink = link
+		if article.FailReason != "" && newArticle.FailReason == "" {
+			newArticle.FailReason = article.FailReason
+		}
+		return newArticle, err
+	}
+
+	article.BizID = MatchBizID(s)
+	if article.BizID == "" {
+		err = errors.New("no biz id")
+		return
+	}
+	article.Name = MatchName(s)
+	if article.Name == "" {
+		err = errors.New("no name")
+		return
+	}
+
+	return
+}

common/structs.go

@@ -0,0 +1,13 @@
+package common
+
+type BizDetail struct {
+	Name    string `csv:"name" json:"name"`
+	BizID   string `csv:"bizid" json:"bizid"`
+	HeadIMG string `json:"head_img"`
+}
+
+type BizInfo struct {
+	Name        string `csv:"name"`
+	BizID       string `csv:"bizid"`
+	Description string `csv:"description"`
+}