diff --git a/main.go b/main.go index b740a0d..7854f81 100644 --- a/main.go +++ b/main.go @@ -6,8 +6,6 @@ import ( "errors" "flag" "fmt" - md "github.com/JohannesKaufmann/html-to-markdown" - "github.com/JohannesKaufmann/html-to-markdown/plugin" "io/ioutil" "net/http" "net/url" @@ -16,6 +14,9 @@ import ( "regexp" "strings" "time" + + md "github.com/JohannesKaufmann/html-to-markdown" + "github.com/JohannesKaufmann/html-to-markdown/plugin" ) type ResultCode struct { @@ -95,35 +96,48 @@ func main() { func fetchFolder(root string, wizUser *WizUser, folder string) error { token := wizUser.Token - cbs, err := Fetch(fmt.Sprintf("%s/ks/note/list/category/%s?start=0&count=200&category=%s&orderBy=created", - wizUser.KbServer, wizUser.KbGuid, url.PathEscape(folder)), token) - if err != nil { - return WrapErr("fetch folder", err) - } - cateResult := new(DocListResult) - if err = json.Unmarshal(cbs, cateResult); err != nil { - return WrapErr("Unmarshal folder result", err) - } - if cateResult.ReturnCode != 200 { - return WrapErr("fetch folder", err) - } + start := 0 + count := 200 + // make root and resource folder parentPath := path.Join(root, folder[1:]) - if err = os.MkdirAll(parentPath, 0755); err != nil { + if err := os.MkdirAll(parentPath, 0755); err != nil { return WrapErr("MkdirAll folder", err) } if err := os.MkdirAll(path.Join(parentPath, "index_files"), 0755); err != nil { return WrapErr("MkdirAll index_files", err) } - // read docs - for _, doc := range cateResult.Result { - fmt.Printf("Doc info:\n\tdocGuid: %s\n\ttitle: %s\n\tattachmentCount:%v\n", - doc.DocGuid, doc.Title, doc.AttachmentCount) - if err := fetchDoc(parentPath, wizUser, doc); err != nil { - fmt.Println("fetchDoc err:", err) + + for { + cbs, err := Fetch(fmt.Sprintf("%s/ks/note/list/category/%s?start=%d&count=%d&category=%s&orderBy=created", + wizUser.KbServer, wizUser.KbGuid, start, count, url.PathEscape(folder)), token) + if err != nil { + return WrapErr("fetch folder", err) } - time.Sleep(100 * time.Millisecond) + cateResult := new(DocListResult) + if err = json.Unmarshal(cbs, cateResult); err != nil { + return WrapErr("Unmarshal folder result", err) + } + if cateResult.ReturnCode != 200 { + return WrapErr("fetch folder", err) + } + + if len(cateResult.Result) == 0 { + break + } + + // read docs + for _, doc := range cateResult.Result { + fmt.Printf("Doc info:\n\tdocGuid: %s\n\ttitle: %s\n\tattachmentCount:%v\n", + doc.DocGuid, doc.Title, doc.AttachmentCount) + if err := fetchDoc(parentPath, wizUser, doc); err != nil { + fmt.Println("fetchDoc err:", err) + } + time.Sleep(100 * time.Millisecond) + } + + start += len(cateResult.Result) } return nil