From 3392c157902941114d0480aa4411e841a90f1fe7 Mon Sep 17 00:00:00 2001 From: jyh Date: Tue, 23 May 2023 14:57:08 +0800 Subject: [PATCH 01/29] =?UTF-8?q?=E5=BF=BD=E7=95=A5=20.idea=20=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=A4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a09c56d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/.idea From 2978db229f1b79a60f3e62d77d142758176b6cf3 Mon Sep 17 00:00:00 2001 From: jyh Date: Tue, 23 May 2023 17:46:14 +0800 Subject: [PATCH 02/29] =?UTF-8?q?=E5=BB=BA=E7=AB=8B=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=A1=86=E6=9E=B6=EF=BC=9B=E5=BC=80=E5=8F=91=20unpack=20?= =?UTF-8?q?=E9=83=A8=E5=88=86=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/phonedatatool/main.go | 99 ++++++++++++++++++++++++++++ phonedatatool/model.go | 41 ++++++++++++ phonedatatool/pack/pack.go | 13 ++++ phonedatatool/phonedatatool.go | 24 +++++++ phonedatatool/query/query.go | 13 ++++ phonedatatool/unpack/index.go | 59 +++++++++++++++++ phonedatatool/unpack/offset.go | 20 ++++++ phonedatatool/unpack/record.go | 70 ++++++++++++++++++++ phonedatatool/unpack/unpack.go | 111 ++++++++++++++++++++++++++++++++ phonedatatool/unpack/version.go | 32 +++++++++ 10 files changed, 482 insertions(+) create mode 100644 cmd/phonedatatool/main.go create mode 100644 phonedatatool/model.go create mode 100644 phonedatatool/pack/pack.go create mode 100644 phonedatatool/phonedatatool.go create mode 100644 phonedatatool/query/query.go create mode 100644 phonedatatool/unpack/index.go create mode 100644 phonedatatool/unpack/offset.go create mode 100644 phonedatatool/unpack/record.go create mode 100644 phonedatatool/unpack/unpack.go create mode 100644 phonedatatool/unpack/version.go diff --git a/cmd/phonedatatool/main.go b/cmd/phonedatatool/main.go new file mode 100644 index 0000000..ece119f --- /dev/null +++ b/cmd/phonedatatool/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "flag" + "fmt" + "github.com/xluohome/phonedata/phonedatatool/pack" + "github.com/xluohome/phonedata/phonedatatool/query" + "github.com/xluohome/phonedata/phonedatatool/unpack" +) + +// 这里编译出来的可执行程序具备打包、查询、解包三个功能。 +// ./phonedatatool -unpack -i phone.dat -o tmp +// ./phonedatatool -pack -i tmp -o phone.dat +// ./phonedatatool -query -i phone.dat -number 13000001234 + +const ( + Name = "phonedatatool" + Version = "0.0.1" + Author = "seedjyh@gmail.com" + FullName = Name + "_" + Version + "(" + Author + ")" + Logo = ` + ____ __ ______ _ ____________ ___ _________ __________ ____ __ + / __ \/ / / / __ \/ | / / ____/ __ \/ |/_ __/ |/_ __/ __ \/ __ \/ / + / /_/ / /_/ / / / / |/ / __/ / / / / /| | / / / /| | / / / / / / / / / / + / ____/ __ / /_/ / /| / /___/ /_/ / ___ |/ / / ___ |/ / / /_/ / /_/ / /___ +/_/ /_/ /_/\____/_/ |_/_____/_____/_/ |_/_/ /_/ |_/_/ \____/\____/_____/` + FullName +) + +func main() { + showVersionFlag := flag.Bool("v", false, "Just show version string") + showHelpFlag := flag.Bool("help", false, "Just print help.") + unpackFlag := flag.Bool("unpack", false, "Unpack phone data to plain text") + packFlag := flag.Bool("pack", false, "Pack plain text to phone data") + queryFlag := flag.Bool("query", false, "Query number from phone data") + source := flag.String("i", "", "Source of operation") + destination := flag.String("o", "", "Destination of operation") + number := flag.String("number", "", "Number to query") + flag.Parse() + if *showVersionFlag { + fmt.Println("Version:", FullName) + return + } + if *showHelpFlag { + fmt.Println("./phonedatatool -unpack -i phone.dat -o tmp") + fmt.Println("./phonedatatool -pack -i tmp -o phone.dat") + fmt.Println("./phonedatatool -query -i phone.dat -n 13000001234") + return + } + if *unpackFlag { + if source == nil { + fmt.Println("ERROR! No source") + return + } + if destination == nil { + fmt.Println("ERROR! No destination") + return + } + if err := unpack.NewUnpacker().Unpack(*source, *destination); err != nil { + fmt.Println("Unpack completed.") + return + } + } + if *packFlag { + if source == nil { + fmt.Println("ERROR! No source") + return + } + if destination == nil { + fmt.Println("ERROR! No destination") + return + } + if err := pack.NewPacker().Pack(*source, *destination); err != nil { + fmt.Println("Pack completed.") + return + } + } + if *queryFlag { + if source == nil { + fmt.Println("ERROR! No source") + return + } + if number == nil { + fmt.Println("ERROR! No number to query") + return + } + if info, err := query.NewQuerier(*source).QueryNumber(*number); err != nil { + fmt.Println("ERROR! ", err) + return + } else { + fmt.Println("PhoneNum: ", info.PhoneNumber) + fmt.Println("AreaZone: ", info.AreaCode) + fmt.Println("CardType: ", info.CardTypeID.ToName().String()) + fmt.Println("City: ", info.CityName) + fmt.Println("ZipCode: ", info.ZipCode) + fmt.Println("Province: ", info.ProvinceName) + } + } + return +} diff --git a/phonedatatool/model.go b/phonedatatool/model.go new file mode 100644 index 0000000..e17fc33 --- /dev/null +++ b/phonedatatool/model.go @@ -0,0 +1,41 @@ +package phonedatatool + +import "github.com/xluohome/phonedata" + +type PhoneNumber string // 手机号码 +type AreaCode string // 区号 +func (ac AreaCode) String() string { + return string(ac) +} + +type CardTypeID byte // 卡类型 ID，单字节 +func (ctid CardTypeID) String() string { + return string(ctid) +} +func (ctid CardTypeID) ToName() CardTypeName { + if v, ok := phonedata.CardTypemap[byte(ctid)]; ok { + return CardTypeName(v) + } else { + return "---" + } +} + +type CardTypeName string // 卡类型中文名 +func (n CardTypeName) String() string { + return string(n) +} + +type CityName string // 城市名 +func (cn CityName) String() string { + return string(cn) +} + +type ProvinceName string // 省名 +func (pn ProvinceName) String() string { + return string(pn) +} + +type ZipCode string // 邮政编码 +func (zc ZipCode) String() string { + return string(zc) +} diff --git a/phonedatatool/pack/pack.go b/phonedatatool/pack/pack.go new file mode 100644 index 0000000..cad3a84 --- /dev/null +++ b/phonedatatool/pack/pack.go @@ -0,0 +1,13 @@ +package pack + +import "github.com/xluohome/phonedata/phonedatatool" + +type Packer struct{} + +func NewPacker() phonedatatool.Packer { + return new(Packer) +} + +func (p *Packer) Pack(plainDirectoryPath string, phoneDataFilePath string) error { + return nil +} diff --git a/phonedatatool/phonedatatool.go b/phonedatatool/phonedatatool.go new file mode 100644 index 0000000..e6e8a08 --- /dev/null +++ b/phonedatatool/phonedatatool.go @@ -0,0 +1,24 @@ +package phonedatatool + +type Unpacker interface { + // Unpack 将压缩文件 phoneDataFilePath 解包后保存到明文目录 plainDirectoryPath 里。 + Unpack(phoneDataFilePath string, plainDirectoryPath string) error +} + +type Packer interface { + // Pack 将明文目录 plainDirectoryPath 的数据打包成压缩文件 phoneDataFilePath。 + Pack(plainDirectoryPath string, phoneDataFilePath string) error +} + +type QueryResult struct { + PhoneNumber PhoneNumber + AreaCode AreaCode + CardTypeID CardTypeID + CityName CityName + ZipCode ZipCode + ProvinceName ProvinceName +} + +type Querier interface { + QueryNumber(number string) (*QueryResult, error) +} diff --git a/phonedatatool/query/query.go b/phonedatatool/query/query.go new file mode 100644 index 0000000..69c4346 --- /dev/null +++ b/phonedatatool/query/query.go @@ -0,0 +1,13 @@ +package query + +import "github.com/xluohome/phonedata/phonedatatool" + +type Querier struct{} + +func NewQuerier(phoneDataFilePath string) phonedatatool.Querier { + return new(Querier) +} + +func (q *Querier) QueryNumber(number string) (*phonedatatool.QueryResult, error) { + return nil, nil +} diff --git a/phonedatatool/unpack/index.go b/phonedatatool/unpack/index.go new file mode 100644 index 0000000..e904b35 --- /dev/null +++ b/phonedatatool/unpack/index.go @@ -0,0 +1,59 @@ +package unpack + +import ( + "bytes" + "github.com/xluohome/phonedata/phonedatatool" + "sort" + "strings" +) + +type PhoneNumberPrefix string + +func (pnp PhoneNumberPrefix) String() string { + return string(pnp) +} + +type PhoneNumberPrefixList []PhoneNumberPrefix + +func (pl PhoneNumberPrefixList) Len() int { + return len(pl) +} +func (pl PhoneNumberPrefixList) Less(i, j int) bool { + return pl[i] < pl[j] +} +func (pl PhoneNumberPrefixList) Swap(i, j int) { + pl[i], pl[j] = pl[j], pl[i] +} + +type IndexItem struct { + CardTypeID phonedatatool.CardTypeID + RecordOffset RecordOffset + RecordID RecordID +} + +type IndexPart struct { + prefix2item map[PhoneNumberPrefix]*IndexItem +} + +func (ip *IndexPart) Bytes() []byte { + w := bytes.NewBuffer(nil) + var prefixList PhoneNumberPrefixList + for k, _ := range ip.prefix2item { + prefixList = append(prefixList, k) + } + sort.Sort(prefixList) + for _, prefix := range prefixList { + item := ip.prefix2item[prefix] + w.WriteString(strings.Join([]string{ + prefix.String(), + item.RecordOffset.String(), + item.CardTypeID.String(), + }, "|")) + w.WriteByte('\n') + } + return w.Bytes() +} + +func (ip *IndexPart) Parse(reader *bytes.Reader) error { + return nil +} diff --git a/phonedatatool/unpack/offset.go b/phonedatatool/unpack/offset.go new file mode 100644 index 0000000..8eb07b9 --- /dev/null +++ b/phonedatatool/unpack/offset.go @@ -0,0 +1,20 @@ +package unpack + +import ( + "bytes" + "encoding/binary" + "fmt" +) + +type IndexPartOffsetPart struct { + IndexPartOffset int64 +} + +func (op *IndexPartOffsetPart) Parse(reader *bytes.Reader) error { + buf := make([]byte, 4) + if _, err := reader.Read(buf); err != nil { + return fmt.Errorf("failed to read: %v", err) + } + op.IndexPartOffset = int64(binary.LittleEndian.Uint64(buf)) + return nil +} diff --git a/phonedatatool/unpack/record.go b/phonedatatool/unpack/record.go new file mode 100644 index 0000000..67612e4 --- /dev/null +++ b/phonedatatool/unpack/record.go @@ -0,0 +1,70 @@ +package unpack + +import ( + "bytes" + "github.com/xluohome/phonedata/phonedatatool" + "sort" + "strconv" + "strings" +) + +type RecordItem struct { + Province phonedatatool.ProvinceName + City phonedatatool.CityName + ZipCode phonedatatool.ZipCode + AreaCode phonedatatool.AreaCode +} +type RecordOffset int64 + +func (ro RecordOffset) String() string { + return strconv.FormatInt(int64(ro), 10) +} + +type RecordID int64 + +func (rid RecordID) String() string { + return strconv.FormatInt(int64(rid), 10) +} + +type RecordIDList []RecordID + +func (l RecordIDList) Len() int { + return len(l) +} +func (l RecordIDList) Less(i, j int) bool { + return l[i] < l[j] +} +func (l RecordIDList) Swap(i, j int) { + l[i], l[j] = l[j], l[i] +} + +type RecordPart struct { + offset2item map[RecordOffset]*RecordItem + offset2id map[RecordOffset]RecordID + id2offset map[RecordID]RecordOffset +} + +func (rp *RecordPart) Bytes() []byte { + w := bytes.NewBuffer(nil) + var idList RecordIDList + for k, _ := range rp.id2offset { + idList = append(idList, k) + } + sort.Sort(idList) + for _, id := range idList { + item := rp.offset2item[rp.id2offset[id]] + w.WriteString(strings.Join([]string{ + id.String(), + item.Province.String(), + item.City.String(), + item.ZipCode.String(), + item.AreaCode.String(), + }, "|")) + w.WriteByte('\n') + } + return w.Bytes() +} + +func (rp *RecordPart) Parse(reader *bytes.Reader) error { + return nil +} diff --git a/phonedatatool/unpack/unpack.go b/phonedatatool/unpack/unpack.go new file mode 100644 index 0000000..a3170ba --- /dev/null +++ b/phonedatatool/unpack/unpack.go @@ -0,0 +1,111 @@ +package unpack + +import ( + "bytes" + "fmt" + "github.com/xluohome/phonedata/phonedatatool" + "os" + "path" +) + +const ( + VersionFileName = "version.txt" + RecordFileName = "record.txt" + IndexFileName = "index.txt" +) + +type Unpacker struct { +} + +func NewUnpacker() phonedatatool.Unpacker { + return new(Unpacker) +} + +func (u *Unpacker) Unpack(phoneDataFilePath string, plainDirectoryPath string) error { + if err := os.MkdirAll(plainDirectoryPath, 0); err != nil { + return fmt.Errorf("target directory %v not exist and can't be created: %v", plainDirectoryPath, err) + } + + versionFilePath := path.Join(plainDirectoryPath, VersionFileName) + recordFilePath := path.Join(plainDirectoryPath, RecordFileName) + indexFilePath := path.Join(plainDirectoryPath, IndexFileName) + + if err := u.assureAllFileNotExist(versionFilePath, recordFilePath, indexFilePath); err != nil { + return err + } + + var rawBuf []byte + if b, err := os.ReadFile(phoneDataFilePath); err != nil { + return err + } else { + rawBuf = b + } + + if res, err := u.parse(rawBuf); err != nil { + return fmt.Errorf("failed to parse raw file data: %v", err) + } else { + if err := os.WriteFile(versionFilePath, res.versionPart.Bytes(), 0); err != nil { + return err + } + if err := os.WriteFile(recordFilePath, res.recordPart.Bytes(), 0); err != nil { + return err + } + if err := os.WriteFile(indexFilePath, res.indexPart.Bytes(), 0); err != nil { + return err + } + return nil + } +} + +func (u *Unpacker) assureFileNotExist(path string) error { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil + } else { + return fmt.Errorf("check file existence %v failed: %v", path, err) + } + } else { + return fmt.Errorf("file %v already exists", path) + } +} + +func (u *Unpacker) assureAllFileNotExist(paths ...string) error { + for _, p := range paths { + if err := u.assureFileNotExist(p); err != nil { + return err + } + } + return nil +} + +type ParseResult struct { + versionPart *VersionPart + recordPart *RecordPart + indexPart *IndexPart +} + +func (u *Unpacker) parse(buf []byte) (*ParseResult, error) { + reader := bytes.NewReader(buf) + versionPart := new(VersionPart) + if err := versionPart.Parse(reader); err != nil { + return nil, fmt.Errorf("failed to read version part: %v", err) + } + offsetPart := new(IndexPartOffsetPart) + if err := offsetPart.Parse(reader); err != nil { + return nil, fmt.Errorf("failed to read index-part-offset part: %v", err) + } + recordPart := new(RecordPart) + if err := recordPart.Parse(bytes.NewReader(buf[:offsetPart.IndexPartOffset])); err != nil { + return nil, fmt.Errorf("failed to read record part: %v", err) + } + indexPart := new(IndexPart) + if err := indexPart.Parse(bytes.NewReader(buf[offsetPart.IndexPartOffset:])); err != nil { + return nil, fmt.Errorf("failed to read index part: %v", err) + } + + return &ParseResult{ + versionPart: versionPart, + recordPart: recordPart, + indexPart: indexPart, + }, nil +} diff --git a/phonedatatool/unpack/version.go b/phonedatatool/unpack/version.go new file mode 100644 index 0000000..065b0b8 --- /dev/null +++ b/phonedatatool/unpack/version.go @@ -0,0 +1,32 @@ +package unpack + +import ( + "bytes" + "fmt" +) + +type Version string + +func (v Version) String() string { + return string(v) +} + +type VersionPart struct { + Version Version +} + +func (vp *VersionPart) Bytes() []byte { + w := bytes.NewBuffer(nil) + w.Write([]byte(vp.Version)) + w.WriteByte('\n') + return w.Bytes() +} + +func (vp *VersionPart) Parse(reader *bytes.Reader) error { + buf := make([]byte, 4) + if _, err := reader.Read(buf); err != nil { + return fmt.Errorf("failed to read: %v", err) + } + vp.Version = Version(buf) + return nil +} From 680ae078da9488010705d70cee999440b8b78653 Mon Sep 17 00:00:00 2001 From: jyh Date: Tue, 23 May 2023 20:05:15 +0800 Subject: [PATCH 03/29] =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/phonedatatool/main.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/cmd/phonedatatool/main.go b/cmd/phonedatatool/main.go index ece119f..86cd6f3 100644 --- a/cmd/phonedatatool/main.go +++ b/cmd/phonedatatool/main.go @@ -18,12 +18,6 @@ const ( Version = "0.0.1" Author = "seedjyh@gmail.com" FullName = Name + "_" + Version + "(" + Author + ")" - Logo = ` - ____ __ ______ _ ____________ ___ _________ __________ ____ __ - / __ \/ / / / __ \/ | / / ____/ __ \/ |/_ __/ |/_ __/ __ \/ __ \/ / - / /_/ / /_/ / / / / |/ / __/ / / / / /| | / / / /| | / / / / / / / / / / - / ____/ __ / /_/ / /| / /___/ /_/ / ___ |/ / / ___ |/ / / /_/ / /_/ / /___ -/_/ /_/ /_/\____/_/ |_/_____/_____/_/ |_/_/ /_/ |_/_/ \____/\____/_____/` + FullName ) func main() { @@ -41,9 +35,7 @@ func main() { return } if *showHelpFlag { - fmt.Println("./phonedatatool -unpack -i phone.dat -o tmp") - fmt.Println("./phonedatatool -pack -i tmp -o phone.dat") - fmt.Println("./phonedatatool -query -i phone.dat -n 13000001234") + showHelp() return } if *unpackFlag { @@ -56,6 +48,9 @@ func main() { return } if err := unpack.NewUnpacker().Unpack(*source, *destination); err != nil { + fmt.Println("ERROR! Unpack failed.", err) + return + } else { fmt.Println("Unpack completed.") return } @@ -70,6 +65,9 @@ func main() { return } if err := pack.NewPacker().Pack(*source, *destination); err != nil { + fmt.Println("ERROR! Pack failed.", err) + return + } else { fmt.Println("Pack completed.") return } @@ -84,7 +82,7 @@ func main() { return } if info, err := query.NewQuerier(*source).QueryNumber(*number); err != nil { - fmt.Println("ERROR! ", err) + fmt.Println("ERROR! Query failed.", err) return } else { fmt.Println("PhoneNum: ", info.PhoneNumber) @@ -93,7 +91,18 @@ func main() { fmt.Println("City: ", info.CityName) fmt.Println("ZipCode: ", info.ZipCode) fmt.Println("Province: ", info.ProvinceName) + fmt.Println("Query completed.") + return } } + fmt.Println("Did nothing.") + showHelp() return } + +func showHelp() { + fmt.Println("<< HELP >>") + fmt.Println("./phonedatatool -unpack -i phone.dat -o tmp") + fmt.Println("./phonedatatool -pack -i tmp -o phone.dat") + fmt.Println("./phonedatatool -query -i phone.dat -n 13000001234") +} From 8b8eba300998e8f9eb39afeba413a2216b9d280e Mon Sep 17 00:00:00 2001 From: jyh Date: Tue, 23 May 2023 20:05:29 +0800 Subject: [PATCH 04/29] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20offset=20=E8=AE=A1?= =?UTF-8?q?=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/unpack/offset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phonedatatool/unpack/offset.go b/phonedatatool/unpack/offset.go index 8eb07b9..c97be3a 100644 --- a/phonedatatool/unpack/offset.go +++ b/phonedatatool/unpack/offset.go @@ -15,6 +15,6 @@ func (op *IndexPartOffsetPart) Parse(reader *bytes.Reader) error { if _, err := reader.Read(buf); err != nil { return fmt.Errorf("failed to read: %v", err) } - op.IndexPartOffset = int64(binary.LittleEndian.Uint64(buf)) + op.IndexPartOffset = int64(binary.LittleEndian.Uint32(buf)) return nil } From 97a17c0a4de506f1785262a40e0bfb386639fe67 Mon Sep 17 00:00:00 2001 From: jyh Date: Tue, 23 May 2023 22:15:05 +0800 Subject: [PATCH 05/29] =?UTF-8?q?=E5=AE=8C=E6=88=90=20unpack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/model.go | 7 ++++-- phonedatatool/unpack/index.go | 41 ++++++++++++++++++++++++++++++++- phonedatatool/unpack/offset.go | 4 ++-- phonedatatool/unpack/record.go | 42 +++++++++++++++++++++++++++++++++- phonedatatool/unpack/unpack.go | 11 +++++---- phonedatatool/util/reader.go | 17 ++++++++++++++ 6 files changed, 112 insertions(+), 10 deletions(-) create mode 100644 phonedatatool/util/reader.go diff --git a/phonedatatool/model.go b/phonedatatool/model.go index e17fc33..3294ee8 100644 --- a/phonedatatool/model.go +++ b/phonedatatool/model.go @@ -1,6 +1,9 @@ package phonedatatool -import "github.com/xluohome/phonedata" +import ( + "github.com/xluohome/phonedata" + "strconv" +) type PhoneNumber string // 手机号码 type AreaCode string // 区号 @@ -10,7 +13,7 @@ func (ac AreaCode) String() string { type CardTypeID byte // 卡类型 ID，单字节 func (ctid CardTypeID) String() string { - return string(ctid) + return strconv.Itoa(int(ctid)) } func (ctid CardTypeID) ToName() CardTypeName { if v, ok := phonedata.CardTypemap[byte(ctid)]; ok { diff --git a/phonedatatool/unpack/index.go b/phonedatatool/unpack/index.go index e904b35..2cc43eb 100644 --- a/phonedatatool/unpack/index.go +++ b/phonedatatool/unpack/index.go @@ -2,8 +2,11 @@ package unpack import ( "bytes" + "encoding/binary" + "fmt" "github.com/xluohome/phonedata/phonedatatool" "sort" + "strconv" "strings" ) @@ -26,15 +29,33 @@ func (pl PhoneNumberPrefixList) Swap(i, j int) { } type IndexItem struct { + NumberPrefix PhoneNumberPrefix CardTypeID phonedatatool.CardTypeID RecordOffset RecordOffset RecordID RecordID } +func (ii *IndexItem) Parse(reader *bytes.Reader) error { + buf := make([]byte, 9) + if _, err := reader.Read(buf); err != nil { + return err + } + ii.NumberPrefix = PhoneNumberPrefix(strconv.Itoa(int(binary.LittleEndian.Uint32(buf[:4])))) + ii.RecordOffset = RecordOffset(binary.LittleEndian.Uint32(buf[4:8])) + ii.CardTypeID = phonedatatool.CardTypeID(buf[8]) + return nil +} + type IndexPart struct { prefix2item map[PhoneNumberPrefix]*IndexItem } +func NewIndexPart() *IndexPart { + return &IndexPart{ + prefix2item: make(map[PhoneNumberPrefix]*IndexItem), + } +} + func (ip *IndexPart) Bytes() []byte { w := bytes.NewBuffer(nil) var prefixList PhoneNumberPrefixList @@ -46,7 +67,7 @@ func (ip *IndexPart) Bytes() []byte { item := ip.prefix2item[prefix] w.WriteString(strings.Join([]string{ prefix.String(), - item.RecordOffset.String(), + item.RecordID.String(), item.CardTypeID.String(), }, "|")) w.WriteByte('\n') @@ -55,5 +76,23 @@ func (ip *IndexPart) Bytes() []byte { } func (ip *IndexPart) Parse(reader *bytes.Reader) error { + for reader.Len() > 0 { + item := new(IndexItem) + if err := item.Parse(reader); err != nil { + return err + } + ip.prefix2item[item.NumberPrefix] = item + } + return nil +} + +func (ip *IndexPart) MatchRecordOffsetToRecordID(offset2id map[RecordOffset]RecordID) error { + for _, v := range ip.prefix2item { + if id, ok := offset2id[v.RecordOffset]; ok { + v.RecordID = id + } else { + return fmt.Errorf("failed to find record id for record offset %v", v.RecordOffset) + } + } return nil } diff --git a/phonedatatool/unpack/offset.go b/phonedatatool/unpack/offset.go index c97be3a..b4867a0 100644 --- a/phonedatatool/unpack/offset.go +++ b/phonedatatool/unpack/offset.go @@ -7,7 +7,7 @@ import ( ) type IndexPartOffsetPart struct { - IndexPartOffset int64 + IndexPartOffset RecordOffset } func (op *IndexPartOffsetPart) Parse(reader *bytes.Reader) error { @@ -15,6 +15,6 @@ func (op *IndexPartOffsetPart) Parse(reader *bytes.Reader) error { if _, err := reader.Read(buf); err != nil { return fmt.Errorf("failed to read: %v", err) } - op.IndexPartOffset = int64(binary.LittleEndian.Uint32(buf)) + op.IndexPartOffset = RecordOffset(binary.LittleEndian.Uint32(buf)) return nil } diff --git a/phonedatatool/unpack/record.go b/phonedatatool/unpack/record.go index 67612e4..1175c79 100644 --- a/phonedatatool/unpack/record.go +++ b/phonedatatool/unpack/record.go @@ -2,7 +2,9 @@ package unpack import ( "bytes" + "fmt" "github.com/xluohome/phonedata/phonedatatool" + "github.com/xluohome/phonedata/phonedatatool/util" "sort" "strconv" "strings" @@ -14,6 +16,23 @@ type RecordItem struct { ZipCode phonedatatool.ZipCode AreaCode phonedatatool.AreaCode } + +func (ri *RecordItem) Parse(reader *bytes.Reader) error { + if buf, err := util.ReadUntil(reader, 0); err != nil { + return fmt.Errorf("no term char for record item: %v", err) + } else { + words := strings.Split(string(buf), "|") + if len(words) != 4 { + return fmt.Errorf("invalid item bytes, %v", string(buf)) + } + ri.Province = phonedatatool.ProvinceName(words[0]) + ri.City = phonedatatool.CityName(words[1]) + ri.ZipCode = phonedatatool.ZipCode(words[2]) + ri.AreaCode = phonedatatool.AreaCode(words[3]) + return nil + } +} + type RecordOffset int64 func (ro RecordOffset) String() string { @@ -44,6 +63,14 @@ type RecordPart struct { id2offset map[RecordID]RecordOffset } +func NewRecordPart() *RecordPart { + return &RecordPart{ + offset2item: make(map[RecordOffset]*RecordItem), + offset2id: make(map[RecordOffset]RecordID), + id2offset: make(map[RecordID]RecordOffset), + } +} + func (rp *RecordPart) Bytes() []byte { w := bytes.NewBuffer(nil) var idList RecordIDList @@ -65,6 +92,19 @@ func (rp *RecordPart) Bytes() []byte { return w.Bytes() } -func (rp *RecordPart) Parse(reader *bytes.Reader) error { +func (rp *RecordPart) Parse(reader *bytes.Reader, endOffset RecordOffset) error { + for nowID := RecordID(1); ; nowID++ { + startOffset := RecordOffset(reader.Size() - int64(reader.Len())) + if startOffset >= endOffset { + break + } + item := new(RecordItem) + if err := item.Parse(reader); err != nil { + return err + } + rp.offset2item[startOffset] = item + rp.offset2id[startOffset] = nowID + rp.id2offset[nowID] = startOffset + } return nil } diff --git a/phonedatatool/unpack/unpack.go b/phonedatatool/unpack/unpack.go index a3170ba..a0b88df 100644 --- a/phonedatatool/unpack/unpack.go +++ b/phonedatatool/unpack/unpack.go @@ -94,14 +94,17 @@ func (u *Unpacker) parse(buf []byte) (*ParseResult, error) { if err := offsetPart.Parse(reader); err != nil { return nil, fmt.Errorf("failed to read index-part-offset part: %v", err) } - recordPart := new(RecordPart) - if err := recordPart.Parse(bytes.NewReader(buf[:offsetPart.IndexPartOffset])); err != nil { + recordPart := NewRecordPart() + if err := recordPart.Parse(reader, offsetPart.IndexPartOffset); err != nil { return nil, fmt.Errorf("failed to read record part: %v", err) } - indexPart := new(IndexPart) - if err := indexPart.Parse(bytes.NewReader(buf[offsetPart.IndexPartOffset:])); err != nil { + indexPart := NewIndexPart() + if err := indexPart.Parse(reader); err != nil { return nil, fmt.Errorf("failed to read index part: %v", err) } + if err := indexPart.MatchRecordOffsetToRecordID(recordPart.offset2id); err != nil { + return nil, fmt.Errorf("failed to match offset to record id: %v", err) + } return &ParseResult{ versionPart: versionPart, diff --git a/phonedatatool/util/reader.go b/phonedatatool/util/reader.go new file mode 100644 index 0000000..c19b6e2 --- /dev/null +++ b/phonedatatool/util/reader.go @@ -0,0 +1,17 @@ +package util + +import "bytes" + +// ReadUntil 读取，直到遇到特定字符 +func ReadUntil(reader *bytes.Reader, term byte) ([]byte, error) { + var buf []byte + for { + if b, err := reader.ReadByte(); err != nil { + return nil, err + } else if b == term { + return buf, nil + } else { + buf = append(buf, b) + } + } +} From 25dd366758e99895f61630077b898e60b3c05a63 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 10:25:20 +0800 Subject: [PATCH 06/29] =?UTF-8?q?=E5=BC=80=E5=8F=91=E6=89=93=E5=8C=85?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=EF=BC=9A=E5=AE=9E=E7=8E=B0=20offset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 ++ phonedatatool/pack/offset.go | 9 +++++++++ phonedatatool/pack/offset_test.go | 8 ++++++++ phonedatatool/pack/version.go | 4 ++++ 4 files changed, 23 insertions(+) create mode 100644 phonedatatool/pack/offset.go create mode 100644 phonedatatool/pack/offset_test.go create mode 100644 phonedatatool/pack/version.go diff --git a/go.mod b/go.mod index 48db9d0..2847f1a 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/xluohome/phonedata go 1.16 + +require github.com/stretchr/testify v1.8.3 diff --git a/phonedatatool/pack/offset.go b/phonedatatool/pack/offset.go new file mode 100644 index 0000000..e2d85fb --- /dev/null +++ b/phonedatatool/pack/offset.go @@ -0,0 +1,9 @@ +package pack + +import "encoding/binary" + +type Offset int64 + +func (o Offset) Bytes() []byte { + return binary.LittleEndian.AppendUint32(nil, uint32(o)) +} diff --git a/phonedatatool/pack/offset_test.go b/phonedatatool/pack/offset_test.go new file mode 100644 index 0000000..4cbb83c --- /dev/null +++ b/phonedatatool/pack/offset_test.go @@ -0,0 +1,8 @@ +package pack + +import "testing" +import "github.com/stretchr/testify/assert" + +func TestOffset_Bytes(t *testing.T) { + assert.Equal(t, []byte{0xd2, 0x04, 0x00, 0x00}, Offset(1234).Bytes()) +} diff --git a/phonedatatool/pack/version.go b/phonedatatool/pack/version.go new file mode 100644 index 0000000..7660917 --- /dev/null +++ b/phonedatatool/pack/version.go @@ -0,0 +1,4 @@ +package pack + +type VersionPart struct { +} From 63e9ed17f17e10dc110d770055ad476eccb7db42 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 10:35:28 +0800 Subject: [PATCH 07/29] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=20version=20part?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/version.go | 28 ++++++++++++++++++++++++++++ phonedatatool/pack/version_test.go | 18 ++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 phonedatatool/pack/version_test.go diff --git a/phonedatatool/pack/version.go b/phonedatatool/pack/version.go index 7660917..da728f7 100644 --- a/phonedatatool/pack/version.go +++ b/phonedatatool/pack/version.go @@ -1,4 +1,32 @@ package pack +import ( + "bytes" + "fmt" +) + type VersionPart struct { + version string +} + +// Bytes 打包成二进制文件里的样子 +func (p *VersionPart) Bytes() []byte { + w := bytes.NewBuffer(nil) + w.WriteString(p.version) + for w.Len() < 4 { + w.WriteByte(0) + } + return w.Bytes() +} + +// ParsePlainText 从文本文件读取 +func (p *VersionPart) ParsePlainText(reader *bytes.Reader) error { + buf := make([]byte, 4) + if n, err := reader.Read(buf); err != nil { + return err + } else if n != 4 { + return fmt.Errorf("expect read 4 bytes, but read %v bytes", n) + } + p.version = string(buf) + return nil } diff --git a/phonedatatool/pack/version_test.go b/phonedatatool/pack/version_test.go new file mode 100644 index 0000000..4976a03 --- /dev/null +++ b/phonedatatool/pack/version_test.go @@ -0,0 +1,18 @@ +package pack + +import ( + "bytes" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestVersionPart_Bytes(t *testing.T) { + assert.Equal(t, []byte{'2', '3', '0', '6'}, (&VersionPart{version: "2306"}).Bytes()) +} + +func TestVersionPart_ParsePlainText(t *testing.T) { + reader := bytes.NewReader([]byte("2306\n")) + versionPart := new(VersionPart) + assert.NoError(t, versionPart.ParsePlainText(reader)) + assert.Equal(t, "2306", versionPart.version) +} From 2c73f252e7ccbd4c0f2f2716b3f457c0ba7af73c Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 11:00:26 +0800 Subject: [PATCH 08/29] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=20record=20part=20pars?= =?UTF-8?q?ePlainText?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/record.go | 60 +++++++++++++++++++++++++++++++ phonedatatool/pack/record_test.go | 28 +++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 phonedatatool/pack/record.go create mode 100644 phonedatatool/pack/record_test.go diff --git a/phonedatatool/pack/record.go b/phonedatatool/pack/record.go new file mode 100644 index 0000000..5251f1e --- /dev/null +++ b/phonedatatool/pack/record.go @@ -0,0 +1,60 @@ +package pack + +import ( + "bytes" + "fmt" + "github.com/xluohome/phonedata/phonedatatool/util" + "strconv" + "strings" +) + +type RecordID int64 + +type RecordItem struct { + province string + city string + zipCode string + areaCode string +} + +type RecordPart struct { + id2item map[RecordID]*RecordItem +} + +func NewRecordPart() *RecordPart { + return &RecordPart{id2item: make(map[RecordID]*RecordItem)} +} + +func (p *RecordPart) ParsePlainText(reader *bytes.Reader) error { + for reader.Len() > 0 { + + var words []string + if b, err := util.ReadUntil(reader, '\n'); err != nil { + return err + } else { + words = strings.Split(string(b), "|") + } + if len(words) != 5 { + return fmt.Errorf("invalid record line. expect 5 words (id, province, city, zipCode, areaCode), got %v words, %v", len(words), words) + } + + var recordID RecordID + if id, err := strconv.Atoi(words[0]); err != nil { + return fmt.Errorf("invalid id format, raw=%v, err=%v", words[0], err) + } else { + recordID = RecordID(id) + } + + if _, ok := p.id2item[recordID]; ok { + return fmt.Errorf("duplicate recordID %v", recordID) + } + + p.id2item[recordID] = &RecordItem{ + province: words[1], + city: words[2], + zipCode: words[3], + areaCode: words[4], + } + } + return nil +} diff --git a/phonedatatool/pack/record_test.go b/phonedatatool/pack/record_test.go new file mode 100644 index 0000000..1a93b6f --- /dev/null +++ b/phonedatatool/pack/record_test.go @@ -0,0 +1,28 @@ +package pack + +import ( + "bytes" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestRecordPart_ParsePlainText(t *testing.T) { + plainText := []byte("\x31\x7C\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\xB7\xA2\xE6\xB9\x96\x7C\x32\x33\x38\x30\x30\x30\x7C\x30\x35\x35\x31\x0A\x32\x7C\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\x90\x88\xE8\x82\xA5\x7C\x32\x33\x30\x30\x30\x30\x7C\x30\x35\x35\x31\x0A") + recordPart := NewRecordPart() + assert.NoError(t, recordPart.ParsePlainText(bytes.NewReader(plainText))) + expectedMap := map[RecordID]*RecordItem{ + 1: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\xB7\xA2\xE6\xB9\x96", + zipCode: "238000", + areaCode: "0551", + }, + 2: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\x90\x88\xE8\x82\xA5", + zipCode: "230000", + areaCode: "0551", + }, + } + assert.Equal(t, expectedMap, recordPart.id2item) +} From 3303b39e5559a3073cc40c66b6200d0cdc676957 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 11:03:04 +0800 Subject: [PATCH 09/29] =?UTF-8?q?=E8=B0=83=E6=95=B4=20version=20part=20?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/version.go | 20 ++++++++++---------- phonedatatool/pack/version_test.go | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/phonedatatool/pack/version.go b/phonedatatool/pack/version.go index da728f7..504c887 100644 --- a/phonedatatool/pack/version.go +++ b/phonedatatool/pack/version.go @@ -9,16 +9,6 @@ type VersionPart struct { version string } -// Bytes 打包成二进制文件里的样子 -func (p *VersionPart) Bytes() []byte { - w := bytes.NewBuffer(nil) - w.WriteString(p.version) - for w.Len() < 4 { - w.WriteByte(0) - } - return w.Bytes() -} - // ParsePlainText 从文本文件读取 func (p *VersionPart) ParsePlainText(reader *bytes.Reader) error { buf := make([]byte, 4) @@ -30,3 +20,13 @@ func (p *VersionPart) ParsePlainText(reader *bytes.Reader) error { p.version = string(buf) return nil } + +// Bytes 打包成二进制文件里的样子 +func (p *VersionPart) Bytes() []byte { + w := bytes.NewBuffer(nil) + w.WriteString(p.version) + for w.Len() < 4 { + w.WriteByte(0) + } + return w.Bytes() +} diff --git a/phonedatatool/pack/version_test.go b/phonedatatool/pack/version_test.go index 4976a03..32d3546 100644 --- a/phonedatatool/pack/version_test.go +++ b/phonedatatool/pack/version_test.go @@ -6,13 +6,13 @@ import ( "testing" ) -func TestVersionPart_Bytes(t *testing.T) { - assert.Equal(t, []byte{'2', '3', '0', '6'}, (&VersionPart{version: "2306"}).Bytes()) -} - func TestVersionPart_ParsePlainText(t *testing.T) { reader := bytes.NewReader([]byte("2306\n")) versionPart := new(VersionPart) assert.NoError(t, versionPart.ParsePlainText(reader)) assert.Equal(t, "2306", versionPart.version) } + +func TestVersionPart_Bytes(t *testing.T) { + assert.Equal(t, []byte{'2', '3', '0', '6'}, (&VersionPart{version: "2306"}).Bytes()) +} From 4ba592be025a387069c6a3c5bf020f77dc1bf3e9 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 11:18:39 +0800 Subject: [PATCH 10/29] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E4=BA=86=20record=20pa?= =?UTF-8?q?rt=20bytes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/record.go | 37 +++++++++++++++++++++++++++++++ phonedatatool/pack/record_test.go | 23 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/phonedatatool/pack/record.go b/phonedatatool/pack/record.go index 5251f1e..0e108ea 100644 --- a/phonedatatool/pack/record.go +++ b/phonedatatool/pack/record.go @@ -4,12 +4,25 @@ import ( "bytes" "fmt" "github.com/xluohome/phonedata/phonedatatool/util" + "sort" "strconv" "strings" ) type RecordID int64 +type RecordIDList []RecordID + +func (idl RecordIDList) Len() int { + return len(idl) +} +func (idl RecordIDList) Less(i, j int) bool { + return idl[i] < idl[j] +} +func (idl RecordIDList) Swap(i, j int) { + idl[i], idl[j] = idl[j], idl[i] +} + type RecordItem struct { province string city string @@ -17,6 +30,13 @@ type RecordItem struct { areaCode string } +func (ri *RecordItem) Bytes() []byte { + w := bytes.NewBuffer(nil) + w.WriteString(strings.Join([]string{ri.province, ri.city, ri.zipCode, ri.areaCode}, "|")) + w.WriteByte(0) + return w.Bytes() +} + type RecordPart struct { id2item map[RecordID]*RecordItem } @@ -58,3 +78,20 @@ func (p *RecordPart) ParsePlainText(reader *bytes.Reader) error { } return nil } + +func (p *RecordPart) Bytes(baseOffset Offset) ([]byte, map[RecordID]Offset) { + w := bytes.NewBuffer(nil) + id2offset := make(map[RecordID]Offset) + + var idList RecordIDList + for k := range p.id2item { + idList = append(idList, k) + } + sort.Sort(idList) + + for _, id := range idList { + id2offset[id] = baseOffset + Offset(w.Len()) + w.Write(p.id2item[id].Bytes()) + } + return w.Bytes(), id2offset +} diff --git a/phonedatatool/pack/record_test.go b/phonedatatool/pack/record_test.go index 1a93b6f..1e68322 100644 --- a/phonedatatool/pack/record_test.go +++ b/phonedatatool/pack/record_test.go @@ -26,3 +26,26 @@ func TestRecordPart_ParsePlainText(t *testing.T) { } assert.Equal(t, expectedMap, recordPart.id2item) } + +func TestRecordPart_Bytes(t *testing.T) { + recordPart := &RecordPart{id2item: map[RecordID]*RecordItem{ + 1: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\xB7\xA2\xE6\xB9\x96", + zipCode: "238000", + areaCode: "0551", + }, + 2: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\x90\x88\xE8\x82\xA5", + zipCode: "230000", + areaCode: "0551", + }, + }} + buf, id2offset := recordPart.Bytes(8) + assert.Equal(t, []byte("\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\xB7\xA2\xE6\xB9\x96\x7C\x32\x33\x38\x30\x30\x30\x7C\x30\x35\x35\x31\x00\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\x90\x88\xE8\x82\xA5\x7C\x32\x33\x30\x30\x30\x30\x7C\x30\x35\x35\x31\x00"), buf) + assert.Equal(t, map[RecordID]Offset{ + 1: 8, + 2: 34, + }, id2offset) +} From a31aa848c49a4590151c8946e7d0a87a67ed7bfd Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 13:21:04 +0800 Subject: [PATCH 11/29] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E4=BA=86=20index=20par?= =?UTF-8?q?t=20parsePlainText?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/index.go | 86 ++++++++++++++++++++++++++++++++ phonedatatool/pack/index_test.go | 32 ++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 phonedatatool/pack/index.go create mode 100644 phonedatatool/pack/index_test.go diff --git a/phonedatatool/pack/index.go b/phonedatatool/pack/index.go new file mode 100644 index 0000000..3b4a078 --- /dev/null +++ b/phonedatatool/pack/index.go @@ -0,0 +1,86 @@ +package pack + +import ( + "bytes" + "fmt" + "github.com/xluohome/phonedata/phonedatatool" + "github.com/xluohome/phonedata/phonedatatool/util" + "strconv" + "strings" +) + +type NumberPrefix int32 +type NumberPrefixList []NumberPrefix + +func (pl NumberPrefixList) Len() int { + return len(pl) +} +func (pl NumberPrefixList) Less(i, j int) bool { + return pl[i] < pl[j] +} +func (pl NumberPrefixList) Swap(i, j int) { + pl[i], pl[j] = pl[j], pl[i] +} + +type IndexItem struct { + recordOffset Offset + cardType phonedatatool.CardTypeID +} + +type IndexPart struct { + prefix2item map[NumberPrefix]*IndexItem +} + +func NewIndexPart() *IndexPart { + return &IndexPart{ + prefix2item: make(map[NumberPrefix]*IndexItem), + } +} + +func (p *IndexPart) ParsePlainText(reader *bytes.Reader, id2offset map[RecordID]Offset) error { + for reader.Len() > 0 { + var words []string + if buf, err := util.ReadUntil(reader, '\n'); err != nil { + return err + } else { + words = strings.Split(string(buf), "|") + } + + if len(words) != 3 { + return fmt.Errorf("expect words len is 3, got %v, %v", len(words), words) + } + + var numberPrefix NumberPrefix + if v, err := strconv.Atoi(words[0]); err != nil { + return fmt.Errorf("invalid number prefix %v: %v", words[0], err) + } else { + numberPrefix = NumberPrefix(v) + } + + var recordOffset Offset + if v, err := strconv.Atoi(words[1]); err != nil { + return fmt.Errorf("invalid record id %v: %v", words[1], err) + } else if offset, ok := id2offset[RecordID(v)]; !ok { + return fmt.Errorf("no offset for record id %v", v) + } else { + recordOffset = offset + } + + var cardTypeID phonedatatool.CardTypeID + if v, err := strconv.Atoi(words[2]); err != nil { + return fmt.Errorf("invalid card type id %v: %v", words[2], err) + } else { + cardTypeID = phonedatatool.CardTypeID(v) + } + + p.prefix2item[numberPrefix] = &IndexItem{ + recordOffset: recordOffset, + cardType: cardTypeID, + } + } + return nil +} + +func (p *IndexPart) Bytes() []byte { + return nil +} diff --git a/phonedatatool/pack/index_test.go b/phonedatatool/pack/index_test.go new file mode 100644 index 0000000..91d5ab5 --- /dev/null +++ b/phonedatatool/pack/index_test.go @@ -0,0 +1,32 @@ +package pack + +import ( + "bytes" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestIndexPart_ParsePlainText(t *testing.T) { + indexPart := NewIndexPart() + reader := bytes.NewReader([]byte("1300000|251|2\n1300001|176|2\n1300002|1|2\n")) + id2offset := map[RecordID]Offset{ + 1: 100, + 176: 200, + 251: 300, + } + assert.NoError(t, indexPart.ParsePlainText(reader, id2offset)) + assert.Equal(t, map[NumberPrefix]*IndexItem{ + 1300000: { + recordOffset: 300, + cardType: 2, + }, + 1300001: { + recordOffset: 200, + cardType: 2, + }, + 1300002: { + recordOffset: 100, + cardType: 2, + }, + }, indexPart.prefix2item) +} From e36cee0f0fec9d25c313d36e8ff3d6c40db0aa3b Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 13:32:32 +0800 Subject: [PATCH 12/29] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E4=BA=86=20index=20par?= =?UTF-8?q?t=20bytes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/model.go | 5 +++++ phonedatatool/pack/index.go | 33 +++++++++++++++++++++++++++++--- phonedatatool/pack/index_test.go | 31 +++++++++++++++++++++++++++--- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/phonedatatool/model.go b/phonedatatool/model.go index 3294ee8..9004618 100644 --- a/phonedatatool/model.go +++ b/phonedatatool/model.go @@ -12,6 +12,11 @@ func (ac AreaCode) String() string { } type CardTypeID byte // 卡类型 ID，单字节 + +func (ctid CardTypeID) Bytes() []byte { + return []byte{byte(ctid)} +} + func (ctid CardTypeID) String() string { return strconv.Itoa(int(ctid)) } diff --git a/phonedatatool/pack/index.go b/phonedatatool/pack/index.go index 3b4a078..d5500a8 100644 --- a/phonedatatool/pack/index.go +++ b/phonedatatool/pack/index.go @@ -2,14 +2,21 @@ package pack import ( "bytes" + "encoding/binary" "fmt" "github.com/xluohome/phonedata/phonedatatool" "github.com/xluohome/phonedata/phonedatatool/util" + "sort" "strconv" "strings" ) type NumberPrefix int32 + +func (p NumberPrefix) Bytes() []byte { + return binary.LittleEndian.AppendUint32(nil, uint32(p)) +} + type NumberPrefixList []NumberPrefix func (pl NumberPrefixList) Len() int { @@ -23,8 +30,17 @@ func (pl NumberPrefixList) Swap(i, j int) { } type IndexItem struct { + numberPrefix NumberPrefix recordOffset Offset - cardType phonedatatool.CardTypeID + cardTypeID phonedatatool.CardTypeID +} + +func (ii IndexItem) Bytes() []byte { + w := bytes.NewBuffer(nil) + w.Write(ii.numberPrefix.Bytes()) + w.Write(ii.recordOffset.Bytes()) + w.Write(ii.cardTypeID.Bytes()) + return w.Bytes() } type IndexPart struct { @@ -74,13 +90,24 @@ func (p *IndexPart) ParsePlainText(reader *bytes.Reader, id2offset map[RecordID] } p.prefix2item[numberPrefix] = &IndexItem{ + numberPrefix: numberPrefix, recordOffset: recordOffset, - cardType: cardTypeID, + cardTypeID: cardTypeID, } } return nil } func (p *IndexPart) Bytes() []byte { - return nil + var prefixList NumberPrefixList + for k := range p.prefix2item { + prefixList = append(prefixList, k) + } + sort.Sort(prefixList) + + w := bytes.NewBuffer(nil) + for _, prefix := range prefixList { + w.Write(p.prefix2item[prefix].Bytes()) + } + return w.Bytes() } diff --git a/phonedatatool/pack/index_test.go b/phonedatatool/pack/index_test.go index 91d5ab5..0511434 100644 --- a/phonedatatool/pack/index_test.go +++ b/phonedatatool/pack/index_test.go @@ -17,16 +17,41 @@ func TestIndexPart_ParsePlainText(t *testing.T) { assert.NoError(t, indexPart.ParsePlainText(reader, id2offset)) assert.Equal(t, map[NumberPrefix]*IndexItem{ 1300000: { + numberPrefix: 1300000, recordOffset: 300, - cardType: 2, + cardTypeID: 2, }, 1300001: { + numberPrefix: 1300001, recordOffset: 200, - cardType: 2, + cardTypeID: 2, }, 1300002: { + numberPrefix: 1300002, recordOffset: 100, - cardType: 2, + cardTypeID: 2, }, }, indexPart.prefix2item) } + +func TestIndexPart_Bytes(t *testing.T) { + var indexPart = &IndexPart{prefix2item: map[NumberPrefix]*IndexItem{ + 1300000: { + numberPrefix: 1300000, + recordOffset: 0x1A4E, + cardTypeID: 2, + }, + 1300001: { + numberPrefix: 1300001, + recordOffset: 0x122C, + cardTypeID: 2, + }, + 1300002: { + numberPrefix: 1300002, + recordOffset: 0x0008, + cardTypeID: 2, + }, + }} + assert.Equal(t, []byte("\x20\xD6\x13\x00\x4E\x1A\x00\x00\x02\x21\xD6\x13\x00\x2C\x12\x00\x00\x02\x22\xD6\x13\x00\x08\x00\x00\x00\x02"), indexPart.Bytes()) + +} From f1a7151174a8b2b47dac08f840d8add49f397462 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 13:41:04 +0800 Subject: [PATCH 13/29] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E4=BA=86=20pack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/pack.go | 61 +++++++++++++++++++++++++++++++++- phonedatatool/phonedatatool.go | 6 ++++ phonedatatool/unpack/unpack.go | 12 ++----- phonedatatool/util/file.go | 27 +++++++++++++++ 4 files changed, 96 insertions(+), 10 deletions(-) create mode 100644 phonedatatool/util/file.go diff --git a/phonedatatool/pack/pack.go b/phonedatatool/pack/pack.go index cad3a84..f1438ed 100644 --- a/phonedatatool/pack/pack.go +++ b/phonedatatool/pack/pack.go @@ -1,13 +1,72 @@ package pack -import "github.com/xluohome/phonedata/phonedatatool" +import ( + "bytes" + "fmt" + "github.com/xluohome/phonedata/phonedatatool" + "github.com/xluohome/phonedata/phonedatatool/util" + "os" + "path" +) type Packer struct{} +const RecordPartBaseOffset = Offset(8) // record part 首字节偏移量 + func NewPacker() phonedatatool.Packer { return new(Packer) } func (p *Packer) Pack(plainDirectoryPath string, phoneDataFilePath string) error { + if err := util.AssureFileNotExist(phoneDataFilePath); err != nil { + return err + } + + versionPart := new(VersionPart) + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.VersionFileName)); err != nil { + return err + } else if err := versionPart.ParsePlainText(bytes.NewReader(buf)); err != nil { + return err + } + versionPartBuf := versionPart.Bytes() + + recordPart := NewRecordPart() + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.RecordFileName)); err != nil { + return err + } else if err := recordPart.ParsePlainText(bytes.NewReader(buf)); err != nil { + return err + } + recordPartBuf, recordID2Offset := recordPart.Bytes(RecordPartBaseOffset) + + indexPartOffsetPart := RecordPartBaseOffset + Offset(len(recordPartBuf)) + + indexPart := NewIndexPart() + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.IndexFileName)); err != nil { + return err + } else if err := indexPart.ParsePlainText(bytes.NewReader(buf), recordID2Offset); err != nil { + return err + } + indexPartBuf := indexPart.Bytes() + + var outFile *os.File + if f, err := os.OpenFile(phoneDataFilePath, os.O_CREATE|os.O_WRONLY, 0); err != nil { + return fmt.Errorf("failed to open file %v to write, %v", phoneDataFilePath, err) + } else { + outFile = f + } + defer outFile.Close() + + if _, err := outFile.Write(versionPartBuf); err != nil { + return err + } + if _, err := outFile.Write(indexPartOffsetPart.Bytes()); err != nil { + return err + } + if _, err := outFile.Write(recordPartBuf); err != nil { + return err + } + if _, err := outFile.Write(indexPartBuf); err != nil { + return err + } return nil } diff --git a/phonedatatool/phonedatatool.go b/phonedatatool/phonedatatool.go index e6e8a08..24b1dd8 100644 --- a/phonedatatool/phonedatatool.go +++ b/phonedatatool/phonedatatool.go @@ -22,3 +22,9 @@ type QueryResult struct { type Querier interface { QueryNumber(number string) (*QueryResult, error) } + +const ( + VersionFileName = "version.txt" + RecordFileName = "record.txt" + IndexFileName = "index.txt" +) diff --git a/phonedatatool/unpack/unpack.go b/phonedatatool/unpack/unpack.go index a0b88df..34b93cb 100644 --- a/phonedatatool/unpack/unpack.go +++ b/phonedatatool/unpack/unpack.go @@ -8,12 +8,6 @@ import ( "path" ) -const ( - VersionFileName = "version.txt" - RecordFileName = "record.txt" - IndexFileName = "index.txt" -) - type Unpacker struct { } @@ -26,9 +20,9 @@ func (u *Unpacker) Unpack(phoneDataFilePath string, plainDirectoryPath string) e return fmt.Errorf("target directory %v not exist and can't be created: %v", plainDirectoryPath, err) } - versionFilePath := path.Join(plainDirectoryPath, VersionFileName) - recordFilePath := path.Join(plainDirectoryPath, RecordFileName) - indexFilePath := path.Join(plainDirectoryPath, IndexFileName) + versionFilePath := path.Join(plainDirectoryPath, phonedatatool.VersionFileName) + recordFilePath := path.Join(plainDirectoryPath, phonedatatool.RecordFileName) + indexFilePath := path.Join(plainDirectoryPath, phonedatatool.IndexFileName) if err := u.assureAllFileNotExist(versionFilePath, recordFilePath, indexFilePath); err != nil { return err diff --git a/phonedatatool/util/file.go b/phonedatatool/util/file.go new file mode 100644 index 0000000..da2ff02 --- /dev/null +++ b/phonedatatool/util/file.go @@ -0,0 +1,27 @@ +package util + +import ( + "fmt" + "os" +) + +func AssureFileNotExist(path string) error { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil + } else { + return fmt.Errorf("check file existence %v failed: %v", path, err) + } + } else { + return fmt.Errorf("file %v already exists", path) + } +} + +func AssureAllFileNotExist(paths ...string) error { + for _, p := range paths { + if err := AssureFileNotExist(p); err != nil { + return err + } + } + return nil +} From 3953ac8c899b4bd70826e83284b897f4c008f010 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 13:44:02 +0800 Subject: [PATCH 14/29] version 0.1.0 --- README.phonedatatool.md | 82 +++++++++++++++++++++++++++++++++++++++ cmd/phonedatatool/main.go | 2 +- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 README.phonedatatool.md diff --git a/README.phonedatatool.md b/README.phonedatatool.md new file mode 100644 index 0000000..f601c87 --- /dev/null +++ b/README.phonedatatool.md @@ -0,0 +1,82 @@ +# README.phonedatatool.md + +工具 phonedatatool 用于对给定 phone.dat 的解包和打包。 + +## 1. 使用方法 + +- 第一步：将原始二进制文件解包成文本。 +- 第二步：根据需要手动修改文本。 +- 第三步：将修改后的文本打包成二进制文件。 + +## 2. 解包 + +```shell +phonedatatool.exe -unpack -i phone.dat -o abc +``` + +可以将二进制文件 phone.dat 解包到一个名为 abc 的目录。里面有三个文本文件。 + +## 3. 打包 + +```shell +phonedatatool.exe -pack -i abc -o phone.2.dat +``` + +可以将目录 abc 里的文本文件打包成二进制文件 phone.2.dat + +## 4. 解包后文件说明 + +解包后的目录下会产生 3 个文本文件，功能分别是： + +| 文件名 | 原始二进制文件 | +| ----------- | --------------------------------------------- | +| version.txt | 版本号 | +| record.txt | 记录区（省、市、邮编、区号） | +| index.txt | 索引区（号码前 7 位、记录区偏移量、号码类型） | + +### 4.1. version.txt + +里面应该是 4 个字符。比如 "2307"。 + +### 4.2. record.txt + +有多行，每行包括一条记录，例如`1|安徽|巢湖|238000|0551`。 + +由竖线分隔成 5 段，含义分别是： + +| 例子 | 含义 | +| ------ | --------- | +| 1 | 记录区 ID | +| 安徽 | 省名 | +| 巢湖 | 城市名 | +| 238000 | 邮政编码 | +| 0551 | 区号 | + +其中「记录区 ID」必须是整数，不一定要连续，但每行的「记录区 ID」必须不同。 + +### 4.3. index.txt + +有多行，每行包括一条索引，例如`1300000|251|2` + +由竖线分隔成 3 段，含义分别是： + +| 例子 | 含义 | +| ------- | ----------------------------------- | +| 1300000 | 手机号码前 7 位 | +| 251 | 记录区 ID（含义见 record.txt 章节） | +| 2 | 卡片类型码 | + +## 5. 备注 + +### 5.1. 卡片类型码 + +卡片类型码的含义由 phonedata 原项目规定。目前（2023-05-24）类型码的含义为： + +| 卡片类型码 | 卡片类型 | +| ---------- | -------------- | +| 1 | 中国移动 | +| 2 | 中国联通 | +| 3 | 中国电信 | +| 4 | 电信虚拟运营商 | +| 5 | 联通虚拟运营商 | +| 6 | 移动虚拟运营商 | diff --git a/cmd/phonedatatool/main.go b/cmd/phonedatatool/main.go index 86cd6f3..2c4880d 100644 --- a/cmd/phonedatatool/main.go +++ b/cmd/phonedatatool/main.go @@ -15,7 +15,7 @@ import ( const ( Name = "phonedatatool" - Version = "0.0.1" + Version = "0.1.0" Author = "seedjyh@gmail.com" FullName = Name + "_" + Version + "(" + Author + ")" ) From c48719f91c95940bc6b8bc992b40434b52e07705 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 15:06:43 +0800 Subject: [PATCH 15/29] =?UTF-8?q?=E9=87=8D=E6=9E=84=EF=BC=9A=E5=B0=86?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=93=8D=E4=BD=9C=E7=A7=BB=E5=8A=A8=E5=88=B0?= =?UTF-8?q?=20main=20=E9=87=8C=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/phonedatatool/main.go | 38 ++++++++++++++++++++++- phonedatatool/pack/pack.go | 55 +++++++++++----------------------- phonedatatool/phonedatatool.go | 4 +-- 3 files changed, 56 insertions(+), 41 deletions(-) diff --git a/cmd/phonedatatool/main.go b/cmd/phonedatatool/main.go index 2c4880d..e904a2d 100644 --- a/cmd/phonedatatool/main.go +++ b/cmd/phonedatatool/main.go @@ -3,9 +3,13 @@ package main import ( "flag" "fmt" + "github.com/xluohome/phonedata/phonedatatool" "github.com/xluohome/phonedata/phonedatatool/pack" "github.com/xluohome/phonedata/phonedatatool/query" "github.com/xluohome/phonedata/phonedatatool/unpack" + "github.com/xluohome/phonedata/phonedatatool/util" + "os" + "path" ) // 这里编译出来的可执行程序具备打包、查询、解包三个功能。 @@ -64,7 +68,7 @@ func main() { fmt.Println("ERROR! No destination") return } - if err := pack.NewPacker().Pack(*source, *destination); err != nil { + if err := Pack(*source, *destination); err != nil { fmt.Println("ERROR! Pack failed.", err) return } else { @@ -106,3 +110,35 @@ func showHelp() { fmt.Println("./phonedatatool -pack -i tmp -o phone.dat") fmt.Println("./phonedatatool -query -i phone.dat -n 13000001234") } + +func Pack(plainDirectoryPath string, phoneDataFilePath string) error { + if err := util.AssureFileNotExist(phoneDataFilePath); err != nil { + return err + } + var versionPlainTextBuf []byte + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.VersionFileName)); err != nil { + return err + } else { + versionPlainTextBuf = buf + } + + var recordPlainTextBuf []byte + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.RecordFileName)); err != nil { + return err + } else { + recordPlainTextBuf = buf + } + + var indexPlainTextBuf []byte + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.IndexFileName)); err != nil { + return err + } else { + indexPlainTextBuf = buf + } + + if buf, err := pack.NewPacker().Pack(versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf); err != nil { + return err + } else { + return os.WriteFile(phoneDataFilePath, buf, 0) + } +} diff --git a/phonedatatool/pack/pack.go b/phonedatatool/pack/pack.go index f1438ed..9232ef1 100644 --- a/phonedatatool/pack/pack.go +++ b/phonedatatool/pack/pack.go @@ -2,11 +2,7 @@ package pack import ( "bytes" - "fmt" "github.com/xluohome/phonedata/phonedatatool" - "github.com/xluohome/phonedata/phonedatatool/util" - "os" - "path" ) type Packer struct{} @@ -17,56 +13,39 @@ func NewPacker() phonedatatool.Packer { return new(Packer) } -func (p *Packer) Pack(plainDirectoryPath string, phoneDataFilePath string) error { - if err := util.AssureFileNotExist(phoneDataFilePath); err != nil { - return err - } - +func (p *Packer) Pack(versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte) ([]byte, error) { versionPart := new(VersionPart) - if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.VersionFileName)); err != nil { - return err - } else if err := versionPart.ParsePlainText(bytes.NewReader(buf)); err != nil { - return err + if err := versionPart.ParsePlainText(bytes.NewReader(versionPlainTextBuf)); err != nil { + return nil, err } versionPartBuf := versionPart.Bytes() recordPart := NewRecordPart() - if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.RecordFileName)); err != nil { - return err - } else if err := recordPart.ParsePlainText(bytes.NewReader(buf)); err != nil { - return err + if err := recordPart.ParsePlainText(bytes.NewReader(recordPlainTextBuf)); err != nil { + return nil, err } recordPartBuf, recordID2Offset := recordPart.Bytes(RecordPartBaseOffset) indexPartOffsetPart := RecordPartBaseOffset + Offset(len(recordPartBuf)) indexPart := NewIndexPart() - if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.IndexFileName)); err != nil { - return err - } else if err := indexPart.ParsePlainText(bytes.NewReader(buf), recordID2Offset); err != nil { - return err + if err := indexPart.ParsePlainText(bytes.NewReader(indexPlainTextBuf), recordID2Offset); err != nil { + return nil, err } indexPartBuf := indexPart.Bytes() - var outFile *os.File - if f, err := os.OpenFile(phoneDataFilePath, os.O_CREATE|os.O_WRONLY, 0); err != nil { - return fmt.Errorf("failed to open file %v to write, %v", phoneDataFilePath, err) - } else { - outFile = f - } - defer outFile.Close() - - if _, err := outFile.Write(versionPartBuf); err != nil { - return err + w := bytes.NewBuffer(nil) + if _, err := w.Write(versionPartBuf); err != nil { + return nil, err } - if _, err := outFile.Write(indexPartOffsetPart.Bytes()); err != nil { - return err + if _, err := w.Write(indexPartOffsetPart.Bytes()); err != nil { + return nil, err } - if _, err := outFile.Write(recordPartBuf); err != nil { - return err + if _, err := w.Write(recordPartBuf); err != nil { + return nil, err } - if _, err := outFile.Write(indexPartBuf); err != nil { - return err + if _, err := w.Write(indexPartBuf); err != nil { + return nil, err } - return nil + return w.Bytes(), nil } diff --git a/phonedatatool/phonedatatool.go b/phonedatatool/phonedatatool.go index 24b1dd8..eec064e 100644 --- a/phonedatatool/phonedatatool.go +++ b/phonedatatool/phonedatatool.go @@ -6,8 +6,8 @@ type Unpacker interface { } type Packer interface { - // Pack 将明文目录 plainDirectoryPath 的数据打包成压缩文件 phoneDataFilePath。 - Pack(plainDirectoryPath string, phoneDataFilePath string) error + // Pack 将版本文件、记录文件、索引文件的内容打包成二进制文件。 + Pack(versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte) ([]byte, error) } type QueryResult struct { From 6618459bc62308d33e2039ce4024a600c12bb642 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 15:19:30 +0800 Subject: [PATCH 16/29] =?UTF-8?q?=E5=B0=86=20unpack=20=E9=83=A8=E5=88=86?= =?UTF-8?q?=E7=9A=84=E6=96=87=E4=BB=B6=E6=93=8D=E4=BD=9C=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E5=88=B0=20main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/phonedatatool/main.go | 38 ++++++++++++++- phonedatatool/phonedatatool.go | 6 +-- phonedatatool/unpack/unpack.go | 88 +++++----------------------------- 3 files changed, 51 insertions(+), 81 deletions(-) diff --git a/cmd/phonedatatool/main.go b/cmd/phonedatatool/main.go index e904a2d..3e63e3a 100644 --- a/cmd/phonedatatool/main.go +++ b/cmd/phonedatatool/main.go @@ -51,7 +51,7 @@ func main() { fmt.Println("ERROR! No destination") return } - if err := unpack.NewUnpacker().Unpack(*source, *destination); err != nil { + if err := Unpack(*source, *destination); err != nil { fmt.Println("ERROR! Unpack failed.", err) return } else { @@ -142,3 +142,39 @@ func Pack(plainDirectoryPath string, phoneDataFilePath string) error { return os.WriteFile(phoneDataFilePath, buf, 0) } } + +func Unpack(phoneDataFilePath string, plainDirectoryPath string) error { + if err := os.MkdirAll(plainDirectoryPath, 0); err != nil { + return fmt.Errorf("target directory %v not exist and can't be created: %v", plainDirectoryPath, err) + } + + versionFilePath := path.Join(plainDirectoryPath, phonedatatool.VersionFileName) + recordFilePath := path.Join(plainDirectoryPath, phonedatatool.RecordFileName) + indexFilePath := path.Join(plainDirectoryPath, phonedatatool.IndexFileName) + + if err := util.AssureAllFileNotExist(versionFilePath, recordFilePath, indexFilePath); err != nil { + return err + } + + var rawBuf []byte + if b, err := os.ReadFile(phoneDataFilePath); err != nil { + return err + } else { + rawBuf = b + } + + if versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf, err := unpack.NewUnpacker().Unpack(rawBuf); err != nil { + return err + } else { + if err := os.WriteFile(versionFilePath, versionPlainTextBuf, 0); err != nil { + return err + } + if err := os.WriteFile(recordFilePath, recordPlainTextBuf, 0); err != nil { + return err + } + if err := os.WriteFile(indexFilePath, indexPlainTextBuf, 0); err != nil { + return err + } + return nil + } +} diff --git a/phonedatatool/phonedatatool.go b/phonedatatool/phonedatatool.go index eec064e..3e3b07a 100644 --- a/phonedatatool/phonedatatool.go +++ b/phonedatatool/phonedatatool.go @@ -1,12 +1,12 @@ package phonedatatool type Unpacker interface { - // Unpack 将压缩文件 phoneDataFilePath 解包后保存到明文目录 plainDirectoryPath 里。 - Unpack(phoneDataFilePath string, plainDirectoryPath string) error + // Unpack 将二进制文件的内容解包成版本文件、记录文件、索引文件的内容。 + Unpack(phoneDataBuf []byte) (versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte, err error) } type Packer interface { - // Pack 将版本文件、记录文件、索引文件的内容打包成二进制文件。 + // Pack 将版本文件、记录文件、索引文件的内容打包成二进制文件的内容。 Pack(versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte) ([]byte, error) } diff --git a/phonedatatool/unpack/unpack.go b/phonedatatool/unpack/unpack.go index 34b93cb..ffac70c 100644 --- a/phonedatatool/unpack/unpack.go +++ b/phonedatatool/unpack/unpack.go @@ -4,8 +4,6 @@ import ( "bytes" "fmt" "github.com/xluohome/phonedata/phonedatatool" - "os" - "path" ) type Unpacker struct { @@ -15,94 +13,30 @@ func NewUnpacker() phonedatatool.Unpacker { return new(Unpacker) } -func (u *Unpacker) Unpack(phoneDataFilePath string, plainDirectoryPath string) error { - if err := os.MkdirAll(plainDirectoryPath, 0); err != nil { - return fmt.Errorf("target directory %v not exist and can't be created: %v", plainDirectoryPath, err) - } - - versionFilePath := path.Join(plainDirectoryPath, phonedatatool.VersionFileName) - recordFilePath := path.Join(plainDirectoryPath, phonedatatool.RecordFileName) - indexFilePath := path.Join(plainDirectoryPath, phonedatatool.IndexFileName) - - if err := u.assureAllFileNotExist(versionFilePath, recordFilePath, indexFilePath); err != nil { - return err - } - - var rawBuf []byte - if b, err := os.ReadFile(phoneDataFilePath); err != nil { - return err - } else { - rawBuf = b - } - - if res, err := u.parse(rawBuf); err != nil { - return fmt.Errorf("failed to parse raw file data: %v", err) - } else { - if err := os.WriteFile(versionFilePath, res.versionPart.Bytes(), 0); err != nil { - return err - } - if err := os.WriteFile(recordFilePath, res.recordPart.Bytes(), 0); err != nil { - return err - } - if err := os.WriteFile(indexFilePath, res.indexPart.Bytes(), 0); err != nil { - return err - } - return nil - } -} - -func (u *Unpacker) assureFileNotExist(path string) error { - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - return nil - } else { - return fmt.Errorf("check file existence %v failed: %v", path, err) - } - } else { - return fmt.Errorf("file %v already exists", path) - } -} - -func (u *Unpacker) assureAllFileNotExist(paths ...string) error { - for _, p := range paths { - if err := u.assureFileNotExist(p); err != nil { - return err - } - } - return nil -} - -type ParseResult struct { - versionPart *VersionPart - recordPart *RecordPart - indexPart *IndexPart -} - -func (u *Unpacker) parse(buf []byte) (*ParseResult, error) { - reader := bytes.NewReader(buf) +func (u *Unpacker) Unpack(phoneDataBuf []byte) (versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte, err error) { + reader := bytes.NewReader(phoneDataBuf) versionPart := new(VersionPart) if err := versionPart.Parse(reader); err != nil { - return nil, fmt.Errorf("failed to read version part: %v", err) + return nil, nil, nil, fmt.Errorf("failed to read version part: %v", err) } offsetPart := new(IndexPartOffsetPart) if err := offsetPart.Parse(reader); err != nil { - return nil, fmt.Errorf("failed to read index-part-offset part: %v", err) + return nil, nil, nil, fmt.Errorf("failed to read index-part-offset part: %v", err) } recordPart := NewRecordPart() if err := recordPart.Parse(reader, offsetPart.IndexPartOffset); err != nil { - return nil, fmt.Errorf("failed to read record part: %v", err) + return nil, nil, nil, fmt.Errorf("failed to read record part: %v", err) } indexPart := NewIndexPart() if err := indexPart.Parse(reader); err != nil { - return nil, fmt.Errorf("failed to read index part: %v", err) + return nil, nil, nil, fmt.Errorf("failed to read index part: %v", err) } if err := indexPart.MatchRecordOffsetToRecordID(recordPart.offset2id); err != nil { - return nil, fmt.Errorf("failed to match offset to record id: %v", err) + return nil, nil, nil, fmt.Errorf("failed to match offset to record id: %v", err) } - return &ParseResult{ - versionPart: versionPart, - recordPart: recordPart, - indexPart: indexPart, - }, nil + versionPlainTextBuf = versionPart.Bytes() + recordPlainTextBuf = recordPart.Bytes() + indexPlainTextBuf = indexPart.Bytes() + return versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf, nil } From 4a101055589fa065e42c9c44b38e4e7208b02178 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 15:31:00 +0800 Subject: [PATCH 17/29] =?UTF-8?q?=E5=BC=80=E5=8F=91=E6=96=B0=E7=9A=84=20ve?= =?UTF-8?q?rsion=20part=20parse=20=E5=92=8C=20bytesPlainText?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/version.go | 16 ++++++++++++++++ phonedatatool/pack/version_test.go | 10 ++++++++++ 2 files changed, 26 insertions(+) diff --git a/phonedatatool/pack/version.go b/phonedatatool/pack/version.go index 504c887..862041a 100644 --- a/phonedatatool/pack/version.go +++ b/phonedatatool/pack/version.go @@ -30,3 +30,19 @@ func (p *VersionPart) Bytes() []byte { } return w.Bytes() } + +func (p *VersionPart) Parse(reader *bytes.Reader) error { + buf := make([]byte, 4) + if _, err := reader.Read(buf); err != nil { + return err + } + p.version = string(buf) + return nil +} + +func (p *VersionPart) BytesPlainText() []byte { + w := bytes.NewBuffer(nil) + w.Write([]byte(p.version)) + w.WriteByte('\n') + return w.Bytes() +} diff --git a/phonedatatool/pack/version_test.go b/phonedatatool/pack/version_test.go index 32d3546..dd6f115 100644 --- a/phonedatatool/pack/version_test.go +++ b/phonedatatool/pack/version_test.go @@ -16,3 +16,13 @@ func TestVersionPart_ParsePlainText(t *testing.T) { func TestVersionPart_Bytes(t *testing.T) { assert.Equal(t, []byte{'2', '3', '0', '6'}, (&VersionPart{version: "2306"}).Bytes()) } + +func TestVersionPart_Parse(t *testing.T) { + versionPart := new(VersionPart) + assert.NoError(t, versionPart.Parse(bytes.NewReader([]byte{'2', '3', '0', '6'}))) + assert.Equal(t, "2306", versionPart.version) +} + +func TestVersionPart_BytesPlainText(t *testing.T) { + assert.Equal(t, []byte("2306\n"), (&VersionPart{version: "2306"}).BytesPlainText()) +} From 54a70b66b5865603e2cbb73cc4c882ab11bb55f4 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 15:34:29 +0800 Subject: [PATCH 18/29] =?UTF-8?q?=E5=BC=80=E5=8F=91=E6=96=B0=E7=9A=84=20of?= =?UTF-8?q?fset=20parse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/offset.go | 14 +++++++++++++- phonedatatool/pack/offset_test.go | 11 ++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/phonedatatool/pack/offset.go b/phonedatatool/pack/offset.go index e2d85fb..4006dd4 100644 --- a/phonedatatool/pack/offset.go +++ b/phonedatatool/pack/offset.go @@ -1,9 +1,21 @@ package pack -import "encoding/binary" +import ( + "bytes" + "encoding/binary" + "fmt" +) type Offset int64 func (o Offset) Bytes() []byte { return binary.LittleEndian.AppendUint32(nil, uint32(o)) } +func (o *Offset) Parse(reader *bytes.Reader) error { + buf := make([]byte, 4) + if _, err := reader.Read(buf); err != nil { + return fmt.Errorf("failed to read: %v", err) + } + *o = Offset(binary.LittleEndian.Uint32(buf)) + return nil +} diff --git a/phonedatatool/pack/offset_test.go b/phonedatatool/pack/offset_test.go index 4cbb83c..8bf34a4 100644 --- a/phonedatatool/pack/offset_test.go +++ b/phonedatatool/pack/offset_test.go @@ -1,8 +1,17 @@ package pack -import "testing" +import ( + "bytes" + "testing" +) import "github.com/stretchr/testify/assert" func TestOffset_Bytes(t *testing.T) { assert.Equal(t, []byte{0xd2, 0x04, 0x00, 0x00}, Offset(1234).Bytes()) } + +func TestOffset_Parse(t *testing.T) { + offset := new(Offset) + assert.NoError(t, offset.Parse(bytes.NewReader([]byte{0xd2, 0x04, 0x00, 0x00}))) + assert.Equal(t, Offset(1234), *offset) +} From b38427f3f0f03a5e002a5d6471f566b81e800853 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 15:55:14 +0800 Subject: [PATCH 19/29] =?UTF-8?q?=E5=BC=80=E5=8F=91=E6=96=B0=E7=9A=84=20re?= =?UTF-8?q?cord=20Parse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/record.go | 39 +++++++++++++++++++++++++++++++ phonedatatool/pack/record_test.go | 25 ++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/phonedatatool/pack/record.go b/phonedatatool/pack/record.go index 0e108ea..06edfcf 100644 --- a/phonedatatool/pack/record.go +++ b/phonedatatool/pack/record.go @@ -37,6 +37,23 @@ func (ri *RecordItem) Bytes() []byte { return w.Bytes() } +// Parse 从压缩文件读取一条 RecordItem。注意 reader 必须以 '\0' 结尾。 +func (ri *RecordItem) Parse(reader *bytes.Reader) error { + if buf, err := util.ReadUntil(reader, 0); err != nil { + return fmt.Errorf("no term char for record item: %v", err) + } else { + words := strings.Split(string(buf), "|") + if len(words) != 4 { + return fmt.Errorf("invalid item bytes, %v", string(buf)) + } + ri.province = words[0] + ri.city = words[1] + ri.zipCode = words[2] + ri.areaCode = words[3] + return nil + } +} + type RecordPart struct { id2item map[RecordID]*RecordItem } @@ -95,3 +112,25 @@ func (p *RecordPart) Bytes(baseOffset Offset) ([]byte, map[RecordID]Offset) { } return w.Bytes(), id2offset } + +func (p *RecordPart) Parse(reader *bytes.Reader, baseOffset Offset) (map[Offset]RecordID, error) { + offset2id := make(map[Offset]RecordID) + offset := baseOffset + for id := RecordID(1); reader.Len() > 0; id++ { + var itemBuf []byte + if buf, err := util.ReadUntil(reader, 0); err != nil { + return nil, err + } else { + itemBuf = buf + itemBuf = append(itemBuf, 0) + } + item := new(RecordItem) + if err := item.Parse(bytes.NewReader(itemBuf)); err != nil { + return nil, err + } + offset2id[offset] = id + p.id2item[id] = item + offset += Offset(len(itemBuf)) + } + return offset2id, nil +} diff --git a/phonedatatool/pack/record_test.go b/phonedatatool/pack/record_test.go index 1e68322..9f07027 100644 --- a/phonedatatool/pack/record_test.go +++ b/phonedatatool/pack/record_test.go @@ -49,3 +49,28 @@ func TestRecordPart_Bytes(t *testing.T) { 2: 34, }, id2offset) } + +func TestRecordPart_Parse(t *testing.T) { + buf := []byte("\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\xB7\xA2\xE6\xB9\x96\x7C\x32\x33\x38\x30\x30\x30\x7C\x30\x35\x35\x31\x00\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\x90\x88\xE8\x82\xA5\x7C\x32\x33\x30\x30\x30\x30\x7C\x30\x35\x35\x31\x00") + recordPart := NewRecordPart() + offset2id, err := recordPart.Parse(bytes.NewReader(buf), 8) + assert.NoError(t, err) + assert.Equal(t, map[Offset]RecordID{ + 8: 1, + 34: 2, + }, offset2id) + assert.Equal(t, map[RecordID]*RecordItem{ + 1: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\xB7\xA2\xE6\xB9\x96", + zipCode: "238000", + areaCode: "0551", + }, + 2: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\x90\x88\xE8\x82\xA5", + zipCode: "230000", + areaCode: "0551", + }, + }, recordPart.id2item) +} From 519fb8573de53625592a3181462369252e5af1b2 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 16:02:22 +0800 Subject: [PATCH 20/29] =?UTF-8?q?=E5=BC=80=E5=8F=91=E6=96=B0=E7=9A=84=20re?= =?UTF-8?q?cord=20part=20BytesPlainText?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/record.go | 26 ++++++++++++++++++++++++++ phonedatatool/pack/record_test.go | 21 +++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/phonedatatool/pack/record.go b/phonedatatool/pack/record.go index 06edfcf..e0506bc 100644 --- a/phonedatatool/pack/record.go +++ b/phonedatatool/pack/record.go @@ -11,6 +11,10 @@ import ( type RecordID int64 +func (rid RecordID) String() string { + return strconv.FormatInt(int64(rid), 10) +} + type RecordIDList []RecordID func (idl RecordIDList) Len() int { @@ -134,3 +138,25 @@ func (p *RecordPart) Parse(reader *bytes.Reader, baseOffset Offset) (map[Offset] } return offset2id, nil } + +func (p *RecordPart) BytesPlainText() []byte { + var idList RecordIDList + for k := range p.id2item { + idList = append(idList, k) + } + sort.Sort(idList) + + w := bytes.NewBuffer(nil) + for _, id := range idList { + item := p.id2item[id] + w.WriteString(strings.Join([]string{ + id.String(), + item.province, + item.city, + item.zipCode, + item.areaCode, + }, "|")) + w.WriteByte('\n') + } + return w.Bytes() +} diff --git a/phonedatatool/pack/record_test.go b/phonedatatool/pack/record_test.go index 9f07027..6bab7b0 100644 --- a/phonedatatool/pack/record_test.go +++ b/phonedatatool/pack/record_test.go @@ -74,3 +74,24 @@ func TestRecordPart_Parse(t *testing.T) { }, }, recordPart.id2item) } + +func TestRecordPart_BytesPlainText(t *testing.T) { + plainText := []byte("\x31\x7C\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\xB7\xA2\xE6\xB9\x96\x7C\x32\x33\x38\x30\x30\x30\x7C\x30\x35\x35\x31\x0A\x32\x7C\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\x90\x88\xE8\x82\xA5\x7C\x32\x33\x30\x30\x30\x30\x7C\x30\x35\x35\x31\x0A") + recordPart := &RecordPart{ + id2item: map[RecordID]*RecordItem{ + 1: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\xB7\xA2\xE6\xB9\x96", + zipCode: "238000", + areaCode: "0551", + }, + 2: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\x90\x88\xE8\x82\xA5", + zipCode: "230000", + areaCode: "0551", + }, + }, + } + assert.Equal(t, plainText, recordPart.BytesPlainText()) +} From a6396735314dc8f3f1b8c6c88f5064ef5ebb675a Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 16:10:37 +0800 Subject: [PATCH 21/29] =?UTF-8?q?=E5=BC=80=E5=8F=91=E6=96=B0=E7=9A=84=20in?= =?UTF-8?q?dex=20part=20parse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/index.go | 21 +++++++++++++++++++++ phonedatatool/pack/index_test.go | 22 ++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/phonedatatool/pack/index.go b/phonedatatool/pack/index.go index d5500a8..efc7f73 100644 --- a/phonedatatool/pack/index.go +++ b/phonedatatool/pack/index.go @@ -42,6 +42,16 @@ func (ii IndexItem) Bytes() []byte { w.Write(ii.cardTypeID.Bytes()) return w.Bytes() } +func (ii *IndexItem) Parse(reader *bytes.Reader) error { + buf := make([]byte, 9) + if _, err := reader.Read(buf); err != nil { + return err + } + ii.numberPrefix = NumberPrefix(binary.LittleEndian.Uint32(buf[:4])) + ii.recordOffset = Offset(binary.LittleEndian.Uint32(buf[4:8])) + ii.cardTypeID = phonedatatool.CardTypeID(buf[8]) + return nil +} type IndexPart struct { prefix2item map[NumberPrefix]*IndexItem @@ -111,3 +121,14 @@ func (p *IndexPart) Bytes() []byte { } return w.Bytes() } + +func (p *IndexPart) Parse(reader *bytes.Reader) error { + for reader.Len() > 0 { + item := new(IndexItem) + if err := item.Parse(reader); err != nil { + return err + } + p.prefix2item[item.numberPrefix] = item + } + return nil +} diff --git a/phonedatatool/pack/index_test.go b/phonedatatool/pack/index_test.go index 0511434..189c429 100644 --- a/phonedatatool/pack/index_test.go +++ b/phonedatatool/pack/index_test.go @@ -53,5 +53,27 @@ func TestIndexPart_Bytes(t *testing.T) { }, }} assert.Equal(t, []byte("\x20\xD6\x13\x00\x4E\x1A\x00\x00\x02\x21\xD6\x13\x00\x2C\x12\x00\x00\x02\x22\xD6\x13\x00\x08\x00\x00\x00\x02"), indexPart.Bytes()) +} +func TestIndexPart_Parse(t *testing.T) { + buf := []byte("\x20\xD6\x13\x00\x4E\x1A\x00\x00\x02\x21\xD6\x13\x00\x2C\x12\x00\x00\x02\x22\xD6\x13\x00\x08\x00\x00\x00\x02") + indexPart := NewIndexPart() + assert.NoError(t, indexPart.Parse(bytes.NewReader(buf))) + assert.Equal(t, map[NumberPrefix]*IndexItem{ + 1300000: { + numberPrefix: 1300000, + recordOffset: 0x1A4E, + cardTypeID: 2, + }, + 1300001: { + numberPrefix: 1300001, + recordOffset: 0x122C, + cardTypeID: 2, + }, + 1300002: { + numberPrefix: 1300002, + recordOffset: 0x0008, + cardTypeID: 2, + }, + }, indexPart.prefix2item) } From a2cd8b1b0dc88c97cc295072fba9401d89d64938 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 16:17:57 +0800 Subject: [PATCH 22/29] =?UTF-8?q?=E5=BC=80=E5=8F=91=E6=96=B0=E7=9A=84=20in?= =?UTF-8?q?dex=20part=20bytesPlainText?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/index.go | 22 ++++++++++++++++++++++ phonedatatool/pack/index_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/phonedatatool/pack/index.go b/phonedatatool/pack/index.go index efc7f73..748a7de 100644 --- a/phonedatatool/pack/index.go +++ b/phonedatatool/pack/index.go @@ -16,6 +16,9 @@ type NumberPrefix int32 func (p NumberPrefix) Bytes() []byte { return binary.LittleEndian.AppendUint32(nil, uint32(p)) } +func (p NumberPrefix) String() string { + return strconv.Itoa(int(p)) +} type NumberPrefixList []NumberPrefix @@ -132,3 +135,22 @@ func (p *IndexPart) Parse(reader *bytes.Reader) error { } return nil } + +func (p *IndexPart) BytesPlainText(offset2id map[Offset]RecordID) []byte { + w := bytes.NewBuffer(nil) + var prefixList NumberPrefixList + for k, _ := range p.prefix2item { + prefixList = append(prefixList, k) + } + sort.Sort(prefixList) + for _, prefix := range prefixList { + item := p.prefix2item[prefix] + w.WriteString(strings.Join([]string{ + prefix.String(), + offset2id[item.recordOffset].String(), + item.cardTypeID.String(), + }, "|")) + w.WriteByte('\n') + } + return w.Bytes() +} diff --git a/phonedatatool/pack/index_test.go b/phonedatatool/pack/index_test.go index 189c429..74e6b50 100644 --- a/phonedatatool/pack/index_test.go +++ b/phonedatatool/pack/index_test.go @@ -77,3 +77,30 @@ func TestIndexPart_Parse(t *testing.T) { }, }, indexPart.prefix2item) } + +func TestIndexPart_BytesPlainText(t *testing.T) { + plainTextBuf := []byte("1300000|251|2\n1300001|176|2\n1300002|1|2\n") + indexPart := &IndexPart{prefix2item: map[NumberPrefix]*IndexItem{ + 1300000: { + numberPrefix: 1300000, + recordOffset: 300, + cardTypeID: 2, + }, + 1300001: { + numberPrefix: 1300001, + recordOffset: 200, + cardTypeID: 2, + }, + 1300002: { + numberPrefix: 1300002, + recordOffset: 100, + cardTypeID: 2, + }, + }} + offset2id := map[Offset]RecordID{ + 100: 1, + 200: 176, + 300: 251, + } + assert.Equal(t, plainTextBuf, indexPart.BytesPlainText(offset2id)) +} From 49d25489022edd5fde3877080526bbcbf8a7ed12 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 16:29:56 +0800 Subject: [PATCH 23/29] =?UTF-8?q?=E5=BC=80=E5=8F=91=E6=96=B0=E7=9A=84=20pa?= =?UTF-8?q?ck.unpack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/phonedatatool/main.go | 3 +-- phonedatatool/pack/unpack.go | 50 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 phonedatatool/pack/unpack.go diff --git a/cmd/phonedatatool/main.go b/cmd/phonedatatool/main.go index 3e63e3a..0083ba5 100644 --- a/cmd/phonedatatool/main.go +++ b/cmd/phonedatatool/main.go @@ -6,7 +6,6 @@ import ( "github.com/xluohome/phonedata/phonedatatool" "github.com/xluohome/phonedata/phonedatatool/pack" "github.com/xluohome/phonedata/phonedatatool/query" - "github.com/xluohome/phonedata/phonedatatool/unpack" "github.com/xluohome/phonedata/phonedatatool/util" "os" "path" @@ -163,7 +162,7 @@ func Unpack(phoneDataFilePath string, plainDirectoryPath string) error { rawBuf = b } - if versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf, err := unpack.NewUnpacker().Unpack(rawBuf); err != nil { + if versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf, err := pack.NewUnpacker().Unpack(rawBuf); err != nil { return err } else { if err := os.WriteFile(versionFilePath, versionPlainTextBuf, 0); err != nil { diff --git a/phonedatatool/pack/unpack.go b/phonedatatool/pack/unpack.go new file mode 100644 index 0000000..c8a68be --- /dev/null +++ b/phonedatatool/pack/unpack.go @@ -0,0 +1,50 @@ +package pack + +import ( + "bytes" + "github.com/xluohome/phonedata/phonedatatool" +) + +type Unpacker struct { +} + +func NewUnpacker() phonedatatool.Unpacker { + return new(Unpacker) +} + +func (u *Unpacker) Unpack(phoneDataBuf []byte) (versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte, err error) { + reader := bytes.NewReader(phoneDataBuf) + versionPart := new(VersionPart) + if err := versionPart.Parse(reader); err != nil { + return nil, nil, nil, err + } + versionPlainTextBuf = versionPart.BytesPlainText() + + var indexPartOffset Offset + if err := indexPartOffset.Parse(reader); err != nil { + return nil, nil, nil, err + } + + recordBuf := make([]byte, indexPartOffset-RecordPartBaseOffset) + if _, err := reader.Read(recordBuf); err != nil { + return nil, nil, nil, err + } + + recordPart := NewRecordPart() + offset2id := make(map[Offset]RecordID) + if o2i, err := recordPart.Parse(bytes.NewReader(recordBuf), RecordPartBaseOffset); err != nil { + return nil, nil, nil, err + } else { + offset2id = o2i + } + recordPlainTextBuf = recordPart.BytesPlainText() + + indexPart := NewIndexPart() + if err := indexPart.Parse(reader); err != nil { + return nil, nil, nil, err + } + indexPart.BytesPlainText(offset2id) + indexPlainTextBuf = indexPart.BytesPlainText(offset2id) + + return versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf, nil +} From 9163b55ce8dadfbb0371455ccc0f35034152bdfd Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 16:32:37 +0800 Subject: [PATCH 24/29] =?UTF-8?q?plaintext=20=E6=96=87=E4=BB=B6=E5=90=8D?= =?UTF-8?q?=E6=94=BE=E5=9C=A8=20main=20=E9=87=8C=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/phonedatatool/main.go | 19 ++++++++++++------- phonedatatool/phonedatatool.go | 6 ------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/phonedatatool/main.go b/cmd/phonedatatool/main.go index 0083ba5..4a0e746 100644 --- a/cmd/phonedatatool/main.go +++ b/cmd/phonedatatool/main.go @@ -3,7 +3,6 @@ package main import ( "flag" "fmt" - "github.com/xluohome/phonedata/phonedatatool" "github.com/xluohome/phonedata/phonedatatool/pack" "github.com/xluohome/phonedata/phonedatatool/query" "github.com/xluohome/phonedata/phonedatatool/util" @@ -23,6 +22,12 @@ const ( FullName = Name + "_" + Version + "(" + Author + ")" ) +const ( + VersionFileName = "version.txt" + RecordFileName = "record.txt" + IndexFileName = "index.txt" +) + func main() { showVersionFlag := flag.Bool("v", false, "Just show version string") showHelpFlag := flag.Bool("help", false, "Just print help.") @@ -115,21 +120,21 @@ func Pack(plainDirectoryPath string, phoneDataFilePath string) error { return err } var versionPlainTextBuf []byte - if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.VersionFileName)); err != nil { + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, VersionFileName)); err != nil { return err } else { versionPlainTextBuf = buf } var recordPlainTextBuf []byte - if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.RecordFileName)); err != nil { + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, RecordFileName)); err != nil { return err } else { recordPlainTextBuf = buf } var indexPlainTextBuf []byte - if buf, err := os.ReadFile(path.Join(plainDirectoryPath, phonedatatool.IndexFileName)); err != nil { + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, IndexFileName)); err != nil { return err } else { indexPlainTextBuf = buf @@ -147,9 +152,9 @@ func Unpack(phoneDataFilePath string, plainDirectoryPath string) error { return fmt.Errorf("target directory %v not exist and can't be created: %v", plainDirectoryPath, err) } - versionFilePath := path.Join(plainDirectoryPath, phonedatatool.VersionFileName) - recordFilePath := path.Join(plainDirectoryPath, phonedatatool.RecordFileName) - indexFilePath := path.Join(plainDirectoryPath, phonedatatool.IndexFileName) + versionFilePath := path.Join(plainDirectoryPath, VersionFileName) + recordFilePath := path.Join(plainDirectoryPath, RecordFileName) + indexFilePath := path.Join(plainDirectoryPath, IndexFileName) if err := util.AssureAllFileNotExist(versionFilePath, recordFilePath, indexFilePath); err != nil { return err diff --git a/phonedatatool/phonedatatool.go b/phonedatatool/phonedatatool.go index 3e3b07a..b0e1a15 100644 --- a/phonedatatool/phonedatatool.go +++ b/phonedatatool/phonedatatool.go @@ -22,9 +22,3 @@ type QueryResult struct { type Querier interface { QueryNumber(number string) (*QueryResult, error) } - -const ( - VersionFileName = "version.txt" - RecordFileName = "record.txt" - IndexFileName = "index.txt" -) From 6be29d42ac7250349bbfcab447b5df7d3b818e00 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 16:33:27 +0800 Subject: [PATCH 25/29] =?UTF-8?q?=E5=BD=BB=E5=BA=95=E5=88=A0=E9=99=A4=20un?= =?UTF-8?q?pack=20=E5=8C=85=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/unpack/index.go | 98 ---------------------------- phonedatatool/unpack/offset.go | 20 ------ phonedatatool/unpack/record.go | 110 -------------------------------- phonedatatool/unpack/unpack.go | 42 ------------ phonedatatool/unpack/version.go | 32 ---------- 5 files changed, 302 deletions(-) delete mode 100644 phonedatatool/unpack/index.go delete mode 100644 phonedatatool/unpack/offset.go delete mode 100644 phonedatatool/unpack/record.go delete mode 100644 phonedatatool/unpack/unpack.go delete mode 100644 phonedatatool/unpack/version.go diff --git a/phonedatatool/unpack/index.go b/phonedatatool/unpack/index.go deleted file mode 100644 index 2cc43eb..0000000 --- a/phonedatatool/unpack/index.go +++ /dev/null @@ -1,98 +0,0 @@ -package unpack - -import ( - "bytes" - "encoding/binary" - "fmt" - "github.com/xluohome/phonedata/phonedatatool" - "sort" - "strconv" - "strings" -) - -type PhoneNumberPrefix string - -func (pnp PhoneNumberPrefix) String() string { - return string(pnp) -} - -type PhoneNumberPrefixList []PhoneNumberPrefix - -func (pl PhoneNumberPrefixList) Len() int { - return len(pl) -} -func (pl PhoneNumberPrefixList) Less(i, j int) bool { - return pl[i] < pl[j] -} -func (pl PhoneNumberPrefixList) Swap(i, j int) { - pl[i], pl[j] = pl[j], pl[i] -} - -type IndexItem struct { - NumberPrefix PhoneNumberPrefix - CardTypeID phonedatatool.CardTypeID - RecordOffset RecordOffset - RecordID RecordID -} - -func (ii *IndexItem) Parse(reader *bytes.Reader) error { - buf := make([]byte, 9) - if _, err := reader.Read(buf); err != nil { - return err - } - ii.NumberPrefix = PhoneNumberPrefix(strconv.Itoa(int(binary.LittleEndian.Uint32(buf[:4])))) - ii.RecordOffset = RecordOffset(binary.LittleEndian.Uint32(buf[4:8])) - ii.CardTypeID = phonedatatool.CardTypeID(buf[8]) - return nil -} - -type IndexPart struct { - prefix2item map[PhoneNumberPrefix]*IndexItem -} - -func NewIndexPart() *IndexPart { - return &IndexPart{ - prefix2item: make(map[PhoneNumberPrefix]*IndexItem), - } -} - -func (ip *IndexPart) Bytes() []byte { - w := bytes.NewBuffer(nil) - var prefixList PhoneNumberPrefixList - for k, _ := range ip.prefix2item { - prefixList = append(prefixList, k) - } - sort.Sort(prefixList) - for _, prefix := range prefixList { - item := ip.prefix2item[prefix] - w.WriteString(strings.Join([]string{ - prefix.String(), - item.RecordID.String(), - item.CardTypeID.String(), - }, "|")) - w.WriteByte('\n') - } - return w.Bytes() -} - -func (ip *IndexPart) Parse(reader *bytes.Reader) error { - for reader.Len() > 0 { - item := new(IndexItem) - if err := item.Parse(reader); err != nil { - return err - } - ip.prefix2item[item.NumberPrefix] = item - } - return nil -} - -func (ip *IndexPart) MatchRecordOffsetToRecordID(offset2id map[RecordOffset]RecordID) error { - for _, v := range ip.prefix2item { - if id, ok := offset2id[v.RecordOffset]; ok { - v.RecordID = id - } else { - return fmt.Errorf("failed to find record id for record offset %v", v.RecordOffset) - } - } - return nil -} diff --git a/phonedatatool/unpack/offset.go b/phonedatatool/unpack/offset.go deleted file mode 100644 index b4867a0..0000000 --- a/phonedatatool/unpack/offset.go +++ /dev/null @@ -1,20 +0,0 @@ -package unpack - -import ( - "bytes" - "encoding/binary" - "fmt" -) - -type IndexPartOffsetPart struct { - IndexPartOffset RecordOffset -} - -func (op *IndexPartOffsetPart) Parse(reader *bytes.Reader) error { - buf := make([]byte, 4) - if _, err := reader.Read(buf); err != nil { - return fmt.Errorf("failed to read: %v", err) - } - op.IndexPartOffset = RecordOffset(binary.LittleEndian.Uint32(buf)) - return nil -} diff --git a/phonedatatool/unpack/record.go b/phonedatatool/unpack/record.go deleted file mode 100644 index 1175c79..0000000 --- a/phonedatatool/unpack/record.go +++ /dev/null @@ -1,110 +0,0 @@ -package unpack - -import ( - "bytes" - "fmt" - "github.com/xluohome/phonedata/phonedatatool" - "github.com/xluohome/phonedata/phonedatatool/util" - "sort" - "strconv" - "strings" -) - -type RecordItem struct { - Province phonedatatool.ProvinceName - City phonedatatool.CityName - ZipCode phonedatatool.ZipCode - AreaCode phonedatatool.AreaCode -} - -func (ri *RecordItem) Parse(reader *bytes.Reader) error { - if buf, err := util.ReadUntil(reader, 0); err != nil { - return fmt.Errorf("no term char for record item: %v", err) - } else { - words := strings.Split(string(buf), "|") - if len(words) != 4 { - return fmt.Errorf("invalid item bytes, %v", string(buf)) - } - ri.Province = phonedatatool.ProvinceName(words[0]) - ri.City = phonedatatool.CityName(words[1]) - ri.ZipCode = phonedatatool.ZipCode(words[2]) - ri.AreaCode = phonedatatool.AreaCode(words[3]) - return nil - } -} - -type RecordOffset int64 - -func (ro RecordOffset) String() string { - return strconv.FormatInt(int64(ro), 10) -} - -type RecordID int64 - -func (rid RecordID) String() string { - return strconv.FormatInt(int64(rid), 10) -} - -type RecordIDList []RecordID - -func (l RecordIDList) Len() int { - return len(l) -} -func (l RecordIDList) Less(i, j int) bool { - return l[i] < l[j] -} -func (l RecordIDList) Swap(i, j int) { - l[i], l[j] = l[j], l[i] -} - -type RecordPart struct { - offset2item map[RecordOffset]*RecordItem - offset2id map[RecordOffset]RecordID - id2offset map[RecordID]RecordOffset -} - -func NewRecordPart() *RecordPart { - return &RecordPart{ - offset2item: make(map[RecordOffset]*RecordItem), - offset2id: make(map[RecordOffset]RecordID), - id2offset: make(map[RecordID]RecordOffset), - } -} - -func (rp *RecordPart) Bytes() []byte { - w := bytes.NewBuffer(nil) - var idList RecordIDList - for k, _ := range rp.id2offset { - idList = append(idList, k) - } - sort.Sort(idList) - for _, id := range idList { - item := rp.offset2item[rp.id2offset[id]] - w.WriteString(strings.Join([]string{ - id.String(), - item.Province.String(), - item.City.String(), - item.ZipCode.String(), - item.AreaCode.String(), - }, "|")) - w.WriteByte('\n') - } - return w.Bytes() -} - -func (rp *RecordPart) Parse(reader *bytes.Reader, endOffset RecordOffset) error { - for nowID := RecordID(1); ; nowID++ { - startOffset := RecordOffset(reader.Size() - int64(reader.Len())) - if startOffset >= endOffset { - break - } - item := new(RecordItem) - if err := item.Parse(reader); err != nil { - return err - } - rp.offset2item[startOffset] = item - rp.offset2id[startOffset] = nowID - rp.id2offset[nowID] = startOffset - } - return nil -} diff --git a/phonedatatool/unpack/unpack.go b/phonedatatool/unpack/unpack.go deleted file mode 100644 index ffac70c..0000000 --- a/phonedatatool/unpack/unpack.go +++ /dev/null @@ -1,42 +0,0 @@ -package unpack - -import ( - "bytes" - "fmt" - "github.com/xluohome/phonedata/phonedatatool" -) - -type Unpacker struct { -} - -func NewUnpacker() phonedatatool.Unpacker { - return new(Unpacker) -} - -func (u *Unpacker) Unpack(phoneDataBuf []byte) (versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte, err error) { - reader := bytes.NewReader(phoneDataBuf) - versionPart := new(VersionPart) - if err := versionPart.Parse(reader); err != nil { - return nil, nil, nil, fmt.Errorf("failed to read version part: %v", err) - } - offsetPart := new(IndexPartOffsetPart) - if err := offsetPart.Parse(reader); err != nil { - return nil, nil, nil, fmt.Errorf("failed to read index-part-offset part: %v", err) - } - recordPart := NewRecordPart() - if err := recordPart.Parse(reader, offsetPart.IndexPartOffset); err != nil { - return nil, nil, nil, fmt.Errorf("failed to read record part: %v", err) - } - indexPart := NewIndexPart() - if err := indexPart.Parse(reader); err != nil { - return nil, nil, nil, fmt.Errorf("failed to read index part: %v", err) - } - if err := indexPart.MatchRecordOffsetToRecordID(recordPart.offset2id); err != nil { - return nil, nil, nil, fmt.Errorf("failed to match offset to record id: %v", err) - } - - versionPlainTextBuf = versionPart.Bytes() - recordPlainTextBuf = recordPart.Bytes() - indexPlainTextBuf = indexPart.Bytes() - return versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf, nil -} diff --git a/phonedatatool/unpack/version.go b/phonedatatool/unpack/version.go deleted file mode 100644 index 065b0b8..0000000 --- a/phonedatatool/unpack/version.go +++ /dev/null @@ -1,32 +0,0 @@ -package unpack - -import ( - "bytes" - "fmt" -) - -type Version string - -func (v Version) String() string { - return string(v) -} - -type VersionPart struct { - Version Version -} - -func (vp *VersionPart) Bytes() []byte { - w := bytes.NewBuffer(nil) - w.Write([]byte(vp.Version)) - w.WriteByte('\n') - return w.Bytes() -} - -func (vp *VersionPart) Parse(reader *bytes.Reader) error { - buf := make([]byte, 4) - if _, err := reader.Read(buf); err != nil { - return fmt.Errorf("failed to read: %v", err) - } - vp.Version = Version(buf) - return nil -} From 78e4db043d522d43e28829121b73064952ffa78e Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 16:44:05 +0800 Subject: [PATCH 26/29] =?UTF-8?q?=E5=BD=BB=E5=BA=95=E5=88=A0=E9=99=A4=20qu?= =?UTF-8?q?ery=20=E5=8C=85=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/phonedatatool/main.go | 29 +++++++++++++++++++++-------- phonedatatool/pack/query.go | 14 ++++++++++++++ phonedatatool/query/query.go | 13 ------------- 3 files changed, 35 insertions(+), 21 deletions(-) create mode 100644 phonedatatool/pack/query.go delete mode 100644 phonedatatool/query/query.go diff --git a/cmd/phonedatatool/main.go b/cmd/phonedatatool/main.go index 4a0e746..8662bb1 100644 --- a/cmd/phonedatatool/main.go +++ b/cmd/phonedatatool/main.go @@ -4,7 +4,6 @@ import ( "flag" "fmt" "github.com/xluohome/phonedata/phonedatatool/pack" - "github.com/xluohome/phonedata/phonedatatool/query" "github.com/xluohome/phonedata/phonedatatool/util" "os" "path" @@ -89,16 +88,10 @@ func main() { fmt.Println("ERROR! No number to query") return } - if info, err := query.NewQuerier(*source).QueryNumber(*number); err != nil { + if err := QueryNumber(*source, *number); err != nil { fmt.Println("ERROR! Query failed.", err) return } else { - fmt.Println("PhoneNum: ", info.PhoneNumber) - fmt.Println("AreaZone: ", info.AreaCode) - fmt.Println("CardType: ", info.CardTypeID.ToName().String()) - fmt.Println("City: ", info.CityName) - fmt.Println("ZipCode: ", info.ZipCode) - fmt.Println("Province: ", info.ProvinceName) fmt.Println("Query completed.") return } @@ -182,3 +175,23 @@ func Unpack(phoneDataFilePath string, plainDirectoryPath string) error { return nil } } + +func QueryNumber(phoneDataFilePath string, number string) error { + var rawBuf []byte + if b, err := os.ReadFile(phoneDataFilePath); err != nil { + return err + } else { + rawBuf = b + } + if info, err := pack.NewQuerier().Query(rawBuf, number); err != nil { + return err + } else { + fmt.Println("PhoneNum: ", info.PhoneNumber) + fmt.Println("AreaZone: ", info.AreaCode) + fmt.Println("CardType: ", info.CardTypeID.ToName().String()) + fmt.Println("City: ", info.CityName) + fmt.Println("ZipCode: ", info.ZipCode) + fmt.Println("Province: ", info.ProvinceName) + return nil + } +} diff --git a/phonedatatool/pack/query.go b/phonedatatool/pack/query.go new file mode 100644 index 0000000..45460d5 --- /dev/null +++ b/phonedatatool/pack/query.go @@ -0,0 +1,14 @@ +package pack + +import "github.com/xluohome/phonedata/phonedatatool" + +type Querier struct { +} + +func NewQuerier() *Querier { + return &Querier{} +} + +func (q *Querier) Query(phoneDataBuf []byte, number string) (*phonedatatool.QueryResult, error) { + return nil, nil +} diff --git a/phonedatatool/query/query.go b/phonedatatool/query/query.go deleted file mode 100644 index 69c4346..0000000 --- a/phonedatatool/query/query.go +++ /dev/null @@ -1,13 +0,0 @@ -package query - -import "github.com/xluohome/phonedata/phonedatatool" - -type Querier struct{} - -func NewQuerier(phoneDataFilePath string) phonedatatool.Querier { - return new(Querier) -} - -func (q *Querier) QueryNumber(number string) (*phonedatatool.QueryResult, error) { - return nil, nil -} From 15d4606e8991fe4001a0a3aa3c3d0796ce5fab18 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 17:04:51 +0800 Subject: [PATCH 27/29] =?UTF-8?q?=E9=87=8D=E6=9E=84=E5=AE=8C=E6=88=90?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E5=8A=A0=E5=85=A5=E4=BA=86=E5=8F=B7=E7=A0=81?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E5=8A=9F=E8=83=BD=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- phonedatatool/pack/query.go | 37 ++++++++++++++++++++++++++++++++++-- phonedatatool/pack/unpack.go | 36 +++++++++++++++++++++++++---------- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/phonedatatool/pack/query.go b/phonedatatool/pack/query.go index 45460d5..384bb79 100644 --- a/phonedatatool/pack/query.go +++ b/phonedatatool/pack/query.go @@ -1,6 +1,11 @@ package pack -import "github.com/xluohome/phonedata/phonedatatool" +import ( + "bytes" + "fmt" + "github.com/xluohome/phonedata/phonedatatool" + "strconv" +) type Querier struct { } @@ -10,5 +15,33 @@ func NewQuerier() *Querier { } func (q *Querier) Query(phoneDataBuf []byte, number string) (*phonedatatool.QueryResult, error) { - return nil, nil + if len(number) < 7 { + return nil, fmt.Errorf("number %v is too short", number) + } + var numberPrefix NumberPrefix + if v, err := strconv.Atoi(number[:7]); err != nil { + return nil, err + } else { + numberPrefix = NumberPrefix(v) + } + + if result, err := new(Unpacker).unpack(bytes.NewReader(phoneDataBuf)); err != nil { + return nil, err + } else { + var indexItem *IndexItem + if item, ok := result.indexPart.prefix2item[numberPrefix]; !ok { + return nil, fmt.Errorf("unknown prefix of number %v", numberPrefix) + } else { + indexItem = item + } + recordItem := result.recordPart.id2item[result.offset2id[indexItem.recordOffset]] + return &phonedatatool.QueryResult{ + PhoneNumber: phonedatatool.PhoneNumber(number), + AreaCode: phonedatatool.AreaCode(recordItem.areaCode), + CardTypeID: indexItem.cardTypeID, + CityName: phonedatatool.CityName(recordItem.city), + ZipCode: phonedatatool.ZipCode(recordItem.zipCode), + ProvinceName: phonedatatool.ProvinceName(recordItem.province), + }, nil + } } diff --git a/phonedatatool/pack/unpack.go b/phonedatatool/pack/unpack.go index c8a68be..094148f 100644 --- a/phonedatatool/pack/unpack.go +++ b/phonedatatool/pack/unpack.go @@ -12,39 +12,55 @@ func NewUnpacker() phonedatatool.Unpacker { return new(Unpacker) } +type unpackResult struct { + versionPart *VersionPart + recordPart *RecordPart + offset2id map[Offset]RecordID + indexPart *IndexPart +} + func (u *Unpacker) Unpack(phoneDataBuf []byte) (versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte, err error) { - reader := bytes.NewReader(phoneDataBuf) + if result, err := u.unpack(bytes.NewReader(phoneDataBuf)); err != nil { + return nil, nil, nil, err + } else { + return result.versionPart.BytesPlainText(), result.recordPart.BytesPlainText(), result.indexPart.BytesPlainText(result.offset2id), nil + } +} + +func (u *Unpacker) unpack(reader *bytes.Reader) (*unpackResult, error) { versionPart := new(VersionPart) if err := versionPart.Parse(reader); err != nil { - return nil, nil, nil, err + return nil, err } - versionPlainTextBuf = versionPart.BytesPlainText() var indexPartOffset Offset if err := indexPartOffset.Parse(reader); err != nil { - return nil, nil, nil, err + return nil, err } recordBuf := make([]byte, indexPartOffset-RecordPartBaseOffset) if _, err := reader.Read(recordBuf); err != nil { - return nil, nil, nil, err + return nil, err } recordPart := NewRecordPart() offset2id := make(map[Offset]RecordID) if o2i, err := recordPart.Parse(bytes.NewReader(recordBuf), RecordPartBaseOffset); err != nil { - return nil, nil, nil, err + return nil, err } else { offset2id = o2i } - recordPlainTextBuf = recordPart.BytesPlainText() indexPart := NewIndexPart() if err := indexPart.Parse(reader); err != nil { - return nil, nil, nil, err + return nil, err } indexPart.BytesPlainText(offset2id) - indexPlainTextBuf = indexPart.BytesPlainText(offset2id) - return versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf, nil + return &unpackResult{ + versionPart: versionPart, + recordPart: recordPart, + offset2id: offset2id, + indexPart: indexPart, + }, nil } From 0ad63e63da3848e6f975c2ca17498a4e36f91b1a Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 17:16:39 +0800 Subject: [PATCH 28/29] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E3=80=82=E5=8A=A0=E5=85=A5=E4=BA=86=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E5=8A=9F=E8=83=BD=E8=AF=B4=E6=98=8E=EF=BC=8C=E4=BB=A5?= =?UTF-8?q?=E5=8F=8A=E6=9B=B4=E8=AF=A6=E7=BB=86=E7=9A=84=E6=96=87=E6=9C=AC?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=AF=B4=E6=98=8E=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.phonedatatool.md | 54 +++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/README.phonedatatool.md b/README.phonedatatool.md index f601c87..ff2a161 100644 --- a/README.phonedatatool.md +++ b/README.phonedatatool.md @@ -7,11 +7,13 @@ - 第一步：将原始二进制文件解包成文本。 - 第二步：根据需要手动修改文本。 - 第三步：将修改后的文本打包成二进制文件。 +- 第四步：用修改后的二进制文件查询号码。 ## 2. 解包 ```shell -phonedatatool.exe -unpack -i phone.dat -o abc +D:\seedjyh\phonedata>phonedatatool.exe -unpack -i phone.dat -o tmp +Unpack completed. ``` 可以将二进制文件 phone.dat 解包到一个名为 abc 的目录。里面有三个文本文件。 @@ -19,12 +21,26 @@ phonedatatool.exe -unpack -i phone.dat -o abc ## 3. 打包 ```shell -phonedatatool.exe -pack -i abc -o phone.2.dat +D:\seedjyh\phonedata>phonedatatool.exe -pack -i tmp -o phone.2.dat +Pack completed. ``` 可以将目录 abc 里的文本文件打包成二进制文件 phone.2.dat -## 4. 解包后文件说明 +## 4. 查询号码 + +```shell +D:\seedjyh\phonedata>phonedatatool.exe -query -i phone.2.dat -number 13336061916 +PhoneNum: 13336061916 +AreaZone: 0571 +CardType: 中国电信 +City: 杭州 +ZipCode: 310000 +Province: 浙江 +Query completed. +``` + +## 5. 解包后文件说明 解包后的目录下会产生 3 个文本文件，功能分别是： @@ -34,11 +50,11 @@ phonedatatool.exe -pack -i abc -o phone.2.dat | record.txt | 记录区（省、市、邮编、区号） | | index.txt | 索引区（号码前 7 位、记录区偏移量、号码类型） | -### 4.1. version.txt +### 5.1. version.txt 里面应该是 4 个字符。比如 "2307"。 -### 4.2. record.txt +### 5.2. record.txt 有多行，每行包括一条记录，例如`1|安徽|巢湖|238000|0551`。 @@ -54,7 +70,7 @@ phonedatatool.exe -pack -i abc -o phone.2.dat 其中「记录区 ID」必须是整数，不一定要连续，但每行的「记录区 ID」必须不同。 -### 4.3. index.txt +### 5.3. index.txt 有多行，每行包括一条索引，例如`1300000|251|2` @@ -66,9 +82,31 @@ phonedatatool.exe -pack -i abc -o phone.2.dat | 251 | 记录区 ID（含义见 record.txt 章节） | | 2 | 卡片类型码 | -## 5. 备注 +### 5.4. 文本文件修改方式 + +#### 5.4.1. 新增一个号码段 + +如果一个号码段（前七位）在索引文件 index.txt 里没有，则可以直接在 index.txt 末尾加入一条记录。 + +如果这个号段前缀归属于 record.txt 里已有的某一地区，则「记录区 ID」直接填该地区的 ID 即可。 + +否则，需要先在 record.txt 里添加一条记录（注意新记录的 ID 必须和已有的所有 ID 均不相同），然后再往 index.txt 里添加记录。 + +#### 5.4.2. 修改一个号码段 + +直接修改该号码段在 index.txt 里的信息即可。例如，将「记录区 ID」修改成另一个数。必要时也要先在 record.txt 里新增记录。 + +#### 5.4.3. 删除一个号码段 + +直接删除 index.txt 里的信息即可。 + +#### 5.4.4. 备注 + +所有文本文件都必须以换行符结尾。 + +## 6. 备注 -### 5.1. 卡片类型码 +### 6.1. 卡片类型码 卡片类型码的含义由 phonedata 原项目规定。目前（2023-05-24）类型码的含义为： From afa57c7c58d77fb4a96d43dd8c0cae00b672b408 Mon Sep 17 00:00:00 2001 From: jyh Date: Wed, 24 May 2023 17:22:14 +0800 Subject: [PATCH 29/29] version 0.2.0 --- CHANGELOG.phoneatatool.md | 24 ++++++++++++++++++++++++ cmd/phonedatatool/main.go | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.phoneatatool.md diff --git a/CHANGELOG.phoneatatool.md b/CHANGELOG.phoneatatool.md new file mode 100644 index 0000000..16b70bb --- /dev/null +++ b/CHANGELOG.phoneatatool.md @@ -0,0 +1,24 @@ +# CHANGELOG.phoneatatool + +## [0.2.0] - 2023-05-21 + +### 新增 + +- 实现了号码查询功能。 + +### 改动 + +- 说明文档更清晰了。 + +### 修复 + +## [0.1.0] - 2023-05-21 + +### 新增 + +- 实现了打包功能。 +- 实习了解包功能。 + +### 改动 + +### 修复 diff --git a/cmd/phonedatatool/main.go b/cmd/phonedatatool/main.go index 8662bb1..4006efe 100644 --- a/cmd/phonedatatool/main.go +++ b/cmd/phonedatatool/main.go @@ -16,7 +16,7 @@ import ( const ( Name = "phonedatatool" - Version = "0.1.0" + Version = "0.2.0" Author = "seedjyh@gmail.com" FullName = Name + "_" + Version + "(" + Author + ")" )