Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions scripts/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,5 @@ module customerio/devices/scripts
go 1.19

require golang.org/x/text v0.3.7

require github.com/wk8/go-ordered-map v1.0.0 // indirect
8 changes: 8 additions & 0 deletions scripts/go.sum
Original file line number Diff line number Diff line change
@@ -1,2 +1,10 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/wk8/go-ordered-map v1.0.0 h1:BV7z+2PaK8LTSd/mWgY12HyMAo5CEgkHqbkVq2thqr8=
github.com/wk8/go-ordered-map v1.0.0/go.mod h1:9ZIbRunKbuvfPKyBP1SIKLcXNlv74YCOZ3t3VTS6gRk=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
174 changes: 172 additions & 2 deletions scripts/ios/main.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,177 @@
package main

import "fmt"
import (
"bufio"
"encoding/json"
"fmt"
"net/http"
"os"
"strings"

orderedmap "github.com/wk8/go-ordered-map"
)

type Device struct {
Identifier string `json:"identifier"`
Generation string `json:"generation"`
}

func main() {
fmt.Println("TODO: Implement this to ease down on the manual process of updating the device data")
// Fetch devices raw data from https://github.com/pluwen/apple-device-model-list

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this script run manually or does some action/statup trigger this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We run it manually whenever we want to update the devices, just like the Android one.

responseAsLines, err := readDevicesRawDataFromGithubAsLines()
if err != nil {
fmt.Println(err)
return
}
// defer devicesRawResponse.Body.Close()

// We are going to be processing the ReadMe file line by line, those variables will help us keep state

// The read me file has different sections for different devices (iPhone, iPad...etc)
// This keeps track of the current section being processed
var currentSection string
// Boolean variable to indicate if we are currently processing lines in a code block
isCodeBlock := false

// A map to keep the overall resultMap
// Key: string -> Section name (iPhone, iPad...etc)
// Value : Map
// - Key: string -> Device Generation
// - Value: []string -> Device identifiers for a single generation
resultMap := orderedmap.New()

for _, line := range responseAsLines {

if strings.HasPrefix(line, "## ") {
// New section detected
currentSection = strings.TrimSpace(strings.TrimPrefix(line, "## "))
resultMap.Set(currentSection, orderedmap.New())
continue
}

if strings.HasPrefix(line, "```") {
// Code block start or end detected
isCodeBlock = !isCodeBlock
continue
}

if isCodeBlock && currentSection != "" {
blockResult, _ := resultMap.Get(currentSection)
typedBlockResult := blockResult.(*orderedmap.OrderedMap)

// Parse a single code block line
// Example: `"iPhone3,1", "iPhone3,2", "iPhone3,3": iPhone 4`
parts := strings.Split(line, ":")
if len(parts) == 2 {
generation := strings.TrimSpace(parts[1])
identifiers := extractIdentifiers(strings.TrimSpace(parts[0]))
typedBlockResult.Set(generation, identifiers)
}

resultMap.Set(currentSection, typedBlockResult)
}
}

// Flatten result into list of OutputItems
devices := make([]Device, 0)
devices = append(devices, addSectionDevicesToOutputList("Apple TV", resultMap)...)
devices = append(devices, addSectionDevicesToOutputList("Apple Watch", resultMap)...)
devices = append(devices, addSectionDevicesToOutputList("iPad", resultMap)...)
devices = append(devices, addSectionDevicesToOutputList("iPhone", resultMap)...)

// Write result to JSON file
outputFile, err := os.Create("../../src/data/ios.json")
if err != nil {
fmt.Println("Error creating output file:", err)
return
}

encoder := json.NewEncoder(outputFile)
encoder.SetIndent("", " ")

if err := encoder.Encode(devices); err != nil {
fmt.Println("Error encoding JSON:", err)
return
}

fmt.Println("Data successfully written to:", outputFile.Name())
}

func readDevicesRawDataFromGithubAsLines() ([]string, error) {
url := "https://raw.githubusercontent.com/pluwen/apple-device-model-list/main/README.md"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what if they delete it? May be we can just hardcode it for now?

Also parsing a README looks dicey.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I agree, it's not the most stable approach but the alternative is manually updating it from a Wikipedia page. So we can use this as long as it works and if it ever breaks, we can go back to manual update or find another source 😇

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1 on parsing from README, very sketchy. Specially given the fact that the readme, has and can possibly have more of non device information like Author etc.

Would it be possible to sanitize/test the output before we populate json files?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can see here that we are picking only devices sections to be processed.

I completely agree that this isn't a perfect approach but isn't this better than manually updating the file?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💯 better than manual. But the only issue I see, which can be resolved with some sanity checks.

  • When we paste things manually, we do not alter anything and items were already verified from Wikipedia.

In this case for script, we are fetching it from source, parsing it and storing it. Unless we are certain the script didn't do anything flaky, we are always gonna be worried or do manual checks.

So, I would say, let's go with the script and source and parsing MD, but maybe let's just have 2 small tests to be sure

  • Before inserting in json file, we make sure old json items are not decreasing
  • There is not item duplication .

If we have these 2 tests, next time we run script and they pass, we won't have to worry about data integrity. What do you think?


resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("error reading response body: %v", err)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("error reading response body: %v", err)
}

fmt.Println("Loaded devices raw data successfully!")

var lines []string
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}

if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("error reading response body: %v", err)
}

return lines, nil
}

// This function accepts input like that: `"iPhone3,1", "iPhone3,2", "iPhone3,3"`
// It's tricky to do a simple split because the values we are interested in are separated by ','
// But they also have ',' in their value
// Expected output for above example is: [iPhone3,1, iPhone3,2, iPhone3,3]
func extractIdentifiers(input string) []string {
result := make([]string, 0)
isProcessingWord := false
wordUnderConstruction := ""

// The order of checks in this loop is important
for _, char := range input {
if char == '"' {
// This could be either start or end of a word
if isProcessingWord {
// End of word detected
result = append(result, wordUnderConstruction)
wordUnderConstruction = ""
}
isProcessingWord = !isProcessingWord
continue
}

if isProcessingWord {
wordUnderConstruction += string(char)
continue
}

// Do this check last to avoid ignoring commands and spaces within an identifier value
if char == ',' || char == ' ' {
continue
}
}

return result
}

func addSectionDevicesToOutputList(sectionName string, data *orderedmap.OrderedMap) []Device {
devices := make([]Device, 0)

sectionMap, _ := data.Get(sectionName)
if innerMap, ok := sectionMap.(*orderedmap.OrderedMap); ok {
for innerPair := innerMap.Oldest(); innerPair != nil; innerPair = innerPair.Next() {
for _, identifier := range innerPair.Value.([]string) {
devices = append(devices, Device{Identifier: identifier, Generation: innerPair.Key.(string)})
}
}
}

return devices
}
Loading
Loading