-
Notifications
You must be signed in to change notification settings - Fork 0
Update devices list for Android and iOS #4
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,10 @@ | ||
| github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | ||
| github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | ||
| github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | ||
| github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= | ||
| github.com/wk8/go-ordered-map v1.0.0 h1:BV7z+2PaK8LTSd/mWgY12HyMAo5CEgkHqbkVq2thqr8= | ||
| github.com/wk8/go-ordered-map v1.0.0/go.mod h1:9ZIbRunKbuvfPKyBP1SIKLcXNlv74YCOZ3t3VTS6gRk= | ||
| golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= | ||
| golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= | ||
| gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= | ||
| gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,177 @@ | ||
| package main | ||
|
|
||
| import "fmt" | ||
| import ( | ||
| "bufio" | ||
| "encoding/json" | ||
| "fmt" | ||
| "net/http" | ||
| "os" | ||
| "strings" | ||
|
|
||
| orderedmap "github.com/wk8/go-ordered-map" | ||
| ) | ||
|
|
||
| type Device struct { | ||
| Identifier string `json:"identifier"` | ||
| Generation string `json:"generation"` | ||
| } | ||
|
|
||
| func main() { | ||
| fmt.Println("TODO: Implement this to ease down on the manual process of updating the device data") | ||
| // Fetch devices raw data from https://github.com/pluwen/apple-device-model-list | ||
| responseAsLines, err := readDevicesRawDataFromGithubAsLines() | ||
| if err != nil { | ||
| fmt.Println(err) | ||
| return | ||
| } | ||
| // defer devicesRawResponse.Body.Close() | ||
|
|
||
| // We are going to be processing the ReadMe file line by line, those variables will help us keep state | ||
|
|
||
| // The read me file has different sections for different devices (iPhone, iPad...etc) | ||
| // This keeps track of the current section being processed | ||
| var currentSection string | ||
| // Boolean variable to indicate if we are currently processing lines in a code block | ||
| isCodeBlock := false | ||
|
|
||
| // A map to keep the overall resultMap | ||
| // Key: string -> Section name (iPhone, iPad...etc) | ||
| // Value : Map | ||
| // - Key: string -> Device Generation | ||
| // - Value: []string -> Device identifiers for a single generation | ||
| resultMap := orderedmap.New() | ||
|
|
||
| for _, line := range responseAsLines { | ||
|
|
||
| if strings.HasPrefix(line, "## ") { | ||
| // New section detected | ||
| currentSection = strings.TrimSpace(strings.TrimPrefix(line, "## ")) | ||
| resultMap.Set(currentSection, orderedmap.New()) | ||
| continue | ||
| } | ||
|
|
||
| if strings.HasPrefix(line, "```") { | ||
| // Code block start or end detected | ||
| isCodeBlock = !isCodeBlock | ||
| continue | ||
| } | ||
|
|
||
| if isCodeBlock && currentSection != "" { | ||
| blockResult, _ := resultMap.Get(currentSection) | ||
| typedBlockResult := blockResult.(*orderedmap.OrderedMap) | ||
|
|
||
| // Parse a single code block line | ||
| // Example: `"iPhone3,1", "iPhone3,2", "iPhone3,3": iPhone 4` | ||
| parts := strings.Split(line, ":") | ||
| if len(parts) == 2 { | ||
| generation := strings.TrimSpace(parts[1]) | ||
| identifiers := extractIdentifiers(strings.TrimSpace(parts[0])) | ||
| typedBlockResult.Set(generation, identifiers) | ||
| } | ||
|
|
||
| resultMap.Set(currentSection, typedBlockResult) | ||
| } | ||
| } | ||
|
|
||
| // Flatten result into list of OutputItems | ||
| devices := make([]Device, 0) | ||
| devices = append(devices, addSectionDevicesToOutputList("Apple TV", resultMap)...) | ||
| devices = append(devices, addSectionDevicesToOutputList("Apple Watch", resultMap)...) | ||
| devices = append(devices, addSectionDevicesToOutputList("iPad", resultMap)...) | ||
| devices = append(devices, addSectionDevicesToOutputList("iPhone", resultMap)...) | ||
|
|
||
| // Write result to JSON file | ||
| outputFile, err := os.Create("../../src/data/ios.json") | ||
| if err != nil { | ||
| fmt.Println("Error creating output file:", err) | ||
| return | ||
| } | ||
|
|
||
| encoder := json.NewEncoder(outputFile) | ||
| encoder.SetIndent("", " ") | ||
|
|
||
| if err := encoder.Encode(devices); err != nil { | ||
| fmt.Println("Error encoding JSON:", err) | ||
| return | ||
| } | ||
|
|
||
| fmt.Println("Data successfully written to:", outputFile.Name()) | ||
| } | ||
|
|
||
| func readDevicesRawDataFromGithubAsLines() ([]string, error) { | ||
| url := "https://raw.githubusercontent.com/pluwen/apple-device-model-list/main/README.md" | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what if they delete it? May be we can just hardcode it for now? Also parsing a README looks dicey.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, I agree, it's not the most stable approach but the alternative is manually updating it from a Wikipedia page. So we can use this as long as it works and if it ever breaks, we can go back to manual update or find another source 😇 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. +1 on parsing from README, very sketchy. Specially given the fact that the readme, has and can possibly have more of non device information like Would it be possible to sanitize/test the output before we populate json files?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You can see here that we are picking only devices sections to be processed. I completely agree that this isn't a perfect approach but isn't this better than manually updating the file? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 💯 better than manual. But the only issue I see, which can be resolved with some sanity checks.
In this case for script, we are fetching it from source, parsing it and storing it. Unless we are certain the script didn't do anything flaky, we are always gonna be worried or do manual checks. So, I would say, let's go with the script and source and parsing MD, but maybe let's just have 2 small tests to be sure
If we have these 2 tests, next time we run script and they pass, we won't have to worry about data integrity. What do you think? |
||
|
|
||
| resp, err := http.Get(url) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error reading response body: %v", err) | ||
| } | ||
| defer resp.Body.Close() | ||
|
|
||
| if resp.StatusCode != http.StatusOK { | ||
| return nil, fmt.Errorf("error reading response body: %v", err) | ||
| } | ||
|
|
||
| fmt.Println("Loaded devices raw data successfully!") | ||
|
|
||
| var lines []string | ||
| scanner := bufio.NewScanner(resp.Body) | ||
| for scanner.Scan() { | ||
| lines = append(lines, scanner.Text()) | ||
| } | ||
|
|
||
| if err := scanner.Err(); err != nil { | ||
| return nil, fmt.Errorf("error reading response body: %v", err) | ||
| } | ||
|
|
||
| return lines, nil | ||
| } | ||
|
|
||
| // This function accepts input like that: `"iPhone3,1", "iPhone3,2", "iPhone3,3"` | ||
| // It's tricky to do a simple split because the values we are interested in are separated by ',' | ||
| // But they also have ',' in their value | ||
| // Expected output for above example is: [iPhone3,1, iPhone3,2, iPhone3,3] | ||
| func extractIdentifiers(input string) []string { | ||
| result := make([]string, 0) | ||
| isProcessingWord := false | ||
| wordUnderConstruction := "" | ||
|
|
||
| // The order of checks in this loop is important | ||
| for _, char := range input { | ||
| if char == '"' { | ||
| // This could be either start or end of a word | ||
| if isProcessingWord { | ||
| // End of word detected | ||
| result = append(result, wordUnderConstruction) | ||
| wordUnderConstruction = "" | ||
| } | ||
| isProcessingWord = !isProcessingWord | ||
| continue | ||
| } | ||
|
|
||
| if isProcessingWord { | ||
| wordUnderConstruction += string(char) | ||
| continue | ||
| } | ||
|
|
||
| // Do this check last to avoid ignoring commands and spaces within an identifier value | ||
| if char == ',' || char == ' ' { | ||
| continue | ||
| } | ||
| } | ||
|
|
||
| return result | ||
| } | ||
|
|
||
| func addSectionDevicesToOutputList(sectionName string, data *orderedmap.OrderedMap) []Device { | ||
| devices := make([]Device, 0) | ||
|
|
||
| sectionMap, _ := data.Get(sectionName) | ||
| if innerMap, ok := sectionMap.(*orderedmap.OrderedMap); ok { | ||
| for innerPair := innerMap.Oldest(); innerPair != nil; innerPair = innerPair.Next() { | ||
| for _, identifier := range innerPair.Value.([]string) { | ||
| devices = append(devices, Device{Identifier: identifier, Generation: innerPair.Key.(string)}) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return devices | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
is this script run manually or does some action/statup trigger this?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We run it manually whenever we want to update the devices, just like the Android one.