Skip to content

Commit 7ca4012

Browse files
npentrelnpentrel
authored
DOCS 3554: Refactor SDK ingestion (#4086)
1 parent c06d91d commit 7ca4012

File tree

15 files changed

+1228
-1180
lines changed

15 files changed

+1228
-1180
lines changed

‎.github/workflows/parse_flutter.py‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
from parser_utils import make_soup
2+
from markdownify import markdownify as md
3+
4+
## Language-specific resource name overrides:
5+
flutter_resource_overrides = {
6+
"generic_component": "Generic",
7+
"movement_sensor": "MovementSensor",
8+
"power_sensor": "PowerSensor",
9+
"vision": "VisionClient",
10+
"robot": "RobotClient",
11+
"data": "DataClient",
12+
"dataset": "DataClient",
13+
"data_sync": "DataClient"
14+
}
15+
16+
## Ignore these specific APIs if they error, are deprecated, etc:
17+
flutter_ignore_apis = [
18+
'getRoverRentalRobots' # internal use
19+
]
20+
21+
## Use these URLs for data types that are built-in to the language:
22+
flutter_datatype_links = {}
23+
24+
class FlutterParser:
25+
def __init__(self, proto_map_file, flutter_staging_url = None):
26+
self.proto_map_file = proto_map_file
27+
self.sdk_url = "https://flutter.viam.dev"
28+
29+
if flutter_staging_url:
30+
self.scrape_url = flutter_staging_url
31+
self.staging = True
32+
else:
33+
self.scrape_url = "https://flutter.viam.dev"
34+
self.staging = False
35+
self.flutter_methods = {}
36+
37+
38+
def parse(self, type, viam_resources, args):
39+
self.flutter_methods[type] = {}
40+
41+
# Skip resources not supported in Flutter
42+
unsupported_resources = [
43+
"base_remote_control", "button", "encoder", "input_controller",
44+
"data_manager", "generic_service", "mlmodel", "motion",
45+
"navigation", "slam", "switch", "app", "billing", "mltraining"
46+
]
47+
48+
for resource in viam_resources:
49+
50+
## Determine URL form for Flutter depending on type (like 'component').
51+
## TEMP: Manually exclude Base Remote Control Service (Go only):
52+
## TODO: Handle resources with 0 implemented methods for this SDK better.
53+
54+
# Initialize Flutter methods dictionary if it doesn't exist
55+
self.flutter_methods[type][resource] = {}
56+
57+
if resource in unsupported_resources:
58+
if args.verbose:
59+
print(f'DEBUG: Skipping unsupported Flutter resource: {resource}')
60+
continue
61+
elif resource in flutter_resource_overrides:
62+
url = f"{self.scrape_url}/viam_sdk/{flutter_resource_overrides[resource]}-class.html"
63+
if args.verbose:
64+
print(f'DEBUG: Parsing Flutter URL: {url}')
65+
else:
66+
url = f"{self.scrape_url}/viam_sdk/{resource.capitalize()}-class.html"
67+
68+
## Scrape each parent method tag and all contained child tags for Flutter by resource.
69+
## TEMP: Manually exclude Base Remote Control Service (Go only).
70+
## TODO: Handle resources with 0 implemented methods for this SDK better.
71+
if resource not in unsupported_resources:
72+
73+
soup = make_soup(url)
74+
75+
if resource in flutter_resource_overrides:
76+
flutter_resource = flutter_resource_overrides[resource]
77+
else:
78+
flutter_resource = resource.capitalize()
79+
## Limit matched class to either 'callable' or 'callable inherited' and remove the constructor (proto id) itself:
80+
flutter_methods_raw = soup.find_all(
81+
lambda tag: tag.name == 'dt'
82+
and not tag.get('id') == flutter_resource
83+
and tag.has_attr("class")
84+
and "callable" in tag.get("class"))
85+
86+
## Loop through scraped tags and select salient data:
87+
for tag in flutter_methods_raw:
88+
89+
## Create new empty dictionary for this specific method, to be appended to ongoing flutter_methods dictionary,
90+
## in form: flutter_methods[type][resource][method_name] = this_method_dict
91+
this_method_dict = {}
92+
93+
method_name = tag.get('id')
94+
95+
if not method_name in flutter_ignore_apis:
96+
97+
## Look up method_name in proto_map file, and return matching proto:
98+
with open(self.proto_map_file, 'r') as f:
99+
for row in f:
100+
## Because Flutter is the final entry in the mapping CSV, we must also rstrip() to
101+
## strip the trailing newline (\n) off the row itself:
102+
row = row.rstrip()
103+
104+
if not row.startswith('#') \
105+
and row.startswith(resource + ',') \
106+
and row.split(',')[5] == method_name:
107+
this_method_dict["proto"] = row.split(',')[1]
108+
109+
## Determine method link:
110+
method_link = tag.find("span", class_="name").a['href'].replace("..", self.sdk_url)
111+
this_method_dict["method_link"] = method_link
112+
113+
## While some method info is available to us on this current Flutter SDK page, the code sample is only found on the
114+
## method_link page. So we scrape that page for everything:
115+
method_soup = make_soup(method_link)
116+
117+
## Method description and code samples are both found within the same section tag:
118+
desc_or_code_sample = method_soup.find('section', class_ = 'desc markdown')
119+
120+
if desc_or_code_sample:
121+
if desc_or_code_sample.p:
122+
this_method_dict["method_description"] = desc_or_code_sample.p.text
123+
if desc_or_code_sample.pre:
124+
this_method_dict["code_sample"] = desc_or_code_sample.pre.text
125+
126+
parameter_tags = method_soup.find_all(
127+
lambda tag: tag.name == 'span'
128+
and tag.get('class') == ['parameter'])
129+
130+
## Parse parameters, if any are found:
131+
if len(parameter_tags) != 0:
132+
133+
## Create new empty dictionary for this_method_dict named "parameters":
134+
this_method_dict["parameters"] = {}
135+
136+
optional = False
137+
138+
# If there is a curly brace before the parameter list all parameters are optional
139+
prev = parameter_tags[0].find_previous('section')
140+
if prev:
141+
if "{<ol" in str(prev):
142+
optional = True
143+
144+
for parameter_tag in parameter_tags:
145+
146+
## Create new empty dictionary this_method_parameters_dict to house all parameter
147+
## keys for this method, to allow for multiple parameters. Also resets the
148+
## previous parameter's data when looping through multiple parameters:
149+
this_method_parameters_dict = {}
150+
151+
## Parse for param name and usage string, convert to string (for markdownify):
152+
param_name = parameter_tag.find('span', class_ = 'parameter-name').text
153+
param_usage = str(parameter_tag.find('span', class_ = 'type-annotation')).replace('>>', '>\\>')
154+
155+
## Markdownify parameter usage and replace relative links with absolute:
156+
formatted_param_usage = md(param_usage, strip=['wbr']).replace("../../", "https://flutter.viam.dev/")
157+
this_method_parameters_dict["param_usage"] = formatted_param_usage
158+
159+
# if a parameter is optional the previous parameter has a curly brace before it
160+
this_method_parameters_dict["optional"] = optional
161+
162+
this_method_dict["parameters"][param_name] = this_method_parameters_dict
163+
164+
if parameter_tag.text.endswith("{"):
165+
optional = True
166+
167+
return_tags = method_soup.find_all(
168+
lambda tag: tag.name == 'span'
169+
and tag.get('class') == ['returntype'])
170+
171+
if len(return_tags) != 0:
172+
173+
## Create new empty dictionary for this_method_dict named "return":
174+
this_method_dict["return"] = {}
175+
176+
for return_tag in return_tags:
177+
178+
## Create new empty dictionary this_method_returns_dict to house all return
179+
## keys for this method, to allow for multiple returns. Also resets the
180+
## previous return's data when looping through multiple returns:
181+
this_method_return_dict = {}
182+
183+
# Parse return usage string, convert to string (for markdownify):
184+
return_usage = str(return_tag)
185+
186+
## Markdownify return usage and replace relative links with absolute:
187+
formatted_return_usage = md(return_usage, strip=['wbr']).replace("../../", "https://flutter.viam.dev/")
188+
this_method_return_dict["return_usage"] = formatted_return_usage.replace('>>', '>\\>').replace("dart-core/Future-class.html", "dart-async/Future-class.html")
189+
190+
# Parse return type:
191+
if return_tag.find('span', class_ = 'type-parameter'):
192+
return_type = return_tag.find('span', class_ = 'type-parameter').text
193+
else:
194+
return_type = return_tag.text
195+
196+
this_method_dict["return"][return_type] = this_method_return_dict
197+
198+
self.flutter_methods[type][resource][method_name] = this_method_dict
199+
200+
elif type == "app":
201+
##Flutter SDK has no APP API!
202+
pass
203+
204+
return self.flutter_methods
205+

0 commit comments

Comments
 (0)