-
Notifications
You must be signed in to change notification settings - Fork 46
/
Copy pathupdate_sdk_methods.py
executable file
·2372 lines (1993 loc) · 153 KB
/
update_sdk_methods.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
from bs4 import BeautifulSoup
from urllib.request import urlopen
from pathlib import Path
from markdownify import markdownify as md
import sys
import os
import subprocess
import urllib.parse
import urllib.error
import re as regex
import argparse
## The full list of SDK languages we scrape. You can use the sdk_languages
## positional parameter to refine this at runtime if desired:
sdks_supported = ["go", "python", "flutter"]
## Arrays of resources to scrape, by type:
## type = ["array", "of", "resources"]
## You can use the target_resources positional parameter to refine this
## at runtime if desired:
components = ["arm", "base", "board", "camera", "encoder", "gantry", "generic_component", "gripper",
"input_controller", "motor", "movement_sensor", "power_sensor", "sensor", "servo"]
services = ["base_remote_control", "data_manager", "generic_service", "mlmodel", "motion", "navigation", "slam", "vision"]
app_apis = ["app", "billing", "data", "dataset", "data_sync", "mltraining"]
robot_apis = ["robot"]
## Parse arguments passed to update_sdk_methods.py. You can:
## - Provide the specific sdk languages to run against as a comma-separated list, or
## omit entirely to run against all sdks_supported.
## - Provide the specific resource(s) to run against as a comma-separated list, or
## omit entirely to run against all resources, across all types. This option
## only supports arbitrary resources within the same resource type (i.e. all
## components, or all services).
## - Use 'verbose' mode to enable DEBUG output.
## - Use 'map' mode to generate a proto map template file only.
parser = argparse.ArgumentParser()
parser.add_argument('sdk_languages', type=str, nargs='?', help="A comma-separated list of the sdks to run against. \
Can be one of: go, python, flutter. Omit to run against all sdks.")
parser.add_argument('target_resources', type=str, nargs='?', help="A comma-separated list of the resources to run against. \
Must be all within the same resource type, like component or service. Omit to run against all resources.")
parser.add_argument('-o', '--overrides', action='store_true', help="Print out the full expected parameter | return description \
override filepath to STDOUT. Use with something like:\n \
for filename in `python3 .github/workflows/update_sdk_methods.py -o go motion`; do touch $filename; done")
parser.add_argument('-m', '--map', action='store_true', help="Generate initial mapping CSV file from upstream protos. \
In this mode, only the initial mapping file is output, no markdown.")
parser.add_argument('-v', '--verbose', action='store_true', help="Run in verbose mode. Writes a debug file containing \
the complete data object from parse() to /tmp/update_sdk_methods_debug.txt. \
Also prints high-level status updates to STDOUT. \
Deletes previous debug file when run again.")
## Parse provided parameters and arguments, if any:
args = parser.parse_args()
if args.map:
## We check for args.map again in both proto_map() and run().
sdks = sdks_supported
else:
sdk_list = ''
resource_list = ''
only_run_against = ''
## Using specific argument names to allow help text to be specific, but checking both
## for sdk or resource values, to allow providing either in any position on the CLI.
if args.sdk_languages is not None:
first_list = [s.strip() for s in args.sdk_languages.split(",")]
if True in tuple(x in sdks_supported for x in first_list):
sdk_list = first_list
else:
resource_list = first_list
if args.target_resources is not None:
second_list = [s.strip() for s in args.target_resources.split(",")]
if True in tuple(x in sdks_supported for x in second_list):
sdk_list = second_list
else:
resource_list = second_list
## If (one of) the list(s) contains an SDK language name:
if sdk_list:
sdks = []
for sdk_lang in sdk_list:
if sdk_lang not in sdks_supported:
print("ERROR: Unsupported SDK language: " + sdk_lang)
print("Exiting ...")
exit(1)
else:
sdks.append(sdk_lang)
if not args.overrides:
print("\nIMPORTANT: You have indicated that you want to run against specific SDKs, instead of all.")
print(" This will ERASE any existing content for any SDKs which you have not specified.")
print(" This is suitable for the initial conversion to autogenerated content, but likely")
print(" not desireable once automation is in full swing. If you do not want this, you should")
print(" CANCEL this run, and re-run without specifying any SDKs, which will run against all.")
else:
sdks = sdks_supported
## If (one of) the list(s) does not contain an SDK language name, assume this list is
## intended to serve as the resource list, and check each item against its matching
## likely resource type. All provided resources must be of the same resource type,
## so we can return an error to the operator as soon as any subsequent resource fails
## to match to another entry on the first's resource type array (i.e. if first is arm,
## we can always safely error and quit if second returns False against same type array: component):
if resource_list:
if True in tuple(x in components for x in resource_list) and not False in tuple(x in components for x in resource_list):
only_run_against = 'components'
components = resource_list
elif True in tuple(x in services for x in resource_list) and not False in tuple(x in services for x in resource_list):
only_run_against = 'services'
services = resource_list
elif True in tuple(x in app_apis for x in resource_list) and not False in tuple(x in app_apis for x in resource_list):
only_run_against = 'app_apis'
app_apis = resource_list
elif True in tuple(x in robot_apis for x in resource_list) and not False in tuple(x in robot_apis for x in resource_list):
only_run_against = 'robot_apis'
robot_apis = resource_list
else:
print("ERROR: Malformed resource list: " + str(resource_list))
print(" Specified resources must all be within a single resource type, like components or services.")
print("Exiting ...")
exit(1)
## If running in verbose mode, print some initial configuration details to the operator:
if args.verbose:
print('\nVERBOSE MODE: See /tmp/update_sdk_methods_debug.txt for debug output.')
print(' Note: This file is deleted at the start of each new verbose run.')
print(' Try, in a separate terminal window:\n')
print(' DURING RUN: tail -f /tmp/update_sdk_methods_debug.txt')
print(' AFTER RUN: less /tmp/update_sdk_methods_debug.txt\n')
if sdk_list and args.verbose:
print('SDKS OVERRIDE: Only running against ' + str(sdks) + '\n')
if resource_list and args.verbose:
print('RESOURCE OVERRIDE: Only running against ' + str(resource_list) + '\n')
if args.overrides and args.verbose:
print("ERROR: You cannot use verbose mode with print overrides mode.")
print(" If you want this script to print the overrides filepaths it needs")
print(" for param | return description overrides, rerun this script with")
print(" the -o flag but WITHOUT the -v flag.")
print("Exiting ...")
exit(1)
## This script must be run within the 'docs' git repo. Here we check
## to make sure this is the case, and get the root of our git-managed
## repo to use later in parse() and write_markdown():
process_result = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'], \
stdout=subprocess.PIPE, \
stderr=subprocess.PIPE)
stdout, stderr = process_result.communicate()
if process_result.returncode == 0:
gitroot = stdout.decode().rstrip()
else:
print("ERROR: You must run this script within a cloned copy of the 'docs' git repo!")
print("Exiting ...")
exit(1)
## Build path to sdk_protos_map.csv file that contains proto-to-methods mapping, used in write_markdown():
proto_map_file = os.path.join(gitroot, '.github/workflows/sdk_protos_map.csv')
## Array mapping language to its root URL:
sdk_url_mapping = {
"go": "https://pkg.go.dev",
"python": "https://python.viam.dev",
"cpp": "https://cpp.viam.dev",
"typescript": "https://ts.viam.dev",
"flutter": "https://flutter.viam.dev"
}
## Dictionary of proto API names, with empty methods array, to be filled in for later use by get_proto_apis():
proto_map = {
"arm": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/arm/v1/arm_grpc.pb.go",
"name": "ArmServiceClient",
"methods": []
},
"base": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/base/v1/base_grpc.pb.go",
"name": "BaseServiceClient",
"methods": []
},
"board": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/board/v1/board_grpc.pb.go",
"name": "BoardServiceClient",
"methods": []
},
"camera": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/camera/v1/camera_grpc.pb.go",
"name": "CameraServiceClient",
"methods": []
},
"encoder": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/encoder/v1/encoder_grpc.pb.go",
"name": "EncoderServiceClient",
"methods": []
},
"gantry": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/gantry/v1/gantry_grpc.pb.go",
"name": "GantryServiceClient",
"methods": []
},
"generic_component": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/generic/v1/generic_grpc.pb.go",
"name": "GenericServiceClient",
"methods": []
},
"gripper": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/gripper/v1/gripper_grpc.pb.go",
"name": "GripperServiceClient",
"methods": []
},
"input_controller": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/inputcontroller/v1/input_controller_grpc.pb.go",
"name": "InputControllerServiceClient",
"methods": []
},
"motor": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/motor/v1/motor_grpc.pb.go",
"name": "MotorServiceClient",
"methods": []
},
"movement_sensor": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/movementsensor/v1/movementsensor_grpc.pb.go",
"name": "MovementSensorServiceClient",
"methods": []
},
"power_sensor": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/powersensor/v1/powersensor_grpc.pb.go",
"name": "PowerSensorServiceClient",
"methods": []
},
"sensor": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/sensor/v1/sensor_grpc.pb.go",
"name": "SensorServiceClient",
"methods": []
},
"servo": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/component/servo/v1/servo_grpc.pb.go",
"name": "ServoServiceClient",
"methods": []
},
"data_manager": {
"url": "https://github.com/viamrobotics/api/blob/main/service/datamanager/v1/data_manager_grpc.pb.go",
"name": "DataManagerServiceClient",
"methods": []
},
"generic_service": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/service/generic/v1/generic_grpc.pb.go",
"name": "GenericServiceClient",
"methods": []
},
"mlmodel": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/service/mlmodel/v1/mlmodel_grpc.pb.go",
"name": "MLModelServiceClient",
"methods": []
},
"motion": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/service/motion/v1/motion_grpc.pb.go",
"name": "MotionServiceClient",
"methods": []
},
"navigation": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/service/navigation/v1/navigation_grpc.pb.go",
"name": "NavigationServiceClient",
"methods": []
},
"slam": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/service/slam/v1/slam_grpc.pb.go",
"name": "SLAMServiceClient",
"methods": []
},
"vision": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/service/vision/v1/vision_grpc.pb.go",
"name": "VisionServiceClient",
"methods": []
},
"app": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/app/v1/app_grpc.pb.go",
"name": "AppServiceClient",
"methods": []
},
"billing": {
"url": "https://github.com/viamrobotics/api/blob/main/app/v1/billing_grpc.pb.go",
"name": "BillingServiceClient",
"methods": []
},
"data": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/app/data/v1/data_grpc.pb.go",
"name": "DataServiceClient",
"methods": []
},
"dataset": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/app/dataset/v1/dataset_grpc.pb.go",
"name": "DatasetServiceClient",
"methods": []
},
"data_sync": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/app/datasync/v1/data_sync_grpc.pb.go",
"name": "DataSyncServiceClient",
"methods": []
},
"robot": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/robot/v1/robot_grpc.pb.go",
"name": "RobotServiceClient",
"methods": []
},
"mltraining": {
"url": "https://raw.githubusercontent.com/viamrobotics/api/main/app/mltraining/v1/ml_training_grpc.pb.go",
"name": "MLTrainingServiceClient",
"methods": []
}
}
## Language-specific resource name overrides:
## "proto_resource_name" : "language-specific_resource_name"
## "as-it-appears-in-type-array": "as-it-is-used-per-sdk"
## Note: Always remap generic component and service, for all languages,
## as this must be unique for this script, but is non-unique across sdks.
go_resource_overrides = {
"generic_component": "generic",
"input_controller": "input",
"movement_sensor": "movementsensor",
"power_sensor": "powersensor",
"generic_service": "generic",
"base_remote_control": "baseremotecontrol",
"data_manager": "datamanager"
}
## Ignore these specific APIs if they error, are deprecated, etc:
## {resource}.{methodname} to exclude a specific method, or
## interface.{interfacename} to exclude an entire Go interface:
go_ignore_apis = [
'interface.NavStore', # motion service interface
'interface.LocalRobot', # robot interface
'interface.RemoteRobot', # robot interface
'robot.RemoteByName', # robot method
'robot.ResourceByName', # robot method
'robot.RemoteNames', # robot method
#'robot.ResourceNames', # robot method
'robot.ResourceRPCAPIs', # robot method
'robot.ProcessManager', # robot method
'robot.OperationManager', # robot method
'robot.SessionManager', # robot method
'robot.PackageManager', # robot method
'robot.Logger' # robot method
]
## Use these URLs for data types (for params, returns, and errors raised) that are
## built-in to the language or provided by a non-Viam third-party package:
## TODO: Not currently using these in parse(), but could do a simple replace()
## or could handle in markdownification instead. TBD. Same with other SDK lang link arrays:
go_datatype_links = {
"context": "https://pkg.go.dev/context",
"map": "https://go.dev/blog/maps",
"bool": "https://pkg.go.dev/builtin#bool",
"int": "https://pkg.go.dev/builtin#int",
"float64": "https://pkg.go.dev/builtin#float64",
"image": "https://pkg.go.dev/image#Image",
"r3.vector": "https://pkg.go.dev/github.com/golang/geo/r3#Vector",
"string": "https://pkg.go.dev/builtin#string",
"*geo.Point": "https://pkg.go.dev/github.com/kellydunn/golang-geo#Point",
"primitive.ObjectID": "https://pkg.go.dev/go.mongodb.org/mongo-driver/bson/primitive#ObjectID",
"error": "https://pkg.go.dev/builtin#error"
}
## Language-specific resource name overrides:
python_resource_overrides = {
"generic_component": "generic",
"input_controller": "input",
"generic_service": "generic",
"data": "data_client",
"app": "app_client",
"billing": "billing_client",
"data": "data_client",
## Python bundles Dataset and Datasync protos in with Data,
## while Flutter does not. HACK:
"dataset": "data_client",
"data_sync": "data_client",
"mltraining": "ml_training_client"
}
## Ignore these specific APIs if they error, are deprecated, etc:
python_ignore_apis = [
'viam.app.app_client.AppClient.get_rover_rental_robots', # internal use
'viam.app.app_client.AppClient.get_rover_rental_parts', # internal use
'viam.app.data_client.DataClient.create_filter', # deprecated
'viam.app.data_client.DataClient.delete_tabular_data_by_filter', # deprecated
'viam.components.input.client.ControllerClient.reset_channel', # GUESS ?
'viam.robot.client.RobotClient.transform_point_cloud', # unimplemented
'viam.robot.client.RobotClient.get_component', # GUESS ?
'viam.robot.client.RobotClient.get_service', # GUESS ?
'viam.components.board.client.BoardClient.write_analog', # Currently borked: https://python.viam.dev/autoapi/viam/components/board/client/index.html#viam.components.board.client.BoardClient.write_analog
'viam.components.board.client.StreamWithIterator.next', # No content upstream
'viam.robot.client.ViamChannel.close', # channel-specific close
'viam.robot.client.SessionsClient.reset' # session-specific reset
]
## Use these URLs for data types that are not otherwise captured by parse(), such as:
## - Well-known built-in data types that are not scrapeable (like 'int')
## - Viam-specific data types, even if scrapeable, that are part of a multiple-data-type return
## (like list_organization_members : Tuple[List[viam.proto.app.OrganizationMember], List[viam.proto.app.OrganizationInvite]]
## Data type links defined here will be used instead of scraped links if both exist:
python_datatype_links = {
## Built-in data types:
"str": "https://docs.python.org/3/library/stdtypes.html#text-sequence-type-str",
"int": "https://docs.python.org/3/library/stdtypes.html#numeric-types-int-float-complex",
"float": "https://docs.python.org/3/library/stdtypes.html#numeric-types-int-float-complex",
"bytes": "https://docs.python.org/3/library/stdtypes.html#bytes-objects",
"bool": "https://docs.python.org/3/library/stdtypes.html#boolean-type-bool",
"datetime.datetime": "https://docs.python.org/3/library/datetime.html",
"datetime.timedelta": "https://docs.python.org/3/library/datetime.html#timedelta-objects",
## Third-party data types:
"numpy.typing.NDArray": "https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray",
## Viam-specific data types:
"viam.proto.app.OrganizationMember": "https://python.viam.dev/autoapi/viam/proto/app/index.html#viam.proto.app.OrganizationMember",
"viam.proto.app.OrganizationInvite": "https://python.viam.dev/autoapi/viam/proto/app/index.html#viam.proto.app.OrganizationInvite",
"viam.components.arm.KinematicsFileFormat.ValueType": "https://python.viam.dev/autoapi/viam/components/arm/index.html#viam.components.arm.KinematicsFileFormat",
"viam.media.video.NamedImage": "https://python.viam.dev/autoapi/viam/media/video/index.html#viam.media.video.NamedImage",
"viam.proto.common.ResponseMetadata": "https://python.viam.dev/autoapi/viam/gen/common/v1/common_pb2/index.html#viam.gen.common.v1.common_pb2.ResponseMetadata",
"viam.proto.component.encoder.PositionType.ValueType": "https://python.viam.dev/autoapi/viam/gen/component/encoder/v1/encoder_pb2/index.html#viam.gen.component.encoder.v1.encoder_pb2.PositionType",
"typing_extensions.Self": "https://python.viam.dev/autoapi/viam/robot/client/index.html#viam.robot.client.RobotClient",
"viam.components.movement_sensor.movement_sensor.MovementSensor.Accuracy": "https://python.viam.dev/autoapi/viam/components/movement_sensor/movement_sensor/index.html#viam.components.movement_sensor.movement_sensor.MovementSensor.Accuracy",
"viam.services.vision.vision.Vision.Properties": "https://python.viam.dev/autoapi/viam/components/audio_input/audio_input/index.html#viam.components.audio_input.audio_input.AudioInput.Properties"
}
## Inject these URLs, relative to 'docs', into param/return/raises descriptions that contain exact matching key text.
## write_markdown() uses override_description_links via link_description() when it goes to write out descriptions.
## Currently only used for method descriptions, but see commented-out code for usage as optional consideration.
## NOTE: I am assuming we want to link matching text across all SDKs and methods. If not, this array
## will need additional field(s): ( method | sdk ) to narrow match.
## NOTE 2: I omitted links to the SDKs (like for 'datetime', and 'dataclass' since these can be
## separately handled uniformly (perhaps with the {sdk}_datatype_links array for example).
## EXAMPLES: The first two items in this dict correspond to these docs examples:
## EXAMPLE 1: https://docs.viam.com/services/frame-system/#transformpose
## EXAMPLE 2: https://docs.viam.com/services/motion/#moveonmap
override_description_links = {
"additional transforms": "/services/frame-system/#additional-transforms",
"SLAM service": "/services/slam/",
"frame": "/services/frame-system/",
"Viam app": "https://app.viam.com/",
"organization settings page": "/manage/reference/organize/",
"image tags": "/data-ai/ai/create-dataset/#label-your-images",
"API key": "/fleet/cli/#authenticate",
"board model": "/dev/reference/apis/components/board/"
}
## Language-specific resource name overrides:
flutter_resource_overrides = {
"generic_component": "Generic",
"movement_sensor": "MovementSensor",
"power_sensor": "PowerSensor",
"vision": "VisionClient",
"robot": "RobotClient"
}
## Ignore these specific APIs if they error, are deprecated, etc:
flutter_ignore_apis = [
'getRoverRentalRobots' # internal use
]
## Use these URLs for data types that are built-in to the language:
flutter_datatype_links = {}
## Map sdk language to specific code fence formatting syntax for that language:
code_fence_fmt = {
'python': 'python',
'go': 'go',
'flutter': 'dart'
}
## Check to see if we have a locally-staged version of any of the supported SDK docs sites.
## If any are detected here, they will be used for all content in parse(), and the live
## version for that SDK docs site will not be scraped at all! First, set empty staging URLs
## to allow us to later action on whether they are empty or not:
python_staging_url = ''
go_staging_url = ''
flutter_staging_url = ''
## Check for GO SDK docs staging on local workstation:
go_process_pid = subprocess.run(["ps -ef | grep pkgsite | grep -v grep | awk {'print $2'}"], shell=True, text=True, capture_output=True).stdout.rstrip()
## If we found a staged local instance of the GO SDK docs, determine which port it is using, and build the staging URL to scrape against:
if go_process_pid != '':
go_process_port = subprocess.run(["lsof -Pp " + go_process_pid + " | grep LISTEN | awk {'print $9'} | sed 's%.*:%%g'"], shell=True, text = True, capture_output=True).stdout.rstrip()
go_staging_url = 'http://localhost:' + go_process_port
if args.verbose:
print('DEBUG: Detected local staged Go SDK docs URL: ' + go_staging_url + '/go.viam.com/rdk')
print(' Using this URL for all Go content (ignoring live version).')
## Check for Python and Flutter SDK docs staging on local workstation.
## Both use http.server with a user-selected port. This command fetches all possible matches:
http_server_process_pids = subprocess.run(["ps -ef | grep http.server | grep -v grep | awk '{print $2}'"], shell=True, text=True, capture_output=True).stdout.rstrip().split('\n')
## For each http.server processes detected, make educated guesses about which is which, determine the port used, and build the staging URL to scrape against:
for pid in http_server_process_pids:
http_server_pwd_result = subprocess.run(["lsof -Pp " + pid + " | grep cwd | awk {'print $9'}"], shell=True, text = True, capture_output=True).stdout.rstrip()
## Quality guess: Python build process always builds to this directory; safe to assume:
if 'docs/_build/html' in http_server_pwd_result:
python_process_port = subprocess.run(["lsof -Pp " + pid + " | grep LISTEN | awk {'print $9'} | sed 's%.*:%%g'"], shell=True, text = True, capture_output=True).stdout.rstrip()
python_staging_url = 'http://localhost:' + python_process_port
if args.verbose:
print('DEBUG: Detected local staged Python SDK docs URL: ' + python_staging_url)
print(' Using this URL for all Python content (ignoring live version).')
## Mediocre guess: Flutter build process likely to have string 'flutter' in cwd, either 'viam-flutter-sdk' as cloned directly, or 'flutter' as renamed by operator.
## TODO: If operators run into instances where this script misses a valid Flutter staging URL because the path to that staged HTML artifacts dir has been
## renamed in a fashion that does not include the string 'flutter', then change this to just always pick up any instances of http.server that aren't already matched to
## Go, above. This would mean that operators cannot run an unrelated http.server instance on this workstation, which they currently can do with present config.
if 'flutter' in http_server_pwd_result:
flutter_process_port = subprocess.run(["lsof -Pp " + pid + " | grep LISTEN | awk {'print $9'} | sed 's%.*:%%g'"], shell=True, text = True, capture_output=True).stdout.rstrip()
flutter_staging_url = 'http://localhost:' + flutter_process_port + '/doc/api'
if args.verbose:
print('DEBUG: Detected local staged Flutter SDK docs URL: ' + flutter_staging_url)
print(' Using this URL for all Flutter content (ignoring live version).')
## Fetch canonical Proto method names.
## Required by Flutter parsing, and for generating the initial mapping file if -m was passed:
def get_proto_apis():
for api in proto_map.keys():
api_url = proto_map[api]["url"]
api_name = proto_map[api]["name"]
api_page = urlopen(api_url)
api_html = api_page.read().decode("utf-8")
## Protos are presented in plaintext, so we must match by expected raw text:
proto_regex = 'type ' + regex.escape(api_name) + r'[^{]*\{([^}]+)\}'
search = regex.search(proto_regex, api_html)
match_output = search.group()
split = match_output.splitlines()
for line in split:
line = line.strip()
if line[0].isupper():
separator = "("
line = line.split(separator, 1)[0]
## Append to proto_map for use later:
proto_map[api]["methods"].append(line)
## Only generate proto mapping template file if 'map' was passed as argument:
if args.map:
## Writing template file with extra '.template' at the end to avoid accidentally clobbering
## the prod file if we've already populated it. When ready, change the filename to exactly
## sdk_protos_map.csv for this script to use it for proto mapping:
proto_map_file_template = os.path.join(gitroot, '.github/workflows/sdk_protos_map.csv.template')
output_file = open('%s' % proto_map_file_template, "w")
output_file.write('## RESOURCE, PROTO, PYTHON METHOD, GO METHOD, FLUTTER METHOD\n')
for api in proto_map.keys():
output_file.write('\n## ' + api.title() + '\n')
for proto in proto_map[api]['methods']:
output_file.write(api + ',' + proto + ',\n')
return proto_map
## Fetch URL content via BS4, used in parse():
def make_soup(url):
try:
page = urlopen(url)
html = page.read().decode("utf-8")
return BeautifulSoup(html, "html.parser")
except urllib.error.HTTPError as err:
print(f'An HTTPError was thrown: {err.code} {err.reason} for URL: {url}')
def shorten_data_type(t):
if '.' in t:
return '.'.join(t.split('.')[-2:])
else:
return t
## Link any matching data types to their reference links, based on {sdk}_datatype_links[] array,
## used in parse() for both param and return data types. Handles data types syntax that includes
## multiple data types (and therefore requires multiple data type links), such as
## ListOrganizationMembers: Tuple[List[viam.proto.app.OrganizationMember], List[viam.proto.app.OrganizationInvite]
## DESIGN DECISION: Ignore well-known, usually leading (containing) data types like List, Tuple, Dict.
## NOTE: Only used in PySDK parsing, for now (but should work for all with minor tweak to support per-language links array):
def link_data_types(sdk, data_type_string):
linked_data_type_string = ""
## If the passed data_type_string matches exactly to a data type defined in python_datatype_links, use that:
if data_type_string in python_datatype_links.keys():
shorter_data_type = shorten_data_type(data_type_string)
linked_data_type_string = '[' + shorter_data_type + '](' + python_datatype_links[data_type_string] + ')'
else:
## Assemble all encountered data types that match to python_datatype_links keys into array.
## This match is a little too greedy, and will match, say, 'int' to 'JointPositions'. To counter
## this, we additionally check for leading and trailing alphanumeric characters further in:
matching_data_types = list(key for key in python_datatype_links if key in data_type_string)
if len(matching_data_types) > 0:
## Ugly hack to allow us to append within the for loop below, sorry:
linked_data_type_string = data_type_string
for data_type_found in matching_data_types:
## Discard string matches that are substrings of other data type strings:
if not regex.search(r'[A-Za-z0-9]' + data_type_found, data_type_string) and not regex.search(data_type_found + r'[A-Za-z0-9]', data_type_string):
shorter_data_type = shorten_data_type(data_type_found)
data_type_linked = '[' + shorter_data_type + '](' + python_datatype_links[data_type_found] + ')'
linked_data_type_string = regex.sub(data_type_found, data_type_linked, linked_data_type_string)
else:
## If we get here, this data_type is actually a substring of another data type. Take no action:
pass
## If we didn't find any matching links, return an empty string so we can know to look elsewhere,
## otherwise return linked data type string:
if linked_data_type_string == data_type_string:
return ""
else:
return linked_data_type_string
## Link matching text, used in write_markdown():
## NOTE: Currently does not support formatting for link titles
## (EXAMPLE: bolded DATA tab here: https://docs.viam.com/dev/reference/apis/data-client/#binarydatabyfilter)
def link_description(format_type, full_description, link_text, link_url):
## Supports 'md' link styling or 'html' link styling.
## The latter in case you want to link raw method usage:
if format_type == 'md':
new_linked_text = '[' + link_text + '](' + link_url + ')'
linked_description = regex.sub(link_text, new_linked_text, full_description)
elif format_type == 'html':
new_linked_text = '<a href="' + link_url + '">' + link_text + '</a>'
linked_description = regex.sub(link_text, new_linked_text, full_description)
return linked_description
## Fetch SDK documentation for each language in sdks array, by language, by type, by resource, by method.
def parse(type, names):
## TODO:
## - Unify returned method object form. Currently returning raw method usage for Go, and by-param, by-return (and by-raise)
## breakdown for each method for Python and Flutter. Let's chat about which is useful, and which I should throw away.
## Raw usage is I think how check_python_methods.py currently does it. Happy to convert Flutter and Py to dump raw usage,
## if you don't need the per-param,per-return,per-raise stuff.
## This parent dictionary will contain all dictionaries:
## all_methods[sdk][type][resource]
all_methods = {}
## Iterate through each sdk (like 'python') in sdks array:
for sdk in sdks:
## Determine SDK URL based on resource type:
sdk_url = sdk_url_mapping[sdk]
scrape_url = sdk_url
## Build empty dict to house methods, and update scrape_url
## with staging link if we detected one earlier:
if sdk == "go":
go_methods = {}
go_methods[type] = {}
if go_staging_url != '':
scrape_url = go_staging_url
elif sdk == "python":
python_methods = {}
python_methods[type] = {}
if python_staging_url != '':
scrape_url = python_staging_url
elif sdk == "flutter":
flutter_methods = {}
flutter_methods[type] = {}
if flutter_staging_url != '':
scrape_url = flutter_staging_url
else:
print("unsupported language!")
## Iterate through each resource (like 'arm') in type (like 'components') array:
for resource in names:
## Determine URL form for Go depending on type (like 'component'):
if sdk == "go":
if type in ("component", "service") and resource in go_resource_overrides:
url = f"{scrape_url}/go.viam.com/rdk/{type}s/{go_resource_overrides[resource]}"
elif type in ("component", "service"):
url = f"{scrape_url}/go.viam.com/rdk/{type}s/{resource}"
elif type == "robot" and resource in go_resource_overrides:
url = f"{scrape_url}/go.viam.com/rdk/{type}/{go_resource_overrides[resource]}"
elif type == "robot":
url = f"{scrape_url}/go.viam.com/rdk/{type}"
elif type == "app":
pass
go_methods[type][resource] = {}
## Determine URL form for Python depending on type (like 'component'):
elif sdk == "python":
if type in ("component", "service") and resource in python_resource_overrides:
url = f"{scrape_url}/autoapi/viam/{type}s/{python_resource_overrides[resource]}/client/index.html"
elif type in ("component", "service"):
url = f"{scrape_url}/autoapi/viam/{type}s/{resource}/client/index.html"
elif type == "app" and resource in python_resource_overrides:
url = f"{scrape_url}/autoapi/viam/{type}/{python_resource_overrides[resource]}/index.html"
elif type == "app":
url = f"{scrape_url}/autoapi/viam/{type}/{resource}/index.html"
else: # robot
url = f"{scrape_url}/autoapi/viam/{type}/client/index.html"
python_methods[type][resource] = {}
## Determine URL form for Flutter depending on type (like 'component').
## TEMP: Manually exclude Base Remote Control Service (Go only):
## TODO: Handle resources with 0 implemented methods for this SDK better.
elif sdk == "flutter" and resource != 'base_remote_control' and resource != 'encoder' and resource != 'input_controller' \
and resource != 'data_manager' and resource != 'generic_service' and resource !='mlmodel' and resource !='motion' \
and resource !='navigation' and resource !='slam' and type !='app':
if resource in flutter_resource_overrides:
url = f"{scrape_url}/viam_sdk/{flutter_resource_overrides[resource]}-class.html"
else:
url = f"{scrape_url}/viam_sdk/{resource.capitalize()}-class.html"
flutter_methods[type][resource] = {}
## If an invalid language was provided:
else:
pass
## Scrape each parent method tag and all contained child tags for Go by resource:
## Skip Go: App (Go has no App client) and the generic component and service, which
## require explicit in-script workaround (DoCommand neither inherited (from resource.Resource)
## nor explicitly defined in-interface (not in Go docs, only in un-doc'd code):
if sdk == "go" and type != "app" and resource != "generic_component" and resource != "generic_service":
soup = make_soup(url)
## Get a raw dump of all go methods by interface for each resource:
go_methods_raw = soup.find_all(
lambda tag: tag.name == 'div'
and tag.get('class') == ['Documentation-declaration']
and tag.pre.text.startswith('type')
and "interface {" in tag.pre.text)
# some resources have more than one interface:
for resource_interface in go_methods_raw:
## Determine the interface name, which we need for the method_link:
interface_name = resource_interface.find('pre').text.splitlines()[0].removeprefix('type ').removesuffix(' interface {')
## Exclude unwanted Go interfaces:
check_interface_name = 'interface.' + interface_name
if not check_interface_name in go_ignore_apis:
## Loop through each method found for this interface:
for tag in resource_interface.find_all('span', attrs={"data-kind" : "method"}):
## Create new empty dictionary for this specific method, to be appended to ongoing go_methods dictionary,
## in form: go_methods[type][resource][method_name] = this_method_dict
this_method_dict = {}
tag_id = tag.get('id')
method_name = tag.get('id').split('.')[1]
## Exclude unwanted Go methods:
check_method_name = resource + '.' + method_name
if not check_method_name in go_ignore_apis:
## Look up method_name in proto_map file, and return matching proto:
with open(proto_map_file, 'r') as f:
for row in f:
if not row.startswith('#') \
and row.startswith(resource + ',') \
and row.split(',')[4] == method_name:
this_method_dict["proto"] = row.split(',')[1]
## Extract the raw text from resource_interface matching method_name.
## Split by method span, throwing out remainder of span tag, catching cases where
## id is first attr or data-kind is first attr, and slicing to omit the first match,
## which is the opening of the method span tag, not needed:
this_method_raw1 = regex.split(r'id="' + tag_id + '"', str(resource_interface))[1].removeprefix('>').removeprefix(' data-kind="method">').lstrip()
## Then, omit all text that begins a new method span, and additionally remove trailing
## element closers for earlier tags we spliced into (pre and span):
this_method_raw2 = regex.split(r'<span .*data-kind="method".*>', this_method_raw1)[0].removesuffix('}</pre>\n</div>').removesuffix('</span>').rstrip()
method_description = ""
## Get method description, if any comment spans are found:
if tag.find('span', class_='comment'):
## Iterate through all comment spans, splitting by opening comment tag, and
## omitting the first string, which is either the opening comment tag itself,
## or the usage of this method, if the comment is appended to the end of usage line:
for comment in regex.split(r'<span class="comment">', this_method_raw2)[1:]:
comment_raw = regex.split(r'</span>.*', comment.removeprefix('//'))[0].lstrip()
method_description = method_description + comment_raw
## Write comment field as appended comments if found, or empty string if none.
this_method_dict["description"] = method_description
## Get full method usage string, by omitting all comment spans:
method_usage_raw = regex.sub(r'<span class="comment">.*</span>', '', this_method_raw2)
method_usage_raw2 = regex.sub(r'</span>', '', method_usage_raw).replace("\t", " ").lstrip().rstrip()
## Some Go params use versioned links, some omit the version (to use latest).
## Standardize on using latest for all cases. This handles parameters and returns:
this_method_dict["usage"] = regex.sub(r'/rdk@v[0-9\.]*/', '/rdk/', method_usage_raw2, flags=regex.DOTALL)
## Not possible to link to the specific functions, so we link to the parent resource instead.
## If we are scraping from a local staging instance, replace host and port with upstream link target URL:
if go_staging_url != '':
this_method_dict["method_link"] = str(url + '#' + interface_name).replace(go_staging_url, 'https://pkg.go.dev')
else:
this_method_dict["method_link"] = url + '#' + interface_name
## Check for code sample for this method.
go_code_samples_raw = soup.find_all(
lambda code_sample_tag: code_sample_tag.name == 'p'
and code_sample_tag.text.startswith(method_name)
and code_sample_tag.text.endswith(method_name + ' example:\n'))
## Determine if a code sample is provided for this method:
if len(go_code_samples_raw) == 1:
## Fetch code sample raw text, preserving newlines but stripping all formatting.
## This string should be suitable for feeding into any python formatter to get proper form:
this_method_dict["code_sample"] = go_code_samples_raw[0].find_next('pre').text.replace("\t", " ")
elif len(go_code_samples_raw) > 1:
## In case we want to support multiple code samples per method down the line,
## this is where to process (and: update write_markdown() accordingly to enable looping
## through possible code sample data objects). For now we just continue to fetch just the
## first-discovered (i.e., at index [0]):
this_method_dict["code_sample"] = go_code_samples_raw[0].find_next('pre').text.replace("\t", " ")
## We have finished collecting all data for this method. Write the this_method_dict dictionary
## in its entirety to the go_methods dictionary by type (like 'component'), by resource (like 'arm'),
## using the method_name as key:
go_methods[type][resource][method_name] = this_method_dict
## If this Go interface inherits from another interface, also fetch data for those inherited methods:
if '\tresource.' in resource_interface.text:
resource_url = f"{scrape_url}/go.viam.com/rdk/resource"
resource_soup = make_soup(resource_url)
## If the resource being considered inherits from resource.Resource (currently all components and services do,
## and no app or robot interfaces do), then add the three inherited methods manually: Reconfigure(), DoCommand(), Close()
if '\tresource.Resource' in resource_interface.text:
go_methods[type][resource]['Reconfigure'] = {'proto': 'Reconfigure', \
'description': 'Reconfigure must reconfigure the resource atomically and in place. If this cannot be guaranteed, then usage of AlwaysRebuild or TriviallyReconfigurable is permissible.', \
'usage': 'Reconfigure(ctx <a href="/context">context</a>.<a href="/context#Context">Context</a>, deps <a href="#Dependencies">Dependencies</a>, conf <a href="#Config">Config</a>) <a href="/builtin#error">error</a>', \
'method_link': 'https://pkg.go.dev/go.viam.com/rdk/resource#Resource'}
code_sample = resource_soup.find_all(lambda code_sample_tag: code_sample_tag.name == 'p' and "Reconfigure example:" in code_sample_tag.text)
if code_sample:
go_methods[type][resource]['Reconfigure']['code_sample'] = code_sample[0].find_next('pre').text.replace("\t", " ")
go_methods[type][resource]['DoCommand'] = {'proto': 'DoCommand', \
'description': 'DoCommand sends/receives arbitrary data.', \
'usage': 'DoCommand(ctx <a href="/context">context</a>.<a href="/context#Context">Context</a>, cmd map[<a href="/builtin#string">string</a>]interface{}) (map[<a href="/builtin#string">string</a>]interface{}, <a href="/builtin#error">error</a>)', \
'method_link': 'https://pkg.go.dev/go.viam.com/rdk/resource#Resource'}
code_sample = resource_soup.find_all(lambda code_sample_tag: code_sample_tag.name == 'p' and "DoCommand example:" in code_sample_tag.text)
if code_sample:
if type == "component":
go_methods[type][resource]['DoCommand']['code_sample'] = 'my' + resource.title().replace("_", "")+ ', err := ' + resource + '.FromRobot(machine, "my_' + resource + '")\n\ncommand := map[string]interface{}{"cmd": "test", "data1": 500}\nresult, err := my' + resource.title().replace("_", "") + '.DoCommand(context.Background(), command)\n'
if resource == "generic_component":
go_methods[type][resource]['DoCommand']['code_sample'] = 'myGenericComponent, err := generic.FromRobot(machine, "my_generic_component")\n\ncommand := map[string]interface{}{"cmd": "test", "data1": 500}\nresult, err := myGenericComponent.DoCommand(context.Background(), command)\n'
else:
go_methods[type][resource]['DoCommand']['code_sample'] = 'my' + resource.title().replace("_", "")+ 'Svc, err := ' + resource.replace("_","") + '.FromRobot(machine, "my_' + resource + '_svc")\n\ncommand := map[string]interface{}{"cmd": "test", "data1": 500}\nresult, err := my' + resource.title().replace("_", "") + 'Svc.DoCommand(context.Background(), command)\n'
if resource == "slam":
go_methods[type][resource]['DoCommand']['code_sample'] = 'mySLAMService, err := slam.FromRobot(machine, "my_slam_svc")\n\ncommand := map[string]interface{}{"cmd": "test", "data1": 500}\nresult, err := mySLAMService.DoCommand(context.Background(), command)\n'
go_methods[type][resource]['Close'] = {'proto': 'Close', \
'description': 'Close must safely shut down the resource and prevent further use. Close must be idempotent. Later reconfiguration may allow a resource to be "open" again.', \
'usage': 'Close(ctx <a href="/context">context</a>.<a href="/context#Context">Context</a>) <a href="/builtin#error">error</a>', \
'method_link': 'https://pkg.go.dev/go.viam.com/rdk/resource#Resource'}
code_sample = resource_soup.find_all(lambda code_sample_tag: code_sample_tag.name == 'p' and "Close example:" in code_sample_tag.text)
if code_sample:
if type == "component":
go_methods[type][resource]['Close']['code_sample'] = 'my' + resource.title().replace("_", "") + ', err := ' + go_resource_overrides.get(resource, resource) + '.FromRobot(machine, "my_' + resource + '")\n\nerr = my' + resource.title().replace("_", "") + '.Close(context.Background())\n'
else:
if resource == "base_remote_control":
go_methods[type][resource]['Close']['code_sample'] = 'baseRCService, err := baseremotecontrol.FromRobot(machine, "my_baseRCService_svc")\n\nerr := baseRCService.Close(context.Background())\n'
elif resource == "data_manager":
go_methods[type][resource]['Close']['code_sample'] = 'data, err := datamanager.FromRobot(machine, "my_data_manager")\n\nerr := data.Close(context.Background())\n'
elif resource == "navigation":
go_methods[type][resource]['Close']['code_sample'] = 'my_nav, err := navigation.FromRobot(machine, "my_nav_svc")\n\nerr := my_nav.Close(context.Background())\n'
elif resource == "mlmodel":
go_methods[type][resource]['Close']['code_sample'] = 'my_mlmodel, err := mlmodel.FromRobot(machine, "my_ml_model")\n\nerr := my_mlmodel.Close(context.Background())\n'
else:
go_methods[type][resource]['Close']['code_sample'] = 'my' + resource.title().replace("_", "") + 'Svc, err := ' + resource + '.FromRobot(machine, "my_' + resource + '_svc")\n\nerr = my' + resource.title().replace("_", "") + 'Svc.Close(context.Background())\n'
## Similarly, if the resource being considered inherits from resource.Actuator (Servo, for example),
## then add the two inherited methods manually: IsMoving() and Stop():
if '\tresource.Actuator' in resource_interface.text:
go_methods[type][resource]['IsMoving'] = {'proto': 'IsMoving', \
'description': 'IsMoving returns whether the resource is moving or not', \
'usage': 'IsMoving(ctx <a href="/context">context</a>.<a href="/context#Context">Context</a>) (<a href="/builtin#bool">bool</a>, <a href="/builtin#error">error</a>)', \
'method_link': 'https://pkg.go.dev/go.viam.com/rdk/resource#Actuator'}
code_sample = resource_soup.find_all(lambda code_sample_tag: code_sample_tag.name == 'p' and "IsMoving example:" in code_sample_tag.text)
if code_sample:
go_methods[type][resource]['IsMoving']['code_sample'] = code_sample[0].find_next('pre').text.replace("\t", " ")
go_methods[type][resource]['Stop'] = {'proto': 'Stop', \
'description': 'Stop stops all movement for the resource', \
'usage': 'Stop(ctx <a href="/context">context</a>.<a href="/context#Context">Context</a>, extra map[<a href="/builtin#string">string</a>]interface{}) <a href="/builtin#error">error</a>', \
'method_link': 'https://pkg.go.dev/go.viam.com/rdk/resource#Actuator'}
code_sample = resource_soup.find_all(lambda code_sample_tag: code_sample_tag.name == 'p' and "Stop example:" in code_sample_tag.text)
if code_sample:
go_methods[type][resource]['Stop']['code_sample'] = code_sample[0].find_next('pre').text.replace("\t", " ")
## Similarly, if the resource being considered inherits from resource.Shaped (Base, for example),
## then add the one inherited method manually: Geometries():
if '\tresource.Shaped' in resource_interface.text:
go_methods[type][resource]['Geometries'] = {'proto': 'GetGeometries', \
'description': 'Geometries returns the list of geometries associated with the resource, in any order. The poses of the geometries reflect their current location relative to the frame of the resource.', \
'usage': 'Geometries(ctx <a href="/context">context</a>.<a href="/context#Context">Context</a>, extra map[<a href="/builtin#string">string</a>]interface{}) ([]<a href="/go.viam.com/rdk/spatialmath">spatialmath</a>.<a href="/go.viam.com/rdk/spatialmath#Geometry">Geometry</a>, <a href="/builtin#error">error</a>)', \
'method_link': 'https://pkg.go.dev/go.viam.com/rdk/resource#Shaped'}
code_sample = resource_soup.find_all(lambda code_sample_tag: code_sample_tag.name == 'p' and "Geometries example:" in code_sample_tag.text)
if code_sample:
go_methods[type][resource]['Geometries']['code_sample'] = code_sample[0].find_next('pre').text.replace("\t", " ").replace('myArm', 'my{}'.format(resource.title().replace('_', ''))).replace('my_arm', 'my_{}'.format(resource)).replace('arm', resource)
## Similarly, if the resource being considered inherits from resource.Sensor (Movement Sensor, for example),
## then add the one inherited method manually: Readings():
if '\tresource.Sensor' in resource_interface.text:
go_methods[type][resource]['Readings'] = {'proto': 'GetReadings', \
'description': 'Readings return data specific to the type of sensor and can be of any type.', \
'usage': 'Readings(ctx <a href="/context">context</a>.<a href="/context#Context">Context</a>, extra map[<a href="/builtin#string">string</a>]interface{}) (map[<a href="/builtin#string">string</a>]interface{}, <a href="/builtin#error">error</a>)', \
'method_link': 'https://pkg.go.dev/go.viam.com/rdk/resource#Sensor'}
code_sample = resource_soup.find_all(lambda code_sample_tag: code_sample_tag.name == 'p' and "Readings example:" in code_sample_tag.text)
if code_sample:
go_methods[type][resource]['Readings']['code_sample'] = code_sample[0].find_next('pre').text.replace("\t", " ")
## For SLAM service only, additionally fetch data for two helper methods defined outside of the resource's interface:
if resource == 'slam':
## Fetch PointCloudMapFull:
pointcloudmapfull_method_raw = soup.find_all(
lambda tag: tag.name == 'div'
and tag.get('class') == ['Documentation-declaration']
and "PointCloudMapFull" in tag.pre.text)
go_methods[type][resource]['PointCloudMapFull'] = {}
go_methods[type][resource]['PointCloudMapFull']['proto'] = 'PointCloudMapFull'
go_methods[type][resource]['PointCloudMapFull']['description'] = pointcloudmapfull_method_raw[0].pre.find_next('p').text
go_methods[type][resource]['PointCloudMapFull']['usage'] = str(pointcloudmapfull_method_raw[0].pre).removeprefix('<pre>func ').removesuffix('</pre>')
go_methods[type][resource]['PointCloudMapFull']['method_link'] = 'https://pkg.go.dev/go.viam.com/rdk/services/slam#PointCloudMapFull'
## Fetch InternalStateFull:
internalstatefull_method_raw = soup.find_all(
lambda tag: tag.name == 'div'
and tag.get('class') == ['Documentation-declaration']
and "InternalStateFull" in tag.pre.text)
go_methods[type][resource]['InternalStateFull'] = {}
go_methods[type][resource]['InternalStateFull']['proto'] = 'InternalStateFull'
go_methods[type][resource]['InternalStateFull']['description'] = internalstatefull_method_raw[0].pre.find_next('p').text
go_methods[type][resource]['InternalStateFull']['usage'] = str(internalstatefull_method_raw[0].pre).removeprefix('<pre>func ').removesuffix('</pre>')
go_methods[type][resource]['InternalStateFull']['method_link'] = 'https://pkg.go.dev/go.viam.com/rdk/services/slam#InternalStateFull'
## We have finished looping through all scraped Go methods. Write the go_methods dictionary
## in its entirety to the all_methods dictionary using "go" as the key:
all_methods["go"] = go_methods
## Assemble workaround data object for DoCommand for Go generic component and service.
## Using code sample and method_link from resource.Resource, because these cannot be found
## in Go docs for these resources:
elif sdk == "go" and (resource == "generic_component" or resource == "generic_service"):
go_methods[type][resource]['DoCommand'] = {}
go_methods[type][resource]['DoCommand'] = {'proto': 'DoCommand', \
'description': 'DoCommand sends/receives arbitrary data.', \
'usage': 'DoCommand(ctx <a href="/context">context</a>.<a href="/context#Context">Context</a>, cmd map[<a href="/builtin#string">string</a>]interface{}) (map[<a href="/builtin#string">string</a>]interface{}, <a href="/builtin#error">error</a>)', \
'method_link': 'https://pkg.go.dev/go.viam.com/rdk/resource#Resource', \
'code_sample': 'my' + resource.title().replace("_", "") + ', err := generic.FromRobot(machine, "my_' + resource.lower() + '")\n\ncommand := map[string]interface{}{"cmd": "test", "data1": 500}\nresult, err := my' + resource.title().replace("_", "") + '.DoCommand(context.Background(), command)\n'}
if resource == "generic_service":
go_methods[type][resource]['DoCommand']['code_sample'] = 'myGenericService, err := generic.FromRobot(machine, "my_generic_service")\n\ncommand := map[string]interface{}{"cmd": "test", "data1": 500}\nresult, err := myGenericService.DoCommand(context.Background(), command)\n'
all_methods["go"] = go_methods
elif sdk == "go" and type == "app":
##Go SDK has no APP API!
pass
## Scrape each parent method tag and all contained child tags for Python by resource.
## TEMP: Manually exclude Base Remote Control Service (Go only) and Data Manager Service (Go + Flutter only).
## TODO: Handle resources with 0 implemented methods for this SDK better.
elif sdk == "python" and resource != 'base_remote_control' and resource != 'data_manager':
soup = make_soup(url)
python_methods_raw = soup.find_all("dl", class_="py method")
## Loop through scraped tags and select salient data:
for tag in python_methods_raw:
## Create new empty dictionary for this specific method, to be appended to ongoing python_methods dictionary,
## in form: python_methods[type][resource][method_name] = this_method_dict
this_method_dict = {}
id = tag.find("dt", class_="sig sig-object py").get("id")
if not id.endswith(".get_operation") \
and not id.endswith(".from_proto") and not id.endswith(".to_proto") \
and not id.endswith(".from_string") and not id.endswith("__") \
and not id.endswith("HasField") and not id.endswith("WhichOneof") \
and not id in python_ignore_apis:
## Determine method name, but don't save to dictionary as value; we use it as a key instead:
method_name = id.rsplit('.',1)[1]