Merge pull request #1143 from OpenC3/target_file

Fix download_file and get_target_file

docs.openc3.com/docs/guides/scripting-api.md

@@ -3235,15 +3235,15 @@ local_screen("TESTING", screen_def, 600, 75)
 Python Example:
 
 ```python
-screen_def = '
+screen_def = """
   SCREEN AUTO AUTO 0.1 FIXED
   VERTICAL
     TITLE "Local Screen"
     VERTICALBOX
       LABELVALUE INST HEALTH_STATUS TEMP1
     END
   END
-'
+"""
 # Here we pass in the screen definition as a string
 local_screen("TESTING", screen_def, 600, 75)
 ```

openc3-cosmos-cmd-tlm-api/app/controllers/storage_controller.rb

@@ -90,10 +90,15 @@ def exists
     raise "Unknown bucket #{params[:bucket]}" unless bucket_name
     path = sanitize_path(params[:object_id])
     bucket = OpenC3::Bucket.getClient()
+    # Returns true or false if the object is found
     result = bucket.check_object(bucket: bucket_name,
                                  key: path,
                                  retries: false)
-    render :json => result, :status => 201
+    if result
+      render :json => result, :status => 200
+    else
+      render :json => result, :status => 404
+    end
   rescue Exception => e
     OpenC3::Logger.error("File exists request failed: #{e.message}", user: username())
     render :json => { status: 'error', message: e.message }, status: 500

openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST/procedures/target_file.rb

@@ -1,9 +1,7 @@
 put_target_file("INST/test.txt", "this is a string test")
-download_file("INST/test.txt") # download via path
+download_file("INST/test.txt")
 file = get_target_file("INST/test.txt")
 puts file.read
-file.rewind # rewind so download_file can read
-download_file(file) # download using file
 file.delete
 delete_target_file("INST/test.txt")
 
@@ -18,6 +16,7 @@
 delete_target_file("INST/test.txt")
 
 put_target_file("INST/test.bin", "\x00\x01\x02\x03\xFF\xEE\xDD\xCC")
+download_file("INST/test.bin")
 file = get_target_file("INST/test.bin")
 puts file.read.formatted
 file.delete

openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST2/procedures/target_file.py

@@ -2,11 +2,9 @@
 import tempfile
 
 put_target_file("INST/test.txt", "this is a string test")
-download_file("INST/test.txt")  # download via path
+download_file("INST/test.txt")
 file = get_target_file("INST/test.txt")
 print(file.read())
-file.seek(0)  # rewind so download_file can read
-download_file(file)  # download using file
 file.close()  # closing deletes tempfile
 delete_target_file("INST/test.txt")
 
@@ -20,9 +18,9 @@
 file.close()
 delete_target_file("INST/test.txt")
 
-# TODO: Binary not yet supported
-# put_target_file("INST/test.bin", b"\x00\x01\x02\x03\xFF\xEE\xDD\xCC")
-# file = get_target_file("INST/test.bin")
-# print(formatted(file.read()))
-# file.close()
-# delete_target_file("INST/test.bin")
+put_target_file("INST/test.bin", b"\x00\x01\x02\x03\xFF\xEE\xDD\xCC")
+download_file("INST/test.bin")
+file = get_target_file("INST/test.bin")
+print(formatted(file.read()))
+file.close()
+delete_target_file("INST/test.bin")

openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST2/tables/procedures/download.py

@@ -9,10 +9,9 @@
 # i = 1
 # num_segments = 5 # calculate based on TBL_FILENAME
 # table_id = 1  # calculate based on TBL_FILENAME
-# while i < num_segments
+# while i < num_segments:
 #   # Request a part of the table buffer
-#   cmd("TGT DUMP with TABLE_ID #{table_id}, SEGMENT #{i}")
+#   cmd(f"TGT DUMP with TABLE_ID {table_id}, SEGMENT {i}")
 #   buffer += tlm("TGT DUMP_PKT DATA")
 #   i += 1
-# end
 put_target_file(os.getenv("TBL_FILENAME"), buffer)

openc3-cosmos-init/plugins/packages/openc3-cosmos-tool-scriptrunner/src/tools/ScriptRunner/ScriptRunner.vue

@@ -1653,12 +1653,9 @@ export default {
             this.screens = []
             break
           case 'downloadfile':
-            const blob = new Blob([data.text], {
-              type: 'text/plain',
-            })
             // Make a link and then 'click' on it to start the download
             const link = document.createElement('a')
-            link.href = URL.createObjectURL(blob)
+            link.href = window.location.origin + data.url
             link.setAttribute('download', data.filename)
             link.click()
             break

openc3-cosmos-init/plugins/packages/openc3-tool-common/src/components/widgets/ImageLoader.js

@@ -32,22 +32,27 @@ export default {
   methods: {
     getPresignedUrl: async function (fileName) {
       var targets = 'targets_modified'
-      // exists returns true / false
-      var response = await Api.get(
+      await Api.get(
         `/openc3-api/storage/exists/${encodeURIComponent(
-          `${window.openc3Scope}/${targets}/${this.target}/public/${fileName}`,
+          `${window.openc3Scope}/${targets}/${this.target}/public/${fileName}`
         )}?bucket=OPENC3_CONFIG_BUCKET`,
-      )
-      // If response was false then 'targets_modified' doesn't exist
-      // so switch to 'targets' and then just try to get the URL
-      // If the file doesn't exist it will throw a 404 when it is actually retrieved
-      if (response.data === false) {
+        {
+          headers: {
+            Accept: 'application/json',
+            // Since we're just checking for existance, 404 is possible so ignore it
+            'Ignore-Errors': '404',
+          },
+        }
+      ).catch((error) => {
+        // If response fails then 'targets_modified' doesn't exist
+        // so switch to 'targets' and then just try to get the URL
+        // If the file doesn't exist it will throw a 404 when it is actually retrieved
         targets = 'targets'
-      }
-      response = await Api.get(
+      })
+      var response = await Api.get(
         `/openc3-api/storage/download/${encodeURIComponent(
-          `${window.openc3Scope}/${targets}/${this.target}/public/${fileName}`,
-        )}?bucket=OPENC3_CONFIG_BUCKET`,
+          `${window.openc3Scope}/${targets}/${this.target}/public/${fileName}`
+        )}?bucket=OPENC3_CONFIG_BUCKET`
       )
       return response.data.url
     },

openc3-cosmos-script-runner-api/app/models/running_script.rb

@@ -166,7 +166,7 @@ def openc3_script_sleep(sleep_time = nil)
         return false
       end
 
-      def display_screen(target_name, screen_name, x = nil, y = nil, scope: $openc3_scope)
+      def display_screen(target_name, screen_name, x = nil, y = nil, scope: RunningScript.instance.scope)
         definition = get_screen_definition(target_name, screen_name, scope: scope)
         OpenC3::Store.publish(["script-api", "running-script-channel:#{RunningScript.instance.id}"].compact.join(":"), JSON.generate({ type: :screen, target_name: target_name, screen_name: screen_name, definition: definition, x: x, y: y }))
       end
@@ -183,15 +183,9 @@ def local_screen(screen_name, definition, x = nil, y = nil)
         OpenC3::Store.publish(["script-api", "running-script-channel:#{RunningScript.instance.id}"].compact.join(":"), JSON.generate({ type: :screen, target_name: "LOCAL", screen_name: screen_name, definition: definition, x: x, y: y }))
       end
 
-      def download_file(file_or_path)
-        if file_or_path.respond_to? :read
-          data = file_or_path.read
-          filename = File.basename(file_or_path.filename)
-        else # path
-          data = ::Script.body(RunningScript.instance.scope, file_or_path)
-          filename = File.basename(file_or_path)
-        end
-        OpenC3::Store.publish(["script-api", "running-script-channel:#{RunningScript.instance.id}"].compact.join(":"), JSON.generate({ type: :downloadfile, filename: filename, text: data.to_utf8 }))
+      def download_file(path, scope: RunningScript.instance.scope)
+        url = get_download_url(path, scope: scope)
+        OpenC3::Store.publish(["script-api", "running-script-channel:#{RunningScript.instance.id}"].compact.join(":"), JSON.generate({ type: :downloadfile, filename: File.basename(path), url: url }))
       end
     end
   end

openc3-cosmos-script-runner-api/scripts/run_script.py

@@ -123,7 +123,7 @@ def run_script_log(id, message, color="BLACK", message_log=True):
     # (defined in cable.yml) and then the channel stream. This isn't typically how you see these
     # topics used in the Rails ActionCable documentation but this is what is happening under the
     # scenes in ActionCable. Throughout the rest of the code we use ActionCable to broadcast
-    #   e.g. ActionCable.server.broadcast("running-script-channel:#{@id}", ...)
+    #   e.g. ActionCable.server.broadcast("running-script-channel:{@id}", ...)
     redis = Store.instance().build_redis()
     p = redis.pubsub(ignore_subscribe_messages=True)
     p.subscribe(f"script-api:cmd-running-script-channel:{id}")
@@ -151,7 +151,18 @@ def run_script_log(id, message, color="BLACK", message_log=True):
                 if type(parsed_cmd) is dict and "method" in parsed_cmd:
                     match parsed_cmd["method"]:
                         # This list matches the list in running_script.py:113
-                        case "ask" | "ask_string" | "message_box" | "vertical_message_box" | "combo_box" | "prompt" | "prompt_for_hazardous" | "metadata_input" | "open_file_dialog" | "open_files_dialog":
+                        case (
+                            "ask"
+                            | "ask_string"
+                            | "message_box"
+                            | "vertical_message_box"
+                            | "combo_box"
+                            | "prompt"
+                            | "prompt_for_hazardous"
+                            | "metadata_input"
+                            | "open_file_dialog"
+                            | "open_files_dialog"
+                        ):
                             if running_script.prompt_id != None:
                                 if (
                                     "prompt_id" in parsed_cmd

openc3-cosmos-script-runner-api/scripts/running_script.py

@@ -1405,25 +1405,13 @@ def local_screen(screen_name, definition, x=None, y=None):
 setattr(openc3.script, "local_screen", local_screen)
 
 
-def download_file(file_or_path):
-    if hasattr(file_or_path, "read") and callable(file_or_path.read):
-        data = file_or_path.read()
-        if hasattr(file_or_path, "name") and callable(file_or_path.name):
-            filename = os.path.basename(file_or_path.name)
-        else:
-            filename = "unnamed_file.bin"
-    else:  # path
-        data = TargetFile.body(RunningScript.instance.scope, file_or_path)
-        if not data:
-            raise RuntimeError(
-                f"Unable to retrieve: {file_or_path} in scope {RunningScript.instance.scope}"
-            )
-        else:
-            data = data.decode()
-        filename = os.path.basename(file_or_path)
+def download_file(path, scope=OPENC3_SCOPE):
+    url = openc3.script.get_download_url(path, scope=scope)
     Store.publish(
-        f"script-api:running-script-channel:#{RunningScript.instance.id}",
-        json.dumps({"type": "downloadfile", "filename": filename, "text": data}),
+        f"script-api:running-script-channel:{RunningScript.instance.id}",
+        json.dumps(
+            {"type": "downloadfile", "filename": os.path.basename(path), "url": url}
+        ),
     )
 
 

openc3/lib/openc3/script/storage.rb

@@ -43,8 +43,8 @@ def delete_target_file(path, scope: $openc3_scope)
         if response.nil? || response.status != 200
           raise "Failed to delete #{delete_path}"
         end
-      rescue => error
-        raise "Failed deleting #{path} due to #{error.message}"
+      rescue => e
+        raise "Failed deleting #{path} due to #{e.message}"
       end
       nil
     end
@@ -84,8 +84,8 @@ def put_target_file(path, io_or_string, scope: $openc3_scope)
             end
           end
         end
-      rescue => error
-        raise "Failed to write #{upload_path} due to #{error.message}"
+      rescue => e
+        raise "Failed to write #{upload_path} due to #{e.message}"
       end
       nil
     end
@@ -115,19 +115,33 @@ def get_target_file(path, original: false, scope: $openc3_scope)
           end
 
           return _get_storage_file("#{part}/#{path}", scope: scope)
-        rescue => error
+        rescue => e
           if part == "targets_modified"
             part = "targets"
             redo
           else
-            raise error
+            raise e
           end
         end
         break
       end
     end
 
-    # download_file(path_or_file) is implemented by running_script to download a file
+    def get_download_url(path, scope: $openc3_scope)
+      targets = "targets_modified" # First try targets_modified
+      response = $api_server.request('get', "/openc3-api/storage/exists/#{scope}/#{targets}/#{path}", query: { bucket: 'OPENC3_CONFIG_BUCKET' }, scope: scope)
+      if response.status != 200
+        targets = "targets" # Next try targets
+        response = $api_server.request('get', "/openc3-api/storage/exists/#{scope}/#{targets}/#{path}", query: { bucket: 'OPENC3_CONFIG_BUCKET' }, scope: scope)
+        if response.status != 200
+          raise "File not found: #{path} in scope: #{scope}"
+        end
+      end
+      endpoint = "/openc3-api/storage/download/#{scope}/#{targets}/#{path}"
+      # external must be true because we're using this URL from the frontend
+      result = _get_presigned_request(endpoint, external: true, scope: scope)
+      return result['url']
+    end
 
     # These are helper methods ... should not be used directly
 
@@ -161,7 +175,9 @@ def _get_uri(url)
       if $openc3_in_cluster
         case ENV['OPENC3_CLOUD']
         when 'local'
-          URI.parse("http://openc3-minio:9000" + url)
+          bucket_url = ENV["OPENC3_BUCKET_URL"] || "openc3-minio:9000"
+          # TODO: Bucket schema for http vs https
+          URI.parse("http://#{bucket_url}#{url}")
         when 'aws'
           URI.parse("https://s3.#{ENV['AWS_REGION']}.amazonaws.com" + url)
         when 'gcp'
@@ -175,11 +191,11 @@ def _get_uri(url)
       end
     end
 
-    def _get_presigned_request(endpoint, scope: $openc3_scope)
-      if $openc3_in_cluster
-        response = $api_server.request('get', endpoint, query: { bucket: 'OPENC3_CONFIG_BUCKET', internal: true }, scope: scope)
-      else
+    def _get_presigned_request(endpoint, external: nil, scope: $openc3_scope)
+      if external or !$openc3_in_cluster
         response = $api_server.request('get', endpoint, query: { bucket: 'OPENC3_CONFIG_BUCKET' }, scope: scope)
+      else
+        response = $api_server.request('get', endpoint, query: { bucket: 'OPENC3_CONFIG_BUCKET', internal: true }, scope: scope)
       end
       if response.nil? || response.status != 201
         raise "Failed to get presigned URL for #{endpoint}"

openc3/python/openc3/microservices/microservice.py

@@ -125,7 +125,7 @@ def __init__(self, name, is_plugin=False):
         self.logger.info(f"Microservice initialized with config:\n{self.config}")
         if not hasattr(self, "topics") or self.topics is None:
             self.topics = []
-        self.microservice_topic = f"MICROSERVICE__#{self.name}"
+        self.microservice_topic = f"MICROSERVICE__{self.name}"
 
         # Get configuration for any targets
         self.target_names = self.config.get("target_names")
@@ -230,7 +230,7 @@ def microservice_cmd(self, topic, msg_id, msg_hash, _):
                         self.topics.append(new_topic)
             else:
                 raise RuntimeError(
-                    f"Invalid topics given to microservice_cmd: #{topics}"
+                    f"Invalid topics given to microservice_cmd: {topics}"
                 )
             Topic.trim_topic(topic, msg_id)
             return True

openc3/python/openc3/packets/packet.py

@@ -1125,7 +1125,7 @@ def from_json(cls, hash):
                     packet.accessor = accessor()
             except RuntimeError as error:
                 Logger.error(
-                    f"#{packet.target_name} #{packet.packet_name} accessor of #{hash['accessor']} could not be found due to #{repr(error)}"
+                    f"{packet.target_name} {packet.packet_name} accessor of {hash['accessor']} could not be found due to {repr(error)}"
                 )
         if "template" in hash:
             packet.template = base64.b64decode(hash["template"])