Draft: Add repository directory to path when importing files from repository

artiq/browser/experiments.py

@@ -486,7 +486,7 @@ def initialize_submission_arguments(self, arginfo):
     async def examine(self, file):
         worker = Worker(self.worker_handlers)
         try:
-            return await worker.examine("examine", file)
+            return await worker.examine("examine", file, self.current_dir)
         finally:
             await worker.close()
 

artiq/dashboard/experiments.py

@@ -602,18 +602,26 @@ def get_submission_scheduling(self, expurl):
             self.submission_scheduling[expurl] = scheduling
             return scheduling
 
+    def initialize_submission_options(self, expurl, repo_dir):
+        # mutated by _ExperimentDock
+        options = {
+            "log_level": logging.WARNING
+        }
+        if expurl[:5] == "repo:":
+            options["repo_rev"] = None
+        elif repo_dir:
+            options["repo_dir"] = repo_dir
+        self.submission_options[expurl] = options
+        return options
+
     def get_submission_options(self, expurl):
         if expurl in self.submission_options:
             return self.submission_options[expurl]
         else:
-            # mutated by _ExperimentDock
-            options = {
-                "log_level": logging.WARNING
-            }
-            if expurl[:5] == "repo:":
-                options["repo_rev"] = None
-            self.submission_options[expurl] = options
-            return options
+            if expurl[:5] != "repo:":
+                raise ValueError("Submission options must be preinitialized "
+                                 "when not using repository")
+            return self.initialize_submission_options(expurl, None)
 
     def initialize_submission_arguments(self, expurl, arginfo):
         arguments = OrderedDict()
@@ -696,6 +704,8 @@ def submit(self, expurl):
         }
         if "repo_rev" in options:
             expid["repo_rev"] = options["repo_rev"]
+        elif "repo_dir" in options:
+            expid["repo_dir"] = options["repo_dir"]
         asyncio.ensure_future(self._submit_task(
             expurl,
             scheduling["pipeline_name"],
@@ -733,19 +743,22 @@ def request_inst_term(self, expurl):
 
     async def compute_expdesc(self, expurl):
         file, class_name, use_repository = self.resolve_expurl(expurl)
+        options = self.get_submission_options(expurl)
         if use_repository:
-            revision = self.get_submission_options(expurl)["repo_rev"]
+            revision = options["repo_rev"]
         else:
             revision = None
+        repo_dir = options.get("repo_dir")
         description = await self.experiment_db_ctl.examine(
-            file, use_repository, revision)
+            file, use_repository=use_repository, revision=revision, wd=repo_dir)
         return description[class_name]
 
-    async def open_file(self, file):
-        description = await self.experiment_db_ctl.examine(file, False)
+    async def open_file(self, file, repo_dir):
+        description = await self.experiment_db_ctl.examine(file, use_repository=False, wd=repo_dir)
         for class_name, class_desc in description.items():
             expurl = "file:{}@{}".format(class_name, file)
             self.initialize_submission_arguments(expurl, class_desc["arginfo"])
+            self.initialize_submission_options(expurl, repo_dir)
             if expurl in self.open_experiments:
                 self.open_experiments[expurl].close()
             self.open_experiment(expurl)

artiq/dashboard/explorer.py

@@ -36,9 +36,16 @@ def __init__(self, explorer, exp_manager, experiment_db_ctl):
         grid.addWidget(self.file_list, 1, 0, 1, 2)
         self.file_list.doubleClicked.connect(self.accept)
 
+        repo_dir_label = QtWidgets.QLabel("\"Repository\" directory:")
+        grid.addWidget(repo_dir_label, 2, 0, 1, 2)
+        self.repo_dir = QtWidgets.QLineEdit(self.location_label.text())
+        grid.addWidget(self.repo_dir, 3, 0, 1, 2)
+        # todo: validator to make sure wd is parent of file
+        #  or just completely different interface for choosing wd
+
         buttons = QtWidgets.QDialogButtonBox(
             QtWidgets.QDialogButtonBox.Ok | QtWidgets.QDialogButtonBox.Cancel)
-        grid.addWidget(buttons, 2, 0, 1, 2)
+        grid.addWidget(buttons, 4, 0, 1, 2)
         buttons.accepted.connect(self.accept)
         buttons.rejected.connect(self.reject)
 
@@ -49,6 +56,8 @@ async def refresh_view(self):
         else:
             self.location_label.setText(self.explorer.current_directory)
 
+        self.repo_dir.setText(self.location_label.text())
+
         item = QtWidgets.QListWidgetItem()
         item.setText("..")
         item.setIcon(QtWidgets.QApplication.style().standardIcon(
@@ -105,7 +114,7 @@ def accept(self):
                 file = self.explorer.current_directory + selected
                 async def open_task():
                     try:
-                        await self.exp_manager.open_file(file)
+                        await self.exp_manager.open_file(file, self.repo_dir.text())
                     except:
                         logger.error("Failed to open file '%s'",
                                      file, exc_info=True)

artiq/master/experiments.py

@@ -24,7 +24,7 @@ async def process_file(self, entry_dict, root, filename):
         logger.debug("processing file %s %s", root, filename)
         try:
             description = await self.worker.examine(
-                "scan", os.path.join(root, filename))
+                "scan", os.path.join(root, filename), root)
         except:
             log_worker_exception()
             raise
@@ -128,15 +128,15 @@ def scan_repository_async(self, new_cur_rev=None):
         asyncio.ensure_future(
             exc_to_warning(self.scan_repository(new_cur_rev)))
 
-    async def examine(self, filename, use_repository=True, revision=None):
+    async def examine(self, filename, use_repository=True, revision=None, wd=None):
         if use_repository:
             if revision is None:
                 revision = self.cur_rev
             wd, _ = self.repo_backend.request_rev(revision)
             filename = os.path.join(wd, filename)
         worker = Worker(self.worker_handlers)
         try:
-            description = await worker.examine("examine", filename)
+            description = await worker.examine("examine", filename, wd)
         finally:
             await worker.close()
         if use_repository:

artiq/master/scheduler.py

@@ -131,8 +131,13 @@ def submit(self, expid, priority, due_date, flush, pipeline_name):
                 expid["repo_rev"] = self.experiment_db.cur_rev
             wd, repo_msg = self.experiment_db.repo_backend.request_rev(
                 expid["repo_rev"])
+        elif "repo_dir" in expid:
+            wd = expid["repo_dir"]
+            repo_msg = None
         else:
             wd, repo_msg = None, None
+        # always expose wd/repo_dir via expid
+        expid["repo_dir"] = wd
         run = Run(rid, pipeline_name, wd, expid, priority, due_date, flush,
                   self, repo_msg=repo_msg)
         self.runs[rid] = run

artiq/master/worker.py

@@ -293,7 +293,7 @@ async def resume(self, request_termination):
     async def analyze(self):
         await self._worker_action({"action": "analyze"})
 
-    async def examine(self, rid, file, timeout=20.0):
+    async def examine(self, rid, file, wd, timeout=20.0):
         self.rid = rid
         self.filename = os.path.basename(file)
 
@@ -303,7 +303,7 @@ async def examine(self, rid, file, timeout=20.0):
         def register(class_name, name, arginfo, scheduler_defaults):
             r[class_name] = {"name": name, "arginfo": arginfo, "scheduler_defaults": scheduler_defaults}
         self.register_experiment = register
-        await self._worker_action({"action": "examine", "file": file},
+        await self._worker_action({"action": "examine", "file": file, "wd": wd},
                                   timeout)
         del self.register_experiment
         return r

artiq/master/worker_impl.py

@@ -129,8 +129,8 @@ class CCB:
     issue = staticmethod(make_parent_action("ccb_issue"))
 
 
-def get_experiment(file, class_name):
-    module = tools.file_import(file, prefix="artiq_worker_")
+def get_experiment(file, class_name, repository_path=None):
+    module = tools.file_import(file, prefix="artiq_worker_", repository_path=repository_path)
     return tools.get_experiment(module, class_name)
 
 
@@ -155,10 +155,10 @@ def update(self, mod):
         pass
 
 
-def examine(device_mgr, dataset_mgr, file):
+def examine(device_mgr, dataset_mgr, file, wd):
     previous_keys = set(sys.modules.keys())
     try:
-        module = tools.file_import(file)
+        module = tools.file_import(file, repository_path=wd)
         for class_name, exp_class in inspect.getmembers(module, is_public_experiment):
             if exp_class.__doc__ is None:
                 name = class_name
@@ -279,7 +279,7 @@ def write_results():
                     experiment_file = expid["file"]
                     repository_path = None
                 setup_diagnostics(experiment_file, repository_path)
-                exp = get_experiment(experiment_file, expid["class_name"])
+                exp = get_experiment(experiment_file, expid["class_name"], repository_path)
                 device_mgr.virtual_devices["scheduler"].set_run_info(
                     rid, obj["pipeline_name"], expid, obj["priority"])
                 start_local_time = time.localtime(start_time)
@@ -311,7 +311,7 @@ def write_results():
                 finally:
                     write_results()
             elif action == "examine":
-                examine(ExamineDeviceMgr, ExamineDatasetMgr, obj["file"])
+                examine(ExamineDeviceMgr, ExamineDatasetMgr, obj["file"], obj["wd"])
                 put_completed()
             elif action == "terminate":
                 break

artiq/tools.py

@@ -70,11 +70,18 @@ def short_format(v):
         return r
 
 
-def file_import(filename, prefix="file_import_"):
-    filename = pathlib.Path(filename)
+def file_import(filename, prefix="file_import_", repository_path=None):
+    filename = pathlib.Path(filename).resolve()
     modname = prefix + filename.stem
 
-    path = str(filename.resolve().parent)
+    if repository_path:
+        repository_path = pathlib.Path(repository_path).resolve()
+        relative_file_path = filename.relative_to(repository_path)
+        modname = '.'.join(relative_file_path.parts[:-1] + (modname,))
+        path = str(repository_path)
+    else:
+        path = str(filename.parent)
+
     sys.path.insert(0, path)
 
     try: