From c230ca213b6441100db3b77b43a68661fdd555d8 Mon Sep 17 00:00:00 2001 From: Aaruni Kaushik Date: Thu, 13 Nov 2025 13:53:21 +0100 Subject: [PATCH 1/3] quick and dirty implementation to export tar --- src/maps | 62 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/src/maps b/src/maps index 6eafb46..00359d5 100755 --- a/src/maps +++ b/src/maps @@ -108,6 +108,8 @@ def addCLI(): default=False, help="Force enable GUI") parser_runtime.add_argument('-e', '--export-url', dest='EXPORTURL', action='store', default=False, help="Export a runtime as a URL which maps can open") + parser_runtime.add_argument('--export-tar', dest='EXPORTTAR', action='store', + default=False, help="Export a runtime as a compressed tarball") # arguments for remote management parser_remote = subparser.add_parser("remote", @@ -1243,6 +1245,8 @@ def mode_runtime(repo, repopath, args): mode_url(repo, repopath, args) elif args.EXPORTURL: mode_export_url(repo, args) + elif args.EXPORTTAR: + export(repo, args.EXPORTTAR) def byteSI(inbytes): @@ -1329,6 +1333,8 @@ def needs_tar(refhash, tarpath, datadir): Otherwise, it returns true. """ tardbpath = f"{datadir}/tardb.toml" + if VERBOSE: + print(f"tardb path is {tardbpath}") if not os.path.isfile(tarpath): if VERBOSE: print("Tarfile doesn't already exist. Needs tar-ing!") @@ -1370,6 +1376,40 @@ def add_hash_to_db(refhash, tarpath, datadir): tardbfile.write(f'"{refhash}"="{tarhash}"\n') +def runtime_to_tar(repo, runtime, remote = None): + + if remote is None: + remote, runtime = disambiguate_runtime(repo, runtime, installed=False) + print(f"remote was disambiguated to {remote}") + + DATADIR = f"{os.getenv('HOME')}/.var/org.mardi.maps" + RUNTIMEDIR = f"{DATADIR}/{remote}/{runtime}/rofs" + REFHASH = repo.list_refs()[1][runtime] + TARPATH = f"{DATADIR}/{remote}/{':'.join(runtime.split('/'))}.tar.gz" + # if runtime is not checked out, do it + if not os.path.isdir(RUNTIMEDIR): + checkout(repo, remote, runtime) + + # if the refhash matches tar hash, don't re-tar + if needs_tar(REFHASH, TARPATH, DATADIR): + if VERBOSE: + print("Making tarball...") + opts = "-cv --use-compress-program=pigz" + else: + opts = "-c --use-compress-program=pigz" + + subprocess.run(f"tar {opts} -C {RUNTIMEDIR[0:-4]} -f {TARPATH} {RUNTIMEDIR[-4:]}".split(), check=True) + add_hash_to_db(REFHASH, TARPATH, DATADIR) + + return TARPATH + + +def export(repo, runtime): + tarpath = runtime_to_tar(repo, runtime) + print(f"Runtime exported to {tarpath}") + return + + def upload(repo, runtime): """ Given a local runtime, tar it and upload it. (Try) @@ -1393,27 +1433,11 @@ def upload(repo, runtime): print("We only allow publishing locally made runtimes!") sys.exit(1) - remote = "Local" + remote = "_local" DATADIR = f"{os.getenv('HOME')}/.var/org.mardi.maps" - RUNTIMEDIR = f"{DATADIR}/{remote}/{runtime}" - TARPATH = f"{DATADIR}/{remote}/{runtime}.tar.gz" STORAGEFILE = f"{DATADIR}/tustorage" - REFHASH = repo.list_refs()[1][runtime] - - # if runtime is not checked out, do it - if not os.path.isdir(RUNTIMEDIR): - checkout(repo, remote, runtime) - - # if the refhash matches tar hash, don't re-tar - if needs_tar(REFHASH, TARPATH, DATADIR): - if VERBOSE: - print("Making tarball...") - opts = "-cv --use-compress-program=pigz -f" - else: - opts = "-c --use-compress-program=pigz -f" - subprocess.run(f"tar {opts} {TARPATH} {RUNTIMEDIR}".split(), check=True) - add_hash_to_db(REFHASH, TARPATH, DATADIR) + tarpath = runtime_to_tar(repo, runtime, remote) # check storagefile good = 0 @@ -1437,7 +1461,7 @@ def upload(repo, runtime): elif good == 2: os.rename(STORAGEFILE, f"{STORAGEFILE}.bak") - if tus_upload(TARPATH, STORAGEFILE, runtime) != 0: + if tus_upload(tarpath, STORAGEFILE, runtime) != 0: print("something very bad happened") return -1 return 0 From b4d9794b67c69884f1c09a6bbbb49744401d92e0 Mon Sep 17 00:00:00 2001 From: Aaruni Kaushik Date: Thu, 13 Nov 2025 14:09:37 +0100 Subject: [PATCH 2/3] Beautify and lint --- pylint.toml | 2 +- src/maps | 78 ++++++++++++++++++++++++++++------------------------- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/pylint.toml b/pylint.toml index 993a23a..e1d09d6 100644 --- a/pylint.toml +++ b/pylint.toml @@ -298,7 +298,7 @@ indent-string = " " max-line-length = 100 # Maximum number of lines in a module. -max-module-lines = 1500 +max-module-lines = 2000 # Allow the body of a class to be on the same line as the declaration if body # contains single statement. diff --git a/src/maps b/src/maps index 00359d5..3654e55 100755 --- a/src/maps +++ b/src/maps @@ -172,8 +172,7 @@ def create_config_file(config_path): if os.getenv("MAPS_NOTELE") is not None and os.getenv("MAPS_NOTELE") != "": telemetry_consent = 'n' elif ( - os.getenv("MAPS_TELEMETRY_CONSENT") is not None - and os.getenv("MAPS_TELEMETRY_CONSENT") != "" + os.getenv("MAPS_TELEMETRY_CONSENT", "") != "" ): telemetry_consent = 'y' else: @@ -252,10 +251,10 @@ def program_init(repopath): except subprocess.CalledProcessError: print( f"{COLORS['FAIL']}mkdir failed. This can happen if {COLORS['WARNING']}" - + f"{repopath}{COLORS['FAIL']} is not a directory, for example, if it " - + "is a regular file, a symlink, or some other type of file. Please check that " - + f"{COLORS['WARNING']}{repopath}{COLORS['FAIL']} is either a " - + f"directory, or does not exist, on a read-write filesystem.{COLORS['ENDC']}" + f"{repopath}{COLORS['FAIL']} is not a directory, for example, if it " + "is a regular file, a symlink, or some other type of file. Please check that " + f"{COLORS['WARNING']}{repopath}{COLORS['FAIL']} is either a " + f"directory, or does not exist, on a read-write filesystem.{COLORS['ENDC']}" ) sys.exit(-1) # lets decide, -1 is for all mkdir errors @@ -301,9 +300,10 @@ def program_init(repopath): # check that the things we want exist if "Core" not in MAPS_CONFIG or "telemetry" not in MAPS_CONFIG["Core"]: if VERBOSE: - print(COLORS["FAIL"] - + "Malformed config file! Printing for verification:" - + COLORS["ENDC"]) + print(f"{COLORS['FAIL']}" + "Malformed config file! Printing for verification:" + f"{COLORS['ENDC']}" + ) print("---" + COLORS["WARNING"]) print(tomli_w.dumps(MAPS_CONFIG).strip() + COLORS["ENDC"]) print("---") @@ -319,9 +319,7 @@ def program_init(repopath): telemetry_consent = TELECONSENT # check if env vars override config file if ( - os.getenv("MAPS_TELEMETRY_CONSENT") is not None - and os.getenv("MAPS_TELEMETRY_CONSENT") != "" - and not TELECONSENT + os.getenv("MAPS_TELEMETRY_CONSENT", "") != "" and not TELECONSENT ): inputmessage = "MAPS_TELEMETRY_CONSENT is set, but telemetry was previously disabled!\n"\ "Would you like to enable telemetry? (Y/N) > " @@ -658,10 +656,10 @@ def mode_run(repo, args): except subprocess.CalledProcessError: print( f"{COLORS['FAIL']}mkdir failed. This can happen if {COLORS['WARNING']}" - + f"{os.getenv('HOME')}/Public{COLORS['FAIL']} is not a directory, for example, if it " - + "is a regular file, a symlink, or some other type of file. Please check that " - + f"{COLORS['WARNING']}{os.getenv('HOME')}/Public{COLORS['FAIL']} is either a " - + f"directory, or does not exist, on a read-write filesystem.{COLORS['ENDC']}" + f"{os.getenv('HOME')}/Public{COLORS['FAIL']} is not a directory, for example, if it " + "is a regular file, a symlink, or some other type of file. Please check that " + f"{COLORS['WARNING']}{os.getenv('HOME')}/Public{COLORS['FAIL']} is either a " + f"directory, or does not exist, on a read-write filesystem.{COLORS['ENDC']}" ) sys.exit(-1) # lets decide, -1 is for all mkdir errors if not os.path.isdir(f"{DATADIR}/rofs/home/runtime/Public"): @@ -670,10 +668,10 @@ def mode_run(repo, args): except subprocess.CalledProcessError: print( f"{COLORS['FAIL']}mkdir failed. This can happen if {COLORS['WARNING']}" - + f"{os.getenv('HOME')}/Public{COLORS['FAIL']} is not a directory, for example, if " - + "it is a regular file, a symlink, or some other type of file. Please check that " - + f"{COLORS['WARNING']}{os.getenv('HOME')}/Public{COLORS['FAIL']} is either a " - + f"directory, or does not exist, on a read-write filesystem.{COLORS['ENDC']}" + f"{os.getenv('HOME')}/Public{COLORS['FAIL']} is not a directory, for example, if " + "it is a regular file, a symlink, or some other type of file. Please check that " + f"{COLORS['WARNING']}{os.getenv('HOME')}/Public{COLORS['FAIL']} is either a " + f"directory, or does not exist, on a read-write filesystem.{COLORS['ENDC']}" ) sys.exit(-1) # lets decide, -1 is for all mkdir errors @@ -873,10 +871,10 @@ def checkout(repo, remote, runtime): except subprocess.CalledProcessError: print( f"{COLORS['FAIL']}mkdir failed. This can happen if {COLORS['WARNING']}" - + f"{DATADIR}{COLORS['FAIL']} is not a directory, for example, if it " - + "is a regular file, a symlink, or some other type of file. Please check that " - + f"{COLORS['WARNING']}{DATADIR}{COLORS['FAIL']} is either a " - + f"directory, or does not exist, on a read-write filesystem.{COLORS['ENDC']}" + f"{DATADIR}{COLORS['FAIL']} is not a directory, for example, if it " + "is a regular file, a symlink, or some other type of file. Please check that " + f"{COLORS['WARNING']}{DATADIR}{COLORS['FAIL']} is either a " + f"directory, or does not exist, on a read-write filesystem.{COLORS['ENDC']}" ) sys.exit(-1) @@ -1170,7 +1168,7 @@ def mode_url(repo, repopath, args): if not check1 and not check3: # here check2 is always false assert not check2 # failing this assert will crash without an error - # this is fine because this is an impossible case ? + # this is fine because this is an impossible case ? # neither the name, nor the remote URL was known # add those to the local repo args.REMOTE = [remote_name, remote_url] @@ -1207,16 +1205,16 @@ def mode_export_url(repo: OSTree.Repo, args: argparse.Namespace): if '@' in remote_url: print( f"{COLORS['WARNING']} Warning: sharing a password protected remote, with the password. " - + f"Please double check! {COLORS['ENDC']}" + f"Please double check! {COLORS['ENDC']}" ) print(f"- Remote name is\t{remote_name}") print(f"- Remote URL is\t\t{remote_url}") print(f"- Runtime is\t\t{runtime}") urlstring = ( "maps://runtime?" - + f"remotename={remote_name}&" - + f"remoteurl={remote_url}&" - + f"runtime={runtime}" + f"remotename={remote_name}&" + f"remoteurl={remote_url}&" + f"runtime={runtime}" ) print(f"\n {urlstring}\n") @@ -1246,7 +1244,7 @@ def mode_runtime(repo, repopath, args): elif args.EXPORTURL: mode_export_url(repo, args) elif args.EXPORTTAR: - export(repo, args.EXPORTTAR) + mode_export_tar(repo, args.EXPORTTAR) def byteSI(inbytes): @@ -1376,16 +1374,19 @@ def add_hash_to_db(refhash, tarpath, datadir): tardbfile.write(f'"{refhash}"="{tarhash}"\n') -def runtime_to_tar(repo, runtime, remote = None): +def runtime_to_tar(repo, runtime, remote=None): + """ + Given a runtime, local or remote, and export its fs tree to a tarball. + """ if remote is None: remote, runtime = disambiguate_runtime(repo, runtime, installed=False) - print(f"remote was disambiguated to {remote}") DATADIR = f"{os.getenv('HOME')}/.var/org.mardi.maps" RUNTIMEDIR = f"{DATADIR}/{remote}/{runtime}/rofs" REFHASH = repo.list_refs()[1][runtime] TARPATH = f"{DATADIR}/{remote}/{':'.join(runtime.split('/'))}.tar.gz" + # if runtime is not checked out, do it if not os.path.isdir(RUNTIMEDIR): checkout(repo, remote, runtime) @@ -1398,16 +1399,21 @@ def runtime_to_tar(repo, runtime, remote = None): else: opts = "-c --use-compress-program=pigz" - subprocess.run(f"tar {opts} -C {RUNTIMEDIR[0:-4]} -f {TARPATH} {RUNTIMEDIR[-4:]}".split(), check=True) + subprocess.run(f"tar {opts} -C {RUNTIMEDIR[0:-4]} -f {TARPATH} {RUNTIMEDIR[-4:]}".split(), + check=True) add_hash_to_db(REFHASH, TARPATH, DATADIR) - + return TARPATH -def export(repo, runtime): +def mode_export_tar(repo, runtime): + """ + User interface for runtime_to_tar + """ + print(f"Exporting {runtime} to tar...") tarpath = runtime_to_tar(repo, runtime) print(f"Runtime exported to {tarpath}") - return + return 0 def upload(repo, runtime): From 54464670c416081d55e5d1844c568079d7250f44 Mon Sep 17 00:00:00 2001 From: Aaruni Kaushik Date: Thu, 13 Nov 2025 14:15:52 +0100 Subject: [PATCH 3/3] add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d51917f..a8f4bac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project tries to adhere to [Semantic Versioning](https://semver.org/spe ## [Upcoming] ### Added +- Add `--export-tar` option to export a runtime to a tarball. ### Changed