Move from archive to reference for local git repos to save space

Author: ctk21

build_ocaml_hash.py

@@ -11,7 +11,7 @@
 parser.add_argument('basedir', type=str, help='location to put the source and the build')
 parser.add_argument('--configure_args', type=str, help='additional configure arguments', default=None)
 parser.add_argument('--repo', type=str, help='alternate URL for the repo', default=REPO)
-parser.add_argument('--use_archive', action='store_true', help='use archive to get just the source (only works on local repos)', default=False)
+parser.add_argument('--use_reference', action='store_true', help='use reference to clone the source (only works on local repos)', default=False)
 parser.add_argument('--no_clean', action='store_true', default=False)
 parser.add_argument('-j', '--jobs', type=int, help='number of jobs for make in build', default=1)
 parser.add_argument('-v', '--verbose', action='store_true', default=False)
@@ -34,16 +34,14 @@ def shell_exec(cmd, verbose=args.verbose, check=True):
 os.mkdir(srcdir)
 
 
-if args.use_archive:
-	# go to the repo and archive out of it
-	os.chdir(args.repo)
-	shell_exec('git archive --format=tar %s | tar x -C %s'%(args.hash, srcdir))
-	os.chdir(srcdir)
+if args.use_reference:
+	shell_exec('git clone --reference %s %s %s'%(args.repo, args.repo, srcdir))
 else:
-	# get git source and checkout the hash
 	shell_exec('git clone %s %s'%(args.repo, srcdir))
-	os.chdir(srcdir)
-	shell_exec('git checkout %s'%args.hash)
+
+os.chdir(srcdir)
+shell_exec('git checkout %s'%args.hash)
+shell_exec('git clean -f -d -x')
 
 # build the source
 xtra_args = "" if args.configure_args is None else args.configure_args

run_backfill.py

@@ -25,7 +25,7 @@ def get_script_dir():
 parser.add_argument('--branch', type=str, help='git branch for the compiler (default: %s)'%DEFAULT_BRANCH, default=DEFAULT_BRANCH)
 parser.add_argument('--main_branch', type=str, help='name of mainline git branch for compiler (default: %s)'%DEFAULT_MAIN_BRANCH, default=DEFAULT_MAIN_BRANCH)
 parser.add_argument('--repo_pull', action='store_true', help="do a pull on the git repo before selecting hashes", default=False)
-parser.add_argument('--use_repo_archive', action='store_true', help="use archive to get source out of repo to save space", default=False)
+parser.add_argument('--use_repo_reference', action='store_true', help="use reference to clone a local git repo", default=False)
 parser.add_argument('--no_first_parent', action='store_true', help="By default we use first-parent on git logs (to keep date ordering sane); this option turns it off", default=False)
 parser.add_argument('--commit_choice_method', type=str, help='commit choice method (version_tags, status_success, hash=XXX, delay=00:05:00, all)', default='version_tags')
 parser.add_argument('--commit_after', type=str, help='select commits after the specified date (e.g. 2017-10-02)', default=None)
@@ -184,8 +184,8 @@ def get_hash_status(h):
 			print('Skipping build for %s as already built'%h)
 		else:
 			log_fname = os.path.join(hashdir, 'build_%s.log'%run_timestamp)
-			use_archive_opt = '--use_archive' if args.use_repo_archive else ''
-			completed_proc = shell_exec_redirect('%s/build_ocaml_hash.py --repo %s %s -j %d --configure_args="%s" %s %s %s'%(SCRIPTDIR, repo_path, use_archive_opt, args.jobs, configure_args, verbose_args, h, builddir), log_fname)
+			use_reference_opt = '--use_reference' if args.use_repo_reference else ''
+			completed_proc = shell_exec_redirect('%s/build_ocaml_hash.py --repo %s %s -j %d --configure_args="%s" %s %s %s'%(SCRIPTDIR, repo_path, use_reference_opt, args.jobs, configure_args, verbose_args, h, builddir), log_fname)
 			if completed_proc.returncode != 0:
 				print('ERROR[%d] in build_ocaml_hash for %s (see %s)'%(completed_proc.returncode, h, log_fname))
 				continue