diff --git a/cget/cli.py b/cget/cli.py index 886dcb1..0bf8363 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -87,8 +87,10 @@ def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, def @click.option('--debug', is_flag=True, help="Install debug version") @click.option('--release', is_flag=True, help="Install release version") @click.option('--insecure', is_flag=True, help="Don't use https urls") +@click.option('--use-build-cache', is_flag=True, help="Cache builds") +@click.option('--recipe-deps-only', is_flag=True, help="only use dependencies from recipes (speeds up cached builds a lot)") @click.argument('pkgs', nargs=-1, type=click.STRING) -def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure): +def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure, use_build_cache, recipe_deps_only): """ Install packages """ if debug and release: click.echo("ERROR: debug and release are not supported together") @@ -102,7 +104,16 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato for pbu in util.flat([prefix.from_file(file), pbs]): pb = pbu.merge_defines(define) with prefix.try_("Failed to build package {}".format(pb.to_name()), on_fail=lambda: prefix.remove(pb)): - click.echo(prefix.install(pb, test=test, test_all=test_all, update=update, generator=generator, insecure=insecure)) + click.echo(prefix.install( + pb, + test=test, + test_all=test_all, + update=update, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=recipe_deps_only + )) @cli.command(name='ignore') @use_prefix diff --git a/cget/package.py b/cget/package.py index 8a93c79..99737b9 100644 --- a/cget/package.py +++ b/cget/package.py @@ -1,4 +1,4 @@ -import base64, copy, argparse, six +import base64, copy, argparse, six, dirhash, hashlib def encode_url(url): x = six.b(url[url.find('://')+3:]) @@ -31,6 +31,14 @@ def get_src_dir(self): return self.url[7:] # Remove "file://" raise TypeError() + def calc_hash(self): + if self.recipe: + print("calculating dirshash of recipe '%s' package '%s'" % (self.recipe, self.to_name())) + return dirhash.dirhash(self.recipe, "sha1") + elif self.url: + print("calculating hash of url '%s' package '%s'" % (self.url, self.to_name())) + return hashlib.sha1(self.url.encode("utf-8")).hexdigest() + raise Exception("no url or recipe: %s" % self.__dict__) def fname_to_pkg(fname): if fname.startswith('_url_'): return PackageSource(name=decode_url(fname), fname=fname) diff --git a/cget/prefix.py b/cget/prefix.py index ad8e658..d08bcb9 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -1,4 +1,4 @@ -import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools +import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools, hashlib from cget.builder import Builder from cget.package import fname_to_pkg @@ -205,7 +205,7 @@ def get_unlink_deps_directory(self, name, *dirs): def parse_src_file(self, name, url, start=None): f = util.actual_path(url, start) self.log('parse_src_file actual_path:', start, f) - if os.path.exists(f): return PackageSource(name=name, url='file://' + f) + if os.path.isfile(f): return PackageSource(name=name, url='file://' + f) return None def parse_src_recipe(self, name, url): @@ -223,6 +223,15 @@ def parse_src_github(self, name, url): if name is None: name = p return PackageSource(name=name, url=url) + def hash_pkg(self, pkg): + pkg_src = self.parse_pkg_src(pkg) + result = pkg_src.calc_hash() + pkg_build = self.parse_pkg_build(pkg) + if pkg_build.requirements: + for dependency in self.from_file(pkg_build.requirements): + result = hashlib.sha1((result + self.hash_pkg(dependency)).encode("utf-8")).hexdigest() + return result + @returns(PackageSource) @params(pkg=PACKAGE_SOURCE_TYPES) def parse_pkg_src(self, pkg, start=None, no_recipe=False): @@ -239,14 +248,14 @@ def parse_pkg_src(self, pkg, start=None, no_recipe=False): @returns(PackageBuild) @params(pkg=PACKAGE_SOURCE_TYPES) def parse_pkg_build(self, pkg, start=None, no_recipe=False): - if isinstance(pkg, PackageBuild): + if isinstance(pkg, PackageBuild): pkg.pkg_src = self.parse_pkg_src(pkg.pkg_src, start, no_recipe) if pkg.pkg_src.recipe: pkg = self.from_recipe(pkg.pkg_src.recipe, pkg) if pkg.cmake: pkg.cmake = find_cmake(pkg.cmake, start) return pkg else: pkg_src = self.parse_pkg_src(pkg, start, no_recipe) - if pkg_src.recipe: return self.from_recipe(pkg_src.recipe, pkg_src.name) + if pkg_src.recipe: return self.from_recipe(pkg_src.recipe, name=pkg_src.name) else: return PackageBuild(pkg_src) def from_recipe(self, recipe, pkg=None, name=None): @@ -256,7 +265,7 @@ def from_recipe(self, recipe, pkg=None, name=None): self.check(lambda:p.pkg_src is not None) requirements = os.path.join(recipe, "requirements.txt") if os.path.exists(requirements): p.requirements = requirements - p.pkg_src.recipe = None + p.pkg_src.recipe = recipe # Use original name if pkg: p.pkg_src.name = pkg.pkg_src.name elif name: p.pkg_src.name = name @@ -285,21 +294,55 @@ def from_file(self, file, url=None, no_recipe=False): def write_parent(self, pb, track=True): if track and pb.parent is not None: util.mkfile(self.get_deps_directory(pb.to_fname()), pb.parent, pb.parent) - def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecure=False): - for dependent in self.from_file(pb.requirements or os.path.join(d, 'requirements.txt'), pb.pkg_src.url): + def install_deps( + self, + pb, + src_dir=None, + test=False, + test_all=False, + generator=None, + insecure=False, + use_build_cache=False, + recipe_deps_only=False + ): + if pb.requirements: + dependents = self.from_file(pb.requirements, pb.pkg_src.url) + elif src_dir: + dependents = self.from_file(os.path.join(src_dir, 'requirements.txt'), pb.pkg_src.url) + else: + return + for dependent in dependents: transient = dependent.test or dependent.build testing = test or test_all installable = not dependent.test or dependent.test == testing if installable: - self.install(dependent.of(pb), test_all=test_all, generator=generator, track=not transient, insecure=insecure) + self.install( + dependent.of(pb), + test_all=test_all, + generator=generator, + track=not transient, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=recipe_deps_only + ) @returns(six.string_types) @params(pb=PACKAGE_SOURCE_TYPES, test=bool, test_all=bool, update=bool, track=bool) - def install(self, pb, test=False, test_all=False, generator=None, update=False, track=True, insecure=False): + def install( + self, + pb, + test=False, + test_all=False, + generator=None, + update=False, + track=True, + insecure=False, + use_build_cache=False, + recipe_deps_only=False + ): pb = self.parse_pkg_build(pb) pkg_dir = self.get_package_directory(pb.to_fname()) unlink_dir = self.get_unlink_directory(pb.to_fname()) - install_dir = self.get_package_directory(pb.to_fname(), 'install') # If its been unlinked, then link it in if os.path.exists(unlink_dir): if update: shutil.rmtree(unlink_dir) @@ -311,26 +354,74 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, self.write_parent(pb, track=track) if update: self.remove(pb) else: return "Package {} already installed".format(pb.to_name()) - with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: - # Fetch package - src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) - # Install any dependencies first - self.install_deps(pb, src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure) - # Setup cmake file - if pb.cmake: - target = os.path.join(src_dir, 'CMakeLists.txt') - if os.path.exists(target): - os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) - shutil.copyfile(pb.cmake, target) - # Configure and build - builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) - builder.build(variant=pb.variant) - # Run tests if enabled - if test or test_all: builder.test(variant=pb.variant) - # Install - builder.build(target='install', variant=pb.variant) - if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) - else: util.copy_dir(install_dir, self.prefix) + package_hash = self.hash_pkg(pb) + self.log("package %s hash %s" % (pb.to_name(), package_hash)) + pkg_install_dir = self.get_package_directory(pb.to_fname(), 'install') + if use_build_cache: + install_dir = util.get_cache_path("builds", pb.to_name(), package_hash) + util.mkdir(pkg_dir) + os.symlink(install_dir, pkg_install_dir) + self.log("using cached install dir '%s'" % install_dir) + else: + install_dir = pkg_install_dir + self.log("using local install dir '%s'" % install_dir) + build_needed = True + if recipe_deps_only: + self.install_deps( + pb, + test=test, + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=True + ) + with util.cache_lock() as cache_lock: + if not update and use_build_cache and os.path.exists(install_dir): + print("retreived Package {} from cache".format(pb.to_name())) + build_needed = False + if build_needed: + with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: + # Fetch package + src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) + # Install any dependencies first + if not recipe_deps_only: + self.install_deps( + pb, + src_dir=src_dir, + test=test, + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=False + ) + with util.cache_lock() as cache_lock: + if not update and use_build_cache and os.path.exists(install_dir): + print("retreived Package {} from cache".format(pb.to_name())) + else: + # Setup cmake file + if pb.cmake: + target = os.path.join(src_dir, 'CMakeLists.txt') + if os.path.exists(target): + os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) + shutil.copyfile(pb.cmake, target) + # Configure and build + builder.configure( + src_dir, + defines=pb.define, + generator=generator, + install_prefix=install_dir, + test=test, + variant=pb.variant + ) + builder.build(variant=pb.variant) + # Run tests if enabled + if test or test_all: builder.test(variant=pb.variant) + # Install + builder.build(target='install', variant=pb.variant) + if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) + else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) return "Successfully installed {}".format(pb.to_name()) diff --git a/cget/util.py b/cget/util.py index b2aa1b5..59b0fe0 100644 --- a/cget/util.py +++ b/cget/util.py @@ -1,4 +1,4 @@ -import click, os, sys, shutil, json, six, hashlib, ssl +import click, os, sys, shutil, json, six, hashlib, ssl, filelock if sys.version_info[0] < 3: try: @@ -87,6 +87,43 @@ def mkfile(d, file, content, always_write=True): write_to(p, content) return p +def cache_lock(): + cache_base_dir = get_cache_path() + mkdir(cache_base_dir) + return filelock.FileLock(os.path.join(cache_base_dir, "lock")) + +def zipdir(src_dir, tgt_file): + print("zipping '%s' to '%s" % (src_dir, tgt_file)) + zipf = zipfile.ZipFile(tgt_file, 'w', zipfile.ZIP_DEFLATED) + for root, dirs, files in os.walk(src_dir): + for file in files: + zipf.write( + os.path.join(root, file), + os.path.relpath( + os.path.join(root, file), + os.path.join(src_dir) + ) + ) + zipf.close() + +def zip_dir_to_cache(prefix, key, src_dir): + with cache_lock(): + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + mkdir(cache_dir) + zipdir(src_dir, zipfile_path) + +def unzip_dir_from_cache(prefix, key, tgt_dir): + with cache_lock(): + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + if os.path.exists(zipfile_path): + f = zipfile.ZipFile(zipfile_path, "r") + f.extractall(tgt_dir) + return True + else: + return False + def ls(p, predicate=lambda x:True): if os.path.exists(p): return (d for d in os.listdir(p) if predicate(os.path.join(p, d))) @@ -97,7 +134,7 @@ def get_app_dir(*args): return os.path.join(click.get_app_dir('cget'), *args) def get_cache_path(*args): - return get_app_dir('cache', *args) + return os.path.join(os.path.expanduser("~"), ".cget", "cache", *args) def adjust_path(p): # Prefixing path to avoid problems with long paths on windows @@ -106,15 +143,17 @@ def adjust_path(p): return p def add_cache_file(key, f): - mkdir(get_cache_path(key)) - shutil.copy2(f, get_cache_path(key, os.path.basename(f))) + with cache_lock(): + mkdir(get_cache_path(key)) + shutil.copy2(f, get_cache_path(key, os.path.basename(f))) def get_cache_file(key): - p = get_cache_path(key) - if os.path.exists(p): - return os.path.join(p, next(ls(p))) - else: - return None + with cache_lock(): + p = get_cache_path(key) + if os.path.exists(p): + return os.path.join(p, next(ls(p))) + else: + return None def delete_dir(path): if path is not None and os.path.exists(path): shutil.rmtree(adjust_path(path)) diff --git a/requirements.txt b/requirements.txt index 1df73e3..4948a96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ click>=6.6 # PyYAML six>=1.10 +dirhash>=0.2.1 +filelock>=2.0.13