diff --git a/cget/cli.py b/cget/cli.py index 886dcb1..bb933a3 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -56,7 +56,8 @@ def w(obj, prefix, verbose, build_path, *args, **kwargs): @click.option('-D', '--define', multiple=True, help="Extra configuration variables to pass to CMake") @click.option('--shared', is_flag=True, help="Set toolchain to build shared libraries by default") @click.option('--static', is_flag=True, help="Set toolchain to build static libraries by default") -def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, define, shared, static): +@click.option('--no-global-include', is_flag=True, help="Don't use global include dir (required for -X header)") +def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, define, shared, static, no_global_include): """ Initialize install directory """ if shared and static: click.echo("ERROR: shared and static are not supported together") @@ -73,7 +74,9 @@ def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, def cxxflags=cxxflags, ldflags=ldflags, std=std, - defines=defines) + defines=defines, + no_global_include=no_global_include + ) @cli.command(name='install') @use_prefix @@ -87,8 +90,10 @@ def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, def @click.option('--debug', is_flag=True, help="Install debug version") @click.option('--release', is_flag=True, help="Install release version") @click.option('--insecure', is_flag=True, help="Don't use https urls") +@click.option('--use-build-cache', is_flag=True, help="Cache builds") +@click.option('--recipe-deps-only', is_flag=True, help="only use dependencies from recipes (speeds up cached builds a lot)") @click.argument('pkgs', nargs=-1, type=click.STRING) -def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure): +def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure, use_build_cache, recipe_deps_only): """ Install packages """ if debug and release: click.echo("ERROR: debug and release are not supported together") @@ -102,7 +107,16 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato for pbu in util.flat([prefix.from_file(file), pbs]): pb = pbu.merge_defines(define) with prefix.try_("Failed to build package {}".format(pb.to_name()), on_fail=lambda: prefix.remove(pb)): - click.echo(prefix.install(pb, test=test, test_all=test_all, update=update, generator=generator, insecure=insecure)) + click.echo(prefix.install( + pb, + test=test, + test_all=test_all, + update=update, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=recipe_deps_only + )) @cli.command(name='ignore') @use_prefix diff --git a/cget/cmake/autotools.cmake b/cget/cmake/autotools.cmake index 3701785..3db6fe0 100644 --- a/cget/cmake/autotools.cmake +++ b/cget/cmake/autotools.cmake @@ -60,14 +60,6 @@ macro(preamble PREFIX) endforeach() adjust_path(${PREFIX}_SYSTEM_PATH) - set(${PREFIX}_PKG_CONFIG_PATH) - foreach(P ${PREFIX_PATH} ${PREFIX_SYSTEM_PATH}) - foreach(SUFFIX lib lib${${PREFIX}_ADDRESS_MODEL} share) - list(APPEND ${PREFIX}_PKG_CONFIG_PATH ${P}/${SUFFIX}/pkgconfig) - endforeach() - endforeach() - adjust_path(${PREFIX}_PKG_CONFIG_PATH) - get_property_list(${PREFIX}_COMPILE_FLAGS COMPILE_OPTIONS) get_directory_property(${PREFIX}_INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES) foreach(DIR ${${PREFIX}_INCLUDE_DIRECTORIES}) @@ -86,11 +78,12 @@ macro(preamble PREFIX) endif() endforeach() get_directory_property(${PREFIX}_LINK_DIRECTORIES LINK_DIRECTORIES) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LIB_DIR ${${PREFIX}_LINK_DIRECTORIES}) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") else() - string(APPEND ${PREFIX}_LINK_FLAGS " -L ${LIB_DIR}") + string(APPEND ${PREFIX}_LINK_FLAGS " -L${LIB_DIR}") endif() endforeach() @@ -112,13 +105,11 @@ macro(preamble PREFIX) set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") endif (APPLE) - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") else() string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_STATIC_LINKER_FLAGS}") endif() - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) @@ -146,14 +137,8 @@ macro(preamble PREFIX) set(${PREFIX}_BASE_ENV_COMMAND ${CMAKE_COMMAND} -E env "PATH=${${PREFIX}_SYSTEM_PATH}${PATH_SEP}$ENV{PATH}" - "PKG_CONFIG_PATH=${${PREFIX}_PKG_CONFIG_PATH}" ) - # TODO: Set also PKG_CONFIG_SYSROOT_DIR - if(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE STREQUAL "ONLY") - list(APPEND ${PREFIX}_BASE_ENV_COMMAND "PKG_CONFIG_LIBDIR=${${PREFIX}_PKG_CONFIG_PATH}") - endif() - set(${PREFIX}_ENV_COMMAND ${${PREFIX}_BASE_ENV_COMMAND} "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" @@ -181,7 +166,19 @@ list(APPEND CONFIGURE_OPTIONS ) endif() -execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/configure --help OUTPUT_VARIABLE AUTOTOOLS_AVAILABLE_OPTIONS) + +set(AUTOTOOLS_RUN_AUTOGEN_SH Off CACHE BOOL "") +if (AUTOTOOLS_RUN_AUTOGEN_SH) + execute_process( + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/autogen.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) +endif (AUTOTOOLS_RUN_AUTOGEN_SH) + +execute_process( + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/configure --help + OUTPUT_VARIABLE AUTOTOOLS_AVAILABLE_OPTIONS +) if(AUTOTOOLS_AVAILABLE_OPTIONS MATCHES "--disable-option-checking") set(AUTOTOOL_IMPLICIT_CONFIGURE_OPTIONS On CACHE BOOL "") @@ -200,6 +197,7 @@ if(AUTOTOOL_IMPLICIT_CONFIGURE_OPTIONS) endif() endif() + message(STATUS "Configure options: ${CONFIGURE_OPTIONS}") # TODO: Check flags of configure script diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index 87127ec..0c850f2 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -53,14 +53,6 @@ macro(preamble PREFIX) endforeach() adjust_path(${PREFIX}_SYSTEM_PATH) - set(${PREFIX}_PKG_CONFIG_PATH) - foreach(P ${PREFIX_PATH} ${PREFIX_SYSTEM_PATH}) - foreach(SUFFIX lib lib${${PREFIX}_ADDRESS_MODEL} share) - list(APPEND ${PREFIX}_PKG_CONFIG_PATH ${P}/${SUFFIX}/pkgconfig) - endforeach() - endforeach() - adjust_path(${PREFIX}_PKG_CONFIG_PATH) - get_property_list(${PREFIX}_COMPILE_FLAGS COMPILE_OPTIONS) get_directory_property(${PREFIX}_INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES) foreach(DIR ${${PREFIX}_INCLUDE_DIRECTORIES}) @@ -79,11 +71,12 @@ macro(preamble PREFIX) endif() endforeach() get_directory_property(${PREFIX}_LINK_DIRECTORIES LINK_DIRECTORIES) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LIB_DIR ${${PREFIX}_LINK_DIRECTORIES}) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") else() - string(APPEND ${PREFIX}_LINK_FLAGS " -L ${LIB_DIR}") + string(APPEND ${PREFIX}_LINK_FLAGS " -L${LIB_DIR}") endif() endforeach() @@ -105,13 +98,11 @@ macro(preamble PREFIX) set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") endif (APPLE) - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") else() string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_STATIC_LINKER_FLAGS}") endif() - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) @@ -139,14 +130,8 @@ macro(preamble PREFIX) set(${PREFIX}_BASE_ENV_COMMAND ${CMAKE_COMMAND} -E env "PATH=${${PREFIX}_SYSTEM_PATH}${PATH_SEP}$ENV{PATH}" - "PKG_CONFIG_PATH=${${PREFIX}_PKG_CONFIG_PATH}" ) - # TODO: Set also PKG_CONFIG_SYSROOT_DIR - if(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE STREQUAL "ONLY") - list(APPEND ${PREFIX}_BASE_ENV_COMMAND "PKG_CONFIG_LIBDIR=${${PREFIX}_PKG_CONFIG_PATH}") - endif() - set(${PREFIX}_ENV_COMMAND ${${PREFIX}_BASE_ENV_COMMAND} "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" @@ -234,24 +219,40 @@ using ${BOOST_TOOLCHAIN} : ${B2_TOOLCHAIN_VERSION} : \"${B2_COMPILER}\" : ${SEARCH_PATHS} ; ") + +set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") +set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") + +if (BOOST_PYTHON) + find_program(BOOST_PYTHON_FOUND ${BOOST_PYTHON} REQUIRED) + get_filename_component(BOOST_PYTHON_FOUND_REAL "${BOOST_PYTHON_FOUND}" REALPATH) + message(STATUS "found python: '${BOOST_PYTHON_FOUND}' -> '${BOOST_PYTHON_FOUND_REAL}'") + set(B2_CONFIG_CONTENT "${B2_CONFIG_CONTENT} + using python : : ${BOOST_PYTHON_FOUND_REAL} ; + ") +endif (BOOST_PYTHON) + message("${B2_CONFIG_CONTENT}") file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") find_program(B2_EXE b2) if(NOT ${B2_EXE}) + if (BOOST_PYTHON) + set(BOOST_BOOTSTRAP_PYTHON_ARG "--with-python=${BOOST_PYTHON_FOUND_REAL}") + endif (BOOST_PYTHON) if(CMAKE_HOST_WIN32) add_custom_target(bootstrap - COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ + COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2.exe") else() add_custom_target(bootstrap - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) - set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2") + set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/b2") endif() install(PROGRAMS ${B2_EXE} DESTINATION bin) endif() diff --git a/cget/cmake/make.cmake b/cget/cmake/make.cmake index 7d5f495..28d6f22 100644 --- a/cget/cmake/make.cmake +++ b/cget/cmake/make.cmake @@ -58,14 +58,6 @@ macro(preamble PREFIX) endforeach() adjust_path(${PREFIX}_SYSTEM_PATH) - set(${PREFIX}_PKG_CONFIG_PATH) - foreach(P ${PREFIX_PATH} ${PREFIX_SYSTEM_PATH}) - foreach(SUFFIX lib lib${${PREFIX}_ADDRESS_MODEL} share) - list(APPEND ${PREFIX}_PKG_CONFIG_PATH ${P}/${SUFFIX}/pkgconfig) - endforeach() - endforeach() - adjust_path(${PREFIX}_PKG_CONFIG_PATH) - get_property_list(${PREFIX}_COMPILE_FLAGS COMPILE_OPTIONS) get_directory_property(${PREFIX}_INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES) foreach(DIR ${${PREFIX}_INCLUDE_DIRECTORIES}) @@ -84,11 +76,12 @@ macro(preamble PREFIX) endif() endforeach() get_directory_property(${PREFIX}_LINK_DIRECTORIES LINK_DIRECTORIES) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LIB_DIR ${${PREFIX}_LINK_DIRECTORIES}) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") else() - string(APPEND ${PREFIX}_LINK_FLAGS " -L ${LIB_DIR}") + string(APPEND ${PREFIX}_LINK_FLAGS " -L${LIB_DIR}") endif() endforeach() @@ -110,13 +103,11 @@ macro(preamble PREFIX) set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") endif (APPLE) - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") else() string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_STATIC_LINKER_FLAGS}") endif() - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) @@ -144,14 +135,8 @@ macro(preamble PREFIX) set(${PREFIX}_BASE_ENV_COMMAND ${CMAKE_COMMAND} -E env "PATH=${${PREFIX}_SYSTEM_PATH}${PATH_SEP}$ENV{PATH}" - "PKG_CONFIG_PATH=${${PREFIX}_PKG_CONFIG_PATH}" ) - # TODO: Set also PKG_CONFIG_SYSROOT_DIR - if(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE STREQUAL "ONLY") - list(APPEND ${PREFIX}_BASE_ENV_COMMAND "PKG_CONFIG_LIBDIR=${${PREFIX}_PKG_CONFIG_PATH}") - endif() - set(${PREFIX}_ENV_COMMAND ${${PREFIX}_BASE_ENV_COMMAND} "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" diff --git a/cget/cmake/meson.cmake b/cget/cmake/meson.cmake index 73e4601..5aa9485 100644 --- a/cget/cmake/meson.cmake +++ b/cget/cmake/meson.cmake @@ -87,14 +87,6 @@ macro(preamble PREFIX) endforeach() adjust_path(${PREFIX}_SYSTEM_PATH) - set(${PREFIX}_PKG_CONFIG_PATH) - foreach(P ${PREFIX_PATH} ${PREFIX_SYSTEM_PATH}) - foreach(SUFFIX lib lib${${PREFIX}_ADDRESS_MODEL} share) - list(APPEND ${PREFIX}_PKG_CONFIG_PATH ${P}/${SUFFIX}/pkgconfig) - endforeach() - endforeach() - adjust_path(${PREFIX}_PKG_CONFIG_PATH) - get_property_list(${PREFIX}_COMPILE_FLAGS COMPILE_OPTIONS) get_directory_property(${PREFIX}_INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES) foreach(DIR ${${PREFIX}_INCLUDE_DIRECTORIES}) @@ -113,11 +105,12 @@ macro(preamble PREFIX) endif() endforeach() get_directory_property(${PREFIX}_LINK_DIRECTORIES LINK_DIRECTORIES) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LIB_DIR ${${PREFIX}_LINK_DIRECTORIES}) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") else() - string(APPEND ${PREFIX}_LINK_FLAGS " -L ${LIB_DIR}") + string(APPEND ${PREFIX}_LINK_FLAGS " -L${LIB_DIR}") endif() endforeach() @@ -139,13 +132,11 @@ macro(preamble PREFIX) set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") endif (APPLE) - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") else() string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_STATIC_LINKER_FLAGS}") endif() - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) @@ -173,14 +164,8 @@ macro(preamble PREFIX) set(${PREFIX}_BASE_ENV_COMMAND ${CMAKE_COMMAND} -E env "PATH=${${PREFIX}_SYSTEM_PATH}${PATH_SEP}$ENV{PATH}" - "PKG_CONFIG_PATH=${${PREFIX}_PKG_CONFIG_PATH}" ) - # TODO: Set also PKG_CONFIG_SYSROOT_DIR - if(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE STREQUAL "ONLY") - list(APPEND ${PREFIX}_BASE_ENV_COMMAND "PKG_CONFIG_LIBDIR=${${PREFIX}_PKG_CONFIG_PATH}") - endif() - set(${PREFIX}_ENV_COMMAND ${${PREFIX}_BASE_ENV_COMMAND} "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" diff --git a/cget/package.py b/cget/package.py index 8a93c79..649829e 100644 --- a/cget/package.py +++ b/cget/package.py @@ -1,4 +1,4 @@ -import base64, copy, argparse, six +import base64, copy, argparse, six, dirhash, hashlib def encode_url(url): x = six.b(url[url.find('://')+3:]) @@ -31,6 +31,12 @@ def get_src_dir(self): return self.url[7:] # Remove "file://" raise TypeError() + def calc_hash(self): + if self.recipe: + return dirhash.dirhash(self.recipe, "sha1") + elif self.url: + return hashlib.sha1(self.url.encode("utf-8")).hexdigest() + raise Exception("no url or recipe: %s" % self.__dict__) def fname_to_pkg(fname): if fname.startswith('_url_'): return PackageSource(name=decode_url(fname), fname=fname) diff --git a/cget/prefix.py b/cget/prefix.py index ad8e658..1e09460 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -1,4 +1,4 @@ -import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools +import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools, hashlib from cget.builder import Builder from cget.package import fname_to_pkg @@ -117,27 +117,24 @@ def write_cmake(self, always_write=False, **kwargs): @returns(inspect.isgenerator) @util.yield_from - def generate_cmake_toolchain(self, toolchain=None, cc=None, cxx=None, cflags=None, cxxflags=None, ldflags=None, std=None, defines=None): + def generate_cmake_toolchain( + self, + toolchain=None, + cc=None, + cxx=None, + cflags=None, + cxxflags=None, + ldflags=None, + std=None, + defines=None, + no_global_include=False + ): set_ = cmake_set if_ = cmake_if else_ = cmake_else append_ = cmake_append yield set_('CGET_PREFIX', self.prefix) - yield set_('CMAKE_PREFIX_PATH', self.prefix) - yield if_('${CMAKE_VERSION} VERSION_LESS "3.6.0"', - ['include_directories(SYSTEM ${CGET_PREFIX}/include)'], - else_( - set_('CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include'), - set_('CMAKE_C_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include') - ) - ) if toolchain: yield ['include({})'.format(util.quote(os.path.abspath(toolchain)))] - yield if_('CMAKE_CROSSCOMPILING', - append_('CMAKE_FIND_ROOT_PATH', self.prefix) - ) - yield if_('CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT', - set_('CMAKE_INSTALL_PREFIX', self.prefix) - ) if cxx: yield set_('CMAKE_CXX_COMPILER', cxx) if cc: yield set_('CMAKE_C_COMPILER', cc) if std: @@ -205,7 +202,7 @@ def get_unlink_deps_directory(self, name, *dirs): def parse_src_file(self, name, url, start=None): f = util.actual_path(url, start) self.log('parse_src_file actual_path:', start, f) - if os.path.exists(f): return PackageSource(name=name, url='file://' + f) + if os.path.isfile(f): return PackageSource(name=name, url='file://' + f) return None def parse_src_recipe(self, name, url): @@ -223,6 +220,15 @@ def parse_src_github(self, name, url): if name is None: name = p return PackageSource(name=name, url=url) + def hash_pkg(self, pkg): + pkg_src = self.parse_pkg_src(pkg) + result = pkg_src.calc_hash() + pkg_build = self.parse_pkg_build(pkg) + if pkg_build.requirements: + for dependency in self.from_file(pkg_build.requirements): + result = hashlib.sha1((result + self.hash_pkg(dependency)).encode("utf-8")).hexdigest() + return result + @returns(PackageSource) @params(pkg=PACKAGE_SOURCE_TYPES) def parse_pkg_src(self, pkg, start=None, no_recipe=False): @@ -239,14 +245,14 @@ def parse_pkg_src(self, pkg, start=None, no_recipe=False): @returns(PackageBuild) @params(pkg=PACKAGE_SOURCE_TYPES) def parse_pkg_build(self, pkg, start=None, no_recipe=False): - if isinstance(pkg, PackageBuild): + if isinstance(pkg, PackageBuild): pkg.pkg_src = self.parse_pkg_src(pkg.pkg_src, start, no_recipe) if pkg.pkg_src.recipe: pkg = self.from_recipe(pkg.pkg_src.recipe, pkg) if pkg.cmake: pkg.cmake = find_cmake(pkg.cmake, start) return pkg else: pkg_src = self.parse_pkg_src(pkg, start, no_recipe) - if pkg_src.recipe: return self.from_recipe(pkg_src.recipe, pkg_src.name) + if pkg_src.recipe: return self.from_recipe(pkg_src.recipe, name=pkg_src.name) else: return PackageBuild(pkg_src) def from_recipe(self, recipe, pkg=None, name=None): @@ -256,7 +262,7 @@ def from_recipe(self, recipe, pkg=None, name=None): self.check(lambda:p.pkg_src is not None) requirements = os.path.join(recipe, "requirements.txt") if os.path.exists(requirements): p.requirements = requirements - p.pkg_src.recipe = None + p.pkg_src.recipe = recipe # Use original name if pkg: p.pkg_src.name = pkg.pkg_src.name elif name: p.pkg_src.name = name @@ -285,21 +291,60 @@ def from_file(self, file, url=None, no_recipe=False): def write_parent(self, pb, track=True): if track and pb.parent is not None: util.mkfile(self.get_deps_directory(pb.to_fname()), pb.parent, pb.parent) - def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecure=False): - for dependent in self.from_file(pb.requirements or os.path.join(d, 'requirements.txt'), pb.pkg_src.url): + def install_deps( + self, + pb, + src_dir=None, + test=False, + test_all=False, + generator=None, + insecure=False, + use_build_cache=False, + recipe_deps_only=False + ): + for dependent in self.get_dependents(pb, src_dir): transient = dependent.test or dependent.build testing = test or test_all installable = not dependent.test or dependent.test == testing if installable: - self.install(dependent.of(pb), test_all=test_all, generator=generator, track=not transient, insecure=insecure) + self.install( + dependent.of(pb), + test_all=test_all, + generator=generator, + track=not transient, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=recipe_deps_only + ) + + def get_dependents(self, pb, src_dir): + if pb.requirements: + return self.from_file(pb.requirements, pb.pkg_src.url) + elif src_dir: + return self.from_file(os.path.join(src_dir, 'requirements.txt'), pb.pkg_src.url) + else: + return [] + + def get_real_install_path(self, pb): + return os.path.realpath(self.get_package_directory(pb.to_fname(), 'install')) @returns(six.string_types) @params(pb=PACKAGE_SOURCE_TYPES, test=bool, test_all=bool, update=bool, track=bool) - def install(self, pb, test=False, test_all=False, generator=None, update=False, track=True, insecure=False): + def install( + self, + pb, + test=False, + test_all=False, + generator=None, + update=False, + track=True, + insecure=False, + use_build_cache=False, + recipe_deps_only=False + ): pb = self.parse_pkg_build(pb) pkg_dir = self.get_package_directory(pb.to_fname()) unlink_dir = self.get_unlink_directory(pb.to_fname()) - install_dir = self.get_package_directory(pb.to_fname(), 'install') # If its been unlinked, then link it in if os.path.exists(unlink_dir): if update: shutil.rmtree(unlink_dir) @@ -311,26 +356,81 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, self.write_parent(pb, track=track) if update: self.remove(pb) else: return "Package {} already installed".format(pb.to_name()) - with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: - # Fetch package - src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) - # Install any dependencies first - self.install_deps(pb, src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure) - # Setup cmake file - if pb.cmake: - target = os.path.join(src_dir, 'CMakeLists.txt') - if os.path.exists(target): - os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) - shutil.copyfile(pb.cmake, target) - # Configure and build - builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) - builder.build(variant=pb.variant) - # Run tests if enabled - if test or test_all: builder.test(variant=pb.variant) - # Install - builder.build(target='install', variant=pb.variant) - if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) - else: util.copy_dir(install_dir, self.prefix) + package_hash = self.hash_pkg(pb) + self.log("package %s hash %s" % (pb.to_name(), package_hash)) + pkg_install_dir = self.get_package_directory(pb.to_fname(), 'install') + if use_build_cache: + install_dir = util.get_cache_path("builds", pb.to_name(), package_hash) + util.mkdir(pkg_dir) + os.symlink(install_dir, pkg_install_dir) + self.log("using cached install dir '%s'" % install_dir) + else: + install_dir = pkg_install_dir + self.log("using local install dir '%s'" % install_dir) + build_needed = True + if recipe_deps_only: + self.install_deps( + pb, + test=test, + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=True + ) + with util.cache_lock() as cache_lock: + if not update and use_build_cache and os.path.exists(install_dir): + print("retreived Package {} from cache".format(pb.to_name())) + build_needed = False + if build_needed: + with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: + # Fetch package + src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) + # Install any dependencies first + if not recipe_deps_only: + self.install_deps( + pb, + src_dir=src_dir, + test=test, + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=False + ) + with util.cache_lock() as cache_lock: + if not update and use_build_cache and os.path.exists(install_dir): + print("retreived Package {} from cache".format(pb.to_name())) + else: + # Setup cmake file + if pb.cmake: + target = os.path.join(src_dir, 'CMakeLists.txt') + if os.path.exists(target): + os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) + shutil.copyfile(pb.cmake, target) + # Configure and build + defines = list(pb.define or []) + [ + "CMAKE_PREFIX_PATH=%s" % ";".join( + ['"%s"' % self.get_real_install_path(dep) for dep in self.get_dependents(pb, src_dir)] + ) + ] + print("defines") + print(defines) + builder.configure( + src_dir, + defines=defines, + generator=generator, + install_prefix=install_dir, + test=test, + variant=pb.variant + ) + builder.build(variant=pb.variant) + # Run tests if enabled + if test or test_all: builder.test(variant=pb.variant) + # Install + builder.build(target='install', variant=pb.variant) + if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) + else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) return "Successfully installed {}".format(pb.to_name()) diff --git a/cget/util.py b/cget/util.py index b2aa1b5..1eaa5c6 100644 --- a/cget/util.py +++ b/cget/util.py @@ -1,4 +1,4 @@ -import click, os, sys, shutil, json, six, hashlib, ssl +import click, os, sys, shutil, json, six, hashlib, ssl, filelock if sys.version_info[0] < 3: try: @@ -16,7 +16,7 @@ else: import subprocess -from six.moves.urllib import request +import requests def to_bool(value): x = str(value).lower() @@ -87,6 +87,52 @@ def mkfile(d, file, content, always_write=True): write_to(p, content) return p +def cache_lock(): + cache_base_dir = get_cache_path() + mkdir(cache_base_dir) + return filelock.FileLock(os.path.join(cache_base_dir, "lock")) + +def zipdir(src_dir, tgt_file): + print("zipping '%s' to '%s" % (src_dir, tgt_file)) + zipf = zipfile.ZipFile(tgt_file, 'w', zipfile.ZIP_DEFLATED) + for root, dirs, files in os.walk(src_dir): + for file in files: + zipf.write( + os.path.join(root, file), + os.path.relpath( + os.path.join(root, file), + os.path.join(src_dir) + ) + ) + zipf.close() + +def unzip(zipname, extract_dir): + def extract_file(zf, info, extract_dir): + zf.extract( info.filename, path=extract_dir ) + out_path = os.path.join( extract_dir, info.filename ) + perm = info.external_attr >> 16 + os.chmod( out_path, perm ) + with zipfile.ZipFile(zipname, 'r') as zf: + for info in zf.infolist(): + extract_file(zf, info, extract_dir) + +def zip_dir_to_cache(prefix, key, src_dir): + with cache_lock(): + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + mkdir(cache_dir) + zipdir(src_dir, zipfile_path) + +def unzip_dir_from_cache(prefix, key, tgt_dir): + with cache_lock(): + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + if os.path.exists(zipfile_path): + unzip(zipfile_path, tgt_dir) + return True + else: + return False + def ls(p, predicate=lambda x:True): if os.path.exists(p): return (d for d in os.listdir(p) if predicate(os.path.join(p, d))) @@ -97,7 +143,7 @@ def get_app_dir(*args): return os.path.join(click.get_app_dir('cget'), *args) def get_cache_path(*args): - return get_app_dir('cache', *args) + return os.path.join(os.path.expanduser("~"), ".cget", "cache", *args) def adjust_path(p): # Prefixing path to avoid problems with long paths on windows @@ -106,14 +152,17 @@ def adjust_path(p): return p def add_cache_file(key, f): - mkdir(get_cache_path(key)) - shutil.copy2(f, get_cache_path(key, os.path.basename(f))) + with cache_lock(): + mkdir(get_cache_path(key)) + shutil.copy2(f, get_cache_path(key, os.path.basename(f))) def get_cache_file(key): - p = get_cache_path(key) - if os.path.exists(p): - return os.path.join(p, next(ls(p))) - else: + with cache_lock(): + p = get_cache_path(key) + if os.path.exists(p): + content = list(ls(p)) + if content: + return os.path.join(p, content[0]) return None def delete_dir(path): @@ -205,31 +254,27 @@ def symlink_to(src, dst_dir): os.symlink(src, target) return target -class CGetURLOpener(request.FancyURLopener): - def http_error_default(self, url, fp, errcode, errmsg, headers): - if errcode >= 400: - raise BuildError("Download failed with error {0} for: {1}".format(errcode, url)) - return request.FancyURLopener.http_error_default(self, url, fp, errcode, errmsg, headers) - def download_to(url, download_dir, insecure=False): name = url.split('/')[-1] - file = os.path.join(download_dir, name) + file_name = os.path.join(download_dir, name) click.echo("Downloading {0}".format(url)) - bar_len = 1000 - with click.progressbar(length=bar_len, width=70) as bar: - def hook(count, block_size, total_size): - percent = int(count*block_size*bar_len/total_size) - if percent > 0 and percent < bar_len: - # Hack because we can't set the position - bar.pos = percent - bar.update(0) - context = None - if insecure: context = ssl._create_unverified_context() - CGetURLOpener(context=context).retrieve(url, filename=file, reporthook=hook, data=None) - bar.update(bar_len) - if not os.path.exists(file): + with open(file_name, "wb") as f: + response = requests.get(url, stream=True) + response.raise_for_status() + total_length = response.headers.get('content-length') + if total_length is None: # no content length header + f.write(response.content) + else: + total_length = int(total_length) + with click.progressbar(length=total_length, width=70) as bar: + for data in response.iter_content(chunk_size=4096): + f.write(data) + bar.pos += len(data) + bar.update(0) + bar.update(total_length) + if not os.path.exists(file_name): raise BuildError("Download failed for: {0}".format(url)) - return file + return file_name def transfer_to(f, dst, copy=False): if USE_SYMLINKS and not copy: return symlink_to(f, dst) @@ -244,10 +289,12 @@ def retrieve_url(url, dst, copy=False, insecure=False, hash=None): f = download_to(url, dst, insecure=insecure) if remote else transfer_to(url[7:], dst, copy=copy) if os.path.isfile(f) and hash: click.echo("Computing hash: {}".format(hash)) - if check_hash(f, hash): + hash_type, hash_value = hash.lower().split(':') + computed = hash_file(f, hash_type) + if computed == hash_value: if remote: add_cache_file(hash.replace(':', '-'), f) else: - raise BuildError("Hash doesn't match for {0}: {1}".format(url, hash)) + raise BuildError("Hash doesn't match for {0}: {1} != {2}".format(url, hash_type + ":" + computed, hash)) return f def extract_ar(archive, dst, *kwargs): @@ -274,10 +321,6 @@ def hash_file(f, t): h.update(open(f, 'rb').read()) return h.hexdigest() -def check_hash(f, hash): - t, h = hash.lower().split(':') - return hash_file(f, t) == h - def which(p, paths=None, throws=True): exes = [p+x for x in ['', '.exe', '.bat']] for dirname in list(paths or [])+os.environ['PATH'].split(os.pathsep): diff --git a/requirements.txt b/requirements.txt index 1df73e3..9a9aaf2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ click>=6.6 # PyYAML six>=1.10 +requests +dirhash>=0.2.1 +filelock>=2.0.13 diff --git a/tools/cmake/autotools.cmake b/tools/cmake/autotools.cmake index 2629822..45c7ebd 100644 --- a/tools/cmake/autotools.cmake +++ b/tools/cmake/autotools.cmake @@ -34,7 +34,19 @@ list(APPEND CONFIGURE_OPTIONS ) endif() -execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/configure --help OUTPUT_VARIABLE AUTOTOOLS_AVAILABLE_OPTIONS) + +set(AUTOTOOLS_RUN_AUTOGEN_SH Off CACHE BOOL "") +if (AUTOTOOLS_RUN_AUTOGEN_SH) + execute_process( + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/autogen.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) +endif (AUTOTOOLS_RUN_AUTOGEN_SH) + +execute_process( + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/configure --help + OUTPUT_VARIABLE AUTOTOOLS_AVAILABLE_OPTIONS +) if(AUTOTOOLS_AVAILABLE_OPTIONS MATCHES "--disable-option-checking") set(AUTOTOOL_IMPLICIT_CONFIGURE_OPTIONS On CACHE BOOL "") @@ -53,6 +65,7 @@ if(AUTOTOOL_IMPLICIT_CONFIGURE_OPTIONS) endif() endif() + message(STATUS "Configure options: ${CONFIGURE_OPTIONS}") # TODO: Check flags of configure script diff --git a/tools/cmake/boost.cmake b/tools/cmake/boost.cmake index 3337730..e5cbe1c 100644 --- a/tools/cmake/boost.cmake +++ b/tools/cmake/boost.cmake @@ -87,24 +87,40 @@ using ${BOOST_TOOLCHAIN} : ${B2_TOOLCHAIN_VERSION} : \"${B2_COMPILER}\" : ${SEARCH_PATHS} ; ") + +set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") +set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") + +if (BOOST_PYTHON) + find_program(BOOST_PYTHON_FOUND ${BOOST_PYTHON} REQUIRED) + get_filename_component(BOOST_PYTHON_FOUND_REAL "${BOOST_PYTHON_FOUND}" REALPATH) + message(STATUS "found python: '${BOOST_PYTHON_FOUND}' -> '${BOOST_PYTHON_FOUND_REAL}'") + set(B2_CONFIG_CONTENT "${B2_CONFIG_CONTENT} + using python : : ${BOOST_PYTHON_FOUND_REAL} ; + ") +endif (BOOST_PYTHON) + message("${B2_CONFIG_CONTENT}") file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") find_program(B2_EXE b2) if(NOT ${B2_EXE}) + if (BOOST_PYTHON) + set(BOOST_BOOTSTRAP_PYTHON_ARG "--with-python=${BOOST_PYTHON_FOUND_REAL}") + endif (BOOST_PYTHON) if(CMAKE_HOST_WIN32) add_custom_target(bootstrap - COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ + COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2.exe") else() add_custom_target(bootstrap - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) - set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2") + set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/b2") endif() install(PROGRAMS ${B2_EXE} DESTINATION bin) endif() diff --git a/tools/preamble.cmake b/tools/preamble.cmake index ae1fed6..f8e1b5a 100644 --- a/tools/preamble.cmake +++ b/tools/preamble.cmake @@ -43,14 +43,6 @@ macro(preamble PREFIX) endforeach() adjust_path(${PREFIX}_SYSTEM_PATH) - set(${PREFIX}_PKG_CONFIG_PATH) - foreach(P ${PREFIX_PATH} ${PREFIX_SYSTEM_PATH}) - foreach(SUFFIX lib lib${${PREFIX}_ADDRESS_MODEL} share) - list(APPEND ${PREFIX}_PKG_CONFIG_PATH ${P}/${SUFFIX}/pkgconfig) - endforeach() - endforeach() - adjust_path(${PREFIX}_PKG_CONFIG_PATH) - get_property_list(${PREFIX}_COMPILE_FLAGS COMPILE_OPTIONS) get_directory_property(${PREFIX}_INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES) foreach(DIR ${${PREFIX}_INCLUDE_DIRECTORIES}) @@ -69,11 +61,12 @@ macro(preamble PREFIX) endif() endforeach() get_directory_property(${PREFIX}_LINK_DIRECTORIES LINK_DIRECTORIES) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LIB_DIR ${${PREFIX}_LINK_DIRECTORIES}) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") else() - string(APPEND ${PREFIX}_LINK_FLAGS " -L ${LIB_DIR}") + string(APPEND ${PREFIX}_LINK_FLAGS " -L${LIB_DIR}") endif() endforeach() @@ -95,13 +88,11 @@ macro(preamble PREFIX) set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") endif (APPLE) - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") else() string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_STATIC_LINKER_FLAGS}") endif() - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) @@ -129,14 +120,8 @@ macro(preamble PREFIX) set(${PREFIX}_BASE_ENV_COMMAND ${CMAKE_COMMAND} -E env "PATH=${${PREFIX}_SYSTEM_PATH}${PATH_SEP}$ENV{PATH}" - "PKG_CONFIG_PATH=${${PREFIX}_PKG_CONFIG_PATH}" ) - # TODO: Set also PKG_CONFIG_SYSROOT_DIR - if(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE STREQUAL "ONLY") - list(APPEND ${PREFIX}_BASE_ENV_COMMAND "PKG_CONFIG_LIBDIR=${${PREFIX}_PKG_CONFIG_PATH}") - endif() - set(${PREFIX}_ENV_COMMAND ${${PREFIX}_BASE_ENV_COMMAND} "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}"