From 37a64d2027ee5700b7145ba4ccbda7ab7d136587 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 8 Jan 2021 16:55:29 +0100 Subject: [PATCH 01/12] implement simple package hashing --- cget/cli.py | 2 ++ cget/package.py | 8 +++++++- cget/prefix.py | 11 ++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cget/cli.py b/cget/cli.py index 886dcb1..f321492 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -100,6 +100,8 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato else: file = 'requirements.txt' pbs = [PackageBuild(pkg, cmake=cmake, variant=variant) for pkg in pkgs] for pbu in util.flat([prefix.from_file(file), pbs]): + hash = prefix.hash_pkg(pbu) + print("package '%s' hash %s" % (prefix.parse_pkg_src(pbu).name, hash)) pb = pbu.merge_defines(define) with prefix.try_("Failed to build package {}".format(pb.to_name()), on_fail=lambda: prefix.remove(pb)): click.echo(prefix.install(pb, test=test, test_all=test_all, update=update, generator=generator, insecure=insecure)) diff --git a/cget/package.py b/cget/package.py index 8a93c79..649829e 100644 --- a/cget/package.py +++ b/cget/package.py @@ -1,4 +1,4 @@ -import base64, copy, argparse, six +import base64, copy, argparse, six, dirhash, hashlib def encode_url(url): x = six.b(url[url.find('://')+3:]) @@ -31,6 +31,12 @@ def get_src_dir(self): return self.url[7:] # Remove "file://" raise TypeError() + def calc_hash(self): + if self.recipe: + return dirhash.dirhash(self.recipe, "sha1") + elif self.url: + return hashlib.sha1(self.url.encode("utf-8")).hexdigest() + raise Exception("no url or recipe: %s" % self.__dict__) def fname_to_pkg(fname): if fname.startswith('_url_'): return PackageSource(name=decode_url(fname), fname=fname) diff --git a/cget/prefix.py b/cget/prefix.py index ad8e658..37de659 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -1,4 +1,4 @@ -import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools +import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools, hashlib from cget.builder import Builder from cget.package import fname_to_pkg @@ -223,6 +223,15 @@ def parse_src_github(self, name, url): if name is None: name = p return PackageSource(name=name, url=url) + def hash_pkg(self, pkg): + pkg_src = self.parse_pkg_src(pkg) + result = pkg_src.calc_hash() + pkg_build = self.parse_pkg_build(pkg) + if pkg_build.requirements: + for dependency in self.from_file(pkg_build.requirements): + result = hashlib.sha1((result + self.hash_pkg(dependency)).encode("utf-8")).hexdigest() + return result + @returns(PackageSource) @params(pkg=PACKAGE_SOURCE_TYPES) def parse_pkg_src(self, pkg, start=None, no_recipe=False): From c00433960336f9506f6ec4a2b58aac613b1c56c6 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 8 Jan 2021 17:24:38 +0100 Subject: [PATCH 02/12] zip builds to cache --- cget/prefix.py | 1 + cget/util.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/cget/prefix.py b/cget/prefix.py index 37de659..8d50e86 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -338,6 +338,7 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, if test or test_all: builder.test(variant=pb.variant) # Install builder.build(target='install', variant=pb.variant) + util.zip_dir_to_cache("builds/%s" % pb.to_name(), self.hash_pkg(pb), install_dir) if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) diff --git a/cget/util.py b/cget/util.py index b2aa1b5..6ef75aa 100644 --- a/cget/util.py +++ b/cget/util.py @@ -87,6 +87,25 @@ def mkfile(d, file, content, always_write=True): write_to(p, content) return p +def zipdir(src_dir, tgt_file): + print("zipping '%s' to '%s" % (src_dir, tgt_file)) + zipf = zipfile.ZipFile(tgt_file, 'w', zipfile.ZIP_DEFLATED) + for root, dirs, files in os.walk(src_dir): + for file in files: + zipf.write( + os.path.join(root, file), + os.path.relpath( + os.path.join(root, file), + os.path.join(src_dir, '..') + ) + ) + zipf.close() + +def zip_dir_to_cache(prefix, key, src_dir): + out_dir = get_cache_path(prefix) + mkdir(out_dir) + zipdir(src_dir, os.path.join(out_dir, key + ".zip")) + def ls(p, predicate=lambda x:True): if os.path.exists(p): return (d for d in os.listdir(p) if predicate(os.path.join(p, d))) From dcb990f771c4b5428be1c7ca1db8e162d1c0a109 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 10:07:20 +0100 Subject: [PATCH 03/12] add dirhash dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 1df73e3..0d55ef5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ click>=6.6 # PyYAML six>=1.10 +dirhash>=0.2.1 From 0ddf42618cf57befad97fea316760f13599b2ef1 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 10:58:51 +0100 Subject: [PATCH 04/12] get builds from cache if possible --- cget/cli.py | 2 -- cget/prefix.py | 35 +++++++++++++++++++++-------------- cget/util.py | 19 +++++++++++++++---- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/cget/cli.py b/cget/cli.py index f321492..886dcb1 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -100,8 +100,6 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato else: file = 'requirements.txt' pbs = [PackageBuild(pkg, cmake=cmake, variant=variant) for pkg in pkgs] for pbu in util.flat([prefix.from_file(file), pbs]): - hash = prefix.hash_pkg(pbu) - print("package '%s' hash %s" % (prefix.parse_pkg_src(pbu).name, hash)) pb = pbu.merge_defines(define) with prefix.try_("Failed to build package {}".format(pb.to_name()), on_fail=lambda: prefix.remove(pb)): click.echo(prefix.install(pb, test=test, test_all=test_all, update=update, generator=generator, insecure=insecure)) diff --git a/cget/prefix.py b/cget/prefix.py index 8d50e86..0d7ff9f 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -320,25 +320,32 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, self.write_parent(pb, track=track) if update: self.remove(pb) else: return "Package {} already installed".format(pb.to_name()) + package_hash = self.hash_pkg(pb) + print("package %s hash %s" % (pb.to_name(), package_hash)) + #return "dry run" + build_cache_prefix = "builds/%s" % pb.to_name() with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: # Fetch package src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) # Install any dependencies first self.install_deps(pb, src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure) - # Setup cmake file - if pb.cmake: - target = os.path.join(src_dir, 'CMakeLists.txt') - if os.path.exists(target): - os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) - shutil.copyfile(pb.cmake, target) - # Configure and build - builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) - builder.build(variant=pb.variant) - # Run tests if enabled - if test or test_all: builder.test(variant=pb.variant) - # Install - builder.build(target='install', variant=pb.variant) - util.zip_dir_to_cache("builds/%s" % pb.to_name(), self.hash_pkg(pb), install_dir) + if util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): + print("retreived Package {} from cache".format(pb.to_name())) + else: + # Setup cmake file + if pb.cmake: + target = os.path.join(src_dir, 'CMakeLists.txt') + if os.path.exists(target): + os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) + shutil.copyfile(pb.cmake, target) + # Configure and build + builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) + builder.build(variant=pb.variant) + # Run tests if enabled + if test or test_all: builder.test(variant=pb.variant) + # Install + builder.build(target='install', variant=pb.variant) + util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) diff --git a/cget/util.py b/cget/util.py index 6ef75aa..3ea77b8 100644 --- a/cget/util.py +++ b/cget/util.py @@ -96,15 +96,26 @@ def zipdir(src_dir, tgt_file): os.path.join(root, file), os.path.relpath( os.path.join(root, file), - os.path.join(src_dir, '..') + os.path.join(src_dir) ) ) zipf.close() def zip_dir_to_cache(prefix, key, src_dir): - out_dir = get_cache_path(prefix) - mkdir(out_dir) - zipdir(src_dir, os.path.join(out_dir, key + ".zip")) + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + mkdir(cache_dir) + zipdir(src_dir, zipfile_path) + +def unzip_dir_from_cache(prefix, key, tgt_dir): + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + if os.path.exists(zipfile_path): + f = zipfile.ZipFile(zipfile_path, "r") + f.extractall(tgt_dir) + return True + else: + return False def ls(p, predicate=lambda x:True): if os.path.exists(p): From 90015b27964e2ab266fd28f4dab4c6aa2bd68681 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 11:10:29 +0100 Subject: [PATCH 05/12] lock cache --- cget/util.py | 49 ++++++++++++++++++++++++++++-------------------- requirements.txt | 2 ++ 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/cget/util.py b/cget/util.py index 3ea77b8..c7b6dea 100644 --- a/cget/util.py +++ b/cget/util.py @@ -1,4 +1,4 @@ -import click, os, sys, shutil, json, six, hashlib, ssl +import click, os, sys, shutil, json, six, hashlib, ssl, filelock if sys.version_info[0] < 3: try: @@ -87,6 +87,11 @@ def mkfile(d, file, content, always_write=True): write_to(p, content) return p +def cache_lock(): + cache_base_dir = get_cache_path() + mkdir(cache_base_dir) + return filelock.FileLock(os.path.join(cache_base_dir, "lock")) + def zipdir(src_dir, tgt_file): print("zipping '%s' to '%s" % (src_dir, tgt_file)) zipf = zipfile.ZipFile(tgt_file, 'w', zipfile.ZIP_DEFLATED) @@ -102,20 +107,22 @@ def zipdir(src_dir, tgt_file): zipf.close() def zip_dir_to_cache(prefix, key, src_dir): - cache_dir = get_cache_path(prefix) - zipfile_path = os.path.join(cache_dir, key + ".zip") - mkdir(cache_dir) - zipdir(src_dir, zipfile_path) + with cache_lock(): + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + mkdir(cache_dir) + zipdir(src_dir, zipfile_path) def unzip_dir_from_cache(prefix, key, tgt_dir): - cache_dir = get_cache_path(prefix) - zipfile_path = os.path.join(cache_dir, key + ".zip") - if os.path.exists(zipfile_path): - f = zipfile.ZipFile(zipfile_path, "r") - f.extractall(tgt_dir) - return True - else: - return False + with cache_lock(): + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + if os.path.exists(zipfile_path): + f = zipfile.ZipFile(zipfile_path, "r") + f.extractall(tgt_dir) + return True + else: + return False def ls(p, predicate=lambda x:True): if os.path.exists(p): @@ -136,15 +143,17 @@ def adjust_path(p): return p def add_cache_file(key, f): - mkdir(get_cache_path(key)) - shutil.copy2(f, get_cache_path(key, os.path.basename(f))) + with cache_lock(): + mkdir(get_cache_path(key)) + shutil.copy2(f, get_cache_path(key, os.path.basename(f))) def get_cache_file(key): - p = get_cache_path(key) - if os.path.exists(p): - return os.path.join(p, next(ls(p))) - else: - return None + with cache_lock(): + p = get_cache_path(key) + if os.path.exists(p): + return os.path.join(p, next(ls(p))) + else: + return None def delete_dir(path): if path is not None and os.path.exists(path): shutil.rmtree(adjust_path(path)) diff --git a/requirements.txt b/requirements.txt index 0d55ef5..2f15aa5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ click>=6.6 # PyYAML six>=1.10 dirhash>=0.2.1 +filelock>=2.0.13 + From 41bd679d736c257f0508908dc89b78541ef08fef Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 11:26:28 +0100 Subject: [PATCH 06/12] make build cache optional --- cget/cli.py | 13 +++++++++++-- cget/prefix.py | 7 ++++--- requirements.txt | 1 - 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cget/cli.py b/cget/cli.py index 886dcb1..f499dd4 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -87,8 +87,9 @@ def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, def @click.option('--debug', is_flag=True, help="Install debug version") @click.option('--release', is_flag=True, help="Install release version") @click.option('--insecure', is_flag=True, help="Don't use https urls") +@click.option('--use-build-cache', is_flag=True, help="Cache builds") @click.argument('pkgs', nargs=-1, type=click.STRING) -def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure): +def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure, use_build_cache): """ Install packages """ if debug and release: click.echo("ERROR: debug and release are not supported together") @@ -102,7 +103,15 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato for pbu in util.flat([prefix.from_file(file), pbs]): pb = pbu.merge_defines(define) with prefix.try_("Failed to build package {}".format(pb.to_name()), on_fail=lambda: prefix.remove(pb)): - click.echo(prefix.install(pb, test=test, test_all=test_all, update=update, generator=generator, insecure=insecure)) + click.echo(prefix.install( + pb, + test=test, + test_all=test_all, + update=update, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache + )) @cli.command(name='ignore') @use_prefix diff --git a/cget/prefix.py b/cget/prefix.py index 0d7ff9f..fe2dd92 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -304,7 +304,7 @@ def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecu @returns(six.string_types) @params(pb=PACKAGE_SOURCE_TYPES, test=bool, test_all=bool, update=bool, track=bool) - def install(self, pb, test=False, test_all=False, generator=None, update=False, track=True, insecure=False): + def install(self, pb, test=False, test_all=False, generator=None, update=False, track=True, insecure=False, use_build_cache=False): pb = self.parse_pkg_build(pb) pkg_dir = self.get_package_directory(pb.to_fname()) unlink_dir = self.get_unlink_directory(pb.to_fname()) @@ -329,7 +329,7 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) # Install any dependencies first self.install_deps(pb, src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure) - if util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): + if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): print("retreived Package {} from cache".format(pb.to_name())) else: # Setup cmake file @@ -345,7 +345,8 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, if test or test_all: builder.test(variant=pb.variant) # Install builder.build(target='install', variant=pb.variant) - util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) + if use_build_cache: + util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) diff --git a/requirements.txt b/requirements.txt index 2f15aa5..4948a96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ click>=6.6 six>=1.10 dirhash>=0.2.1 filelock>=2.0.13 - From 4478066efbb836c7818fb8f24e251d367593bb2b Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 14:31:24 +0100 Subject: [PATCH 07/12] propagate build cache setting to dependencies --- cget/prefix.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/cget/prefix.py b/cget/prefix.py index fe2dd92..3ff72c0 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -294,13 +294,20 @@ def from_file(self, file, url=None, no_recipe=False): def write_parent(self, pb, track=True): if track and pb.parent is not None: util.mkfile(self.get_deps_directory(pb.to_fname()), pb.parent, pb.parent) - def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecure=False): + def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecure=False, use_build_cache=False): for dependent in self.from_file(pb.requirements or os.path.join(d, 'requirements.txt'), pb.pkg_src.url): transient = dependent.test or dependent.build testing = test or test_all installable = not dependent.test or dependent.test == testing if installable: - self.install(dependent.of(pb), test_all=test_all, generator=generator, track=not transient, insecure=insecure) + self.install( + dependent.of(pb), + test_all=test_all, + generator=generator, + track=not transient, + insecure=insecure, + use_build_cache=use_build_cache + ) @returns(six.string_types) @params(pb=PACKAGE_SOURCE_TYPES, test=bool, test_all=bool, update=bool, track=bool) @@ -328,7 +335,15 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, # Fetch package src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) # Install any dependencies first - self.install_deps(pb, src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure) + self.install_deps( + pb, + src_dir, + test=test, + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache + ) if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): print("retreived Package {} from cache".format(pb.to_name())) else: From e5021aa325ced6a6bd18c12f5a71a58f5dfb253b Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 15:03:54 +0100 Subject: [PATCH 08/12] allow fast path cached builds by ignoring in source deps --- cget/cli.py | 6 ++- cget/prefix.py | 102 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 77 insertions(+), 31 deletions(-) diff --git a/cget/cli.py b/cget/cli.py index f499dd4..0bf8363 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -88,8 +88,9 @@ def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, def @click.option('--release', is_flag=True, help="Install release version") @click.option('--insecure', is_flag=True, help="Don't use https urls") @click.option('--use-build-cache', is_flag=True, help="Cache builds") +@click.option('--recipe-deps-only', is_flag=True, help="only use dependencies from recipes (speeds up cached builds a lot)") @click.argument('pkgs', nargs=-1, type=click.STRING) -def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure, use_build_cache): +def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure, use_build_cache, recipe_deps_only): """ Install packages """ if debug and release: click.echo("ERROR: debug and release are not supported together") @@ -110,7 +111,8 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato update=update, generator=generator, insecure=insecure, - use_build_cache=use_build_cache + use_build_cache=use_build_cache, + recipe_deps_only=recipe_deps_only )) @cli.command(name='ignore') diff --git a/cget/prefix.py b/cget/prefix.py index 3ff72c0..bc0bfc5 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -294,8 +294,24 @@ def from_file(self, file, url=None, no_recipe=False): def write_parent(self, pb, track=True): if track and pb.parent is not None: util.mkfile(self.get_deps_directory(pb.to_fname()), pb.parent, pb.parent) - def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecure=False, use_build_cache=False): - for dependent in self.from_file(pb.requirements or os.path.join(d, 'requirements.txt'), pb.pkg_src.url): + def install_deps( + self, + pb, + src_dir=None, + test=False, + test_all=False, + generator=None, + insecure=False, + use_build_cache=False, + recipe_deps_only=False + ): + if pb.requirements: + dependents = self.from_file(pb.requirements, pb.pkg_src.url) + elif src_dir: + dependents = self.from_file(os.path.join(src_dir, 'requirements.txt'), pb.pkg_src.url) + else: + return + for dependent in dependents: transient = dependent.test or dependent.build testing = test or test_all installable = not dependent.test or dependent.test == testing @@ -306,12 +322,24 @@ def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecu generator=generator, track=not transient, insecure=insecure, - use_build_cache=use_build_cache + use_build_cache=use_build_cache, + recipe_deps_only=recipe_deps_only ) @returns(six.string_types) @params(pb=PACKAGE_SOURCE_TYPES, test=bool, test_all=bool, update=bool, track=bool) - def install(self, pb, test=False, test_all=False, generator=None, update=False, track=True, insecure=False, use_build_cache=False): + def install( + self, + pb, + test=False, + test_all=False, + generator=None, + update=False, + track=True, + insecure=False, + use_build_cache=False, + recipe_deps_only=False + ): pb = self.parse_pkg_build(pb) pkg_dir = self.get_package_directory(pb.to_fname()) unlink_dir = self.get_unlink_directory(pb.to_fname()) @@ -329,41 +357,57 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, else: return "Package {} already installed".format(pb.to_name()) package_hash = self.hash_pkg(pb) print("package %s hash %s" % (pb.to_name(), package_hash)) - #return "dry run" build_cache_prefix = "builds/%s" % pb.to_name() - with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: - # Fetch package - src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) - # Install any dependencies first + need_build = True + if recipe_deps_only: self.install_deps( pb, - src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure, - use_build_cache=use_build_cache + use_build_cache=use_build_cache, + recipe_deps_only=True ) if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): print("retreived Package {} from cache".format(pb.to_name())) - else: - # Setup cmake file - if pb.cmake: - target = os.path.join(src_dir, 'CMakeLists.txt') - if os.path.exists(target): - os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) - shutil.copyfile(pb.cmake, target) - # Configure and build - builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) - builder.build(variant=pb.variant) - # Run tests if enabled - if test or test_all: builder.test(variant=pb.variant) - # Install - builder.build(target='install', variant=pb.variant) - if use_build_cache: - util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) - if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) - else: util.copy_dir(install_dir, self.prefix) + need_build = False + if need_build: + with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: + # Fetch package + src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) + # Install any dependencies first + if not recipe_deps_only: + self.install_deps( + pb, + src_dir=src_dir, + test=test, + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=False + ) + if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): + print("retreived Package {} from cache".format(pb.to_name())) + else: + # Setup cmake file + if pb.cmake: + target = os.path.join(src_dir, 'CMakeLists.txt') + if os.path.exists(target): + os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) + shutil.copyfile(pb.cmake, target) + # Configure and build + builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) + builder.build(variant=pb.variant) + # Run tests if enabled + if test or test_all: builder.test(variant=pb.variant) + # Install + builder.build(target='install', variant=pb.variant) + if use_build_cache: + util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) + if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) + else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) return "Successfully installed {}".format(pb.to_name()) From eaf45c112f6f8bfe042cd3fbced2590af5be330a Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Tue, 12 Jan 2021 12:10:20 +0100 Subject: [PATCH 09/12] preserve recipe in package source for hashing --- cget/package.py | 2 ++ cget/prefix.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cget/package.py b/cget/package.py index 649829e..99737b9 100644 --- a/cget/package.py +++ b/cget/package.py @@ -33,8 +33,10 @@ def get_src_dir(self): def calc_hash(self): if self.recipe: + print("calculating dirshash of recipe '%s' package '%s'" % (self.recipe, self.to_name())) return dirhash.dirhash(self.recipe, "sha1") elif self.url: + print("calculating hash of url '%s' package '%s'" % (self.url, self.to_name())) return hashlib.sha1(self.url.encode("utf-8")).hexdigest() raise Exception("no url or recipe: %s" % self.__dict__) diff --git a/cget/prefix.py b/cget/prefix.py index bc0bfc5..3f2a97b 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -205,7 +205,7 @@ def get_unlink_deps_directory(self, name, *dirs): def parse_src_file(self, name, url, start=None): f = util.actual_path(url, start) self.log('parse_src_file actual_path:', start, f) - if os.path.exists(f): return PackageSource(name=name, url='file://' + f) + if os.path.isfile(f): return PackageSource(name=name, url='file://' + f) return None def parse_src_recipe(self, name, url): @@ -248,14 +248,14 @@ def parse_pkg_src(self, pkg, start=None, no_recipe=False): @returns(PackageBuild) @params(pkg=PACKAGE_SOURCE_TYPES) def parse_pkg_build(self, pkg, start=None, no_recipe=False): - if isinstance(pkg, PackageBuild): + if isinstance(pkg, PackageBuild): pkg.pkg_src = self.parse_pkg_src(pkg.pkg_src, start, no_recipe) if pkg.pkg_src.recipe: pkg = self.from_recipe(pkg.pkg_src.recipe, pkg) if pkg.cmake: pkg.cmake = find_cmake(pkg.cmake, start) return pkg else: pkg_src = self.parse_pkg_src(pkg, start, no_recipe) - if pkg_src.recipe: return self.from_recipe(pkg_src.recipe, pkg_src.name) + if pkg_src.recipe: return self.from_recipe(pkg_src.recipe, name=pkg_src.name) else: return PackageBuild(pkg_src) def from_recipe(self, recipe, pkg=None, name=None): @@ -265,7 +265,7 @@ def from_recipe(self, recipe, pkg=None, name=None): self.check(lambda:p.pkg_src is not None) requirements = os.path.join(recipe, "requirements.txt") if os.path.exists(requirements): p.requirements = requirements - p.pkg_src.recipe = None + p.pkg_src.recipe = recipe # Use original name if pkg: p.pkg_src.name = pkg.pkg_src.name elif name: p.pkg_src.name = name From 9237e2a7227848be998f7c42f1a93cab71a8ac15 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 22 Jan 2021 13:43:14 +0100 Subject: [PATCH 10/12] fix build cache --- cget/prefix.py | 62 ++++++++++++++++++++++++++++---------------------- cget/util.py | 2 +- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/cget/prefix.py b/cget/prefix.py index 3f2a97b..936a281 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -343,7 +343,6 @@ def install( pb = self.parse_pkg_build(pb) pkg_dir = self.get_package_directory(pb.to_fname()) unlink_dir = self.get_unlink_directory(pb.to_fname()) - install_dir = self.get_package_directory(pb.to_fname(), 'install') # If its been unlinked, then link it in if os.path.exists(unlink_dir): if update: shutil.rmtree(unlink_dir) @@ -356,9 +355,16 @@ def install( if update: self.remove(pb) else: return "Package {} already installed".format(pb.to_name()) package_hash = self.hash_pkg(pb) - print("package %s hash %s" % (pb.to_name(), package_hash)) - build_cache_prefix = "builds/%s" % pb.to_name() - need_build = True + self.log("package %s hash %s" % (pb.to_name(), package_hash)) + pkg_install_dir = self.get_package_directory(pb.to_fname(), 'install') + if use_build_cache: + install_dir = util.get_cache_path("builds", pb.to_name(), package_hash) + util.mkdir(pkg_dir) + os.symlink(install_dir, pkg_install_dir) + self.log("using cached install dir '%s'" % install_dir) + else: + install_dir = pkg_install_dir + self.log("using local install dir '%s'" % install_dir) if recipe_deps_only: self.install_deps( pb, @@ -369,26 +375,23 @@ def install( use_build_cache=use_build_cache, recipe_deps_only=True ) - if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): - print("retreived Package {} from cache".format(pb.to_name())) - need_build = False - if need_build: - with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: - # Fetch package - src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) - # Install any dependencies first - if not recipe_deps_only: - self.install_deps( - pb, - src_dir=src_dir, - test=test, - test_all=test_all, - generator=generator, - insecure=insecure, - use_build_cache=use_build_cache, - recipe_deps_only=False - ) - if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): + with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: + # Fetch package + src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) + # Install any dependencies first + if not recipe_deps_only: + self.install_deps( + pb, + src_dir=src_dir, + test=test, + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=False + ) + with util.cache_lock() as cache_lock: + if not update and use_build_cache and os.path.exists(install_dir): print("retreived Package {} from cache".format(pb.to_name())) else: # Setup cmake file @@ -398,14 +401,19 @@ def install( os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) shutil.copyfile(pb.cmake, target) # Configure and build - builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) + builder.configure( + src_dir, + defines=pb.define, + generator=generator, + install_prefix=install_dir, + test=test, + variant=pb.variant + ) builder.build(variant=pb.variant) # Run tests if enabled if test or test_all: builder.test(variant=pb.variant) # Install builder.build(target='install', variant=pb.variant) - if use_build_cache: - util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) diff --git a/cget/util.py b/cget/util.py index c7b6dea..59b0fe0 100644 --- a/cget/util.py +++ b/cget/util.py @@ -134,7 +134,7 @@ def get_app_dir(*args): return os.path.join(click.get_app_dir('cget'), *args) def get_cache_path(*args): - return get_app_dir('cache', *args) + return os.path.join(os.path.expanduser("~"), ".cget", "cache", *args) def adjust_path(p): # Prefixing path to avoid problems with long paths on windows From 85ea62cce5467975eb15b61f87136db7c1cda5c8 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 22 Jan 2021 16:01:11 +0100 Subject: [PATCH 11/12] avoid package fetch if possible --- cget/prefix.py | 73 +++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/cget/prefix.py b/cget/prefix.py index 936a281..d3c2234 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -375,45 +375,50 @@ def install( use_build_cache=use_build_cache, recipe_deps_only=True ) - with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: - # Fetch package - src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) - # Install any dependencies first - if not recipe_deps_only: - self.install_deps( - pb, - src_dir=src_dir, - test=test, - test_all=test_all, - generator=generator, - insecure=insecure, - use_build_cache=use_build_cache, - recipe_deps_only=False - ) with util.cache_lock() as cache_lock: if not update and use_build_cache and os.path.exists(install_dir): print("retreived Package {} from cache".format(pb.to_name())) - else: - # Setup cmake file - if pb.cmake: - target = os.path.join(src_dir, 'CMakeLists.txt') - if os.path.exists(target): - os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) - shutil.copyfile(pb.cmake, target) - # Configure and build - builder.configure( - src_dir, - defines=pb.define, - generator=generator, - install_prefix=install_dir, + build_needed = False + if build_needed: + with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: + # Fetch package + src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) + # Install any dependencies first + if not recipe_deps_only: + self.install_deps( + pb, + src_dir=src_dir, test=test, - variant=pb.variant + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=False ) - builder.build(variant=pb.variant) - # Run tests if enabled - if test or test_all: builder.test(variant=pb.variant) - # Install - builder.build(target='install', variant=pb.variant) + with util.cache_lock() as cache_lock: + if not update and use_build_cache and os.path.exists(install_dir): + print("retreived Package {} from cache".format(pb.to_name())) + else: + # Setup cmake file + if pb.cmake: + target = os.path.join(src_dir, 'CMakeLists.txt') + if os.path.exists(target): + os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) + shutil.copyfile(pb.cmake, target) + # Configure and build + builder.configure( + src_dir, + defines=pb.define, + generator=generator, + install_prefix=install_dir, + test=test, + variant=pb.variant + ) + builder.build(variant=pb.variant) + # Run tests if enabled + if test or test_all: builder.test(variant=pb.variant) + # Install + builder.build(target='install', variant=pb.variant) if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) From cd6ce6627919f2189579990e482e6018788ec859 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 22 Jan 2021 16:12:31 +0100 Subject: [PATCH 12/12] fix uninitialized variable --- cget/prefix.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cget/prefix.py b/cget/prefix.py index d3c2234..d08bcb9 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -365,6 +365,7 @@ def install( else: install_dir = pkg_install_dir self.log("using local install dir '%s'" % install_dir) + build_needed = True if recipe_deps_only: self.install_deps( pb,