diff --git a/README.md b/README.md index 6b517dc..e3c77fd 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ Upload local files up to S3. * -s/--sync-check: check md5 hash to avoid uploading the same content. * -f/--force: override existing file instead of showing error message. * -n/--dry-run: emulate the operation without real upload. +* --no-filelinks: do not upload files, if they are symbolic links #### `s4cmd get [source] [target]` @@ -143,6 +144,8 @@ Synchronize the contents of two directories. The directory can either be local o * -f/--force: override existing file instead of showing error message. * -n/--dry-run: emulate the operation without real sync. * --delete-removed: delete files not in source directory. +* --exclude: exclude files with the given pattern(s). Only works for local files to be uploaded to s3 directory. This option can be provided several times. +* --no-filelinks: do not upload (local) files to S3, if they are symbolic links #### `s4cmd sync [source] [target]` @@ -153,6 +156,7 @@ Synchronize the contents of two directories. The directory can either be local o * -f/--force: override existing file instead of showing error message. * -n/--dry-run: emulate the operation without real sync. * --delete-removed: delete files not in source directory. Only works when syncing local directory to s3 directory. +* --no-filelinks: do not upload (local) files, if they are symbolic links #### `s4cmd cp [source] [target]` @@ -263,6 +267,9 @@ before given parameter. Condition on files where their last modified dates are after given parameter. +##### `--exclude` +(local) Filenames, matching the given pattern (https://docs.python.org/3.4/library/fnmatch.html) will not be synced to s3 directory. You may provide this param several times. + ## S3 API Pass-through Options diff --git a/s4cmd.py b/s4cmd.py index b12a57b..05220ea 100755 --- a/s4cmd.py +++ b/s4cmd.py @@ -745,7 +745,7 @@ def local_walk(self, basedir): '''Walk through local directories from root basedir''' result = [] - for root, dirs, files in os.walk(basedir): + for root, dirs, files in os.walk(basedir, followlinks=True): for f in files: result.append(os.path.join(root, f)) return result @@ -994,8 +994,14 @@ def dsync_files(self, source, target): for src, dest in sync_list: pool.download(src, dest) elif not src_s3_url and dst_s3_url: + expats = [] + if self.opt.exclude: + expats = self.opt.exclude + info("excluded patterns: %s", expats) + for src, dest in sync_list: - pool.upload(src, dest) + if not any(fnmatch.fnmatch(src, expat) for expat in expats): + pool.upload(src, dest) elif src_s3_url and dst_s3_url: for src, dest in sync_list: pool.copy(src, dest) @@ -1316,9 +1322,13 @@ def upload(self, source, target, mpi=None, pos=0, chunk=0, part=0): if not mpi: fsize = os.path.getsize(source) md5cache = LocalMD5Cache(source) + filelink = os.path.islink(source) and os.path.isfile(source) # optional checks - if self.opt.dry_run: + if self.opt.no_filelinks and filelink: + message('%s => %s (symlink skipped)', source, target) + return + elif self.opt.dry_run: message('%s => %s', source, target) return elif self.opt.sync_check and self.sync_check(md5cache, obj): @@ -1815,6 +1825,18 @@ def check_dict(self, opt, value): TYPE_CHECKER['datetime'] = check_datetime TYPE_CHECKER['dict'] = check_dict + # enable multiple option values (e.g. --exclude '*.thumbnail.jpg' --exclude '*.layout.jpg') + ACTIONS = optparse.Option.ACTIONS + ("extend",) + STORE_ACTIONS = optparse.Option.STORE_ACTIONS + ("extend",) + TYPED_ACTIONS = optparse.Option.TYPED_ACTIONS + ("extend",) + ALWAYS_TYPED_ACTIONS = optparse.Option.ALWAYS_TYPED_ACTIONS + ("extend",) + + def take_action(self, action, dest, opt, value, values, parser): + if action == "extend": + values.ensure_value(dest, []).append(value) + else: + optparse.Option.take_action(self, action, dest, opt, value, values, parser) + def main(): try: if not sys.argv[0]: sys.argv[0] = '' # Workaround for running with optparse from egg @@ -1910,6 +1932,12 @@ def main(): '--last-modified-after', help='Condition on files where their last modified dates are after given parameter.', type='datetime', default=None) + parser.add_option( + '--exclude', help='Exclude all files from src dsync (file-)list matching the specified pattern', + dest='exclude', action="extend", type='string', default=None) + parser.add_option( + '--no-filelinks', help='do not upload files, which are symbolic links', + dest='no_filelinks', action='store_true', default=False) # Extra S3 API arguments BotoClient.add_options(parser)