From 48085d98c27de76329ccf74253f79b7189bc1b8b Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Sat, 1 Aug 2026 14:32:34 +0200 Subject: [PATCH 1/2] fix(aws): stop s3fs caching missing FC/kernel/envd objects The four s3fs mounts passed "-o enable_noobj_cache", which is the s3fs default rather than an override, so negative lookups were cached for stat_cache_expire (900s). A host that looked for a Firecracker/kernel/envd version before it was uploaded kept returning ENOENT for 15 minutes after the upload landed, and the only workaround was to unmount and remount. Pass "-o disable_noobj_cache" instead so a missed lookup is retried against S3 on the next access. Positive stat cache entries are unaffected. --- .../nodepool-client/scripts/start-client.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/iac/provider-aws/modules/nodepool-client/scripts/start-client.sh b/iac/provider-aws/modules/nodepool-client/scripts/start-client.sh index bf9b561956..50cf271aa9 100755 --- a/iac/provider-aws/modules/nodepool-client/scripts/start-client.sh +++ b/iac/provider-aws/modules/nodepool-client/scripts/start-client.sh @@ -75,24 +75,32 @@ modprobe nbd nbds_max=4096 mkdir -p /fc-vm # Mount envd buckets +# +# disable_noobj_cache: s3fs caches "this object does not exist" answers for +# stat_cache_expire (900s by default). These buckets are written out-of-band - +# a new Firecracker/kernel/envd version is uploaded after a host has already +# looked for it - so a cached miss makes the freshly uploaded object invisible +# on that host until the entry expires or the mount is recreated. +# Note that "enable_noobj_cache" is the s3fs default, so passing it was a no-op; +# "disable_noobj_cache" is the option that actually turns the negative cache off. envd_dir="/fc-envd" mkdir -p $envd_dir -s3fs "${FC_ENV_PIPELINE_BUCKET_NAME}" "$envd_dir" -o allow_other -o umask=000 -o nonempty -o iam_role -o enable_noobj_cache +s3fs "${FC_ENV_PIPELINE_BUCKET_NAME}" "$envd_dir" -o allow_other -o umask=000 -o nonempty -o iam_role -o disable_noobj_cache # Mount kernels kernels_dir="/fc-kernels" mkdir -p $kernels_dir -s3fs "${FC_KERNELS_BUCKET_NAME}" "$kernels_dir" -o allow_other -o umask=000 -o nonempty -o iam_role -o enable_noobj_cache +s3fs "${FC_KERNELS_BUCKET_NAME}" "$kernels_dir" -o allow_other -o umask=000 -o nonempty -o iam_role -o disable_noobj_cache # Mount FC versions fc_versions_dir="/fc-versions" mkdir -p $fc_versions_dir -s3fs "${FC_VERSIONS_BUCKET_NAME}" "$fc_versions_dir" -o allow_other -o umask=000 -o nonempty -o iam_role -o enable_noobj_cache +s3fs "${FC_VERSIONS_BUCKET_NAME}" "$fc_versions_dir" -o allow_other -o umask=000 -o nonempty -o iam_role -o disable_noobj_cache # Mount busybox busybox_dir="/fc-busybox" mkdir -p $busybox_dir -s3fs "${FC_BUSYBOX_BUCKET_NAME}" "$busybox_dir" -o allow_other -o umask=000 -o nonempty -o iam_role -o enable_noobj_cache +s3fs "${FC_BUSYBOX_BUCKET_NAME}" "$busybox_dir" -o allow_other -o umask=000 -o nonempty -o iam_role -o disable_noobj_cache # These variables are passed in via Terraform template interpolation aws s3 cp "s3://${SCRIPTS_BUCKET}/run-consul-${RUN_CONSUL_FILE_HASH}.sh" /opt/consul/bin/run-consul.sh From ed2d6bb928eb4685e2ef70a93deb961e851f1c9b Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 3 Aug 2026 11:22:35 +0200 Subject: [PATCH 2/2] style: strip narration comments Co-Authored-By: Claude Fable 5 --- .../modules/nodepool-client/scripts/start-client.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/iac/provider-aws/modules/nodepool-client/scripts/start-client.sh b/iac/provider-aws/modules/nodepool-client/scripts/start-client.sh index 50cf271aa9..7565a829c1 100755 --- a/iac/provider-aws/modules/nodepool-client/scripts/start-client.sh +++ b/iac/provider-aws/modules/nodepool-client/scripts/start-client.sh @@ -75,14 +75,8 @@ modprobe nbd nbds_max=4096 mkdir -p /fc-vm # Mount envd buckets -# -# disable_noobj_cache: s3fs caches "this object does not exist" answers for -# stat_cache_expire (900s by default). These buckets are written out-of-band - -# a new Firecracker/kernel/envd version is uploaded after a host has already -# looked for it - so a cached miss makes the freshly uploaded object invisible -# on that host until the entry expires or the mount is recreated. -# Note that "enable_noobj_cache" is the s3fs default, so passing it was a no-op; -# "disable_noobj_cache" is the option that actually turns the negative cache off. +# disable_noobj_cache: s3fs caches negative lookups for stat_cache_expire (900s +# default), hiding freshly uploaded envd/kernel/FC versions until the entry expires. envd_dir="/fc-envd" mkdir -p $envd_dir s3fs "${FC_ENV_PIPELINE_BUCKET_NAME}" "$envd_dir" -o allow_other -o umask=000 -o nonempty -o iam_role -o disable_noobj_cache