Range-based input stream cherry pick

build.gradle

@@ -16,6 +16,41 @@ allprojects {
 
 ext {
 	publishedProjects = []
+    isBuildSnapshot = version.endsWith('-SNAPSHOT')
+    isReleaseVersion = !isBuildSnapshot
+}
+
+group = 'cloud.wondrify'
+
+
+if (isReleaseVersion) {
+    apply plugin: "io.github.gradle-nexus.publish-plugin"
+    nexusPublishing {
+        repositories {
+            sonatype {
+								nexusUrl.set(uri("https://ossrh-staging-api.central.sonatype.com/service/local/"))
+                if(project.hasProperty('mavenUser')) {
+                    username = mavenDavydotcomUser
+                    password = mavenDavydotcomPassword
+                }   
+            }
+        }
+    }
+} else {
+
+    publishing {
+        repositories {
+            maven {
+                url = "http://nexus.bertramlabs.com/content/repositories/snapshots"
+                if(project.hasProperty('labsNexusUser')) {
+                    credentials {
+                        username = labsNexusUser
+                        password = labsNexusPassword
+                    }
+                }
+            }
+        }
+    }
 }
 
 subprojects { project ->

karman-aws/src/main/groovy/com/bertramlabs/plugins/karman/aws/S3CloudFile.groovy

@@ -31,10 +31,7 @@ import org.apache.http.auth.AuthScope
 import org.apache.http.auth.NTCredentials
 import org.apache.http.client.CredentialsProvider
 import org.apache.http.client.HttpClient
-import org.apache.http.client.methods.HttpDelete
 import org.apache.http.client.methods.HttpGet
-import org.apache.http.client.methods.HttpHead
-import org.apache.http.client.methods.HttpPut
 import org.apache.http.client.utils.URIBuilder
 import org.apache.http.config.MessageConstraints
 import org.apache.http.config.Registry
@@ -47,11 +44,8 @@ import org.apache.http.conn.socket.ConnectionSocketFactory
 import org.apache.http.conn.socket.PlainConnectionSocketFactory
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory
 import org.apache.http.conn.ssl.X509HostnameVerifier
-import org.apache.http.entity.InputStreamEntity
-import org.apache.http.entity.StringEntity
 import org.apache.http.impl.DefaultHttpResponseFactory
 import org.apache.http.impl.client.BasicCredentialsProvider
-import org.apache.http.impl.client.DefaultHttpClient
 import org.apache.http.impl.client.HttpClientBuilder
 import org.apache.http.impl.client.HttpClients
 import org.apache.http.impl.client.ProxyAuthenticationStrategy
@@ -67,21 +61,17 @@ import org.apache.http.io.SessionInputBuffer
 import org.apache.http.message.BasicHeader
 import org.apache.http.message.BasicLineParser
 import org.apache.http.message.LineParser
-import org.apache.http.params.HttpConnectionParams
-import org.apache.http.params.HttpParams
 import org.apache.http.protocol.HttpContext
 import org.apache.http.ssl.SSLContexts
 import org.apache.http.util.CharArrayBuffer
-import org.apache.commons.beanutils.PropertyUtils;
-import org.apache.http.util.EntityUtils
+import org.apache.commons.beanutils.PropertyUtils
 
 import javax.net.ssl.SSLContext
 import javax.net.ssl.SSLSession
 import javax.net.ssl.SSLSocket
 import javax.net.ssl.TrustManager
 import javax.net.ssl.X509TrustManager
 import java.lang.reflect.InvocationTargetException
-import java.net.URLEncoder
 import java.security.SecureRandom
 import java.security.cert.X509Certificate
 
@@ -224,10 +214,17 @@ class S3CloudFile extends CloudFile {
 	 * @return inputStream
 	 */
 	InputStream getInputStream() {
-		if(provider.baseUrls && provider.baseUrls[parent.name]) {
+		return getInputStreamInternal()
+	}
+
+	private BufferedInputStream getInputStreamInternal(Long offset = null, Long length = null) {
+		if (provider.baseUrls && provider.baseUrls[parent.name]) {
 			//if we are using a custom base url to fetch it like a cloudfront edge server
 			URIBuilder uriBuilder = new URIBuilder("${provider.baseUrls[parent.name]}/${encodedName}".toString())
 			HttpGet request = new HttpGet(uriBuilder.build())
+			if (offset && length) {
+				request.addHeader('Range', "bytes=$offset-${offset+length}")
+			}
 			HttpClientBuilder clientBuilder = HttpClients.custom()
 			clientBuilder.setHostnameVerifier(new X509HostnameVerifier() {
 				public boolean verify(String host, SSLSession sess) {
@@ -252,15 +249,15 @@ class S3CloudFile extends CloudFile {
 			SSLConnectionSocketFactory sslConnectionFactory = new SSLConnectionSocketFactory(sslcontext) {
 				@Override
 				public Socket connectSocket(int connectTimeout, Socket socket, HttpHost host, InetSocketAddress remoteAddress, InetSocketAddress localAddress, HttpContext context) throws IOException, ConnectTimeoutException {
-					if(socket instanceof SSLSocket) {
+					if (socket instanceof SSLSocket) {
 						try {
 							socket.setEnabledProtocols(['SSLv3', 'TLSv1', 'TLSv1.1', 'TLSv1.2'] as String[])
 							PropertyUtils.setProperty(socket, "host", host.getHostName());
-						} catch(NoSuchMethodException ex) {
+						} catch (NoSuchMethodException ex) {
 						}
-						catch(IllegalAccessException ex) {
+						catch (IllegalAccessException ex) {
 						}
-						catch(InvocationTargetException ex) {
+						catch (InvocationTargetException ex) {
 						}
 					}
 					return super.connectSocket(30000, socket, host, remoteAddress, localAddress, context)
@@ -283,12 +280,12 @@ class S3CloudFile extends CloudFile {
 
 					};
 					return new DefaultHttpResponseParser(
-						ibuffer, lineParser, DefaultHttpResponseFactory.INSTANCE, constraints ?: MessageConstraints.DEFAULT) {
+							ibuffer, lineParser, DefaultHttpResponseFactory.INSTANCE, constraints ?: MessageConstraints.DEFAULT) {
 
 						@Override
 						protected boolean reject(final CharArrayBuffer line, int count) {
 							//We need to break out of forever head reads
-							if(count > 100) {
+							if (count > 100) {
 								return true
 							}
 							return false;
@@ -301,25 +298,25 @@ class S3CloudFile extends CloudFile {
 			};
 			clientBuilder.setSSLSocketFactory(sslConnectionFactory)
 			Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory> create()
-				.register("https", sslConnectionFactory)
-				.register("http", PlainConnectionSocketFactory.INSTANCE)
-				.build();
+					.register("https", sslConnectionFactory)
+					.register("http", PlainConnectionSocketFactory.INSTANCE)
+					.build();
 
 			HttpMessageWriterFactory<HttpRequest> requestWriterFactory = new DefaultHttpRequestWriterFactory();
 
 			HttpConnectionFactory<HttpRoute, ManagedHttpClientConnection> connFactory = new ManagedHttpClientConnectionFactory(
-				requestWriterFactory, responseParserFactory);
+					requestWriterFactory, responseParserFactory);
 			BasicHttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry, connFactory)
 
 			clientBuilder.setConnectionManager(connectionManager)
 
 			//Proxy Settings
-			if(provider.proxyHost) {
+			if (provider.proxyHost) {
 				clientBuilder.setProxy(new HttpHost(provider.proxyHost, provider.proxyPort))
-				if(provider.proxyUser) {
+				if (provider.proxyUser) {
 					CredentialsProvider credsProvider = new BasicCredentialsProvider();
 					NTCredentials ntCreds = new NTCredentials(provider.proxyUser, provider.proxyPassword, provider.proxyWorkstation, provider.proxyDomain)
-					credsProvider.setCredentials(new AuthScope(provider.proxyHost,provider.proxyPort), ntCreds)
+					credsProvider.setCredentials(new AuthScope(provider.proxyHost, provider.proxyPort), ntCreds)
 
 					clientBuilder.setDefaultCredentialsProvider(credsProvider)
 					clientBuilder.setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy())
@@ -331,7 +328,7 @@ class S3CloudFile extends CloudFile {
 			HttpEntity entity = response.getEntity()
 			return new BufferedInputStream(entity.content, 8000)
 		} else {
-			loadObject()
+			loadObject(offset, length)
 			return new BufferedInputStream(s3Object.objectContent, 8000)
 		}
 	}
@@ -535,9 +532,13 @@ class S3CloudFile extends CloudFile {
 		object
 	}
 
-	private void loadObject() {
+	private void loadObject(Long offset = null, Long length = null) {
 		if(valid) {
-			object = s3Client.getObject(parent.name, name)
+			def req = new GetObjectRequest(parent.name, name)
+			if (offset && length) {
+				req.setRange(offset, offset+length)
+			}
+			object = s3Client.getObject(req)
 			loaded = true
 			metaDataLoaded = false
 		}
@@ -580,4 +581,20 @@ class S3CloudFile extends CloudFile {
 		// set up a TrustManager that trusts everything
 		sslContext.init(null, trustAllCerts, new SecureRandom());
 	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	boolean supportsRangeBasedInputStream() {
+		return true
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	InputStream getInputStream(long offset, long length) {
+		getInputStreamInternal(offset, length)
+	}
 }

karman-aws/src/main/groovy/com/bertramlabs/plugins/karman/aws/S3StorageProvider.groovy

@@ -151,6 +151,13 @@ class S3StorageProvider extends StorageProvider {
         proxyPassword = options.proxyPassword ?: proxyPassword
         proxyDomain = options.proxyDomain ?: proxyDomain
         noProxy = options.noProxy ?: noProxy
+		if (noProxy) {
+			// aws sdk doesn't support `,` as a separator in no proxy, they follow the standard
+			// for java 'http.nonProxyHosts' you need to use pipes instead
+			// see https://github.com/aws/aws-sdk-java-v2/issues/5573
+			// and https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/net/doc-files/net-properties.html
+			noProxy = noProxy.replace(',', '|')
+		}
         proxyWorkstation = options.proxyWorkstation ?: proxyWorkstation
         chunkSize = options.chunkSize ?: chunkSize
 		tempDir = options.tempDir ?: tempDir

karman-cifs/src/main/groovy/com/bertramlabs/plugins/karman/cifs/CifsCloudFile.groovy

@@ -218,4 +218,19 @@ class CifsCloudFile extends CloudFile {
     }
   }
 
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	InputStream getInputStream(long offset, long length) {
+		return new SmbRandomAccessFileInputStream(getCifsFile(), offset)
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	boolean supportsRangeBasedInputStream() {
+		return true
+	}
 }

karman-cifs/src/main/groovy/com/bertramlabs/plugins/karman/cifs/SmbRandomAccessFileInputStream.groovy

@@ -0,0 +1,81 @@
+package com.bertramlabs.plugins.karman.cifs
+
+import jcifs.smb.SmbFile
+import jcifs.smb.SmbRandomAccessFile
+
+/**
+ * SMB input stream for random access into a SmbFile
+ * This allows starting at a specific offset within the smbfile
+ */
+class SmbRandomAccessFileInputStream extends InputStream {
+	private final SmbRandomAccessFile file
+
+	SmbRandomAccessFileInputStream(SmbFile smbFile, long offset) throws IOException {
+		if (smbFile == null) {
+			throw new IllegalArgumentException("smbFile cannot be null")
+		}
+		if (offset < 0) {
+			throw new IllegalArgumentException("offset cannot be negative")
+		}
+		this.file = new SmbRandomAccessFile(smbFile, "r")
+		try {
+			this.file.seek(offset)
+		} catch (IOException e) {
+			this.file.close()
+			throw e
+		}
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	int read() throws IOException {
+		return file.read()
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	int read(byte[] b) throws IOException {
+		return file.read(b)
+	}
+
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	int read(byte[] b, int off, int len) throws IOException {
+		return file.read(b, off, len)
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	long skip(long n) throws IOException {
+		long pos = file.getFilePointer()
+		long len = file.length()
+		long newPos = Math.min(pos + n, len)
+		file.seek(newPos)
+		return newPos - pos
+	}
+
+	@Override
+	int available() throws IOException {
+		long remaining = file.length() - file.getFilePointer()
+		if (remaining <= 0) {
+			return 0
+		}
+		return (int) Math.min(remaining, Integer.MAX_VALUE)
+	}
+
+	@Override
+	void close() throws IOException {
+		file.close()
+	}
+
+}
+

karman-core/src/main/groovy/com/bertramlabs/plugins/karman/CloudFile.groovy

@@ -159,6 +159,17 @@ abstract class CloudFile implements CloudFileInterface {
 		}
 	}
 
+	@Override
+	InputStream getInputStream(long offset, long length) {
+		throw new UnsupportedOperationException('Range based input streams are not supported.')
+	}
 
+	@Override
+	boolean supportsRangeBasedInputStream() {
+		return false
+	}
 
+	// to satisfy the groovy static typechecker since we don't have a default impl
+	// for the parameterless variant
+	abstract InputStream getInputStream()
 }

karman-core/src/main/groovy/com/bertramlabs/plugins/karman/CloudFileInterface.groovy

@@ -29,6 +29,16 @@ interface CloudFileInterface {
 	URL getURL()
 
 	InputStream getInputStream()
+
+	/**
+	 * Returns an input stream over a range of bytes of the CloudFile specified by offset and length.
+	 *
+	 * Note: Mark support for this by overriding {@link CloudFileInterface#supportsRangeBasedInputStream} to return true.
+	 * @param offset the offset this stream will start at in the CloudFile
+	 * @param length the total length of this chunk in the CloudFile
+	 * @return ranged input stream
+	 */
+	InputStream getInputStream(long offset,long length)
 	void setInputStream(InputStream is)
 	OutputStream getOutputStream()
 
@@ -72,4 +82,10 @@ interface CloudFileInterface {
 	def getMetaAttributes()
 
 	void removeMetaAttribute(key)
+
+	/**
+	 * Indicates if we support range based input streams via {@link CloudFileInterface#getInputStream(long, long)}
+	 * @return True if this CloudFile supports range based input streams
+	 */
+	boolean supportsRangeBasedInputStream()
 }

karman-differential/src/main/groovy/com/bertramlabs/plugins/karman/differential/DifferentialCloudFile.groovy

@@ -31,7 +31,7 @@ public class DifferentialCloudFile extends CloudFile {
 	@Override
 	@CompileStatic
 	InputStream getInputStream() {
-		CloudFile manifestFile = parent.sourceDirectory[sourceFile.name + "/karman.diff"]
+		CloudFileInterface manifestFile = parent.sourceDirectory[sourceFile.name + "/karman.diff"]
 		if(manifestFile.exists()) {
 			//we need to copy the index table to local storage for read in case of slow connections
 			Path localManifestCache = Files.createTempFile("karman",".diff")