diff --git a/build.gradle b/build.gradle index 490e286..1f6932d 100644 --- a/build.gradle +++ b/build.gradle @@ -16,6 +16,41 @@ allprojects { ext { publishedProjects = [] + isBuildSnapshot = version.endsWith('-SNAPSHOT') + isReleaseVersion = !isBuildSnapshot +} + +group = 'cloud.wondrify' + + +if (isReleaseVersion) { + apply plugin: "io.github.gradle-nexus.publish-plugin" + nexusPublishing { + repositories { + sonatype { + nexusUrl.set(uri("https://ossrh-staging-api.central.sonatype.com/service/local/")) + if(project.hasProperty('mavenUser')) { + username = mavenDavydotcomUser + password = mavenDavydotcomPassword + } + } + } + } +} else { + + publishing { + repositories { + maven { + url = "http://nexus.bertramlabs.com/content/repositories/snapshots" + if(project.hasProperty('labsNexusUser')) { + credentials { + username = labsNexusUser + password = labsNexusPassword + } + } + } + } + } } subprojects { project -> diff --git a/karman-aws/src/main/groovy/com/bertramlabs/plugins/karman/aws/S3CloudFile.groovy b/karman-aws/src/main/groovy/com/bertramlabs/plugins/karman/aws/S3CloudFile.groovy index 9b67e54..f0efbe1 100644 --- a/karman-aws/src/main/groovy/com/bertramlabs/plugins/karman/aws/S3CloudFile.groovy +++ b/karman-aws/src/main/groovy/com/bertramlabs/plugins/karman/aws/S3CloudFile.groovy @@ -31,10 +31,7 @@ import org.apache.http.auth.AuthScope import org.apache.http.auth.NTCredentials import org.apache.http.client.CredentialsProvider import org.apache.http.client.HttpClient -import org.apache.http.client.methods.HttpDelete import org.apache.http.client.methods.HttpGet -import org.apache.http.client.methods.HttpHead -import org.apache.http.client.methods.HttpPut import org.apache.http.client.utils.URIBuilder import org.apache.http.config.MessageConstraints import org.apache.http.config.Registry @@ -47,11 +44,8 @@ import org.apache.http.conn.socket.ConnectionSocketFactory import org.apache.http.conn.socket.PlainConnectionSocketFactory import org.apache.http.conn.ssl.SSLConnectionSocketFactory import org.apache.http.conn.ssl.X509HostnameVerifier -import org.apache.http.entity.InputStreamEntity -import org.apache.http.entity.StringEntity import org.apache.http.impl.DefaultHttpResponseFactory import org.apache.http.impl.client.BasicCredentialsProvider -import org.apache.http.impl.client.DefaultHttpClient import org.apache.http.impl.client.HttpClientBuilder import org.apache.http.impl.client.HttpClients import org.apache.http.impl.client.ProxyAuthenticationStrategy @@ -67,13 +61,10 @@ import org.apache.http.io.SessionInputBuffer import org.apache.http.message.BasicHeader import org.apache.http.message.BasicLineParser import org.apache.http.message.LineParser -import org.apache.http.params.HttpConnectionParams -import org.apache.http.params.HttpParams import org.apache.http.protocol.HttpContext import org.apache.http.ssl.SSLContexts import org.apache.http.util.CharArrayBuffer -import org.apache.commons.beanutils.PropertyUtils; -import org.apache.http.util.EntityUtils +import org.apache.commons.beanutils.PropertyUtils import javax.net.ssl.SSLContext import javax.net.ssl.SSLSession @@ -81,7 +72,6 @@ import javax.net.ssl.SSLSocket import javax.net.ssl.TrustManager import javax.net.ssl.X509TrustManager import java.lang.reflect.InvocationTargetException -import java.net.URLEncoder import java.security.SecureRandom import java.security.cert.X509Certificate @@ -224,10 +214,17 @@ class S3CloudFile extends CloudFile { * @return inputStream */ InputStream getInputStream() { - if(provider.baseUrls && provider.baseUrls[parent.name]) { + return getInputStreamInternal() + } + + private BufferedInputStream getInputStreamInternal(Long offset = null, Long length = null) { + if (provider.baseUrls && provider.baseUrls[parent.name]) { //if we are using a custom base url to fetch it like a cloudfront edge server URIBuilder uriBuilder = new URIBuilder("${provider.baseUrls[parent.name]}/${encodedName}".toString()) HttpGet request = new HttpGet(uriBuilder.build()) + if (offset && length) { + request.addHeader('Range', "bytes=$offset-${offset+length}") + } HttpClientBuilder clientBuilder = HttpClients.custom() clientBuilder.setHostnameVerifier(new X509HostnameVerifier() { public boolean verify(String host, SSLSession sess) { @@ -252,15 +249,15 @@ class S3CloudFile extends CloudFile { SSLConnectionSocketFactory sslConnectionFactory = new SSLConnectionSocketFactory(sslcontext) { @Override public Socket connectSocket(int connectTimeout, Socket socket, HttpHost host, InetSocketAddress remoteAddress, InetSocketAddress localAddress, HttpContext context) throws IOException, ConnectTimeoutException { - if(socket instanceof SSLSocket) { + if (socket instanceof SSLSocket) { try { socket.setEnabledProtocols(['SSLv3', 'TLSv1', 'TLSv1.1', 'TLSv1.2'] as String[]) PropertyUtils.setProperty(socket, "host", host.getHostName()); - } catch(NoSuchMethodException ex) { + } catch (NoSuchMethodException ex) { } - catch(IllegalAccessException ex) { + catch (IllegalAccessException ex) { } - catch(InvocationTargetException ex) { + catch (InvocationTargetException ex) { } } return super.connectSocket(30000, socket, host, remoteAddress, localAddress, context) @@ -283,12 +280,12 @@ class S3CloudFile extends CloudFile { }; return new DefaultHttpResponseParser( - ibuffer, lineParser, DefaultHttpResponseFactory.INSTANCE, constraints ?: MessageConstraints.DEFAULT) { + ibuffer, lineParser, DefaultHttpResponseFactory.INSTANCE, constraints ?: MessageConstraints.DEFAULT) { @Override protected boolean reject(final CharArrayBuffer line, int count) { //We need to break out of forever head reads - if(count > 100) { + if (count > 100) { return true } return false; @@ -301,25 +298,25 @@ class S3CloudFile extends CloudFile { }; clientBuilder.setSSLSocketFactory(sslConnectionFactory) Registry registry = RegistryBuilder. create() - .register("https", sslConnectionFactory) - .register("http", PlainConnectionSocketFactory.INSTANCE) - .build(); + .register("https", sslConnectionFactory) + .register("http", PlainConnectionSocketFactory.INSTANCE) + .build(); HttpMessageWriterFactory requestWriterFactory = new DefaultHttpRequestWriterFactory(); HttpConnectionFactory connFactory = new ManagedHttpClientConnectionFactory( - requestWriterFactory, responseParserFactory); + requestWriterFactory, responseParserFactory); BasicHttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry, connFactory) clientBuilder.setConnectionManager(connectionManager) //Proxy Settings - if(provider.proxyHost) { + if (provider.proxyHost) { clientBuilder.setProxy(new HttpHost(provider.proxyHost, provider.proxyPort)) - if(provider.proxyUser) { + if (provider.proxyUser) { CredentialsProvider credsProvider = new BasicCredentialsProvider(); NTCredentials ntCreds = new NTCredentials(provider.proxyUser, provider.proxyPassword, provider.proxyWorkstation, provider.proxyDomain) - credsProvider.setCredentials(new AuthScope(provider.proxyHost,provider.proxyPort), ntCreds) + credsProvider.setCredentials(new AuthScope(provider.proxyHost, provider.proxyPort), ntCreds) clientBuilder.setDefaultCredentialsProvider(credsProvider) clientBuilder.setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy()) @@ -331,7 +328,7 @@ class S3CloudFile extends CloudFile { HttpEntity entity = response.getEntity() return new BufferedInputStream(entity.content, 8000) } else { - loadObject() + loadObject(offset, length) return new BufferedInputStream(s3Object.objectContent, 8000) } } @@ -535,9 +532,13 @@ class S3CloudFile extends CloudFile { object } - private void loadObject() { + private void loadObject(Long offset = null, Long length = null) { if(valid) { - object = s3Client.getObject(parent.name, name) + def req = new GetObjectRequest(parent.name, name) + if (offset && length) { + req.setRange(offset, offset+length) + } + object = s3Client.getObject(req) loaded = true metaDataLoaded = false } @@ -580,4 +581,20 @@ class S3CloudFile extends CloudFile { // set up a TrustManager that trusts everything sslContext.init(null, trustAllCerts, new SecureRandom()); } + + /** + * {@inheritDoc} + */ + @Override + boolean supportsRangeBasedInputStream() { + return true + } + + /** + * {@inheritDoc} + */ + @Override + InputStream getInputStream(long offset, long length) { + getInputStreamInternal(offset, length) + } } diff --git a/karman-aws/src/main/groovy/com/bertramlabs/plugins/karman/aws/S3StorageProvider.groovy b/karman-aws/src/main/groovy/com/bertramlabs/plugins/karman/aws/S3StorageProvider.groovy index 5907788..c95bcb2 100644 --- a/karman-aws/src/main/groovy/com/bertramlabs/plugins/karman/aws/S3StorageProvider.groovy +++ b/karman-aws/src/main/groovy/com/bertramlabs/plugins/karman/aws/S3StorageProvider.groovy @@ -151,6 +151,13 @@ class S3StorageProvider extends StorageProvider { proxyPassword = options.proxyPassword ?: proxyPassword proxyDomain = options.proxyDomain ?: proxyDomain noProxy = options.noProxy ?: noProxy + if (noProxy) { + // aws sdk doesn't support `,` as a separator in no proxy, they follow the standard + // for java 'http.nonProxyHosts' you need to use pipes instead + // see https://github.com/aws/aws-sdk-java-v2/issues/5573 + // and https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/net/doc-files/net-properties.html + noProxy = noProxy.replace(',', '|') + } proxyWorkstation = options.proxyWorkstation ?: proxyWorkstation chunkSize = options.chunkSize ?: chunkSize tempDir = options.tempDir ?: tempDir diff --git a/karman-cifs/src/main/groovy/com/bertramlabs/plugins/karman/cifs/CifsCloudFile.groovy b/karman-cifs/src/main/groovy/com/bertramlabs/plugins/karman/cifs/CifsCloudFile.groovy index 8c9fef9..2b971f9 100644 --- a/karman-cifs/src/main/groovy/com/bertramlabs/plugins/karman/cifs/CifsCloudFile.groovy +++ b/karman-cifs/src/main/groovy/com/bertramlabs/plugins/karman/cifs/CifsCloudFile.groovy @@ -218,4 +218,19 @@ class CifsCloudFile extends CloudFile { } } + /** + * {@inheritDoc} + */ + @Override + InputStream getInputStream(long offset, long length) { + return new SmbRandomAccessFileInputStream(getCifsFile(), offset) + } + + /** + * {@inheritDoc} + */ + @Override + boolean supportsRangeBasedInputStream() { + return true + } } diff --git a/karman-cifs/src/main/groovy/com/bertramlabs/plugins/karman/cifs/SmbRandomAccessFileInputStream.groovy b/karman-cifs/src/main/groovy/com/bertramlabs/plugins/karman/cifs/SmbRandomAccessFileInputStream.groovy new file mode 100644 index 0000000..3b737ec --- /dev/null +++ b/karman-cifs/src/main/groovy/com/bertramlabs/plugins/karman/cifs/SmbRandomAccessFileInputStream.groovy @@ -0,0 +1,81 @@ +package com.bertramlabs.plugins.karman.cifs + +import jcifs.smb.SmbFile +import jcifs.smb.SmbRandomAccessFile + +/** + * SMB input stream for random access into a SmbFile + * This allows starting at a specific offset within the smbfile + */ +class SmbRandomAccessFileInputStream extends InputStream { + private final SmbRandomAccessFile file + + SmbRandomAccessFileInputStream(SmbFile smbFile, long offset) throws IOException { + if (smbFile == null) { + throw new IllegalArgumentException("smbFile cannot be null") + } + if (offset < 0) { + throw new IllegalArgumentException("offset cannot be negative") + } + this.file = new SmbRandomAccessFile(smbFile, "r") + try { + this.file.seek(offset) + } catch (IOException e) { + this.file.close() + throw e + } + } + + /** + * {@inheritDoc} + */ + @Override + int read() throws IOException { + return file.read() + } + + /** + * {@inheritDoc} + */ + @Override + int read(byte[] b) throws IOException { + return file.read(b) + } + + + /** + * {@inheritDoc} + */ + @Override + int read(byte[] b, int off, int len) throws IOException { + return file.read(b, off, len) + } + + /** + * {@inheritDoc} + */ + @Override + long skip(long n) throws IOException { + long pos = file.getFilePointer() + long len = file.length() + long newPos = Math.min(pos + n, len) + file.seek(newPos) + return newPos - pos + } + + @Override + int available() throws IOException { + long remaining = file.length() - file.getFilePointer() + if (remaining <= 0) { + return 0 + } + return (int) Math.min(remaining, Integer.MAX_VALUE) + } + + @Override + void close() throws IOException { + file.close() + } + +} + diff --git a/karman-core/src/main/groovy/com/bertramlabs/plugins/karman/CloudFile.groovy b/karman-core/src/main/groovy/com/bertramlabs/plugins/karman/CloudFile.groovy index 366d6b8..3fc246f 100644 --- a/karman-core/src/main/groovy/com/bertramlabs/plugins/karman/CloudFile.groovy +++ b/karman-core/src/main/groovy/com/bertramlabs/plugins/karman/CloudFile.groovy @@ -159,6 +159,17 @@ abstract class CloudFile implements CloudFileInterface { } } + @Override + InputStream getInputStream(long offset, long length) { + throw new UnsupportedOperationException('Range based input streams are not supported.') + } + @Override + boolean supportsRangeBasedInputStream() { + return false + } + // to satisfy the groovy static typechecker since we don't have a default impl + // for the parameterless variant + abstract InputStream getInputStream() } diff --git a/karman-core/src/main/groovy/com/bertramlabs/plugins/karman/CloudFileInterface.groovy b/karman-core/src/main/groovy/com/bertramlabs/plugins/karman/CloudFileInterface.groovy index 0fb356b..77e7677 100644 --- a/karman-core/src/main/groovy/com/bertramlabs/plugins/karman/CloudFileInterface.groovy +++ b/karman-core/src/main/groovy/com/bertramlabs/plugins/karman/CloudFileInterface.groovy @@ -29,6 +29,16 @@ interface CloudFileInterface { URL getURL() InputStream getInputStream() + + /** + * Returns an input stream over a range of bytes of the CloudFile specified by offset and length. + * + * Note: Mark support for this by overriding {@link CloudFileInterface#supportsRangeBasedInputStream} to return true. + * @param offset the offset this stream will start at in the CloudFile + * @param length the total length of this chunk in the CloudFile + * @return ranged input stream + */ + InputStream getInputStream(long offset,long length) void setInputStream(InputStream is) OutputStream getOutputStream() @@ -72,4 +82,10 @@ interface CloudFileInterface { def getMetaAttributes() void removeMetaAttribute(key) + + /** + * Indicates if we support range based input streams via {@link CloudFileInterface#getInputStream(long, long)} + * @return True if this CloudFile supports range based input streams + */ + boolean supportsRangeBasedInputStream() } diff --git a/karman-differential/src/main/groovy/com/bertramlabs/plugins/karman/differential/DifferentialCloudFile.groovy b/karman-differential/src/main/groovy/com/bertramlabs/plugins/karman/differential/DifferentialCloudFile.groovy index c6a6885..a411074 100644 --- a/karman-differential/src/main/groovy/com/bertramlabs/plugins/karman/differential/DifferentialCloudFile.groovy +++ b/karman-differential/src/main/groovy/com/bertramlabs/plugins/karman/differential/DifferentialCloudFile.groovy @@ -31,7 +31,7 @@ public class DifferentialCloudFile extends CloudFile { @Override @CompileStatic InputStream getInputStream() { - CloudFile manifestFile = parent.sourceDirectory[sourceFile.name + "/karman.diff"] + CloudFileInterface manifestFile = parent.sourceDirectory[sourceFile.name + "/karman.diff"] if(manifestFile.exists()) { //we need to copy the index table to local storage for read in case of slow connections Path localManifestCache = Files.createTempFile("karman",".diff") diff --git a/karman-nfs/src/main/groovy/com/bertramlabs/plugins/karman/nfs/NfsCloudFile.groovy b/karman-nfs/src/main/groovy/com/bertramlabs/plugins/karman/nfs/NfsCloudFile.groovy index 88551e5..8878fdf 100644 --- a/karman-nfs/src/main/groovy/com/bertramlabs/plugins/karman/nfs/NfsCloudFile.groovy +++ b/karman-nfs/src/main/groovy/com/bertramlabs/plugins/karman/nfs/NfsCloudFile.groovy @@ -190,4 +190,19 @@ class NfsCloudFile extends CloudFile{ log.warn("Karman CloudFile Meta Attributes Not Available for NfsCloudFile") } + /** + * {@inheritDoc} + */ + @Override + boolean supportsRangeBasedInputStream() { + return true + } + + @Override + InputStream getInputStream(long offset, long length) { + if(baseFile.exists()) { + return new NfsFileInputStream(baseFile, offset) + } + return null + } } diff --git a/karman-nfs/src/main/groovy/com/bertramlabs/plugins/karman/nfs/NfsFileInputStream.java b/karman-nfs/src/main/groovy/com/bertramlabs/plugins/karman/nfs/NfsFileInputStream.java index 1f77985..995478a 100644 --- a/karman-nfs/src/main/groovy/com/bertramlabs/plugins/karman/nfs/NfsFileInputStream.java +++ b/karman-nfs/src/main/groovy/com/bertramlabs/plugins/karman/nfs/NfsFileInputStream.java @@ -114,6 +114,29 @@ public NfsFileInputStream(NfsFile nfsFile, long offset, int maximumBufferS _bytes = makeBytes(maximumBufferSize); } + /** + * Creates a NfsFileInputStream by opening a connection to an + * actual NFS file, using the specified offset and + * maximumBufferSize. + *

+ * If the named file does not exist, is a directory rather than a regular + * file, or for some other reason cannot be opened for reading then a + * FileNotFoundException is thrown. + *

+ * + * @param nfsFile + * The NFS file instance to be read. + * @param offset + * The offset at which reading should start, in bytes. + * @throws IOException + * If the file does not exist, is a directory rather than a + * regular file, or for some other reason cannot be opened for + * reading. + */ + public NfsFileInputStream(NfsFile nfsFile, long offset) throws IOException { + this(nfsFile, offset, (int) Math.min(nfsFile.fsinfo().getFsInfo().rtpref, Integer.MAX_VALUE)); + } + /** * Creates a NfsFileInputStream by opening a connection to an * actual NFS file, starting to read at offset 0 and using the specified