New deploy service

lib/remote-storage.js

@@ -10,162 +10,162 @@ OF ANY KIND, either express or implied. See the License for the specific languag
 governing permissions and limitations under the License.
 */
 
-const { S3 } = require('@aws-sdk/client-s3')
 const path = require('path')
 const mime = require('mime-types')
 const fs = require('fs-extra')
-const joi = require('joi')
 const klaw = require('klaw')
 const http = require('http')
-// Proxy support for AWS SDK v3 (inspired by PR #224 by pat-lego, with compatibility fixes)
-const { NodeHttpHandler } = require('@smithy/node-http-handler')
-const { ProxyAgent } = require('proxy-agent')
+const { EnvHttpProxyAgent } = require('undici')
 const { codes, logAndThrow } = require('./StorageError')
+const aioLibEnv = require('@adobe/aio-lib-env')
 
-const fileExtensionPattern = /\*\.[0-9a-zA-Z]+$/
-
-module.exports = class RemoteStorage {
-  /**
-   * @param  {object} creds
-   * @param  {string} creds.accessKeyId
-   * @param  {string} creds.secretAccessKey
-   * @param  {string} creds.params.Bucket
-   * @param  {string} [creds.sessionToken]
-   */
-  constructor (creds) {
-    const res = joi.object().keys({
-      sessionToken: joi.string(),
-      accessKeyId: joi.string().required(),
-      secretAccessKey: joi.string().required(),
-      // hacky needs s3Bucket in creds.params.Bucket
-      params: joi.object().keys({ Bucket: joi.string().required() }).required()
-    }).unknown()
-      .validate(creds)
-    if (res.error) {
-      throw res.error
-    }
-
-    // the TVM response could be passed as is to the v2 client constructor, but the v3 client follows a different format
-    // see https://github.com/adobe/aio-tvm/issues/85
-    const region = creds.region || 'us-east-1'
-    // note this must supports TVM + BYO use cases
-    // see https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/credentials.html
-    const credentials = {
-      accessKeyId: creds.accessKeyId,
-      secretAccessKey: creds.secretAccessKey,
-      sessionToken: creds.sessionToken,
-      expiration: creds.expiration ? new Date(creds.expiration) : undefined
-    }
-    this.bucket = creds.params.Bucket
+// Set AIO_DEPLOYMENT_SERVICE_URL env var to override the default deploy-service url
+// dev deploy => https://deploy-service.dev.app-builder.adp.adobe.io
+// run local =>  http://localhost:3000
+const getDeploymentServiceUrl = () => {
+  if (process.env.AIO_DEPLOYMENT_SERVICE_URL) {
+    return process.env.AIO_DEPLOYMENT_SERVICE_URL
+  }
+  return aioLibEnv.getCliEnv() === aioLibEnv.PROD_ENV
+    ? 'https://deploy-service.app-builder.adp.adobe.io'
+    : 'https://deploy-service.stg.app-builder.corp.adp.adobe.io'
+}
 
-    // Configure proxy support for AWS SDK v3
-    // ProxyAgent automatically handles proxy environment variables via proxy-from-env
-    const agent = new ProxyAgent()
-    const s3Config = {
-      credentials,
-      region,
-      requestHandler: new NodeHttpHandler({
-        httpAgent: agent,
-        httpsAgent: agent
-      })
-    }
+const fileExtensionPattern = /\*\.[0-9a-zA-Z]+$/
 
-    // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/classes/s3.html#constructor
-    this.s3 = new S3(s3Config)
+let fetchDispatcher
+const getFetchDispatcher = () => {
+  if (!fetchDispatcher) {
+    fetchDispatcher = new EnvHttpProxyAgent()
   }
+  return fetchDispatcher
+}
+
+const deployServiceFetch = (url, init) => {
+  return fetch(url, { ...init, dispatcher: getFetchDispatcher() })
+}
 
+module.exports = class RemoteStorage {
   /**
-   * Checks if prefix exists
-   * @param  {string} prefix
-   * @returns {boolean}
+   * Checks if any files exist for the namespace
+   * @param  {string} authToken - Authorization header value
+   * @param  {string} prefix - unused, kept for API compatibility
+   * @param  {Object} appConfig - application config
+   * @returns {Promise<boolean>} true if files exist, false otherwise
    */
-  async folderExists (prefix) {
+  async folderExists (authToken, prefix, appConfig) {
     if (typeof prefix !== 'string') {
       throw new Error('prefix must be a valid string')
     }
-    const listParams = {
-      Bucket: this.bucket,
-      Prefix: prefix
+    // Call the list files endpoint (GET /files) - there is no GET /files/:key route
+    const response = await deployServiceFetch(`${getDeploymentServiceUrl()}/cdn-api/namespaces/${appConfig.ow.namespace}/files`, {
+      method: 'GET',
+      headers: {
+        Authorization: authToken
+      }
+    })
+    if (!response.ok) {
+      throw new Error(`cannot check if folder exists, request failed: ${response.status} ${response.statusText}`)
     }
-    const listedObjects = await this.s3.listObjectsV2(listParams)
-
-    return listedObjects.KeyCount > 0
+    const files = await response.json()
+    // Check if there are any files (folder "exists" if it has content)
+    return Array.isArray(files) && files.length > 0
   }
 
   /**
-   * Deletes all files in a prefix location
-   * @param  {string} prefix
+   * Empties all files for the namespace or deletes a specific file
+   * @param {string} authToken - Authorization header value
+   * @param {string} prefix - '/' to delete all files, or a specific file path
+   * @param {Object} appConfig - application config
+   * @returns {Promise<boolean>} true if the folder was emptied, false otherwise
    */
-  async emptyFolder (prefix) {
-    if (typeof prefix !== 'string') throw new Error('prefix must be a valid string')
-    const listParams = {
-      Bucket: this.bucket,
-      Prefix: prefix
+  async emptyFolder (authToken, prefix, appConfig) {
+    if (typeof prefix !== 'string') {
+      throw new Error('prefix must be a valid string')
     }
-    const listedObjects = await this.s3.listObjectsV2(listParams)
+    // Server route is DELETE /files/:key
+    // When key='/' the server triggers emptyStorageForNamespace
+    // URL construction: /files/ (trailing slash makes :key = '/')
+    const deploymentServiceUrl = getDeploymentServiceUrl()
+    const url = prefix === '/'
+      ? `${deploymentServiceUrl}/cdn-api/namespaces/${appConfig.ow.namespace}/files/`
+      : `${deploymentServiceUrl}/cdn-api/namespaces/${appConfig.ow.namespace}/files/${prefix}`
 
-    if (listedObjects.KeyCount < 1) {
-      return
-    }
-    const deleteParams = {
-      Bucket: this.bucket,
-      Delete: { Objects: [] }
-    }
-    listedObjects.Contents.forEach(({ Key }) => {
-      deleteParams.Delete.Objects.push({ Key })
+    const response = await deployServiceFetch(url, {
+      method: 'DELETE',
+      headers: {
+        Authorization: authToken
+      }
     })
-    await this.s3.deleteObjects(deleteParams)
-    if (listedObjects.IsTruncated) {
-      await this.emptyFolder(prefix)
-    }
+    return response.ok
   }
 
   /**
-   * Uploads a file
-   * @param  {string} file
-   * @param  {string} prefix - prefix to upload the file to
+   * Uploads a file to the CDN API
+   * @param  {string} authToken - Authorization header value
+   * @param  {string} file - Full local file path
+   * @param  {string} filePath - Path relative to namespace (e.g., 'images/photo.jpg' or 'index.html')
+   *                             This becomes file.name in the API request. The server will prepend the namespace.
    * @param  {Object} appConfig - application config
-   * @param  {string} distRoot - Distribution root dir
+   * @param  {string} distRoot - Distribution root dir (used for header matching)
    */
-  async uploadFile (file, prefix, appConfig, distRoot) {
-    if (typeof prefix !== 'string') {
-      throw new Error('prefix must be a valid string')
+  async uploadFile (authToken, file, filePath, appConfig, distRoot) {
+    if (typeof filePath !== 'string') {
+      throw new Error('filePath must be a valid string')
     }
+    const url = `${getDeploymentServiceUrl()}/cdn-api/namespaces/${appConfig.ow.namespace}/files`
+
     const content = await fs.readFile(file)
     const mimeType = mime.lookup(path.extname(file))
     // first we will grab it from the global config: htmlCacheDuration, etc.
-    const cacheControlString = this._getCacheControlConfig(mimeType, appConfig.app)
-    const uploadParams = {
-      Bucket: this.bucket,
-      Key: this._urlJoin(prefix, path.basename(file)),
-      Body: content
-    }
-    // if we found it in the global config, we will use it ( for now )
-    if (cacheControlString) {
-      uploadParams.CacheControl = cacheControlString
-    }
+    let cacheControlString = this._getCacheControlConfig(mimeType, appConfig.app)
     // add response headers if specified in manifest
     const responseHeaders = this.getResponseHeadersForFile(file, distRoot, appConfig) ?? {}
     // here we allow overriding the cache control if specified in response headers
     // this is considered more specific than the general cache control config
     // ideally we deprecate cache control config in favor of response headers directly
     if (responseHeaders?.['adp-cache-control']) {
-      uploadParams.CacheControl = responseHeaders['adp-cache-control']
+      cacheControlString = responseHeaders['adp-cache-control']
       delete responseHeaders['adp-cache-control']
     }
-
-    // we only set metadata if we have added anything to responseHeaders object
-    // it is not null, but could be empty
-    if (Object.keys(responseHeaders).length > 0) {
-      uploadParams.Metadata = responseHeaders
+    // server expected body is: { contentType, cacheControl, customHeaders: {}, file: { name, content } }
+    // file.name is the path relative to namespace (e.g., 'images/photo.jpg' or 'index.html')
+    // The server will prepend the namespace to create the S3 key: ${namespace}/${file.name}
+    const fileName = path.basename(file)
+    let relativeFilePath = filePath
+    if (relativeFilePath.startsWith('/')) {
+      relativeFilePath = relativeFilePath.substring(1)
     }
-    // s3 misses some mime types like for css files
-    if (mimeType) {
-      uploadParams.ContentType = mimeType
+    // Use _urlJoin to normalize backslashes and construct the file path
+    const filePathForServer = this._urlJoin(relativeFilePath, fileName)
+    // NOTE: File content is base64-encoded in the JSON body
+    // which increases payload size by ~33%. Look at getting server to handle
+    // multipart uploads to avoid this.
+    const data = {
+      file: {
+        contentType: mimeType,
+        cacheControl: cacheControlString,
+        customHeaders: responseHeaders,
+        name: filePathForServer,
+        content: Buffer.from(content).toString('base64')
+      }
+    }
+    const response = await deployServiceFetch(url, {
+      method: 'PUT',
+      body: JSON.stringify(data),
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: authToken
+      }
+    }).catch(error => {
+      console.error('Error uploading file:', file)
+      throw error
+    })
+    if (!response.ok) {
+      console.error('Failed to upload file:', file)
+      throw new Error(`Failed to upload file: ${response.statusText}`)
     }
-    // Note: putObject is recommended for files < 100MB and has a limit of 5GB, which is ok for our use case of storing static web assets
-    // if we intend to store larger files, we should use multipart upload and https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/modules/_aws_sdk_lib_storage.html
-    return this.s3.putObject(uploadParams)
+    return response.status
   }
 
   getResponseHeadersForFile (file, distRoot, appConfig) {
@@ -250,15 +250,17 @@ module.exports = class RemoteStorage {
   }
 
   /**
-   * Uploads all files in a dir to - recursion is supported
-   * @param  {string} dir - directory with files to upload
-   * @param  {string} prefix - prefix to upload the dir to
+   * Uploads all files in a directory recursively to the CDN API
+   * @param  {string} authToken - Authorization header value
+   * @param  {string} dir - Local directory with files to upload
+   * @param  {string} basePath - Base path prefix for all files (e.g., from config.s3.folder)
+   *                              This is combined with each file's relative directory path.
    * @param  {Object} appConfig - application config
    * @param  {function} [postFileUploadCallback] - called for each uploaded file
    */
-  async uploadDir (dir, prefix, appConfig, postFileUploadCallback) {
-    if (typeof prefix !== 'string') {
-      throw new Error('prefix must be a valid string')
+  async uploadDir (authToken, dir, basePath, appConfig, postFileUploadCallback) {
+    if (typeof basePath !== 'string') {
+      throw new Error('basePath must be a valid string')
     }
     // walk the whole directory recursively using klaw.
     const files = await this.walkDir(dir)
@@ -271,18 +273,28 @@ module.exports = class RemoteStorage {
     let fileBatch = files.splice(0, batchSize)
     const allResults = []
     while (fileBatch.length > 0) {
-      const res = await Promise.all(fileBatch.map(async f => {
-        // get file's relative folder to the base directory.
-        let prefixDirectory = path.dirname(path.relative(dir, f))
-        // base directory returns ".", ignore that.
-        prefixDirectory = prefixDirectory === '.' ? '' : prefixDirectory
-        // newPrefix is now the initial prefix plus the files relative directory path.
-        const newPrefix = this._urlJoin(prefix, prefixDirectory)
-        const s3Res = await this.uploadFile(f, newPrefix, appConfig, dir)
+      // this used to sleep, but I don't think we need it here.
+      // I am leaving it here in case we see issues with rate limiting.
+      // sleep for 100ms to prevent rate limiting
+      // await new Promise(resolve => setTimeout(resolve, 100))
+      const res = await Promise.all(fileBatch.map(async file => {
+        // Calculate the file's relative directory path from the base directory
+        // e.g., if dir='/dist' and file='/dist/images/photo.jpg', relativeDir='images'
+        let relativeDir = path.dirname(path.relative(dir, file))
+        // path.relative returns '.' for files in the root directory, normalize to empty string
+        relativeDir = relativeDir === '.' ? '' : relativeDir
+
+        // Combine basePath with relativeDir to get the full file path relative to namespace
+        // e.g., basePath='' + relativeDir='images' = 'images'
+        //       basePath='assets' + relativeDir='images' = 'assets/images'
+        const filePath = this._urlJoin(basePath, relativeDir)
+
+        // Upload file with the calculated filePath (server will prepend namespace)
+        const s3Result = await this.uploadFile(authToken, file, filePath, appConfig, dir)
         if (postFileUploadCallback) {
-          postFileUploadCallback(f)
+          postFileUploadCallback(file)
         }
-        return s3Res
+        return s3Result
       }))
       allResults.push(res)
       fileBatch = files.splice(0, batchSize)