gene's encode_uri added/adapted to python

CHANGELOG.md

@@ -15,6 +15,7 @@ All notable changes to the Zowe Client Python SDK will be documented in this fil
 - Updated the `pyo3` dependency of the Secrets SDK for technical currency. [#399](https://github.com/zowe/zowe-client-python-sdk/pull/399)
 - Updated the `secrets_core` dependency of the Secrets SDK to Zowe CLI 8.35.1 and pinned it to a commit for reproducible builds. [#407](https://github.com/zowe/zowe-client-python-sdk/pull/407)
 - Updated the `Tso.issue_command` SDK method to accept a `command_timeout` parameter and raise a `TimeoutError` if the "TSO PROMPT" message is not received within that time, preventing the method from looping indefinitely. [#406](https://github.com/zowe/zowe-client-python-sdk/pull/406)
+- Fixed URI-encoding of USS paths, dataset/job names, and zFS file system names to match what z/OSMF and API-ML actually require. Also fixed `Console.issue_command`/`get_response` corrupting URLs when the host or base path contains `defcn`. [#408](https://github.com/zowe/zowe-client-python-sdk/pull/408)
 
 ## `1.0.0-dev26`
 

src/core/zowe/core_for_zowe_sdk/sdk_api.py

@@ -11,6 +11,7 @@
 """
 
 import copy
+import posixpath
 import urllib
 
 from . import session_constants
@@ -19,6 +20,24 @@
 from .session import ISession, Session
 from typing import Any, Optional, Type
 
+# Characters that fail against both z/OSMF and API-ML unless they are encoded.
+_USS_CHARS_TO_ENCODE = {" ": "%20", "%": "%25", "+": "%2B", "?": "%3F"}
+
+# Characters that API-ML rejects with an HTTP 400 unless they are encoded.
+# None of these are encoded for a direct z/OSMF connection.
+_APIML_CHARS_TO_ENCODE = {
+    "#": "%23",
+    ";": "%3B",
+    "<": "%3C",
+    ">": "%3E",
+    "[": "%5B",
+    "]": "%5D",
+    "^": "%5E",
+    "{": "%7B",
+    "|": "%7C",
+    "}": "%7D",
+}
+
 
 class SdkApi:
     """
@@ -112,3 +131,91 @@ def _encode_uri_component(self, str_to_adjust: str) -> str:
             A string with special characters, acceptable for a URL
         """
         return urllib.parse.quote(str_to_adjust, safe="!~*'()") if str_to_adjust is not None else None
+
+    def _is_using_apiml(self) -> bool:
+        """
+        Determine whether requests are routed through API-ML.
+
+        Returns
+        -------
+        bool
+            True if the session connects through API-ML, False otherwise
+        """
+        if self.session.token_type == session_constants.TOKEN_TYPE_APIML:
+            return True
+        return self.session.base_path is not None
+
+    def _encode_uri_path_for_zos(self, zos_uri_path: str) -> str:
+        """
+        Encode a z/OS resource (dataset, job, or volser) path for the path component of a URI.
+
+        None of the documented z/OS resource naming special characters require encoding
+        to be processed successfully by z/OSMF. API-ML rejects a literal "#" with an
+        HTTP 400 error unless it is encoded, so it is the only character adjusted here.
+
+        Parameters
+        ----------
+        zos_uri_path : str
+            The URI path to encode
+
+        Returns
+        -------
+        str
+            The path, with "#" encoded when the session is routed through API-ML
+        """
+        if self._is_using_apiml():
+            return zos_uri_path.replace("#", "%23")
+        return zos_uri_path
+
+    def _encode_uri_path_for_uss(self, uss_uri_path: str) -> str:
+        """
+        Encode a USS file path for the path component of a URI.
+
+        Many documented USS file name special characters cause an HTTP 500 error
+        unless they are encoded. Forward slashes are preserved rather than encoded
+        as %2F, since encoded slashes are expected to be rejected in future.
+
+        Parameters
+        ----------
+        uss_uri_path : str
+            The USS path to encode
+
+        Returns
+        -------
+        str
+            The normalized and encoded USS path, without a leading slash
+
+        Raises
+        ------
+        ValueError
+            Thrown when the path contains a backslash or a double-quote character.
+        """
+        # Normalizing against root collapses // and resolves /../ without escaping the service path
+        normalized = posixpath.normpath("/" + uss_uri_path).lstrip("/")
+        encode_for_apiml = self._is_using_apiml()
+
+        encoded_path = []
+        for next_char in normalized:
+            if next_char == "\\":
+                # Both encoded and unencoded backslashes fail in REST requests
+                self.logger.error(f"The USS path '{uss_uri_path}' contains a backslash character.")
+                raise ValueError(
+                    f"The supplied USS path '{uss_uri_path}' contains a backslash \\ character. "
+                    "When a backslash is present, z/OSMF and API-ML servers fail with an HTTP 400 "
+                    "or 500 error code, or the backslash is ignored. This request was not sent."
+                )
+            if next_char == '"':
+                # Both encoded and unencoded double-quotes fail in REST requests
+                self.logger.error(f"The USS path '{uss_uri_path}' contains a double-quote character.")
+                raise ValueError(
+                    f'The supplied USS path \'{uss_uri_path}\' contains a double-quote " character. '
+                    "When a double-quote is present, z/OSMF and API-ML servers fail with an HTTP 400 "
+                    "or 500 error code. This request was not sent."
+                )
+            if next_char in _USS_CHARS_TO_ENCODE:
+                encoded_path.append(_USS_CHARS_TO_ENCODE[next_char])
+            elif encode_for_apiml and next_char in _APIML_CHARS_TO_ENCODE:
+                encoded_path.append(_APIML_CHARS_TO_ENCODE[next_char])
+            else:
+                encoded_path.append(next_char)
+        return "".join(encoded_path)

src/core/zowe/core_for_zowe_sdk/session_constants.py

@@ -32,6 +32,10 @@
 AUTH_TYPE_CERT_PEM = "cert-pem"
 
 
+# Token type property value for an API-ML authentication token
+TOKEN_TYPE_APIML = "apimlAuthenticationToken"
+
+
 # https protocol defaults
 DEFAULT_HTTPS_PORT = 443
 HTTPS_PROTOCOL = "https"

src/zos_console/zowe/zos_console_for_zowe_sdk/console.py

@@ -16,6 +16,8 @@
 
 from .response import ConsoleResponse, IssueCommandResponse
 
+_DEFAULT_CONSOLE_NAME = "defcn"
+
 
 class Console(SdkApi):  # type: ignore
     """
@@ -30,7 +32,7 @@ class Console(SdkApi):  # type: ignore
     """
 
     def __init__(self, connection: dict[str, Any], log: bool = True):
-        super().__init__(connection, "/zosmf/restconsoles/consoles/defcn", logger_name=__name__, log=log)
+        super().__init__(connection, "/zosmf/restconsoles/consoles", logger_name=__name__, log=log)
 
     def issue_command(self, command: str, console: Optional[str] = None) -> IssueCommandResponse:
         """Issues a command on z/OS Console.
@@ -48,7 +50,9 @@ def issue_command(self, command: str, console: Optional[str] = None) -> IssueCom
             A JSON containing the response from the console command
         """
         custom_args = self._create_custom_request_arguments()
-        custom_args["url"] = self._request_endpoint.replace("defcn", console or "defcn")
+        custom_args["url"] = "{}/{}".format(
+            self._request_endpoint, self._encode_uri_component(console or _DEFAULT_CONSOLE_NAME)
+        )
         request_body = {"cmd": command}
         custom_args["json"] = request_body
         response_json = self.request_handler.perform_request("PUT", custom_args)
@@ -71,7 +75,10 @@ def get_response(self, response_key: str, console: Optional[str] = None) -> Cons
             A JSON containing the response to the command
         """
         custom_args = self._create_custom_request_arguments()
-        request_url = "{}/solmsgs/{}".format(console or "defcn", response_key)
-        custom_args["url"] = self._request_endpoint.replace("defcn", request_url)
+        custom_args["url"] = "{}/{}/solmsgs/{}".format(
+            self._request_endpoint,
+            self._encode_uri_component(console or _DEFAULT_CONSOLE_NAME),
+            self._encode_uri_component(response_key),
+        )
         response_json = self.request_handler.perform_request("GET", custom_args)
         return ConsoleResponse(response_json)

src/zos_files/zowe/zos_files_for_zowe_sdk/datasets.py

@@ -375,7 +375,7 @@ def list_members(
         if member_pattern is not None:
             additional_parms["pattern"] = member_pattern
         custom_args["params"] = additional_parms
-        custom_args["url"] = "{}ds/{}/member".format(self._request_endpoint, self._encode_uri_component(dataset_name))
+        custom_args["url"] = "{}ds/{}/member".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name))
         custom_args["headers"]["X-IBM-Max-Items"] = "{}".format(limit)
         custom_args["headers"]["X-IBM-Attributes"] = attributes
         response_json = self.request_handler.perform_request("GET", custom_args)
@@ -439,7 +439,7 @@ def copy_data_set_or_member(
 
         custom_args = self._create_custom_request_arguments()
         custom_args["json"] = data
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(path_to_member))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(path_to_member))
         self.request_handler.perform_request("PUT", custom_args, expected_code=[200])
 
     def create(self, dataset_name: str, options: Optional[DatasetOption] = None) -> None:
@@ -483,7 +483,7 @@ def create(self, dataset_name: str, options: Optional[DatasetOption] = None) ->
                     break
 
         custom_args = self._create_custom_request_arguments()
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name))
         custom_args["json"] = options.to_dict() if options else {}
         self.request_handler.perform_request("POST", custom_args, expected_code=[201])
 
@@ -561,7 +561,7 @@ def create_default(self, dataset_name: str, default_type: str) -> None:
                 "dirblk": 25,
             }
 
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name))
         self.request_handler.perform_request("POST", custom_args, expected_code=[201])
 
     def retrieve_content(
@@ -590,7 +590,7 @@ def retrieve_content(
             or a Response object with content of the file if `as_stream == True`
         """
         custom_args = self._create_custom_request_arguments()
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name))
         custom_args["headers"]["X-IBM-Data-Type"] = content_type.value
         if content_type == ContentType.RECORD or content_type == ContentType.BINARY:
             custom_args["headers"]["Accept"] = "application/octet-stream"
@@ -600,7 +600,7 @@ def retrieve_content(
     def get_content(self, dataset_name: str, stream: bool = False) -> Union[str, None, Response]:
         """Use `retrieve_content()` instead of this deprecated function."""
         custom_args = self._create_custom_request_arguments()
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name))
         response: Union[str, Response] = self.request_handler.perform_request("GET", custom_args, stream=stream)
         return response
 
@@ -609,7 +609,7 @@ def get_binary_content(
     ) -> Union[bytes, Response]:
         """Use `retrieve_content(content_type=ContentType.BINARY)` instead of this deprecated function."""
         custom_args = self._create_custom_request_arguments()
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name))
         custom_args["headers"]["Accept"] = "application/octet-stream"
         if with_prefixes:
             custom_args["headers"]["X-IBM-Data-Type"] = "record"
@@ -637,7 +637,7 @@ def write(self, dataset_name: str, data: Union[str, bytes], encoding: str = _ZOW
             Data must be either a string or bytes.
         """
         custom_args = self._create_custom_request_arguments()
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name))
 
         # Check if the data is a string (text content)
         if isinstance(data, str):
@@ -768,7 +768,7 @@ def recall_migrated(self, dataset_name: str, wait: bool = False) -> None:
 
         custom_args = self._create_custom_request_arguments()
         custom_args["json"] = data
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name))
 
         self.request_handler.perform_request("PUT", custom_args, expected_code=[200])
 
@@ -793,7 +793,7 @@ def delete_migrated(self, dataset_name: str, purge: bool = False, wait: bool = F
 
         custom_args = self._create_custom_request_arguments()
         custom_args["json"] = data
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name))
 
         self.request_handler.perform_request("PUT", custom_args, expected_code=[200])
 
@@ -812,7 +812,7 @@ def migrate(self, dataset_name: str, wait: bool = False) -> None:
 
         custom_args = self._create_custom_request_arguments()
         custom_args["json"] = data
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name))
 
         self.request_handler.perform_request("PUT", custom_args, expected_code=[200])
 
@@ -833,7 +833,7 @@ def rename(self, before_dataset_name: str, after_dataset_name: str) -> None:
         custom_args = self._create_custom_request_arguments()
         custom_args["json"] = data
         custom_args["url"] = "{}ds/{}".format(
-            self._request_endpoint, self._encode_uri_component(after_dataset_name).strip()
+            self._request_endpoint, self._encode_uri_path_for_zos(after_dataset_name).strip()
         )
 
         self.request_handler.perform_request("PUT", custom_args, expected_code=[200])
@@ -877,7 +877,7 @@ def rename_member(self, dataset_name: str, before_member_name: str, after_member
 
         custom_args = self._create_custom_request_arguments()
         custom_args["json"] = data
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(path_to_member))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(path_to_member))
 
         self.request_handler.perform_request("PUT", custom_args, expected_code=[200])
 
@@ -897,9 +897,13 @@ def delete(self, dataset_name: str, volume: Optional[str] = None, member_name: O
         custom_args = self._create_custom_request_arguments()
         if member_name is not None:
             dataset_name = f"{dataset_name}({member_name})"
-        url = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name))
+        url = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name))
         if volume is not None:
-            url = "{}ds/-{}/{}".format(self._request_endpoint, volume, self._encode_uri_component(dataset_name))
+            url = "{}ds/-{}/{}".format(
+                self._request_endpoint,
+                self._encode_uri_path_for_zos(str(volume)),
+                self._encode_uri_path_for_zos(dataset_name),
+            )
         custom_args["url"] = url
         self.request_handler.perform_request("DELETE", custom_args, expected_code=[200, 202, 204])
 
@@ -936,5 +940,5 @@ def copy_uss_to_data_set(
         path_to_member = f"{to_dataset_name}({to_member_name})" if to_member_name else to_dataset_name
         custom_args = self._create_custom_request_arguments()
         custom_args["json"] = data
-        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(path_to_member))
+        custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(path_to_member))
         self.request_handler.perform_request("PUT", custom_args, expected_code=[200])

src/zos_files/zowe/zos_files_for_zowe_sdk/file_system.py

@@ -68,7 +68,9 @@ def create(self, file_system_name: str, options: dict[str, Any] = {}) -> None:
                     raise MaxAllocationQuantityExceeded()
 
         custom_args = self._create_custom_request_arguments()
-        custom_args["url"] = "{}mfs/zfs/{}".format(self._request_endpoint, file_system_name)
+        custom_args["url"] = "{}mfs/zfs/{}".format(
+            self._request_endpoint, self._encode_uri_path_for_zos(file_system_name)
+        )
         custom_args["json"] = options
         self.request_handler.perform_request("POST", custom_args, expected_code=[201])
 
@@ -82,7 +84,9 @@ def delete(self, file_system_name: str) -> None:
             Name of the file system
         """
         custom_args = self._create_custom_request_arguments()
-        custom_args["url"] = "{}mfs/zfs/{}".format(self._request_endpoint, file_system_name)
+        custom_args["url"] = "{}mfs/zfs/{}".format(
+            self._request_endpoint, self._encode_uri_path_for_zos(file_system_name)
+        )
         self.request_handler.perform_request("DELETE", custom_args, expected_code=[204])
 
     def mount(
@@ -109,7 +113,7 @@ def mount(
         options["action"] = "mount"
         options["mount-point"] = mount_point
         custom_args = self._create_custom_request_arguments()
-        custom_args["url"] = "{}mfs/{}".format(self._request_endpoint, file_system_name)
+        custom_args["url"] = "{}mfs/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(file_system_name))
         custom_args["json"] = options
         custom_args["headers"]["Content-Type"] = "text/plain; charset={}".format(encoding)
         self.request_handler.perform_request("PUT", custom_args, expected_code=[204])
@@ -131,7 +135,7 @@ def unmount(
         """
         options["action"] = "unmount"
         custom_args = self._create_custom_request_arguments()
-        custom_args["url"] = "{}mfs/{}".format(self._request_endpoint, file_system_name)
+        custom_args["url"] = "{}mfs/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(file_system_name))
         custom_args["json"] = options
         custom_args["headers"]["Content-Type"] = "text/plain; charset={}".format(encoding)
         self.request_handler.perform_request("PUT", custom_args, expected_code=[204])