diff --git a/CHANGELOG.md b/CHANGELOG.md index d579cf23..495b3dd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ All notable changes to the Zowe Client Python SDK will be documented in this fil - Updated the `pyo3` dependency of the Secrets SDK for technical currency. [#399](https://github.com/zowe/zowe-client-python-sdk/pull/399) - Updated the `secrets_core` dependency of the Secrets SDK to Zowe CLI 8.35.1 and pinned it to a commit for reproducible builds. [#407](https://github.com/zowe/zowe-client-python-sdk/pull/407) - Updated the `Tso.issue_command` SDK method to accept a `command_timeout` parameter and raise a `TimeoutError` if the "TSO PROMPT" message is not received within that time, preventing the method from looping indefinitely. [#406](https://github.com/zowe/zowe-client-python-sdk/pull/406) +- Fixed URI-encoding of USS paths, dataset/job names, and zFS file system names to match what z/OSMF and API-ML actually require. Also fixed `Console.issue_command`/`get_response` corrupting URLs when the host or base path contains `defcn`. [#408](https://github.com/zowe/zowe-client-python-sdk/pull/408) ## `1.0.0-dev26` diff --git a/src/core/zowe/core_for_zowe_sdk/sdk_api.py b/src/core/zowe/core_for_zowe_sdk/sdk_api.py index e26a393a..6761b93e 100644 --- a/src/core/zowe/core_for_zowe_sdk/sdk_api.py +++ b/src/core/zowe/core_for_zowe_sdk/sdk_api.py @@ -11,6 +11,7 @@ """ import copy +import posixpath import urllib from . import session_constants @@ -19,6 +20,24 @@ from .session import ISession, Session from typing import Any, Optional, Type +# Characters that fail against both z/OSMF and API-ML unless they are encoded. +_USS_CHARS_TO_ENCODE = {" ": "%20", "%": "%25", "+": "%2B", "?": "%3F"} + +# Characters that API-ML rejects with an HTTP 400 unless they are encoded. +# None of these are encoded for a direct z/OSMF connection. +_APIML_CHARS_TO_ENCODE = { + "#": "%23", + ";": "%3B", + "<": "%3C", + ">": "%3E", + "[": "%5B", + "]": "%5D", + "^": "%5E", + "{": "%7B", + "|": "%7C", + "}": "%7D", +} + class SdkApi: """ @@ -112,3 +131,91 @@ def _encode_uri_component(self, str_to_adjust: str) -> str: A string with special characters, acceptable for a URL """ return urllib.parse.quote(str_to_adjust, safe="!~*'()") if str_to_adjust is not None else None + + def _is_using_apiml(self) -> bool: + """ + Determine whether requests are routed through API-ML. + + Returns + ------- + bool + True if the session connects through API-ML, False otherwise + """ + if self.session.token_type == session_constants.TOKEN_TYPE_APIML: + return True + return self.session.base_path is not None + + def _encode_uri_path_for_zos(self, zos_uri_path: str) -> str: + """ + Encode a z/OS resource (dataset, job, or volser) path for the path component of a URI. + + None of the documented z/OS resource naming special characters require encoding + to be processed successfully by z/OSMF. API-ML rejects a literal "#" with an + HTTP 400 error unless it is encoded, so it is the only character adjusted here. + + Parameters + ---------- + zos_uri_path : str + The URI path to encode + + Returns + ------- + str + The path, with "#" encoded when the session is routed through API-ML + """ + if self._is_using_apiml(): + return zos_uri_path.replace("#", "%23") + return zos_uri_path + + def _encode_uri_path_for_uss(self, uss_uri_path: str) -> str: + """ + Encode a USS file path for the path component of a URI. + + Many documented USS file name special characters cause an HTTP 500 error + unless they are encoded. Forward slashes are preserved rather than encoded + as %2F, since encoded slashes are expected to be rejected in future. + + Parameters + ---------- + uss_uri_path : str + The USS path to encode + + Returns + ------- + str + The normalized and encoded USS path, without a leading slash + + Raises + ------ + ValueError + Thrown when the path contains a backslash or a double-quote character. + """ + # Normalizing against root collapses // and resolves /../ without escaping the service path + normalized = posixpath.normpath("/" + uss_uri_path).lstrip("/") + encode_for_apiml = self._is_using_apiml() + + encoded_path = [] + for next_char in normalized: + if next_char == "\\": + # Both encoded and unencoded backslashes fail in REST requests + self.logger.error(f"The USS path '{uss_uri_path}' contains a backslash character.") + raise ValueError( + f"The supplied USS path '{uss_uri_path}' contains a backslash \\ character. " + "When a backslash is present, z/OSMF and API-ML servers fail with an HTTP 400 " + "or 500 error code, or the backslash is ignored. This request was not sent." + ) + if next_char == '"': + # Both encoded and unencoded double-quotes fail in REST requests + self.logger.error(f"The USS path '{uss_uri_path}' contains a double-quote character.") + raise ValueError( + f'The supplied USS path \'{uss_uri_path}\' contains a double-quote " character. ' + "When a double-quote is present, z/OSMF and API-ML servers fail with an HTTP 400 " + "or 500 error code. This request was not sent." + ) + if next_char in _USS_CHARS_TO_ENCODE: + encoded_path.append(_USS_CHARS_TO_ENCODE[next_char]) + elif encode_for_apiml and next_char in _APIML_CHARS_TO_ENCODE: + encoded_path.append(_APIML_CHARS_TO_ENCODE[next_char]) + else: + encoded_path.append(next_char) + return "".join(encoded_path) diff --git a/src/core/zowe/core_for_zowe_sdk/session_constants.py b/src/core/zowe/core_for_zowe_sdk/session_constants.py index 72bb3970..b54aa924 100644 --- a/src/core/zowe/core_for_zowe_sdk/session_constants.py +++ b/src/core/zowe/core_for_zowe_sdk/session_constants.py @@ -32,6 +32,10 @@ AUTH_TYPE_CERT_PEM = "cert-pem" +# Token type property value for an API-ML authentication token +TOKEN_TYPE_APIML = "apimlAuthenticationToken" + + # https protocol defaults DEFAULT_HTTPS_PORT = 443 HTTPS_PROTOCOL = "https" diff --git a/src/zos_console/zowe/zos_console_for_zowe_sdk/console.py b/src/zos_console/zowe/zos_console_for_zowe_sdk/console.py index f4190fe6..fdf9c2ac 100644 --- a/src/zos_console/zowe/zos_console_for_zowe_sdk/console.py +++ b/src/zos_console/zowe/zos_console_for_zowe_sdk/console.py @@ -16,6 +16,8 @@ from .response import ConsoleResponse, IssueCommandResponse +_DEFAULT_CONSOLE_NAME = "defcn" + class Console(SdkApi): # type: ignore """ @@ -30,7 +32,7 @@ class Console(SdkApi): # type: ignore """ def __init__(self, connection: dict[str, Any], log: bool = True): - super().__init__(connection, "/zosmf/restconsoles/consoles/defcn", logger_name=__name__, log=log) + super().__init__(connection, "/zosmf/restconsoles/consoles", logger_name=__name__, log=log) def issue_command(self, command: str, console: Optional[str] = None) -> IssueCommandResponse: """Issues a command on z/OS Console. @@ -48,7 +50,9 @@ def issue_command(self, command: str, console: Optional[str] = None) -> IssueCom A JSON containing the response from the console command """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = self._request_endpoint.replace("defcn", console or "defcn") + custom_args["url"] = "{}/{}".format( + self._request_endpoint, self._encode_uri_component(console or _DEFAULT_CONSOLE_NAME) + ) request_body = {"cmd": command} custom_args["json"] = request_body response_json = self.request_handler.perform_request("PUT", custom_args) @@ -71,7 +75,10 @@ def get_response(self, response_key: str, console: Optional[str] = None) -> Cons A JSON containing the response to the command """ custom_args = self._create_custom_request_arguments() - request_url = "{}/solmsgs/{}".format(console or "defcn", response_key) - custom_args["url"] = self._request_endpoint.replace("defcn", request_url) + custom_args["url"] = "{}/{}/solmsgs/{}".format( + self._request_endpoint, + self._encode_uri_component(console or _DEFAULT_CONSOLE_NAME), + self._encode_uri_component(response_key), + ) response_json = self.request_handler.perform_request("GET", custom_args) return ConsoleResponse(response_json) diff --git a/src/zos_files/zowe/zos_files_for_zowe_sdk/datasets.py b/src/zos_files/zowe/zos_files_for_zowe_sdk/datasets.py index 8a5f95f8..3f84acc8 100644 --- a/src/zos_files/zowe/zos_files_for_zowe_sdk/datasets.py +++ b/src/zos_files/zowe/zos_files_for_zowe_sdk/datasets.py @@ -375,7 +375,7 @@ def list_members( if member_pattern is not None: additional_parms["pattern"] = member_pattern custom_args["params"] = additional_parms - custom_args["url"] = "{}ds/{}/member".format(self._request_endpoint, self._encode_uri_component(dataset_name)) + custom_args["url"] = "{}ds/{}/member".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name)) custom_args["headers"]["X-IBM-Max-Items"] = "{}".format(limit) custom_args["headers"]["X-IBM-Attributes"] = attributes response_json = self.request_handler.perform_request("GET", custom_args) @@ -439,7 +439,7 @@ def copy_data_set_or_member( custom_args = self._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(path_to_member)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(path_to_member)) self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) def create(self, dataset_name: str, options: Optional[DatasetOption] = None) -> None: @@ -483,7 +483,7 @@ def create(self, dataset_name: str, options: Optional[DatasetOption] = None) -> break custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name)) custom_args["json"] = options.to_dict() if options else {} self.request_handler.perform_request("POST", custom_args, expected_code=[201]) @@ -561,7 +561,7 @@ def create_default(self, dataset_name: str, default_type: str) -> None: "dirblk": 25, } - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name)) self.request_handler.perform_request("POST", custom_args, expected_code=[201]) def retrieve_content( @@ -590,7 +590,7 @@ def retrieve_content( or a Response object with content of the file if `as_stream == True` """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name)) custom_args["headers"]["X-IBM-Data-Type"] = content_type.value if content_type == ContentType.RECORD or content_type == ContentType.BINARY: custom_args["headers"]["Accept"] = "application/octet-stream" @@ -600,7 +600,7 @@ def retrieve_content( def get_content(self, dataset_name: str, stream: bool = False) -> Union[str, None, Response]: """Use `retrieve_content()` instead of this deprecated function.""" custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name)) response: Union[str, Response] = self.request_handler.perform_request("GET", custom_args, stream=stream) return response @@ -609,7 +609,7 @@ def get_binary_content( ) -> Union[bytes, Response]: """Use `retrieve_content(content_type=ContentType.BINARY)` instead of this deprecated function.""" custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name)) custom_args["headers"]["Accept"] = "application/octet-stream" if with_prefixes: custom_args["headers"]["X-IBM-Data-Type"] = "record" @@ -637,7 +637,7 @@ def write(self, dataset_name: str, data: Union[str, bytes], encoding: str = _ZOW Data must be either a string or bytes. """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name)) # Check if the data is a string (text content) if isinstance(data, str): @@ -768,7 +768,7 @@ def recall_migrated(self, dataset_name: str, wait: bool = False) -> None: custom_args = self._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name)) self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) @@ -793,7 +793,7 @@ def delete_migrated(self, dataset_name: str, purge: bool = False, wait: bool = F custom_args = self._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name)) self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) @@ -812,7 +812,7 @@ def migrate(self, dataset_name: str, wait: bool = False) -> None: custom_args = self._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name)) self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) @@ -833,7 +833,7 @@ def rename(self, before_dataset_name: str, after_dataset_name: str) -> None: custom_args = self._create_custom_request_arguments() custom_args["json"] = data custom_args["url"] = "{}ds/{}".format( - self._request_endpoint, self._encode_uri_component(after_dataset_name).strip() + self._request_endpoint, self._encode_uri_path_for_zos(after_dataset_name).strip() ) self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) @@ -877,7 +877,7 @@ def rename_member(self, dataset_name: str, before_member_name: str, after_member custom_args = self._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(path_to_member)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(path_to_member)) self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) @@ -897,9 +897,13 @@ def delete(self, dataset_name: str, volume: Optional[str] = None, member_name: O custom_args = self._create_custom_request_arguments() if member_name is not None: dataset_name = f"{dataset_name}({member_name})" - url = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(dataset_name)) + url = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(dataset_name)) if volume is not None: - url = "{}ds/-{}/{}".format(self._request_endpoint, volume, self._encode_uri_component(dataset_name)) + url = "{}ds/-{}/{}".format( + self._request_endpoint, + self._encode_uri_path_for_zos(str(volume)), + self._encode_uri_path_for_zos(dataset_name), + ) custom_args["url"] = url self.request_handler.perform_request("DELETE", custom_args, expected_code=[200, 202, 204]) @@ -936,5 +940,5 @@ def copy_uss_to_data_set( path_to_member = f"{to_dataset_name}({to_member_name})" if to_member_name else to_dataset_name custom_args = self._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_component(path_to_member)) + custom_args["url"] = "{}ds/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(path_to_member)) self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) diff --git a/src/zos_files/zowe/zos_files_for_zowe_sdk/file_system.py b/src/zos_files/zowe/zos_files_for_zowe_sdk/file_system.py index fb9c2325..1715c420 100644 --- a/src/zos_files/zowe/zos_files_for_zowe_sdk/file_system.py +++ b/src/zos_files/zowe/zos_files_for_zowe_sdk/file_system.py @@ -68,7 +68,9 @@ def create(self, file_system_name: str, options: dict[str, Any] = {}) -> None: raise MaxAllocationQuantityExceeded() custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}mfs/zfs/{}".format(self._request_endpoint, file_system_name) + custom_args["url"] = "{}mfs/zfs/{}".format( + self._request_endpoint, self._encode_uri_path_for_zos(file_system_name) + ) custom_args["json"] = options self.request_handler.perform_request("POST", custom_args, expected_code=[201]) @@ -82,7 +84,9 @@ def delete(self, file_system_name: str) -> None: Name of the file system """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}mfs/zfs/{}".format(self._request_endpoint, file_system_name) + custom_args["url"] = "{}mfs/zfs/{}".format( + self._request_endpoint, self._encode_uri_path_for_zos(file_system_name) + ) self.request_handler.perform_request("DELETE", custom_args, expected_code=[204]) def mount( @@ -109,7 +113,7 @@ def mount( options["action"] = "mount" options["mount-point"] = mount_point custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}mfs/{}".format(self._request_endpoint, file_system_name) + custom_args["url"] = "{}mfs/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(file_system_name)) custom_args["json"] = options custom_args["headers"]["Content-Type"] = "text/plain; charset={}".format(encoding) self.request_handler.perform_request("PUT", custom_args, expected_code=[204]) @@ -131,7 +135,7 @@ def unmount( """ options["action"] = "unmount" custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}mfs/{}".format(self._request_endpoint, file_system_name) + custom_args["url"] = "{}mfs/{}".format(self._request_endpoint, self._encode_uri_path_for_zos(file_system_name)) custom_args["json"] = options custom_args["headers"]["Content-Type"] = "text/plain; charset={}".format(encoding) self.request_handler.perform_request("PUT", custom_args, expected_code=[204]) diff --git a/src/zos_files/zowe/zos_files_for_zowe_sdk/uss.py b/src/zos_files/zowe/zos_files_for_zowe_sdk/uss.py index 05ee29b2..b036ff30 100644 --- a/src/zos_files/zowe/zos_files_for_zowe_sdk/uss.py +++ b/src/zos_files/zowe/zos_files_for_zowe_sdk/uss.py @@ -73,7 +73,7 @@ def delete(self, filepath_name: str, recursive: bool = False) -> None: If specified as True, all the files and sub-directories will be deleted. """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}fs/{}".format(self._request_endpoint, filepath_name.lstrip("/")) + custom_args["url"] = "{}fs/{}".format(self._request_endpoint, self._encode_uri_path_for_uss(filepath_name)) if recursive: custom_args["headers"]["X-IBM-Option"] = "recursive" @@ -96,7 +96,7 @@ def create(self, file_path: str, type: str, mode: Optional[str] = None) -> None: custom_args = self._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "{}fs/{}".format(self._request_endpoint, file_path.lstrip("/")) + custom_args["url"] = "{}fs/{}".format(self._request_endpoint, self._encode_uri_path_for_uss(file_path)) self.request_handler.perform_request("POST", custom_args, expected_code=[201]) def write(self, filepath_name: str, data: Union[str, bytes], encoding: str = _ZOWE_FILES_DEFAULT_ENCODING) -> None: @@ -118,7 +118,7 @@ def write(self, filepath_name: str, data: Union[str, bytes], encoding: str = _ZO Data must be either a string or bytes. """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}fs/{}".format(self._request_endpoint, filepath_name.lstrip("/")) + custom_args["url"] = "{}fs/{}".format(self._request_endpoint, self._encode_uri_path_for_uss(filepath_name)) custom_args["data"] = data # Check if the data is a string (text content) @@ -172,7 +172,7 @@ def retrieve_content( custom_args = self._create_custom_request_arguments() custom_args["url"] = "{}fs/{}".format( self._request_endpoint, - self._encode_uri_component(file_path.lstrip("/")) + self._encode_uri_path_for_uss(file_path) ) if content_type == ContentType.BINARY: custom_args["headers"]["X-IBM-Data-Type"] = "binary" @@ -196,7 +196,7 @@ def get_content( ) -> Optional[str]: """Use `retrieve_content()` instead of this deprecated function.""" custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}fs{}".format(self._request_endpoint, filepath_name) + custom_args["url"] = "{}fs/{}".format(self._request_endpoint, self._encode_uri_path_for_uss(filepath_name)) custom_args["headers"]["X-IBM-Data-Type"] = "text;fileEncoding={}".format(file_encoding) custom_args["headers"]["Content-Type"] = "text/plain; charset={}".format(receive_encoding) response_json = self.request_handler.perform_request("GET", custom_args) @@ -211,7 +211,7 @@ def get_content_streamed( ) -> Response: """Use `retrieve_content(as_stream=True)` instead of this deprecated function.""" custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}fs/{}".format(self._request_endpoint, self._encode_uri_component(file_path.lstrip("/"))) + custom_args["url"] = "{}fs/{}".format(self._request_endpoint, self._encode_uri_path_for_uss(file_path)) if binary: custom_args["headers"]["X-IBM-Data-Type"] = "binary" else: @@ -372,7 +372,7 @@ def get_file_tag(self, filepath_name: str) -> USSFileTag: Tag info of a given file. """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}fs{}".format(self._request_endpoint, filepath_name) + custom_args["url"] = "{}fs/{}".format(self._request_endpoint, self._encode_uri_path_for_uss(filepath_name)) custom_args["json"] = { "request": "chtag", "action": "list" } response_json = self.request_handler.perform_request("PUT", custom_args) return USSFileTag(response_json) diff --git a/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py b/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py index 3d1e0786..2e1bc342 100644 --- a/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py +++ b/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py @@ -54,7 +54,7 @@ def get_job_status(self, jobname: str, jobid: str) -> JobResponse: """ custom_args = self._create_custom_request_arguments() job_url = "{}/{}".format(jobname, jobid) - request_url = "{}{}".format(self._request_endpoint, self._encode_uri_component(job_url)) + request_url = "{}{}".format(self._request_endpoint, self._encode_uri_path_for_zos(job_url)) custom_args["url"] = request_url response_json = self.request_handler.perform_request("GET", custom_args) return JobResponse(response_json) @@ -89,7 +89,7 @@ def cancel_job(self, jobname: str, jobid: str, modify_version: str = "2.0") -> S custom_args = self._create_custom_request_arguments() job_url = "{}/{}".format(jobname, jobid) - request_url = "{}{}".format(self._request_endpoint, self._encode_uri_component(job_url)) + request_url = "{}{}".format(self._request_endpoint, self._encode_uri_path_for_zos(job_url)) custom_args["url"] = request_url custom_args["json"] = {"request": "cancel", "version": modify_version} @@ -126,7 +126,7 @@ def delete_job(self, jobname: str, jobid: str, modify_version: str = "2.0") -> S custom_args = self._create_custom_request_arguments() job_url = "{}/{}".format(jobname, jobid) - request_url = "{}{}".format(self._request_endpoint, self._encode_uri_component(job_url)) + request_url = "{}{}".format(self._request_endpoint, self._encode_uri_path_for_zos(job_url)) custom_args["url"] = request_url custom_args["headers"]["X-IBM-Job-Modify-Version"] = modify_version @@ -156,7 +156,7 @@ def _issue_job_request(self, req: dict[str, Any], jobname: str, jobid: str, modi """ custom_args = self._create_custom_request_arguments() job_url = "{}/{}".format(jobname, jobid) - request_url = "{}{}".format(self._request_endpoint, self._encode_uri_component(job_url)) + request_url = "{}{}".format(self._request_endpoint, self._encode_uri_path_for_zos(job_url)) custom_args["url"] = request_url custom_args["json"] = {**req, "version": modify_version} @@ -388,7 +388,7 @@ def get_spool_files(self, correlator: str) -> list[SpoolResponse]: """ custom_args = self._create_custom_request_arguments() job_url = "{}/files".format(correlator) - request_url = "{}{}".format(self._request_endpoint, self._encode_uri_component(job_url)) + request_url = "{}{}".format(self._request_endpoint, self._encode_uri_path_for_zos(job_url)) custom_args["url"] = request_url response_json = self.request_handler.perform_request("GET", custom_args) response = [] @@ -412,7 +412,7 @@ def get_jcl_text(self, correlator: str) -> str: """ custom_args = self._create_custom_request_arguments() job_url = "{}/files/JCL/records".format(correlator) - request_url = "{}{}".format(self._request_endpoint, self._encode_uri_component(job_url)) + request_url = "{}{}".format(self._request_endpoint, self._encode_uri_path_for_zos(job_url)) custom_args["url"] = request_url response_json: str = self.request_handler.perform_request("GET", custom_args) return response_json @@ -436,7 +436,7 @@ def get_spool_file_contents(self, correlator: str, id: str) -> str: """ custom_args = self._create_custom_request_arguments() job_url = "{}/files/{}/records".format(correlator, id) - request_url = "{}{}".format(self._request_endpoint, self._encode_uri_component(job_url)) + request_url = "{}{}".format(self._request_endpoint, self._encode_uri_path_for_zos(job_url)) custom_args["url"] = request_url response_json: str = self.request_handler.perform_request("GET", custom_args) return response_json diff --git a/tests/unit/core/test_sdk_api.py b/tests/unit/core/test_sdk_api.py index 174fd522..e1e29b28 100644 --- a/tests/unit/core/test_sdk_api.py +++ b/tests/unit/core/test_sdk_api.py @@ -110,3 +110,70 @@ def test_encode_uri_component(self): actual_none = sdk_api._encode_uri_component(None) expected_none = None self.assertEqual(actual_none, expected_none) + + def test_is_using_apiml(self): + """Session should be detected as API-ML from a base path or an API-ML token.""" + sdk_api = SdkApi(self.basic_props, self.default_url) + self.assertFalse(sdk_api._is_using_apiml()) + + base_path_api = SdkApi({**self.basic_props, "basePath": "/api/v1"}, self.default_url) + self.assertTrue(base_path_api._is_using_apiml()) + + token_props = {**self.token_props, "tokenType": session_constants.TOKEN_TYPE_APIML} + self.assertTrue(SdkApi(token_props, self.default_url)._is_using_apiml()) + + def test_encode_uri_path_for_zos_leaves_zosmf_path_unchanged(self): + """None of the documented z/OS resource special characters require encoding for z/OSMF.""" + sdk_api = SdkApi(self.basic_props, self.default_url) + + self.assertEqual(sdk_api._encode_uri_path_for_zos("MY.DS#NAME$HERE"), "MY.DS#NAME$HERE") + self.assertEqual(sdk_api._encode_uri_path_for_zos("JOB$0010/JOB00010"), "JOB$0010/JOB00010") + + def test_encode_uri_path_for_zos_encodes_hash_for_apiml(self): + """API-ML rejects a literal '#' with an HTTP 400 error unless it is encoded.""" + sdk_api = SdkApi({**self.basic_props, "basePath": "/api/v1"}, self.default_url) + + self.assertEqual(sdk_api._encode_uri_path_for_zos("MY.DS#NAME$HERE"), "MY.DS%23NAME$HERE") + + def test_encode_uri_path_for_uss_normalizes_path(self): + """USS paths should be normalized and stripped of their leading slash.""" + sdk_api = SdkApi(self.basic_props, self.default_url) + + self.assertEqual(sdk_api._encode_uri_path_for_uss("/u/user/file"), "u/user/file") + self.assertEqual(sdk_api._encode_uri_path_for_uss("u/user/file"), "u/user/file") + self.assertEqual(sdk_api._encode_uri_path_for_uss("/u/user//file"), "u/user/file") + self.assertEqual(sdk_api._encode_uri_path_for_uss("/u/user/../other"), "u/other") + # Normalizing against root means .. cannot climb past the service path + self.assertEqual(sdk_api._encode_uri_path_for_uss("/u/a/../../../../etc/passwd"), "etc/passwd") + + def test_encode_uri_path_for_uss_encodes_special_characters(self): + """Characters that z/OSMF rejects should be encoded, and slashes should be preserved.""" + sdk_api = SdkApi(self.basic_props, self.default_url) + + self.assertEqual(sdk_api._encode_uri_path_for_uss("/u/my file.txt"), "u/my%20file.txt") + self.assertEqual(sdk_api._encode_uri_path_for_uss("/u/a%b"), "u/a%25b") + self.assertEqual(sdk_api._encode_uri_path_for_uss("/u/a+b"), "u/a%2Bb") + self.assertEqual(sdk_api._encode_uri_path_for_uss("/u/f?x=1"), "u/f%3Fx=1") + # API-ML characters stay unencoded on a direct z/OSMF connection + self.assertEqual(sdk_api._encode_uri_path_for_uss("/u/a#b;c"), "u/a#b;c") + + def test_encode_uri_path_for_uss_encodes_apiml_characters(self): + """API-ML rejects these characters with an HTTP 400 unless they are encoded.""" + sdk_api = SdkApi({**self.basic_props, "basePath": "/api/v1"}, self.default_url) + + self.assertEqual( + sdk_api._encode_uri_path_for_uss("/u/a#b;c[e]^{f}|g"), + "u/a%23b%3Bc%3Cd%3E%5Be%5D%5E%7Bf%7D%7Cg", + ) + + def test_encode_uri_path_for_uss_rejects_unusable_characters(self): + """Backslashes and double-quotes fail server side either way, so the request is not sent.""" + sdk_api = SdkApi(self.basic_props, self.default_url) + + with self.assertRaises(ValueError) as backslash: + sdk_api._encode_uri_path_for_uss("/u/a\\b") + self.assertIn("backslash", str(backslash.exception)) + + with self.assertRaises(ValueError) as double_quote: + sdk_api._encode_uri_path_for_uss('/u/a"b') + self.assertIn("double-quote", str(double_quote.exception)) diff --git a/tests/unit/files/datasets/test_rename.py b/tests/unit/files/datasets/test_rename.py index 9352e88a..4dc5281b 100644 --- a/tests/unit/files/datasets/test_rename.py +++ b/tests/unit/files/datasets/test_rename.py @@ -98,8 +98,8 @@ def test_rename_data_set_member_parameterized(self): custom_args = files_test_profile.ds._create_custom_request_arguments() custom_args["json"] = data ds_path = "{}({})".format(test_case[0][0], test_case[0][2]) - ds_path_adjusted = files_test_profile._encode_uri_component(ds_path) - self.assertNotRegex(ds_path_adjusted, r"[\$\@\#]") + ds_path_adjusted = files_test_profile.ds._encode_uri_path_for_zos(ds_path) + self.assertEqual(ds_path_adjusted, ds_path) self.assertRegex(ds_path_adjusted, r"[\(" + re.escape(test_case[0][2]) + r"\)]") custom_args["url"] = "https://mock-url.com:443/zosmf/restfiles/ds/{}".format(ds_path_adjusted) files_test_profile.ds.request_handler.perform_request.assert_called_once_with( diff --git a/tests/unit/files/uss/test_uss.py b/tests/unit/files/uss/test_uss.py index 4177f328..8dc86404 100644 --- a/tests/unit/files/uss/test_uss.py +++ b/tests/unit/files/uss/test_uss.py @@ -398,6 +398,16 @@ def test_perform_upload_fail_file_not_found(self, mock_is_file, mock_send_reques mock_is_file.assert_called_once() mock_is_file.assert_called_once() + @mock.patch("requests.Session.send") + def test_create_uss(self, mock_send_request): + """Test creating a USS file sends request""" + mock_send_request.return_value = mock.Mock(headers={"Content-Type": "application/json"}, status_code=201) + + Files(self.test_profile).uss.create("/some/test/path", "file", mode="rwxr-xr-x") + mock_send_request.assert_called_once() + prepared_request = mock_send_request.call_args[0][0] + self.assertEqual(prepared_request.method, "POST") + @mock.patch("requests.Session.send") def test_get_file_tag(self, mock_send_request): """Test get a USS file tag sends request""" diff --git a/tests/unit/test_zos_jobs.py b/tests/unit/test_zos_jobs.py index 3a018e37..5d834e81 100644 --- a/tests/unit/test_zos_jobs.py +++ b/tests/unit/test_zos_jobs.py @@ -38,6 +38,69 @@ def test_cancel_job(self, mock_send_request): Jobs(self.test_profile).cancel_job("TESTJOB2", "JOB00084") mock_send_request.assert_called_once() + @mock.patch("requests.Session.send") + def test_get_job_status(self, mock_send_request): + """Test getting job status sends a request""" + mock_response = mock.Mock() + mock_response.headers = {"Content-Type": "application/json"} + mock_response.status_code = 200 + mock_response.json.return_value = {"jobname": "TESTJOB2", "jobid": "JOB00084"} + mock_send_request.return_value = mock_response + + Jobs(self.test_profile).get_job_status("TESTJOB2", "JOB00084") + mock_send_request.assert_called_once() + + @mock.patch("requests.Session.send") + def test_delete_job(self, mock_send_request): + """Test deleting a job sends a request""" + mock_response = mock.Mock() + mock_response.headers = {"Content-Type": "application/json"} + mock_response.status_code = 202 + mock_response.json.return_value = {} + mock_send_request.return_value = mock_response + + Jobs(self.test_profile).delete_job("TESTJOB2", "JOB00084") + mock_send_request.assert_called_once() + + @mock.patch("requests.Session.send") + def test_get_spool_files(self, mock_send_request): + """Test retrieving spool files sends a request""" + mock_response = mock.Mock() + mock_response.headers = {"Content-Type": "application/json"} + mock_response.status_code = 200 + mock_response.json.return_value = [{"id": 1, "ddname": "JESMSGLG"}] + mock_send_request.return_value = mock_response + + result = Jobs(self.test_profile).get_spool_files("J0000001") + mock_send_request.assert_called_once() + self.assertEqual(len(result), 1) + + @mock.patch("requests.Session.send") + def test_get_jcl_text(self, mock_send_request): + """Test retrieving JCL text sends a request""" + mock_response = mock.Mock() + mock_response.headers = {"Content-Type": "text/plain"} + mock_response.status_code = 200 + mock_response.text = "//JOBCARD JOB\n" + mock_send_request.return_value = mock_response + + result = Jobs(self.test_profile).get_jcl_text("J0000001") + mock_send_request.assert_called_once() + self.assertEqual(result, "//JOBCARD JOB\n") + + @mock.patch("requests.Session.send") + def test_get_spool_file_contents(self, mock_send_request): + """Test retrieving spool file contents sends a request""" + mock_response = mock.Mock() + mock_response.headers = {"Content-Type": "text/plain"} + mock_response.status_code = 200 + mock_response.text = "spool content" + mock_send_request.return_value = mock_response + + result = Jobs(self.test_profile).get_spool_file_contents("J0000001", "2") + mock_send_request.assert_called_once() + self.assertEqual(result, "spool content") + @mock.patch("requests.Session.send") def test_hold_job(self, mock_send_request): """Test holding a job sends a request""" @@ -127,8 +190,8 @@ def test_cancel_job_modify_version_parameterized(self): "version": test_case[0][2], } job_url = "{}/{}".format(test_case[0][0], test_case[0][1]) - job_url_adjusted = jobs_test_object._encode_uri_component(job_url) - self.assertNotRegex(job_url_adjusted, r"\$") + job_url_adjusted = jobs_test_object._encode_uri_path_for_zos(job_url) + self.assertEqual(job_url_adjusted, job_url) custom_args["url"] = "https://mock-url.com:443/zosmf/restjobs/jobs/{}".format(job_url_adjusted) jobs_test_object.request_handler.perform_request.assert_called_once_with( "PUT", custom_args, expected_code=[202, 200]