diff --git a/.gitignore b/.gitignore index 6adfc05..d172c3a 100644 --- a/.gitignore +++ b/.gitignore @@ -65,19 +65,6 @@ __pycache__/ !e.* !libkurl.* !libobjstor.* -**/4-1-libs/*.q -**/4-1-libs/*.q_ -**/4-1-libs/*.k_ -**/4-1-libs/*.k -!**/4-1-libs/q.k -**/4-1-libs/**/libe.so -**/4-1-libs/**/e.* - -#ToDo revert in KXI-44621 -src/pykx/lib/4-1-libs/l64/kxreaper -src/pykx/lib/4-1-libs/l64/libkurl.so -src/pykx/lib/4-1-libs/l64/libobjstor.so -src/pykx/lib/4-1-libs/l64/pg # Distribution / packaging .Python @@ -252,3 +239,5 @@ symsEnumsSplay/** src/pykx/pykx_init.q_ qcumber_results.xml + +src/pykx/lib/4-1-libs diff --git a/DEVELOPING.md b/DEVELOPING.md index ff467b3..bb2aebf 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -1,15 +1,15 @@ # Development Guidelines -This document is for developers of PyKX, and strives to clarify some otherwise unspoken assumptions regarding PyKX, and to provide some guidelines for its development. +This document is for developers of KDB-X Python, and strives to clarify some otherwise unspoken assumptions regarding KDB-X Python, and to provide some guidelines for its development. ## Goals -PyKX aims to provide a Python-first interface to q. This means that the API should be as Pythonic as possible while ideally still being able to efficiently perform any task that one could be done with q itself. +KDB-X Python aims to provide a Python-first interface to q. This means that the API should be as Pythonic as possible while ideally still being able to efficiently perform any task that one could be done with q itself. -The primary target audience of this package is naive Python users. These would be data scientists, and the majority of programmers who have at least a passing familiarity with Python. These users will rely heavily on the documentation, and examples. They will not be familiar with any advanced Python concepts, or even have a working knowledge of the [Python object model](https://docs.python.org/3/reference/datamodel.html). Additionally, and more importantly for any developers of PyKX who are primarily q programmers, these users will not be familiar with any q-isms, or vector languages in general. The closest thing most of the will have experienced to a vector language is the Python package [Numpy](https://numpy.org/), and as such we can expect that many of them will use PyKX primarily as a way to convert query results from their q database to Numpy arrays. +The primary target audience of this package is naive Python users. These would be data scientists, and the majority of programmers who have at least a passing familiarity with Python. These users will rely heavily on the documentation, and examples. They will not be familiar with any advanced Python concepts, or even have a working knowledge of the [Python object model](https://docs.python.org/3/reference/datamodel.html). Additionally, and more importantly for any developers of KDB-X Python who are primarily q programmers, these users will not be familiar with any q-isms, or vector languages in general. The closest thing most of the will have experienced to a vector language is the Python package [Numpy](https://numpy.org/), and as such we can expect that many of them will use KDB-X Python primarily as a way to convert query results from their q database to Numpy arrays. -Other kinds of users who should also be considered include skilled q users, and skilled Python users. For them we try to include extra opt-in features which are entirely unnecessary to get the core experience of PyKX. For example the q console is, in part, provided for skilled q users, and it does not follow any Python conventions. +Other kinds of users who should also be considered include skilled q users, and skilled Python users. For them we try to include extra opt-in features which are entirely unnecessary to get the core experience of KDB-X Python. For example the q console is, in part, provided for skilled q users, and it does not follow any Python conventions. ## What does Pythonic means? @@ -74,7 +74,7 @@ Other miscellaneous Pythonic tips include: ## API Development Considerations -It's easy to provide new functions/methods, decide on argument order, calling conventions, etc., but it's terribly difficult to change those decisions after PyKX has been released. For this reason we should err on the side of not releasing such changes until we're confident that we have made the best decision, and that we probably won't want to change it in the future. +It's easy to provide new functions/methods, decide on argument order, calling conventions, etc., but it's terribly difficult to change those decisions after KDB-X Python has been released. For this reason we should err on the side of not releasing such changes until we're confident that we have made the best decision, and that we probably won't want to change it in the future. ## Cython Tips diff --git a/LICENSE.txt b/LICENSE.txt index 43e0d3b..d49f2bd 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -218,8 +218,8 @@ Software License for q.so KX Confidential -Version Number: 1.0 -Date Last Revised: April 2023 +Version Number: 1.1 +Date Last Revised: June 2026 Software License Agreement for use of q.so (“Agreement”) @@ -231,7 +231,7 @@ This Agreement is made between KX Systems, Inc. (“KX” or “we”) and the E  1. LICENSE GRANTS -1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. +1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX or KDB-X software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX or KDB-X software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. 1.2 Software Use Restrictions. End User may not: (a) modify any part of the Software or create derivative works thereof, (b) sell, lease, license or distribute the Software to any third party, (c) attempt to decompile, disassemble or reverse engineer the Software, (d) copy the Software, except for purposes of installing and executing it within the limitations set out at clause 1.1, (e) use or attempt to use the Software in any way that is unlawful or fraudulent or has any unlawful or fraudulent purpose or effect, (f) use or attempt to use the Software in any way that would breach the license granted herein. diff --git a/README.md b/README.md index 08bc1b9..d966548 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -# PyKX +# KDB-X Python ## Introduction -PyKX is a Python first interface to the worlds fastest time-series database kdb+ and it's underlying vector programming language q. PyKX takes a Python first approach to integrating q/kdb+ with Python following 10+ years of integrations between these two languages. Fundamentally it provides users with the ability to efficiently query and analyze huge amounts of in-memory and on-disk time-series data. +KDB-X Python is a Python first interface to the worlds fastest time-series database kdb+ and it's underlying vector programming language q. KDB-X Python takes a Python first approach to integrating q/kdb+ with Python following 10+ years of integrations between these two languages. Fundamentally it provides users with the ability to efficiently query and analyze huge amounts of in-memory and on-disk time-series data. This interface exposes q as a domain-specific language (DSL) embedded within Python, taking the approach that q should principally be used for data processing and management of databases. This approach does not diminish the ability for users familiar with q or those wishing to learn more about it from making the most of advanced analytics and database management functionality but rather empowers those who want to make use of the power of kdb+/q who lack this expertise to get up and running fast. -PyKX supports three principal use cases: +KDB-X Python supports three principal use cases: - It allows users to store, query, manipulate and use q objects within a Python process. - It allows users to query external q processes via an IPC interface. @@ -14,7 +14,7 @@ PyKX supports three principal use cases: Users wishing to install the library can do so following the instructions [here](https://code.kx.com/pykx/getting-started/installing.html). -Once you have the library installed you can get up and running with PyKX following the quickstart guide [here](https://code.kx.com/pykx/getting-started/quickstart.html). +Once you have the library installed you can get up and running with KDB-X Python following the quickstart guide [here](https://code.kx.com/pykx/getting-started/quickstart.html). ### What is q/kdb+? @@ -31,7 +31,7 @@ For more information on using q/kdb+ and getting started with see the following ## Installation -### Installing PyKX using `pip` +### Installing KDB-X Python using `pip` Ensure you have a recent version of pip: @@ -39,13 +39,13 @@ Ensure you have a recent version of pip: pip install --upgrade pip ``` -Then install the latest version of PyKX with the following command: +Then install the latest version of KDB-X Python with the following command: ```bash -pip install pykx +pip install --upgrade pykx ``` -To install a specific version of PyKX run the following command replacing with a specific released semver version of the interface +To install a specific version of KDB-X Python run the following command replacing with a specific released semver version of the interface ```bash pip install pykx== @@ -53,48 +53,29 @@ pip install pykx== **Warning:** Python packages should typically be installed in a virtual environment. [This can be done with the venv package from the standard library](https://docs.python.org/3/library/venv.html). -### PyKX License access and enablement +### KDB-X Python License access and enablement -Installation of PyKX via pip provides users with access to the library with limited functional scope, full details of these limitations can be found [here](docs/user-guide/advanced/modes.md). To access the full functionality of PyKX you must first download and install a kdb+ license, this can be achieved either through use of a personal evaluation license or receipt of a commercial license. +Installation of KDB-X Python via pip provides users with access to the library with limited functional scope, full details of these limitations can be found [here](docs/user-guide/advanced/modes.md). To access the full functionality of KDB-X Python you must first download and install a license. -#### Personal Evaluation License +If you do not have a license go to the [KX Developer Center](https://developer.kx.com/products/kdb-x/install) to obtain a KDB-X license. -The following steps outline the process by which a user can gain access to an install a kdb Insights license which provides access to PyKX - -1. Visit https://kx.com/kdb-insights-sdk-personal-edition-download/ and fill in the attached form following the instructions provided. -2. On receipt of an email from KX providing access to your license download this file and save to a secure location on your computer. -3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). - * Variable Name: `QLIC` - * Variable Value: `/user/path/to/folder` - -#### Commercial Evaluation License - -The following steps outline the process by which a user can gain access to an install a kdb Insights license which provides access to PyKX - -1. Contact you KX sales representative or sales@kx.com requesting a trial license for PyKX evaluation. Alternately apply through https://kx.com/book-demo. -2. On receipt of an email from KX providing access to your license download this file and save to a secure location on your computer. -3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). - * Variable Name: `QLIC` - * Variable Value: `/user/path/to/folder` - -__Note:__ PyKX will not operate with a vanilla or legacy kdb+ license which does not have access to specific feature flags embedded within the license. In the absence of a license with appropriate feature flags PyKX will fail to initialise with full feature functionality. +Full instructions on license installation are covered on the [Installing page](docs/getting-started/installing.md). ### Supported Environments -KX only officially supports versions of PyKX built by KX, i.e. versions of PyKX installed from wheel files. Support for user-built installations of PyKX (e.g. built from the source distribution) is only provided on a best-effort basis. Currently, PyKX provides wheels for the following environments: +KX only officially supports versions of KDB-X Python built by KX, i.e. versions of KDB-X Python installed from wheel files. Support for user-built installations of KDB-X Python (e.g. built from the source distribution) is only provided on a best-effort basis. Currently, KDB-X Python provides wheels for the following environments: -- Linux (`manylinux_2_17_x86_64`) with CPython 3.8-3.11 -- macOS (`macosx_10_10_x86_64`) with CPython 3.8-3.11 -- Windows (`win_amd64`) with CPython 3.8-3.11 +- Linux (`manylinux2014_x86_64`, `manylinux2014_aarch64`) with CPython 3.9-3.14 +- macOS (`macosx_10_15_x86_64`, `macosx_10_15_arm64`) with CPython 3.9-3.14 +- Windows (`win_amd64`) with CPython 3.9-3.14 ### Dependencies #### Python Dependencies -PyKX depends on the following third-party Python packages: +KDB-X Python depends on the following third-party Python packages: -- `pandas>=1.2, <2.0; python_version=='3.8'` -- `pandas>=1.2, <3.0; python_version>'3.8'` +- `pandas>=1.2; python_version>'3.8'` - `numpy>=1.22; python_version<'3.11'` - `numpy>=1.23; python_version=='3.11'` - `numpy>=1.26; python_version>'3.11'` @@ -103,13 +84,13 @@ PyKX depends on the following third-party Python packages: - `dill>=0.2.0` - `requests>=2.25.0` -They are installed automatically by `pip` when PyKX is installed. +They are installed automatically by `pip` when KDB-X Python is installed. -PyKX also has an optional Python dependency of `pyarrow>=3.0.0`, which can be included by installing the `pyarrow` extra, e.g. `pip install pykx[pyarrow]` +KDB-X Python also has an optional Python dependency of `pyarrow>=3.0.0`, which can be included by installing the `pyarrow` extra, e.g. `pip install pykx[pyarrow]` -When using PyKX with KX Dashboards users will be required to install `ast2json~=0.3` this can be installed using the `dashboards` extra, e.g. `pip install pykx[dashboards]` +When using KDB-X Python with KX Dashboards users will be required to install `ast2json~=0.3` this can be installed using the `dashboards` extra, e.g. `pip install pykx[dashboards]` -When using PyKX Streaming users may require the ability to stop processes initialized in a now unavailable process to facilitate this PyKX can make use of `psutil` this can be installed using the `streaming` extra, e.g. `pip install pykx[streaming]` +When using KDB-X Python Streaming users may require the ability to stop processes initialized in a now unavailable process to facilitate this KDB-X Python can make use of `psutil` this can be installed using the `streaming` extra, e.g. `pip install pykx[streaming]` When using Streamlit users will be required to install `streamlit~=1.28` this can be installed using the `streamlit` extra, e.g. `pip install pykx[streamlit]` @@ -162,16 +143,16 @@ python3 -m venv pykx-dev source pykx-dev/bin/activate ``` -Build and install PyKX: +Build and install KDB-X Python: ```bash cd pykx pip3 install -U '.[all]' ``` -To run PyKX in licensed mode ensure to follow the steps to receive a [Personal Evaluation License](https://code.kx.com/pykx/getting-started/installing.html#personal-evaluation-license) +To run in licensed mode see [here](docs/getting-started/installing.md). -Now you can run/test PyKX: +Now you can run/test KDB-X Python: ```bash (pykx-dev) /data/pykx$ python @@ -198,7 +179,7 @@ export QHOME=/location/of/your/q #q needs QHOME available python -m pytest -vvv -n 0 --no-cov --junitxml=report.xml ``` -## PyKX Licenses +## KDB-X Python Licenses This work is dual licensed under [Apache 2.0](https://code.kx.com/pykx/license.html#apache-2-license) and the [Software License for q.so](https://code.kx.com/pykx/license.html#qso-license) and users are required to abide by the terms of both licenses in their entirety. diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index b3a1c31..5f2fff5 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -1,5 +1,4 @@ python: - - 3.8 - 3.9 - 3.10 - 3.11 diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 71a1532..6629fc4 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -26,8 +26,6 @@ requirements: - numpy==2.0.* # [py==311] - numpy==2.0.* # [py==310] - numpy==2.0.* # [py==39] - - numpy==1.22.* # [py==38] - - numpy==1.20.* # [py==37] - tomli>=2.0.1 - wheel>=0.36 - sysroot_linux-64 ==2.17 # [linux and py<314] @@ -37,12 +35,10 @@ requirements: run: - python - - numpy>=1.20 # [py==37] - - numpy>=1.22 # [py>37 , py<314] + - numpy>=1.22 # [py>38 , py<314] - numpy>=2.3.0 # [py==314] - - pandas>=1.2, <3.0 # [py>38 , py<314] - - pandas>=2.3.3, <3.0 # [py==314] - - pandas<2.0 # [py==38] + - pandas>=1.2 # [py>38 , py<314] + - pandas>=2.3.3 # [py==314] - pyarrow>=3.0.0 - pytz>=2022.1 - toml>=0.10.2 diff --git a/custom_theme/main.html b/custom_theme/main.html index 4c73cdc..b56eff9 100644 --- a/custom_theme/main.html +++ b/custom_theme/main.html @@ -14,7 +14,7 @@ delightedThumbsP.survey({ properties: { // extra context (optional) - product: "PyKX" + product: "KDB-X Python" } }); diff --git a/custom_theme/partials/header.html.tpl b/custom_theme/partials/header.html.tpl index 5ede20a..3e9f6a0 100644 --- a/custom_theme/partials/header.html.tpl +++ b/custom_theme/partials/header.html.tpl @@ -126,5 +126,5 @@ } - + diff --git a/docs/api/columns.md b/docs/api/columns.md index 9e7988d..9c2bd63 100644 --- a/docs/api/columns.md +++ b/docs/api/columns.md @@ -1,4 +1,4 @@ -# PyKX Column Objects +# KDB-X Python Column Objects ::: pykx.wrappers.Column diff --git a/docs/api/compress.md b/docs/api/compress.md index 95425a0..b21aa92 100644 --- a/docs/api/compress.md +++ b/docs/api/compress.md @@ -1,6 +1,6 @@ --- title: Compression and encryption -description: API reference page for using kdb+ compression and encryption through PyKX +description: API reference page for using kdb+ compression and encryption through KDB-X Python author: KX Systems date: September 2024 tags: compression, encryption, dare diff --git a/docs/api/db.md b/docs/api/db.md index 46176a4..c8feafa 100644 --- a/docs/api/db.md +++ b/docs/api/db.md @@ -1,6 +1,6 @@ --- title: DB Admin -description: API reference page for managing kdb+ databases with PyKX +description: API reference page for managing kdb+ databases with KDB-X Python author: KX Systems date: September 2024 tags: admin, management diff --git a/docs/api/exceptions.md b/docs/api/exceptions.md index 3b0e8e6..f95264a 100644 --- a/docs/api/exceptions.md +++ b/docs/api/exceptions.md @@ -1,6 +1,6 @@ --- title: Exceptions -description: Reference page explaining the different custom exceptions implemented in PyKX +description: Reference page explaining the different custom exceptions implemented in KDB-X Python author: KX Systems date: September 2024 tags: exceptions, errors diff --git a/docs/api/license.md b/docs/api/license.md index 858b305..a654c58 100644 --- a/docs/api/license.md +++ b/docs/api/license.md @@ -1,13 +1,13 @@ --- title: License management -description: Utilities for the management of PyKX licenses +description: Utilities for the management of KDB-X Python licenses author: KX Systems date: September 2024 tags: license --- # License management -_This page documents utility functions that allow users to manage their PyKX licenses._ +_This page documents utility functions that allow users to manage their KDB-X Python licenses._ ::: pykx.license rendering: diff --git a/docs/api/module.md b/docs/api/module.md new file mode 100644 index 0000000..5f52b02 --- /dev/null +++ b/docs/api/module.md @@ -0,0 +1,13 @@ +--- +title: Loading KDB-X modules +description: API reference page for loading KDB-X modules within KDB-X Python +author: KX Systems +date: May 2026 +tags: pykx, embed +--- +# Loading KDB-X Modules + +_This page documents how to load KDB-X q modules into Python with `kx.use`, the `QModule` objects returned, and the `SearchPath` used to locate them._ + +::: pykx.module + diff --git a/docs/api/pykx-execution/console.md b/docs/api/pykx-execution/console.md index 2837925..d77b1eb 100644 --- a/docs/api/pykx-execution/console.md +++ b/docs/api/pykx-execution/console.md @@ -5,7 +5,7 @@ author: KX Systems date: September 2024 tags: console,repl --- -# PyKX Console +# KDB-X Python Console _This page documents the use of the emulated q console available during use of a Python repl._ diff --git a/docs/api/pykx-execution/ctx.md b/docs/api/pykx-execution/ctx.md index 2b32999..9c5c575 100644 --- a/docs/api/pykx-execution/ctx.md +++ b/docs/api/pykx-execution/ctx.md @@ -1,6 +1,6 @@ --- title: q context interface -description: PyKX interface for q contexts/namespaces +description: KDB-X Python interface for q contexts/namespaces author: KX Systems date: September 2024 tags: namespace diff --git a/docs/api/pykx-execution/embedded_q.md b/docs/api/pykx-execution/embedded_q.md index 4baf23c..19b7b2f 100644 --- a/docs/api/pykx-execution/embedded_q.md +++ b/docs/api/pykx-execution/embedded_q.md @@ -1,4 +1,4 @@ -# PyKX Execution Classes +# KDB-X Python Execution Classes _This page documents the base class for all interfaces between Python and the embedded q process._ diff --git a/docs/api/pykx-execution/q.md b/docs/api/pykx-execution/q.md index 8cb0a7e..6e9995d 100644 --- a/docs/api/pykx-execution/q.md +++ b/docs/api/pykx-execution/q.md @@ -1,17 +1,17 @@ --- -title: PyKX q functions and operators -description: PyKX implementation of a subset of the q language's functions and operators +title: KDB-X Python q functions and operators +description: KDB-X Python implementation of a subset of the q language's functions and operators author: KX Systems date: September 2024 tags: operators --- # q functions and operators -_This page documents the PyKX implementations of a selection of keywords and operators available in q._ +_This page documents the KDB-X Python implementations of a selection of keywords and operators available in q._ -The functions listed here are accessible in PyKX as attributes of `#!python pykx.q`, or as attributes of `#!python pykx.QConnection` instances. Refer to [the q reference card in the q docs](https://code.kx.com/q/ref/#by-category) for more details about these functions as they are used in a q process. This page documents using them in Python via PyKX. +The functions listed here are accessible in KDB-X Python as attributes of `#!python pykx.q`, or as attributes of `#!python pykx.QConnection` instances. Refer to [the q reference card in the q docs](https://code.kx.com/q/ref/#by-category) for more details about these functions as they are used in a q process. This page documents using them in Python via KDB-X Python. -These functions take and return q objects, which are wrapped in PyKX as `#!python pykx.K` objects. Any arguments of other types are converted appropriately. Refer to [the PyKX wrappers documentation](../pykx-q-data/wrappers.md) for more information about `#!python pykx.K` objects. +These functions take and return q objects, which are wrapped in KDB-X Python as `#!python pykx.K` objects. Any arguments of other types are converted appropriately. Refer to [the KDB-X Python wrappers documentation](../pykx-q-data/wrappers.md) for more information about `#!python pykx.K` objects. ## By Category @@ -38,7 +38,7 @@ Some keywords listed on [the q reference card](https://code.kx.com/q/ref/#by-cat - functions that have names which would result in syntax errors in Python, such as `#!q not` and `#!q or` -The unavailable functions can still be used in PyKX by executing q code with `#!python pykx.q`, i.e. `#!python pykx.q('not')` instead of `#!python pykx.q.not`. For the qSQL functions (`#!q select`, `#!q exec`, `#!q update`, and `#!q delete`) use [PyKX qSQL](../query.md). +The unavailable functions can still be used in KDB-X Python by executing q code with `#!python pykx.q`, i.e. `#!python pykx.q('not')` instead of `#!python pykx.q.not`. For the qSQL functions (`#!q select`, `#!q exec`, `#!q update`, and `#!q delete`) use [KDB-X Python qSQL](../query.md). ## Environment @@ -2784,7 +2784,7 @@ pykx.EnumVector(pykx.q('`d$`a`b`c`b`a`b`c')) ### [pad](https://code.kx.com/q/ref/pad/) -Pad a supplied PyKX string (Python bytes) to the length supplied by the user. +Pad a supplied KDB-X Python string (Python bytes) to the length supplied by the user. In the case that you are padding the front of a string use a negative value. ```python @@ -2813,8 +2813,8 @@ pykx.BooleanVector(pykx.q('0111111111b')) ### [tok](https://code.kx.com/q/ref/tok/) -Interpret a PyKX string as a data value(s), this should use a single upper case character byte or -a non-positive PyKX short value. +Interpret a KDB-X Python string as a data value(s), this should use a single upper case character byte or +a non-positive KDB-X Python short value. See https://code.kx.com/q/ref/tok/ for more information on accepted lists for casting ```python diff --git a/docs/api/pykx-q-data/register.md b/docs/api/pykx-q-data/register.md index de4b045..734016a 100644 --- a/docs/api/pykx-q-data/register.md +++ b/docs/api/pykx-q-data/register.md @@ -8,11 +8,11 @@ tags: PyKX, register, api # Registering Custom Operations -The purpose of this functionality is to provide an extension mechanism for PyKX allowing users to register extension logic for PyKX. +The purpose of this functionality is to provide an extension mechanism for KDB-X Python allowing users to register extension logic for KDB-X Python. Specifically this allows users to: -1. Extend the supported conversions from Pythonic types to PyKX objects when using the `#!python pykx.toq` function +1. Extend the supported conversions from Pythonic types to KDB-X Python objects when using the `#!python pykx.toq` function 2. Extend the supported custom functions on `#!python pykx.Column` objects ::: pykx.register diff --git a/docs/api/pykx-q-data/toq.md b/docs/api/pykx-q-data/toq.md index ba296dc..40b8459 100644 --- a/docs/api/pykx-q-data/toq.md +++ b/docs/api/pykx-q-data/toq.md @@ -1,12 +1,12 @@ --- -title: Python to PyKX Data Conversion -description: Documentation for API that handles converting Python data types into PyKX K objects +title: Python to KDB-X Python Data Conversion +description: Documentation for API that handles converting Python data types into KDB-X Python K objects author: KX Systems date: September 2024 tags: conversion,types,toq --- -# Convert Pythonic data to PyKX +# Convert Pythonic data to KDB-X Python -_This page documents converting python data types to PyKX data types via the `#!python pykx.toq` function._ +_This page documents converting python data types to KDB-X Python data types via the `#!python pykx.toq` function._ ::: pykx.toq diff --git a/docs/api/pykx-q-data/type_conversions.md b/docs/api/pykx-q-data/type_conversions.md index 7afd8d7..b998ff9 100644 --- a/docs/api/pykx-q-data/type_conversions.md +++ b/docs/api/pykx-q-data/type_conversions.md @@ -1,10 +1,10 @@ -# PyKX to Pythonic data type mapping +# KDB-X Python to Pythonic data type mapping A breakdown of each of the `pykx.K` types and their analogous `Python`, `NumPy`, `Pandas`, and `PyArrow` types. ??? "Cheat Sheet: `Python`, `NumPy`, `PyArrow`" - | PyKX type | Python type | Numpy dtype | PyArrow type | + | KDB-X Python type | Python type | Numpy dtype | PyArrow type | | ------------------------------- | ----------- | --------------- | -------------- | | [List](#pykxlist) | list | object | Not Supported | | [Boolean](#pykxbooleanatom) | bool | bool | Not Supported | @@ -31,7 +31,7 @@ A breakdown of each of the `pykx.K` types and their analogous `Python`, `NumPy`, **Note:** Creating PyArrow backed Pandas objects uses `as_arrow=True` using NumPy arrays as an intermediate data format. - | PyKX type | Pandas 1.\* dtype | Pandas 2.\* dtype | Pandas 2.\* as_arrow=True dtype | + | KDB-X Python type | Pandas 1.\* dtype | Pandas 2.\* dtype | Pandas 2.\* as_arrow=True dtype | | ------------------------------- | ----------------- | ----------------- | ------------------------------- | | [List](#pykxlist) | object | object | object | | [Boolean](#pykxbooleanatom) | bool | bool | bool[pyarrow] | diff --git a/docs/api/pykx-q-data/wrappers.md b/docs/api/pykx-q-data/wrappers.md index a336379..1510e21 100644 --- a/docs/api/pykx-q-data/wrappers.md +++ b/docs/api/pykx-q-data/wrappers.md @@ -1,4 +1,4 @@ -# PyKX type wrappers +# KDB-X Python type wrappers ::: pykx.wrappers selection: diff --git a/docs/api/pykx-save-load/fileio.md b/docs/api/pykx-save-load/fileio.md index 2f55cd1..f61f921 100644 --- a/docs/api/pykx-save-load/fileio.md +++ b/docs/api/pykx-save-load/fileio.md @@ -1,4 +1,4 @@ -# Reading and Writing files with PyKX +# Reading and Writing files with KDB-X Python ::: pykx.read diff --git a/docs/api/query.md b/docs/api/query.md index fe38d45..6cab453 100644 --- a/docs/api/query.md +++ b/docs/api/query.md @@ -1,6 +1,6 @@ --- title: Database queries -description: API reference page for querying kdb+ databases using PyKX +description: API reference page for querying kdb+ databases using KDB-X Python author: KX Systems date: September 2024 tags: query, database, update, delete, upsert, insert, select, exec, kdb+ diff --git a/docs/api/random.md b/docs/api/random.md index 81152e7..c50ce4b 100644 --- a/docs/api/random.md +++ b/docs/api/random.md @@ -1,6 +1,6 @@ --- title: Random data generation -description: PyKX interface for accessing q random data generation +description: KDB-X Python interface for accessing q random data generation author: KX Systems date: September 2024 tags: random diff --git a/docs/api/reimporting.md b/docs/api/reimporting.md index fc4b637..96f3dd6 100644 --- a/docs/api/reimporting.md +++ b/docs/api/reimporting.md @@ -1,6 +1,6 @@ --- title: Reimporting module -description: API reference page for reimporting the PyKX module in a Python session +description: API reference page for reimporting the KDB-X Python module in a Python session author: KX Systems date: September 2024 tags: reimport, pykx, import diff --git a/docs/api/serialize.md b/docs/api/serialize.md index c19c75d..8ce9e08 100644 --- a/docs/api/serialize.md +++ b/docs/api/serialize.md @@ -1,6 +1,6 @@ --- title: Serialization and deserialization -description: API reference page for serializing and deserializing PyKX objects +description: API reference page for serializing and deserializing KDB-X Python objects author: KX Systems date: September 2024 tags: serialize, deserialize diff --git a/docs/api/streamlit.md b/docs/api/streamlit.md index bb93e47..5fdfd2c 100644 --- a/docs/api/streamlit.md +++ b/docs/api/streamlit.md @@ -7,7 +7,7 @@ tags: streamlit, ipc --- # Streamlit integration -_This page documents the API for using the Streamlit library with PyKX._ +_This page documents the API for using the Streamlit library with KDB-X Python._ ::: pykx.streamlit rendering: diff --git a/docs/api/tick.md b/docs/api/tick.md index 9db0add..6c84240 100644 --- a/docs/api/tick.md +++ b/docs/api/tick.md @@ -1,6 +1,6 @@ --- title: Streaming tickerplant -description: API reference page for the PyKX tick module +description: API reference page for the KDB-X Python tick module author: KX Systems date: September 2024 tags: tick, rdb, hdb, idb, streaming, stream diff --git a/docs/api/util.md b/docs/api/util.md index 677ba0a..347af2c 100644 --- a/docs/api/util.md +++ b/docs/api/util.md @@ -1,10 +1,10 @@ -# PyKX Utilities +# KDB-X Python Utilities -The purpose of this page is to provide users with documentation for utility functions located within various modules within PyKX. +The purpose of this page is to provide users with documentation for utility functions located within various modules within KDB-X Python. !!! Note - This functionality presently is not located in a centralized module but it is expected that with the next major release version of PyKX 3.0.0 they + This functionality presently is not located in a centralized module but it is expected that with the next major release version of KDB-X Python 3.0.0 they ## `pykx.ssl_info` @@ -12,13 +12,13 @@ The purpose of this page is to provide users with documentation for utility func pykx.ssl_info() ``` -View information relating to the TLS Settings used by PyKX from your process +View information relating to the TLS Settings used by KDB-X Python from your process **Returns:** | Type | Description | |-------------------|------------------------------------------------------| -| `pykx.Dictionary` | A dictionary outlining the TLS settings used by PyKX | +| `pykx.Dictionary` | A dictionary outlining the TLS settings used by KDB-X Python | **Example:** @@ -84,7 +84,7 @@ pykx.util.debug_environment(detailed=False, return_info=False) ```python >>> import pykx as kx >>> kx.util.debug_environment() -**** PyKX information **** +**** KDB-X Python information **** pykx.args: () pykx.qhome: /usr/local/anaconda3/envs/qenv/q pykx.qlic: /usr/local/anaconda3/envs/qenv/q @@ -105,21 +105,20 @@ find_libpython: /usr/local/anaconda3/lib/libpython3.8.dylib **** Platform information **** platform.platform: macOS-10.16-x86_64-i386-64bit -**** PyKX Configuration Variables **** +**** KDB-X Python Configuration Variables **** PYKX_IGNORE_QHOME: False PYKX_KEEP_LOCAL_TIMES: False -PYKX_ALLOCATOR: False +PYKX_NO_ALLOCATOR: False PYKX_GC: False PYKX_LOAD_PYARROW_UNSAFE: False PYKX_MAX_ERROR_LENGTH: 256 -PYKX_NOQCE: False +PYKX_QCE: False PYKX_RELEASE_GIL: False PYKX_Q_LIB_LOCATION: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib PYKX_Q_LOCK: False PYKX_SKIP_UNDERQ: False PYKX_Q_EXECUTABLE: /usr/local/anaconda3/envs/qenv/q/m64/q PYKX_THREADING: False -PYKX_4_1_ENABLED: False PYKX_QDEBUG: False PYKX_DEBUG_INSIGHTS_LIBRARIES: False PYKX_DEFAULT_CONVERSION: @@ -131,13 +130,13 @@ PYKX_DIR: /usr/local/anaconda3/lib/python3.8/site-packages/pykx PYKX_USE_FIND_LIBPYTHON: PYKX_UNLICENSED: PYKX_LICENSED: -PYKX_4_1_ENABLED: **** q Environment Variables **** QARGS: QHOME: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib QLIC: /usr/local/anaconda3/envs/qenv/q QINIT: +QCFG: **** License information **** pykx.qlic directory: True diff --git a/docs/beta-features/index.md b/docs/beta-features/index.md index f150488..9b21a3e 100644 --- a/docs/beta-features/index.md +++ b/docs/beta-features/index.md @@ -1,23 +1,23 @@ --- -title: PyKX Beta Features -description: PyKX features in beta status +title: KDB-X Python Beta Features +description: KDB-X Python features in beta status date: January 2025 author: KX Systems, Inc., -tags: PyKX, beta features, +tags: PyKX, beta features, KDB-X Python --- # Beta Features -_This page provides an overview of PyKX Beta Features, including what they are, how to enable them, and what features are available._ +_This page provides an overview of KDB-X Python Beta Features, including what they are, how to enable them, and what features are available._ ## What is a Beta Feature? -As used commonly within software development "Beta Features" within PyKX describe features which have completed an initial development process phase and are being released in an opt-in manner to users of PyKX wishing to test these features. These features are not intended to be for production use while in beta and are subject to change prior to release as full features. Usage of these features will not effect the default behavior of the library outside of the scope of the new functionality being added. +As used commonly within software development "Beta Features" within KDB-X Python describe features which have completed an initial development process phase and are being released in an opt-in manner to users of KDB-X Python wishing to test these features. These features are not intended to be for production use while in beta and are subject to change prior to release as full features. Usage of these features will not effect the default behavior of the library outside of the scope of the new functionality being added. -Feedback on Beta Feature development is incredibly helpful and helps to determine when these features are promoted to fully supported production features. If you run into any issues while making use of these features please raise an issue on the PyKX Github [here](https://github.com/KxSystems/pykx/issues). +Feedback on Beta Feature development is incredibly helpful and helps to determine when these features are promoted to fully supported production features. If you run into any issues while making use of these features please raise an issue on the KDB-X Python Github [here](https://github.com/KxSystems/pykx/issues). ## How do I enable Beta Features? -Enable PyKX beta features using the `#!python PYKX_BETA_FEATURES` configuration/environment variable. Set this before importing PyKX in a Python session, as shown below, to view available beta features through `#!python kx.beta_features`: +Enable KDB-X Python beta features using the `#!python PYKX_BETA_FEATURES` configuration/environment variable. Set this before importing KDB-X Python (`pykx`) in a Python session, as shown below, to view available beta features through `#!python kx.beta_features`: ```python >>> import os @@ -27,7 +27,7 @@ Enable PyKX beta features using the `#!python PYKX_BETA_FEATURES` configuration/ ['PyTorch Conversions'] ``` -Alternatively you can set beta features to be available at all times by adding `PYKX_BETA_FEATURES` to your `.pykx-config` file as outlined [here](../user-guide/configuration.md#configuration-file). An example of a configuration making use of this is as follows: +Alternatively you can set beta features to be available at all times by adding `PYKX_BETA_FEATURES` to your `config-pykx` file as outlined [here](../user-guide/configuration.md#configuration-file). An example of a configuration making use of this is as follows: ```bash [default] @@ -47,4 +47,4 @@ As mentioned above, the `beta_features` property contains the list of available ['PyTorch Conversions'] ``` -1. [`PyTorch Conversions`](torch.md): Allow users to convert numeric type PyKX vectors and N-Dimensional lists to PyTorch Tensor objects. +1. [`PyTorch Conversions`](torch.md): Allow users to convert numeric type `pykx` vectors and N-Dimensional lists to PyTorch Tensor objects. diff --git a/docs/beta-features/torch.md b/docs/beta-features/torch.md index cd0ef87..5b007ba 100644 --- a/docs/beta-features/torch.md +++ b/docs/beta-features/torch.md @@ -1,13 +1,13 @@ --- title: PyTorch Conversions -description: PyTorch Tensor Conversions beta feature in PyKX +description: PyTorch Tensor Conversions beta feature in KDB-X Python date: January 2025 author: KX Systems, Inc., -tags: PyKX, PyTorch Tensor +tags: PyKX, PyTorch Tensor, KDB-X Python --- # PyTorch Conversions -_This page provides an overview of PyTorch Conversions, a beta feature in PyKX._ +_This page provides an overview of PyTorch Conversions, a beta feature in KDB-X Python (`pykx`)._ !!! Warning @@ -17,7 +17,7 @@ _This page provides an overview of PyTorch Conversions, a beta feature in PyKX._ Commonly used in the development of complex machine learning algorithms, PyTorch is a machine learning library based on the Torch library and is used in applications such as computer vision and natural language processing. Originally developed by Meta AI it is now widely used in the open-source community for algorithm development. -This beta feature allows PyKX users to convert PyKX Vector/List objects into their PyTorch [Tensor](https://pytorch.org/docs/stable/tensors.html) equivalents. +This beta feature allows KDB-X Python users to convert `pykx` Vector/List objects into their PyTorch [Tensor](https://pytorch.org/docs/stable/tensors.html) equivalents. ## Requirements and limitations @@ -32,13 +32,13 @@ pip install pykx[torch] This walkthrough demonstrates the following steps: -1. Convert a PyKX Vector object to a Tensor object. -1. Convert a PyKX List object to a Tensor object. -1. Convert a Tensor object to a PyKX equivalent object. +1. Convert a `pykx` Vector object to a Tensor object. +1. Convert a `pykx` List object to a Tensor object. +1. Convert a Tensor object to a `pykx` equivalent object. ### Vector to Tensor -Use the `*.pt()` methods to convert PyKX numeric data representations to Tensor objects. In the example below we convert PyKX numeric types to their PyTorch Tensor equivalents: +Use the `*.pt()` methods to convert `pykx` numeric data representations to Tensor objects. In the example below we convert `pykx` numeric types to their PyTorch Tensor equivalents: ```python >>> import os @@ -62,7 +62,7 @@ In particular note in the above that the data types are converted to their Tenso ### List to Tensor -To convert PyKX List objects to Tensors, two criteria must be met: +To convert `pykx` List objects to Tensors, two criteria must be met: 1. The `#!python pykx.List` contains only data of a single type. 1. The `#!python pykx.List` is an N-Dimensional regularly shaped/rectangular structure. diff --git a/docs/blogs.md b/docs/blogs.md index fbccacf..a74682c 100644 --- a/docs/blogs.md +++ b/docs/blogs.md @@ -1,6 +1,6 @@ # Blogs, Articles, Podcasts and Videos -KX, Partners and members of the public regularly post articles, blogs and videos relating to their usage of PyKX and how it can be used as part of solutions to real-world problems. The intention of this page is to centralise these blogs and articles and will be kept up to date regularly. +KX, Partners and members of the public regularly post articles, blogs and videos relating to their usage of KDB-X Python and how it can be used as part of solutions to real-world problems. The intention of this page is to centralise these blogs and articles and will be kept up to date regularly. !!! note "Want to contribute?" @@ -17,7 +17,7 @@ _Last updated:_ 10th May 2024 | [PyKX Boosts Trade Analytics](https://www.treliant.com/knowledge-center/pykx-boosts-trade-analytics/) | An introduction to the fundamental features and functionality of PyKX | Paul Douglas, Paul Walsh, and Thomas Smyth | June 26th 2023 | | [PyKX Highlights 2023](https://kx.com/blog/pykx-highlights-2023/) | A breakdown of new features and functionality added from January 2023 to version 2.1.1 in October 2023. | Rian Ó Cuinneagáin | 25th October 2023 | | [Build and Manage Databases using PyKX](https://kx.com/blog/how-to-build-and-manage-databases-using-pykx/) | A breakdown of how PyKX can be used to generate and maintain kdb+ databases using newly released functionality | Conor McCarthy | 24th January 2024 | -| [Contributing to PyKX](https://www.habla.dev/blog/2024/04/10/Contributing-to-PyKX.html) | Outlining how new developers can contribute to PyKX | Oscar Nydza Nicpoñ | 10th April 2024 | +| [Contributing to PyKX](https://www.habla.dev/blog/2024/04/10/Contributing-to-PyKX.html) | Outlining how new developers can contribute to PyKX| Oscar Nydza Nicpoñ | 10th April 2024 | ## Articles @@ -36,7 +36,7 @@ Conor McCarthy Introduces how PyKX can be used generate data, run analytics and -### Accelerating Application Development with PyKX +### Accelerating Application Development with PyKX Jack Kiernan outlines the fundamentals of PyKX. diff --git a/docs/examples/AsynchronousQueries/async_querying.md b/docs/examples/AsynchronousQueries/async_querying.md index 3078819..366a8bb 100644 --- a/docs/examples/AsynchronousQueries/async_querying.md +++ b/docs/examples/AsynchronousQueries/async_querying.md @@ -5,9 +5,9 @@ author: KX Systems, Inc. tags: PyKX, q, asyncio, IPC, asynchronous --- -# PyKX Calling into multiple q servers without blocking +# KDB-X Python Calling into multiple q servers without blocking -_This example provides a quick start for setting up a Python process using `PyKX` to call into +_This example provides a quick start for setting up a Python process using KDB-X Python to call into multiple q servers without blocking each other._ To follow along, feel free to download this zip archive that @@ -16,7 +16,7 @@ contains a copy of the python scripts and this writeup. ## Quickstart This example creates a python process that sends 2 queries meant to simulate long running queries to -two separate q servers to show how to query q servers without blocking using `PyKX`. +two separate q servers to show how to query q servers without blocking. ### Run the Example diff --git a/docs/examples/charting.ipynb b/docs/examples/charting.ipynb index 9820119..e85c396 100644 --- a/docs/examples/charting.ipynb +++ b/docs/examples/charting.ipynb @@ -7,9 +7,9 @@ "source": [ "# Python charting libraries\n", "\n", - "_This workbook details example of interfacing PyKX with Python charting libraries._\n", + "_This workbook details example of interfacing KDB-X Python with Python charting libraries._\n", "\n", - "PyKX supports rich datatype mapping meaning you can convert data from PyKX objects to:\n", + "KDB-X Python supports rich datatype mapping meaning you can convert data from `pykx` objects to:\n", "- Python objects using `.py()`\n", "- NumPy objects using `.np()`\n", "- Pandas objects using `.pd()`\n", @@ -19,7 +19,7 @@ "\n", "The resulting objects behave as expected with all Python libraries.\n", "\n", - "For efficiency and exactness the examples below aim to use PyKX objects directly, minimising conversions when possible." + "For efficiency and exactness the examples below aim to use `pykx` objects directly, minimising conversions when possible." ] }, { @@ -34,7 +34,7 @@ "outputs": [], "source": [ "import os\n", - "os.environ['PYKX_IGNORE_QHOME'] = '1' # Ignore symlinking PyKX q libraries to QHOME\n", + "os.environ['PYKX_IGNORE_QHOME'] = '1' # Ignore symlinking pykx q libraries to QHOME\n", "os.environ['PYKX_Q_LOADED_MARKER'] = '' # Only used here for running Notebook under mkdocs-jupyter during document generation." ] }, @@ -147,7 +147,7 @@ "\n", "Let's generate a scatter plot using the `price` and `size` columns of our table. \n", "\n", - "Use the `scatter(tab['price'], tab['quantity'])` notation to access PyKX objects directly. \n", + "Use the `scatter(tab['price'], tab['quantity'])` notation to access `pykx` objects directly. \n", "\n", "The `x=` and `y=` syntax requires conversion to a dataframe using `.pd()` .i.e `scatter(tab.pd(), x='price' ,y='quantity')` \n", "\n", @@ -248,7 +248,7 @@ "id": "07595eb5-26ef-45c9-a15d-0edbe2bee955", "metadata": {}, "source": [ - "Unlike with Pandas, a PyKX table cannot be passed as the first argument with the following data being passed as column names. Each axis must be explicitly set. \n", + "Unlike with Pandas, a `pykx` table cannot be passed as the first argument with the following data being passed as column names. Each axis must be explicitly set. \n", "\n", "To use this feature, first convert to Pandas using the `.pd()` function." ] @@ -290,7 +290,7 @@ "source": [ "## Seaborn\n", "\n", - "[Seaborn](https://seaborn.pydata.org/) allows you to set `data` as a PyKX table name without conversions and then call the `x` and `y` parameters using only the column names of that table.\n", + "[Seaborn](https://seaborn.pydata.org/) allows you to set `data` as a `pykx` table name without conversions and then call the `x` and `y` parameters using only the column names of that table.\n", "\n", "The bar chart below demonstrates this by setting the data as the table object and using the column names for all parameters, without any conversions." ] diff --git a/docs/examples/compress_and_encrypt/readme.md b/docs/examples/compress_and_encrypt/readme.md index c7fb1ae..28bf678 100644 --- a/docs/examples/compress_and_encrypt/readme.md +++ b/docs/examples/compress_and_encrypt/readme.md @@ -1,14 +1,14 @@ --- -title: PyKX Compress and encrypt +title: KDB-X Python Compress and encrypt description: Compress and encrypt Examples date: October 2024 author: KX Systems, Inc., -tags: compression, encryption, PyKX +tags: compression, encryption, PyKX, KDB-X Python --- # Compress and encrypt example -_This example shows how to use various `#!python q` compression and encryption algorithms on a `#!python PyKX` table._ +_This example shows how to use various `#!python q` compression and encryption algorithms on a `#!python pykx` table._ To follow along, download this zip archive that contains a copy of the python script and this writeup. diff --git a/docs/examples/db-management.ipynb b/docs/examples/db-management.ipynb index e7be2f0..b4a4e54 100644 --- a/docs/examples/db-management.ipynb +++ b/docs/examples/db-management.ipynb @@ -7,9 +7,9 @@ "source": [ "# Database Creation and Management\n", "\n", - "This notebook provides a walkthrough of some of the functionality available for users looking to create and maintain large databases using PyKX.\n", + "This notebook provides a walkthrough of some of the functionality available for users looking to create and maintain large databases using KDB-X Python (`pykx`).\n", "\n", - "This notebook refers to creating and maintaining large [partitioned kdb+ databases](https://code.kx.com/q/kb/partition/) using PyKX. Go to [Q for Mortals](https://code.kx.com/q4m3/14_Introduction_to_Kdb+/#143-partitioned-tables) for more in-depth information about partitioned databases in kdb+.\n", + "This notebook refers to creating and maintaining large [partitioned KDB-X databases](https://code.kx.com/q/kb/partition/) using `pykx`. Go to [Q for Mortals](https://code.kx.com/q4m3/14_Introduction_to_Kdb+/#143-partitioned-tables) for more in-depth information about partitioned databases in KDB-X.\n", "\n", "You can download this walkthrough as a `.ipynb` notebook file.\n", "\n", @@ -45,7 +45,7 @@ "outputs": [], "source": [ "import os\n", - "os.environ['PYKX_IGNORE_QHOME'] = '1' # Ignore symlinking PyKX q libraries to QHOME\n", + "os.environ['PYKX_IGNORE_QHOME'] = '1' # Ignore symlinking KDB-X Python (`pykx`) q libraries to QHOME\n", "os.environ['PYKX_Q_LOADED_MARKER'] = '' # Only used here for running Notebook under mkdocs-jupyter during document generation." ] }, @@ -57,7 +57,6 @@ "outputs": [], "source": [ "import os\n", - "os.environ['PYKX_BETA_FEATURES'] = 'true'\n", "\n", "import pykx as kx\n", "from datetime import date\n", diff --git a/docs/examples/interface-overview.ipynb b/docs/examples/interface-overview.ipynb index bd7c79a..0fced04 100644 --- a/docs/examples/interface-overview.ipynb +++ b/docs/examples/interface-overview.ipynb @@ -4,29 +4,29 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# PyKX introduction notebook\n", + "# KDB-X Python introduction notebook\n", "\n", - "_The purpose of this notebook is to introduce you to PyKX capabilities and functionality._\n", + "_The purpose of this notebook is to introduce you to KDB-X Python capabilities and functionality._\n", "\n", - "For the best experience, visit [what is PyKX](../getting-started/what_is_pykx.html) and the [quickstart guide](../getting-started/quickstart.html) first.\n", + "For the best experience, visit [what is KDB-X Python](../getting-started/what_is_pykx.html) and the [quickstart guide](../getting-started/quickstart.html) first.\n", "\n", "To follow along, we recommend to download the notebook. \n", "\n", "Now let's go through the following sections:\n", "\n", - "1. [Import PyKX](#1-import-pykx)\n", - "1. [Basic PyKX data structures](#2-basic-pykx-data-structures)\n", - "1. [Access and create PyKX objects](#3-access-and-create-pykx-objects)\n", - "1. [Run analytics on PyKX objects](#4-run-analytics-on-pykx-objects)" + "1. [Import KDB-X Python](#1-import-kdb-x-python)\n", + "1. [Basic KDB-X Python data structures](#2-basic-pykx-data-structures)\n", + "1. [Access and create `pykx` objects](#3-access-and-create-pykx-objects)\n", + "1. [Run analytics on `pykx` objects](#4-run-analytics-on-pykx-objects)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 1. Import PyKX\n", + "## 1. Import KDB-X Python\n", "\n", - "To access PyKX and its functions, import it in your Python code as follows:" + "To access KDB-X Python and its functions, import it in your Python code as follows:" ] }, { @@ -40,7 +40,7 @@ "outputs": [], "source": [ "import os\n", - "os.environ['PYKX_IGNORE_QHOME'] = '1' # Ignore symlinking PyKX q libraries to QHOME\n", + "os.environ['PYKX_IGNORE_QHOME'] = '1' # Ignore symlinking KDB-X Python q libraries to QHOME\n", "os.environ['PYKX_Q_LOADED_MARKER'] = '' # Only used here for running Notebook under mkdocs-jupyter during document generation." ] }, @@ -58,7 +58,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Tip: We recommend to always use `import pykx as kx`. The shortened import name `kx` makes the code more readabile and is standard for the PyKX library. \n", + "Tip: We recommend to always use `import pykx as kx`. The shortened import name `kx` makes the code more readabile and is standard for the KDB-X Python library. \n", "\n", "Below we load additional libraries used through this notebook:" ] @@ -77,11 +77,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 2. Basic PyKX data structures\n", + "## 2. Basic `pykx` data structures\n", "\n", - "Central to your interaction with PyKX are the data types supported by the library. PyKX is built atop the `q` programming language. This provides small footprint data structures for analytic calculations and the creation of highly-performant databases. The types we show below are generated from Python-equivalent types.\n", + "Central to your interaction with KDB-X Python are the data types supported by the library. KDB-X Python (`pykx`) is built atop the `q` programming language. This provides small footprint data structures for analytic calculations and the creation of highly-performant databases. The types we show below are generated from Python-equivalent types.\n", "\n", - "This section describes the basic elements in the PyKX library and explains why/how they are different:\n", + "This section describes the basic elements in the KDB-X Python library and explains why/how they are different:\n", "\n", "- 2.1 [Atom](#21-atom)\n", "- 2.2 [Vector](#22-vector)\n", @@ -93,7 +93,7 @@ "\n", "### 2.1 Atom\n", "\n", - "In PyKX, an `atom` is a single irreducible value of a specific type. For example, you may come across `pykx.FloatAtom` or `pykx.DateAtom` objects which may have been generated as follows, from an equivalent Pythonic representation. " + "In KDB-X Python, an **atom** is a single irreducible value of a specific type. For example, you may come across `pykx.FloatAtom` or `pykx.DateAtom` objects which may have been generated as follows, from an equivalent Pythonic representation. " ] }, { @@ -121,13 +121,13 @@ "source": [ "### 2.2 Vector\n", "\n", - "Like PyKX atoms, PyKX `Vectors` are a data structure with multiple elements of a single type. These objects in PyKX, along with lists described below, form the basis for most of the other important data structures that you will encounter including dictionaries and tables.\n", + "Like `pykx` atoms, `pykx` vectors are a data structure with multiple elements of a single type. These objects in `pykx`, along with lists described below, form the basis for most of the other important data structures that you will encounter including dictionaries and tables.\n", "\n", - "Vector objects provide significant benefits when applying analytics over Python lists. Like Numpy, PyKX gains from the underlying speed of its analytic engine when operating on these strictly typed objects.\n", + "Vector objects provide significant benefits when applying analytics over Python lists. Like Numpy, `pykx` gains from the underlying speed of its analytic engine when operating on these strictly typed objects.\n", "\n", "Vector type objects are always 1-D and are/can be indexed along a single axis.\n", "\n", - "In the following example, we create PyKX vectors from common Python equivalent `numpy` and `pandas` objects:" + "In the following example, we create `pykx` vectors from common Python equivalent `numpy` and `pandas` objects:" ] }, { @@ -154,7 +154,7 @@ "source": [ "### 2.3 List\n", "\n", - "A PyKX `List` is an untyped vector object. Unlike vectors which are optimised for the performance of analytics, lists are mostly used for storing reference information or matrix data.\n", + "A `pykx.List` is an untyped vector object. Unlike vectors which are optimised for the performance of analytics, lists are mostly used for storing reference information or matrix data.\n", "\n", "Unlike vector objects which are 1-D in shape, lists can be ragged N-Dimensional objects. This makes them useful for storing complex data structures, but limits their performance when dealing with data-access/data modification tasks." ] @@ -174,7 +174,7 @@ "source": [ "### 2.4 Dictionary\n", "\n", - "A PyKX `Dictionary` is a mapping between a direct key-value association. The list of keys and values to which they are associated must have the same count. While it can be considered as a key-value pair, it's physically stored as a pair of lists." + "A `pykx.Dictionary` is a mapping between a direct key-value association. The list of keys and values to which they are associated must have the same count. While it can be considered as a key-value pair, it's physically stored as a pair of lists." ] }, { @@ -192,9 +192,9 @@ "source": [ "### 2.5 Table\n", "\n", - "PyKX `Tables` are a first-class typed entity which lives in memory. They're a collection of named columns implemented as a dictionary. This mapping construct means that PyKX tables are column oriented. This makes analytic operations on columns much faster than for a relational database equivalent.\n", + "`pykx.Table` objects are a first-class typed entity which lives in memory. They're a collection of named columns implemented as a dictionary. This mapping construct means that `pykx` tables are column oriented. This makes analytic operations on columns much faster than for a relational database equivalent.\n", "\n", - "PyKX Tables come in many forms, but the key table types are as follows:\n", + "`pykx` tables come in many forms, but the key table types are as follows:\n", "\n", " - `pykx.Table` \n", " - `pykx.KeyedTable`\n", @@ -278,11 +278,11 @@ "source": [ "### 2.6 Other data types\n", "\n", - "Below we outlined some of the important PyKX data type structures that you will run into through the rest of this notebook.\n", + "Below we outlined some of the important `pykx` data type structures that you will run into through the rest of this notebook.\n", "\n", "#### pykx.Lambda\n", "\n", - "A `pykx.Lambda` is the most basic kind of function within PyKX. They take between 0 and 8 parameters and are the building blocks for most analytics written by users when interacting with data from PyKX." + "A `pykx.Lambda` is the most basic kind of function within `pykx`. They take between 0 and 8 parameters and are the building blocks for most analytics written by users when interacting with data from KDB-X Python." ] }, { @@ -310,7 +310,7 @@ "source": [ "#### pykx.Projection\n", "\n", - "Like [functools.partial](https://docs.python.org/3/library/functools.html#functools.partial), functions in PyKX can have some of their parameters set in advance, resulting in a new function, which is called a projection. When you call this projection, the set parameters are no longer required and cannot be provided.\n", + "Like [functools.partial](https://docs.python.org/3/library/functools.html#functools.partial), functions in `pykx` can have some of their parameters set in advance, resulting in a new function, which is called a projection. When you call this projection, the set parameters are no longer required and cannot be provided.\n", "\n", "If the original function had `n` total parameters and `m` provided, the result would be a function (projection) that requires the user to input `n-m` parameters." ] @@ -338,26 +338,26 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 3. Access and create PyKX objects\n", + "## 3. Access and create `pykx` objects \n", "\n", - "Now that you're familiar with the PyKX object types, let's see how they work in real-world scenarios, such as:\n", + "Now that you're familiar with the `pykx` object types, let's see how they work in real-world scenarios, such as:\n", "\n", - "- 3.1 [Create PyKX objects from Pythonic data types](#31-create-pykx-objects-from-pythonic-data-types)\n", + "- 3.1 [Create `pykx` objects from Pythonic data types](#31-create-pykx-objects-from-pythonic-data-types)\n", "- 3.2 [Random data generation](#32-random-data-generation)\n", "- 3.3 [Run q code to generate data](#33-run-q-code-to-generate-data)\n", "- 3.4 [Read data from a CSV file](#34-read-data-from-a-csv-file)\n", "- 3.5 [Query external processes via IPC](#35-query-external-processes-via-ipc)\n", "\n", - "### 3.1 Create PyKX objects from Pythonic data types\n", + "### 3.1 Create `pykx` objects from Pythonic data types\n", "\n", - "One of the most common ways to generate PyKX data is by converting equivalent Pythonic data types. PyKX natively supports conversions to and from the following common Python data formats:\n", + "One of the most common ways to generate `pykx` data is by converting equivalent Pythonic data types. `pykx` natively supports conversions to and from the following common Python data formats:\n", "\n", "- Python\n", "- Numpy\n", "- Pandas\n", "- PyArrow\n", "\n", - "You can generate PyKX objects by using the `kx.toq` PyKX function:" + "You can generate `pykx` objects by using the `kx.toq` `pykx` function:" ] }, { @@ -396,7 +396,7 @@ "source": [ "### 3.2 Random data generation\n", "\n", - "PyKX provides a module to create random data of user-specified PyKX types or their equivalent Python types. The creation of random data helps in prototyping analytics.\n", + "KDB-X Python provides a module to create random data of user-specified `pykx` types or their equivalent Python types. The creation of random data helps in prototyping analytics.\n", "\n", "As a first example, generate a list of 1,000,000 random floating-point values between 0 and 1 as follows:" ] @@ -430,7 +430,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Random data does not only come in 1-Dimensional forms. To create multi-Dimensional PyKX Lists, turn the first argument into a list. The following examples include a PyKX trick that uses nulls/infinities to generate random data across the full allowable range:" + "Random data does not only come in 1-Dimensional forms. To create multi-Dimensional `pykx` lists, turn the first argument into a list. The following examples include a `pykx` trick that uses nulls/infinities to generate random data across the full allowable range:" ] }, { @@ -483,7 +483,7 @@ "source": [ "### 3.3 Run q code to generate data\n", "\n", - "PyKX is an entry point to the vector programming language q. This means that PyKX users can execute q code directly via PyKX within a Python session, by calling `kx.q`.\n", + "KDB-X Python (`pykx`) is an entry point to the vector programming language q. This means that KDB-X Python users can execute q code directly via `pykx` within a Python session, by calling `kx.q`.\n", "\n", "For example, to create q data, run the following command:" ] @@ -528,7 +528,7 @@ "source": [ "### 3.4 Read data from a CSV file\n", "\n", - "A lot of data that you run into for data analysis tasks comes in the form of CSV files. PyKX, like Pandas, provides a CSV reader called via `kx.q.read.csv`. In the next cell we create a CSV that can be read in PyKX:" + "A lot of data that you run into for data analysis tasks comes in the form of CSV files. KDB-X Python, like Pandas, provides a CSV reader called via `kx.q.read.csv`. In the next cell we create a CSV that can be read in `pykx`:" ] }, { @@ -574,9 +574,9 @@ "source": [ "### 3.5 Query external processes via IPC\n", "\n", - "One of the most common usage patterns in organizations with access to data in kdb+/q is to query data from an external server process infrastructure. For the example below you need to [install q](https://kx.com/kdb-insights-sdk-personal-edition-download/).\n", + "One of the most common usage patterns in organizations with access to data in KDB-X/q is to query data from an external server process infrastructure. For the example below you need to [install q](https://code.kx.com/kdb-x/get_started/kdb-x-install.html).\n", "\n", - "First, set up a q/kdb+ server. Set it on port 5050 and populate it with some data in the form of a table `tab`:" + "First, set up a q/KDB-X server. Set it on port 5050 and populate it with some data in the form of a table `tab`:" ] }, { @@ -635,7 +635,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Alternatively, use the PyKX query API:" + "Alternatively, use the `pykx` query API:" ] }, { @@ -651,7 +651,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Or use PyKX's context interface to run SQL server side if you have access to it:" + "Or use the `pykx` context interface to run SQL server side if you have access to it:" ] }, { @@ -660,7 +660,6 @@ "metadata": {}, "outputs": [], "source": [ - "conn('\\l s.k_')\n", "conn.sql('SELECT * FROM tab where col2>=0.5')" ] }, @@ -691,18 +690,18 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 4. Run analytics on PyKX objects\n", + "## 4. Run analytics on `pykx` objects\n", "\n", - "Like many Python libraries (including Numpy and Pandas), PyKX provides many ways to use its data with analytics that you generated and defined within the library. Let's explore the following:\n", + "Like many Python libraries (including Numpy and Pandas), KDB-X Python (`pykx`) provides many ways to use its data with analytics that you generated and defined within the library. Let's explore the following:\n", "\n", - "- 4.1 [Use in-built methods on PyKX Vectors](#41-use-in-built-methods-on-pykx-vectors)\n", - "- 4.2 [Use in-built methods on PyKX Tables](#42-use-in-built-methods-on-pykx-tables)\n", - "- 4.3 [Use PyKX/q native functions](#43-use-pykxq-native-functions)\n", + "- 4.1 [Use in-built methods on `pykx` vectors](#41-use-in-built-methods-on-pykx-vectors)\n", + "- 4.2 [Use in-built methods on `pykx` tables](#42-use-in-built-methods-on-pykx-tables)\n", + "- 4.3 [Use `pykx`/q native functions](#43-use-pykxq-native-functions)\n", "\n", "\n", - "### 4.1 Use in-built methods on PyKX Vectors\n", + "### 4.1 Use in-built methods on `pykx` Vectors\n", "\n", - "When you interact with PyKX Vectors, you may wish to gain insights into these objects through the application of basic analytics such as calculation of the `mean`/`median`/`mode` of the vector:" + "When you interact with `pykx` vectors, you may wish to gain insights into these objects through the application of basic analytics such as calculation of the `mean`/`median`/`mode` of the vector:" ] }, { @@ -755,9 +754,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 4.2 Use in-built methods on PyKX Tables\n", + "### 4.2 Use in-built methods on `pykx` tables\n", "\n", - "In addition to the vector processing capabilities of PyKX, it's important to have the ability to manage tables. Highlighted in depth within the Pandas-Like API documentation [here](../user-guide/advanced/Pandas_API.ipynb), these methods allow you to apply functions and gain insights into your data in a familiar way.\n", + "In addition to the vector processing capabilities of KDB-X Python, it's important to have the ability to manage tables. Highlighted in depth within the Pandas-Like API documentation [here](../user-guide/advanced/Pandas_API.ipynb), these methods allow you to apply functions and gain insights into your data in a familiar way.\n", "\n", "The example below uses combinations of the most used elements of this Table API operating on the following table:" ] @@ -814,7 +813,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Next, you can set the index columns of a table. In PyKX, this means converting the table from a `pykx.Table` object to a `pykx.KeyedTable` object:" + "Next, you can set the index columns of a table. In KDB-X Python, this means converting the table from a `pykx.Table` object to a `pykx.KeyedTable` object:" ] }, { @@ -850,7 +849,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Next, use the `groupby` method to group PyKX tabular data so you can use it for analytic purposes.\n", + "Next, use the `groupby` method to group `pykx` tabular data so you can use it for analytic purposes.\n", "\n", "In the first example, let's start by grouping the dataset based on the `sym` column and calculate the `mean` for each column based on their `sym`:" ] @@ -868,7 +867,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To extend the above `groupby`, consider a more complex example which uses `numpy` to run calculations on the PyKX data. You will notice later that you can simplify this specific use-case further." + "To extend the above `groupby`, consider a more complex example which uses `numpy` to run calculations on the `pykx` data. You will notice later that you can simplify this specific use-case further." ] }, { @@ -960,9 +959,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 4.3 Use PyKX/q native functions\n", + "### 4.3 Use `pykx`/q native functions\n", "\n", - "While the Pandas-like API and methods provided off PyKX Vectors provides an effective method of applying analytics on PyKX data, the most efficient and performant way to run analytics on your data is by using PyKX/q primitives available through the `kx.q` module.\n", + "While the Pandas-like API and methods provided off `pykx` vectors provides an effective method of applying analytics on `pykx` data, the most efficient and performant way to run analytics on your data is by using `pykx`/q primitives available through the `kx.q` module.\n", "\n", "These include functionality for calculating moving averages, asof/window joins, column reversal etc. Now let's see a few examples with how you can use these functions, grouped into the following sections:\n", "\n", @@ -1039,7 +1038,7 @@ "\n", "##### each\n", "\n", - "Supplied both as a standalone primitive and as a method for PyKX Lambdas `each` allows you to pass individual elements of a PyKX object to a function:" + "Supplied both as a standalone primitive and as a method for `pykx` Lambdas `each` allows you to pass individual elements of a `pykx` object to a function:" ] }, { diff --git a/docs/examples/ipc/README.md b/docs/examples/ipc/README.md index d9b6f5c..1310a35 100644 --- a/docs/examples/ipc/README.md +++ b/docs/examples/ipc/README.md @@ -1,16 +1,16 @@ --- -title: PyKX IPC interface +title: KDB-X Python IPC interface description: IPC interface example date: October 2024 author: KX Systems, Inc., -tags: compression, encryption, PyKX +tags: compression, encryption, PyKX, KDB-X Python --- # IPC interface example -_This example provides a quickstart for interfacing with external q processes using PyKX._ +_This example provides a quickstart for interfacing with external q processes using KDB-X Python (`pykx`)._ -This example should work whether or not a `#!python k4.lic` file is present. The purpose is to show the flexibility of this interface for users who had previously used PyKX or are familiar with qPython. +This example should work whether or not a `#!python k4.lic` file is present. The purpose is to show the flexibility of this interface for users who had previously used KDB-X Python (`pykx`) or are familiar with qPython. To follow along, feel free to download this zip archive that contains a copy of the Python script and this writeup. @@ -40,7 +40,7 @@ q) $ q tick/feed.q q) -// Start the q process to receive data from PyKX +// Start the q process to receive data from `pykx` $ q -p 5130 q) ``` diff --git a/docs/examples/jupyter-integration.ipynb b/docs/examples/jupyter-integration.ipynb index 6de2326..3aaf90e 100644 --- a/docs/examples/jupyter-integration.ipynb +++ b/docs/examples/jupyter-integration.ipynb @@ -10,11 +10,11 @@ "_This notebook demonstrates how to use the q Magic command in a Jupyter notebook._\n", "\n", "\n", - "The Jupyter q magic command in PyKX allows you to execute q code within a Jupyter notebook. It provides seamless integration with the q programming language.\n", + "The Jupyter q magic command in KDB-X Python allows you to execute q code within a Jupyter notebook. It provides seamless integration with the q programming language.\n", "\n", "This example Notebook has the following sections:\n", "\n", - "1. [Import PyKX](#1-import-pykx)\n", + "1. [Import KDB-X Python](#1-import-KDB-X-Python)\n", "1. [Create the external q process](#2-create-the-external-q-process)\n", "1. [Execute against Embedded q](#3-execute-against-embedded-q)\n", "1. [SQL interface](#4-sql-interface)\n", @@ -36,7 +36,7 @@ "outputs": [], "source": [ "import os\n", - "os.environ['PYKX_IGNORE_QHOME'] = '1' # Ignore symlinking PyKX q libraries to QHOME\n", + "os.environ['PYKX_IGNORE_QHOME'] = '1' # Ignore symlinking KDB-X Python q libraries to QHOME\n", "os.environ['PYKX_Q_LOADED_MARKER'] = '' # Only used here for running Notebook under mkdocs-jupyter during document generation." ] }, @@ -45,9 +45,9 @@ "id": "688b9ed0", "metadata": {}, "source": [ - "## 1. Import PyKX\n", + "## 1. Import KDB-X Python\n", "\n", - "To run this example, first import the PyKX library:" + "To run this example, first import the KDB-X Python library:" ] }, { @@ -110,7 +110,7 @@ "source": [ "## 3. Execute against Embedded q\n", "\n", - "To execute q code within PyKX's `EmbeddedQ` module, type `%%q` at the beginning of the cell:" + "To execute q code within KDB-X Python's `EmbeddedQ` module, type `%%q` at the beginning of the cell:" ] }, { @@ -166,9 +166,7 @@ "source": [ "## 4. SQL interface\n", "\n", - "The `s)` syntax runs SQL queries against local tables within the `q` process.\n", - "\n", - "Note: To use the SQL interface, first you need to load the `s.k_` library." + "The `s)` syntax runs SQL queries against local tables within the `q` process.\n" ] }, { @@ -181,9 +179,8 @@ "outputs": [], "source": [ "%%q\n", - "\\l s.k_\n", "tab:([]a:1000?1000; b:1000?500.0; c:1000?`AAPL`MSFT`GOOG);\n", - "s) select * from tab where a>500 and b<250.0 limit 5" + "s)select * from tab where a>500 and b<250.0 limit 5" ] }, { @@ -323,7 +320,7 @@ "metadata": {}, "source": [ "## 7. q first mode\n", - "q first mode can be enabled by importing PyKX after setting the environment variable `PYKX_JUPYTERQ` to `true`, or at runtime use:" + "q first mode can be enabled by importing `pykx` after setting the environment variable `PYKX_JUPYTERQ` to `true`, or at runtime use:" ] }, { diff --git a/docs/examples/server/server.md b/docs/examples/server/server.md index fc224db..6bb5f0c 100644 --- a/docs/examples/server/server.md +++ b/docs/examples/server/server.md @@ -1,20 +1,20 @@ --- -title: PyKX as q server -description: PyKX as q server example +title: KDB-X Python as q server +description: KDB-X Python as q server example date: October 2024 author: KX Systems, Inc., -tags: PyKX, q, server +tags: PyKX, q, server, KDB-X Python --- -# Use PyKX as a `#!python q` Server +# Use KDB-X Python as a `#!python q` Server -_This example provides a quick start for setting up PyKX as a `#!python q` server that other `#!python q` and PyKX sessions can connect to._ +_This example provides a quick start for setting up KDB-X Python (`pykx`) as a `#!python q` server that other `#!python q` and `pykx` sessions can connect to._ To follow along, feel free to download this zip archive that contains a copy of the python script and this writeup. ## Quick start -To run this example, run the `#!python server.py` script to launch a `#!python PyKX` server on port 5000. Alternatively, run `#!python server_async.py` to run an asynchronous version of the server. +To run this example, run the `#!python server.py` script to launch a `#!python pykx` server on port 5000. Alternatively, run `#!python server_async.py` to run an asynchronous version of the server. The server prints out any queries it receives as well as the result of executing the query before replying. @@ -91,4 +91,4 @@ set the `#!python conn_gc_time` to `#!python 10.0` then this clean-up happens ev !!! Note - [reval](../../api/pykx-execution/q.md#reval) will not impose read only execution on a PyKX server as Python manages the sockets rather than `q`. \ No newline at end of file + [reval](../../api/pykx-execution/q.md#reval) will not impose read only execution on a KDB-X Python server as Python manages the sockets rather than `q`. \ No newline at end of file diff --git a/docs/examples/streaming/Evolving System.ipynb b/docs/examples/streaming/Evolving System.ipynb index 784744a..140aa2e 100644 --- a/docs/examples/streaming/Evolving System.ipynb +++ b/docs/examples/streaming/Evolving System.ipynb @@ -19,7 +19,7 @@ "\n", "\"Drawing\"\n", "\n", - "For a full breakdown on streaming within PyKX see our documentation [here](https://code.kx.com/pykx/user-guide/advanced/streaming/index.html) to start your journey." + "For a full breakdown on streaming within KDB-X Python see our documentation [here](https://code.kx.com/pykx/user-guide/advanced/streaming/index.html) to start your journey." ] }, { @@ -27,7 +27,7 @@ "id": "8345fa7e", "metadata": {}, "source": [ - "#### Initialise PyKX" + "#### Initialise KDB-X Python" ] }, { diff --git a/docs/examples/streaming/generate_hdb.py b/docs/examples/streaming/generate_hdb.py index d8bba9a..3360f81 100644 --- a/docs/examples/streaming/generate_hdb.py +++ b/docs/examples/streaming/generate_hdb.py @@ -1,6 +1,4 @@ import argparse -import os -os.environ['PYKX_BETA_FEATURES'] = "true" import pykx as kx diff --git a/docs/examples/streaming/index.md b/docs/examples/streaming/index.md index 01d5fba..4f7c30d 100644 --- a/docs/examples/streaming/index.md +++ b/docs/examples/streaming/index.md @@ -1,14 +1,14 @@ --- title: Example: Real-Time Streaming -description: The development of a basic streaming workflow using PyKX +description: The development of a basic streaming workflow using KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, streaming, basic +tags: PyKX, q, streaming, basic, KDB-X Python --- -# Example: Real-Time Streaming using PyKX +# Real-Time Streaming using KDB-X Python -_This page outlines the steps taken and functionality shown in demonstrating your first PyKX streaming application_ +_This page outlines the steps taken and functionality shown in demonstrating your first KDB-X Python (`pykx`) streaming application._ To run this example please download the [zip](./real-time-pykx.zip) file containing the notebook or visit our github repository [here](https://github.com/KxSystems/pykx/tree/main/docs/examples/streaming) to view the code directly. @@ -20,11 +20,11 @@ In this example we will generate a real-time and historical analysis system whic 4. Develop a number of query analytics on the historical database and real-time database which provide the count of the number of trades/quotes for a specified ticker symbol. 5. Generate a username/password protected gateway process which a user can query to combine the results from the real-time and historical data view into one value. -Each of the analytics provided in steps 3, 4 and 5 are Python analytics operating on data in kdb+/PyKX format. +Each of the analytics provided in steps 3, 4 and 5 are Python analytics operating on data in KDB-X/`pykx` format. ## Want more information? -The documentation surrounding real-time streaming with PyKX is extensively outlined [here](../../user-guide/advanced/streaming/index.md). For information on specific parts of the infrastructures that can be generated you might find the following links useful: +The documentation surrounding real-time streaming with KDB-X Python is extensively outlined [here](../../user-guide/advanced/streaming/index.md). For information on specific parts of the infrastructures that can be generated you might find the following links useful: | Title | Description | |--------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| diff --git a/docs/examples/subscriber/readme.md b/docs/examples/subscriber/readme.md index 292ed81..da68756 100644 --- a/docs/examples/subscriber/readme.md +++ b/docs/examples/subscriber/readme.md @@ -1,21 +1,21 @@ --- -title: PyKX installation guide +title: KDB-X Python installation guide description: Subscriber Examples date: October 2024 author: KX Systems, Inc., -tags: subscriber, synchronous, asynchronous, PyKX +tags: subscriber, synchronous, asynchronous, PyKX, KDB-X Python --- -# PyKX Subscribing to a `q` Process +# KDB-X Python Subscribing to a `q` Process -_This example demonstrates using `PyKX` to setup a python process as a subscriber to data messages published from a q process._ +_This example demonstrates using `pykx` to setup a python process as a subscriber to data messages published from a q process._ ## Pre-requisites -A kdb+ license is required to complete this example. [Sign-up for a license](https://code.kx.com/q/learn/licensing/). +A KDB-X license is required to complete this example. [Sign-up for a license](https://code.kx.com/q/learn/licensing/). The following python libraries are required to run this example: -1. pykx +1. KDB-X Python (`pykx`) 1. asyncio The source code for this example is available in the examples directory here: diff --git a/docs/examples/threaded_execution/asyncio_threading.py b/docs/examples/threaded_execution/asyncio_threading.py index 9d96f06..e729f0f 100644 --- a/docs/examples/threaded_execution/asyncio_threading.py +++ b/docs/examples/threaded_execution/asyncio_threading.py @@ -1,7 +1,6 @@ import os import asyncio os.environ['PYKX_THREADING'] = '1' -os.environ['PYKX_BETA_FEATURES'] = '1' import pykx as kx diff --git a/docs/examples/threaded_execution/threading.md b/docs/examples/threaded_execution/threading.md index 24e75af..e1a1216 100644 --- a/docs/examples/threaded_execution/threading.md +++ b/docs/examples/threaded_execution/threading.md @@ -1,14 +1,14 @@ --- title: Multithreaded Execution Example -description: Example of PyKX Calling into q from multiple threads +description: Example of KDB-X Python Calling into q from multiple threads date: July 2024 author: KX Systems, Inc., tags: PyKX, q, threading, python, asyncio, multithreaded --- -# PyKX Calling into q from multiple threads +# KDB-X Python Calling into q from multiple threads -_This example provides a quick start for setting up a Python process using `#!python PyKX` to call into `#!python EmbeddedQ` from multiple threads._ +_This example provides a quick start for setting up a Python process using `#!python KDB-X Python` to call into `#!python EmbeddedQ` from multiple threads._ To follow along, feel free to download this zip archive that contains a copy of the python scripts and this writeup. diff --git a/docs/extras/glossary.md b/docs/extras/glossary.md index a24c45c..9a857eb 100644 --- a/docs/extras/glossary.md +++ b/docs/extras/glossary.md @@ -1,6 +1,6 @@ --- -title: PyKX Glossary -description: Common terms explained for PyKX +title: KDB-X Python Glossary +description: Common terms explained for KDB-X Python date: June 2024 author: KX Systems, Inc., tags: glossary @@ -8,10 +8,10 @@ tags: glossary # Glossary -_This page contains descriptions of commonly used terms in PyKX._ +_This page contains descriptions of commonly used terms in KDB-X Python._ ## Attributes -PyKX attributes are characteristics and features of PyKX objects that define their behavior and interaction within the Python environment. +`pykx` attributes are characteristics and features of `pykx` objects that define their behavior and interaction within the Python environment. ## Grouped attribute The `#!python grouped` attribute ensures that all items in the `#!python Vector`/`#!python Table` column are stored in a different format to help reduce memory usage. @@ -23,22 +23,22 @@ An HDB is a mount for historical data within a database. It’s the ultimate rep An IDB serves as a mount to store interval data in a database. It collects data from a real-time database (RDB), retains it for a specified duration, such as 10 minutes, and then transfers the data to a historical database (HDB). Learn how [IDB](https://code.kx.com/insights/1.9/enterprise/database/configuration/assembly/database.html) works in database configuration. ## IPC -Interprocess Communication (IPC) forms a central mechanism by which you can connect to and query existing kdb+/q infrastructures. Read more about [communicating via IPC](../user-guide/advanced/ipc.md). +Interprocess Communication (IPC) forms a central mechanism by which you can connect to and query existing KDB-X/q infrastructures. Read more about [communicating via IPC](../user-guide/advanced/ipc.md). ## kdb+ -kdb+ is a powerful, ultra-fast column-based relational time series database (TSDB) with in-memory (IMDB) capabilities. Operating with temporal data types as a first class entity, the use of q and its query language qSQL against this database creates a highly performant time-series analysis tool. Learn more about [kdb+](https://code.kx.com/q/). +kdb+ (now KDB-X) is a powerful, ultra-fast column-based relational time series database (TSDB) with in-memory (IMDB) capabilities. Operating with temporal data types as a first class entity, the use of q and its query language qSQL against this database creates a highly performant time-series analysis tool. Learn more about [kdb+](https://code.kx.com/q/). ## Mount In databases, mounting refers to making a set of databases available online. A mounted database is ready for use. A database may have three types of mounts: real-time (RDB), interval (IDB), and/or historic (HDB). Learn more about [mounts](https://code.kx.com/insights/1.9/enterprise/database/configuration/assembly/database.html#mounts) in database configuration. ## Multithreading -Multithreading means running multiple threads concurrently within a single process. It allows better resource utilization and improved responsiveness. When a program has multiple threads, the operating system switches between them rapidly, giving the illusion of parallel execution. Learn about [multithreading in PyKX](../user-guide/advanced/threading.md) and run an [example](../examples/threaded_execution/threading.md). +Multithreading means running multiple threads concurrently within a single process. It allows better resource utilization and improved responsiveness. When a program has multiple threads, the operating system switches between them rapidly, giving the illusion of parallel execution. Learn about [multithreading in KDB-X Python](../user-guide/advanced/threading.md) and run an [example](../examples/threaded_execution/threading.md). ## Objects -PyKX objects are Python representations of kdb+ data structures that allow developers to interact with kdb+ databases, perform complex queries, and manipulate data efficiently. +`pykx` objects are Python representations of KDB-X data structures that allow developers to interact with KDB-X databases, perform complex queries, and manipulate data efficiently. ## Object storage -Object storage is a data storage architecture designed to handle large amounts of unstructured data. A data storage system that manages data as objects is distinct from file hierarchy or block-based storage architectures. Object storage is ideal for unstructured data because it overcomes the scaling limitations of traditional file storage systems. The capacity for limitless scaling is why object storage is the backbone of cloud storage; major players like Amazon, Google, and Microsoft utilize object storage as their primary data storage solution. Learn [how to interact with PyKX objects](../user-guide/fundamentals/creating.md) or [how to index PyKX objects](../user-guide/fundamentals/indexing.md). +Object storage is a data storage architecture designed to handle large amounts of unstructured data. A data storage system that manages data as objects is distinct from file hierarchy or block-based storage architectures. Object storage is ideal for unstructured data because it overcomes the scaling limitations of traditional file storage systems. The capacity for limitless scaling is why object storage is the backbone of cloud storage; major players like Amazon, Google, and Microsoft utilize object storage as their primary data storage solution. Learn [how to interact with `pykx` objects](../user-guide/fundamentals/creating.md) or [how to index `pykx` objects](../user-guide/fundamentals/indexing.md). ## Parallelization Parallelization involves distributing computational tasks across multiple threads to improve performance and efficiency. @@ -47,10 +47,10 @@ Parallelization involves distributing computational tasks across multiple thread The `#!python parted` attribute is similar to the `#!python grouped` attribute with the additional requirement that each unique value must be adjacent to its other copies, where the grouped attribute allows them to be dispersed throughout the `#!python Vector`/`#!python Table`. ## Partitioned database -A partitioned database is a database that is divided into smaller, more manageable units, improving scalability while maintaining security. These partitions can be created for various reasons, such as manageability, performance optimization, availability, or load balancing. Learn more about [creating and maintaining partitioned kdb+ databases](https://code.kx.com/q/kb/partition/). Go to [Q for Mortals](https://code.kx.com/q4m3/14_Introduction_to_Kdb+/#143-partitioned-tables) for in-depth information about partitioned databases in kdb+. +A partitioned database is a database that is divided into smaller, more manageable units, improving scalability while maintaining security. These partitions can be created for various reasons, such as manageability, performance optimization, availability, or load balancing. Learn more about [creating and maintaining partitioned KDB-X databases](https://code.kx.com/q/kb/partition/). Go to [Q for Mortals](https://code.kx.com/q4m3/14_Introduction_to_Kdb+/#143-partitioned-tables) for in-depth information about partitioned databases in KDB-X. ## Partitioning -When writing a data table to a database, it must be partitioned to ensure compatibility with a kdb+ time series database. In PyKX, partitioning is managed through a `#!python Timestamp` column defined in the schema, and each table is required to have a `#!python Timestamp` column. Learn more about [partitioning tables across directories](https://code.kx.com/q/kb/partition/). +When writing a data table to a database, it must be partitioned to ensure compatibility with a KDB-X time series database. In KDB-X Python, partitioning is managed through a `#!python Timestamp` column defined in the schema, and each table is required to have a `#!python Timestamp` column. Learn more about [partitioning tables across directories](https://code.kx.com/q/kb/partition/). ## Persisted database A persisted database (or on-disk database) stores data on non-volatile storage like a hard drive or SSD, ensuring the data remains intact even after the application is closed or the system is restarted. In contrast, in-memory databases store data in RAM and lose all data when the system is powered down. Persisted databases are crucial for applications needing long-term data storage and reliability, such as financial systems, customer databases, and many web applications. @@ -59,25 +59,25 @@ A persisted database (or on-disk database) stores data on non-volatile storage l A Python byte object is an immutable sequence of bytes, used to handle binary data. Each byte is an integer between 0 and 255. Byte objects are essential for tasks like file I/O and network communication. ## q -q is a versatile vector programming language mainly used to query a kdb+ database. q is known both for its speed and expressiveness. [Learn more about q](https://code.kx.com/q/learn/) including [q terminology](https://code.kx.com/q/basics/glossary/). +q is a versatile vector programming language mainly used to query a KDB-X database. q is known both for its speed and expressiveness. [Learn more about q](https://code.kx.com/q/learn/) including [q terminology](https://code.kx.com/q/basics/glossary/). ## q/SQL -q/SQL is a collection of SQL-like functions for interacting with a kdb+ database. Learn more about [q/SQL](https://code.kx.com/q4m3/9_Queries_q-sql/). +q/SQL is a collection of SQL-like functions for interacting with a KDB-X database. Learn more about [q/SQL](https://code.kx.com/q4m3/9_Queries_q-sql/). ## RDB Real-time event data is stored on an RDB mount of the database, before being moved to the interval database (IDB). Learn how [RDB](https://code.kx.com/insights/1.9/enterprise/database/configuration/assembly/database.html) works in database configuration. ## Schema -A database schema is a fundamental concept in database management. It describes how data is structured within a relational database. It serves as a blueprint for the database’s architecture, outlining the relationships between different entities, such as tables, columns, data types, views, stored procedures, primary keys, and foreign keys. Learn more about [API schema generation in PyKX](../api/schema.md) or [schema configuration in kdb Insights Enterprise](https://code.kx.com/insights/1.9/enterprise/database/configuration/assembly/schema.html). +A database schema is a fundamental concept in database management. It describes how data is structured within a relational database. It serves as a blueprint for the database’s architecture, outlining the relationships between different entities, such as tables, columns, data types, views, stored procedures, primary keys, and foreign keys. Learn more about [API schema generation in KDB-X Python](../api/schema.md) or [schema configuration in kdb Insights Enterprise](https://code.kx.com/insights/1.9/enterprise/database/configuration/assembly/schema.html). ## Sorted attribute The `#!python sorted` attribute ensures that all items in the `#!python Vector`/`#!python Table` column are sorted in ascending order. ## Time-series analysis -Time-series analysis is a specific way of analyzing a sequence of data points collected over an interval of time. Unlike sporadic or random data collection, time-series analysis involves recording data points at consistent intervals within a set period. Use cases include finance, Internet of Things (IoT), and other domains where data evolves over time. PyKX time-series analysis uses q’s query language (qSQL) against kdb+. Learn more about [time-series models](https://code.kx.com/insights/api/machine-learning/q/analytics/api/variadic/timeseries.html). +Time-series analysis is a specific way of analyzing a sequence of data points collected over an interval of time. Unlike sporadic or random data collection, time-series analysis involves recording data points at consistent intervals within a set period. Use cases include finance, Internet of Things (IoT), and other domains where data evolves over time. KDB-X Python time-series analysis uses q’s query language (qSQL) against KDB-X. Learn more about [time-series models](https://code.kx.com/insights/api/machine-learning/q/analytics/api/variadic/timeseries.html). ## Thread -A thread is an independent sequence of instructions within a program that can be executed independently of other code. Threads share the same memory space as the process they belong to, allowing them to communicate and share data efficiently. In Python, the threading module provides an intuitive API for working with threads. Learn about [PyKX calling into q from multiple threads](../examples/threaded_execution/threading.md). +A thread is an independent sequence of instructions within a program that can be executed independently of other code. Threads share the same memory space as the process they belong to, allowing them to communicate and share data efficiently. In Python, the threading module provides an intuitive API for working with threads. Learn about [KDB-X Python calling into q from multiple threads](../examples/threaded_execution/threading.md). ## Unique attribute The `#!python unique` attribute ensures that all items in the `#!python Vector`/`#!python Table` column are unique (there are no duplicated values). @@ -86,4 +86,4 @@ The `#!python unique` attribute ensures that all items in the `#!python Vector`/ In the context of databases, upsert is an operation that combines both updating and inserting data into a table. When you perform an upsert, the database checks whether a record with a specific key already exists in the table. If a record with that key exists, the database updates the existing record with new values. If no record with that key exists, the database inserts a new record with the provided data. Learn more about [upsert](../api/pykx-execution/q.md#upsert). ## Vector -A vector is a mathematical concept used to represent quantities that have both magnitude (size) and direction. In other words, vectors are arrays of numerical values that represent points in multidimensional space. A vector is typically represented as an arrow in space, pointing from one point to another. Learn more about [using in-built methods on PyKX vectors](../examples/interface-overview.ipynb#using-in-built-methods-on-pykx-vectors) and [adding values to PyKX vectors/lists](../user-guide/fundamentals/indexing.md#b-assigning-and-adding-values-to-vectorslists). +A vector is a mathematical concept used to represent quantities that have both magnitude (size) and direction. In other words, vectors are arrays of numerical values that represent points in multidimensional space. A vector is typically represented as an arrow in space, pointing from one point to another. Learn more about [using in-built methods on KDB-X Python vectors](../examples/interface-overview.ipynb#using-in-built-methods-on-pykx-vectors) and [adding values to KDB-X Python vectors/lists](../user-guide/fundamentals/indexing.md#b-assigning-and-adding-values-to-vectorslists). diff --git a/docs/getting-started/installing.md b/docs/getting-started/installing.md index 9d642de..d0a23a8 100644 --- a/docs/getting-started/installing.md +++ b/docs/getting-started/installing.md @@ -1,50 +1,50 @@ --- -title: PyKX installation guide -description: Getting started with PyKX -date: April 2024 +title: KDB-X Python installation guide +description: Getting started with KDB-X Python +last_updated: October 2025 author: KX Systems, Inc., -tags: PyKX, setup, install, +tags: KDB-X Python, setup, install, --- -# PyKX installation guide +# KDB-X Python installation guide -_This page explains how to install PyKX on your machine._ +_This page explains how to install KDB-X Python on your machine._ !!! License - PyKX is released under a dual license covering the files within the [PyKX repository](https://github.com/kxsystems/pykx) as outlined [here](../license.md). + KDB-X Python is released under a dual license covering the files within the [KDB-X Python repository](https://github.com/kxsystems/pykx) as outlined [here](../license.md). **Acceptance of license terms:** - By downloading, installing, or using PyKX, you acknowledge and agree that you have read, understood, and accept the license [link](../license.md) and will adhere to its terms. + By downloading, installing, or using KDB-X Python, you acknowledge and agree that you have read, understood, and accept the license [link](../license.md) and will adhere to its terms. ## Pre-requisites Before you start, make sure you have: -- [**Python**](https://www.python.org/downloads/) (versions 3.8-3.14) +- [**Python**](https://www.python.org/downloads/) (versions 3.9-3.14) - [**pip**](https://pypi.org/project/pip/) Recommended: a virtual environment with packages such as [venv](https://docs.python.org/3/library/venv.html) from the standard library. ## Supported environments -KX only supports versions of PyKX built by KX (installed from wheel files) for: +KX only supports versions of KDB-X Python built by KX (installed from wheel files) for: -- **Linux** (`manylinux_2_17_x86_64`, `linux-arm64`) with CPython 3.8-3.14 -- **macOS** (`macosx_10_15_x86_64`, `macosx_10_15_arm`) with CPython 3.8-3.14 -- **Windows** (`win_amd64`) with CPython 3.8-3.14 +- **Linux** (`manylinux2014_x86_64`, `manylinux2014_aarch64`) with CPython 3.9-3.14 +- **macOS** (`macosx_10_15_x86_64`, `macosx_10_15_arm64`) with CPython 3.9-3.14 +- **Windows** (`win_amd64`) with CPython 3.9-3.14 -We provide assistance to user-built installations of PyKX only on a best-effort basis. +We provide assistance to user-built installations of KDB-X Python only on a best-effort basis. -## 1. Install PyKX +## 1. Install KDB-X Python -You can install PyKX from three sources: +You can install KDB-X Python from three sources: !!! Note "Installing in air-gapped environments" If you are installing in a location without internet connection you may find [this section](#installing-in-an-air-gapped-environment) useful. -=== "Install PyKX from PyPI" +=== "Install KDB-X Python from PyPI" Ensure you have a recent version of `#!bash pip`: @@ -52,34 +52,34 @@ You can install PyKX from three sources: pip install --upgrade pip ``` - Then install the latest version of PyKX with the following command: + Then install the latest version of KDB-X Python with the following command: ```sh - pip install pykx + pip install --upgrade pykx ``` -=== "Install PyKX from Anaconda" +=== "Install KDB-X Python from Anaconda" - For Linux x86 and arm-based architectures, you can install PyKX from the `#!bash kx` channel on Anaconda as follows: + For Linux x86 and arm-based architectures, you can install KDB-X Python from the `#!bash kx` channel on Anaconda as follows: ```sh - conda install -c kx pykx + conda install -c kx KDB-X Python ``` Type `#!bash y` when prompted to accept the installation. -=== "Install PyKX from GitHub" +=== "Install KDB-X Python from GitHub" - Clone the PyKX repository: + Clone the `pykx` repository: ```sh git clone https://github.com/kxsystems/pykx ``` - Enter the cloned repository and install PyKX using `#!bash pip`: + Enter the cloned repository and install KDB-X Python using `#!bash pip`: ```sh cd pykx @@ -87,47 +87,42 @@ You can install PyKX from three sources: ``` -=== "Install PyKX from UV" +=== "Install KDB-X Python from UV" Ensure you have `#!bash uv` installed, instructions can be found [here.](https://docs.astral.sh/uv/getting-started/installation/) - Then install the latest version of PyKX with the following command: + Then install the latest version of KDB-X Python with the following command: ```sh uv pip install pykx ``` -=== "Install PyKX from pipx" +=== "Install KDB-X Python from pipx" Ensure you have `#!bash pipx` installed, installation guide can be found [here.](https://pipx.pypa.io/stable/installation/) - Then install PyKX using the flag to install dependencies: + Then install KDB-X Python using the flag to install dependencies: ```sh pipx install pykx --include-deps ``` - This will create a virtual environment to run PyKX in, make sure you activate this to access the package. + This will create a virtual environment to run KDB-X Python in, make sure you activate this to access the package. -At this point you have [partial access to PyKX](../user-guide/advanced/modes.md#1a-running-in-unlicensed-mode). To gain access to all PyKX features, follow the steps in the next section, otherwise go straight to [3. Verify PyKX Installation](#3-verify-pykx-installation). +At this point you have [partial access to KDB-X Python](../user-guide/advanced/modes.md#1a-running-in-unlicensed-mode). To gain access to all KDB-X Python features, follow the steps in the next section, otherwise go straight to [3. Verify KDB-X Python Installation](#3-verify-pykx-installation). -## 2. Install a kdb Insights license -To use all PyKX functionalities, you need to download and install a kdb Insights license. +## 2. Install a KDB-X license -!!! Warning "Legacy kdb+/q licenses do not support all PyKX features." +To use all KDB-X Python functionalities, you need to download and install a KDB-X license. +To obtain a license, go to the [KX Developer Center](https://developer.kx.com/products/kdb-x/install). -There are two types of kdb Insights licenses for PyKX: personal and commercial. For either of them, you have two installation options: +### 2.a Install an existing license in Python - - a) from Python - - b) using environment variables - -### 2.a Install license in Python - -Follow the steps below to install a kdb Insights license for PyKX from Python: +Follow the steps below to install a license for KDB-X Python: 1. Start your Python session: @@ -135,90 +130,56 @@ Follow the steps below to install a kdb Insights license for PyKX from Python: $ python ``` -2. Import the PyKX library. When prompted to accept the installation, type `#!python Y` or press `#!python Enter`: +2. Import the KDB-X Python library. When prompted to accept the installation, type `#!python Y` or press `#!python Enter`: ```python >>> import pykx as kx - Thank you for installing PyKX! + Thank you for installing KDB-X Python! - We have been unable to locate your license for PyKX. Running PyKX in unlicensed mode has reduced functionality. + We have been unable to locate your license for KDB-X Python. Running KDB-X Python in unlicensed mode has reduced functionality. Would you like to install a license? [Y/n]: ``` -3. Indicate whether you have access to an existing PyKX enabled license or not, type `#!python N` or press `#!python Enter` to continue with accessing a new license: - - ```python - Do you have access to an existing license for PyKX that you would like to use? [N/y]: - ``` - -4. Choose whether you wish to install a personal or commercial license, type `#!python Y` or press `#!python Enter` to choose a personal license +3. Indicate whether you have access to an existing KDB-X Python enabled license or not, type `#!python Y` if you have a license: ```python - Is the intended use of this software for: - [1] Personal use (Default) - [2] Commercial use - Enter your choice here [1/2]: + Do you have access to an existing license for KDB-X Python that you would like to use? [N/y]: ``` -5. When asked if you would like to apply for a license, type `#!python Y` or press `#!python Enter`: - - === "Personal license" - - ```bash - To apply for a PyKX license, navigate to https://kx.com/kdb-insights-sdk-personal-edition-download - Shortly after you submit your license application, you will receive a welcome email containing your license information. - Would you like to open this page? [Y/n]: - ``` - - === "Commercial license" - - ```bash - To apply for your PyKX license, contact your KX sales representative or sales@kx.com. - Alternately apply through https://kx.com/book-demo. - Would you like to open this page? [Y/n]: - ``` - -6. For personal use, complete the form to receive your welcome email. For commercial use, the license will be provided over email after the commercial evaluation process has been followed with the support of your sales representative. - -7. Choose the desired method to activate your license by typing `#!python 1`, `#!python 2`, or `#!python 3` as appropriate: +4. Choose the desired method to activate your license by typing `#!python 1` or `#!python 2` as appropriate: ```bash - Select the method you wish to use to activate your license: - [1] Download the license file provided in your welcome email and input the file path (Default) - [2] Input the activation key (base64 encoded string) provided in your welcome email - [3] Proceed with unlicensed mode - Enter your choice here [1/2/3]: + Please select the method you wish to use to activate your license: + [1] Provide the location of your license file + [2] Paste the license key ``` -8. Depending on your choice (`#!python 1`, `#!python 2`, or `#!python 3`), complete the installation by following the final step as below: +5. Depending on your choice (`#!python 1` or `#!python 2`), complete the installation by following the final steps as below: === "1" - === "Personal license" - - ```bash - Provide the download location of your license (for example, ~/path/to/kc.lic): - ``` - - === "Commercial license" - - ```bash - Provide the download location of your license (for example, ~/path/to/k4.lic): - ``` + ```bash + Provide the download location of your license (for example, ~/path/to/kc.lic): + ``` === "2" + Specify the license type: + ```bash - Provide your activation key (base64 encoded string) provided with your welcome email: + Please confirm the license type: + [1] kc.lic - The default + [2] k4.lic - Used in some scenarios ``` - === "3" + + Then paste your license key and hit enter: ```bash - No further actions needed. + Provide your kc.lic license key (base64 encoded string) : ``` -9. Validate the correct installation of your license: +6. Validate the correct installation of your license: ```python >>> kx.q.til(10) @@ -227,31 +188,27 @@ Follow the steps below to install a kdb Insights license for PyKX from Python: ### 2.b Install license with environment variables -For environment-specific flexibility, there are two ways to install your license: by using a file or by copying text. Both are sourced in your welcome email. Click on the tabs below, read the instructions, and choose the method you wish to follow: +For environment-specific flexibility, there are two ways to install your license: by using a file or by copying text. Click on the tabs below, read the instructions, and choose the method you wish to follow: !!! Note "" === "Using a file" - 1. For personal usage, navigate to the [personal license](https://kx.com/kdb-insights-sdk-personal-edition-download/) and complete the form. For commercial usage, contact your KX sales representative or sales@kx.com or apply through https://kx.com/book-demo. + 1. Download your license file. - 2. On receipt of an email from KX, download and save the license file to a secure location on your computer. - - 3. Set an environment variable pointing to the folder with the license file. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). + 2. Set an environment variable pointing to the folder with the license file. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). * **Variable Name**: `#!bash QLIC` * **Variable Value**: `#!bash /user/path/to/folder` === "Using text" - 1. For personal usage, navigate to the [personal license](https://kx.com/kdb-insights-sdk-personal-edition-download/) and complete the form. For commercial usage, contact your KX sales representative or sales@kx.com or apply through https://kx.com/book-demo. - - 2. On receipt of an email from KX, copy the `#!bash base64` encoded contents of your license provided in plain-text within the email. + 1. Copy the `#!bash base64` encoded contents of your license key provided. - 3. On your computer, set an environment variable `#!bash KDB_LICENSE_B64` when using a personal license or `KDB_K4LICENSE_B64` for a commercial license, pointing with the value copied in step 2. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). + 2. On your computer, set an environment variable with the contents of your `kc.lic` license key named `#!bash KDB_LICENSE_B64`. Or use `KDB_K4LICENSE_B64` if you have an older `k4.lic` license key. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). * **Variable Name**: `KDB_LICENSE_B64` / `KDB_K4LICENSE_B64` - * **Variable Value**: `` + * **Variable Value**: `` -To validate if you successfully installed your license with environment variables, start Python and import PyKX as follows: +To validate if you successfully installed your license with environment variables, start Python and import KDB-X Python as follows: ```bash $ python @@ -262,7 +219,7 @@ pykx.LongVector(pykx.q('0 1 2 3 4')) !!! Tip "Tip: automatic license renewal setup" - When your license nears its expiry date, you can set PyKX to automatically renew it. To do this, modify the environment variable `#!bash KDB_LICENSE_B64` or `#!bash KDB_K4LICENSE_B64` with your new license information. When PyKX initializes with the expired license, it will attempt to overwrite it with the new value: + When your license nears its expiry date, you can set KDB-X Python to automatically renew it. To do this, modify the environment variable `#!bash KDB_LICENSE_B64` or `#!bash KDB_K4LICENSE_B64` with your new license information. When KDB-X Python initializes with the expired license, it will attempt to overwrite it with the new value: ```shell $python @@ -273,26 +230,25 @@ pykx.LongVector(pykx.q('0 1 2 3 4')) License write location: /user/path/to/license/k4.lic ``` -## 3. Verify PyKX installation +## 3. Verify KDB-X Python installation -To verify if you successfully installed PyKX on your system, run: +To verify if you successfully installed KDB-X Python on your system, run: ```bash python -c"import pykx;print(pykx.__version__)" ``` -This command should display the installed version of PyKX. +This command should display the installed version of KDB-X Python. ## Dependencies -??? Info "Expand for Required and Optional PyKX dependencies" +??? Info "Expand for Required and Optional KDB-X Python dependencies" === "Required" - PyKX depends on the following third-party Python packages: + KDB-X Python depends on the following third-party Python packages: - - `pandas>=1.2, <2.0; python_version=='3.8'` - - `pandas>=1.2, <3.0; python_version>'3.8'` + - `pandas>=1.2; python_version>'3.8'` - `numpy>=1.22; python_version<'3.11'` - `numpy>=1.23; python_version=='3.11'` - `numpy>=1.26; python_version>='3.12'` @@ -301,15 +257,15 @@ This command should display the installed version of PyKX. - `dill>=0.2.0` - `requests>=2.25.0` - **Note**: All are installed automatically by `#!bash pip` when you install PyKX. + **Note**: All are installed automatically by `#!bash pip` when you install KDB-X Python. - Here's a breakdown of how PyKX uses these libraries: + Here's a breakdown of how KDB-X Python uses these libraries: - - [NumPy](https://pypi.org/project/numpy): converts data from PyKX objects to NumPy equivalent Array/Recarray style objects; direct calls to NumPy functions such as `numpy.max` with PyKX objects relies on the NumPy Python API. - - [Pandas](https://pypi.org/project/pandas): converts PyKX data to Pandas Series/DataFrame equivalent objects or to PyArrow data formats. Pandas is used as an intermediary data format. - - [pytz](https://pypi.org/project/pytz/): converts data with timezone information to PyKX objects to ensure that the offsets are accurately applied. - - [toml](https://pypi.org/project/toml/): for configuration parsing and management, with `.pykx-config` as outlined [here](../user-guide/configuration.md). - - [dill](https://pypi.org/project/dill): use in the serialization and deserialization of Python objects when interfacing between kdb+ and Python processes using [remote functions](../user-guide/advanced/remote-functions.md) or [real-time capture](../user-guide/advanced/streaming/index.md) functionality. + - [NumPy](https://pypi.org/project/numpy): converts data from `pykx` objects to NumPy equivalent Array/Recarray style objects; direct calls to NumPy functions such as `numpy.max` with `pykx` objects relies on the NumPy Python API. + - [Pandas](https://pypi.org/project/pandas): converts KDB-X Python data to Pandas Series/DataFrame equivalent objects or to PyArrow data formats. Pandas is used as an intermediary data format. + - [pytz](https://pypi.org/project/pytz/): converts data with timezone information to `pykx` objects to ensure that the offsets are accurately applied. + - [toml](https://pypi.org/project/toml/): for configuration parsing and management, with `config-KDB-X Python` as outlined [here](../user-guide/configuration.md). + - [dill](https://pypi.org/project/dill): used in the serialization and deserialization of Python objects when interfacing between q and Python processes using [remote functions](../user-guide/advanced/remote-functions.md) or [real-time capture](../user-guide/advanced/streaming/index.md) functionality. === "Optional" @@ -323,13 +279,13 @@ This command should display the installed version of PyKX. - **`psutil >=5.0.0`**: install via pip, with `streaming` extra, for example `pip install pykx[streaming]` - **`torch >2.1`**: install via pip, with `torch` extra, for example `pip install pykx[torch]` - Here's a breakdown of how PyKX uses these libraries: + Here's a breakdown of how KDB-X Python uses these libraries: - - [PyArrow](https://pypi.org/project/pyarrow): converts PyKX objects to and from their PyArrow equivalent table/array objects. - - [find-libpython](https://pypi.org/project/find-libpython): provides the `libpython.{so|dll|dylib}` file required by [PyKX under q](../pykx-under-q/intro.md). + - [PyArrow](https://pypi.org/project/pyarrow): converts `pykx` objects to and from their PyArrow equivalent table/array objects. + - [find-libpython](https://pypi.org/project/find-libpython): provides the `libpython.{so|dll|dylib}` file required by [KDB-X Python under q](../pykx-under-q/intro.md). - [ast2json](https://pypi.org/project/ast2json/): required for KX Dashboards Direct integration. - [psutil](https://pypi.org/project/psutil/): facilitates the stopping and killing of a q process on a specified port allowing for orphaned q processes to be stopped, functionality defined [here](../api/util.md#pykxutilkill_q_process). - - [torch](https://pytorch.org/docs/stable/): required for conversions between `#!python torch.Tensor` objects and their PyKX equivalents. + - [torch](https://pytorch.org/docs/stable/): required for conversions between `#!python torch.Tensor` objects and their KDB-X Python equivalents. **Optional non-Python dependencies:** @@ -347,45 +303,40 @@ This command should display the installed version of PyKX. ## Asset Information -| Platform | Mode | File | Version | -| --------- | ---------- | ----------- | ---------- | -| Linux ARM | kdb+ 4.0 | libq.so | 2025.02.18 | -| Linux x86 | kdb+ 4.0 | libq.so | 2025.02.18 | -| Mac ARM | kdb+ 4.0 | libq.dylib | 2025.02.18 | -| Mac x86 | kdb+ 4.0 | libq.dylib | 2025.02.18 | -| Windows | kdb+ 4.0 | q.dll/q.lib | 2025.02.18 | -| Linux ARM | kdb+ 4.1 | libq.so | 2026.01.23 | -| Linux x86 | kdb+ 4.1 | libq.so | 2026.01.23 | -| Mac ARM | kdb+ 4.1 | libq.dylib | 2026.01.23 | -| Mac x86 | kdb+ 4.1 | libq.dylib | 2026.01.23 | -| Windows | kdb+ 4.1 | q.dll/q.lib | 2026.01.23 | -| Linux ARM | Unlicensed | libe.so | 2023.11.22 | -| Linux x86 | Unlicensed | libe.so | 2023.11.22 | -| Mac ARM | Unlicensed | libe.so | 2023.11.22 | -| Mac x86 | Unlicensed | libe.so | 2023.11.22 | -| Windows | Unlicensed | e.dll/e.lib | 2024.08.21 | +| Platform | Mode | File | Version | +| --------- | ------------- | ----------- | ------------ | +| Linux ARM | KDB-X | libq.so | 5.0.20260501 | +| Linux x86 | KDB-X | libq.so | 5.0.20260501 | +| Mac ARM | KDB-X | libq.dylib | 5.0.20260501 | +| Mac x86 | KDB-X | libq.dylib | 5.0.20260501 | +| Windows | KDB-X | q.dll/q.lib | 5.0.20260501 | +| Linux ARM | Unlicensed | libe.so | 2023.11.22 | +| Linux x86 | Unlicensed | libe.so | 2023.11.22 | +| Mac ARM | Unlicensed | libe.so | 2023.11.22 | +| Mac x86 | Unlicensed | libe.so | 2023.11.22 | +| Windows | Unlicensed | e.dll/e.lib | 2024.08.21 | ## Optional: Installing a q executable -The following section is optional and primarily required if you are looking to make use of the [Real-Time Capture](../user-guide/advanced/streaming/index.md) functionality provided by PyKX. +The following section is optional and primarily required if you are looking to make use of the [Real-Time Capture](../user-guide/advanced/streaming/index.md) functionality provided by KDB-X Python. ### Do I need a q executable? -For the majority of functionality provided by PyKX you do not explicitly need access to a q executable. Users within a Python process who do not have a q executable will be able to complete tasks such as the following: +For the majority of functionality provided by KDB-X Python you do not explicitly need access to a q executable. Users within a Python process who do not have a q executable will be able to complete tasks such as the following: - Convert data to/from Python types - Run analytics on in-memory and on-disk databases - Create databases -- Query remote q/kdb+ processes via IPC -- Execute numpy functions with PyKX data +- Query remote q processes via IPC +- Execute numpy functions with KDB-X Python data If however you need to make use of the [Real-Time Capture](../user-guide/advanced/streaming/index.md) functionality you will need access to a q executable. Fundamentally the capture and persistence of real-time data and the application of analytics on this streaming data is supported via deployment of code on q processes. -### Configuring PyKX to use an existing executable +### Configuring KDB-X Python to use an existing executable -By default when attempting to start a q process for use within the Real-Time Capture workflows PyKX will attempt to call `q` directly, this method however is not fully reliable when using the Python `subprocess` module. As such the following setup can be completed to point more explicitly at your executable. +By default, when initializing a q process within Real-Time Capture workflows, KDB-X Python attempts to call `q` directly. However, this approach can be unreliable when the Python `subprocess` module is used to launch the process. To ensure consistency, configure the environment to explicitly reference the full path to your q executable. -If you already have a q executable, PyKX can use this when initializing the Real-Time Capture APIs through the setting of the following in your [configuration file](../user-guide/configuration.md#configuration-file) or as [environment variables](../user-guide/configuration.md#environment-variables): +If you already have a q executable, KDB-X Python can use this when initializing the Real-Time Capture APIs through the setting of the following in your [configuration file](../user-guide/configuration.md#configuration-file) or as [environment variables](../user-guide/configuration.md#environment-variables): | **Variable** | **Explanation** | | :------------------ | :--------------------------------------------------------------------------------------------------------------- | @@ -394,36 +345,7 @@ If you already have a q executable, PyKX can use this when initializing the Real ### Installing an executable -#### Installing using PyKX - -For users who do not have access to a q executable, PyKX provides a utility function `kx.util.install_q` to allow users access q locally. - -The following default information is used when installing the q executable: - -| **Parameter** | **Default** | **Explanation** | -| :--------------- | :------------------ | :--------------------------------------------------------------------------------------------------------------------- | -| location | `'~/q'` or `'C:\q'` | The location to which q will be installed if not otherwise specified. | -| date | `'2024.07.08'` | The dated version of kdb+ 4.0 which is to be installed. | - -The following provide a number of examples of using the installation functionality under various conditions. - -- Installing to default location - - ```python - >>> kx.util.install_q() - ``` - -- Installing to a specified location - - ```python - >>> kx.util.install_q('~/custom') - ``` - -Installation of q via this method will update the configuration file `.pykx-config` at either `~/q` or `C:\q` to include the location of `QHOME` and `PYKX_Q_EXECUTABLE` to be used. - -#### Installing without PyKX - -The installed q executable is not required to be installed via PyKX. If you wish to install q following the traditional approach you can follow the install instructions outlined [here](https://code.kx.com/q/learn/install/) or through signing up for a free-trial [here](https://kx.com/download-kdb/). +Go to [KDB-X](https://kdb-x.kx.com/) and follow the instructions. ### Installing in an air-gapped environment @@ -431,12 +353,12 @@ Installing Python libraries in air-gapped environments requires users to first d !!! Note "Build using the same environment as you're installing" - When downloading the `.whl` files and dependencies make sure you are using the same OS and Python version as you will be when installing in your isolated environment + When downloading the `.whl` files and dependencies make sure you are using the same OS and Python version as you will be when installing in your isolated environment. -In the case of PyKX users can in a internet enabled environment either +Prepare the installation package from an internet-enabled system in one of the following ways: 1. Download the `.whl` file for the OS, library version and Python version you are intending to use on the air-gapped environment. These files can be sourced from [here](https://pypi.org/project/pykx/#files). -2. Generate the `.whl` file from a git clone of the [PyKX repository](https://github.com/kxsystems/pykx). An example of this is as follows: +2. Generate the `.whl` file from a git clone of the [KDB-X Python repository](https://github.com/kxsystems/pykx). An example of this is as follows: ```bash $ git clone https://github.com/kxsystems/pykx @@ -445,7 +367,7 @@ In the case of PyKX users can in a internet enabled environment either # The below will install the `.whl` to a `dist/` folder $ python -m build . ``` -Once locally downloaded the dependencies of the `*.whl` file can be downloaded as follows: +After downloading the `*.whl` file locally, you can retrieve and save its required dependencies using the following command: ```bash $ pip download dist/*.whl @@ -459,9 +381,9 @@ Install the wheels which for simplicity are stored at a location `/opt/airgap/wh pip install --no-cache /opt/airgap/wheels/* ``` -### Verify PyKX can use the executable +### Verify KDB-X Python can use the executable -Verifying that PyKX has access to the executable can be done through execution of the function `#!python kx.util.start_q_subprocess` and requires either your configuration file or environment variables to include `PYKX_Q_EXECUTABLE`. This is outlined [here](#configuring-pykx-to-use-an-existing-executable). +Verifying that KDB-X Python has access to the executable can be done through execution of the function `#!python kx.util.start_q_subprocess` and requires either your configuration file or environment variables to include `PYKX_Q_EXECUTABLE`. This is outlined [here](#configuring-kdb-x-python-to-use-an-existing-executable). ```python >>> import pykx as kx @@ -474,7 +396,7 @@ pykx.LongAtom(pykx.q('2')) ## Next steps -That's it! You can now start using PyKX in your Python projects: +That's it! You can now start using KDB-X Python in your Python projects: - [Quickstart guide](quickstart.md) - [Updating/Upgrading your license](../user-guide/advanced/license.md) diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index aae05bf..313a6ef 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -1,30 +1,30 @@ --- -title: Quickstart for PyKX -description: Quickstart guide for setting up PyKX -date: June 2024 +title: Quickstart for KDB-X Python +description: Quickstart guide for setting up KDB-X Python +last_updated: October 2025 author: KX Systems, Inc., -tags: PyKX, quickstart, import PyKX, use PyKX objects +tags: KDB-X Python, quickstart, import KDB-X Python, use pykx objects --- # Quickstart -_This quickstart guide provides first time users with essential instructions for using the PyKX library._ +_This quickstart guide provides first time users with essential instructions for using the KDB-X Python library._ ## Prerequisites Before you start, make sure to: -- [Install the PyKX library](installing.md#1-install-pykx). -- [Have a kdb Insights license](installing.md#2-install-a-kdb-insights-license). +- [Install the KDB-X Python library](installing.md#1-install-pykx). +- [Have a license](installing.md#2-install-a-license). -## 1. Import PyKX +## 1. Import KDB-X Python -To access PyKX, import it within your Python code using the following syntax: +To access KDB-X Python, import it within your Python code using the following syntax: ```python >>> import pykx as kx ``` -!!! Info "The use of the shortened name `#!python kx` is optional and provides a terse convention for interacting with methods and objects from the PyKX library." +!!! Info "The use of the shortened name `#!python kx` is optional and provides a terse convention for interacting with methods and objects from the KDB-X Python library." Many examples in this guide make use of NumPy so we import this: @@ -32,13 +32,13 @@ Many examples in this guide make use of NumPy so we import this: >>> import numpy as np ``` -## 2. Generate PyKX objects +## 2. Generate `pykx` objects -You can generate PyKX objects in three ways. Click on the tabs below to follow the instructions: +You can generate `pykx` objects in three ways. Click on the tabs below to follow the instructions: -=== "Use PyKX functions" +=== "Use KDB-X Python functions" - Generate PyKX objects using `#!python pykx` helper functions: + Generate `pykx` objects using `#!python pykx` helper functions: ```python >>> kx.random.random([3, 4], 10.0) @@ -62,7 +62,7 @@ You can generate PyKX objects in three ways. Click on the tabs below to follow t === "From Python data types" - Generate PyKX objects from Python, NumPy, Pandas and PyArrow objects by using the `#!python kx.toq` method: + Generate `pykx` objects from Python, NumPy, Pandas and PyArrow objects by using the `#!python kx.toq` method: ```python >>> pylist = [10, 20, 30] @@ -107,7 +107,7 @@ You can generate PyKX objects in three ways. Click on the tabs below to follow t === "Execute q code" - Generate PyKX objects using q by calling `#!python kx.q`: + Generate `pykx` objects using q by calling `#!python kx.q`: ```python >>> kx.q('10 20 30') @@ -125,11 +125,18 @@ You can generate PyKX objects in three ways. Click on the tabs below to follow t ')) ``` -## 3. Interact with PyKX objects +## 3. Interact with `pykx` objects -You can interact with PyKX objects in a variety of ways, for example, through [indexing using Pythonic syntax](../user-guide/fundamentals/indexing.md), passing [PyKX objects to q/NumPy](../user-guide/fundamentals/creating.md#2-convert-pykx-objects-to-pythonic-types) functions, [querying via Python/SQL/qSQL](..//user-guide/fundamentals/query/index.md) syntax or by [using the q functionality](../user-guide/advanced/context_interface.md) via the context interface. Each way is described in more depth under the the User guide > Fundamentals section. For now, we recommend a few examples: +You can interact with KDB-X Python objects in a variety of ways, for example: -* Create a PyKX list and interact with it using indexing and slices: +- through [indexing using Pythonic syntax](../user-guide/fundamentals/indexing.md) +- passing [`pykx` objects to q/NumPy](../user-guide/fundamentals/creating.md#converting-pykx-objects-to-pythonic-types) functions +- [querying via Python/SQL/qSQL](..//user-guide/fundamentals/query/index.md) syntax +- by [using the q functionality](../user-guide/advanced/context_interface.md) via the context interface. + +Each way is described in more depth under the **How to** section. For now, we recommend a few examples: + +* Create a KDB-X Python list and interact with it using indexing and slices: ```python >>> qarray = kx.random.random(10, 1.0) @@ -141,7 +148,7 @@ You can interact with PyKX objects in a variety of ways, for example, through [i pykx.FloatVector(pykx.q('0.08123546 0.9367503 0.2782122')) ``` -* Assign objects to PyKX lists: +* Assign objects to KDB-X Python lists: ```python >>> qarray = kx.random.random(3, 10.0, seed=10) @@ -151,7 +158,7 @@ You can interact with PyKX objects in a variety of ways, for example, through [i pykx.FloatVector(pykx.q('0.891041 0.1 3.621949')) ``` -* Create a PyKX table and manipulate using Pythonic syntax: +* Create a KDB-X Python table and manipulate using Pythonic syntax: ```python >>> N = 100 @@ -205,7 +212,7 @@ You can interact with PyKX objects in a variety of ways, for example, through [i ')) ``` -* Pass a PyKX object to a q function: +* Pass a KDB-X Python object to a q function: ```python >>> qfunction = kx.q('{x+til 10}') @@ -213,7 +220,7 @@ You can interact with PyKX objects in a variety of ways, for example, through [i pykx.FloatVector(pykx.q('0.3992327 1.726329 2.488636 3.653597 4.028107 5.444905 6.542917 7.00628 8.152..')) ``` -* Apply a Python function on a PyKX Vector: +* Apply a Python function on a KDB-X Python vector: ```python >>> qvec = kx.random.random(10, 10, seed=42) @@ -223,7 +230,7 @@ You can interact with PyKX objects in a variety of ways, for example, through [i pykx.LongVector(pykx.q('5 8 3 3 10 5 3 1 9 1')) ``` -* Pass PyKX arrays of objects to NumPy functions: +* Pass `pykx` arrays of objects to NumPy functions: ```python >>> qarray1 = kx.random.random(10, 1.0) @@ -323,9 +330,9 @@ You can interact with PyKX objects in a variety of ways, for example, through [i pykx.LongAtom(q('5')) ``` -## 4. Convert PyKX objects to Python types +## 4. Convert `pykx` objects to Python types -To convert the objects generated via the PyKX library to the corresponding `#!python Python`, `#!python NumPy`, `#!python Pandas`, and `#!python PyArrow` types, use `#!python py`, `#!python np`, `#!python pd`, and `#!python pa` methods. Click on the tabs below to go through the examples: +To convert the objects generated via the KDB-X Python library to the corresponding `#!python Python`, `#!python NumPy`, `#!python Pandas`, and `#!python PyArrow` types, use `#!python py`, `#!python np`, `#!python pd`, and `#!python pa` methods. Click on the tabs below to go through the examples: === "Convert to Python" @@ -395,7 +402,7 @@ To convert the objects generated via the PyKX library to the corresponding `#!py ``` If using `#!python pandas>=2.0` it is possible to also use the `#!python as_arrow` keyword argument to convert to - pandas types using pyarrow as the backend instead of the default numpy backed pandas objects. + pandas types using pyarrow as the backend instead of the default NumPy-backed Pandas objects. ```python >>> qvec = kx.toq(np.random.randint(5, size=10)) diff --git a/docs/getting-started/what_is_pykx.md b/docs/getting-started/what_is_pykx.md index 543e39b..bc0adb4 100644 --- a/docs/getting-started/what_is_pykx.md +++ b/docs/getting-started/what_is_pykx.md @@ -1,82 +1,82 @@ --- -title: What is PyKX? -description: Overview of Pykx -date: June 2024 +title: What is KDB-X Python +description: Overview of KDB-X Python +last_updated: October 2025 author: KX Systems, Inc., -tags: about PyKX, q/kdb+, use cases, +tags: about KDB-X Python, q, use cases, --- -# What is PyKX? +# What is KDB-X Python? -_This page briefly describes PyKX, its use cases, and its connection with q/kdb+._ +_This page briefly describes KDB-X Python, its use cases, and its connection with q._ ## Introduction -**PyKX** is a Python-first interface to **kdb+** (the world's fastest time-series database) and **q** (kdb+'s underlying vector programming language). PyKX is the result of 10+ years of integrations between two languages: Python and q. Its aim is to help users query and analyze huge amounts of in-memory and on-disk time-series data, significantly faster than other libraries. +**KDB-X Python** is a Python-first interface to **KDB-X** (the world's fastest time-series database) and **q** (KDB-X's underlying vector programming language). KDB-X Python is the result of 10+ years of integrations between two languages: Python and q. Its aim is to help users query and analyze huge amounts of in-memory and on-disk time-series data, significantly faster than other libraries. -- For **Python developers**, PyKX unlocks the speed and power of kdb+ for data processing and storage from within your Python environment. It enables anyone with Python knowledge to apply analytics against vast amounts of data, both in-memory and on-disk, in a fraction of the time, allowing you to focus on getting the best from your data. +- For **Python developers**, KDB-X Python unlocks the speed and power of kdb+ for data processing and storage from within your Python environment. It enables anyone with Python knowledge to apply analytics against vast amounts of data, both in-memory and on-disk, in a fraction of the time, allowing you to focus on getting the best from your data. -- For **q developers**, PyKX brings together Python's data science ecosystem and the power of kdb+'s vector and time-series analytics. This makes them available in both q and Python environments. You can use it to run q code within a Python environment or embed Python analytics within your q session. +- For **q developers**, KDB-X Python brings together Python's data science ecosystem and the power of kdb+'s vector and time-series analytics. This makes them available in both q and Python environments. You can use it to run q code within a Python environment or embed Python analytics within your q session. ## Use cases -PyKX supports three main use cases, allowing Python data engineers and data scientists to: +KDB-X Python supports three main use cases, allowing Python data engineers and data scientists to: 1. Store, query, manipulate and use q objects within a Python process. 2. Query external q processes via an [Inter-Process Communication (IPC)](..//user-guide/advanced/ipc.md) interface. 3. Embed Python functionality within a native q session using its [under q](../pykx-under-q/intro.md) functionality. -??? Note "Expand to learn more about q/kdb+" +??? Note "Expand to learn more about q/KDB-X" - Used throughout the financial sector for 25+ years, q and kdb+ have been a cornerstone of modern financial markets. This technology provides a storage mechanism for historical market data and performant tooling to analyze this vast streaming, real-time and historical data. + Used throughout the financial sector for 25+ years, q and KDB-X (formerly known as kdb+) have been a cornerstone of modern financial markets. This technology provides a storage mechanism for historical market data and performant tooling to analyze this vast streaming, real-time and historical data. - - **Kdb+** is a high-performance column-oriented database designed to process and store large amounts of data. Commonly accessed data is available in RAM which makes it faster to access than disk stored data. Operating with temporal data types as a first class entity the use of q and it's query language qsql against this database creates a highly performant time-series analysis tool available. + - **KDB-X** is a high-performance column-oriented database designed to process and store large amounts of data. Commonly accessed data is available in RAM which makes it faster to access than disk stored data. Operating with temporal data types as a first class entity the use of q and it's query language qsql against this database creates a highly performant time-series analysis tool available. - - **q** is the vector programming language which is used for all interactions with kdb+ databases, known both for its speed and expressiveness. PyKX exposes q as a domain-specific language (DSL) embedded within Python. The assumption is that q is mainly used for data processing and database management. + - **q** is the vector programming language which is used for all interactions with KDB-X databases, known both for its speed and expressiveness. KDB-X Python exposes q as a domain-specific language (DSL) embedded within Python. The assumption is that q is mainly used for data processing and database management. This approach benefits users in two ways: - Helps users familiar with q to make the most of its advanced analytics and database management. - - Empowers kdb+/q users who lack q expertise to get up and running quickly. + - Empowers KDB-X/q users who lack q expertise to get up and running quickly. - For more information on using q/kdb+ and getting started with see the following links: + For more information on using q/KDB-X and getting started with see the following links: - - [An introduction to q/kdb+](https://code.kx.com/q/learn/tour/) - - [Tutorial videos introducing kdb+/q](https://code.kx.com/q/learn/q-for-all/) + - [An introduction to q/KDB-X](https://code.kx.com/q/learn/tour/) + - [Tutorial videos introducing q/KDB-X](https://code.kx.com/q/learn/q-for-all/) -## PyKX vs. Python/q interfaces +## KDB-X Python vs. Python/q interfaces -There are three historical interfaces which allow interoperability between Python and q/kdb+: +There are three historical interfaces which allow interoperability between Python and q/KDB-X: 1. [Embedpy](https://code.kx.com/q/ml/embedpy) 2. [PyQ](https://github.com/KxSystems/pyq) 3. [qPython](https://github.com/KxSystems/pyq) -How does PyKX compare to other q interfaces for Python? Here’s a TL;DR comparison table highlighting the key differences between EmbedPy, PyQ, qPython, and PyKX: +How does KDB-X Python compare to other q interfaces for Python? Here’s a TL;DR comparison table highlighting the key differences between EmbedPy, PyQ, qPython, and KDB-X Python: -| **Feature** | **EmbedPy** | **PyQ** | **qPython** | **PyKX** | +| **Feature** | **EmbedPy** | **PyQ** | **qPython** | **KDB-X Python** | | ---------------- | ------------------------------------------------ | ------------------------------------- | ----------------------------------------------------------------------- | -------------------------------------------------------------------------------- | | Interoperability | Python from q | Python & q in same process | Python-first, over IPC | Python-first, in-process & IPC | | Execution | Designed for q developers, run in q session | Requires PyQ binary or start from q | Processing completed on q session via IPC, deserialization using Python | Run from Python session, class-based type system | | Use Case | Leverage Python functionality not available in q | Operate on same data across languages | Common use case, expensive in processing & memory | Store, query, manipulate q objects within Python, via IPC or Python in q session | | Flexibility | \- | Locked into using PyQ binary | \- | Pythonic interface, context interface for q scripts, q first mode, IPC available | -| Data Conversion | q to/from Numpy/Python only | q to/from Numpy/Pandas only | Data converted directly from socket | Leverages q memory space embedded within Python, supports NumPy, Pandas, PyArrow | +| Data Conversion | q to/from NumPy/Python only | q to/from Numpy/Pandas only | Data converted directly from socket | Leverages q memory space embedded within Python, supports NumPy, Pandas, PyArrow | -??? Note "Expand to learn more about EmbedPy, PyQ, qPython, and PyKX" +??? Note "Expand to learn more about EmbedPy, PyQ, qPython, and KDB-X Python" To give you a clear understanding of how each interface operates and their suitability for different use cases, here are some additional details: - **EmbedPy** allows using Python from q but does not interface with q from Python. It’s mainly for q developers to access Python functionalities like machine learning, statistical methods, and plotting. - **PyQ** integrates Python and q interpreters in the same process, but it requires executing a special PyQ binary or starting from q, which is not ideal for Python use cases that require a standard Python binary. - **qPython** takes a Python-first approach but works entirely over IPC (Inter-Process Communication), meaning Python objects sent to q and q objects returned are serialized, sent over a socket, and then deserialized, which can be resource-intensive. - - **PyKX** supports storing, querying, manipulating, and using q objects within a Python process and querying external q processes via IPC. PyKX provides a more Pythonic approach with a class-based hierarchical type system and a context interface for interacting with q scripts in a Pythonic manner. + - **KDB-X Python** supports storing, querying, manipulating, and using q objects within a Python process and querying external q processes via IPC. KDB-X Python provides a more Pythonic approach with a class-based hierarchical type system and a context interface for interacting with q scripts in a Pythonic manner. !!! tip "EmbedPy, PyQ, qPython: Interface support" - KX maintains both embedPy and PyQ on a best-efforts basis under the [Fusion](https://code.kx.com/q/interfaces) initiative. qPython is in maintenance mode, not supported by KX. If you're using EmbedPy, PyQ, or qPython, we recommend switching to PyKX to pick up the latest updates from KX. + KX maintains both embedPy and PyQ on a best-efforts basis under the [Fusion](https://code.kx.com/q/interfaces) initiative. qPython is in maintenance mode, not supported by KX. If you're using EmbedPy, PyQ, or qPython, we recommend switching to KDB-X Python to pick up the latest updates from KX. ## Next steps - Follow the [Installation guide](installing.md) -- Get up and running with the PyKX [Quickstart guide](quickstart.md) -- Get to know more functionalities with the [PyKX Introduction Notebook](../examples/interface-overview.ipynb) +- Get up and running with the KDB-X Python [Quickstart guide](quickstart.md) +- Get to know more functionalities with the [KDB-X Python Introduction Notebook](../examples/interface-overview.ipynb) diff --git a/docs/help/faq.md b/docs/help/faq.md index cd481a8..4e719df 100644 --- a/docs/help/faq.md +++ b/docs/help/faq.md @@ -1,9 +1,9 @@ --- title: FAQ -description: Frequently questions for PyKX +description: Frequently questions for KDB-X Python maintained by: KX Systems, Inc. date: Aug 2024 -tags: PyKX, FAQ +tags: KDB-X Python, FAQ --- # FAQ @@ -15,7 +15,7 @@ tags: PyKX, FAQ '2022.09.15T10:32:13.419 license error: cores ``` -This error indicates PyKX tried to use more cores than your license allows. You can fix this by limiting the number of cores used by the python process. +This error indicates that KDB-X Python tried to use more cores than your license allows. You can fix this by limiting the number of cores used by the python process. - On Linux you can use `#!bash taskset` to limit the number of cores used by a process: @@ -24,7 +24,7 @@ This error indicates PyKX tried to use more cores than your license allows. You $ taskset -c 0-3 python ``` -- You can also do this in python before importing PyKX (Linux only): +- You can also do this in python before importing KDB-X Python (Linux only): ```python >>> import os @@ -42,15 +42,15 @@ pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) (above, `#!bat 0xf = 00001111b`, so the python process will only use the four cores for which the mask bits are equal to 1) -## How does PyKX determine the license that is used? +## How does KDB-X Python determine the license that is used? -The following steps are run by PyKX to find the license when you execute `#!python import pykx`: +The following steps are run by `pykx` to find the license when you execute `#!python import pykx`: 1. Search for **kx.lic**, **kc.lic** and **k4.lic** license files in this order within the following locations: 1. Current working directory 1. Location defined by environment variable `#!bash QLIC` if set 1. Location defined by environment variable `#!bash QHOME` if set -2. If a license is not found PyKX will use the following environment variables (if they are set) to install and make use of a license: +2. If a license is not found KDB-X Python will use the following environment variables (if they are set) to install and make use of a license: 1. `#!bash KDB_LICENSE_B64` containing a base64 encoded version of a **kc.lic** license 1. `#!bash KDB_K4LICENSE_B64` containing a base64 encoded version of a **k4.lic** license 3. If a license has not been located you will be guided to install a license following a prompt based license installation. diff --git a/docs/help/issues.md b/docs/help/issues.md index 78a9f32..966d957 100644 --- a/docs/help/issues.md +++ b/docs/help/issues.md @@ -3,19 +3,19 @@ title: Issues and Limitations description: Known issues that occur when using python and limitations when using q embedded in python maintained by: KX Systems, Inc. date: Aug 2024 -tags: PyKX, issues, embedded +tags: PyKX, issues, embedded, KDB-X Python --- # Issues and Limitations -_This page details known issues and functional limitations when using PyKX either as q embedded in a python process via the `#!python import pykx` command, or as a python processes embedded in q via `#!q \l pykx.q`._ +_This page details known issues and functional limitations when using KDB-X Python either as q embedded in a python process via the `#!python import pykx` command, or as a python processes embedded in q via `#!q \l pykx.q`._ -## PyKX +## KDB-X Python (`pykx`) ### Known issues * Enabling the NEP-49 NumPy allocators will often segfault when running in a multiprocess setting. * The timeout value is always set to 0 when using PYKX_Q_LOCK. * Enabling PYKX_ALLOCATOR and using PyArrow tables can cause segfaults. -* Multiprocessing is not available on Windows. When using PyKX in a multiprocessing setup, PyKX must be imported and initialized within each subprocess where it is needed, regardless of the multiprocessing start method used (spawn, fork, or forkserver). +* Multiprocessing is not available on Windows. When using KDB-X Python in a multiprocessing setup, KDB-X Python must be imported and initialized within each subprocess where it is needed, regardless of the multiprocessing start method used (spawn, fork, or forkserver). * Multithreading mode is not available on Windows. * `#!python kurl` functions require their `#!python options` dictionary to have mixed type values. Add a `#!python None` value to bypass: `#!python {'': None, ...}` * `#!python None` and `#!python pykx.Identity(pykx.q('::'))` do not pass through to single argument Python functions set under q as outlined in this example: @@ -35,7 +35,7 @@ pykx.LongAtom(pykx.q('2')) >>> kx.toq(df, no_allocator=True) ``` -Or for PyKX under q you can use. +Or for KDB-X Python under q you can use. ```q q) df: .pykx.pyeval"pd.read_parquet('nested_arrs.parquet')"; q) .pykx.toq .pykx.noalloc df; @@ -57,12 +57,12 @@ pykx.Identity(pykx.q('::')) Now, in a Python or q process attempt to connect to the above embedded q server. ```python >>> import pykx as kx ->>> q = kx.QConnection(port=5001) # Attempt to create a q connection to a PyKX embedded q instance +>>> q = kx.QConnection(port=5001) # Attempt to create a q connection to a `pykx` embedded q instance # This process is now hung indefinitely as the embedded q server cannot respond ``` ```q -q)h:hopen`::5001 /Attempting to create an IPC connection to a PyKX embedded q instance +q)h:hopen`::5001 /Attempting to create an IPC connection to a `pykx` embedded q instance /This process is now hung indefinitely as the embedded q server cannot respond ``` @@ -74,7 +74,7 @@ Timers in q rely on the main loop of the standalone executable so they will not >>> kx.q('\t 1000') # Set timer to tick every 1000ms pykx.Identity(pykx.q('::')) # No output follows because the timer never ticks ``` -Attempting to use the timer callback function directly using PyKX will raise an AttributeError: +Attempting to use the timer callback function directly using `pykx` will raise an AttributeError: ```python >>> kx.q.z.ts AttributeError: ts: .z.ts is not exposed through the context interface because there is no main loop in the embedded q process @@ -111,7 +111,7 @@ q)(::)~{x}[] /x parameter received by lambda is the generic null :: 1b ``` -Using `#!q ::` as an argument to PyKX functions presents some difficulties: +Using `#!q ::` as an argument to `pykx` functions presents some difficulties: ```q q)f:.pykx.eval["lambda x: x";<] q)f[::] /the Python process cannot tell the difference between f[] and f[::] so throws an error @@ -158,7 +158,7 @@ False ``` **Cause:** -Pandas now checks the type of the `_mgr` (dataframes manager) property. PyKX uses a custom `_mgr` implementation for performance optimization. +Pandas now checks the type of the `_mgr` (dataframes manager) property. KDB-X Python uses a custom `_mgr` implementation for performance optimization. ```python >>> type(df1._mgr) diff --git a/docs/help/support.md b/docs/help/support.md index 5363819..a659736 100644 --- a/docs/help/support.md +++ b/docs/help/support.md @@ -1,17 +1,17 @@ --- title: Help and Support -description: Summary of help and support for PyKX +description: Summary of help and support for KDB-X Python (`pykx`) maintained by: KX Systems, Inc. date: Aug 2024 tags: support, PyKX --- # Support -_This page provides links users can follow to receive help and support for the PyKX library_ +_This page provides links users can follow to receive help and support for the KDB-X Python library._ ## Community Help -If you have any issues or questions you can post them to the following locations, each of which is monitored by the PyKX development team: +If you have any issues or questions you can post them to the following locations, each of which is monitored by the KDB-X Python development team: - Ask a question to the KX community at [community.kx.com](https://community.kx.com/t5/PyKX/bd-p/PyKX). - Use Stack Overflow with the tags [pykx](https://stackoverflow.com/questions/tagged/pykx) or [kdb](https://stackoverflow.com/questions/tagged/kdb) depending on the subject. diff --git a/docs/help/troubleshooting.md b/docs/help/troubleshooting.md index 3f96ba0..2706054 100644 --- a/docs/help/troubleshooting.md +++ b/docs/help/troubleshooting.md @@ -2,16 +2,15 @@ ## License issues -The following section outlines practical information useful when dealing with getting access to and managing licenses for PyKX. +The following section outlines practical information useful when dealing with getting access to and managing licenses for KDB-X Python. -### Accessing a license valid for PyKX +### Accessing a license valid for KDB-X Python -A number of trial and enterprise type licenses exist for q/kdb+. Not all licenses for q/kdb+ however are valid for PyKX. In particular users require access to a license which contains the feature flags **pykx** and **embedq** which provide access to the PyKX functionality. The following locations can be used for the retrieval of evaluation/personal licenses +KDB-X licenses use flags **py** and **eq** to enable KDB-X Python (these work with KDB-X Python `pykx>=4.0` only and not PyKX `pykx<4.0`). +Older licenses using long flags **pykx** and **embedq** also work with KDB-X Python. -- For non-commercial personal users you can access a 12 month kdb+ license with PyKX enabled [here](https://kx.com/kdb-insights-sdk-personal-edition-download). -- For commercial evaluation, contact your KX sales representative or sales@kx.com requesting a PyKX trial license. Alternately apply through https://kx.com/book-demo. - -For non-personal or non-commercial usage please contact sales@kx.com. +Go to the [KX Developer Center](https://developer.kx.com/products/kdb-x/install) to obtain a KDB-X license. +For other enquiries email sales@kx.com. Once you have access to your license you can install the license following the steps provided [here](../getting-started/installing.md) or through installation using the function `#!python kx.license.install` as follows @@ -22,11 +21,16 @@ Once you have access to your license you can install the license following the s ### Initialization failing with a `#!python embedq` error -Failure to initialize PyKX while raising an error `#!python embedq` indicates that the license you are attempting to use for PyKX in [licensed modality](../user-guide/advanced/modes.md) does not have the sufficient feature flags necessary to run PyKX. To access a license which does allow for running PyKX in this modality please following the instructions [here](#accessing-a-license-valid-for-pykx) to get a new license with appropriate feature flags. +Failure to initialize `pykx` while raising an error `#!python embedq` indicates that the license you are attempting to use for `pykx` in [licensed modality](../user-guide/advanced/modes.md) does not have the sufficient feature flags necessary to run KDB-X Python. To access a license which does allow for running `pykx` in this modality please following the instructions [here](#accessing-a-license-valid-for-pykx) to get a new license with appropriate feature flags. + +KDB-X licenses use flags **py** and **eq** to enable KDB-X Python (these work with KDB-X Python `pykx>=4.0` only and not PyKX `pykx<4.0`). +Older licenses using long flags **pykx** and **embedq** also work with KDB-X Python. + +Check the output of `.z.l` in a q process to view the flags in your license. ### Initialization failing with a `#!python kc.lic` error -If after initially completing the installation guide for PyKX [here](../getting-started/installing.md) you receive the following error: +If after initially completing the installation guide for `pykx` [here](../getting-started/installing.md) you receive the following error: ```python pykx.exceptions.PyKXException: Failed to initialize embedded q. Captured output from initialization attempt: @@ -91,16 +95,16 @@ It usually indicates that your license was not correctly written to disk or a li ## Environment issues -### Using PyKX under q is raising a `'libpython` error +### Using KDB-X Python under q is raising a `'libpython` error -If you are getting a `'libpython` error when starting PyKX within a q session, this may indicate that PyKX has been unable to source the Python shared libraries that are required to run Python within an embedded setting. To fix this issue users can either +If you are getting a `'libpython` error when starting `pykx` within a q session, this may indicate that KDB-X Python has been unable to source the Python shared libraries that are required to run Python within an embedded setting. To fix this issue users can either - Find the absolute path to the appropriate shared object and set the environment variable `PYKX_PYTHON_LIB_PATH` with this location. - Set the environment variable `PYKX_USE_FIND_LIBPYTHON` to `"true"`, this will use the Python library [`find-libpython`](https://pypi.org/project/find-libpython/) to locate the `libpython` shared library and automatically set `PYKX_PYTHON_LIB_PATH` to the returned location. ### Getting more information about your environment -The following section outlines how a user can get access to a verbose set of environment configuration associated with PyKX. This information is helpful when debugging your environment and should be provided if possible with support requests. +The following section outlines how a user can get access to a verbose set of environment configuration associated with KDB-X Python. This information is helpful when debugging your environment and should be provided if possible with support requests. ```python >>> import pykx as kx @@ -111,7 +115,7 @@ The following section outlines how a user can get access to a verbose set of env ```python >>> kx.util.debug_environment() - **** PyKX information **** + **** KDB-X Python information **** pykx.args: () pykx.qhome: /usr/local/anaconda3/envs/qenv/q pykx.qlic: /usr/local/anaconda3/envs/qenv/q @@ -131,29 +135,28 @@ The following section outlines how a user can get access to a verbose set of env **** Platform information **** platform.platform: macOS-13.0.1-x86_64-i386-64bit - **** PyKX Configuration File **** - File location: /usr/local/.pykx-config + **** KDB-X Python Configuration File **** + File location: /usr/local/.kx/config-pykx Used profile: default Profile content: {'PYKX_Q_EXECUTABLE': '/usr/local/anaconda3/envs/qenv/q/m64/q'} - **** PyKX Configuration Variables **** + **** KDB-X Python Configuration Variables **** PYKX_IGNORE_QHOME: False PYKX_KEEP_LOCAL_TIMES: False - PYKX_ALLOCATOR: False + PYKX__NO_ALLOCATOR: False PYKX_GC: False PYKX_LOAD_PYARROW_UNSAFE: False PYKX_MAX_ERROR_LENGTH: 256 - PYKX_NOQCE: False + PYKX_QCE: False PYKX_RELEASE_GIL: False PYKX_Q_LIB_LOCATION: /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/pykx/lib PYKX_Q_LOCK: False PYKX_SKIP_UNDERQ: False PYKX_Q_EXECUTABLE: /usr/local/anaconda3/envs/qenv/q/m64/q PYKX_THREADING: False - PYKX_4_1_ENABLED: False PYKX_QDEBUG: False PYKX_DEBUG_INSIGHTS_LIBRARIES: False - PYKX_CONFIGURATION_LOCATION: . + PYKX_CONFIGURATION_LOCATION: PYKX_NO_SIGNAL: False PYKX_CONFIG_PROFILE: default PYKX_BETA_FEATURES: True @@ -168,13 +171,13 @@ The following section outlines how a user can get access to a verbose set of env PYKX_USE_FIND_LIBPYTHON: PYKX_UNLICENSED: PYKX_LICENSED: - PYKX_4_1_ENABLED: **** q Environment Variables **** QARGS: QHOME: /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/pykx/lib QLIC: /usr/local/anaconda3/envs/qenv/q - QINIT: + QINIT: + QCFG: **** License information **** pykx.qlic directory: True @@ -193,7 +196,7 @@ The following section outlines how a user can get access to a verbose set of env ### Debugging q code issues -If you are developing a library of q code, by default PyKX does not provide the full backtrace on error. As an example assume you have developed the a function and pass it an incorrect input +If you are developing a library of q code, by default KDB-X Python does not provide the full backtrace on error. As an example assume you have developed the a function and pass it an incorrect input ```python >>> import pykx as kx diff --git a/docs/index.md b/docs/index.md index 651c04b..8dbcc80 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,8 +1,8 @@ -# PyKX Home +# KDB-X Python Home -_Welcome to the home of PyKX technical documentation._ +_Welcome to the home of KDB-X Python technical documentation._ -[PyKX](./getting-started/what_is_pykx.md) is a Python-first interface to kdb+, the world's fastest time-series database, and its underlying vector programming language q. Whether you're a Python developer looking to harness kdb+'s analytics, or a q developer wanting to leverage Python's data science ecosystem, PyKX bridges both worlds. +[KDB-X Python](./getting-started/what_is_pykx.md) is a Python-first interface to kdb+, the world's fastest time-series database, and its underlying vector programming language q. Whether you're a Python developer looking to harness kdb+'s analytics, or a q developer wanting to leverage Python's data science ecosystem, KDB-X Python bridges both worlds. ## Ready to dive in? @@ -10,8 +10,8 @@ _Welcome to the home of PyKX technical documentation._ @@ -26,7 +26,7 @@ _Welcome to the home of PyKX technical documentation._

How-to Guides

@@ -50,7 +50,7 @@ _Welcome to the home of PyKX technical documentation._

Examples

@@ -59,9 +59,8 @@ _Welcome to the home of PyKX technical documentation._ ## What's next? -Discover how PyKX builds upon kdb+ and what's coming soon: +Discover how KDB-X Python builds upon KDB-X and what's coming soon: - Submit a [feature request](https://forum.kx.com/c/ideas-feature-requests/10) - Stay updated with the latest [release notes](release-notes/changelog.md){target="_blank"} - Join our [community channels](https://kx-community.slack.com/) to get assistance - diff --git a/docs/learn/objects.md b/docs/learn/objects.md index af138f3..d5e279c 100644 --- a/docs/learn/objects.md +++ b/docs/learn/objects.md @@ -1,26 +1,26 @@ --- -title: PyKX Objects and Attributes -description: Objects explained for PyKX +title: pykx Objects and Attributes +description: Objects explained for KDB-X Python date: September 2024 author: KX Systems, Inc., -tags: PyKX, object +tags: KDB-X Python, object --- -# PyKX objects and attributes +# `pykx` objects and attributes -_This page explains objects and attributes in PyKX._ +_This page explains objects and attributes in KDB-X Python._ -## What are PyKX objects? +## What are `pykx` objects? -PyKX objects are Python representations of kdb+ data structures. They allow Python developers to interact with kdb+ databases, perform complex queries, and manipulate data efficiently. +`pykx` objects are Python representations of KDB-X data structures. They allow Python developers to interact with KDB-X databases, perform complex queries, and manipulate data efficiently. -When you call or connect to a q instance, it returns a PyKX object. This object is an instance of the [`#!python pykx.K`](../api/pykx-q-data/wrappers.md#pykx.wrappers.K) class or one of its subclasses, as documented on the [PyKX wrappers API](..//api/pykx-q-data/wrappers.md) page. +When you call or connect to a q instance, it returns a KDB-X Python object. This object is an instance of the [`#!python pykx.K`](../api/pykx-q-data/wrappers.md#pykx.wrappers.K) class or one of its subclasses, as documented on the [KDB-X Python wrappers API](..//api/pykx-q-data/wrappers.md) page. -PyKX objects act as wrappers around objects in q’s memory space within the Python process where PyKX (and your program) runs. These wrappers are efficient to create since they don’t require copying data out of q’s memory space. +`pykx` objects act as wrappers around objects in q’s memory space within the Python process where KDB-X Python (and your program) runs. These wrappers are efficient to create since they don’t require copying data out of q’s memory space. -PyKX objects support various Python features like iteration, slicing, and calling, so converting them to other types (for example, from [`#!python pykx.Vector`](../api/pykx-q-data/wrappers.md#pykx.wrappers.Vector) to `#!python numpy.ndarray`) is often unnecessary. +`pykx` objects support various Python features like iteration, slicing, and calling, so converting them to other types (for example, from [`#!python pykx.Vector`](../api/pykx-q-data/wrappers.md#pykx.wrappers.Vector) to `#!python numpy.ndarray`) is often unnecessary. -Examples of PyKX objects: +Examples of `pykx` objects: - **Atoms**: Single values, such as integers, floats, or symbols. - **Vectors**: Arrays of values of the same type. @@ -28,20 +28,20 @@ Examples of PyKX objects: - **Tables**: Collections of columns, where each column is a vector. - **Lists**: These can contain elements of different types. -### How to use PyKX objects +### How to use `pykx` objects -To leverage the power of kdb+ within a Python environment, you can perform the following key operations with PyKX objects: +To leverage the power of KDB-X within a Python environment, you can perform the following key operations with `pykx` objects: | **Operation** | **Description** | |--------------------------------------------------------------|-------------| -| [Create and convert](../user-guide/fundamentals/creating.md) | Create PyKX objects from and to various Python objects, such as lists, dictionaries, and NumPy arrays. | -| [Use](../user-guide/fundamentals/evaluating.md) | Once created, interact with PyKX objects using familiar Pythonic syntax. For example [querying tables](../user-guide/fundamentals/query/pyquery.md) using Python. | -| [Index](../user-guide/fundamentals/indexing.md) | Indexing PyKX objects allows you to access and manipulate elements within these objects, similar to how you would with standard Python sequences.| +| [Create and convert](../user-guide/fundamentals/creating.md) | Create `pykx` objects from and to various Python objects, such as lists, dictionaries, and NumPy arrays. | +| [Use](../user-guide/fundamentals/evaluating.md) | Once created, interact with `pykx` objects using familiar Pythonic syntax. For example [querying tables](../user-guide/fundamentals/query/pyquery.md) using Python. | +| [Index](../user-guide/fundamentals/indexing.md) | Indexing `pykx` objects allows you to access and manipulate elements within these objects, similar to how you would with standard Python sequences.| -## What are PyKX attributes? +## What are `pykx` attributes? -Attributes are metadata that you attach to lists with special forms. They are also used on table columns to speed up retrieval for certain operations. PyKX can optimize based on the list structure implied by the attribute. +Attributes are metadata that you attach to lists with special forms. They are also used on table columns to speed up retrieval for certain operations. KDB-X Python can optimize based on the list structure implied by the attribute. -Attributes (except for ``#!python `g#``) are descriptive rather than prescriptive. This means that by applying an attribute, you are asserting that the list has a special form, which PyKX will verify. It does not instruct PyKX to create or remake the list into its special form; that is your responsibility. If a list operation respects the form specified by the attribute, the attribute remains intact (except for ``#!python `p#``). However, if an operation breaks the form, the attribute is removed from the result. +Attributes (except for ``#!python `g#``) are descriptive rather than prescriptive. This means that by applying an attribute, you are asserting that the list has a special form, which KDB-X Python will verify. It does not instruct KDB-X Python to create or remake the list into its special form; that is your responsibility. If a list operation respects the form specified by the attribute, the attribute remains intact (except for ``#!python `p#``). However, if an operation breaks the form, the attribute is removed from the result. -Learn how to [apply attributes](../user-guide/advanced/attributes.md) in PyKX. +Learn how to [apply attributes](../user-guide/advanced/attributes.md) in KDB-X Python. diff --git a/docs/license.md b/docs/license.md index 7421799..269f500 100644 --- a/docs/license.md +++ b/docs/license.md @@ -1,6 +1,6 @@ # Licenses -All files contained within the [PyKX repository](https://github.com/kxsystems/pykx) are not covered by a single license. The following outlines the differences. +All files contained within the [PyKX Python repository](https://github.com/kxsystems/pykx) are not covered by a single license. The following outlines the differences. 1. All files and folders contained within the source code directory 'src/pykx/q.so/' are licensed under the terms of the 'Software License for q.so' which are included below 2. All other files within this repository are licensed under the "Apache 2.0" license include below @@ -63,8 +63,8 @@ END OF TERMS AND CONDITIONS Software License for q.so -Version Number: 1.0 -Date Last Revised: April 2023 +Version Number: 1.1 +Date Last Revised: June 2026 Software License Agreement for use of q.so (“Agreement”) @@ -76,7 +76,7 @@ This Agreement is made between KX Systems, Inc. (“KX” or “we”) and the E  1. LICENSE GRANTS -1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. +1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX or KDB-X software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX or KDB-X software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. 1.2 Software Use Restrictions. End User may not: (a) modify any part of the Software or create derivative works thereof, (b) sell, lease, license or distribute the Software to any third party, (c) attempt to decompile, disassemble or reverse engineer the Software, (d) copy the Software, except for purposes of installing and executing it within the limitations set out at clause 1.1, (e) use or attempt to use the Software in any way that is unlawful or fraudulent or has any unlawful or fraudulent purpose or effect, (f) use or attempt to use the Software in any way that would breach the license granted herein. diff --git a/docs/mkdocstrings_filter/mkdocstrings_filter/__init__.py b/docs/mkdocstrings_filter/mkdocstrings_filter/__init__.py index 3553994..00b8f5d 100644 --- a/docs/mkdocstrings_filter/mkdocstrings_filter/__init__.py +++ b/docs/mkdocstrings_filter/mkdocstrings_filter/__init__.py @@ -2,7 +2,7 @@ Older versions of mkdocstrings do not support Cython projects. Newer versions do not yet have element filtering implemented. This plugin exists as a temporary measure to enable filtering for -the PyKX docs until mkdocstrings officially supports filtering. +the KDB-X Python docs until mkdocstrings officially supports filtering. """ from griffe.dataclasses import Alias diff --git a/docs/pykx-under-q/api.md b/docs/pykx-under-q/api.md index 96b4ad0..af7f611 100644 --- a/docs/pykx-under-q/api.md +++ b/docs/pykx-under-q/api.md @@ -1,11 +1,10 @@ # pykx.q Library Reference Card -!!! tip "Tip: For the best experience, read [How to use PyKX within q](../pykx-under-q/intro.md) and [Why upgrade from embedPy](../pykx-under-q/upgrade.md) first." +!!! tip "Tip: For the best experience, read [How to use KDB-X Python within q](../pykx-under-q/intro.md) and [Why upgrade from embedPy](../pykx-under-q/upgrade.md) first." This page documents the functions found in the `pykx.q` q library that are available. -This library can be installed by calling a helper function within `PyKX`, this function will move -all the required files and libraries into your `QHOME` directory. +This library can be installed by calling a helper function within `pykx`, this function will move all the required files and libraries into your `QHOME` directory. ```python import pykx as kx @@ -44,14 +43,14 @@ Gain access to the `.pykx` namespace within the `q` session q)\l pykx.q ``` -**PyKX q API functionality:** +**KDB-X Python q API functionality:**
**.pykx.** **General:** [console open an interactive Python REPL](#pykxconsole) -[version retrieve PyKX version](#pykxversion) +[version retrieve KDB-X Python version](#pykxversion) [print print a Python object directly to stdout](#pykxprint) [repr evaluate the Python function repr() on supplied Python object](#pykxrepr) [debugInfo print useful process debug information to q session](#pykxdebuginfo) @@ -59,9 +58,9 @@ q)\l pykx.q **Data Conversions:** [setdefault define the default conversion for KX objects to Python](#pykxsetdefault) -[toq convert an (un)wrapped `PyKX` foreign object into a q type](#pykxtoq) -[toq0 convert an (un)wrapped `PyKX` foreign object into a q type, use 2nd parameter to allow str objects to return as strings](#pykxtoq0) -[tok tag a q object to be indicate conversion to a Pythonic PyKX object when called in Python](#pykxtok) +[toq convert an (un)wrapped `pykx` foreign object into a q type](#pykxtoq) +[toq0 convert an (un)wrapped `pykx` foreign object into a q type, use 2nd parameter to allow str objects to return as strings](#pykxtoq0) +[tok tag a q object to be indicate conversion to a Pythonic `pykx` object when called in Python](#pykxtok) [topy tag a q object to be indicate conversion to a Python object when called in Python](#pykxtopy) [tonp tag a q object to be indicate conversion to a Numpy object when called in Python](#pykxtonp) [topd tag a q object to be indicate conversion to a Pandas object when called in Python](#pykxtopd) @@ -89,7 +88,7 @@ q)\l pykx.q [get retrieve a named item from the Python memory](#pykxget) [getattr retrieve an attribute of a Python object](#pykxgetattr) -**Foreign and PyKX object Handling:** +**Foreign and `pykx` object Handling:** [wrap convert a foreign object generated from Python execution to a callable q object](#pykxwrap) [unwrap convert a wrapped foreign object generated from this interface into a python foreign](#pykxunwrap) @@ -125,7 +124,7 @@ type | description **Example:** ```q -Enter PyKX console and evaluate Python code +Enter `pykx` console and evaluate Python code q).pykx.console[] >>> 1+1 2 @@ -134,7 +133,7 @@ q).pykx.console[] >>> quit() q) -// Enter PyKX console setting q objects using PyKX +// Enter `pykx` console setting q objects using pykx q).pykx.console[] >>> import pykx as kx >>> kx.q['table'] = kx.q('([]2?1f;2?0Ng;2?`3)' @@ -145,7 +144,7 @@ x x1 x2 0.439081 49f2404d-5aec-f7c8-abba-e2885a580fb6 mil 0.5759051 656b5e69-d445-417e-bfe7-1994ddb87915 igf -// Enter PyKX console setting Python objects using PyKX +// Enter `pykx` console setting Python objects using pykx q).pykx.console[] >>> a = list(range(5)) >>> quit() @@ -172,7 +171,7 @@ type | description ```q q).pykx.debugInfo[] -"**** PyKX information ****" +"**** pykx information ****" "pykx.args: ()" "pykx.qhome: /usr/local/anaconda3/envs/qenv/q" "pykx.qlic: /usr/local/anaconda3/envs/qenv/q" @@ -300,7 +299,7 @@ _List all q scripts in the extensions directory which can be loaded_ type | description -------|------------ -`list` | A list of strings denoting the available extensions in your version of PyKX +`list` | A list of strings denoting the available extensions in your version of KDB-X Python **Example:** @@ -313,7 +312,7 @@ q).pykx.listExtensions[] ## `.pykx.loadExtension` -_Loading of a PyKX extension_ +_Loading of a `pykx` extension_ ```q .pykx.loadExtension[ext] @@ -425,7 +424,7 @@ q).pykx.print .pykx.topd ([]5?1f;5?0b) ## `.pykx.toq` -_Convert an (un)wrapped `PyKX` foreign object into an analogous q type._ +_Convert an (un)wrapped `pykx` foreign object into an analogous q type._ ```q .pykx.toq[pythonObject] @@ -444,19 +443,19 @@ type | description `any` | A q object converted from Python ```q -// Convert a wrapped PyKX foreign object to q +// Convert a wrapped `pykx` foreign object to q q)show a:.pykx.eval["1+1"] {[f;x].pykx.util.pykx[f;x]}[foreign]enlist q).pykx.toq a 2 -// Convert an unwrapped PyKX foreign object to q +// Convert an unwrapped `pykx` foreign object to q q)show b:a`. foreign q).pykx.toq b 2 -// Convert a PyKX conversion object back to q +// Convert a `pykx` conversion object back to q q).pykx.toq .pykx.topd ([]5?1f;5?`a`b`c) x x1 @@ -677,7 +676,7 @@ q).pykx.repr til 5 ## `.pykx.safeReimport` -_Isolated execution of a q function which relies on importing PyKX_ +_Isolated execution of a q function which relies on importing `pykx`_ ```q .pykx.safeReimport[qFunction] @@ -692,7 +691,7 @@ For more information on the reimporter module which this functionality calls see name | type | description -------------|------------|------------- -`qFunction` | `function` | A function which is to be run following unsetting of PyKX environment variables and prior to their reset +`qFunction` | `function` | A function which is to be run following unsetting of `pykx` environment variables and prior to their reset **Returns:** @@ -702,7 +701,7 @@ type | description **Example:** -Initializing a Python process which imports PyKX +Initializing a Python process which imports `pykx` ```q q)\l pykx.q @@ -902,7 +901,7 @@ q).pykx.util.defaultConv ## `.pykx.todefault` -_Tag a q object to indicate it should use the PyKX default conversion when called in Python_ +_Tag a q object to indicate it should use the `pykx` default conversion when called in Python_ ```q .pykx.todefault[qObject] @@ -929,7 +928,7 @@ type | description q).pykx.todefault til 10 enlist[`..numpy;;][0 1 2 3 4 5 6 7 8 9] -// Pass a q list to Python treating the Python object as PyKX default +// Pass a q list to Python treating the Python object as `pykx` default q).pykx.typepy .pykx.todefault (til 10;til 10) "" @@ -941,7 +940,7 @@ q).pykx.typepy .pykx.todefault ([]til 10;til 10) ## `.pykx.tok` -_Tag a q object to be indicate conversion to a Pythonic PyKX object when called in Python_ +_Tag a q object to be indicate conversion to a Pythonic `pykx` object when called in Python_ ```q .pykx.tok[qObject] @@ -951,16 +950,16 @@ _Tag a q object to be indicate conversion to a Pythonic PyKX object when called name | type | description | ----------|---------|-------------| -`qObject` | `any` | A q object which is to be defined as a PyKX object in Python. | +`qObject` | `any` | A q object which is to be defined as a `pykx` object in Python. | **Return:** type | description -------------|------------ -`projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a PyKX type object. | +`projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a `pykx` type object. | ```q -// Denote that a q object once passed to Python should be managed as a PyKX object +// Denote that a q object once passed to Python should be managed as a `pykx` object q).pykx.tok til 10 enlist[`..k;;][0 1 2 3 4 5 6 7 8 9] @@ -968,7 +967,7 @@ enlist[`..k;;][0 1 2 3 4 5 6 7 8 9] q).pykx.typepy til 10 "" -// Pass a q object to Python treating the Python object as a PyKX object +// Pass a q object to Python treating the Python object as a `pykx` object q).pykx.typepy .pykx.tok til 10 "" ``` @@ -1158,7 +1157,7 @@ q).pykx.typepy .pykx.topy til 10 ## `.pykx.toq0` -_Convert an (un)wrapped `PyKX` foreign object into an analogous q type._ +_Convert an (un)wrapped `pykx` foreign object into an analogous q type._ ```q .pykx.toq0[pythonObject;strAsChar] @@ -1178,13 +1177,13 @@ type | description `any` | A q object converted from Python ```q -// Convert a wrapped PyKX foreign object to q +// Convert a wrapped `pykx` foreign object to q q)show a:.pykx.eval["1+1"] {[f;x].pykx.util.pykx[f;x]}[foreign]enlist q).pykx.toq0 a 2 -// Convert an unwrapped PyKX foreign object to q +// Convert an unwrapped `pykx` foreign object to q q)show b:a`. foreign q).pykx.toq0 b @@ -1306,7 +1305,7 @@ foreign ## `.pykx.version` -_Retrieve the version of PyKX presently being used by a q process_ +_Retrieve the version of KDB-X Python presently being used by a q process_ ```q .pykx.version[] @@ -1316,7 +1315,7 @@ _Retrieve the version of PyKX presently being used by a q process_ type | description ---------|------------ -`string` | The version number of PyKX installed within the users q session +`string` | The version number of KDB-X Python installed within the users q session ```q q).pykx.version[] diff --git a/docs/pykx-under-q/intro.md b/docs/pykx-under-q/intro.md index 6c36be6..8994bea 100644 --- a/docs/pykx-under-q/intro.md +++ b/docs/pykx-under-q/intro.md @@ -1,32 +1,32 @@ --- -title: PyKX within q -description: How to use PyKX in a q session +title: KDB-X Python within q +description: How to use KDB-X Python in a q session date: June 2024 author: KX Systems, Inc., -tags: PyKX, q, setup, +tags: KDB-X Python, q, setup, --- -# How to use PyKX within q +# How to use KDB-X Python within q -_This page provides details on how to run PyKX within a q session, including how to evaluate and execute Python code, how to interact with objects, and how to call a function._ +_This page provides details on how to run KDB-X Python within a q session, including how to evaluate and execute Python code, how to interact with objects, and how to call a function._ !!! tip "Tip: For the best experience, we recommend reading [Why upgrade from embedPy](../pykx-under-q/upgrade.md) first." ## Introduction -PyKX is a Python-first interface to the programming language q and its underlying database kdb+. To overcome a few [limitations](../help/issues.md), PyKX allows you to run Python within q, similarly to [embedPy](https://github.com/kxsystems/embedpy). The ability to execute and manipulate Python objects within a q session helps two types of users in the following ways: +KDB-X Python is a Python-first interface to the programming language q and its underlying database KDB-X. To overcome a few [limitations](../help/issues.md), KDB-X Python allows you to run Python within q, similarly to [embedPy](https://github.com/kxsystems/embedpy). The ability to execute and manipulate Python objects within a q session helps two types of users in the following ways: - - kdb+/q users can build applications which embed machine learning/data science libraries in production q infrastructures. + - KDB-X/q users can build applications which embed machine learning/data science libraries in production q infrastructures. - Users of Python plotting libraries can visualize and explore the outcomes of their analyses. ## Getting started ### Prerequisites -Before you run PyKX within q, make sure you: +Before you run KDB-X Python within q, make sure you: 1. Have access to a running `#!python q` environment. [Follow [the q installation guide](https://code.kx.com/q/learn/install/).] -2. Have [installed](../getting-started/installing.md) the licensed version of PyKX. +2. Have [installed](../getting-started/installing.md) the licensed version of KDB-X Python. ### Install @@ -92,7 +92,7 @@ Use this library to complete a wide variety of tasks, from the simple execution #### Evaluate Python code -To evaluate Python code with PyKX, pass a string of Python code to a variety of PyKX functions as shown below. +To evaluate Python code with KDB-X Python, pass a string of Python code to a variety of KDB-X Python functions as shown below. For example, if you want to evaluate and return the result to `#!python q`, use the function `#!python .pykx.qeval`: @@ -134,7 +134,7 @@ q).pykx.pyexec"print(array)" [0 1 2 3] ``` -b) Use the PyKX console functionality +b) Use the `pykx` console functionality This is useful when interacting within a q session and you need to prototype a functionality in Python: @@ -188,7 +188,7 @@ q)f[1;2] 3 ``` -### Interact with PyKX objects +### Interact with `pykx` objects #### Foreign objects @@ -198,23 +198,23 @@ You can store foreign objects in variables just like any other q datatype, or as ??? "Serialization and IPC" - Kdb+ cannot serialize foreign objects, nor send them over IPC. Foreign objects live in the embedded Python memory space. To pass them over IPC, first you have to convert them to q. + KDB-X cannot serialize foreign objects, nor send them over IPC. Foreign objects live in the embedded Python memory space. To pass them over IPC, first you have to convert them to q. -#### Create PyKX objects +#### Create `pykx` objects -q doesn't allow you to operate directly with foreign objects. Instead, Python objects are represented as PyKX objects, which wrap the underlying foreign objects. This helps to get and set attributes, index, call or convert the underlying foreign object to a q object. +q doesn't allow you to operate directly with foreign objects. Instead, Python objects are represented as `pykx` objects, which wrap the underlying foreign objects. This helps to get and set attributes, index, call or convert the underlying foreign object to a q object. -Use `#!python .pykx.wrap` to create a PyKX object from a foreign object. +Use `#!python .pykx.wrap` to create a `pykx` object from a foreign object. ```q q)x foreign q)p:.pykx.wrap x -q)p /how a PyKX object looks +q)p /how a `pykx` object looks {[f;x].pykx.util.pykx[f;x]}[foreign]enlist ``` -To retrieve PyKX objects directly from Python, choose between the following functions: +To retrieve `pykx` objects directly from Python, choose between the following functions: **Function** | **Argument** | **Example** ---------------|--------------------------------------------------|----------------------- @@ -229,7 +229,7 @@ To retrieve PyKX objects directly from Python, choose between the following func #### Convert data -For `#!python obj`, a PyKX object representing Python data, to obtain the underlying data (as foreign object or q) use: +For `#!python obj`, a `pykx` object representing Python data, to obtain the underlying data (as foreign object or q) use: ```q obj`. / get data as foreign @@ -258,14 +258,14 @@ Python `#!python None` maps to the q identity function `#!python ::` when conver #### Get attributes and properties -Given `#!python obj`, a PyKX object representing a Python object, you can get an attribute or property by using: +Given `#!python obj`, a `pykx` object representing a Python object, you can get an attribute or property by using: ```q obj`:attr / equivalent to obj.attr in Python obj`:attr1.attr2 / equivalent to obj.attr1.attr2 in Python ``` -These expressions return PyKX objects, allowing you to chain operations together: +These expressions return `pykx` objects, allowing you to chain operations together: ```q obj[`:attr1]`:attr2 / equivalent to obj.attr1.attr2 in Python @@ -292,7 +292,7 @@ q)obj[`:y]` #### Set attributes and properties -Given `#!python obj`, a PyKX object representing a Python object, you can set an attribute or property by using: +Given `#!python obj`, a `pykx` object representing a Python object, you can set an attribute or property by using: ```q obj[:;`:attr;val] / equivalent to obj.attr=val in Python @@ -315,7 +315,7 @@ q)obj[`:y]` #### How to index -Given `#!python lst`, a PyKX object representing an indexable container object in Python, you can access the element at index `#!python i` by using: +Given `#!python lst`, a `pykx` object representing an indexable container object in Python, you can access the element at index `#!python i` by using: ```q lst[@;i] / equivalent to lst[i] in Python @@ -327,7 +327,7 @@ Set the element at index `#!python i` (to object `#!pythonx`) with this command: lst[=;i;x] / equivalent to lst[i]=x in Python ``` -These expressions return PyKX objects, for instance: +These expressions return `pykx` objects, for instance: ```q q)lst:.pykx.eval"[True,2,3.0,'four']" @@ -351,39 +351,39 @@ q)lst` #### Get methods -Given `#!python obj`, a PyKX object representing a Python object, you can access a method by using: +Given `#!python obj`, a `pykx` object representing a Python object, you can access a method by using: ```q obj`:method / equivalent to obj.method in Python ``` -When calling PyKX objects representing Python methods, the return of evaluation is a PyKX object. For example: +When calling `pykx` objects representing Python methods, the return of evaluation is a `pykx` object. For example: ```q q)np:.pykx.import`numpy q)np`:arange {[f;x].pykx.util.pykx[f;x]}[foreign]enlist -q)arange:np`:arange / callable returning PyKX object +q)arange:np`:arange / callable returning `pykx` object q)arange 12 {[f;x].pykx.util.pykx[f;x]}[foreign]enlist q)arange[12]` 0 1 2 3 4 5 6 7 8 9 10 11 ``` -#### PyKX function API +#### KDB-X Python function API Use the function API to achieve the following: -- Call PyKX objects (to get PyKX objects). -- Declare PyKX objects callable (to get q or `#!python foreign` data). +- Call `pykx` objects (to get `pykx` objects). +- Declare `pykx` objects callable (to get q or `#!python foreign` data). -The default return is a PyKX object. For q or foreign return type, you need to specify it. +The default return is a `pykx` object. For q or foreign return type, you need to specify it. -Given `#!python func`, a `#!python PyKX` object representing a callable Python function or method, you can carry out the following operations: +Given `#!python func`, a `#!python KDB-X Python` object representing a callable Python function or method, you can carry out the following operations: ```q -func / func is callable by default (returning PyKX) -func arg / call func(arg) (returning PyKX) +func / func is callable by default (returning KDB-X Python) +func arg / call func(arg) (returning KDB-X Python) func[<] / declare func callable (returning q) func[<]arg / call func(arg) (returning q) func[<;arg] / equivalent @@ -394,10 +394,10 @@ func[>;arg] / equivalent !!! info "How to chain operations?" - To chain together sequences of operations, return another PyKX object from a function or method call. Alternatively, call `.pykx.import`, `.pykx.get` and `.pykx.eval`. + To chain together sequences of operations, return another `pykx` object from a function or method call. Alternatively, call `.pykx.import`, `.pykx.get` and `.pykx.eval`. -#### PyKX examples +#### KDB-X Python examples === "Example #1" @@ -494,17 +494,17 @@ func[>;arg] / equivalent #### Function argument types -PyKX supports data type conversions between q and Python for Python native objects, NumPy objects, Pandas objects, PyArrow objects, and PyKX objects. +KDB-X Python supports data type conversions between q and Python for Python native objects, NumPy objects, Pandas objects, PyArrow objects, and `pykx` objects. By default, when passing a q object to a callable function, it's converted to the most "natural" analogous type, as detailed below: -- PyKX/q generic list objects become Python lists. -- PyKX/q table/keyed table objects become Pandas equivalent DataFrames. -- All other PyKX/q objects become their analogous NumPy equivalent types. +- `pykx`/q generic list objects become Python lists. +- `pykx`/q table/keyed table objects become Pandas equivalent DataFrames. +- All other `pykx`/q objects become their analogous NumPy equivalent types. !!! Warning - Prior to PyKX 2.1.0, all conversions from q objects to Python would convert to their NumPy equivalent. To achieve this now, set the environment variable `PYKX_DEFAULT_CONVERSION="np"` + Prior to KDB-X Python 2.1.0, all conversions from q objects to Python would convert to their NumPy equivalent. To achieve this now, set the environment variable `PYKX_DEFAULT_CONVERSION="np"` For function/method calls, control the default behavior of the conversions by setting `#!python .pykx.util.defaultConv`: @@ -514,7 +514,7 @@ q).pykx.util.defaultConv ``` You can apply one of the following values: -|**Python type**|Default|Python|NumPy|Pandas|PyArrow|PyKX| +|**Python type**|Default|Python|NumPy|Pandas|PyArrow|KDB-X Python| |---------------|-------|------|-----|------|-------|----| |**Value**: |"default"|"py"|"np"|"pd"|"pa"|"k"| @@ -566,7 +566,7 @@ In the example below, we start with NumPy and update the default types across al q).pykx.typepy ([]100?1f;100?1f); "" ``` -=== "PyKX" +=== "KDB-X Python" ```q q).pykx.util.defaultConv:"k" @@ -591,7 +591,7 @@ q)typeFunc[til 10].pykx.topd til 10; // Pass in the second argumen q)typeFunc[.pykx.topa([]100?1f);til 10]; // Pass in first argument as PyArrow Table -q)typeFunc[.pykx.tok til 10;.pykx.tok ([]100?1f)]; // Pass in two PyKX objects +q)typeFunc[.pykx.tok til 10;.pykx.tok ([]100?1f)]; // Pass in two `pykx` objects ``` @@ -619,9 +619,9 @@ Python allows you to call functions with: - A mixture of positional and keyword arguments - Implicit (default) arguments -This is available in the PyKX function-call interface, as detailed below: +This is available in the KDB-X Python function-call interface, as detailed below: -- Callable PyKX objects are variadic (they accept a variable number of arguments). +- Callable `pykx` objects are variadic (they accept a variable number of arguments). - Default arguments are applied where no explicit arguments are given. - Individual keyword arguments are specified using the (infix) `#!python pykw` operator. - A list of positional arguments can be passed using `#!python pyarglist` (like Python \*args). @@ -730,9 +730,9 @@ func() #call with no arguments func(None) #call with argument None ``` -!!! warning "PyKX function called with `::` calls Python with no arguments" +!!! warning "KDB-X Python function called with `::` calls Python with no arguments" - Although `::` in q corresponds to `None` in Python, if a PyKX function is called with `::` as its only argument, the corresponding Python function will be called with _no_ arguments. + Although `::` in q corresponds to `None` in Python, if a KDB-X Python function is called with `::` as its only argument, the corresponding Python function will be called with _no_ arguments. To call a Python function with `#!python None` as its sole argument, retrieve `#!python None` as a foreign object in q and pass that as the argument: @@ -763,7 +763,7 @@ None #### Print or return -`#!python .pykx.repr` returns the string representation of a Python object, either PyKX or foreign. You can print this representation to `#!python stdout` by using `#!python .pykx.print`. Here's how to use this function with a q object: +`#!python .pykx.repr` returns the string representation of a Python object, either KDB-X Python or foreign. You can print this representation to `#!python stdout` by using `#!python .pykx.print`. Here's how to use this function with a q object: ```q q)x:.pykx.eval"{'a':1,'b':2}" diff --git a/docs/pykx-under-q/upgrade.md b/docs/pykx-under-q/upgrade.md index edc3ac4..f12b55b 100644 --- a/docs/pykx-under-q/upgrade.md +++ b/docs/pykx-under-q/upgrade.md @@ -1,22 +1,22 @@ --- title: Upgrade from embedPy -description: How to upgrade from embedPy to PyKX within q +description: How to upgrade from embedPy to KDB-X Python within q date: June 2024 author: KX Systems, Inc., -tags: embedPy, PyKX, q, +tags: embedPy, KDB-X Python, q, --- # Why upgrade from embedPy -_This page outlines differences and function mappings when upgrading from embedPy to PyKX in a q session._ +_This page outlines differences and function mappings when upgrading from embedPy to KDB-X Python in a q session._ -Just like [PyKX](../getting-started/what_is_pykx.md), [embedPy](https://github.com/kxsystems/embedpy) is a tool that allows to execute Python code and call Python functions. +Just like [KDB-X Python](../getting-started/what_is_pykx.md), [embedPy](https://github.com/kxsystems/embedpy) is a tool that allows to execute Python code and call Python functions. ## Functional differences ### q symbol and string support -EmbedPy doesn't allow users to discern between q `#!python string` and `#!python symbol` types when converting to Python. In both cases, these are converted to `#!python str` objects in Python. As a result, embedPy doesn't support round-trip conversions for symbols, but PyKX does: +EmbedPy doesn't allow users to discern between q `#!python string` and `#!python symbol` types when converting to Python. In both cases, these are converted to `#!python str` objects in Python. As a result, embedPy doesn't support round-trip conversions for symbols, but KDB-X Python does: === "embedPy" @@ -29,7 +29,7 @@ EmbedPy doesn't allow users to discern between q `#!python string` and `#!python 0b ``` -=== "PyKX" +=== "KDB-X Python" ```q q).pykx.set[`a;"test"] @@ -42,9 +42,9 @@ EmbedPy doesn't allow users to discern between q `#!python string` and `#!python ### Functionality mapping -The following table describes function mapping from PyKX to embedPy: +The following table describes function mapping from KDB-X Python to embedPy: -| Description | PyKX | embedPy | +| Description | KDB-X Python | embedPy | |-----------------------------------------------------------------------|---------------------------------|-----------------| | Load library | `\l pykx.q` | `\l p.q` | | Import Python Libraries as wrapped Python objects | `.pykx.import` | `.p.import` | @@ -54,7 +54,7 @@ The following table describes function mapping from PyKX to embedPy: | Execute Python code returning as intermediary q/Python object | `.pykx.eval` | `.p.eval` | | Execute Python code returning a q object | `.pykx.qeval` | `.p.qeval` | | Execute Python code returning a Python foreign object | `.pykx.pyeval` | `.p.eval` | -| Retrieve a printable representation of a supplied PyKX/q object | `.pykx.repr` | `.p.repr` | +| Retrieve a printable representation of a supplied pykx/q object | `.pykx.repr` | `.p.repr` | | Set an attribute on a supplied Python object | `.pykx.setattr` | `.p.setattr` | | Retrieve an attribute from a supplied Python object | `.pykx.getattr` | `.p.getattr` | | Convert a Python foreign object to a wrapped object for conversion | `.pykx.wrap` | `.p.wrap` | @@ -69,26 +69,26 @@ The following table describes function mapping from PyKX to embedPy: | Create a Python closure using a q function | Unsupported | `.p.closure` | | Create a Python generator using a q function | Unsupported | `.p.generator` | -## PyKX under q benefits over embedPy +## KDB-X Python under q benefits over embedPy -When generating workloads that integrate Python and q code, PyKX under q provides a few key functional benefits over embedPy alone: +When generating workloads that integrate Python and q code, KDB-X Python under q provides a few key functional benefits over embedPy alone: 1. [Flexibility in supported data formats and conversions](#1-flexibility-in-supported-data-formats-and-conversions) 2. [Python code interoperability](#2-python-interoperability) -3. [Access to PyKX as a Python module](#3-access-to-pykx-as-a-python-module) +3. [Access to KDB-X Python as a Python module](#3-access-to-pykx-as-a-python-module) ### 1. Flexibility in supported data formats and conversions When using EmbedPy to convert data between q and Python, there’s a fundamental limitation related to supported data formats. Specifically, when passed to Python functions, q objects use the analogous Python/NumPy representation. This means that if an embedPy user requires data in a Pandas/PyArrow format, they need to convert it manually. -As PyKX supports Python, NumPy, Pandas, and PyArrow data formats, it improves the workflow coverage and flexibility. For instance, PyKX by default converts q tables to Pandas DataFrames when passed to a Python function as follows: +As KDB-X Python supports Python, NumPy, Pandas, and PyArrow data formats, it improves the workflow coverage and flexibility. For instance, KDB-X Python by default converts q tables to Pandas DataFrames when passed to a Python function as follows: ```q q).pykx.eval["lambda x:type(x)"] ([]10?1f;10?1f) ``` -Additionally, PyKX provides helper functions, allowing you to choose the target data formats used when passing to multivariable functions. For example: +Additionally, KDB-X Python provides helper functions, allowing you to choose the target data formats used when passing to multivariable functions. For example: ```q q).pykx.eval["lambda x, y:print(type(x), type(y))"][.pykx.tonp ([]10?1f);.pykx.topd til 10]; @@ -112,7 +112,7 @@ Alternatively, you could create a `#!python .py`/`#!python .p` file and access y Both solutions are not intuitive to users versed both in Python and q. -That's why PyKX provides a Python `#!python .pykx.console` function that you can run within a q session to generate your functions/variables. The following example uses PyKX 2.3.0: +That's why KDB-X Python provides a Python `#!python .pykx.console` function that you can run within a q session to generate your functions/variables. The following example uses KDB-X Python 2.3.0: ```q q).pykx.console[] @@ -127,11 +127,11 @@ q)pyfunc[2] This function allows you to iterate your analytics development faster than when operating with embedPy. -### 3. Access to PyKX as a Python module +### 3. Access to KDB-X Python as a Python module -Access to PyKX in its Python-first mode adds more flexibility to users who develop analytics to use within q. +Access to KDB-X Python in its Python-first mode adds more flexibility to users who develop analytics to use within q. -With embedPy, when you pass q/kdb+ data to Python to complete a "Python-first" analysis, you're restricted to your Python libraries and can't get performance benefits from having access to q/kdb+. +With embedPy, when you pass q/KDB-X data to Python to complete a "Python-first" analysis, you're restricted to your Python libraries and can't get performance benefits from having access to q/KDB-X. Take for example a case where a user wishes to run a Python function which queries a table available in their q process using SQL and calculates the mean value for all numeric columns. @@ -151,5 +151,5 @@ x2| 0.486176 ## Next steps -- Learn [how to use PyKX within q](../pykx-under-q/intro.md). +- Learn [how to use KDB-X Python within q](../pykx-under-q/intro.md). - Use the [pykx.q Library Reference Card](../pykx-under-q/api.md). \ No newline at end of file diff --git a/docs/release-notes/changelog.md b/docs/release-notes/changelog.md index 10f951b..6911bff 100644 --- a/docs/release-notes/changelog.md +++ b/docs/release-notes/changelog.md @@ -1,16 +1,495 @@ -# PyKX Changelog +# KDB-X Python Changelog + +This changelog tracks all KDB-X Python releases and their associated additions, fixes, and improvements. !!! Note - The changelog presented here outlines changes to PyKX when operating within a Python environment specifically, if you require changelogs associated with PyKX operating under a q environment see [here](./underq-changelog.md). + For changes to KDB-X Python within a q environment, refer to the [KDB-X Python under q changelog](./underq-changelog.md). + +## KDB-X + +KDB-X is the evolution of kdb+. For full details and to install go [here](https://developer.kx.com/products/kdb-x/install). + +## KDB-X Python + +KDB-X Python `pykx>=4.0` is the evolution of PyKX `pykx<4.0`. For more details and a migration guide [here](../upgrades/3040.md). + +## KDB-X Python 4.0.0 + +**Release Date** + +2026-06-24 + +**Additions** + +- Added [`kx.use`](../api/module.md#pykx.module.use) for importing `q` modules within KDB-X Python. Refer to the [module documentation](../api/module.md) for more. + ```Python + >>> import pykx as kx + >>> pq = kx.use('kx.pq') + >>> pq + pykx.QModule(pykx.q(' + pq| pykx.Lambda + op| pykx.Lambda + rd| pykx.Lambda + ')) + >>> pq.t = kx.use('kx.pq.t') + >>> pq + pykx.QModule(pykx.q(' + pq| pykx.Lambda + op| pykx.Lambda + rd| pykx.Lambda + t | pykx.QModule(pykx.q(' + mkT| pykx.Lambda + mkP| pykx.Lambda + tt | pykx.Lambda + mt | pykx.Lambda + fv | pykx.Lambda + ')) + ')) + ``` + +- Added Pandas 3.0 support. + + !!! Warning + `pd(as_arrow=True)` conversions containing null values will raise a `ValueError` due to a [known bug in Pandas 3.0](https://github.com/pandas-dev/pandas/issues/63879). This is expected to be resolved in Pandas 3.1. + + ```python + >>> import pykx as kx + >>> kx.q('([] s:1 0Nh)').pd() + s + 0 1 + 1 -- + >>> kx.q('([] s:1 0Nh)').pd(as_arrow=True) + Traceback (most recent call last): + ... + ValueError: Cannot pass a numpy masked array and specify a mask at the same time: Error while type casting for column 's' + ``` + +**Fixes and Improvements** + +- Moved to [KDB-X 5.0.20260501](https://code.kx.com/kdb-x/releases/release-notes-latest.html). +- Fixed `.pykx.console` under q when [kxline](https://code.kx.com/kdb-x/how_to/general_guidance/embedded-line-editor-kxline.html) is in use. +- Includes all changes from PyKX [PyKX 3.1.8](#pykx-318), [PyKX 3.1.9](#pykx-319), and [PyKX 3.2.0](#pykx-320). + +## PyKX 3.2.0 + +**Release Date** + +2026-06-24 + +**Additions** + + +- Added `by` keyword to `merge_asof`. + + ```python + >>> import pandas as pd + >>> left = pd.DataFrame({"a": [1, 5, 10], "b":["a", "a", "b"], "left_val": ["a", "b", "c"]}) + >>> right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "b":["a", "a", "b", "b", "a"], "right_val": [1, 2, 3, 6, 7]}) + >>> import pykx as kx + >>> qleft = kx.toq(left) + >>> qright = kx.toq(right) + >>> kx.merge_asof(qleft, qright, on='a') + pykx.Table(pykx.q(' + a b_x left_val b_y right_val + ----------------------------- + 1 a a a 1 + 5 a b b 3 + 10 b c a 7 + ')) + >>> kx.merge_asof(qleft, qright, on='a', by="b") + pykx.Table(pykx.q(' + a b left_val right_val + ----------------------- + 1 a a 1 + 5 a b 2 + 10 b c 6 + ')) + ``` + +- Added `__getitem__` to `pykx.Column` class. + + ```python + >>> t = kx.q('([] a:5 10 15 20)') + # Equivalent to qsql: select a 1 2 from t + >>> t.select(kx.Column('a')[[1,2]]) + pykx.Table(pykx.q(' + a + -- + 10 + 15 + ')) + >>> kx.q['myvar'] = [1, 2] + # Equivalent to qsql: select a myvar from t + >>> t.select(kx.Column('a')[kx.Variable('myvar')]) + pykx.Table(pykx.q(' + a + -- + 10 + 15 + ')) + >>> kx.q('mylookup:5 10 15 20!`five`ten`fifteen`twenty') + pykx.Identity(pykx.q('::')) + # Equivalent to qsql: update word:mylookup a from t + >>> t.update(kx.Column('word', data=kx.Variable('mylookup'))[kx.Column('a')]) + pykx.Table(pykx.q(' + a word + ---------- + 5 five + 10 ten + 15 fifteen + 20 twenty + ')) + ``` + +- Added `.enlist()` method to `pykx.Column` class to enlist data in a `pykx.Column` object. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 2 4)') + >>> t.exec(kx.Column('n').enlist(iterator='each')) + pykx.List(pykx.q(' + 2 + 4 + ')) + ``` + +- Added `.all()` method to `pykx.Column` class to return `1b` when all items are nonzero. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 5#1b)') + >>> t.exec(kx.Column('n').all()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + + +- Added `.any()` method to `pykx.Column` class which returns `1b` when at least one item is nonzero. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 0010b)') + >>> t.exec(kx.Column('n').any()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + +- Added `._in()` method to `pykx.Column` class to check whether an item is in the column. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:1 2 3 4 5)') + >>> t.select(kx.Column('n')._in([0,2,4], iterator=r'\:')) + pykx.Table(pykx.q(' + n + - + 0 + 1 + 0 + 1 + 0 + ')) + ``` + +- Added `._except()` method to `pykx.Column` class to return column values that are not present in the supplied variable. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: til 5)') + >>> t.exec(kx.Column('n')._except(3)) + pykx.LongVector(pykx.q('0 1 2 4')) + ``` + +- Added `.hsym()` method to `pykx.Column` class which returns a column of symbol objects prefixed with a colon. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: `a`b`c)') + >>> t.exec(kx.Column('n').hsym()) + pykx.SymbolVector(pykx.q('`:a`:b`:c')) + ``` + +- Added `.raze()` method to `pykx.Column` class to collapse column values. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').raze(iterator='over')) + pykx.LongVector(pykx.q('1 2 3 4 5 6 7 8')) + ``` + +- Added `.type()` method to `pykx.Column` class to return type of values in a column. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').type(iterator='each')) + pykx.ShortVector(pykx.q('7 0 -7 -7h')) + ``` + +- Added `.key()` method to `pykx.Column` class to return keys from a column of dictionaries. + ```python + >>> import pykx as kx + >>> d1 = kx.q('`a`b`c ! 1 2 3') + >>> d2 = kx.q('`d`e`f ! 4 5 6') + >>> t = kx.Table(data={'n':[d1, d2]}) + >>> t.exec(kx.Column('n').key(iterator='each')) + pykx.Table(pykx.q(' + n + ----- + a b c + d e f + ')) + ``` + +- Added `.get()` method to `pykx.Column` class to read from a column of named global variables or file names. + ```python + >>> import pykx as kx + >>> kx.q('`:tb1 set ([] n: `a`b`c)') + pykx.SymbolAtom(pykx.q('`:tb1')) + >>> kx.q('`:tb2 set ([] n: 1 2 3)') + pykx.SymbolAtom(pykx.q('`:tb2')) + >>> t = kx.q('([] n: `:tb1`:tb2)') + >>> t.exec(kx.Column('n').get(iterator='each')) + pykx.List(pykx.q(' + +(,`n)!,`a`b`c + +(,`n)!,1 2 3 + ')) + ``` + +- Added `.read0()` method to `pykx.Column` class to read text from a file or process handle. + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read0(iterator='each')) + pykx.List(pykx.q(' + "hello" + "goodbye" + ')) + ``` + +- Added `.read1()` method to `pykx.Column` class to read bytes from a file or named pipe. + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read1(iterator='each')) + pykx.List(pykx.q(' + 0x68656c6c6f0a + 0x676f6f646279650a + ')) + ``` + +- Added `.next()` method to `pykx.Column` class to return the next item in a list. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:3 6 9 12)') + >>> t.exec(kx.Column('n').call('next')) + pykx.LongVector(pykx.q('6 9 12 0N')) + ``` + +- Added `.prior()` method to `pykx.Column` class to apply a function across current and previous value. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:1 2 3 3 2 1)') + >>> t.update(kx.Column('n').prior(kx.q('<')).name('lessThanPrevious')) + pykx.Table(pykx.q(' + n lessThanPrevious + ------------------ + 1 0 + 2 0 + 3 0 + 3 0 + 2 1 + 1 1 + ')) + ``` + +- Added `.where()` method to `pykx.Column` class to return indexes of True values. + + ```python + >>> import pykx as kx + >>> tab = kx.q('([] a:101010b)') + >>> tab.exec(kx.Column('a').where()) + pykx.LongVector(pykx.q('0 2 4')) + ``` + +- Added `.fills()` method to `pykx.Table`, `pykx.KeyedTable`, and `pykx.Vector` classes to replace nulls with preceding non-nulls. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:0N 2 0N 3 0N;b:``h`j``;g:0N 1 0N 0N 2)') + >>> t + pykx.Table(pykx.q(' + n b g + ----- + + 2 h 1 + j + 3 + 2 + ')) + >>> t.fills() + pykx.Table(pykx.q(' + n b g + ----- + + 2 h 1 + 2 j 1 + 3 j 1 + 3 j 2 + ')) + ``` + +- Added `.ungroup()` method to `pykx.Table` and `pykx.KeyedTable` classes to ungroup nested values. + + ```python + >>> import pykx as kx + >>> t = q('([] n:1 2 3;nn:(1 1;2 2 2;3 3 3))') + >>> t + pykx.Table(pykx.q(' + n nn + ------- + 1 1 1 + 2 2 2 2 + 3 3 3 3 + ')) + >>> t.ungroup() + pykx.Table(pykx.q(' + n nn + ---- + 1 1 + 1 1 + 2 2 + 2 2 + 2 2 + 3 3 + 3 3 + 3 3 + ')) + ``` + +**Fixes and Improvements** + +- Updated 4.1 to 2026.05.01 for all platforms. +- Fixed setting the contents of `Column` objects using `data=` or `.data()` for inputs: `SymbolAtom`, `SymbolVector`, `str`, and lists of `str`. + + === "Behavior prior to change" + + ```python + >>> import pykx as kx + >>> t = kx.q('([] a:1 2 3)') + >>> t.update(kx.Column('new', data='this')) + Traceback (most recent call last): + ... + pykx.exceptions.QError: this + ``` + + === "Behavior after change" + + ```python + >>> import pykx as kx + >>> t = kx.q('([] a:1 2 3)') + >>> t.update(kx.Column('new', data='this')) + pykx.Table(pykx.q(' + a new + ------ + 1 this + 2 this + 3 this + ')) + ``` + +- Fixed dictionary item assignment that could result in errors or incorrect types: + + === "Behavior prior to change" + + ```python + # Single row table saved incorrectly as dictionary + >>> a = kx.q('`a`b!(`;2)') + >>> c = kx.q('([] a:enlist 2;b:enlist 6)') + >>> a['c'] = c + >>> a['c'] + pykx.Dictionary(pykx.q(' + a| 2 + b| 6 + ')) + + # Multiple row table fails + >>> a = kx.q('`a`b!(`;2)') + >>> d = kx.q('([] a:1 2;b:3 6)') + >>> a['d'] = d + Traceback (most recent call last): + File "", line 1, in + a['d'] = d + ~^^^^^ + ... + pykx.exceptions.QError: length + ``` + + === "Behavior after change" + + ```python + >>> a = kx.q('`a`b!(`;2)') + >>> c = kx.q('([] a:enlist 2;b:enlist 6)') + >>> a['c'] = c + >>> a['c'] + pykx.Table(pykx.q(' + a b + --- + 2 6 + ')) + + >>> a = kx.q('`a`b!(`;2)') + >>> d = kx.q('([] a:1 2;b:3 6)') + >>> a['d'] = d + >>> a['d'] + pykx.Table(pykx.q(' + a b + --- + 1 3 + 2 6 + ')) + ``` + +- Removed `.pykx.i.errorList` from `astype` calls to fix `'noupdate` errors when called from a thread. +- Updated `help` on `kx.q` methods to use local documentation instead of making a web request. +- Fixed a memory leak when using `.pykx.getattr` on foreign objects in `q`. +- Improved `.cast('symbol')` on `pykx.Column` to correctly cast `Char` columns to `Symbol`. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] a:("this";"that"))') + >>> t + pykx.Table(pykx.q(' + a + ------ + "this" + "that" + ')) + >>> t.select(kx.Column('a').cast('symbol')) + pykx.Table(pykx.q(' + a + ---- + this + that + ')) + ``` + +**Version Support Changes** + +- Python 3.8 is no longer supported as it is [end of life](https://devguide.python.org/versions/#unsupported-versions). ## PyKX 3.1.9 -#### Release Date +**Release Date** 2026-04-02 -### Additions +**Additions** - Added `.np()` method to `pykx.serialize` function, this returns the serialized data as a NumPy array. - Added support for calling `pykx.deserialize` on NumPy arrays of serialized `K` objects. @@ -46,32 +525,32 @@ ')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Updated 4.1 to 2026.04.01 for all platforms. ## PyKX 3.1.8 -#### Release Date +**Release Date** 2026-02-18 -### Additions +**Additions** - Added `connection_timeout` keyword argument to `AsyncQConnection` objects allowing the specification of a timeout for the initial connection to a `q` server. - Added the usage of `.q.Q.ens` for `sym_enum` argument in `DB` objects. ## PyKX 3.1.7 -#### Release Date +**Release Date** 2026-02-10 -### Additions +**Additions** - Added support for Python 3.14. -### Fixes and Improvements +**Fixes and Improvements** - Updated 4.1 to 2026.01.23 for all platforms. - Added more strict type checks for naming `Column` objects, allowing only `str` and `SymbolAtom` values. @@ -121,17 +600,63 @@ ')) ``` -### Deprecations & Removals +**Deprecations & Removals** - Deprecated the `value` keyword when creating `kx.Column` objects, use `data` instead. +## KDB-X Python 4.0.0b5 + +**Release Date** + +2026-02-09 + +**Additions** + +- New `VirtualTable` class for use with [Parquet module](https://code.kx.com/kdb-x/modules/parquet/overview.html): + + ```q + >>> import pykx as kx + >>> from pathlib import Path + >>> pq = kx.q.use('kx.pq') + >>> t = pq['pq'](Path('types.parquet')) + >>> t + pykx.VirtualTable(pykx.q('`T!`f`m`t!(k){[f;t;c;b;a;v]g:$[s:-1h=@b;0;#b];(bf;b1):$[g;df[t;0,0b;b];(::;()..')) + >>> t.select() + pykx.Table(pykx.q(' + col0 col1 col2 + --------------- + 1 1 "asd" + 1 2 "bsd" + 0 3 "csd" + 1 4 "asd" + 0 5 "bsd" + 0 6 "csd" + 1 7 "asd" + 0 8 "bsd" + 0 9 "csd" + ')) + >>> t.select(kx.Column('col1').max(), by=kx.Column('col2')) + pykx.KeyedTable(pykx.q(' + col2 | col1 + -----| ---- + "asd"| 7 + "bsd"| 8 + "csd"| 9 + ')) + ``` + +**Fixes and Improvements** + +- Moved to [KDB-X 5.0.20260122](https://code.kx.com/kdb-x/releases/release-notes-latest.html). +- Includes bug fixes from [PyKX 3.1.6](#pykx-316) and [PyKX 3.1.7](#pykx-317). + ## PyKX 3.1.6 -#### Release Date +**Release Date** 2025-12-01 -### Fixes and Improvements +**Fixes and Improvements** - Updated 4.1 to 2025.11.25 for all platforms. - Fixed issue where setting `sort=True` when calling `merge` on two `kx.Tables` ignored the supplied `on` parameter. @@ -168,13 +693,24 @@ ')) ``` +## KDB-X Python 4.0.0b4 + +**Release Date** + +2025-11-17 + +**Fixes and Improvements** + +- Moved to [KDB-X 5.0.20251114](https://code.kx.com/kdb-x/releases/release-notes-latest.html). +- Includes bug fixes from [PyKX 3.1.5](#pykx-315). + ## PyKX 3.1.5 -#### Release Date +**Release Date** 2025-10-21 -### Fixes and Improvements +**Fixes and Improvements** - Availability for Intel Macs on PyPI has been restored. - Paths starting with '~' will now have it expanded to users home path in `PYKX_CONFIGURATION_LOCATION`, `Encrypt`, `DB`, and `system.load` functionality (using `os.path.expanduser`). @@ -253,28 +789,62 @@ pykx.wrappers.Dictionary ``` -### Deprecations & Removals +**Deprecations & Removals** - Creating more than one `DB` instance without specifying `overwrite=True` +## KDB-X Python 4.0.0b3 + +**Release Date** + +2025-09-30 (expired 2026-01-04) + +**Fixes and Improvements** + +- Moved to KDB-X 0.1.1. + +## KDB-X Python 4.0.0b2 + +**Release Date** + +2025-07-30 (expired 2025-09-30) + +**Fixes and Improvements** + +- Removes `invalid escape sequence` warnings upon import for some users. +- Turns on NumPy allocator by default. + - Replaces `PYKX_ALLOCATOR` with `PYKX_NO_ALLOCATOR` + - Replaces `--pykxalloc` with `--pykxnoalloc` +- Includes bug fixes from [PyKX 3.1.4](#pykx-314). + ## PyKX 3.1.4 -#### Release Date +**Release Date** 2025-07-17 -### Fixes and Improvements +**Fixes and Improvements** - Resolved error on import when `PYKX_THREADING` was set. - Fixed an issue where data could be corrupted in keyed columns of PyArrow backed Pandas dataframes. +## KDB-X Python 4.0.0b1 + +**Release Date** + +2025-07-02 (expired 2025-09-30) + +**Additions** + +- Initial release. Uses [KDB-X 0.1.0](https://code.kx.com/kdb-x/releases/release-notes-latest.html). + ## PyKX 3.1.3 -#### Release Date +**Release Date** 2025-06-12 -### Additions +**Additions** - Addition of the utility function `kx.util.delete_q_variable` which can be used to delete a named variable stored q memory within a namespace or in the global namespace @@ -391,7 +961,7 @@ - Added the current number of secondary threads being used by q to `kx.util.debug_environment()` output - Added option to pass `str` or `Path` to `pykx.install_into_QHOME` keyword `to_local_folder` -### Fixes and Improvements +**Fixes and Improvements** - Updated 4.0 to 2025.02.18 for all platforms. - Updated 4.1 to 2025.04.28 for all platforms. @@ -567,27 +1137,27 @@ - PYKX_MAX_ERROR_LENGTH = 1 ``` -### Deprecations & Removals +**Deprecations & Removals** - Deprecated `kx.q.system.console_size`, use `kx.q.system.display_size` instead. ## PyKX 3.1.2 -#### Release Date +**Release Date** 2025-03-18 -### Fixes and Improvements +**Fixes and Improvements** - Fixes an issue when using `PYKX_THREADING` that could cause a segfault in some scenarios. ## PyKX 3.1.1 -#### Release Date +**Release Date** 2025-02-14 -### Fixes and Improvements +**Fixes and Improvements** - Fixed issue whereby PyKX would prompt for user inputs if a license was not found in a non-interactive session, now correctly falls back to unlicensed mode. - Fixed issue whereby PyKX could list splayed tables loaded outside of initialization of the `kx.DB` module as being supported elements of the database @@ -622,11 +1192,11 @@ ## PyKX 3.1.0 -#### Release Date +**Release Date** 2025-02-11 -### Additions +**Additions** - Added support for `Python 3.13`. - Added support for `NumPy 2.0` @@ -753,7 +1323,7 @@ pykx.LongVector(pykx.q('0 1 2 3 4')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Application of the `#!python str` function on empty PyKX objects could return unexpected results @@ -936,7 +1506,7 @@ - Error message when checking a license referenced a function `pykx.util.install_license` which is deprecated, this has now been updated to reference `pykx.license.install` -### Beta Features +**Beta Features** - Added ability for users to convert between PyKX numeric vectors or N-Dimensional Lists and PyTorch Tensor objects using the `pt` method. @@ -955,7 +1525,7 @@ ## PyKX 3.0.1 -#### Release Date +**Release Date** 2024-12-04 @@ -963,7 +1533,7 @@ PyKX 3.0.1 is currently not available for Mac x86/ARM for all Python versions. Updated builds will be provided once available. To install PyKX 3.0.1 on Mac please install from source [here](https://github.com/kxsystems/pykx). -### Additions +**Additions** - Addition of `.replace()` function to `kx.Vector` and `kx.List` objects to search for and replace items in each collection, retaining typing where appropriate. @@ -990,7 +1560,7 @@ pykx.IntVector(pykx.q('7 10 12..')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Added warning to `kx.q.system.load` and context registration when attempting to load path with a space. Can be suppressed by enabling `PYKX_SUPPRESS_WARNINGS`. - Changed `%%python` heading to `%%py` when calling Python code during `jupyter_qfirst` mode so as not to conflict with inbuilt Jupyter cell magics. @@ -1041,7 +1611,7 @@ ## PyKX 3.0.0 -#### Release Date +**Release Date** 2024-11-12 @@ -1049,7 +1619,7 @@ PyKX 3.0.0 is currently not available for Mac x86/ARM for all Python versions. Updated builds will be provided once available. To install PyKX 3.0.0 on Mac please install from source [here](https://github.com/kxsystems/pykx). -### Additions +**Additions** - Addition of functionality to allow for development of end-to-end streaming workflows consisting of data-ingestion, persistence and query. This functionality is outlined in-depth [here](../user-guide/advanced/streaming/index.md) with a worked example [here](../examples/streaming/index.md). @@ -1413,7 +1983,7 @@ - All infinity and null properties on atom classes are now usable in unlicensed mode: `null`, `inf`, `inf_pos`, `inf_neg`, `is_inf`, `is_neg_inf`, `is_pos_inf`. -### Fixes and Improvements +**Fixes and Improvements** - Previously built-in `#!python help()` python function, when called on q keywords such as `#!python kx.q.rand` or `#!python kx.q.max` was displaying the generic `wrappers` object help message. As noted in [the installation docs](../getting-started/installing.md#dependencies), this functionality requires the installation of optional libraries available through `pip install pykx[help]`. @@ -2101,24 +2671,24 @@ /usr/python/site-packages/pykx/config.py: RuntimeWarning: '-t' argument unsupported in QARGS.. ``` -### Upgrade considerations +**Upgrade considerations** - PyKX 3.0.0 is a major version update which includes changes requiring review before upgrading from 2.5.*. A page details these changes in full [here](../upgrades/2030.md) ## PyKX 2.5.5 -#### Release Date +**Release Date** 2024-11-28 -### Fixes and Improvements +**Fixes and Improvements** - PyKX Pandas dependency has been raised to allow `<=2.2.3` for Python>3.8 - PyKX Pandas dependency for Python 3.8 has been clamped to `<2.0` due to support being dropped for it by Pandas after 2.0.3. ## PyKX 2.5.4 -#### Release Date +**Release Date** 2024-10-22 @@ -2126,7 +2696,7 @@ PyKX 2.5.4 is currently not available for Mac x86/ARM for all Python versions. Updated builds will be provided once available. -### Fixes and Improvements +**Fixes and Improvements** - Resolved context interface failing to load files on Windows. @@ -2168,18 +2738,18 @@ ``` -### Version Support Changes +**Version Support Changes** - Version 2.5.4 marks the removal of support for releases to PyPI/Anaconda of Python 3.7 supported versions of PyKX ## PyKX 2.5.2 -#### Release Date +**Release Date** 2024-07-05 -### Fixes and Improvements +**Fixes and Improvements** - Converting PyKX generic lists using the keyword parameter `raw=True` would previously return incorrect results, the values received being the memory address of the individual elements of the list, this has now been resolved @@ -2287,11 +2857,11 @@ ## PyKX 2.5.1 -#### Release Date +**Release Date** 2024-06-11 -### Additions +**Additions** - Addition of methods to `pykx.Table` and `pykx.KeyedTable` objects for the application of `sql`, `select`, `exec`, `delete` and `update` queries @@ -2376,7 +2946,7 @@ ')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Fix to regression in PyKX 2.5.0 where PyKX initialisation on Windows would result in a segmentation fault when using an `k4.lic` license type. - Previously users could not make direct use of `kx.SymbolicFunction` type objects against a remote process, this has been rectified @@ -2461,11 +3031,11 @@ ## PyKX 2.5.0 -#### Release Date +**Release Date** 2024-05-15 -### Additions +**Additions** - Addition of a method for `pykx.Table` objects to apply `xbar` calculations on specified columns names @@ -2621,7 +3191,7 @@ - Added `as_arrow` keyword to the `.pd()` method on PyKX Wrapped objects, using `as_arrow=True` will use PyArrow backed data types instead of the default NumPy backed data types. -### Fixes and Improvements +**Fixes and Improvements** - When importing PyKX from a source file path containing a space initialisation would fail with an `nyi` error message, this has now been resolved @@ -2924,7 +3494,7 @@ dtype: int64 ``` -### Upgrade considerations +**Upgrade considerations** - Since 2.1.0 when using Pandas >= 2.0 dataframe columns of type `datetime64[s]` converted to `DateVector` under `toq`. Now correctly converts to `TimestampVector`. See [conversion condsideratons](../user-guide/fundamentals/conversion_considerations.md#temporal-data-types) for further details. @@ -2961,27 +3531,27 @@ - Configuration option `PYKX_DISABLE_PANDAS_WARNING` has been removed. - Deprecated `.pd(raw_guids)` keyword. -### Beta Features +**Beta Features** - Addition of [streamlit](https://streamlit.io/) connection class `pykx.streamlit.Connection` to allow querying of q processes when building a streamlit application. For an example of this functionality and an introduction to it's usage see [here](../user-guide/advanced/streamlit.md). ## PyKX 2.4.2 -#### Release Date +**Release Date** 2024-04-03 -### Fixes and Improvements +**Fixes and Improvements** - Updated `libq` to 2024.03.28 for all supported OS's. ## PyKX 2.4.1 -#### Release Date +**Release Date** 2024-03-27 -### Fixes and Improvements +**Fixes and Improvements** - Previously calls to `qsql.select`, `qsql.exec`, `qsql.update` and `qsql.delete` would require multiple calls to parse the content of `where`, `columns` and `by` clauses. These have now been removed with all parsing now completed within the functional query when called via IPC or local to the Python process. - Linux x86 and Mac x86/ARM unlicensed mode `e.o` library updated to 2023.11.22. Fixes subnormals issue: @@ -3014,11 +3584,11 @@ ## PyKX 2.4.0 -#### Release Date +**Release Date** 2024-03-20 -### Additions +**Additions** - Support for q/kdb+ `4.1` documentation [here](https://code.kx.com/q/releases/ChangesIn4.1/) added as an opt-in capability, this functionality is enabled through setting `PYKX_4_1_ENABLED` environment variable. @@ -3107,7 +3677,7 @@ pykx.exceptions.QError: type ``` -### Fixes and Improvements +**Fixes and Improvements** - Added instructions for script to install Windows dependencies. - Resolved segfaults on Windows when PyKX calls Python functions under q. @@ -3172,9 +3742,9 @@ >>> conn('enlist 0Np').py() Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/wrappers.py", line 2443, in py + File "/home/user/.local/lib/python3.10/site-packages/pykx/wrappers.py", line 2443, in py converted_vector[i]=q('0Np') - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 216, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 216, in __call__ raise LicenseException("run q code via 'pykx.q'") pykx.exceptions.LicenseException: A valid q license must be in a known location (e.g. `$QLIC`) to run q code via 'pykx.q'. ``` @@ -3283,7 +3853,7 @@ >>> tab1.merge(tab2_keyed, how='left', q_join=True) ``` -### Beta Features +**Beta Features** - Addition of `Compress` and `Encrypt` classes to allow users to set global configuration and for usage within Database partition persistence. @@ -3326,22 +3896,22 @@ ## PyKX 2.3.2 -#### Release Date +**Release Date** 2024-02-12 -### Fixes and Improvements +**Fixes and Improvements** - Update of PyKX 4.0 linux shared object to version 2024.02.09, this update is to facilitate deployments on more secure linux/linux-arm environments. - Update `Table.rename()` to skip over columns not in table instead of throwing error to match `pandas`. ## PyKX 2.3.1 -#### Release Date +**Release Date** 2024-02-07 -### Fixes and Improvements +**Fixes and Improvements** - Python functions saved to q would error if passed `''` or `'.'`. These now pass without issue. @@ -3355,7 +3925,7 @@ >>> kx.q('func', '') Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ return factory(result, False) File "pykx/_wrappers.pyx", line 493, in pykx._wrappers._factory File "pykx/_wrappers.pyx", line 486, in pykx._wrappers.factory @@ -3363,7 +3933,7 @@ >>> kx.q('func', '.') Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ return factory(result, False) File "pykx/_wrappers.pyx", line 493, in pykx._wrappers._factory File "pykx/_wrappers.pyx", line 486, in pykx._wrappers.factory @@ -3432,7 +4002,7 @@ - `df.select_dtypes()` updated to now accept `kx.*Atom` values for `include`/`exclude` params. Use of `kx.CharVector` will return error. - To align with other areas of PyKX the `upsert` and `insert` methods for PyKX tables and keyed tables now support the keyword argument `inplace`, this change will deprecate usage of `replace_self` with the next major release of PyKX. -### Beta Features +**Beta Features** - Addition of the concept of `Remote Function` execution to PyKX, this allows users, from a Python session to define Python functions which will be executed on a remote q/kdb+ server running PyKX under q. The intention with this feature is to allow onboarding of Python first operations within existing or q/kdb+ first infrastructures @@ -3450,11 +4020,11 @@ ## PyKX 2.3.0 -#### Release Date +**Release Date** 2024-01-22 -### Additions +**Additions** - PyKX now supports the use of `KDB_LICENSE_B64` or `KDB_K4LICENSE_B64` configuration values/environment variables to define the content of a `kc.lic` or `k4.lic` license respectively if no license is found on initial usage of PyKX. - Shortcut provided for access to current date, time and timestamp information using `'today'` and `'now'`. @@ -3513,7 +4083,7 @@ - `%%q` Jupyter Notebook magic adds `--debug` option which prints the q backtrace if the cell execution fails. - Release 2.3.0 adds to PyKX the concept of Beta features, these features are available to users through setting the configuration/environment variable `PYKX_BETA_FEATURES`. For more information on Beta features see further documentation [here](../beta-features/index.md) -### Fixes and Improvements +**Fixes and Improvements** - `%%q` Jupyter Notebook magic now returns all outputs up to and including an error when thrown. Previously only the error was returned. - `%%q` Jupyter Notebook magic ignores accidental whitespace in execution options. Below example no longer fails with `Received unknown argument` error: @@ -3545,7 +4115,7 @@ - Improved error message for conversion failures. - Fixes an issue where a user would receive a length error when attempting to apply `min`, `max`, `prod` and `sum` functions on `pykx.KeyedTable` objects. -### Beta Features +**Beta Features** - Database Management functionality has been added for the creation, loading and maintenance of PyKX Partitioned Databases. A full worked example of this functionality can be found [here](../examples/db-management.ipynb) along with full API documentation which includes examples of each function [here](../api/db.md). The API includes but is not limited to the following: @@ -3563,11 +4133,11 @@ ## PyKX 2.2.3 -#### Release Date +**Release Date** 2024-01-11 -### Fixes and Improvements +**Fixes and Improvements** - PyKX now raises an error appropriately when failing to locate `msvcr100.dll` when loading on Windows. - Config values now default to `False` when not set rather than `None`. @@ -3585,11 +4155,11 @@ Please skip this release and use 2.2.3 or newer. This is due to potential segfaults when reading compressed files. -#### Release Date +**Release Date** 2023-12-12 -### Fixes and Improvements +**Fixes and Improvements** - Conversions between `UUID` and `pykx.GUID` types could produce invalid results under various conditions in both licensed and unlicensed mode. - A regression in 2.2.1 resulted in `SIGINT` signals being incorrectly treated as `SIGTERM` style signals, PyKX now resets all signals overwritten by PyKX to their values prior to import. @@ -3601,11 +4171,11 @@ Please skip this release and use 2.2.3 or newer. This is due to potential segfaults when reading compressed files. -#### Release Date +**Release Date** 2023-11-30 -### Fixes and Improvements +**Fixes and Improvements** - Some messages to `stdout` were not being captured when redirecting. Now all are captured. - Deprecation of internally used environment variable `UNDER_PYTHON` which has been replaced by `PYKX_UNDER_PYTHON` to align with other internally used environment variables. @@ -3742,17 +4312,17 @@ ``` -### Upgrade considerations +**Upgrade considerations** - If dependent on the environment variable `UNDER_PYTHON` please upgrade your code to use `PYKX_UNDER_PYTHON` ## PyKX 2.2.0 -#### Release Date +**Release Date** 2023-11-09 -### Additions +**Additions** - Addition of `agg` method for application of aggregation functions on `pykx.Table` and `pykx.GroupbyTable` objects @@ -3968,7 +4538,7 @@ ')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Fix to allow users to use Python functions when operating on a `pykx.GroupbyTable` with an `apply` function @@ -4062,11 +4632,11 @@ ## PyKX 2.1.2 -#### Release Date +**Release Date** 2023-10-24 -### Fixes and Improvements +**Fixes and Improvements** - Fix to issue where functions retrieved using the Context Interface with names `update/delete/select/exec` would result in an `AttributeError` @@ -4096,11 +4666,11 @@ ## PyKX 2.1.1 -#### Release Date +**Release Date** 2023-10-10 -### Fixes and Improvements +**Fixes and Improvements** - Fix to regression in PyKX 2.1.0 where execution of `from pykx import *` would result in the following behavior @@ -4112,11 +4682,11 @@ ## PyKX 2.1.0 -#### Release Date +**Release Date** 2023-10-09 -### Additions +**Additions** - Added functionality to the CSV Reader to allow for the input of data structures while defining column types. For example, the following reads a CSV file and specifies the types of the three columns named `x1`, `x2` and `x3` to be of type `Integer`, `GUID` and `Timestamp`. @@ -4173,7 +4743,7 @@ the following reads a CSV file and specifies the types of the three columns name the expected type a warning will be raised and the `DateVector` type will be used as a default. -### Fixes and Improvements +**Fixes and Improvements** - Empty PyKX keyed tables can now be converted to Pandas DataFrames, previously this would raise a `ValueError` @@ -4244,17 +4814,17 @@ the following reads a CSV file and specifies the types of the three columns name - Users with [`QINIT`](https://code.kx.com/q/basics/by-topic/#environment) environment variable would previously load twice on initialization within PyKX - Users installing PyKX under q on Windows had been missing installation of required files using `pykx.install_into_QHOME()` -### Dependency Updates +**Dependency Updates** - The version of `Cython` used to build `PyKX` was updated to the full `3.0.x` release version. ## PyKX 2.0.1 -#### Release Date +**Release Date** 2023-09-21 -### Fixes and Improvements +**Fixes and Improvements** - User input based license initialization introduced in 2.0.0 no longer expects user input when operating in a non-interactive modality, use of PyKX in this mode will revert to previous behavior - Use of the environment variables `QARGS='--unlicensed'` or `QARGS='--licensed'` operate correctly following regression in 2.0.0 @@ -4262,7 +4832,7 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 2.0.0 -#### Release Date +**Release Date** 2023-09-18 @@ -4270,7 +4840,7 @@ the following reads a CSV file and specifies the types of the three columns name - Pandas API functionality is enabled permanently which will modify data indexing and retrieval of `pykx.Table` objects. Users should ensure to review and test their codebase before upgrading. - EmbedPy replacement functionality for PyKX under q is now non-beta for Linux and MacOS installations, see [here](underq-changelog.md) for full information on 2.0.0 changelog. -### Additions +**Additions** - [Pandas API](../user-guide/advanced/Pandas_API.ipynb) is enabled by default allowing users to treat PyKX Tables similarly to Pandas Dataframes for a limited subset of Pandas like functionality. As a result of this change the environment variable `PYKX_ENABLE_PANDAS_API` is no longer required. - Addition of file based configuration setting allowing users to define profiles for various PyKX modalities through definition of the file `.pykx.config` see [here](../user-guide/configuration.md) for more information. @@ -4337,7 +4907,7 @@ the following reads a CSV file and specifies the types of the three columns name ')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Update to environment variable definitions in all cases to be prefixed with `PYKX_*` - Return of Pandas API functions `dtypes`, `columns`, `empty`, `ndim`, `size` and `shape` return `kx` objects rather than Pythonic objects @@ -4408,21 +4978,21 @@ the following reads a CSV file and specifies the types of the three columns name pykx.q('1b') ``` -### Breaking Changes +**Breaking Changes** - Pandas API functionality is enabled permanently which will modify data indexing and retrieval. Users should ensure to review and test their codebase before upgrading. ## PyKX 1.6.3 -#### Release Date +**Release Date** 2023-08-18 -### Additions +**Additions** - Addition of argument `return_info` to `pykx.util.debug_environment` allowing user to optionally return the result as a `str` rather than to stdout -### Fixes and Improvements +**Fixes and Improvements** - Fixed Pandas API use of `ndim` functionality which should return `2` when interacting with tables following the expected Pandas behavior. - Fixed an error when using the Pandas API to update a column with a `Symbols`, `Characters`, and `Generic Lists`. @@ -4431,17 +5001,17 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.6.2 -#### Release Date +**Release Date** 2023-08-15 -### Additions +**Additions** - Added `to_local_folder` kwarg to `install_into_QHOME` to enable use of `pykx.q` without write access to `QHOME`. - Added [an example](../examples/threaded_execution/threading.md) that shows how to use `EmbeddedQ` in a multithreaded context where the threads need to modify global state. - Added [PYKX_NO_SIGINT](../user-guide/configuration.md#environment-variables) environment variable. -### Fixes and Improvements +**Fixes and Improvements** - Fixed an issue causing a crash when closing `QConnection` instances on Windows. - Updated q 4.0 libraries to 2023.08.11. Note: Mac ARM release remains on 2022.09.30. @@ -4451,18 +5021,18 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.6.1 -#### Release Date +**Release Date** 2023-07-19 -### Additions +**Additions** - Added `sorted`, `grouped`, `parted`, and `unique`. As methods off of `Tables` and `Vectors`. - Added `PyKXReimport` class to allow subprocesses to reimport `PyKX` safely. - Also includes `.pykx.safeReimport` in `pykx.q` to allows this behavior when running under q as well. - Added environment variables to specify a path to `libpython` in the case `pykx.q` cannot find it. -### Fixes and Improvements +**Fixes and Improvements** - Fixed memory leaks within the various `QConnection` subclasses. - Added deprecation warning around the discontinuing of support for Python 3.7. @@ -4476,11 +5046,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.6.0 -#### Release Date +**Release Date** 2023-06-16 -### Additions +**Additions** - Added `merge_asof` to the Pandas like API. - See [here](../user-guide/advanced/Pandas_API.ipynb#tablemerge_asof) for details of supported keyword arguments and limitations. @@ -4495,7 +5065,7 @@ the following reads a CSV file and specifies the types of the three columns name - Added the ability to use `kx.RawQConnection` as a Python based `q` server using `kx.RawQConnection(port=x, as_server=True)`. - More documentation around using this functionality can be found [here](../examples/server/server.md). -### Fixes and Improvements +**Fixes and Improvements** - Improved error on Windows if `msvcr100.dll` is not found - Updated q libraries to 2023.04.17 @@ -4528,50 +5098,50 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.5.3 -#### Release Date +**Release Date** 2023-05-18 -### Additions +**Additions** - Added support for Pandas `Float64Index`. - Wheels for ARM64 based Macs are now available for download. ## PyKX 1.5.2 -#### Release Date +**Release Date** 2023-04-30 -### Additions +**Additions** - Added support for ARM 64 Linux. ## PyKX 1.5.1 -#### Release Date +**Release Date** 2023-04-28 -### Fixes and Improvements +**Fixes and Improvements** - Fixed an issue with `pykx.q` that caused errors to not be raised properly under q. - Fixed an issue when using `.pykx.get` and `.pykx.getattr` that caused multiple calls to be made. ## PyKX 1.5.0 -#### Release Date +**Release Date** 2023-04-17 -### Additions +**Additions** - Added wrappers around various `q` [system commands](https://code.kx.com/q/basics/syscmds/). - Added `merge` method to tables when using the `Pandas API`. - Added `mean`/`median`/`mode` functions to tables when using the `Pandas API`. - Added various functions around type conversions on tables when using the `Pandas API`. -### Fixes and Improvements +**Fixes and Improvements** - Fix to allow GUIDs to be sent over IPC. - Fix an issue related to IPC connection using compression. @@ -4585,21 +5155,21 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.4.2 -#### Release Date +**Release Date** 2023-03-08 -### Fixes and Improvements +**Fixes and Improvements** - Fixed an issue that would cause `EmbeddedQ` to fail to load. ## PyKX 1.4.1 -#### Release Date +**Release Date** 2023-03-06 -### Fixes and Improvements +**Fixes and Improvements** - Added constructors for `Table` and `KeyedTable` objects to allow creation of these objects from dictionaries and list like objects. - Fixed a memory leak around calling wrapped `Foreign` objects in `pykx.q`. @@ -4607,11 +5177,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.4.0 -#### Release Date +**Release Date** 2023-01-23 -### Additions +**Additions** - Addition of a utility function `kx.ssl_info()` to retrieve the SSL configuration when running in unlicensed mode (returns the same info as kx.q('-26!0') with a license). - Addition of a utility function `kx.schema.builder` to allow for the generation of `pykx.Table` and `pykx.KeyedTable` types with a defined schema and zero rows, this provides an alternative to writing q code to create an empty table. @@ -4624,7 +5194,7 @@ the following reads a CSV file and specifies the types of the three columns name - Added environment variable `PYKX_Q_LOCK=1` to enable a Mutex Lock around calls into q, setting this environment variable to a number greater than 0 will set the max length in time to block before raising an error, a value of '-1' will block indefinitely and will not error, any other value will cause an error to be raised immediately if the lock cannot be acquired. - Added `insert` and `upsert` methods to `Table` and `KeyedTable` objects. -### Fixes and Improvements +**Fixes and Improvements** - Fixed `has_nulls` and `has_infs` properties for subclasses of `k.Collection`. - Improved error output of `kx.QConnection` objects when an error is raised within the context interface. @@ -4634,28 +5204,28 @@ the following reads a CSV file and specifies the types of the three columns name - Fixed conversion of null `GUIDAtom`'s to and from Numpy types. - Improved performance of converting `q` enums to pandas Categoricals. -### Beta Features +**Beta Features** - Added support for a Pandas like API around `Table` and `KeyedTable` instances, documentation for the specific functionality can be found [here](../user-guide/advanced/Pandas_API.ipynb). - Added `.pykx.setdefault` to `pykx.q` which allows the default conversion type to be set without using environment variables. ## PyKX 1.3.2 -#### Release Date +**Release Date** 2023-01-06 -### Features and Fixes +**Features and Fixes** - Fixed support for using TLS with `SyncQConnection` instances. ## PyKX 1.3.1 -#### Release Date +**Release Date** 2022-11-16 -### Features and Fixes +**Features and Fixes** - Added environment variable `PYKX_Q_LIB_LOCATION` to specify a path to load the PyKX q libraries from. - Required files in this directory @@ -4674,11 +5244,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.3.0 -#### Release Date +**Release Date** 2022-10-20 -### Features and Fixes +**Features and Fixes** - Support for converting `datetime.datetime` objects with time zone information into `pykx.TimestampAtom`s and `pykx.TimestampVector`s. - Added a magic command to run cells of q code in a Jupyter Notebook. The addition of `%%q` at the start of a Jupyter Notebook cell will allow a user to execute q code locally similarly to loading a q file. @@ -4687,38 +5257,38 @@ the following reads a CSV file and specifies the types of the three columns name - Fix to memory leak seen when converting Pandas Dataframes to q tables. - Removed unnecessary copy when sending `q` objects over IPC. -### Beta Features +**Beta Features** - EmbedPy replacement functionality `pykx.q` updated significantly to provide parity with embedPy from a syntax perspective. Documentation of the interface [here](../pykx-under-q/intro.md) provides API usage. Note that initialization requires the first version of Python to be retrieved on a users `PATH` to have PyKX installed. Additional flexibility with respect to installation location is expected in `1.4.0` please provide any feedback to `pykx@kx.com` ## PyKX 1.2.2 -#### Release Date +**Release Date** 2022-10-01 -### Features and Fixes +**Features and Fixes** - Fixed an issue causing the timeout argument for `QConnection` instances to not work properly. ## PyKX 1.2.1 -#### Release Date +**Release Date** 2022-09-27 -### Features and Fixes +**Features and Fixes** - Added support for OpenSSLv3 for IPC connections created when in 'licensed' mode. - Updated conversion functionality for timestamps to support conversions within Pandas 1.5.0 ## PyKX 1.2.0 -#### Release Date +**Release Date** 2022-09-01 -### Features and Fixes +**Features and Fixes** - Support for converting any python type to a `q` Foreign object has been added. - Support for converting Pandas categorical types into `pykx.EnumVector` type objects. @@ -4734,7 +5304,7 @@ the following reads a CSV file and specifies the types of the three columns name - Fixed conversion of `None` Python objects into analogous null `K` types if a `ktype` is specified. - Added `event_loop` parameter to `pykx.AsyncQConnection` that takes a running event loop as a parameter and allows the event loop to manage `pykx.QFuture` objects. -### Beta Features +**Beta Features** - Added extra functionality to `pykx.q` related to the calling and use of python foreign objects directly within a `q` process. - Support for [NEP-49](https://numpy.org/neps/nep-0049.html), which allows Numpy arrays to be converted into `q` Vectors without copying the underlying data. This behavior is opt-in and you can do so by setting the environment variable `PYKX_ALLOCATOR` to 1, "1" or True or by adding the flag `--pykxalloc` to the `QARGS` environment variable. Note: This feature also requires a python version of at least 3.8. @@ -4742,25 +5312,25 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.1.1 -#### Release Date +**Release Date** 2022-06-13 -### Features & Fixes +**Features & Fixes** - Added ability to skip symlinking `$QHOME` to `PyKX`'s local `$QHOME` by setting the environment variable `IGNORE_QHOME`. ## PyKX 1.1.0 -#### Release Date +**Release Date** 2022-06-07 -### Dependencies +**Dependencies** - The dependency on the system library `libcurl` has been made optional for Linux. If it is missing on Linux, a warning will be emitted instead of an error being raised, and the KX Insights Core library `kurl` will not be fully loaded. Windows and macOS are unaffected, as they don't support the KX Insights Core features to begin with. -### Features & Fixes +**Features & Fixes** - Splayed and partitioned tables no longer emit warnings when instantiated. - Added `pykx.Q.sql`, which is a wrapper around [KXI Core SQL](https://code.kx.com/insights/core/sql.html#sql-language-support). @@ -4787,15 +5357,15 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.0.1 -#### Release Date +**Release Date** 2022-03-18 -### Deprecations & Removals +**Deprecations & Removals** - The `sync` parameter for `pykx.QConnection` and `pykx.QConnection.__call__` has been renamed to the less confusing name `wait`. The `sync` parameter remains, but its usage will result in a `DeprecationWarning` being emitted. The `sync` parameter will be removed in a future version. -### Features & Fixes +**Features & Fixes** - Updated to stable classifier (`Development Status :: 5 - Production/Stable`) in project metadata. Despite this update being done in version 1.0.1, version 1.0.0 is still the first stable release of PyKX. - PyKX now provides source distributions (`sdist`). It can be downloaded from PyPI using `pip download --no-binary=:all: --no-deps pykx`. As noted in [the installation docs](../getting-started/installing.md#supported-environments), installations built from the source will only receive support on a best-effort basis. - Fixed Pandas NaT conversion to q types. Now `pykx.toq(pandas.NaT, ktype=ktype)` produces a null temporal atom for any given `ktype` (e.g. `pykx.TimeAtom`). @@ -4810,11 +5380,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.0.0 -#### Release Date +**Release Date** 2022-02-14 -### Migration Notes +**Migration Notes** To switch from Pykdb to PyKX, you will need to update the name of the dependency from `pykdb` to `pykx` in your `pyproject.toml`/`requirements.txt`/`setup.cfg`/etc. When Pykdb was renamed to PyKX, its version number was reset. The first public release of PyKX has the version number 1.0.0, and will employ [semantic versioning](https://semver.org/). @@ -4848,7 +5418,7 @@ kx.QARGS # List of arguments provided to the embedded q instance at startup, set You can no longer rely on the [context](../api/pykx-execution/ctx.md) being reset to the global context after each call into embedded q, however IPC calls are unaffected. -### Renames +**Renames** - Pykdb has been renamed to PyKX. `Pykdb` -> `PyKX`; `PYKDB` -> `PYKX`; `pykdb` -> `pykx`. - The `adapt` module has been renamed to `toq`, and it can be called directly. Instead of `pykdb.adapt.adapt(x)` one should write `pykx.toq(x)`. - The `k` module has been renamed to `wrappers`. All wrapper classes can be accessed from the top-level, i.e. `pykx.K`, `pykx.SymbolAtom`, etc. @@ -4858,18 +5428,18 @@ You can no longer rely on the [context](../api/pykx-execution/ctx.md) being rese - The `Connection` class in the IPC module has been renamed to `QConnection`, and is now available at the top-level, i.e. `pykx.QConnection`. - The q type wrapper `DynamicLoad` has been renamed to `Foreign` (`pykdb.k.DynamicLoad` -> `pykx.Foreign`). -### Deprecations & Removals +**Deprecations & Removals** - The `pykdb.q.ipc` attribute has been removed. The IPC module can be accessed directly instead at `pykx.ipc`, but generally one will only need to access the `QConnection` class, which can be accessed at the top-level: `pykx.QConnection`. - The `pykdb.q.K` attribute has been removed. Instead, `K` types can be used as constructors for that type by leveraging the `toq` module. For example, instead of `pykdb.q.K(x)` one should write `pykx.K(x)`. Instead of `pykx.q.K(x, k_type=pykx.k.SymbolAtom)` one should write `pykx.SymbolAtom(x)` or `pykx.toq(x, ktype=pykx.SymbolAtom)`. - Most `KdbError`/`QError` subclasses have been removed, as identifying them is error prone, and we are unable to provide helpful error messages for most of them. - The `pykx.kdb` singleton class has been removed. -### Dependencies +**Dependencies** - More Numpy, Pandas, and PyArrow versions are supported. Current `pandas~=1.0`, `numpy~=1.20,<1.22`, and `pyarrow>=3.0.0` are supported. PyArrow remains an optional dependency. - A dependency on `find-libpython~=0.2` was added. This is only used when running PyKX under a q process (see details in the section below about new alpha features). - A dependency on the system library `libcurl` was added for Linux. This dependency will be made optional in a future release. -### Features & Fixes +**Features & Fixes** - The `pykx.Q` class has been added as the base class for `pykx.EmbeddedQ` (the class for `pykx.q`) and `pykx.QConnection`. - The `pykx.EmbeddedQ` process now persists its [context](../api/pykx-execution/ctx.md) between calls. - The console now works over IPC. @@ -4894,13 +5464,13 @@ You can no longer rely on the [context](../api/pykx-execution/ctx.md) being rese - Misc other bug fixes. - Misc doc improvements. -### Performance Improvements +**Performance Improvements** - Converting nested lists from q to Python is much faster. - Internally, PyKX now calls q functions with arguments directly instead of creating a `pykx.Function` instance then calling it. This results in modest performance benefits in some cases. - The context interface no longer loads every element of a context when the context is first accessed, thereby removing the computation spike, which could be particularly intense for large q contexts. -### New Alpha Features +**New Alpha Features** !!! danger "Alpha features are subject to change" diff --git a/docs/release-notes/underq-changelog.md b/docs/release-notes/underq-changelog.md index e5b3e51..0ad6e74 100644 --- a/docs/release-notes/underq-changelog.md +++ b/docs/release-notes/underq-changelog.md @@ -1,28 +1,46 @@ -# PyKX under q Changelog +# KDB-X Python under q Changelog -This changelog provides updates from PyKX 2.0.0 and above, for information relating to versions of PyKX prior to this version see the changelog linked below. +This changelog covers KDB-X Python under q from version 2.0.0 onwards. !!! Note - The changelog presented here outlines changes to PyKX when operating within a q environment specifically, if you require changelogs associated with PyKX operating within a Python environment see [here](./changelog.md). + For changes to KDB-X Python within a Python environment, refer to the [Python changelog](./changelog.md). + +## KDB-X + +KDB-X is the evolution of kdb+. For full details and to install, refer to the [KDB-X developer center](https://developer.kx.com/products/kdb-x/install). + +## KDB-X Python + +KDB-X Python `pykx>=4.0` is the evolution of PyKX `pykx<4.0`. For more details, refer to the migration guide [here](../upgrades/3040.md). + +## PyKX 3.2.0 + +**Release Date** + +2026-06-24 + +**Fixes and Improvements** + +- Fixed a memory leak when using `.pykx.getattr` on foreign object in `q`. ## PyKX 3.1.5 -#### Release Date +**Release Date** 2025-10-21 -### Fixes and Improvements +**Fixes and Improvements** - Added `.pykx.noalloc` which can be used to wrap a foreign object. When used with `.pykx.toq` the conversion will not use the `PYKX_ALLOCATOR`. See [here](../help/issues.md#known-issues) for details. ## PyKX 3.1.4 -#### Release Date +**Release Date** 2025-07-17 -### Fixes and Improvements +**Fixes and Improvements** - Resolved `object has no attribute 't'` error for certain conversions @@ -49,11 +67,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.1.3 -#### Release Date +**Release Date** 2025-06-12 -### Fixes and Improvements +**Fixes and Improvements** - Dashboards integration now lists missing libraries by name. @@ -78,17 +96,17 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat - Resolved `double free or corruption (out)` error when `pykx.q` was loaded in `QINIT` or `q.q`. - `pykx.q` load time has been roughly halved. -### Deprecations & Removals +**Deprecations & Removals** - `.pykx.console[]` has been removed on Windows due to incompatibility. Will now error with `'.pykx.console is not available on Windows` if called. ## PyKX 3.1.1 -#### Release Date +**Release Date** 2025-02-14 -### Fixes and Improvements +**Fixes and Improvements** - Fix throwing of errors for `.pykx.safeReimport`, now throws an error instead of returning a value. @@ -116,11 +134,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.1.0 -#### Release Date +**Release Date** 2025-02-11 -### Additions +**Additions** - Addition of `.pykx.typepy` which returns an objects datatype as a `CharVector` after being passed to python. @@ -133,7 +151,7 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat "" ``` -### Fixes and Improvements +**Fixes and Improvements** - Using `.pykx.toq`/`.pykx.toq0` now return the q representation of an object when passed a wrapped type conversion object @@ -189,7 +207,7 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat - When attempting to load PyKX after embedPy has already been loaded, an error will be thrown and PyKX will not continue to load. -### Beta Features +**Beta Features** - Added ability for users to convert between PyKX numeric vectors or N-Dimensional Lists and PyTorch Tensor objects using the `.pykx.topt` function. @@ -200,11 +218,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.0.1 -#### Release Date +**Release Date** 2024-12-04 -### Additions +**Additions** - Added a function `.pykx.loadPy` which can be used to execute/load a Python file following expected Python syntax, this removes limitations which exist due to use of q parsing syntax when loading a Python file using `system"l /path/to/file.p"` for example @@ -222,16 +240,16 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.0.0 -#### Release Date +**Release Date** 2024-11-12 -### Additions +**Additions** - Added `cloud_libraries` kwarg to `install_into_QHOME` allowing installation of the kdb Insights cloud libraries to QHOME. - Addition of support for new environment variable `PYKX_USE_FIND_LIBPYTHON` which will use the Python package [`find_libpython`](https://pypi.org/project/find-libpython/) to specify the location from which `libpython.[so|dll]` will be taken. -### Fixes and Improvements +**Fixes and Improvements** - Addition of function `.pykx.toq0` to support conversion of Python strings to q strings rather than q symbols as is the default behavior @@ -293,21 +311,21 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.5.4 -#### Release Date +**Release Date** 2024-10-22 -### Fixes and Improvements +**Fixes and Improvements** - `.pykx.util.loadfile` now loads a file using it's full path unless it contains a space. This is to avoid issues loading scripts which are sensitive to their working directory. ## PyKX 2.5.3 -#### Release Date +**Release Date** 2024-08-20 -### Fixes and Improvements +**Fixes and Improvements** - Previously PyKX conversions of generic lists (type 0h) would convert this data to it's `raw` representation rather than it's `python` representation as documented. This had the effect of restricting the usability of some types within PyKX under q in non-trivial use-cases. With the `2.5.2` changes to more accurately represent `raw` data at depth this became more obvious as an issue. @@ -329,21 +347,21 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.5.0 -#### Release Date +**Release Date** 2024-05-15 -### Fixes and Improvements +**Fixes and Improvements** - When loading PyKX under from a source file path containing a space initialisation would fail with an `nyi` error message, this has now been resolved. ## PyKX 2.4.1 -#### Release Date +**Release Date** 2024-03-27 -### Fixes and Improvements +**Fixes and Improvements** - When loading PyKX under q users who had previously loaded [embedPy](https://github.com/KxSystems/embedPy) into their process would cause a segfault of unspecified origin. With this release we have added a warning prior to loading of PyKX which specifies that if a value of `.p.e` has been specified which does not match that expected of PyKX a user should consider installing PyKX under q fully: @@ -359,11 +377,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.3.1 -#### Release Date +**Release Date** 2024-02-07 -### Fixes and Improvements +**Fixes and Improvements** - `.pykx.eval` is now variadic allowing an optional second parameter to be passed to define return type. Previously would error with `rank`. @@ -424,11 +442,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.3.0 -#### Release Date +**Release Date** 2024-01-22 -### Fixes and Improvements +**Fixes and Improvements** - A bug was fixed when using `.pykx.console`, it is now possible to access python variables set using the console with `.pykx.(eval|pyexec|pyeval)` functions. @@ -455,21 +473,21 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.2.2 -#### Release Date +**Release Date** 2023-12-07 -### Fixes and Improvements +**Fixes and Improvements** - When loaded in a q process loading `pykx.q` would not allow `Ctrl+C` (SIGINT) interruption. ## PyKX 2.2.1 -#### Release Date +**Release Date** 2023-11-30 -### Fixes and Improvements +**Fixes and Improvements** - `.pykx.print` was using `repr` representation for some objects. Now consistently calls `print`. - `.pykx.safeReimport` now resets environment variables correctly before throwing any error raised by the function supplied to it. @@ -519,15 +537,15 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.2.0 -#### Release Date +**Release Date** 2023-11-09 -### Additions +**Additions** - Addition of `PYKX_EXECUTABLE` environment/configuration variable to allow control of which Python executable is used under q. -### Fixes and Improvements +**Fixes and Improvements** - Failure to access and load PyKX resulting in an `os` error now returns Python backtrace outlining the underlying Python error allowing for easier debugging @@ -578,11 +596,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.1.0 -#### Release Date +**Release Date** 2023-10-09 -### Fixes and Improvements +**Fixes and Improvements** - Update to default conversion logic for q objects passed to PyKX functions to more closely match embedPy based conversion expectations.For version <=2.0 conversions of KX lists would produce N Dimensional Numpy arrays of singular type. This results in issues when applying to many analytic libraries which rely on lists of lists rather than singular N Dimensional arrays. Additionally q tables and keyed tables would be converted to Numpy recarrays, these are now converted to Pandas DataFrames. To maintain previous behavior please set the following environment variable `PYKX_DEFAULT_CONVERSION="np"`. @@ -606,11 +624,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.0.0 -#### Release Date +**Release Date** 2023-09-18 -### Additions +**Additions** - Addition of `.pykx.qcallable` and `.pykx.pycallable` functions which allow wrapping of a foreign Python callable function returning the result as q or Python foreign respectively. - Addition of `.pykx.version` allowing users to programmatically access their version from a q process. @@ -628,7 +646,7 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat 4 0.691953 0.375638 ``` -### Fixes and Improvements +**Fixes and Improvements** - Application of object setting on a Python list returns generic null rather than wrapped foreign object. - Use of environment variables relating to `PyKX under q` must use `"true"` as accepted value, previously any value set for such environment variables would be supported. diff --git a/docs/roadmap.md b/docs/roadmap.md deleted file mode 100644 index 89a2514..0000000 --- a/docs/roadmap.md +++ /dev/null @@ -1,26 +0,0 @@ -# PyKX Roadmap - -This page outlines areas of development focus for the PyKX team to provide you with an understanding of the development direction of the library. This is not an exhaustive list of all features/areas of focus but should give you a view on what to expect from the team over the coming months. Additionally this list is subject to change, particularly for any example code provided based on the complexity of the features and any customer feature requests raised following the publishing of this list. - -If you need a feature that's not included in this list please let us know by raising a [Github issue](https://github.com/KxSystems/pykx/issues)! - -## Upcoming Changes - -- Addition of support for q primitives as methods off PyKX Vector and Table objects. Syntax for this will be similar to the following: - - ```python - >>> import pykx as kx - >>> N = 1000 - >>> vec = kx.random.random(N, 100.0) - >>> vec.mavg(3) - >>> vec.abs() - ``` - -- Performance improvements for conversions from PyKX Vector objects to Numpy arrays through enhanced use of C++ over Cython. -- Configurable initialisation logic in the absence of a license. Thus allowing users who have their own workflows for license access to modify the instructions for their users. - -## Future - -- Tighter integration between PyKX/q objects and PyArrow arrays/Tables -- Expansion of supported datatypes for translation to/from PyKX -- Data pre-processing and statistics modules for operation on PyKX tables and vector objects diff --git a/docs/stylesheets/2026-edited.css b/docs/stylesheets/2026-edited.css index cf82b53..41fb386 100644 --- a/docs/stylesheets/2026-edited.css +++ b/docs/stylesheets/2026-edited.css @@ -1,5 +1,5 @@ /* - Title: PyKX extra CSS for code.kx.com in MkDocs + Title: KDB-X Python extra CSS for code.kx.com in MkDocs */ /* countermand MkDocs insertion of href attribute after link anchor @@ -254,7 +254,6 @@ background-color: transparent; margin-top: 0; } - [data-md-color-scheme="kx-light"] .md-header { background-color: #194387; color: white; diff --git a/docs/upgrades/3040.md b/docs/upgrades/3040.md new file mode 100644 index 0000000..12eed76 --- /dev/null +++ b/docs/upgrades/3040.md @@ -0,0 +1,123 @@ +--- +title: Migrating from PyKX 3.* to KDB-X Python 4.* +description: How to migrate from PyKX 3.* to KDB-X Python 4.* +last_updated: April 2026 +author: KX Systems, Inc., +tags: PyKX, KDB-X, KDB-X Python, migrate, remote Python execution +--- + +# Migrating from PyKX 3.* to KDB-X Python 4.* + +*This page outlines the key changes that affect developers moving from PyKX (`pykx~=3.0`) to KDB-X Python (`pykx>=4.0`). It covers updated configuration behavior, new defaults, deprecated options, and important differences in how SQL, threading, and cloud libraries load.* + +## KDB-X and Python integration + +KDB-X is the evolution of kdb+. KDB-X Python is the evolution of PyKX, and now develops directly alongside the KDB-X platform – new KDB-X features integrate rapidly and completely into the Python library. + +To obtain KDB-X and a license, visit the [KDB-X install](https://developer.kx.com/products/kdb-x/install) page. + +## Key migration changes at a glance + +- KDB-X Python operates using KDB-X (unlike PyKX which used kdb+ 4.0/4.1). +- Configuration is now primarily managed through configuration files; environment variables are still supported and take precedence. +- Cloud libraries are no longer loaded by default – they have moved from opt-out to opt-in. +- The SQL interface is embedded directly into KDB-X and is auto-loaded on first use. +- The NEP-49 allocator is now enabled by default (opt-out instead of opt-in). +- No default value for secondary threads. + +## KDB-X configuration changes + +For the full list of platform changes, refer to the [KDB-X release notes](https://code.kx.com/kdb-x/releases/release-notes-latest.html). The updates below directly impact KDB-X Python: + +- **Added `QCFG` configuration option.** Defaults to `~/.kx/config` if that file exists. +- **Updated license lookup behavior.** KDB-X no longer checks the current working directory for a license. Only `QLIC` and `QHOME` are checked. +- **Added `.z.v` to the `ZContext`.** Holds initialization values for: `version`, `QCFG`, `QHOME`, `QLIC`, and `QINIT`. + +## KDB-X Python configuration changes + +### Configuration file location + +- The default config file has changed from `.pykx-config` to `config-pykx`. +- `PYKX_CONFIGURATION_LOCATION` now points to a **file**, not a folder. Users can set a custom filename if needed. +- Configuration is searched in this order: + 1. `PYKX_CONFIGURATION_LOCATION` (if set) + 2. `~/.kx/config-pykx` (if it exists) +- The current directory (`.`) and home directory (`~`) are no longer searched. + +### `kx.util.add_to_config` changes + +- Added a `profile` argument to specify which profile should receive the configuration values. + - Defaults to `kx.config.pykx_config_profile`. + - Falls back to `default` if no profile is set. +- The `folder` argument has been replaced with `file`. + - Defaults to `kx.config.pykx_config_location`. + - If unset, creates a new file at `~/.kx/config-pykx`. +- Values are no longer added to `os.environ`. Configuration is written only to the config file. + +### `kx.util.install_q` changes + +- Added `config_file` argument (defaults to `pykx_config_location`). +- Added `config_profile` argument (defaults to `pykx_config_profile`). + +## Removed and replaced options + +- Removed `PYKX_4_1_ENABLED` configuration option. +- **NEP-49 allocator is now enabled by default** to support zero-copy conversions. + - Removed `PYKX_ALLOCATOR`. + - Added `PYKX_NO_ALLOCATOR` to opt out. + - Removed `--pykxalloc` (QARGS / CLI). + - Added `--pykxnoalloc` as the new opt-out flag. + +## Library loading and command-line changes + +### Cloud library loading + +KDB-X Python no longer loads cloud libraries (`kurl`, `objstor`, `qlog`, `sql`) by default on import. These libraries have moved from **opt-out to opt-in** in order to: + +- Speed up imports. +- Prevent errors and warnings for users who don't need these features. + +Libraries must now be explicitly enabled using the environment variables or `QARGS` flags below. + +### Environment variable changes + +| Removed | Replacement | +|---------|-------------| +| `PYKX_NOQCE` | `PYKX_QCE` | + +### QARGS flag updates + +The following flags have moved from opt-out to **opt-in**: + +| Removed flag | Replacement | +|----------------|--------------| +| `--no-qce` | `--qce` | +| `--no-kurl` | `--kurl` | +| `--no-objstor` | `--objstor` | +| `--no-qlog` | `--qlog` | +| `--no-sql` | `--sql` | + +## SQL interface changes + +KDB-X now **embeds its SQL interface directly**, eliminating the need to load it from an external file (`s.k_`). + +You can initialize the SQL interface in any of the following ways: + +- Set any of: `QARGS="--qce"`, `QARGS="--sql"`, `PYKX_QCE=True`. +- From Python: the first call to `kx.q.sql` automatically loads the interface. +- From q: call `.s.init[]` or start a query with `s)` – both auto-load on first use. + +!!! note + If your code calls `.s` functions directly by name (for example, `.s.e`), ensure the SQL interface is loaded before those calls run using one of the methods above. + +## Threading behavior + +KDB-X Python no longer sets a default value for secondary threads. To enable them: + +- Set `QARGS="-s N"`, or +- Add `s=N` to your `QCFG` file. + + +## Changelog + +For the full changelog, refer to the [KDB-X Python changelog](../release-notes/changelog.md). diff --git a/docs/user-guide/advanced/Pandas_API.ipynb b/docs/user-guide/advanced/Pandas_API.ipynb index 7cb4833..38ce1b7 100644 --- a/docs/user-guide/advanced/Pandas_API.ipynb +++ b/docs/user-guide/advanced/Pandas_API.ipynb @@ -5,23 +5,23 @@ "id": "d2a3ccf7", "metadata": {}, "source": [ - "# Pandas Like API for PyKX Tables\n", + "# Pandas Like API for KDB-X Python Tables\n", "\n", - "_This page demonstrates the PyKX functionality that aligns with the Pandas API for DataFrame interactions._\n", + "_This page demonstrates the KDB-X Python functionality that aligns with the Pandas API for DataFrame interactions._\n", "\n", - "Only operations on PyKX tables that adhere to Pandas API conventions are covered. The focus is on areas where PyKX/q can offer a performance advantage over Pandas, particularly in terms of memory footprint and execution time. \n", + "Only operations on KDB-X Python tables that adhere to Pandas API conventions are covered. The focus is on areas where KDB-X Python/q can offer a performance advantage over Pandas, particularly in terms of memory footprint and execution time. \n", "\n", "A full breakdown of the the available functionality and examples of its use can be found in the [Pandas API](#pandas-api) section below.\n", "\n", "## Covered sections of the Pandas API\n", "\n", - "In this context, _coverage_ refers to the functionality provided by the PyKX API for Tables, which matches the methods and attributes supported by the Pandas DataFrame API. It does not include functionality for interacting with Pandas Series objects or for reading/writing CSV/JSON files.\n", + "In this context, _coverage_ refers to the functionality provided by the KDB-X Python API for Tables, which matches the methods and attributes supported by the Pandas DataFrame API. It does not include functionality for interacting with Pandas Series objects or for reading/writing CSV/JSON files.\n", "\n", "If there's any functionality you would like us to add to this library, please open an issue [here](https://github.com/KxSystems/pykx/issues) or open a pull request [here](https://github.com/KxSystems/pykx/pulls).\n", "\n", "### Property/metadata type information\n", "\n", - "| **DataFrame properties**| **PyKX supported?** | **PyKX API documentation link** | \n", + "| **DataFrame properties**| **KDB-X Python supported?** | **KDB-X Python API documentation link** | \n", "|----------------------|-----------------|-----------------------------|\n", "| [columns](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.columns.html) | Yes | [link](Pandas_API.html#tablecolumns) | \n", "| [dtypes](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.dtypes.html) | Yes | [link](Pandas_API.html#tabledtypes) |\n", @@ -32,7 +32,7 @@ "\n", "### Analytic functionality\n", "\n", - "| **DataFrame method** | **PyKX supported?** | **PyKX API documentation link** |\n", + "| **DataFrame method** | **KDB-X Python supported?** | **KDB-X Python API documentation link** |\n", "|----------------------|-----------------|-----------------------------|\n", "| [abs](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.abs.html) | Yes | [link](Pandas_API.html#tableabs) |\n", "| [agg](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.agg.html) | Yes | [link](Pandas_API.html#tableagg) |\n", @@ -56,7 +56,7 @@ "\n", "### Querying and data interrogation\n", "\n", - "| **DataFrame method** | **PyKX supported?** | **PyKX API documentation link** |\n", + "| **DataFrame method** | **KDB-X Python supported?** | **KDB-X Python API documentation link** |\n", "|----------------------|-----------------|-----------------------------|\n", "| [all](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.all.html) | Yes | [link](Pandas_API.html#tableall) |\n", "| [any](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.any.html) | Yes | [link](Pandas_API.html#tableany) |\n", @@ -76,7 +76,7 @@ "\n", "### Data preprocessing\n", "\n", - "| **DataFrame method** | **PyKX supported?** | **PyKX API documentation link** |\n", + "| **DataFrame method** | **KDB-X Python supported?** | **KDB-X Python API documentation link** |\n", "|----------------------|-----------------|-----------------------------|\n", "| [add_prefix](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.add_prefix.html) | Yes | [link](Pandas_API.html#tableas_prefix) |\n", "| [add_suffix](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.add_suffix.html) | Yes | [link](Pandas_API.html#tableas_suffix) |\n", @@ -90,14 +90,14 @@ "\n", "### Data joins/merge\n", "\n", - "| **DataFrame method** | **PyKX supported?** | **PyKX API documentation link** |\n", + "| **DataFrame method** | **KDB-X Python supported?** | **KDB-X Python API documentation link** |\n", "|----------------------|-----------------|-----------------------------|\n", "| [merge](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.merge.html) | Yes | [link](Pandas_API.html#tablemerge) |\n", "| [merge_asof](https://pandas.pydata.org/docs/reference/api/pandas.merge_asof.html) | Yes | [link](Pandas_API.html#tablemerge_asof) |\n", "\n", "### Data sorting\n", "\n", - "| **DataFrame method** | **PyKX supported?** | **PyKX API documentation link** | \n", + "| **DataFrame method** | **KDB-X Python supported?** | **KDB-X Python API documentation link** | \n", "|----------------------|-----------------|-----------------------------|\n", "| [sort_values](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.sort_values.html) | Yes | [link](Pandas_API.html#tablesort_values) |\n", "| [nlargest](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.nlargest.html) | Yes | [link](Pandas_API.html#tablenlargest) |\n", @@ -105,12 +105,12 @@ "\n", "### Unsupported functionality\n", "\n", - "| **DataFrame methods** | **PyKX supported?** | **Additional information** |\n", + "| **DataFrame methods** | **KDB-X Python supported?** | **Additional information** |\n", "|----------------------|------------------|------------------------|\n", - "| `*from*` | No | Functionality for the creation of PyKX Tables from alternative data sources is not supported at this time. |\n", + "| `*from*` | No | Functionality for the creation of KDB-X Python Tables from alternative data sources is not supported at this time. |\n", "| `*plot*` | No | Functionality for the plotting of columns/tables is not supported at this time. |\n", "| `*sparse*` | No | Sparse data like interactions presently not supported. |\n", - "| `to_*` | No | Functionality for the conversion/persistence of PyKX Tables to other formats is not supported at this time. |\n" + "| `to_*` | No | Functionality for the conversion/persistence of KDB-X Python Tables to other formats is not supported at this time. |\n" ] }, { @@ -119,7 +119,7 @@ "metadata": {}, "source": [ "## Pandas API\n", - "_This notebook provides a demonstration of the capabilities of the pandas-like API for PyKX Table objects._\n", + "_This notebook provides a demonstration of the capabilities of the pandas-like API for KDB-X Python Table objects._\n", "\n", "To follow along, download this notebook.\n", "\n", @@ -145,7 +145,7 @@ "outputs": [], "source": [ "import os\n", - "os.environ['PYKX_IGNORE_QHOME'] = '1' # Ignore symlinking PyKX q libraries to QHOME\n", + "os.environ['PYKX_IGNORE_QHOME'] = '1' # Ignore symlinking KDB-X Python q libraries to QHOME\n", "os.environ['PYKX_Q_LOADED_MARKER'] = '' # Only used here for running Notebook under mkdocs-jupyter during document generation." ] }, @@ -2171,7 +2171,7 @@ "| right_on | str | The column name in the right table to join on. | None |\n", "| left_index | bool | Use the index of the left Table. | False |\n", "| right_index | bool | Use the index of the right Table. | False |\n", - "| by | str | Not yet implemented. | None |\n", + "| by | Union[str/list] | Match on these columns before performing merge operation. | None |\n", "| left_by | str | Field names to match on in the left table. | None |\n", "| right_by | str | Field names to match on in the right table. | None |\n", "| suffixes | Tuple(str, str) | The number of rows to return. | ('\\_x', '\\_y') |\n", @@ -2303,6 +2303,16 @@ "trades.merge_asof(quotes, on=\"time\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc094351", + "metadata": {}, + "outputs": [], + "source": [ + "trades.merge_asof(quotes, on=\"time\", by=\"ticker\")" + ] + }, { "cell_type": "markdown", "id": "e6280a9a", @@ -4759,7 +4769,7 @@ " names=None)\n", "```\n", "\n", - "Reset the keys/index of a keyed PyKX table. This can be used to remove/unset one or more keys within a table.\n", + "Reset the keys/index of a keyed KDB-X Python table. This can be used to remove/unset one or more keys within a table.\n", "\n", "**Parameters:**\n", "\n", diff --git a/docs/user-guide/advanced/attributes.md b/docs/user-guide/advanced/attributes.md index 56720dd..e658f81 100644 --- a/docs/user-guide/advanced/attributes.md +++ b/docs/user-guide/advanced/attributes.md @@ -1,17 +1,17 @@ --- -title: Apply attributes in PyKX -description: How to use attributes in PyKX +title: Apply attributes in KDB-X Python +description: How to use attributes in KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, PyKX objects, +tags: KDB-X Python, q, `pykx` objects, --- # Apply Attributes -_This page provides details on how to apply attributes in PyKX._ +_This page provides details on how to apply attributes in KDB-X Python._ -!!! tip "Tip: For the best experience, we recommend reading about [PyKX attributes](..//../learn/objects.md#what-are-pykx-attributes) first." +!!! tip "Tip: For the best experience, we recommend reading about [KDB-X Python attributes](..//../learn/objects.md#what-are-pykx-attributes) first." -In PyKX, you can apply attributes to various data structures, including `#!python Vector`/`#!python List` types, `#!python Tables`, and `#!python KeyedTable`s. To apply the attributes, call the `#!python sorted`, `#!python unique`, `#!python grouped`, and `#!python parted` methods on these objects. +In KDB-X Python, you can apply attributes to various data structures, including `#!python Vector`/`#!python List` types, `#!python Tables`, and `#!python KeyedTable`s. To apply the attributes, call the `#!python sorted`, `#!python unique`, `#!python grouped`, and `#!python parted` methods on these objects. ### Sorted @@ -141,7 +141,7 @@ b| s p ## Performance -When attributes are set on PyKX objects, various functions can use these attributes to speed up their +When attributes are set on `pykx` objects, various functions can use these attributes to speed up their execution, by using different algorithms. For example, searching through a list without an attribute requires checking every single value. However, setting the `#!python sorted` attribute allows a search algorithm to use a binary search instead and then only a fraction of the values actually needs to be checked. diff --git a/docs/user-guide/advanced/compress-encrypt.md b/docs/user-guide/advanced/compress-encrypt.md index 0d5a153..0354b49 100644 --- a/docs/user-guide/advanced/compress-encrypt.md +++ b/docs/user-guide/advanced/compress-encrypt.md @@ -1,30 +1,30 @@ --- -title: PyKX compress and encrypt -description: How to compress and encrypt data in PyKX +title: KDB-X Python compress and encrypt +description: How to compress and encrypt data in KDB-X Python date: October 2024 author: KX Systems, Inc., -tags: compression, encryption, PyKX +tags: compression, encryption, KDB-X Python --- # Compress and encrypt data -_This page explains how to compress and encrypt data in PyKX._ +_This page explains how to compress and encrypt data in KDB-X Python._ -With the volumes of sensitive data being produced within real-time applications today the ability to securely store this data and quickly access it can be challenging. PyKX provides several utilities, in the form of class objects, for the management of how data is compressed and encrypted when being persisted. +With the volumes of sensitive data being produced within real-time applications today the ability to securely store this data and quickly access it can be challenging. KDB-X Python provides several utilities, in the form of class objects, for the management of how data is compressed and encrypted when being persisted. ### Compress -PyKX supports the compression of data to disk, allowing you to reduce disk space required for your persisted historical data. PyKX gives you a variety of compression/decompression options through the following algorithms: +KDB-X Python supports the compression of data to disk, allowing you to reduce disk space required for your persisted historical data. KDB-X Python gives you a variety of compression/decompression options through the following algorithms: - [`#!python gzip`](https://en.wikipedia.org/wiki/Gzip) - [`#!python snappy`](https://en.wikipedia.org/wiki/Snappy_(compression)) - [`#!python zstd`](https://en.wikipedia.org/wiki/Zstd) - [`#!python LZ4HC`](https://en.wikipedia.org/wiki/LZ4_(compression_algorithm)) -In addition to this, you can compress data to KX's own qIPC format. For full information, go to [KX file compression within kdb+/q](https://code.kx.com/q/kb/file-compression/). +In addition to this, you can compress data to KX's own qIPC format. For full information, go to [KX file compression within KDB-X/q](https://code.kx.com/q/kb/file-compression/). ### Encrypt -PyKX supports Data At Rest Encryption (DARE) with an explicit requirement on at least OpenSSL v1.0.2. To find out which version of OpenSSL is available to you via PyKX, use the following: +KDB-X Python supports Data At Rest Encryption (DARE) with an explicit requirement on at least OpenSSL v1.0.2. To find out which version of OpenSSL is available to you via KDB-X Python, use the following: ```python >>> import pykx as kx @@ -41,7 +41,7 @@ SSL_VERIFY_SERVER| YES ')) ``` -The encryption provided by this functionality is Transparent Disk Encryption (TDE). TDE protects data at rest by encrypting database files on the hard drive and as a result on backup media. Encrypting your data with PyKX is fully transparent to queries requiring no change to the logic used when querying data but results in a time penalty. +The encryption provided by this functionality is Transparent Disk Encryption (TDE). TDE protects data at rest by encrypting database files on the hard drive and as a result on backup media. Encrypting your data with KDB-X Python is fully transparent to queries requiring no change to the logic used when querying data but results in a time penalty. To use this functionality, you must have a password-protected master key available, ideally with a unique password of high-entropy. For more information on the generation of a master key and a password, go to the [DARE configuration](https://code.kx.com/q/kb/dare/#configuration) section. @@ -55,7 +55,7 @@ This walkthrough demonstrates the following steps: ### Generate compression objects -With PyKX, you can create compression and encryption class objects to set global configurations or use in specific individual functions. These respectively are supported via the `#!python kx.Compress` and `#!python kx.Encrypt` classes. For this section we will deal only with compression. +With KDB-X Python, you can create compression and encryption class objects to set global configurations or use in specific individual functions. These respectively are supported via the `#!python kx.Compress` and `#!python kx.Encrypt` classes. For this section we will deal only with compression. The full list of algorithms is part of the `#!python kx.CompressionAlgorithm` enumeration: @@ -126,7 +126,7 @@ In cases where your are dealing with data security is important or where data is ### Persist database partitions with various configurations -Not all data is created equally, in time-series applications such as algorithmic trading it is often the case that older data is less valuable than newer data. As a result, when backfilling historical data, you may more aggressively compress older datasets. The PyKX compression logic allows you to persist different partitions within a historical database to different levels. +Not all data is created equally, in time-series applications such as algorithmic trading it is often the case that older data is less valuable than newer data. As a result, when backfilling historical data, you may more aggressively compress older datasets. The KDB-X Python compression logic allows you to persist different partitions within a historical database to different levels. 1. Create a database with the most recent data uncompressed @@ -200,7 +200,7 @@ zipLevel | 10i Global initialization of compression and encryption allows all data that is persisted within, from a process, to be compressed. This can be useful when completing large batch operations on data where being specific about per partition/per file operations isn't necessary. In the below section we will deal with compression and encryption separately. -PyKX uses compression settings that are globally readable via `#!python kx.q.z.zd`. When unset, this value returns a PyKX Identity value as follows: +KDB-X Python uses compression settings that are globally readable via `#!python kx.q.z.zd`. When unset, this value returns a KDB-X Python Identity value as follows: ```python >>> kx.q.z.zd diff --git a/docs/user-guide/advanced/context_interface.md b/docs/user-guide/advanced/context_interface.md index 89db4d4..14674bc 100644 --- a/docs/user-guide/advanced/context_interface.md +++ b/docs/user-guide/advanced/context_interface.md @@ -1,20 +1,20 @@ --- title: Import existing q functions -description: How to use q functions in a Pythonic way in PyKX +description: How to use q functions in a Pythonic way in KDB-X Python date: October 2024 author: KX Systems, Inc., -tags: interface, q, PyKX +tags: interface, q, KDB-X Python --- # Import existing q functions -For you and many users of PyKX the q programming language may not be your primary language of choice when developing analytics and applications. There are a number of circumstances under which access to q functionality or the ability to use functions written in q may be critical to your use-case: +For you and many users of KDB-X Python the q programming language may not be your primary language of choice when developing analytics and applications. There are a number of circumstances under which access to q functionality or the ability to use functions written in q may be critical to your use-case: -- When dealing with large volumes of kdb+ data or operations where vector analytic performance is of paramount importance. +- When dealing with large volumes of KDB-X data or operations where vector analytic performance is of paramount importance. - When you wish to make use of existing q code/libraries in a Python first way. - When you need access to functions of the q language directly. -The below sections make use of what is known as "The Context Interface". In q, a context (known as a namespace when at the top level) is an organizational structure which is used to organize code into libraries/common utilities. For more information on contexts/namespaces in q please refer to [Chapter 12 of Q for Mortals](https://code.kx.com/q4m3/12_Workspace_Organization/). PyKX exposes these contexts as special [`kx.QContext`](../../api/pykx-execution/ctx.md#pykx.ctx.QContext) objects. These context objects have attributes for their members, which can either be sub-contexts or K objects. For example: +The below sections make use of what is known as "The Context Interface". In q, a context (known as a namespace when at the top level) is an organizational structure which is used to organize code into libraries/common utilities. For more information on contexts/namespaces in q please refer to [Chapter 12 of Q for Mortals](https://code.kx.com/q4m3/12_Workspace_Organization/). KDB-X Python exposes these contexts as special [`kx.QContext`](../../api/pykx-execution/ctx.md#pykx.ctx.QContext) objects. These context objects have attributes for their members, which can either be sub-contexts or K objects. For example: * `#!python pykx.q.Q` is a KdbContext instance for the builtin `#!q .Q` context/namespace * `#!python pykx.q.ctxA.ctxB` is a KdbContext instance for the `#!q .ctxA.ctxB` context @@ -22,7 +22,7 @@ The below sections make use of what is known as "The Context Interface". In q, a ## Use the in-built q functionality -When you start a q process there are 4 namespaces loaded which provide useful functionality to users of PyKX. +When you start a q process there are 4 namespaces loaded which provide useful functionality to users of KDB-X Python. | **Namespace** | **Contents** | **Link** | | :------------ | :-------------------------------------------------------------- | :-------------------------------------- | @@ -48,7 +48,7 @@ As can be seen above, just as in q, the .q context is accessible at the top-leve ## Using executed q code Python first -Much of the code you write or need to access will come from executed code locally in your process or will be contained in scripts which you have access to. The flow chart below shows the hierarchy of search/loading that happens when PyKX cannot find a requested context. +Much of the code you write or need to access will come from executed code locally in your process or will be contained in scripts which you have access to. The flow chart below shows the hierarchy of search/loading that happens when KDB-X Python cannot find a requested context. ```mermaid graph LR @@ -116,7 +116,7 @@ If the namespace/context you are requesting doesn't exist in the `q` memory spac 15. `/opt/kdb/script.k` 16. `/opt/kdb/script.k_` -To show the script search logic in action you can first write a file to the `#!python kx.qhome` location used by PyKX containing a namespace matching the name of the script +To show the script search logic in action you can first write a file to the `#!python kx.qhome` location used by KDB-X Python containing a namespace matching the name of the script ```python >>> demo_extension_source = ''' @@ -170,7 +170,7 @@ c| 0.9830794 4.433333 0 ')) ``` -## Extend where PyKX searches for scripts +## Extend where KDB-X Python searches for scripts In addition to the default search locations you can add additional locations to be searched through appending of additional search paths to the `kx.q.paths` list which is used in the search. @@ -188,7 +188,7 @@ Traceback (most recent call last): ``` -If PyKX fails to find a script an `#!python AttributeError` will be raised, the expanding section below provides an example of this +If KDB-X Python fails to find a script an `#!python AttributeError` will be raised, the expanding section below provides an example of this ??? Note "Failed to find a script" @@ -249,7 +249,7 @@ The following shows an example of the retrieval of a function from a context vs ## Use Contexts via IPC -The context interface is also supported against remote processes thus allowing you to run analytic operations Python first against a remote kdb+/q server. The syntax and operational restrictions outlined in the previous sections also exist for the IPC instance which you can call as follows +The context interface is also supported against remote processes thus allowing you to run analytic operations Python first against a remote KDB-X/q server. The syntax and operational restrictions outlined in the previous sections also exist for the IPC instance which you can call as follows ```python >>> with kx.SyncQConnection(port=5050) as conn: diff --git a/docs/user-guide/advanced/database/db_gen.md b/docs/user-guide/advanced/database/db_gen.md index 4c1fa53..99cc903 100644 --- a/docs/user-guide/advanced/database/db_gen.md +++ b/docs/user-guide/advanced/database/db_gen.md @@ -1,19 +1,19 @@ --- title: Generating and extending a database -description: Introduction to the PyKX database creation and management functionality +description: Introduction to the KDB-X Python database creation and management functionality date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, database, maintenance, management, generation +tags: KDB-X Python, q, database, maintenance, management, generation --- # Generate and extend a database -_This page explains how to create and expand databases using PyKX._ +_This page explains how to create and expand databases using KDB-X Python._ -!!! tip "Tip: For the best experience, we recommend reading [Databases in PyKX](index.md) first. If you already have access to a database and only need to load it, you can skip this page and jump right to [load database](db_loading.md)." +!!! tip "Tip: For the best experience, we recommend reading [Databases in KDB-X Python](index.md) first. If you already have access to a database and only need to load it, you can skip this page and jump right to [load database](db_loading.md)." -Before leveraging the performance of PyKX when querying on-disk data, you need to create a [persisted database](../../../extras/glossary.md#persisted-database). In the following sections we complete the following: +Before leveraging the performance of KDB-X Python when querying on-disk data, you need to create a [persisted database](../../../extras/glossary.md#persisted-database). In the following sections we complete the following: 1. [Create a new database](#1-create-database) containing a single table `#!python trade` and multiple days of data. 1. [Add a new day worth of data](#2-add-new-database-partition) for `#!python today` to the database for the `#!python trade` table. @@ -22,11 +22,11 @@ Before leveraging the performance of PyKX when querying on-disk data, you need t !!! note "Bring your own data" - The below example makes use of randomly-generated data using PyKX, where we use `#!python trade` or `#!python quote` tables generated in that manner. You can replace them with an equivalent Pandas/PyArrow table which will be converted to a PyKX table before being persisted. + The below example makes use of randomly-generated data using KDB-X Python, where we use `#!python trade` or `#!python quote` tables generated in that manner. You can replace them with an equivalent Pandas/PyArrow table which will be converted to a `pykx` table before being persisted. ## 1. Create database -For more information on database structures, see the linked section on [what is a database](index.md#whats-a-pykx-database). With PyKX, use the `#!python pykx.DB` class for all database interactions in Python. This class lets you create, expand, and maintain on-disk splayed/partitioned databases. First, we need to create a database. +For more information on database structures, see the linked section on [what is a database](index.md#whats-a-pykx-database). With KDB-X Python, use the `#!python pykx.DB` class for all database interactions in Python. This class lets you create, expand, and maintain on-disk splayed/partitioned databases. First, we need to create a database. In the next cell, we create a `#!python trade` table with data from multiple days in the chat. @@ -115,7 +115,7 @@ Note that in comparison to the original database creation logic, we do not have === "Encrypt persisted data" - In the below example, we encrypt the data persisted for the added partition. For further details on how encryption works within PyKX see [here](../compress-encrypt.md) or look at the API reference [here](../../../api/compress.md). + In the below example, we encrypt the data persisted for the added partition. For further details on how encryption works within KDB-X Python see [here](../compress-encrypt.md) or look at the API reference [here](../../../api/compress.md). ```python >>> encrypt = kx.Encrypt('/path/to/mykey.key', 'mySuperSecretPassword') diff --git a/docs/user-guide/advanced/database/db_loading.md b/docs/user-guide/advanced/database/db_loading.md index 89b7e59..dcd82c8 100644 --- a/docs/user-guide/advanced/database/db_loading.md +++ b/docs/user-guide/advanced/database/db_loading.md @@ -3,16 +3,16 @@ title: Load an existing database description: How to load an existing database into a Python process date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, database, loading +tags: KDB-X Python, q, database, loading --- # Load an existing database _This page explains how to load an existing database into a Python process._ -!!! tip "Tip: For the best experience, we recommend reading [Databases in PyKX](index.md) and [generate a database](db_gen.md) first." +!!! tip "Tip: For the best experience, we recommend reading [Databases in KDB-X Python](index.md) and [generate a database](db_gen.md) first." -By default, you can only load one database into a Python process when using PyKX. To automatically load a database when initializing the `#!python pykx.DB` class, set the database location as the path: +By default, you can only load one database into a Python process when using KDB-X Python. To automatically load a database when initializing the `#!python pykx.DB` class, set the database location as the path: ```python >>> import pykx as kx diff --git a/docs/user-guide/advanced/database/db_mgmt.md b/docs/user-guide/advanced/database/db_mgmt.md index 3df2f14..37dc282 100644 --- a/docs/user-guide/advanced/database/db_mgmt.md +++ b/docs/user-guide/advanced/database/db_mgmt.md @@ -1,18 +1,18 @@ --- -title: Manage a PyKX Database +title: Manage a KDB-X Python Database description: How to modify an existing database date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, database, loading +tags: KDB-X Python, q, database, loading --- # Manage a database -_This page explains how to modify databases generated in PyKX._ +_This page explains how to modify databases generated in KDB-X Python._ -!!! tip "Tip: For the best experience, we recommend reading [Databases in PyKX](index.md), [Generate a database](db_gen.md) and [Load a database](db_loading.md) first." +!!! tip "Tip: For the best experience, we recommend reading [Databases in KDB-X Python](index.md), [Generate a database](db_gen.md) and [Load a database](db_loading.md) first." -With PyKX, you can use various methods to modify your on-disk database. These changes can take many forms: +With KDB-X Python, you can use various methods to modify your on-disk database. These changes can take many forms: - Add new columns to the database - Apply functions to existing columns diff --git a/docs/user-guide/advanced/database/index.md b/docs/user-guide/advanced/database/index.md index e7d474d..14907eb 100644 --- a/docs/user-guide/advanced/database/index.md +++ b/docs/user-guide/advanced/database/index.md @@ -1,22 +1,22 @@ --- -title: Databases in PyKX -description: PyKX database creation and management +title: Databases in KDB-X Python +description: KDB-X Python database creation and management date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, database, maintenance, management, generation +tags: KDB-X Python, q, database, maintenance, management, generation --- -# Databases in PyKX +# Databases in KDB-X Python -_This page explains the concept of databases in PyKX, including the creation and management of databases._ +_This page explains the concept of databases in KDB-X Python, including the creation and management of databases._ -## What's a PyKX database? +## What's a KDB-X Python database? -In PyKX, the term database refers to a kdb+ database which can hold a set of [splayed](https://code.kx.com/q/kb/splayed-tables/) and [partitioned](https://code.kx.com/q/kb/partition/) tables. +In KDB-X Python, the term database refers to a KDB-X database which can hold a set of [splayed](https://code.kx.com/q/kb/splayed-tables/) and [partitioned](https://code.kx.com/q/kb/partition/) tables. ### Splayed Database -A splayed kdb+ database consists of a single table stored on-disk with each column stored as a separate file rather than using a single file for the whole table. Tables of medium-size with < 100 million rows and many columns are good candidates for being stored as splayed tables, in particular when only a small subset of columns are being accessed often. +A splayed KDB-X database consists of a single table stored on-disk with each column stored as a separate file rather than using a single file for the whole table. Tables of medium-size with < 100 million rows and many columns are good candidates for being stored as splayed tables, in particular when only a small subset of columns are being accessed often. ```bash quotes @@ -28,7 +28,7 @@ quotes !!! note "More information on splayed databases" - The splayed database format used by PyKX has been used in production environments for decades. As such there is a significant amount of information available on the creation and use of these databases. Below are some articles. + The splayed database format used by KDB-X Python has been used in production environments for decades. As such there is a significant amount of information available on the creation and use of these databases. Below are some articles. - [q knowledge base splayed databases](https://code.kx.com/q/kb/splayed-tables/) - [Q for Mortals splayed tables](https://code.kx.com/q4m3/14_Introduction_to_Kdb%2B/#142-splayed-tables) @@ -36,7 +36,7 @@ quotes ### Partitioned Database -A partitioned kdb+ database consists of one or more tables saved on-disk, where they are split into separate folders called partitions. These partitions are most often based on a temporal field within the dataset, such as date or month. Each table within the database must follow the same partition structure. +A partitioned KDB-X database consists of one or more tables saved on-disk, where they are split into separate folders called partitions. These partitions are most often based on a temporal field within the dataset, such as date or month. Each table within the database must follow the same partition structure. A visual representation of a database containing 2 tables (trade and quote) partitioned by date would be as follows, where `#!python price`, `#!python sym`, `#!python time` in the quotes folder are columns within the table: @@ -62,17 +62,17 @@ db !!! note "More information on partitioned databases" - The partitioned database format used by PyKX has been used in production environments for decades in many of the world's best-performing tier-1 investment banks. Today, there is a significant amount of information available on the creation and maintenance of these databases. Below are some articles related to their creation and querying. + The partitioned database format used by KDB-X Python has been used in production environments for decades in many of the world's best-performing tier-1 investment banks. Today, there is a significant amount of information available on the creation and maintenance of these databases. Below are some articles related to their creation and querying. - - [Blog: Partitioning data with kdb+](https://kx.com/blog/partitioning-data-in-kdb/) + - [Blog: Partitioning data with KDB-X](https://kx.com/blog/partitioning-data-in-kdb/) - [Q for Mortals Partitioned Tables](https://code.kx.com/q4m3/14_Introduction_to_Kdb%2B/#143-partitioned-tables) - [Partitioned Tables](https://thinqkdb.wordpress.com/partitioned-tables/) -## How to use databases in PyKX +## How to use databases in KDB-X Python Creating and managing databases is crucial for handling large amounts of data. The `#!python pykx.DB` module helps make these tasks easier, Pythonic, and more user-friendly. -PyKX Database API supports the following operations: +KDB-X Python Database API supports the following operations: | **Operation** | **Description** | |:-------------------------|:--------------------------------------------------------------------------------------------------| diff --git a/docs/user-guide/advanced/examples/streamlit.py b/docs/user-guide/advanced/examples/streamlit.py index 5b881da..3acb528 100644 --- a/docs/user-guide/advanced/examples/streamlit.py +++ b/docs/user-guide/advanced/examples/streamlit.py @@ -12,7 +12,7 @@ def main(): - st.header('PyKX Demonstration') + st.header('KDB-X Python Demonstration') connection = st.connection('pykx', type=kx.streamlit.PyKXConnection, port=5050, diff --git a/docs/user-guide/advanced/ipc.md b/docs/user-guide/advanced/ipc.md index 182a53b..bffed89 100644 --- a/docs/user-guide/advanced/ipc.md +++ b/docs/user-guide/advanced/ipc.md @@ -1,36 +1,36 @@ --- title: Communicate via IPC -description: Use PyKX via IPC +description: Use KDB-X Python via IPC date: June 2024 author: KX Systems, Inc., -tags: PyKX, IPC, +tags: KDB-X Python, IPC, --- # Communicate via IPC -_This page explains how to use PyKX to communicate with q processes via IPC._ +_This page explains how to use KDB-X Python to communicate with q processes via IPC._ -Interprocess Communication (IPC) forms a central mechanism by which you can connect to and query existing kdb+/q infrastructures. +Interprocess Communication (IPC) forms a central mechanism by which you can connect to and query existing KDB-X/q infrastructures. The processes to which users are connecting and running queries often connect into a central server/gateway that contains vast amounts of historical data. -There are 4 main types of IPC connections in PyKX. +There are 4 main types of IPC connections in KDB-X Python. | **Connection Name** | **When it's often used** | | :-------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------- | | [`kx.SyncQConnection`](../../api/ipc.md#pykx.ipc.SyncQConnection) | When you need to retrieve data from a server. | | [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection) | When you need to integrate with Python's `asyncio` library or when integration running queries on an event loop. | -| [`kx.SecureQConnection`](../../api/ipc.md#pykx.ipc.SecureQConnection) | When you need to connect to a kdb+/q server which has TLS enabled. | +| [`kx.SecureQConnection`](../../api/ipc.md#pykx.ipc.SecureQConnection) | When you need to connect to a KDB-X/q server which has TLS enabled. | | [`kx.RawQConnection`](../../api/ipc.md#pykx.ipc.RawQConnection) | Used when more fine-grained control is required by a user to handle when messages are read, also used if emulating a q server from Python. | In the below sections you will learn more about these connections and how to -- Establish a connection to an existing kdb+/q process -- Run analytics/queries on existing kdb+/q processes +- Establish a connection to an existing KDB-X/q process +- Run analytics/queries on existing KDB-X/q processes - Reconnect to a process - Execute a local file - Integrate with Python asynchronous frameworks -- Create your own IPC Server using PyKX +- Create your own IPC Server using KDB-X Python !!! Note "To run the examples" @@ -225,7 +225,7 @@ For more information on the context interface and how to use your q code Python ### Run a local Python function on a server -While not explicitly part of the IPC module of PyKX the ability to run your local Python functions on remote servers makes use of the IPC logic provided by PyKX heavily. Outlined in full detail [here](remote-functions.md), this functionality works by sending across to your server instructions to import relevant libraries, evaluate the function being run and pass data to this function for execution. +While not explicitly part of the IPC module of KDB-X Python the ability to run your local Python functions on remote servers makes use of the IPC logic provided by KDB-X Python heavily. Outlined in full detail [here](remote-functions.md), this functionality works by sending across to your server instructions to import relevant libraries, evaluate the function being run and pass data to this function for execution. In the examples below we can see the registration and use of these functions in practice where the `#!python kx.remote.session` objects are a form of IPC connection. In each case the function is defined in your local session but executed remotely: @@ -265,11 +265,11 @@ In the examples below we can see the registration and use of these functions in pykx.LongAtom(pykx.q('30')) ``` -## Reconnect to a kdb+ server +## Reconnect to a KDB-X server When a server with active connections becomes unavailable, restarts, or suffers an outage, all active connections will need to reconnect whenever the server recovers. This could mean closing an existing stale connection and reconnecting using the same credentials. -PyKX allows you to manually configure reconnection attempts for clients connecting to servers using the #!python reconnection_attempts keyword argument. The following example shows the output of when attempting to make use of a connection which has been cancelled and is subsequently re-established: +KDB-X Python allows you to manually configure reconnection attempts for clients connecting to servers using the #!python reconnection_attempts keyword argument. The following example shows the output of when attempting to make use of a connection which has been cancelled and is subsequently re-established: ```python >>> conn = kx.SyncQConnection(port=5050, reconnection_attempts=5) @@ -364,7 +364,7 @@ pykx.Identity(pykx.q('::')) ### Integrate with Python Async libraries -To make integrate with Python's async libraries such as `#!python asyncio` with `#!python PyKX`, you must use a [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection). When calling an instance of an [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection), the query is sent to the `#!python q` server and control is immediately handed back to the running Python program. The `#!python __call__` function returns a [`kx.QFuture`](../../api/ipc.md#pykx.ipc.QFuture) instance that can later be awaited on to block until it receives a result. +To make integrate with Python's async libraries such as `#!python asyncio` with `#!python KDB-X Python`, you must use a [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection). When calling an instance of an [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection), the query is sent to the `#!python q` server and control is immediately handed back to the running Python program. The `#!python __call__` function returns a [`kx.QFuture`](../../api/ipc.md#pykx.ipc.QFuture) instance that can later be awaited on to block until it receives a result. If you're using a third-party library that runs an eventloop to manage asynchronous calls, ensure you use the `#!python event_loop` keyword argument to pass the event loop into the [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection) instance. This allows the eventloop to properly manage the returned [`kx.QFuture`](../../api/ipc.md#pykx.ipc.QFuture) objects and its lifecycle. @@ -382,9 +382,9 @@ async with kx.AsyncQConnection('localhost', 5001, event_loop=asyncio.get_event_l await fut # await the future object to get the result ``` -## Create your own IPC Server using PyKX +## Create your own IPC Server using KDB-X Python -There are several cases where providing the ability for users to open IPC connections to Python processes via the q native IPC protocol provides advantages. In particular if you are looking to manage infrastructure in Python which kdb+ users are likely to communicate with using q. +There are several cases where providing the ability for users to open IPC connections to Python processes via the q native IPC protocol provides advantages. In particular if you are looking to manage infrastructure in Python which KDB-X users are likely to communicate with using q. The [`server.py`](scripts/server.py) file that you may have called at the start of this page makes use of this functionality and specifically uses a [`kx.RawQConnection`](../../api/ipc.md#pykx.ipc.RawQConnection) to allow connections to be made, this script is defined in plain text as follows: @@ -416,7 +416,7 @@ def qval_async(query): async def main(): kx.q.z.pg = qval_sync kx.q.z.ps = qval_async - kx.q('@[system"l ",;"s.k_";{show "Failed to load SQL"}]') + kx.q('@[value;"s) ";{show "Failed to load SQL"}]') kx.q('tab:([]1000?`a`b`c;1000?1f;1000?10)') async with kx.RawQConnection(port=port, as_server=True, conn_gc_time=20.0) as q: print('Server Initialized') diff --git a/docs/user-guide/advanced/license.md b/docs/user-guide/advanced/license.md index 4c5d77c..ac09ae9 100644 --- a/docs/user-guide/advanced/license.md +++ b/docs/user-guide/advanced/license.md @@ -3,23 +3,23 @@ title: Manage your license description: Tips and tricks for managing licenses date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, license, licenses +tags: KDB-X Python, q, license, licenses --- # Manage your license _Tips and tricks for managing licenses_ -In the [install page](../../getting-started/installing.md) you can follow along with how to install your first license, this is an important first step for you to get the most out of the functionality provided by PyKX. However, there are a number of cases where upgrading this license will be necessary: +In the [install page](../../getting-started/installing.md) you can follow along with how to install your first license, this is an important first step for you to get the most out of the functionality provided by KDB-X Python. However, there are a number of cases where upgrading this license will be necessary: 1. Your license has expired 2. You need to upgrade from a personal to enterprise/commercial license ## When will your license expire? -For personal edition licenses your license will expire one year after initial download, for enterprise/commercial licenses the expiry date will vary. To provide some forewarning of when your license will expire PyKX provides the following: +To provide some forewarning of when your license will expire KDB-X Python provides the following: -- For the 10 days before expiry PyKX will print a message at start up indicating the license will expire in N days +- For the 10 days before expiry KDB-X Python will print a message at start up indicating the license will expire in N days - The utility function `#!python kx.license.expires` provides a programmatic method of finding days to expiry. As an example the following shows you both in action: @@ -28,7 +28,7 @@ As an example the following shows you both in action: ```python >>> import pykx as kx - WARNING:root:PyKX license set to expire in 8 days, please consider installing an updated license + WARNING:root:KDB-X Python license set to expire in 8 days, please consider installing an updated license ``` === "Checking expiry" @@ -43,18 +43,18 @@ As an example the following shows you both in action: When your license is expired/expiring you will need to update it to ensure you can continue to use the software. -There are three methods by which updating your license is possible with PyKX. +There are three methods by which updating your license is possible with KDB-X Python. -- You have allowed your license to expire and on restart of PyKX you will be presented with an new license install walkthrough similar to [installing](../../getting-started/installing.md). -- You pre-emptively install a newly downloaded license using `#!python kx.license.install`. +- You have allowed your license to expire and on restart of KDB-X Python you will be presented with an new license install walkthrough similar to [installing](../../getting-started/installing.md). +- You preemptively install a newly downloaded license using `#!python kx.license.install`. === "After Expiry" - Now that your license has expired importing PyKX will result in the following walkthrough being presented, following this will allow you to install a new license. + Now that your license has expired importing KDB-X Python will result in the following walkthrough being presented, following this will allow you to install a new license. ```python >>> import pykx as kx - Your PyKX license has now expired. + Your KDB-X Python license has now expired. Captured output from initialization attempt: '2023.10.18T13:27:59.719 licence error: exp @@ -64,7 +64,7 @@ There are three methods by which updating your license is possible with PyKX. Would you like to renew your license? [Y/n]: Y - Do you have access to an existing license for PyKX that you would like to use? [N/y]: + Do you have access to an existing license for KDB-X Python that you would like to use? [N/y]: ``` === "Pre-emptive install" @@ -90,9 +90,9 @@ There are three methods by which updating your license is possible with PyKX. ... force=True) ``` -## Upgrade to a commercial license +## Upgrade to a `k4.lic` from a `kc.lic` license -If you are currently using a `kc.lic` personal license and need to upgrade to a `k4.lic` license the following steps allow you to ensure this can be done effectively. +If you are currently using a `kc.lic` license and need to change to a `k4.lic` license the following steps allow you to ensure this can be done effectively. 1. Delete your existing `kc.lic` license. 2. Install your new license following the [license installation](../../getting-started/installing.md) diff --git a/docs/user-guide/advanced/modes.md b/docs/user-guide/advanced/modes.md index fafa356..ebaa73f 100644 --- a/docs/user-guide/advanced/modes.md +++ b/docs/user-guide/advanced/modes.md @@ -1,30 +1,30 @@ --- -title: PyKX modes of operation -description: Operating PyKX in Python and q +title: KDB-X Python modes of operation +description: Operating KDB-X Python in Python and q date: June 2024 author: KX Systems, Inc., -tags: PyKX, q language, Python, licensed, unlincensed, +tags: KDB-X Python, q language, Python, licensed, unlincensed, --- # Modes of operation -_This page explains how to operate PyKX in Python and q, with or without a KDB Insights license._ +_This page explains how to operate KDB-X Python, with or without a license._ -PyKX can operate in different modes. Each mode has its limitations and requirements, so we're breaking them down into the following: +KDB-X Python can operate in different modes. Each mode has its limitations and requirements, so we're breaking them down into the following: -1. PyKX within Python +1. KDB-X Python within Python - 1.a Unlicensed - 1.b Licensed -2. PyKX within q with a kdb Insights license +2. KDB-X Python within q - licensed -## 1. PyKX within Python +## 1. KDB-X Python within Python -The purpose of operating PyKX within a Python session is to replace [qPython](https://github.com/exxeleron/qPython) and [PyQ](https://github.com/kxsystems/pyq). Within Python, PyKX has two modes of operation: +The purpose of operating KDB-X Python within a Python session is to replace [qPython](https://github.com/exxeleron/qPython) and [PyQ](https://github.com/kxsystems/pyq). Within Python, KDB-X Python has two modes of operation: -- `#!python Licensed` (this means you have a kdb Insights license with PyKX enabled) -- `#!python Unlicensed` (this means you don't have a kdb Insights license or a license in which PyKX is not enabled) +- `#!python Licensed` (this means you have a license with KDB-X Python enabled) +- `#!python Unlicensed` (this means you don't have a license or a license in which KDB-X Python is not enabled) -The main difference between the two is that the `#!python Unlicensed` mode is for IPC-based communication. This mean that it allows to create IPC connections and convert data from Pythonic representations to PyKX objects. +The main difference between the two is that the `#!python Unlicensed` mode is for IPC-based communication. This mean that it allows to create IPC connections and convert data from Pythonic representations to `pykx` objects. The following table outlines more key differences: @@ -33,19 +33,19 @@ The following table outlines more key differences: | Convert objects from q to Pythonic types and vice-versa | :material-check: | :material-check: | | Query synchronously and asynchronously a q server via IPC | :material-check: | :material-check: | | Query synchronously and asynchronously a q server with TLS enabled | :material-check: | :material-close: | -| Interact with PyKX tables via a Pandas like API | :material-check: | :material-close: | +| Interact with `pykx` tables via a Pandas like API | :material-check: | :material-close: | | Run arbitrary q code within a Python session | :material-check: | :material-close: | -| Display PyKX/q objects within a Python session | :material-check: | :material-close: | -| Load kdb+ Databases within a Python session | :material-check: | :material-close: | +| Display `pykx`/q objects within a Python session | :material-check: | :material-close: | +| Load KDB-X Databases within a Python session | :material-check: | :material-close: | | Read/write JSON, CSV and q formats to/from disk | :material-check: | :material-close: | | Access to Python classes for SQL, schema creation, custom data conversion | :material-check: | :material-close: | -| Run Python within a q session using PyKX under q | :material-check: | :material-close: | +| Run Python within a q session using KDB-X Python under q | :material-check: | :material-close: | | Full support for nulls, infinities, data slicing and casting | :material-check: | :material-close: | | Production support | :material-check: | :material-close: | ### 1.a Running in Unlicensed mode -Unlicensed mode is a feature-limited mode of operation for PyKX. Its aim is to replace qPython, which has the benefit of not requiring a valid q license (except for the q license required to run the remote q process that PyKX connects to in this mode). +Unlicensed mode is a feature-limited mode of operation for KDB-X Python. Its aim is to replace qPython, which has the benefit of not requiring a valid q license (except for the q license required to run the remote q process that KDB-X Python connects to in this mode). This mode cannot run q embedded within it. Also, it lacks the ability to run q code within the local Python process or any functionality that depends on running q code. Despite this limitation, it provides the following features (which are all also available in licensed mode): @@ -55,12 +55,12 @@ This mode cannot run q embedded within it. Also, it lacks the ability to run q c ### 1.b Running in Licensed mode -Licensed mode is the standard way to operate PyKX, wherein it's running under a Python process [with a valid q license](../../getting-started/installing.md#2-install-a-kdb-insights-license). This modality aims to replace PyQ as the Python-first library for KX. All PyKX features are available in this mode. +Licensed mode is the standard way to operate KDB-X Python, wherein it's running under a Python process [with a valid q license](../../getting-started/installing.md#2-install-a-license). This modality aims to replace PyQ as the Python-first library for KX. All KDB-X Python features are available in this mode. -The differences provided through operating with a valid kdb Insights license are: +The differences provided through operating with a valid license are: -1. You can execute PyKX/q functionalities directly within a Python session. -2. PyKX objects can be represented in a human readable format rather than as a memory address, namely: +1. You can execute KDB-X Python/q functionalities directly within a Python session. +2. `pykx` objects can be represented in a human readable format rather than as a memory address, namely: === "Licensed mode" @@ -82,7 +82,7 @@ The differences provided through operating with a valid kdb Insights license are pykx.Table._from_addr(0x7f5b72ef8860) ``` -3. You can analyze PyKX objects through indexing: +3. You can analyze `pykx` objects through indexing: === "Licensed mode" @@ -107,7 +107,7 @@ The differences provided through operating with a valid kdb Insights license are pykx.exceptions.LicenseException: A valid q license must be in a known location (e.g. `$QLIC`) to index into K object. ``` -4. Licensed mode allows users to cast between PyKX object types. Unlicensed mode doesn't support this, showing an error as below: +4. Licensed mode allows users to cast between `pykx` object types. Unlicensed mode doesn't support this, showing an error as below: === "Licensed mode" @@ -141,7 +141,7 @@ The differences provided through operating with a valid kdb Insights license are 3. kx.q.write 4. kx.q.schema 5. kx.q.console -6. [Pandas API](Pandas_API.ipynb) functionality for interactions with and PyKX Table objects. +6. [Pandas API](Pandas_API.ipynb) functionality for interactions with and `pykx` Table objects. 6. You can convert keyed tables to equivalent Numpy types. 7. All types can be disambiguated, generic null can be discerned from a projection null, and similar for regular vs splayed tables. 8. Numpy list object conversion is optimized only in licensed mode. @@ -174,24 +174,24 @@ The differences provided through operating with a valid kdb Insights license are ### How to choose between Licensed and Unlicensed -You can choose to initialise PyKX under one of these modes through the use of the `#!python QARGS` environment variable as follows: +You can choose to initialise KDB-X Python under one of these modes through the use of the `#!python QARGS` environment variable as follows: | **Mode argument** | **Description** | | :---------------- | :--------------------------------------------------------------------------------------------------------------------------------------- | -| `--unlicensed` | Starts PyKX in unlicensed mode. No license check is performed, and no warning is emitted at start-up if embedded q initialization fails. | +| `--unlicensed` | Starts KDB-X Python in unlicensed mode. No license check is performed, and no warning is emitted at start-up if embedded q initialization fails. | | `--licensed` | Raises a `PyKXException` (as opposed to emitting a `PyKXWarning`) if embedded q initialization fails. | -In addition to the PyKX specific start-up arguments, you can also use `#!python QARGS` to set the standard [q command-line arguments](https://code.kx.com/q/basics/cmdline/). +In addition to the KDB-X Python specific start-up arguments, you can also use `#!python QARGS` to set the standard [q command-line arguments](https://code.kx.com/q/basics/cmdline/). -Alternatively, if you wish to access PyKX in unlicensed mode, you set the environment variable `#!python PYKX_UNLICENSED="true"` or define this in your `#!python .pykx-config` file as outlined [here](../configuration.md). +Alternatively, if you wish to access KDB-X Python in unlicensed mode, you set the environment variable `#!python PYKX_UNLICENSED="true"` or define this in your `#!python config-pykx` file as outlined [here](../configuration.md). -## 2. PyKX within q +## 2. KDB-X Python within q -Fully described [here](../../pykx-under-q/intro.md), the ability to use PyKX within a q session allows you to achieve the following: +Fully described [here](../../pykx-under-q/intro.md), the ability to use KDB-X Python within a q session allows you to achieve the following: - Replace [embedPy](https://github.com/kxsystems/embedpy) functionally with an updated, more flexible interface. -- Use Python within a q environment without the [limitations for PyKX](../../help/issues.md). -- Use Python code in conjunction with timers and subscriptions within a q/kdb+ ecosystem. +- Use Python within a q environment without the [limitations for KDB-X Python](../../help/issues.md). +- Use Python code in conjunction with timers and subscriptions within a q/KDB-X ecosystem. -Similar to the use of PyKX in licensed mode, PyKX running under q requires a user to have access to an appropriate license containing the `#!python insights.lib.pykx` and `#!python insights.lib.embedq` licensing flags. +Similar to the use of KDB-X Python in licensed mode, KDB-X Python running under q requires a user to have access to an appropriate license containing the `insights.lib.pykx` and `insights.lib.embedq` (or `py` and `eq`) licensing flags. diff --git a/docs/user-guide/advanced/numpy.md b/docs/user-guide/advanced/numpy.md index c534d1e..27bbd3d 100644 --- a/docs/user-guide/advanced/numpy.md +++ b/docs/user-guide/advanced/numpy.md @@ -1,15 +1,15 @@ --- title: NumPy Integration -description: Integrate PyKX with NumPy +description: Integrate KDB-X Python with NumPy date: July 2024 author: KX Systems, Inc., -tags: PyKX, NumPy +tags: PyKX, KDB-X Python, NumPy --- # NumPy Integration -_This page explains how to integrate PyKX with NumPy._ +_This page explains how to integrate KDB-X Python with NumPy._ -PyKX is designed for advanced integration with NumPy. This integration is built on three pillars: +KDB-X Python is designed for advanced integration with NumPy. This integration is built on three pillars: - [NEP-49](https://numpy.org/neps/nep-0049-data-allocation-strategies.html) - the NumPy [array interface](https://numpy.org/doc/stable/reference/arrays.interface.html) @@ -17,24 +17,25 @@ PyKX is designed for advanced integration with NumPy. This integration is built ## Support for NEP-49 and 0-copy data transfer from Numpy to q (when possible) -To use NEP-49 and benefit from 0-copy data transfers from NumPy to q, you need to set the `#!python PYKX_ALLOCATOR=1` environment variable before importing PyKX. -Once enabled, PyKX leverages NEP-49 to replace NumPy's memory allocator with the q/k memory allocator. This makes NumPy arrays directly available to q (by passing only a pointer) and accelerates the conversion time from NumPy arrays to q significantly. +NEP-49 is used to benefit from 0-copy data transfers from NumPy to q. +NEP-49 replaces NumPy's memory allocator with the q/k memory allocator. This makes NumPy arrays directly available to q (by passing only a pointer) and accelerates the conversion time from NumPy arrays to q significantly. +With NEP-49 (default) -Without NEP-49 (`#!python PYKX_ALLOCATOR=0`): ```python In [1]: arr = np.random.rand(1000000) In [2]: %timeit kx.toq(arr) -421 µs ± 9.42 µs per loop (mean ± std. dev. of 7 runs, 1,000 loops each) +5.4 µs ± 150 ns per loop (mean ± std. dev. of 7 runs, 100,000 loops each) ``` -With NEP-49 (`#!python PYKX_ALLOCATOR=1`): +Without NEP-49 (`#!python PYKX_NO_ALLOCATOR=True`): + ```python In [1]: arr = np.random.rand(1000000) In [2]: %timeit kx.toq(arr) -5.4 µs ± 150 ns per loop (mean ± std. dev. of 7 runs, 100,000 loops each) +421 µs ± 9.42 µs per loop (mean ± std. dev. of 7 runs, 1,000 loops each) ``` -In the example above, transferring a NumPy array of one million `#!python float64` numbers runs 80x faster with NEP-49 enabled (`#!python PYKX_ALLOCATOR=1`). +In the example above, transferring a NumPy array of one million `#!python float64` numbers runs 80x faster with NEP-49 enabled. !!! Note @@ -44,7 +45,7 @@ In the example above, transferring a NumPy array of one million `#!python float6 ## Support for NumPy array interface and universal functions on pykx/q vectors -PyKX vectors implement the NumPy array interface and are compatible with universal functions. This means all those NumPy functions (and more) can be used directly on PyKX vectors and hence, on q vectors. +`pykx` vectors implement the NumPy array interface and are compatible with universal functions. This means all those NumPy functions (and more) can be used directly on `pykx` vectors and hence, on q vectors. Here are several helpful links related to universal functions that you can use with this: @@ -110,7 +111,7 @@ qb = kx.toq(b) We can use IPython to load this script and benchmark the different implementations with `#!python %timeit`. We will also compare to `#!python np.gcd`, the NumPy ufunc for GCD calculation. ```bash -$ PYKX_ALLOCATOR=1 ipython -i test_numpy_ufuncs.py +$ ipython -i test_numpy_ufuncs.py ``` ```python diff --git a/docs/user-guide/advanced/performance.md b/docs/user-guide/advanced/performance.md index 0db62c2..b031954 100644 --- a/docs/user-guide/advanced/performance.md +++ b/docs/user-guide/advanced/performance.md @@ -1,16 +1,16 @@ --- -title: PyKX Performance -description: How to optimize PyKX +title: KDB-X Python Performance +description: How to optimize KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, performance, parallelization, secondary q threads, multithreading, peach +tags: KDB-X Python, q, performance, parallelization, secondary q threads, multithreading, peach --- # Performance tips -_This page includes PyKX performance optimization tips, including insights on parallelization, secondary q threads, multithreading, and peach._ +_This page includes KDB-X Python performance optimization tips, including insights on parallelization, secondary q threads, multithreading, and peach._ -To get the best performance out of PyKX, follow these guidelines. Note that this page focuses on efficiently interfacing between Python and q, rather than optimizing Python or q individually. +To get the best performance out of KDB-X Python, follow these guidelines. Note that this page focuses on efficiently interfacing between Python and q, rather than optimizing Python or q individually. ## General guidelines @@ -37,11 +37,11 @@ To get the best performance out of PyKX, follow these guidelines. Note that this ## Parallelization Parallelization involves distributing computational tasks across multiple threads to improve performance and efficiency. -Use the following methods if you want to allow PyKX to handle large-scale data processing tasks efficiently by utilizing the available computational resources: secondary q threads, multithreading, or `#!python peach`. +Use the following methods if you want to allow KDB-X Python to handle large-scale data processing tasks efficiently by utilizing the available computational resources: secondary q threads, multithreading, or `#!python peach`. ### Secondary q threads -PyKX starts embedded q with as many secondary q threads enabled as are available. q automatically uses these threads to parallelize some computations as it deems appropriate. You can use the `#!python QARGS` environment variable to provide command-line arguments and other startup flags to q/PyKX, including the number of secondary threads: +KDB-X Python starts embedded q with as many secondary q threads enabled as are available. q automatically uses these threads to parallelize some computations as it deems appropriate. You can use the `#!python QARGS` environment variable to provide command-line arguments and other startup flags to q/KDB-X Python, including the number of secondary threads: ```sh QARGS='-s 0' python # disable secondary threads @@ -51,14 +51,14 @@ QARGS='-s 0' python # disable secondary threads QARGS='-s 12' python # use 12 secondary threads by default ``` -- The value set using `#!python -s` sets both the default and the maximum available to the process; you can't change it after importing PyKX. +- The value set using `#!python -s` sets both the default and the maximum available to the process; you can't change it after importing KDB-X Python. - `#!python pykx.q.system.max_num_threads` shows the maximum number of threads and cannot be changed. - `#!python pykx.q.system.num_threads` shows the current number of threads in use. It starts at the maximum value but can be set to a lower number. ### Multithreading -By default, PyKX doesn’t support calling q from multiple threads in a Python process due to the Global Interpreter Lock [GIL](https://wiki.python.org/moin/GlobalInterpreterLock). Enabling the `#!python PYKX_RELEASE_GIL` environment variable drops the GIL when calling q, making it unsafe to call q from multiple threads. To ensure thread safety, you can also enable the `#!python PYKX_Q_LOCK` environment variable, which adds a re-entrant lock around q. Learn [how to enable multithreaded execution](threading.md) and set up a Python process using PyKX to [call into EmbeddedQ from multiple threads](../../examples/threaded_execution/threading.md) +By default, KDB-X Python doesn’t support calling q from multiple threads in a Python process due to the Global Interpreter Lock [GIL](https://wiki.python.org/moin/GlobalInterpreterLock). Enabling the `#!python PYKX_RELEASE_GIL` environment variable drops the GIL when calling q, making it unsafe to call q from multiple threads. To ensure thread safety, you can also enable the `#!python PYKX_Q_LOCK` environment variable, which adds a re-entrant lock around q. Learn [how to enable multithreaded execution](threading.md) and set up a Python process using KDB-X Python to [call into EmbeddedQ from multiple threads](../../examples/threaded_execution/threading.md) ### Peach diff --git a/docs/user-guide/advanced/remote-functions.md b/docs/user-guide/advanced/remote-functions.md index 4fe83dc..98a9e2e 100644 --- a/docs/user-guide/advanced/remote-functions.md +++ b/docs/user-guide/advanced/remote-functions.md @@ -1,26 +1,26 @@ --- -title: PyKX Remote Functions -description: How to execute Python functions on q servers in PyKX +title: KDB-X Python Remote Functions +description: How to execute Python functions on q servers in KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, performance, parallelization, secondary q threads, multithreading, peach +tags: KDB-X Python, q, performance, parallelization, secondary q threads, multithreading, peach --- # Remote Function Execution -_This page explains how to execute Python functions on q servers in PyKX._ +_This page explains how to execute Python functions on q servers in KDB-X Python._ -Remote Functions let you define Python functions within your Python environment which can interact with kdb+ data on a q process. Once defined, these functions are registered to a [remote session object](../../api/remote.md) along with any Python dependencies which need to be imported. The [remote session object](../../api/remote.md) establishes and manages the remote connection to the kdb+/q server. +Remote Functions let you define Python functions within your Python environment which can interact with KDB-X data on a q process. Once defined, these functions are registered to a [remote session object](../../api/remote.md) along with any Python dependencies which need to be imported. The [remote session object](../../api/remote.md) establishes and manages the remote connection to the KDB-X/q server. -To execute kdb+/q functions using PyKX, go to [PyKX under q](../../pykx-under-q/intro.md) +To execute KDB-X/q functions using KDB-X Python, go to [KDB-X Python under q](../../pykx-under-q/intro.md) ## Requirements and limitations Before you start: - Make sure all necessary Python requirements are installed on the client server. For this functionality you need `#!python dill>=0.2`. -- Confirm that the kdb+/q server you connect to can load PyKX under q. -- Ensure that you have the correct versions of Python library dependencies in your kdb+/q environment at runtime. +- Confirm that the KDB-X/q server you connect to can load KDB-X Python under q. +- Ensure that you have the correct versions of Python library dependencies in your KDB-X/q environment at runtime. - Run the following command: ```bash @@ -31,17 +31,17 @@ pip install pykx[remote] This walkthrough demonstrates the following steps: -1. Initialize a q/kdb+ server loading PyKX under q on a specified port. -1. Import PyKX and generate a remote session object which denotes the process against which the Python functions will be executed. -1. Define a number of Python functions which will be executed on the remote q/kdb+ server. +1. Initialize a q/KDB-X server loading KDB-X Python under q on a specified port. +1. Import KDB-X Python and generate a remote session object which denotes the process against which the Python functions will be executed. +1. Define a number of Python functions which will be executed on the remote q/KDB-X server. -### Initializea q/kdb+ server with PyKX under q +### Initialize a q/KDB-X server with KDB-X Python under q -This step ensures you have a q process running with PyKX under q, as well as having a kdb+ table available to query. If you have this already, proceed to the next step. +This step ensures you have a q process running with KDB-X Python under q, as well as having a KDB-X table available to query. If you have this already, proceed to the next step. -Ensure that you have q installed. If you do not have this installed please follow the guide provided [here](https://code.kx.com/q/learn/install/), retrieving your license following the instructions provided [here](https://kx.com/kdb-insights-sdk-personal-edition-download). +Ensure that you have q installed. If you do not have this installed please follow the guide provided [here](https://developer.kx.com/products/kdb-x/install). -Install PyKX under q using the following command. +Install KDB-X Python under q using the following command. ```bash python -c "import pykx;pykx.install_into_QHOME()" @@ -66,9 +66,9 @@ Set a requirement for users to provide a username/password if you wish to add se .z.pw:{[u;p]$[(u~`user)&p~`password;1b;0b]} ``` -### Import PyKX and create a session +### Import KDB-X Python and create a session -Create a session object from a Python environment of your choice, which establishes and manages the remote connection to the kdb+/q server. +Create a session object from a Python environment of your choice, which establishes and manages the remote connection to the KDB-X/q server. ```python >>> import pykx as kx diff --git a/docs/user-guide/advanced/serialization.md b/docs/user-guide/advanced/serialization.md index 105b8bd..bbe1a53 100644 --- a/docs/user-guide/advanced/serialization.md +++ b/docs/user-guide/advanced/serialization.md @@ -1,16 +1,16 @@ --- title: Serialization and De-serialization -description: Learn how to serialize and de-serialize in PyKX +description: Learn how to serialize and de-serialize in KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, python, serialize, de-serialize +tags: KDB-X Python, q, python, serialize, de-serialize --- # Serialize and de-serialize data -_This page explains how to use PyKX to serialize and de-serialize kdb+/q data structures directly to and from Python byte objects._ +_This page explains how to use KDB-X Python to serialize and de-serialize KDB-X/q data structures directly to and from Python byte objects._ -There are two main ways to serialize/de-serialize data with PyKX: +There are two main ways to serialize/de-serialize data with KDB-X Python: - By interfacing with Python's [`pickle`](https://docs.python.org/3/library/pickle.html) library to persist data to disk in a Python friendly format. - By using the [`kx.serialize`](../../api/serialize.md) module to prepare data in q IPC data format. @@ -22,11 +22,11 @@ There are two main ways to serialize/de-serialize data with PyKX: ## Serialization using pickle -Serializing data is extremely useful in cases where you need to convert a data object into a format that is easily transmittable, such as storing data or transferring it to a remote process. When serializing your PyKX data in most cases it is suggested that you make use of the integration between PyKX and Pickle. +Serializing data is extremely useful in cases where you need to convert a data object into a format that is easily transmittable, such as storing data or transferring it to a remote process. When serializing your KDB-X Python data in most cases it is suggested that you make use of the integration between KDB-X Python and Pickle. -In the following three examples you can see the serialization and de-serialization of various PyKX objects: +In the following three examples you can see the serialization and de-serialization of various `pykx` objects: -1. PyKX Table +1. `pykx.Table` ```python >>> import pykx as kx @@ -45,7 +45,7 @@ In the following three examples you can see the serialization and de-serializati 1 2 3 ``` -2. PyKX Float Vector +2. `pykx.FloatVector` ```python >>> import pykx as kx @@ -60,7 +60,7 @@ In the following three examples you can see the serialization and de-serializati 0.7855048 1.034182 1.031959 0.8133284 0.3561677 0.6035445 1.570066 1.069419 1.. ``` -3. PyKX List +3. `pykx.List` ```python >>> import pykx as kx @@ -84,11 +84,11 @@ In the following three examples you can see the serialization and de-serializati While using `#!python pickle` will be sufficient in most cases, there will be times where you are required to convert data to or from the q IPC format byte representation. Using the `#!python kx.serialize` and `#!python kx.deserialize` functions will provide better performance in these situations. -Unlike with `#!python pickle`, which returns the byte representation immediately on serialization, PyKX allows the generation of this byte object to be deferred by creating a [`memoryview`](https://docs.python.org/3/library/stdtypes.html#memoryview). Deserialization can be completed directly from this `#!python memoryview` or from the raw byte objects +Unlike with `#!python pickle`, which returns the byte representation immediately on serialization, KDB-X Python allows the generation of this byte object to be deferred by creating a [`memoryview`](https://docs.python.org/3/library/stdtypes.html#memoryview). Deserialization can be completed directly from this `#!python memoryview` or from the raw byte objects -Similar to the examples in the previous section in the below we will serialize and deserialize various PyKX objects: +Similar to the examples in the previous section in the below we will serialize and deserialize various `pykx` objects: -1. PyKX Table +1. `pykx.Table` ```python >>> import pykx as kx @@ -112,7 +112,7 @@ Similar to the examples in the previous section in the below we will serialize a 1 2 3 ``` -2. PyKX Float Vector +2. `pykx.FloatVector` ```python >>> import pykx as kx @@ -131,7 +131,7 @@ Similar to the examples in the previous section in the below we will serialize a 0.7855048 1.034182 1.031959 0.8133284 0.3561677 0.6035445 1.570066 1.069419 1.. ``` -3. PyKX List +3. `pykx.List` ```python >>> import pykx as kx @@ -159,7 +159,7 @@ Similar to the examples in the previous section in the below we will serialize a ## What are the limitations? -Serialization of PyKX objects is limited to objects which are purely generated from kdb+/q data. Serialization of `pykx.Foreign` objects, for example, is not supported as these represent underlying objects defined in C of arbitrary complexity. +Serialization of `pykx` objects is limited to objects which are purely generated from KDB-X/q data. Serialization of `pykx.Foreign` objects, for example, is not supported as these represent underlying objects defined in C of arbitrary complexity. ```python >>> import pykx as kx diff --git a/docs/user-guide/advanced/streaming/basic.md b/docs/user-guide/advanced/streaming/basic.md index b66138d..ce3e6dd 100644 --- a/docs/user-guide/advanced/streaming/basic.md +++ b/docs/user-guide/advanced/streaming/basic.md @@ -1,14 +1,14 @@ --- title: Basic Streaming Ingest -description: How to start basic data ingest with PyKX +description: How to start basic data ingest with KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, streaming, simple +tags: KDB-X Python, q, streaming, simple --- # Basic streaming ingest -_This page outlines the steps required to create a high-velocity data ingest infrastructure using PyKX._ +_This page outlines the steps required to create a high-velocity data ingest infrastructure using KDB-X Python._ !!! warning "Disclaimer" @@ -22,7 +22,7 @@ The most fundamental task in managing high-velocity data is consuming and persis 2. Maintain a record of the most recent data available and allow users/analytics to access it. 3. Persist and make available data from previous days to facilitate more complex analytics. -These steps are managed by three separate processes known as the Tickerplant, Real-Time Database (RDB), and Historical Database (HDB). In combination, these processes form the 'basic' building block of any ingest workflow using PyKX. We will break these processes down in more depth later. For now, let's visualize one of their simplest arrangement: +These steps are managed by three separate processes known as the Tickerplant, Real-Time Database (RDB), and Historical Database (HDB). In combination, these processes form the 'basic' building block of any ingest workflow using KDB-X Python. We will break these processes down in more depth later. For now, let's visualize one of their simplest arrangement: ![basic](../images/simple-no-feed.png) @@ -52,7 +52,7 @@ The call to `#!python kx.tick.BASIC` above provides several keyword arguments wh 1. `#!python tables`: Provides a dictionary mapping the names of tables to be ingested and persisted to their schema. This is the only parameter required for initializing the basic infrastructure. 2. `#!python log_directory`: Generates a directory (if not currently available) and persists a log of each incoming message to a log-file associated with today's date. -3. `#!python database`: If omitted, the HDB process outlined above will not be initialized. The `#!python database` denoted here should point to a directory containing a partitioned kdb+ database, for more information on this form of database see [here](../database/index.md). +3. `#!python database`: If omitted, the HDB process outlined above will not be initialized. The `#!python database` denoted here should point to a directory containing a partitioned KDB-X database, for more information on this form of database see [here](../database/index.md). For a full breakdown of the BASIC API functionality go [here](../../../api/tick.md#pykx.tick.BASIC). @@ -84,7 +84,7 @@ The role of a tickerplant is to coordinate the following actions within the life Should the Tickerplant process be killed and need to recover or data be replayed to repopulate the Database at a later point, the log file is centrally important. When the tickerplant receives messages, it persists each of them to the disk, in a file named `#!python log` within a user-specified directory. -Data is logged as a list with the first argument being the function which should be executed on log replay and the remaining elements being the arguments to this function. Within the PyKX streaming workflows, the function called is `#!python .u.upd` and it takes two arguments: `#!python table_name` and `#!python message`. As such, our log file may consist of the following information (in human readable form below). +Data is logged as a list with the first argument being the function which should be executed on log replay and the remaining elements being the arguments to this function. Within the KDB-X Python streaming workflows, the function called is `#!python .u.upd` and it takes two arguments: `#!python table_name` and `#!python message`. As such, our log file may consist of the following information (in human readable form below). ```q (`.u.upd;`trade;(0D11:31:30.758665000;`AAPL;42.0)) @@ -100,11 +100,11 @@ A Real-Time Database (RDB) stores today’s data in-memory and writes it to the ### Historical Databases -The Historical Database (HDB) contains data for all available days of processed data before the current day. This data is stored on-disk and loaded into the process as a memory-mapped dataset. Interactions with this data and its management are more complex than in-memory data. You can manage this data in the [Database API for PyKX](../database/index.md) and you can also [query](../../fundamentals/query/index.md) it. +The Historical Database (HDB) contains data for all available days of processed data before the current day. This data is stored on-disk and loaded into the process as a memory-mapped dataset. Interactions with this data and its management are more complex than in-memory data. You can manage this data in the [Database API for KDB-X Python](../database/index.md) and you can also [query](../../fundamentals/query/index.md) it. !!! info "Important" - PyKX allows to load only one database at a time on an HDB. The HDB connecting to an RDB/RTE should have the same tables defined to avoid issues with missing partitions. + KDB-X Python allows to load only one database at a time on an HDB. The HDB connecting to an RDB/RTE should have the same tables defined to avoid issues with missing partitions. ## Next steps diff --git a/docs/user-guide/advanced/streaming/complex.md b/docs/user-guide/advanced/streaming/complex.md index 3ea6ffc..8a7754c 100644 --- a/docs/user-guide/advanced/streaming/complex.md +++ b/docs/user-guide/advanced/streaming/complex.md @@ -1,9 +1,9 @@ --- title: Complex Streaming Control -description: How to edit/manage your streaming workflows with PyKX +description: How to edit/manage your streaming workflows with KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, streaming, simple +tags: KDB-X Python, q, streaming, simple --- # Complex streaming control @@ -27,7 +27,7 @@ The sections below discuss in detail why it's important and how to update the ex ## Fine-grained ingest control -In the [basic infrastructure](basic.md) section we made use of the function [`#!python kx.tick.BASIC`](../../../api/tick.md#pykx.tick.BASIC) to start the component parts of a PyKX streaming workflow namely: +In the [basic infrastructure](basic.md) section we made use of the function [`#!python kx.tick.BASIC`](../../../api/tick.md#pykx.tick.BASIC) to start the component parts of a KDB-X Python streaming workflow namely: - [Tickerplant](basic.md#tickerplant): The ingestion point which logs incoming messages and publishes messages to down-stream subscribers. - [Real-Time Database(RDB)](basic.md#real-time-database): A process which contains the current day's data in-memory and writes the data to disk at end-of-day. @@ -137,7 +137,7 @@ Each process type supports a keyword argument `#!python process_logs` which can ## How to stop processes -While we hope that we will always generate the perfect code, there can be times when being able to stop processing of our system is a requirement. As the streaming infrastructure for PyKX operates by starting sub-processes from Python, the control of these processes is more complex than it would be, should the parent process be in full control. +While we hope that we will always generate the perfect code, there can be times when being able to stop processing of our system is a requirement. As the streaming infrastructure for KDB-X Python operates by starting sub-processes from Python, the control of these processes is more complex than it would be, should the parent process be in full control. For each of the `#!python BASIC`, `#!python TICK`, `#!python RTP`, `#!python HDB` and `#!python GATEWAY` classes, the initialized class objects have an associated `#!python stop` function. Call this function if you want to gracefully shut down processing and kill the underlying process. You can invoke it using the `#!python rtp` process started in the previous section as an example: diff --git a/docs/user-guide/advanced/streaming/custom_apis.md b/docs/user-guide/advanced/streaming/custom_apis.md index 055e34d..dd1a4de 100644 --- a/docs/user-guide/advanced/streaming/custom_apis.md +++ b/docs/user-guide/advanced/streaming/custom_apis.md @@ -3,7 +3,7 @@ title: Custom Query API Development description: How to generate a custom query API date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, streaming, publishing +tags: KDB-X Python, q, streaming, publishing --- # Custom query API development diff --git a/docs/user-guide/advanced/streaming/gateways.md b/docs/user-guide/advanced/streaming/gateways.md index 2b324b1..ca50c5e 100644 --- a/docs/user-guide/advanced/streaming/gateways.md +++ b/docs/user-guide/advanced/streaming/gateways.md @@ -3,7 +3,7 @@ title: Managing query routing description: How to manage what and how users can query data date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, query, routing, analytics +tags: KDB-X Python, q, query, routing, analytics --- # Manage query routing @@ -22,7 +22,7 @@ When providing users with access to data within your system, you must consider t A `#!python Gateway` process can handle each of these. The gateway is responsible for defining the processes that can be queried within your system and regulates via user-configured logic what is required for a user to establish a connection to the gateway. -PyKX provides a simplistic gateway which allows connections to multiple processes and supports synchronous queries against your processes. Although it doesn't scale to large numbers of high traffic use-cases, it provides a starting infrastructure suitable for small teams of developers. +KDB-X Python provides a simplistic gateway which allows connections to multiple processes and supports synchronous queries against your processes. Although it doesn't scale to large numbers of high traffic use-cases, it provides a starting infrastructure suitable for small teams of developers. !!! note "Have your say" diff --git a/docs/user-guide/advanced/streaming/index.md b/docs/user-guide/advanced/streaming/index.md index a8a106a..686b1f0 100644 --- a/docs/user-guide/advanced/streaming/index.md +++ b/docs/user-guide/advanced/streaming/index.md @@ -1,23 +1,23 @@ --- -title: PyKX Real-Time Data Capture -description: Introduction to the PyKX Real-Time Data Capture functionality +title: KDB-X Python Real-Time Data Capture +description: Introduction to the KDB-X Python Real-Time Data Capture functionality date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, streaming, publishing, real-time data +tags: KDB-X Python, q, streaming, publishing, real-time data --- # Real-Time Data Capture -_This page is an introduction to the PyKX Real-Time Data Capture functionality._ +_This page is an introduction to the KDB-X Python Real-Time Data Capture functionality._ The capture, persistence and presentation of high velocity real-time data provides significant challenges to users at all levels from new users attempting to capture this form of data for the first time, to seasoned data-engineers building complex ingestion workflows. !!! Note "Install q" - The Real-Time Data Capture functionality provided by PyKX requires you to have access to a `q` executable. A workflow is provided by PyKX to install q as outlined [here](../../../getting-started/installing.md). Alternatively, you will be prompted to install q if not detected when initializing the Real-Time Capture functionality. + The Real-Time Data Capture functionality provided by KDB-X Python requires you to have access to a `q` executable. A workflow is provided by KDB-X Python to install q as outlined [here](../../../getting-started/installing.md). Alternatively, you will be prompted to install q if not detected when initializing the Real-Time Capture functionality. -The PyKX Real-Time Data Capture functionality described in this documentation provides a framework for users at all levels of their journey to build highly performant real-time systems which can quickly provide users with the following: +The KDB-X Python Real-Time Data Capture functionality described in this documentation provides a framework for users at all levels of their journey to build highly performant real-time systems which can quickly provide users with the following: - Capture and logging of raw ingested data to facilitate data replay in failure events. - A Real-Time Database maintaining reference to data from the current day and persisting this data at end of day. @@ -29,7 +29,7 @@ Once you're happy with how your data is being captured and persisted, you can bu - Generate real-time and historical query analytics which allow you to derive insights into vast quantities of historical data. - Control how users query your captured data through centralized gateways which keep users away from mission critical data-ingest. -Below we're breaking down the documentation sections to guide you through the process of generating these systems using PyKX and what to consider while building up your infrastructure. +Below we're breaking down the documentation sections to guide you through the process of generating these systems using KDB-X Python and what to consider while building up your infrastructure. ## Sections @@ -41,7 +41,7 @@ Below we're breaking down the documentation sections to guide you through the pr |4. |[Real-Time analytics](rta.md) | Analysing real-time data allows for insights to be derived as you need them. Generate insights into your real-time data and account for common problems. | |5. |[Custom query APIs](custom_apis.md) | Querying historical and real-time data using custom Python APIs allows you and the consumers of your data to gain complex insights into your data. | |6. |[Query access gateways](gateways.md) | Not all users will have free-form access to query your data. They will instead query via authenticated gateway processes. We will outline why this is useful and how to configure this.| -|7. |[Complex streaming control](complex.md) |Learn how to further edit/manage your streaming workflows with PyKX. Methods include: fine-grained ingest control, process logs and stopping processed. | +|7. |[Complex streaming control](complex.md) |Learn how to further edit/manage your streaming workflows with KDB-X Python. Methods include: fine-grained ingest control, process logs and stopping processed. | !!! Warning "Disclaimer" diff --git a/docs/user-guide/advanced/streaming/publish.md b/docs/user-guide/advanced/streaming/publish.md index b418ae1..0fe1e52 100644 --- a/docs/user-guide/advanced/streaming/publish.md +++ b/docs/user-guide/advanced/streaming/publish.md @@ -3,7 +3,7 @@ title: Publish Data description: How to publish data to your streaming infrastructure date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, streaming, publishing +tags: KDB-X Python, q, streaming, publishing --- # Publish Data @@ -14,7 +14,7 @@ _This page outlines how you can publish new data to your streaming infrastructur The functionality outlined below provides the necessary tools for users to build complex streaming infrastructures. The generation and management of such workflows rest solely with the users. KX supports only individual elements used to create these workflows, not the end-to-end applications. -Publishing data to a PyKX streaming workflow is completed by publishing messages to a [tickerplant process](basic.md#tickerplant) using [Interprocess Communication (IPC)](../ipc.md). The sections below show how to achieve this with Python and q in a [basic streaming infrastructure](basic.md). Commonly in KX literature and whitepapers, processes which publish data to a tickerplant are described as Feedhandlers. +Publishing data to a KDB-X Python streaming workflow is completed by publishing messages to a [tickerplant process](basic.md#tickerplant) using [Interprocess Communication (IPC)](../ipc.md). The sections below show how to achieve this with Python and q in a [basic streaming infrastructure](basic.md). Commonly in KX literature and whitepapers, processes which publish data to a tickerplant are described as Feedhandlers. Any messages that is published to a tickerplant is a triplet list with the following structure `#!python [Function;Table;Data]`, where: @@ -91,10 +91,10 @@ In the above code we open a connection to the Tickerplant process on port 5010. ### Other languages -It's possible to publish data to PyKX streaming infrastructures using other languages, such as C and Java: +It's possible to publish data to KDB-X Python streaming infrastructures using other languages, such as C and Java: -- [Publishing to kdb+ using Java](https://www.timestored.com/kdb-guides/kdb-java-api#feedhandling) -- [Publishing to kdb+ tickerplant using C](https://code.kx.com/q/wp/capi/#publishing-to-a-kdb-tickerplant) +- [Publishing to KDB-X using Java](https://www.timestored.com/kdb-guides/kdb-java-api#feedhandling) +- [Publishing to KDB-X tickerplant using C](https://code.kx.com/q/wp/capi/#publishing-to-a-kdb-tickerplant) ## Continuous streaming example diff --git a/docs/user-guide/advanced/streaming/rta.md b/docs/user-guide/advanced/streaming/rta.md index 8748122..bffc725 100644 --- a/docs/user-guide/advanced/streaming/rta.md +++ b/docs/user-guide/advanced/streaming/rta.md @@ -3,7 +3,7 @@ title: Analyze streaming data description: How to apply real-time analytics to streaming data date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, streaming, analytics +tags: KDB-X Python, q, streaming, analytics --- # Analyze streaming data @@ -16,7 +16,7 @@ _This page outlines how you can apply analytics to your streaming data._ In previous sections we have walked through how to build a [basic streaming infrastructure](basic.md), [publish data](publish.md) and [subscribe to updates](subscribe.md). In more complex scenarios users may wish to apply analytics to the real-time data that is being consumed by their infrastructure. This can be completed in a subscribing process or, more formally, in a Real-Time Processor (RTP). -With PyKX, an RTP subscribes to data from a tickerplant and completes one of the following: +With KDB-X Python, an RTP subscribes to data from a tickerplant and completes one of the following: 1. Operates as a Real-Time Database (RDB) ingesting data as quickly as possible, making it available for query during the day and coordinating data persistence at end of day. 1. Allows analytic application on data prior to and post insertion into the in-memory database (RDB). @@ -62,7 +62,7 @@ In the above code we are setting the chained tickerplant on port 5013 to provide Now that a chained tickerplant has been established which can be used for slow subscribers, we can generate some analytics from the data supplied by this process. To achieve this, we use a Real-Time Processor (RTP) which runs pre-processing and post-processing analytics. We generate Python analytics that achieve the following: 1. Pre-process all messages to remove any data which is sourced from a table other than the `#!python trade` table. -2. A post-processing function written in Python which uses PyKX and numpy to calculate the `#!python min`, `#!python max` and `#!python spread` of prices per symbol from the processed trade data. +2. A post-processing function written in Python which uses KDB-X Python and numpy to calculate the `#!python min`, `#!python max` and `#!python spread` of prices per symbol from the processed trade data. 3. Publishes the results of your aggregated information back to the primary tickerplant to ensure the aggregate data is persisted for future inspection. The reference architecture used for this is as follows: @@ -100,7 +100,7 @@ rtp.libraries({'np': 'numpy', 'kx': 'pykx'}) The above example is equivalent to running `#!python import numpy as np` and `#!python import pykx as kx` on the RTP. -Now that the RTP process has access to PyKX and Numpy, we can define the pre-processor function. If this pre-processor returns a `#!python None` object, the data received will not be inserted into the RTP's in-memory database. This function must take two parameters: +Now that the RTP process has access to KDB-X Python and Numpy, we can define the pre-processor function. If this pre-processor returns a `#!python None` object, the data received will not be inserted into the RTP's in-memory database. This function must take two parameters: 1. Name of the table whose data is being processed 2. The data that is being processed diff --git a/docs/user-guide/advanced/streaming/subscribe.md b/docs/user-guide/advanced/streaming/subscribe.md index b7f2954..8ff6ddc 100644 --- a/docs/user-guide/advanced/streaming/subscribe.md +++ b/docs/user-guide/advanced/streaming/subscribe.md @@ -3,7 +3,7 @@ title: Subscribing to data description: How to subscribe to real-time updates from your streaming infrastructure date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, streaming, subscribing +tags: KDB-X Python, q, streaming, subscribing --- # Subscribe to data diff --git a/docs/user-guide/advanced/streamlit.md b/docs/user-guide/advanced/streamlit.md index 10bcd74..9d67f87 100644 --- a/docs/user-guide/advanced/streamlit.md +++ b/docs/user-guide/advanced/streamlit.md @@ -1,16 +1,16 @@ --- title: Streamlit Integration -description: Integrate PyKX Connections into you Streamlit application +description: Integrate KDB-X Python Connections into you Streamlit application date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, streamlit, visualisation, query, web, application +tags: PyKX, KDB-X Python, q, streamlit, visualisation, query, web, application --- # Streamlit Integration !!! Warning - Streamlit makes use of a caching mechanism which makes use of multiple threads, to make use of PyKX under these conditions it is suggested that users set `PYKX_THREADING` as `True`, for more information on the threading feature see [here](threading.md), for information on setting configuration see [here](../configuration.md). + Streamlit makes use of a caching mechanism which makes use of multiple threads, to make use of KDB-X Python under these conditions it is suggested that users set `PYKX_THREADING` as `True`, for more information on the threading feature see [here](threading.md), for information on setting configuration see [here](../configuration.md). [Streamlit](https://streamlit.io) provides an open source framework allowing users to turn Python scripts into sharable web applications. Functionally, Streamlit provides access to external data-sources using the concept of `connections` which allow users to develop conforming APIs which will integrate directly with streamlit applications as an extension connection types. @@ -22,21 +22,21 @@ A full breakdown of the API documentation of this class can be found [here](../. To run this functionality, users must have `streamlit>=1.28` installed local to their Python session. -This can be installed using the following command when installing PyKX: +This can be installed using the following command when installing KDB-X Python: ```bash pip install pykx[streamlit] ``` -## Using PyKX with Streamlit +## Using KDB-X Python with Streamlit -The PyKX Streamlit integration provides users with the ability to do the following: +The KDB-X Python Streamlit integration provides users with the ability to do the following: 1. Establish a Streamlit compliant connection to a q/kdb+ process 1. Check health of a connection and restart connection as necessary 1. Query the remote process using `q`, `SQL` and `qSQL` -As mentioned above PyKX provides a streamlit connection type `pykx.streamlit.PyKXConnection` which can be used with the streamlit [`st.connection`](https://docs.streamlit.io/develop/api-reference/connections/st.connection) functionality to integrate your streamlit application with PyKX. +As mentioned above KDB-X Python provides a streamlit connection type `pykx.streamlit.PyKXConnection` which can be used with the streamlit [`st.connection`](https://docs.streamlit.io/develop/api-reference/connections/st.connection) functionality to integrate your streamlit application with KDB-X Python. In the below section we will discuss how these connections are established, maintained and used for query. @@ -90,7 +90,7 @@ In the below example we connect to a variety of kdb+ processes on port 5050 with In streamlit, your application may be running for a significant period of time. In such situations it is not uncommon for your original connection to a server to drop. -To help with such cases there are a number of methods provided by PyKX to recover your environment: +To help with such cases there are a number of methods provided by KDB-X Python to recover your environment: - The addition of an `is_healthy` method to facilitate checking if the remote server can be interacted with. - The availability of a `reset` method to allow a connection which is deemed not to be healthy to be re-established. @@ -188,7 +188,7 @@ The following script generates a simple streamlit application which 1. Name the streamlit application 1. Create a connection to the q process initialised on port 5050 1. Query the q process retrieving a small tabular subset of data using the Pythonic Query API - 1. Generates a Matplotlib graph directly using the PyKX table + 1. Generates a Matplotlib graph directly using the `pykx` table 1. Displays both the table and graph The script which follows can be downloaded [here](examples/streamlit.py) @@ -208,7 +208,7 @@ The script which follows can be downloaded [here](examples/streamlit.py) import matplotlib.pyplot as plt def main(): - st.header('PyKX Demonstration') + st.header('KDB-X Python Demonstration') connection = st.connection('pykx', type=kx.streamlit.PyKXConnection, port=5050) diff --git a/docs/user-guide/advanced/subprocess.md b/docs/user-guide/advanced/subprocess.md index d954ab9..9baa803 100644 --- a/docs/user-guide/advanced/subprocess.md +++ b/docs/user-guide/advanced/subprocess.md @@ -1,22 +1,22 @@ --- -title: PyKX in Subprocesses +title: KDB-X Python in Subprocesses description: Outline using pykx in python subprocesses date: August 2024 author: KX Systems, Inc. -tags: PyKX +tags: KDB-X Python --- -# Using PyKX in python subprocesses +# Using KDB-X Python in python subprocesses -_This page outlines using PyKX in a Python subprocess._ +_This page outlines using KDB-X Python in a Python subprocess._ -To use PyKX in a python subprocess you should spawn the process using the `#!python kx.PyKXReimport` function as follows: +To use KDB-X Python in a python subprocess you should spawn the process using the `#!python kx.PyKXReimport` function as follows: ```python import pykx as kx import subprocess with kx.PyKXReimport(): - subprocess.Popen(['python', 'file.py']) #_Run a python subprocess that loads a python script containing a PyKX import + subprocess.Popen(['python', 'file.py']) #_Run a python subprocess that loads a python script containing a import pykx ``` -Failing to reimport the PyKX package running in the parent process can cause the subprocess to crash with a segmentation fault. The `#!python PyKXReimport` function and possible causes of segmentation faults is covered in more detail [here](../../api/reimporting.md). +Failing to reimport the KDB-X Python package running in the parent process can cause the subprocess to crash with a segmentation fault. The `#!python PyKXReimport` function and possible causes of segmentation faults is covered in more detail [here](../../api/reimporting.md). diff --git a/docs/user-guide/advanced/threading.md b/docs/user-guide/advanced/threading.md index 3201f02..e0d93b5 100644 --- a/docs/user-guide/advanced/threading.md +++ b/docs/user-guide/advanced/threading.md @@ -1,25 +1,25 @@ --- title: Multithreaded Execution -description: Learn how multithreaded integration for PyKX works +description: Learn how multithreaded integration for KDB-X Python works date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, threading, python, asyncio, multithreaded, noupdate +tags: KDB-X Python, q, threading, python, asyncio, multithreaded, noupdate --- # Multithreaded execution _This page explains how to execute q code on multiple threads._ -When used in its default configuration, PyKX does not support multithreaded execution of q code on Python threads. This limitation arises because only the main thread (the one importing PyKX and loading `#!python libq`) can modify the state in PyKX’s assigned memory. +When used in its default configuration, KDB-X Python does not support multithreaded execution of q code on Python threads. This limitation arises because only the main thread (the one importing KDB-X Python and loading `#!python libq`) can modify the state in KDB-X Python’s assigned memory. -As a result, PyKX’s integration with Python’s multithreading libraries, such as [`#!python threading`](https://docs.python.org/3/library/threading.html), [`#!python asyncio`](https://docs.python.org/3/library/asyncio.html), is restricted. This also affects other Python libraries that utilize multiple threads simultaneously, including [`#!python streamlit`](https://streamlit.io/), which uses multiple threads to manage data caching. Read more information about [PyKX’s integration with Streamlit](streamlit.md). +As a result, KDB-X Python’s integration with Python’s multithreading libraries, such as [`#!python threading`](https://docs.python.org/3/library/threading.html), [`#!python asyncio`](https://docs.python.org/3/library/asyncio.html), is restricted. This also affects other Python libraries that utilize multiple threads simultaneously, including [`#!python streamlit`](https://streamlit.io/), which uses multiple threads to manage data caching. Read more information about [KDB-X Python’s integration with Streamlit](streamlit.md). -Use cases for multithreading with PyKX: +Use cases for multithreading with KDB-X Python: - **Upserting Data**: Insert or update data in a global table from multiple sources. - **Querying Multiple Processes**: Open `#!python QConnection` instances to query several processes simultaneously and combine their results. -If you don’t configure PyKX for multithreading, you might encounter a `#!python noupdate` error. To avoid this, consider enabling the feature described here. This feature allows multithreading by creating a background thread that `#!python loads` libq. +If you don’t configure KDB-X Python for multithreading, you might encounter a `#!python noupdate` error. To avoid this, consider enabling the feature described here. This feature allows multithreading by creating a background thread that `#!python loads` libq. All calls to q from other threads are run on this background thread, created using `#!python libpthread` for minimal overhead. This setup enables safe state modification in multithreaded programs with minimal performance impact. @@ -34,7 +34,7 @@ Before globally enabling this functionality, consider the following: ## How to enable multithreaded execution -By default, PyKX doesn't start with multithreading support enabled. To enable this feature, you must set `#!python PYKX_THREADING=True` during [configuration](../configuration.md). You can do this either as an environment variable or by adding this configuration to a `#!python .pykx-config` file as outlined [here](../configuration.md#configuration-file). +By default, KDB-X Python doesn't start with multithreading support enabled. To enable this feature, you must set `#!python PYKX_THREADING=True` during [configuration](../configuration.md). You can do this either as an environment variable or by adding this configuration to a `#!python config-pykx` file as outlined [here](../configuration.md#configuration-file). ## Example usage diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 567d6cd..d360c68 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -1,22 +1,22 @@ --- -title: Configure PyKX -description: How to configure PyKX +title: Configure KDB-X Python +description: How to configure KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, data, convert +tags: KDB-X Python, data, convert --- -# Configure PyKX +# Configure KDB-X Python -_This page provides details on how to configure PyKX using a configuration file and/or environment variables._ +_This page provides details on how to configure KDB-X Python using a configuration file and/or environment variables._ -To modify the underlying behavior of PyKX based on your specific use-case, check out your [options](#options) for use-case/performance tuned optimizations of the library. You can configure them using a [configuration file](#configuration-file) or [environment variables](#environment-variables) as described below. +To modify the underlying behavior of KDB-X Python based on your specific use-case, check out your [options](#options) for use-case/performance tuned optimizations of the library. You can configure them using a [configuration file](#configuration-file) or [environment variables](#environment-variables) as described below. !!! warning "Important: In all cases, environment variable definitions take precedence over definitions within the configuration file." ## Configuration file -If you choose to use a configuration file `#!python .pykx-config` to define your options for PyKX initialization, here's an example of a `#!python .pykx-config` file which operates according to `#!python *.toml` syntax: +If you choose to use a configuration file `#!python config-pykx` to define your options for KDB-X Python initialization, here's an example of a `#!python config-pykx` file which operates according to `#!python *.toml` syntax: ```bash [default] @@ -31,15 +31,14 @@ PYKX_RELEASE_GIL="true" PYKX_BETA_FEATURES="true" ``` -On import of PyKX, the file `#!python .pykx-config` is searched for according to the following path ordering. The first location containing a `#!python .pykx-config` file is used for definition of the PyKX configuration: +On import of KDB-X Python, the file `#!python config-pykx` is searched for according to the following path ordering. The first found file is used for definition of the KDB-X Python configuration: | **Order** | **Location** | |-----------|--------------------------------------------------| -| 1. | `Path('.')` | -| 2. | `Path(os.getenv('PYKX_CONFIGURATION_LOCATION'))` | -| 3. | `Path.home()` | +| 1. | `Path(os.getenv('PYKX_CONFIGURATION_LOCATION'))` | +| 2. | `Path.home()/'.kx/config-pykx'` | -When loading this file, unless otherwise specified, PyKX uses the profile `#!python default`. To configure non-default profiles from within this file, set an environment variable `#!python PYKX_PROFILE` prior to loading of PyKX, for example using the above configuration file. +When loading this file, unless otherwise specified, KDB-X Python uses the profile `#!python default`. To configure non-default profiles from within this file, set an environment variable `#!python PYKX_PROFILE` prior to loading of KDB-X Python, for example using the above configuration file. === "default" @@ -65,7 +64,7 @@ To add values to your configuration file you can modify the file directly or alt >>> import pykx as kx >>> kx.util.add_to_config({'PYKX_GC': 'True', 'PYKX_BETA_FEATURES': 'True'}) -Configuration updated at: /Users/conormccarthy/.pykx-config. +Configuration updated at: /Users/user/config-pykx. Profile updated: default. Successfully added: - PYKX_GC = True @@ -74,7 +73,7 @@ Successfully added: ## Environment variables -If you wish to configure the [options](#options) as environment variables, before importing PyKX, you can, for example, define the environment variables to use: +If you wish to configure the [options](#options) as environment variables, before importing KDB-X Python, you can, for example, define the environment variables to use: ```python >>> import os @@ -87,74 +86,74 @@ True ## Options -You have various options to tune PyKX behavior at run time. You must set these variables before importing PyKX. They remain effective throughout the execution of the PyKX process. +You have various options to tune KDB-X Python behavior at run time. You must set these variables before importing KDB-X Python. They remain effective throughout the execution of the KDB-X Python process. ### General -To enable or disable advanced features of PyKX across all modes of operation, use the following variables: +To enable or disable advanced features of KDB-X Python across all modes of operation, use the following variables: | **Option** | **Default** | **Values** | **Description** | |---------------------------------|-------------|-----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `PYKX_BETA_FEATURES` | `False` | `1` or `true` | Enable all Beta features supplied with PyKX allowing users to test and prototype code slated for later releases. | +| `PYKX_BETA_FEATURES` | `False` | `1` or `true` | Enable all Beta features supplied with KDB-X Python allowing users to test and prototype code slated for later releases. | | `PYKX_QDEBUG` | `False` | `1` or `true` | Enable retrieval of backtrace information on error being raised when executing q functions, this can alternatively be enabled by setting `debug=True` as a keyword in calls to `kx.q`. | -| `PYKX_IGNORE_QHOME` | `False` | `1` or `true` | When loading PyKX on a machine with an existing q installation (and the environment variable `QHOME` set to the installation folder), PyKX will look within this directory for q scripts their dependencies. It will then symlink these files to make them available to load under PyKX. This variable instructs PyKX to not perform this symlinking. | -| `PYKX_KEEP_LOCAL_TIMES` | `False` | `1` or `true` | When converting a Python datetime object to q, PyKX will translate the Python datetime into UTC before the conversion. This variable instructs PyKX to convert the Python datetime using the local time zone. | -| `PYKX_ALLOCATOR` | `False` | `1` or `true` | When converting a Numpy array to q, PyKX implements a full data copy in order to translate the Numpy array to q representation in memory. When this is set PyKX implements [NEP-49](https://numpy.org/neps/nep-0049.html) which allows q to handle memory allocation of all Numpy arrays so they can be converted more efficiently to q. This avoids the need to resort to a copy where possible. | -| `PYKX_GC` | `False` | `1` or `true` | When PYKX_ALLOCATOR is enabled, PyKX can trigger q garbage collector when Numpy arrays allocated by PyKX are deallocated. This variable enables this behavior which will release q memory to the OS following deallocation of the Numpy array at the cost of a small overhead. | -| `PYKX_LOAD_PYARROW_UNSAFE` | `False` | `1` or `true` | By default, PyKX uses a subprocess to import pyarrow as it can result in a crash when the version of pyarrow is incompatible. This variable will trigger a normal import of pyarrow and importing PyKX should be slightly faster. | -| `PYKX_MAX_ERROR_LENGTH` | `256` | size in characters | By default, PyKX reports IPC connection errors with a message buffer of size 256 characters. This allows the length of these error messages to be modified reducing the chance of excessive error messages polluting logs. | -| `PYKX_NOQCE` | `False` | `1` or `true` | On Linux, PyKX comes with q Cloud Edition features from [Insights Core](https://code.kx.com/insights/core/). This variable allows a user to skip the loading of q Cloud Edition functionality, saving some time when importing PyKX but removing access to possibly supported additional functionality. | -| `PYKX_Q_LIB_LOCATION` | `UNSET` | Path to a directory containing q libraries necessary for loading PyKX | See [here](../release-notes/changelog.md#pykx-131) for detailed information. This allows a user to store the PyKX libraries: `q.so`, `q.k` etc. separately from their Python installation. This is required for some enterprise use-cases. | +| `PYKX_IGNORE_QHOME` | `False` | `1` or `true` | When loading KDB-X Python on a machine with an existing q installation (and the environment variable `QHOME` set to the installation folder), KDB-X Python will look within this directory for q scripts their dependencies. It will then symlink these files to make them available to load under KDB-X Python. This variable instructs KDB-X Python to not perform this symlinking. | +| `PYKX_KEEP_LOCAL_TIMES` | `False` | `1` or `true` | When converting a Python datetime object to q, KDB-X Python will translate the Python datetime into UTC before the conversion. This variable instructs KDB-X Python to convert the Python datetime using the local time zone. | +| `PYKX_NO_ALLOCATOR` | `False` | `1` or `true` | KDB-X Python implements [NEP-49](https://numpy.org/neps/nep-0049.html) which allows q to handle memory allocation of all Numpy arrays so they can be converted more efficiently to q. This avoids the need to resort to a copy where possible. Use `PYKX_NO_ALLOCATOR` to turn this behaviour off, resulting in full data copies. | +| `PYKX_GC` | `False` | `1` or `true` | KDB-X Python can trigger q garbage collector when Numpy arrays allocated by KDB-X Python are deallocated. This variable enables this behavior which will release q memory to the OS following deallocation of the Numpy array at the cost of a small overhead. Only usable when `PYKX_NO_ALLOCATOR` is not set. | +| `PYKX_LOAD_PYARROW_UNSAFE` | `False` | `1` or `true` | By default, KDB-X Python uses a subprocess to import pyarrow as it can result in a crash when the version of pyarrow is incompatible. This variable will trigger a normal import of pyarrow and importing KDB-X Python should be slightly faster. | +| `PYKX_MAX_ERROR_LENGTH` | `256` | size in characters | By default, KDB-X Python reports IPC connection errors with a message buffer of size 256 characters. This allows the length of these error messages to be modified reducing the chance of excessive error messages polluting logs. | +| `PYKX_QCE` | `False` | `1` or `true` | On Linux, KDB-X Python comes with q Cloud Edition features from [Insights Core](https://code.kx.com/insights/core/). This variable allows a user to opt in the loading of q Cloud Edition functionality. | +| `PYKX_Q_LIB_LOCATION` | `UNSET` | Path to a directory containing q libraries necessary for loading KDB-X Python | See [here](../release-notes/changelog.md#pykx-131) for detailed information. This allows a user to store the KDB-X Python libraries: `q.so`, `q.k` etc. separately from their Python installation. This is required for some enterprise use-cases. **Note:** If using this override you must manage the folder to ensure its contents match the version of KDB-X Python you are using | | `PYKX_RELEASE_GIL` | `False` | `1` or `true` | When PYKX_RELEASE_GIL is enabled the Python Global Interpreter Lock will not be held when calling into q. | | `PYKX_Q_LOCK` | `False` | `1` or `true` | When PYKX_Q_LOCK is enabled a re-entrant lock is added around calls into q, this lock will stop multiple threads from calling into q at the same time. This allows embedded q to be thread safe even when using PYKX_RELEASE_GIL. | | `PYKX_DEBUG_INSIGHTS_LIBRARIES` | `False` | `1` or `true` | If the insights libraries failed to load this variable can be used to print out the full error output for debugging purposes. | -| `PYKX_UNLICENSED` | `False` | `1` or `true` | Set PyKX to make use of the library in `unlicensed` mode at all times. | -| `PYKX_LICENSED` | `False` | `1` or `true` | Set PyKX to make use of the library in `licensed` mode at all times. If licensed initialisation fails the import will error rather than bringing up the interactive license helper. Fallback to unlicensed mode is blocked. | -| `PYKX_THREADING` | `False` | `1` or `true` | When importing PyKX start EmbeddedQ within a background thread. This allows calls into q from any thread to modify state, this environment variable is only supported for licensed users. | -| `PYKX_NO_SIGNAL` | `False` | `1` or `true` | Skip overwriting of [signal](https://docs.python.org/3/library/signal.html) definitions by PyKX, these are presently overwritten by default to reset Pythonic default definitions with are reset by PyKX on initialisation in licensed modality. | -| `PYKX_4_1_ENABLED` | `False` | `1` or `true` | Load version 4.1 of `libq` when starting `PyKX` in licensed mode, this environment variable does not work without a valid `q` license. | -| `PYKX_JUPYTERQ` | `False` | `1` or `true` | When enabled, any Jupyter Notebook will start in q first mode by default when PyKX is imported. | +| `PYKX_UNLICENSED` | `False` | `1` or `true` | Set KDB-X Python to make use of the library in `unlicensed` mode at all times. | +| `PYKX_LICENSED` | `False` | `1` or `true` | Set KDB-X Python to make use of the library in `licensed` mode at all times. If licensed initialisation fails the import will error rather than bringing up the interactive license helper. Fallback to unlicensed mode is blocked. | +| `PYKX_THREADING` | `False` | `1` or `true` | When importing KDB-X Python start EmbeddedQ within a background thread. This allows calls into q from any thread to modify state, this environment variable is only supported for licensed users. | +| `PYKX_NO_SIGNAL` | `False` | `1` or `true` | Skip overwriting of [signal](https://docs.python.org/3/library/signal.html) definitions by KDB-X Python, these are presently overwritten by default to reset Pythonic default definitions with are reset by KDB-X Python on initialisation in licensed modality. | +| `PYKX_JUPYTERQ` | `False` | `1` or `true` | When enabled, any Jupyter Notebook will start in q first mode by default when KDB-X Python is imported. | | `PYKX_Q_EXECUTABLE` | `q` | string denoting path to q executable | This allows users to specify the location of the q executable which should be called when using making use of the `tick` module for defining streaming infrastructures | -| `PYKX_SUPPRESS_WARNINGS` | `False` | `1` or `true` | This allows the user to suppress warnings that have been suggested as sensible to be raised by users for PyKX in situations where edge cases can result in unexpected behavior. Warnings in scenarios where a decision has been made to not support behavior explicitly rather than where user discretion is required are still maintained. | -| `PYKX_CONFIGURATION_LOCATION` | `.` | The path to the folder containing the `.pykx-config` file. | This allows users to specify a location other than the `.` or a users `home` directory to store their configuration file outlined [here](#configuration-file) | -| `PYKX_CONFIGURATION_PROFILE` | `default` | The "profile" defined in `.pykx-config` file to be used. | Users can specify which set of configuration variables are to be used by modifying the `PYKX_CONFIGURATION_PROFILE` variable see [here](#configuration-file) for more details. Note that this configuration can only be used as an environment variable. | +| `PYKX_SUPPRESS_WARNINGS` | `False` | `1` or `true` | This allows the user to suppress warnings that have been suggested as sensible to be raised by users for KDB-X Python in situations where edge cases can result in unexpected behavior. Warnings in scenarios where a decision has been made to not support behavior explicitly rather than where user discretion is required are still maintained. | +| `PYKX_CONFIGURATION_LOCATION` | `.` | The path to the folder containing the `config-pykx` file. | This allows users to specify a location other than the `.` or a users `home` directory to store their configuration file outlined [here](#configuration-file) | +| `PYKX_CONFIGURATION_PROFILE` | `default` | The "profile" defined in `config-pykx` file to be used. | Users can specify which set of configuration variables are to be used by modifying the `PYKX_CONFIGURATION_PROFILE` variable see [here](#configuration-file) for more details. Note that this configuration can only be used as an environment variable. | -To set the environment for q (embedded in PyKX, in licensed mode), use the variables below: +To set the environment for q (embedded in KDB-X Python, in licensed mode), use the variables below: | **Variable** | **Values** | **Description** | |--------------|--------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| | `QARGS` | See link | Command-line flags to pass to q, see [here](https://code.kx.com/q/basics/cmdline/) for more information. | | `QHOME` | Path to the users q installation folder | See [here](https://code.kx.com/q/learn/install/#step-5-edit-your-profile) for more information. | | `QLIC` | Path to the folder where the q license should be found | See [here](https://code.kx.com/q/learn/install/#step-5-edit-your-profile) for more information. | -| `QINIT` | Path to an additional `*.q` file loaded after `PyKX` has initialized | See [here](https://code.kx.com/q4m3/14_Introduction_to_Kdb%2B/#1481-the-environment-variables) for more information. | +| `QINIT` | Path to an additional `*.q` file loaded after KDB-X Python has initialized | See [here](https://code.kx.com/q4m3/14_Introduction_to_Kdb%2B/#1481-the-environment-variables) for more information. | +| `QCFG` | Location of KX config file. Defaults to `~/.kx/config` | | -If no license is found, set the following variables either in configuration or as environment variables to define the `kc.lic` or `k4.lic` license used by PyKX: +If no license is found, set the following variables either in configuration or as environment variables to define the `kc.lic` or `k4.lic` license used by KDB-X Python: | **Variable** | **Description** | |---------------------|------------------------------------------------------------------------------------------------| | `KDB_LICENSE_B64` | This should contain the base-64 encoded contents of a valid `kc.lic` file with `pykx` enabled. | | `KDB_K4LICENSE_B64` | This should contain the base-64 encoded contents of a valid `k4.lic` file with `pykx` enabled. | -#### PyKX QARGS supported additions +#### KDB-X Python QARGS supported additions -When using PyKX, you can define `#!python QARGS` to modify its behavior during initialization in a Linux environment. Here are some of the values you can use for `#!python QARGS`: +When using KDB-X Python, you can define `#!python QARGS` to modify its behavior during initialization in a Linux environment. Here are some of the values you can use for `#!python QARGS`: -| **Input** | **Description** | -|----------------|---------------------------------------------------------------------------------| -| `--no-qce` | Ensure that no kdb Insights libraries are loaded at initialization of PyKX. | -| `--no-kurl` | Ensure that the kdb Insights `kurl` library is not loaded at initialization. | -| `--no-objstor` | Ensure that the kdb Insights `objstor` library is not loaded at initialization. | -| `--no-qlog` | Ensure that the kdb Insights `qlog` library is not loaded at initialization. | -| `--no-sql` | Ensure that the kdb Insights `sql` library is not loaded at initialization. | +| **Input** | **Description** | +|-------------|---------------------------------------------------------------------------------| +| `--qce` | Loads KDB-X libraries at initialization of KDB-X. | +| `--kurl` | Ensure that the KDB-X `kurl` library is not loaded at initialization. | +| `--objstor` | Ensure that the KDB-X `objstor` library is not loaded at initialization. | +| `--qlog` | Ensure that the KDB-X `qlog` library is not loaded at initialization. | +| `--sql` | Ensure that the KDB-X `sql` library is not loaded at initialization. | -### PyKX under q +### KDB-X Python under q -You can load PyKX and [use it from a q session](../pykx-under-q/intro.md). The following variables are specific to this mode of operation: +You can load KDB-X Python and [use it from a q session](../pykx-under-q/intro.md). The following variables are specific to this mode of operation: | **Variable** | **Values** | **Description** | |---------------------------|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `PYKX_DEFAULT_CONVERSION` | `py`, `np`, `pd`, `pa` or `k` | Default conversion to apply when passing q objects to Python. Converting to Numpy (`np`) by default. | -| `PYKX_SKIP_UNDERQ` | `1` or `true` | When importing PyKX from Python, PyKX also loads `pykx.q` under its embedded q. This variable skips this step. | +| `PYKX_SKIP_UNDERQ` | `1` or `true` | When importing KDB-X Python from Python, KDB-X Python also loads `pykx.q` under its embedded q. This variable skips this step. | | `PYKX_EXECUTABLE` | File path | The path to use for the Python executable | | `PYKX_USE_FIND_LIBPYTHON` | `1` or `true` | Should the Python package [`find-libpython`](https://pypi.org/project/find-libpython/) be used to determine the location of `libpython.[so|dll]`, this manually could be done by setting the location `PYKX_PYTHON_LIB_PATH` | | `PYKX_PYTHON_LIB_PATH` | File path | The path to use for loading libpython. | diff --git a/docs/user-guide/fundamentals/conversion_considerations.md b/docs/user-guide/fundamentals/conversion_considerations.md index efe081e..cfcc881 100644 --- a/docs/user-guide/fundamentals/conversion_considerations.md +++ b/docs/user-guide/fundamentals/conversion_considerations.md @@ -1,30 +1,30 @@ --- -title: Convert data types in PyKX -description: Converting data types in PyKX +title: Convert data types in KDB-X Python +description: Converting data types in KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, data, convert +tags: KDB-X Python, data, convert --- -# PyKX conversion considerations +# KDB-X Python conversion considerations -_This page provides details on data types and conversions in PyKX._ +_This page provides details on data types and conversions in KDB-X Python._ -PyKX attempts to make conversions between q and Python as seamless as possible. +KDB-X Python attempts to make conversions between q and Python as seamless as possible. However due to differences in their underlying implementations there are cases where 1 to 1 mappings are not possible. ## Data types and conversions -The key PyKX APIs around data types and conversions are outlined under: +The key KDB-X Python APIs around data types and conversions are outlined under: -* [Convert Pythonic data to PyKX](../../api/pykx-q-data/toq.md) -* [PyKX type wrappers](../../api/pykx-q-data/wrappers.md) -* [PyKX to Pythonic data type mapping](../../api/pykx-q-data/type_conversions.md) +* [Convert Pythonic data to KDB-X Python](../../api/pykx-q-data/toq.md) +* [KDB-X Python type wrappers](../../api/pykx-q-data/wrappers.md) +* [KDB-X Python to Pythonic data type mapping](../../api/pykx-q-data/type_conversions.md) * [Registering Custom Conversions](../../api/pykx-q-data/register.md) -## Text representation in PyKX +## Text representation in KDB-X Python -Handling and converting [text in PyKX](./text.md) requires consideration as there are some key differences between the `Symbol` and `Char` data types. +Handling and converting [text in KDB-X Python](./text.md) requires consideration as there are some key differences between the `Symbol` and `Char` data types. ## Nulls and Infinites @@ -32,11 +32,11 @@ Most q datatypes have the concepts of null, negative infinity, and infinity. Pyt ## Temporal data types -Converting [temporal data types](./temporal.md) in PyKX involves handling [timestamp/datetime](./temporal.md#timestampdatetime-types) types and [duration](./temporal.md#duration-types) types, each with specific considerations due to differences in how Python and q (the language used by kdb+) represent these data types. +Converting [temporal data types](./temporal.md) in KDB-X Python involves handling [timestamp/datetime](./temporal.md#timestampdatetime-types) types and [duration](./temporal.md#duration-types) types, each with specific considerations due to differences in how Python and q (the language used by KDB-X) represent these data types. ## List conversion considerations -By default the library converts generic PyKX List objects `#!python pykx.List` to NumPy as an array of NumPy arrays. This conversion is chosen as it allows for the most flexible representation of data allowing ragged array representations and mixed lists of objects to be converted easily. However, this representation can be difficult to work with if/when dealing with multi-dimensional numeric data as is common in machine learning tasks for example. +By default the library converts generic `#!python pykx.List` objects to NumPy as an array of NumPy arrays. This conversion is chosen as it allows for the most flexible representation of data allowing ragged array representations and mixed lists of objects to be converted easily. However, this representation can be difficult to work with if/when dealing with multi-dimensional numeric data as is common in machine learning tasks for example. As an example we can look at the conversion of a 3-Dimensional regularly shaped `#!python pykx.List` object to a NumPy array as follows: diff --git a/docs/user-guide/fundamentals/creating.md b/docs/user-guide/fundamentals/creating.md index 630ea50..23c78e2 100644 --- a/docs/user-guide/fundamentals/creating.md +++ b/docs/user-guide/fundamentals/creating.md @@ -1,40 +1,40 @@ --- -title: Create and convert PyKX objects -description: How to generate PyKX objects +title: Create and convert `pykx` objects +description: How to generate `pykx` objects date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, PyKX objects, +tags: KDB-X Python, q, `pykx` objects, --- -# Create and convert PyKX objects +# Create and convert `pykx` objects -_This page provides details on how to generate and convert PyKX objects._ +_This page provides details on how to generate and convert `pykx` objects._ -!!! tip "Tip: For the best experience, we recommend reading [PyKX objects and attributes](..//../learn/objects.md) first." +!!! tip "Tip: For the best experience, we recommend reading [`pykx` objects and attributes](..//../learn/objects.md) first." -To use the power of q and the functionality provided by PyKX, at some point you must interact with PyKX objects. At their most basic level, objects are allocated C representations of q/kdb+ objects within a memory space managed by q. Keeping the data in this format allows it to be used directly for query/analytic execution in q without any translation overhead. +To use the power of q and the functionality provided by KDB-X Python, at some point you must interact with `pykx` objects. At their most basic level, objects are allocated C representations of q/KDB-X objects within a memory space managed by q. Keeping the data in this format allows it to be used directly for query/analytic execution in q without any translation overhead. -## 1. Create PyKX objects +## 1. Create `pykx` objects -There are five ways to create PyKX objects: +There are five ways to create `pykx` objects: -- a. [Convert Python objects to PyKX objects](#1a-convert-python-objects-to-pykx-objects) -- b. [Generate data using PyKX inbuilt functions](#1b-generate-data-using-pykx-inbuilt-functions) +- a. [Convert Python objects to KDB-X Python objects](#1a-convert-python-objects-to-pykx-objects) +- b. [Generate data using KDB-X Python inbuilt functions](#1b-generate-data-using-pykx-inbuilt-functions) - c. [Evaluate q code using `#!python kx.q`](#1c-evaluate-q-code-using-kxq) - d. [Assign Python data to q's memory](#1d-assign-python-data-to-qs-memory) - e. [Retrieve a named entity from q's memory](#1e-retrieve-a-named-entity-from-qs-memory) - f. [Query an external q session](#1f-query-an-external-q-session) -### 1.a Convert Python objects to PyKX objects +### 1.a Convert Python objects to `pykx` objects -The simplest way to create a PyKX object is by converting a similar Python type into a PyKX object. You can do this with the `#!python pykx.toq function`, which supports conversions from Python, NumPy, Pandas, PyArrow, and PyTorch (Beta) types to PyKX objects. Open the tabs that interest you to see conversion examples: +The simplest way to create a `pykx` object is by converting a similar Python type into a `pykx` object. You can do this with the `#!python pykx.toq function`, which supports conversions from Python, NumPy, Pandas, PyArrow, and PyTorch (Beta) types to `pykx` objects. Open the tabs that interest you to see conversion examples: ??? Note "Specify target types" - When converting Pythonic objects to PyKX types, you can use the `ktype` named argument: + When converting Pythonic objects to KDB-X Python types, you can use the `ktype` named argument: - - To convert lists/atomic elements, use [PyKX types](../../api/pykx-q-data/type_conversions.md); - - To convert Pandas DataFrames or PyArrow Tables, use the `#!python ktype` argument with a dictionary input mapping the column name to the [PyKX type](../../api/pykx-q-data/type_conversions.md). + - To convert lists/atomic elements, use [`pykx` types](../../api/pykx-q-data/type_conversions.md); + - To convert Pandas DataFrames or PyArrow Tables, use the `#!python ktype` argument with a dictionary input mapping the column name to the [`pykx` type](../../api/pykx-q-data/type_conversions.md). === "Python" @@ -173,7 +173,7 @@ The simplest way to create a PyKX object is by converting a similar Python type === "PyTorch (Beta)" - When converting data from PyTorch types to PyKX support is only provided for `#!python torch.Tensor` object conversions to PyKX at this time and requires setting of the configuration `PYKX_BETA_FEATURES=True` as shown below + When converting data from PyTorch types to KDB-X Python support is only provided for `#!python torch.Tensor` object conversions to KDB-X Python at this time and requires setting of the configuration `PYKX_BETA_FEATURES=True` as shown below ```python >>> import os @@ -194,7 +194,7 @@ The simplest way to create a PyKX object is by converting a similar Python type ')) ``` -By default, when you convert Python strings to PyKX, they are returned as `#!python pykx.SymbolAtom` objects. This ensures a clear distinction between `#!python str` (string) and `#!python byte` objects. However, you might prefer Python strings to be returned as `#!python pykx.CharVector` objects, to achieve memory efficiency or greater flexibility in analytic development. To do this, use the keyword argument `#!python strings_as_char`, which ensures that all `#!python str` objects are converted to `#!python pykx.CharVector` objects. +By default, when you convert Python strings to KDB-X Python, they are returned as `#!python pykx.SymbolAtom` objects. This ensures a clear distinction between `#!python str` (string) and `#!python byte` objects. However, you might prefer Python strings to be returned as `#!python pykx.CharVector` objects, to achieve memory efficiency or greater flexibility in analytic development. To do this, use the keyword argument `#!python strings_as_char`, which ensures that all `#!python str` objects are converted to `#!python pykx.CharVector` objects. ```python >>> import pykx as kx @@ -207,7 +207,7 @@ b| "test1" ')) ``` -### 1.b Generate data using PyKX inbuilt functions +### 1.b Generate data using KDB-X Python inbuilt functions For users who want to generate objects directly but are not familiar with q, and wish to quickly prototype this functionality, several helper functions are available. @@ -218,7 +218,7 @@ Create a vector of random floating point precision values: pykx.FloatVector(pykx.q('9.030751 7.750292 3.869818')) ``` -Additionally, when generating random data, you can use PyKX null/infinite data to create data across larger data ranges as follows: +Additionally, when generating random data, you can use KDB-X Python null/infinite data to create data across larger data ranges as follows: ```python >>> kx.random.random(2, kx.GUIDAtom.null) @@ -270,7 +270,7 @@ pykx.TimestampAtom(pykx.q('2024.01.05T16:22:21.012631000')) ### 1.c Evaluate q code using `#!python kx.q` -If you're more familiar with q, generate PyKX objects by evaluating q code: +If you're more familiar with q, generate `pykx` objects by evaluating q code: ```python >>> kx.q('til 10') @@ -314,7 +314,7 @@ Assignment of data from Python's memory space to q can take a number of forms: ??? Note "Why `__setattr__` is discouraged" - Data retrieval using `__getattr__` on the `kx.q` object is designed for use with the PyKX [context interface](../../api/pykx-execution/ctx.md). To comply with round-trip retrieval the assignment completed with `__setattr__` syntax persists data to a name with a leading `.`. + Data retrieval using `__getattr__` on the `kx.q` object is designed for use with the KDB-X Python [context interface](../../api/pykx-execution/ctx.md). To comply with round-trip retrieval the assignment completed with `__setattr__` syntax persists data to a name with a leading `.`. To see the effect of this in practice we can look at the following example: @@ -329,7 +329,7 @@ Assignment of data from Python's memory space to q can take a number of forms: ### 1.e Retrieve a named entity from q's memory -As PyKX objects exist in a memory space accessed and controlled by interactions with q, the items created in q may not be immediately available as Python objects. For example, if you created a named variable in q as a side effect of a function call or just explicitly created it, you can retrieve it by its name: +As `pykx` objects exist in a memory space accessed and controlled by interactions with q, the items created in q may not be immediately available as Python objects. For example, if you created a named variable in q as a side effect of a function call or just explicitly created it, you can retrieve it by its name: ```python >>> kx.q('t:([]5?1f;5?1f)') # Generate a named variable in a single object @@ -352,7 +352,7 @@ pykx.FloatVector(pykx.q('0.3927524 0.5170911 0.5159796 0.4066642 0.1780839')) ### 1.f Query an external q session -PyKX provides an IPC interface allowing users to query and retrieve data from a q server. If you have a q server with no username/password exposed on `#!python port 5000`, it's possible to run synchronous and asynchronous events against this server: +KDB-X Python provides an IPC interface allowing users to query and retrieve data from a q server. If you have a q server with no username/password exposed on `#!python port 5000`, it's possible to run synchronous and asynchronous events against this server: ```python >>> conn = kx.QConnection('localhost', 5000) # Open a connection to the q server @@ -387,19 +387,19 @@ a 0.02810674 0.481821 ')) ``` -## 2. Convert PyKX objects to Pythonic types +## 2. Convert `pykx` objects to Pythonic types -Converting data to a PyKX format allows for easy interaction with these objects using q or the analytic functionality provided by PyKX. However, this format may not be suitable for all use cases. For instance, if a function requires a Pandas DataFrame as input, a PyKX object must be converted to a Pandas DataFrame. +Converting data to a KDB-X Python format allows for easy interaction with these objects using q or the analytic functionality provided by KDB-X Python. However, this format may not be suitable for all use cases. For instance, if a function requires a Pandas DataFrame as input, a `pykx` object must be converted to a Pandas DataFrame. Once the data is ready for use in Python, it may be more appropriate to convert it into a representation using Python, NumPy, Pandas, PyArrow, or PyTorch (Beta) by using the following methods: | **Method** | **Description** | |-----------------|----------------------------------| -| `*.py()` | Convert a PyKX object to Python | -| `*.np()` | Convert a PyKX object to Numpy | -| `*.pd()` | Convert a PyKX object to Pandas | -| `*.pa()` | Convert a PyKX object to PyArrow | -| `*.pt()` (Beta) | Convert a PyKX object to PyTorch | +| `*.py()` | Convert a `pykx` object to Python | +| `*.np()` | Convert a `pykx` object to Numpy | +| `*.pd()` | Convert a `pykx` object to Pandas | +| `*.pa()` | Convert a `pykx` object to PyArrow | +| `*.pt()` (Beta) | Convert a `pykx` object to PyTorch | ??? example "Example" diff --git a/docs/user-guide/fundamentals/evaluating.md b/docs/user-guide/fundamentals/evaluating.md index 7d43d02..8b19a4a 100644 --- a/docs/user-guide/fundamentals/evaluating.md +++ b/docs/user-guide/fundamentals/evaluating.md @@ -1,34 +1,34 @@ --- -title: Use PyKX objects -description: How to use PyKX objects and evaluate q code with PyKX +title: Use `pykx` objects +description: How to use `pykx` objects and evaluate q code with KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, PyKX objects, +tags: KDB-X Python, q, `pykx` objects, --- -# Use PyKX objects and evaluate q code with PyKX +# Use `pykx` objects and evaluate q code with KDB-X Python -_This page provides details on how to use PyKX objects and how to evaluate q code with PyKX._ +_This page provides details on how to use `pykx` objects and how to evaluate q code with KDB-X Python._ -!!! tip "Tip: For the best experience, we recommend reading [PyKX objects and attributes](..//../learn/objects.md) and [Create and convert PyKX objects](creating.md) first." +!!! tip "Tip: For the best experience, we recommend reading [`pykx` objects and attributes](..//../learn/objects.md) and [Create and convert `pykx` objects](creating.md) first." -There are four ways to manipulate PyKX objects and evaluate q code in PyKX: +There are four ways to manipulate `pykx` objects and evaluate q code in KDB-X Python: - a. By calling `#!python pykx.q` directly, for example, `#!python pykx.q('10 {x,sum -2#x}/ 0 1')` - b. By dropping into the [interactive console][pykx.QConsole] - c. By using `#!python q` keyword functions, for example, `#!python pykx.q.til(10)` - d. Over [IPC][pykx.QConnection] -The first three methods evaluate the code locally within the Python process and require a q license. The final method evaluates the code in a separate q process and can be used with or without a q license, provided the server your PyKX instance is connected to is appropriately licensed. +The first three methods evaluate the code locally within the Python process and require a q license. The final method evaluates the code in a separate q process and can be used with or without a q license, provided the server your KDB-X Python instance is connected to is appropriately licensed. !!! Warning - Functions pulled in over IPC are executed locally in PyKX. Go to the [IPC documentation](../../api/ipc.md) + Functions pulled in over IPC are executed locally in KDB-X Python. Go to the [IPC documentation](../../api/ipc.md) for more information on how to ensure the `q` code is executed on the server and not locally. ## a. Call q using `#!python pykx.q` -For users familiar with kdb+/q code, the `#!python pykx.q` (or `#!python kx.q`) method allows the evaluation of q code to take place providing the return of the function as a `#!python PyKX` object. This method is variadic, meaning it can accept a variable number of arguments. You can use in two different ways:” +For users familiar with KDB-X/q code, the `#!python pykx.q` (or `#!python kx.q`) method allows the evaluation of q code to take place providing the return of the function as a `#!python KDB-X Python` object. This method is variadic, meaning it can accept a variable number of arguments. You can use in two different ways:” 1. Direct evaluation of single lines of code 2. Application of functions that take multiple arguments @@ -48,7 +48,7 @@ pykx.FloatVector(pykx.q('0.06165008 0.285799 0.6684724 0.9133033 0.1485357')) ### a.2 Application of functions taking multiple arguments -If the first argument of `#!python pykx.q` is a function, the `#!python N` following arguments are treated as arguments to that function. Arguments can be Python or PyKX objects. All objects passed to a q function are converted to a PyKX object using the method `#!python pykx.toq`. For example: +If the first argument of `#!python pykx.q` is a function, the `#!python N` following arguments are treated as arguments to that function. Arguments can be Python or `pykx` objects. All objects passed to a q function are converted to a `pykx` object using the method `#!python pykx.toq`. For example: ```python >>> import pykx as kx @@ -72,7 +72,7 @@ x x1 !!! Note - The application of arguments to functions within PyKX is limited to a maximum of 8 arguments. This limitation is imposed by the evaluation of q code. + The application of arguments to functions within KDB-X Python is limited to a maximum of 8 arguments. This limitation is imposed by the evaluation of q code. Users wishing to debug failed evaluation of q code can do so, either by globally setting the environment variable `#!python PYKX_QDEBUG` or through a `#!python debug` keyword: @@ -122,7 +122,7 @@ Users wishing to debug failed evaluation of q code can do so, either by globally pykx.exceptions.QError: type ``` -## b. Use the q console within PyKX +## b. Use the q console within KDB-X Python For users more comfortable prototyping q code within a q terminal, it's possible within a Python terminal to run an emulation of a q session directly in Python through the `#!python kx.q.console` method: @@ -137,7 +137,7 @@ q)\\ !!! Note - This is not a fully-featured q terminal. It shares the same core [limitations](../../help/issues.md) as PyKX, particularly regarding the running of timers and subscriptions. + This is not a fully-featured q terminal. It shares the same core [limitations](../../help/issues.md) as KDB-X Python, particularly regarding the running of timers and subscriptions. ## c. Use q keywords diff --git a/docs/user-guide/fundamentals/indexing.md b/docs/user-guide/fundamentals/indexing.md index f569c5b..41ab55f 100644 --- a/docs/user-guide/fundamentals/indexing.md +++ b/docs/user-guide/fundamentals/indexing.md @@ -1,27 +1,27 @@ --- -title: IndexPyKX objects -description: How to index PyKX object +title: Index `pykx` objects +description: How to index `pykx` object date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, PyKX objects, index +tags: KDB-X Python, q, `pykx` objects, index --- -# Index PyKX Objects +# Index `pykx` objects -_This page provides details on how indexing works within PyKX._ +_This page provides details on how indexing works within KDB-X Python._ -Indexing in q works differently than you may be used to, and that behavior largely carries over into PyKX for indexing K objects. +Indexing in q works differently than you may be used to, and that behavior largely carries over into KDB-X Python for indexing K objects. !!! info "Resources" - For more information about how indexing in q works (and by extension, how indexing K objects in PyKX work), refer to the following sections of the q tutorial book [Q For Mortals](https://code.kx.com/q4m3/): + For more information about how indexing in q works (and by extension, how indexing K objects in KDB-X Python work), refer to the following sections of the q tutorial book [Q For Mortals](https://code.kx.com/q4m3/): - [Indexing](https://code.kx.com/q4m3/3_Lists/#34-indexing) - [Iterated indexing and indexing at depth](https://code.kx.com/q4m3/3_Lists/#38-iterated-indexing-and-indexing-at-depth) - [Indexing with lists](https://code.kx.com/q4m3/3_Lists/#39-indexing-with-lists) - [Elided indices](https://code.kx.com/q4m3/3_Lists/#310-elided-indices) -Indexes used on K objects in PyKX are converted to equivalent K objects in q using the [toq module](../../api/pykx-q-data/toq.md), just like any other Python to q conversion. To guarantee that the index used against a K object is what you intend it to be, you may perform the conversion of the index yourself before applying it. When K objects are used as the index for another K object, the index object is applied to the [`#!python pykx.Collection`][pykx.Collection] object as it would be in q, for example as described in Q For Mortals. +Indexes used on K objects in KDB-X Python are converted to equivalent K objects in q using the [toq module](../../api/pykx-q-data/toq.md), just like any other Python to q conversion. To guarantee that the index used against a K object is what you intend it to be, you may perform the conversion of the index yourself before applying it. When K objects are used as the index for another K object, the index object is applied to the [`#!python pykx.Collection`][pykx.Collection] object as it would be in q, for example as described in Q For Mortals. Examples of applying indexing to various q objects include: @@ -31,11 +31,11 @@ Examples of applying indexing to various q objects include: ## a. Basic Vectors Indexing -Indexing in PyKX spans elements `#!python 0` to element `#!python N-1` where `#!python N` is the length of the object being indexed. +Indexing in KDB-X Python spans elements `#!python 0` to element `#!python N-1` where `#!python N` is the length of the object being indexed. ### a.1 Single element indexing -Single element indexing works like any other standard Python sequence. Similar to Numpy, PyKX supports negative indices to allow retrieval of indexes at the end of an array. For example: +Single element indexing works like any other standard Python sequence. Similar to Numpy, KDB-X Python supports negative indices to allow retrieval of indexes at the end of an array. For example: ```python >>> x = kx.q.til(10) @@ -81,7 +81,7 @@ pykx.FloatAtom(pykx.q('0.6919531')) ### a.2 Slicing -Slicing vectors in PyKX is simpler than the functionality provided by Numpy. You can index vectors of N dimensions by using `#!python obj[start:stop:step]` semantics. This slice syntax operates where `#!python start` is the starting index, `#!python stop` is the stopping index and `#!python step` is the number of steps between the elements where `#!python step` is non zero: +Slicing vectors in KDB-X Python is simpler than the functionality provided by Numpy. You can index vectors of N dimensions by using `#!python obj[start:stop:step]` semantics. This slice syntax operates where `#!python start` is the starting index, `#!python stop` is the stopping index and `#!python step` is the number of steps between the elements where `#!python step` is non zero: ```python >>> x = kx.q.til(10) @@ -139,7 +139,7 @@ a b c ## b. Assigning and adding values to vectors/lists -Vector assignment in PyKX operates similarly to that provided by Numpy and operations supported on basic Python lists. As with the previous sections this functionality supports both individual element assignment and slice assignment as follows: +Vector assignment in KDB-X Python operates similarly to that provided by Numpy and operations supported on basic Python lists. As with the previous sections this functionality supports both individual element assignment and slice assignment as follows: ```python >>> vec = kx.q.til(10) @@ -240,7 +240,7 @@ In addition to positional assignment, you can use the `#!python append` and `#!p ## c. Indexing non-vector objects -In addition to being able to index and slice PyKX `#!python vector` and `#!python list` objects, it's also possible to apply index and slicing semantics on PyKX Table objects. Application of slice/index semantics on tabular objects returns table-like objects: +In addition to being able to index and slice `pykx` `#!python vector` and `#!python list` objects, it's also possible to apply index and slicing semantics on `pykx` Table objects. Application of slice/index semantics on tabular objects returns table-like objects: ```python >>> import pandas as pd diff --git a/docs/user-guide/fundamentals/nulls_and_infinities.md b/docs/user-guide/fundamentals/nulls_and_infinities.md index e00da72..1b709e1 100644 --- a/docs/user-guide/fundamentals/nulls_and_infinities.md +++ b/docs/user-guide/fundamentals/nulls_and_infinities.md @@ -1,18 +1,18 @@ --- title: Convert nulls and infinities -description: How to handle nulls and infinities in PyKX +description: How to handle nulls and infinities in KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, nulls, infinity +tags: KDB-X Python, q, nulls, infinity --- # Convert nulls and infinities -_This page explains how to handle nulls and infinities in PyKX._ +_This page explains how to handle nulls and infinities in KDB-X Python._ -PyKX handles nulls and infinities in ways that are subtly different from familiar libraries like NumPy, due to its q legacy. +KDB-X Python handles nulls and infinities in ways that are subtly different from familiar libraries like NumPy, due to its q legacy. -PyKX provides typed null and infinity values for most types. [As shown in the q docs](https://code.kx.com/q/ref/#datatypes): +KDB-X Python provides typed null and infinity values for most types. [As shown in the q docs](https://code.kx.com/q/ref/#datatypes): - nulls can be expressed as `#!python 0N` followed by a type character (or no type character for long integer null). - infinities can be expressed as `#!python 0W` followed by a type character (or no type character for a long integer infinity). @@ -84,7 +84,7 @@ NotImplementedError: Retrieval of infinite values not supported for this type ## Checking for nulls and infinities -If you apply [the q function named null](https://code.kx.com/q/ref/null/) to most PyKX objects, it returns `#!python 1b` if the object is null. If it contains nulls, returns a collection of booleans whose shape matches the object. Like with any function from the `#!python .q` namespace, you can access it via the [context interface](../../api/pykx-execution/ctx.md): [`#!python q.null`](../../api/pykx-execution/q.md#null). +If you apply [the q function named null](https://code.kx.com/q/ref/null/) to most `pykx` objects, it returns `#!python 1b` if the object is null. If it contains nulls, returns a collection of booleans whose shape matches the object. Like with any function from the `#!python .q` namespace, you can access it via the [context interface](../../api/pykx-execution/ctx.md): [`#!python q.null`](../../api/pykx-execution/q.md#null). ```python >>> import pykx as kx @@ -138,7 +138,7 @@ See also the page with specifics on [temporal](./temporal.md) conversions. !!! note "Note" - PyKX null conversion behavior changed in version 3.0.0. The below table outlines the before and after conversions. + KDB-X Python null conversion behavior changed in version 3.0.0. The below table outlines the before and after conversions. === ".py()" @@ -332,7 +332,7 @@ x x1 -9223372036854775808 2 3 ``` -To illustrate this as a limitation of Pandas rather than PyKX consider the following: +To illustrate this as a limitation of Pandas rather than KDB-X Python consider the following: ```python >>> tab = kx.Table(data = { @@ -388,7 +388,7 @@ Additional to the above inconsistency with Pandas you may also run into issues w [1000 rows x 3 columns] ``` -While `#!python -9223372036854778080` represents an underlying PyKX Null value, for display purposes it's visually distracting. To display the DataFrame with the masked values, set its `#!python display.max_rows` to be longer than the length of the specified table. Notice the result below: +While `#!python -9223372036854778080` represents an underlying KDB-X Python Null value, for display purposes it's visually distracting. To display the DataFrame with the masked values, set its `#!python display.max_rows` to be longer than the length of the specified table. Notice the result below: ```python >>> import pandas as pd @@ -417,7 +417,7 @@ See also the page with specifics on [temporal](./temporal.md) conversions to exp !!! note "Note" - PyKX infinite conversion behavior changed in version 3.0.0. The below tables outline the before and after conversions. + KDB-X Python infinite conversion behavior changed in version 3.0.0. The below tables outline the before and after conversions. #### Positive Infinity conversions @@ -583,7 +583,7 @@ See also the page with specifics on [temporal](./temporal.md) conversions to exp #### Infinite weirdness -Other than real/float infinities, which follow the IEEE standard for infinities and so are ignored in this section, infinite values in kdb+ do not behave how you would expect them to. PyKX opts to expose their behavior as-is, since the alternatives (error for infinities, or always expose them as their underlying values) are undesirable. For this reason you should take care when using them. +Other than real/float infinities, which follow the IEEE standard for infinities and so are ignored in this section, infinite values in KDB-X do not behave how you would expect them to. KDB-X Python opts to expose their behavior as-is, since the alternatives (error for infinities, or always expose them as their underlying values) are undesirable. For this reason you should take care when using them. Arithmetic operations on infinities are applied directly to the underlying values. As such, adding 1 to many positive infinities in q will result in the null for that type, as the value overflows and becomes the smallest value in that type's range. Subtracting 1 from positive infinities merely yields the second largest number for that type. For instance, `#!python 2147483646 == q('0Wi') - 1`. @@ -593,7 +593,7 @@ Wherever possible, the conversions from Q to Python are symmetric. Therefore, yo ## Performance -By default, whenever PyKX converts a q vector to a Python representation (e.g. a NumPy array) it checks where the nulls (if any) are located. This requires operating on every element of the array, which can be rather expensive. +By default, whenever KDB-X Python converts a q vector to a Python representation (e.g. a NumPy array) it checks where the nulls (if any) are located. This requires operating on every element of the array, which can be rather expensive. If you know ahead of time that your q vector/table has no nulls in it, you can provide the keyword argument `#!python has_nulls=False` to `#!python .py`/`#!python .np`/`#!python .pd`/`#!python .pa`. This will skip the null-check. If you set this keyword argument to false, but there are still nulls in the data, they will come through as the underlying values from q, for example, `#!python -32768` for a short integer. diff --git a/docs/user-guide/fundamentals/query/index.md b/docs/user-guide/fundamentals/query/index.md index 5f50c99..88501c6 100644 --- a/docs/user-guide/fundamentals/query/index.md +++ b/docs/user-guide/fundamentals/query/index.md @@ -1,14 +1,14 @@ --- -title: Querying data using PyKX -description: Introduction to the concept of querying PyKX databases and tables +title: Querying data using KDB-X Python +description: Introduction to the concept of querying KDB-X Python databases and tables date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, query, historical, SQL, qSQL +tags: KDB-X Python, q, query, historical, SQL, qSQL --- -# Querying data using PyKX +# Querying data using KDB-X Python -_This page provides an introduction to querying your data using PyKX._ +_This page provides an introduction to querying your data using KDB-X Python._ | Title | Description | |:--------------------------------------|:------------------------------------------------------------------------------------| @@ -20,4 +20,4 @@ _This page provides an introduction to querying your data using PyKX._ ## Next Steps - If you don't have a historical database available see [here](../../advanced/database/index.md). -- To learn about creating PyKX Table objects see [here](../../../examples/interface-overview.ipynb). +- To learn about creating `pykx` Table objects see [here](../../../examples/interface-overview.ipynb). diff --git a/docs/user-guide/fundamentals/query/perf.md b/docs/user-guide/fundamentals/query/perf.md index 16ad99e..041e466 100644 --- a/docs/user-guide/fundamentals/query/perf.md +++ b/docs/user-guide/fundamentals/query/perf.md @@ -1,21 +1,21 @@ --- -title: Querying data using PyKX -description: Introduction to the concept of querying PyKX databases and tables +title: Querying data using KDB-X Python +description: Introduction to the concept of querying KDB-X Python databases and tables date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, query, historical, SQL, qSQL +tags: KDB-X Python, q, query, historical, SQL, qSQL --- -# Query performance considerations using PyKX +# Query performance considerations using KDB-X Python -_This page explains how to efficiently query your data using PyKX._ +_This page explains how to efficiently query your data using KDB-X Python._ ## qSQL The whitepapers detailed below outline optimizations which can be useful for qSQL queries. However, the core lessons/concepts which apply in the [q query](./qquery.md) case also apply to using the [Pythonic query API](./pyquery.md) and [SQL](./sql.md) modes: - [Columnar database and query optimization](https://code.kx.com/q/wp/columnar-database/) -- [kdb+ query scaling](https://code.kx.com/q/wp/query-scaling/) +- [KDB-X query scaling](https://code.kx.com/q/wp/query-scaling/) The following provides a tangible example of two impactful optimizations @@ -85,6 +85,6 @@ To optimize frequently called SQL queries you can make use of the [prepare](../. ## Next Steps -- Learn how to query your data using the PyKX Pythonic Query API [here](pyquery.md). +- Learn how to query your data using the KDB-X Python Query API [here](pyquery.md). - If you don't have a historical database available see [here](../../advanced/database/index.md). -- To learn about creating PyKX Table objects see [here](../../../examples/interface-overview.ipynb). +- To learn about creating `pykx` Table objects see [here](../../../examples/interface-overview.ipynb). diff --git a/docs/user-guide/fundamentals/query/pyquery.md b/docs/user-guide/fundamentals/query/pyquery.md index c696ac1..b7c0231 100644 --- a/docs/user-guide/fundamentals/query/pyquery.md +++ b/docs/user-guide/fundamentals/query/pyquery.md @@ -1,14 +1,14 @@ --- -title: Querying data using PyKX -description: Introduction to the concept of querying PyKX databases and tables +title: Querying data using KDB-X Python +description: Introduction to the concept of querying KDB-X Python databases and tables date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, query, historical, SQL, qSQL +tags: KDB-X Python, q, query, historical, SQL, qSQL --- -# Querying data using the query API with PyKX +# Querying data using the query API with KDB-X Python -_This page explains how to query your data with PyKX using the query API._ +_This page explains how to query your data with KDB-X Python using the query API._ Before we get started the following dataset will be used throughout the remainder of this page. @@ -30,7 +30,7 @@ Creating a sample table: ## Query basics -The PyKX [query API](../../../api/query.md) provides a Pythonic way to query kdb+ table. This API builds [qSQL](https://code.kx.com/q/basics/qsql/) queries in their [functional](https://code.kx.com/q/basics/funsql/) allowing you to query in-memory and on-disk data. +The KDB-X Python [query API](../../../api/query.md) provides a Pythonic way to query KDB-X table. This API builds [qSQL](https://code.kx.com/q/basics/qsql/) queries in their [functional](https://code.kx.com/q/basics/funsql/) allowing you to query in-memory and on-disk data. In the following sections we will introduce the functions, their arguments and how they can be used to perform queries of increasing complexity. @@ -91,7 +91,7 @@ table.delete(columns=None, where=None, by=None, inplace=False) !!! Note - The following sections makes use of `kx.Column` objects which are only enabled in PyKX licensed mode. For unlicensed query examples using `str` objects see the [query API](../../../api/query.md) page. + The following sections makes use of `kx.Column` objects which are only enabled in KDB-X Python licensed mode. For unlicensed query examples using `str` objects see the [query API](../../../api/query.md) page. ### Query arguments @@ -645,7 +645,7 @@ MSFT 2022.01.03 46.11964 93 ### Query Types -While this page discusses primarily the Pythonic API for querying kdb+ tables locally. The following describes some of the other ways that queries can be completed +While this page discusses primarily the Pythonic API for querying KDB-X tables locally. The following describes some of the other ways that queries can be completed #### Local Queries @@ -870,9 +870,9 @@ The following are a few examples of this various operations in use ')) ``` -#### PyKX methods +#### KDB-X Python methods -In addition to support for the Python operators outlined above PyKX provides a number of analytic methods and properties for the `kx.Column` objects. In total there are more than 100 analytic methods supported ranging from a basic method to retrieve the maximum value of a column, to more complex analytics for the calculation of the weighted average between two vectors. +In addition to support for the Python operators outlined above KDB-X Python provides a number of analytic methods and properties for the `kx.Column` objects. In total there are more than 100 analytic methods supported ranging from a basic method to retrieve the maximum value of a column, to more complex analytics for the calculation of the weighted average between two vectors. The following drop-down provides a list of the supported methods, with full details on the API page [here](../../../api/columns.md). @@ -1015,7 +1015,7 @@ Not all analytics that you may wish to run on your table will expect to take the r example in some cases you may wish to apply an analytic on each row of a column. While operations which rely on iterato rs may be slower than purely vectorised operations they may be necessary. -PyKX supports the following iterators, a number of examples are provided below +KDB-X Python supports the following iterators, a number of examples are provided below | Iterator | Type | Link | | :------- | :---------- | :------------------------------------------------------------------------------- | @@ -1110,4 +1110,4 @@ Now that you have learnt how to query your data using the Pythonic API you may b For some further reading, here are some related topics: - If you don't have a historical database available see [here](../../advanced/database/index.md). -- To learn about creating PyKX Table objects see [here](../../../examples/interface-overview.ipynb). +- To learn about creating `pykx` Table objects see [here](../../../examples/interface-overview.ipynb). diff --git a/docs/user-guide/fundamentals/query/qquery.md b/docs/user-guide/fundamentals/query/qquery.md index 088a547..622cf85 100644 --- a/docs/user-guide/fundamentals/query/qquery.md +++ b/docs/user-guide/fundamentals/query/qquery.md @@ -1,14 +1,14 @@ --- -title: Querying data using PyKX -description: Introduction to the concept of querying PyKX databases and tables +title: Querying data using KDB-X Python +description: Introduction to the concept of querying KDB-X Python databases and tables date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, query, historical, SQL, qSQL +tags: KDB-X Python, q, query, historical, SQL, qSQL --- -# Querying data using qSQL with PyKX +# Querying data using qSQL with KDB-X Python -_This page explains how to query your data with PyKX using qSQL._ +_This page explains how to query your data with KDB-X Python using qSQL._ ## Querying tables using qSQL @@ -86,4 +86,4 @@ Now that you have learnt how to query your data using q you may be interested in For some further reading, here are some related topics: - If you don't have a historical database available see [here](../../advanced/database/index.md). -- To learn about creating PyKX Table objects see [here](../../../examples/interface-overview.ipynb). +- To learn about creating `pykx` Table objects see [here](../../../examples/interface-overview.ipynb). diff --git a/docs/user-guide/fundamentals/query/sql.md b/docs/user-guide/fundamentals/query/sql.md index 5386d6a..1b32124 100644 --- a/docs/user-guide/fundamentals/query/sql.md +++ b/docs/user-guide/fundamentals/query/sql.md @@ -1,42 +1,20 @@ --- -title: Querying data using SQL with PyKX -description: Introduction to querying data using SQL with PyKX +title: Querying data using SQL with KDB-X Python +description: Introduction to querying data using SQL with KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, q, query, historical, SQL, qSQL +tags: KDB-X Python, q, query, historical, SQL, qSQL --- -# Querying data using SQL with PyKX +# Querying data using SQL with KDB-X Python -_This page explains how to query your data with PyKX using SQL._ +_This page explains how to query your data with KDB-X Python using SQL._ -PyKX exposes a wrapper around the [KX Insights Core ANSI SQL interface](https://code.kx.com/insights/core/sql.html). +KDB-X Python exposes a wrapper around the [KX Insights Core ANSI SQL interface](https://code.kx.com/insights/core/sql.html). This allows SQL to be used to query in-memory and on-disk data. The interface is accessed through the `kx.q.sql` class or via the `sql` method on table type objects. Full documentation of the class is included [here](../../../api/query.md#pykx.query.SQL). -## Loading the SQL interface - -When you `import pykx as kx` an attempt will be made to load the SQL interface. If this fails you will see: - -```python -WARN: Failed to load KX Insights Core library 's.k'. -``` - -To debug this you can set the [configuration option](../../configuration.md) `PYKX_DEBUG_INSIGHTS_LIBRARIES` before importing PyKX: - -```python -import os -os.environ['PYKX_DEBUG_INSIGHTS_LIBRARIES'] = 'true' -import pykx as kx -``` - -This will print a more detailed error message, for example: - -```python -PyKXWarning: Failed to load KX Insights Core library 's.k': s.k_. OS reports: No such file or directory -``` - ## Querying tables using SQL Creating a sample table: @@ -129,4 +107,4 @@ Now that you have learnt the fundamentals of how to query your data using the SQ For some further reading, here are some related topics: - If you don't have a historical database available see [here](../../advanced/database/index.md). -- To learn about creating PyKX Table objects see [here](../../../examples/interface-overview.ipynb). +- To learn about creating `pykx` Table objects see [here](../../../examples/interface-overview.ipynb). diff --git a/docs/user-guide/fundamentals/temporal.md b/docs/user-guide/fundamentals/temporal.md index 75fee55..0724363 100644 --- a/docs/user-guide/fundamentals/temporal.md +++ b/docs/user-guide/fundamentals/temporal.md @@ -1,16 +1,16 @@ --- -title: Convert temporal data types in PyKX -description: How to convert temporal data types in PyKX +title: Convert temporal data types in KDB-X Python +description: How to convert temporal data types in KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, data, convert +tags: KDB-X Python, data, convert --- -# Convert temporal data types in PyKX +# Convert temporal data types in KDB-X Python -_This page provides details on how to convert temporal data types in PyKX._ +_This page provides details on how to convert temporal data types in KDB-X Python._ -Converting temporal data types in PyKX involves handling timestamp/datetime types and duration types. +Converting temporal data types in KDB-X Python involves handling timestamp/datetime types and duration types. ### Timestamp/Datetime types @@ -100,11 +100,11 @@ d * In NumPy further data types exist `datetime64[us]`, `datetime64[ms]`, `datetime64[s]` which due to their lower precision have a wider range of dates they can represent. When converted using to q using `toq` these all present as q `Timestamp` type and as such only dates within the range this data type can represent should be converted. - * Pandas 2.* changes behavior and conversions should be reviewed as part of an upgrade of this package. [PyKX to Pythonic data type mapping](../../api/pykx-q-data/type_conversions.md) includes examples showing differences seen when calling `.pd()`. + * Pandas 2.* changes behavior and conversions should be reviewed as part of an upgrade of this package. [KDB-X Python to Pythonic data type mapping](../../api/pykx-q-data/type_conversions.md) includes examples showing differences seen when calling `.pd()`. ### Duration types -Duration types do not have the issue of epoch offsets, but some range limitations exist when converting between Python and PyKX. +Duration types do not have the issue of epoch offsets, but some range limitations exist when converting between Python and KDB-X Python. `#!python kx.SecondVector` and `#!python kx.MinuteVector` convert to `#!python timedelta64[s]`: diff --git a/docs/user-guide/fundamentals/text.md b/docs/user-guide/fundamentals/text.md index f4b1978..24118b1 100644 --- a/docs/user-guide/fundamentals/text.md +++ b/docs/user-guide/fundamentals/text.md @@ -1,29 +1,29 @@ --- -title: Convert text in PyKX -description: How to convert text in PyKX +title: Convert text in KDB-X Python +description: How to convert text in KDB-X Python date: July 2024 author: KX Systems, Inc., -tags: PyKX, text, +tags: KDB-X Python, text, --- -# Convert text in PyKX +# Convert text in KDB-X Python -_This page provides details on how to represent, handle, and convert text in PyKX._ +_This page provides details on how to represent, handle, and convert text in KDB-X Python._ -In PyKX, text can be represented in various ways. Here are the basic building blocks for handling text within the library: +In KDB-X Python, text can be represented in various ways. Here are the basic building blocks for handling text within the library: | **Type** | **Description** | **Example Generation** | |---------------------|--------------------------------------------------------------------------------------------------|------------------------------| -| `pykx.SymbolAtom` | A symbol atom in PyKX is an irreducible atomic entity storing an arbitrary number of characters. | ```pykx.q('`test')``` | +| `pykx.SymbolAtom` | A symbol atom in KDB-X Python is an irreducible atomic entity storing an arbitrary number of characters. | ```pykx.q('`test')``` | | `pykx.SymbolVector` | A symbol vector is a collected list of symbol atoms. | ```pykx.q('`test`vector')``` | | `pykx.CharAtom` | A char atom holds a single ASCII or 8-but unicode character stored as 1 byte. | `pykx.q('"a"')` | | `pykx.CharVector` | A char vector is a collected list of char vectors. | `pykx.q('"test"')` | !!! info "Head to our [Text data](https://code.kx.com/q4m3/2_Basic_Data_Types_Atoms/#24-text-data) section for a deeper dive into the underlying text representation." -## Convert text to/from PyKX +## Convert text to/from KDB-X Python -To convert Pythonic text data to PyKX objects, use the `#!python pykx.SymbolAtom` and `#!python pykx.CharVector` functions as shown below: +To convert Pythonic text data to `pykx` objects, use the `#!python pykx.SymbolAtom` and `#!python pykx.CharVector` functions as shown below: ```python >>> import pykx as kx @@ -34,9 +34,9 @@ pykx.SymbolAtom(pykx.q('`test string')) pykx.CharVector(pykx.q('"test string"')) ``` -Alternatively, you use the automatic conversion function `#!python pykx.toq` which takes an incoming Python type and converts it to its analogous PyKX type. The following table shows the mapping between the two types: +Alternatively, you use the automatic conversion function `#!python pykx.toq` which takes an incoming Python type and converts it to its analogous KDB-X Python type. The following table shows the mapping between the two types: -| **Python Type**| **PyKX Type** | +| **Python Type**| **KDB-X Python Type** | |-------------|-----------------------------------| | `str` | `pykx.SymbolAtom` | | `byte` | `pykx.CharAtom`/`pykx.CharVector` | @@ -61,7 +61,7 @@ pykx.CharVector(pykx.q('"string"')) pykx.SymbolAtom(pykx.q('`bytes')) ``` -The `#!python pykx.toq` conversion is used by default when passing Python data to PyKX functions, for example: +The `#!python pykx.toq` conversion is used by default when passing Python data to `pykx` functions, for example: ```python >>> import pykx as kx @@ -81,11 +81,11 @@ pykx.SymbolAtom(pykx.q('`test')) pykx.CharVector(pykx.q('"test"')) ``` -## PyKX Under q +## KDB-X Python Under q For more information on executing Python code in a q process see [evaluate and execute python](../../pykx-under-q/intro.html#evaluate-and-execute-python) -Using text conversion under q we can convert PyKX text objects into q. This function call converts the Python `str` into a `SymbolAtom` +Using text conversion under q we can convert `pykx` text objects into q. This function call converts the Python `str` into a `SymbolAtom` ```q q)\l pykx.q @@ -139,7 +139,7 @@ q).pykx.qeval"'testtest'" q)`testtest ``` -A backtick `` ` `` can be used to convert a PyKX object to q. This uses the same underlying logic as .pykx.toq: +A backtick `` ` `` can be used to convert a `pykx` object to q. This uses the same underlying logic as .pykx.toq: ```q q)s:.pykx.eval["('test1', 'test2')"] @@ -151,9 +151,9 @@ For more detail on text conversion under q see our page on [.pykx.toq0](../../py ## Differences between `Symbol` and `Char` data objects -While there may appear to be limited differences between `#!python Symbol` and `#!python Char` representations of objects, the choice of underlying representation can have an impact on the performance and memory profile of many applications of PyKX. This section will describe a number of these differences and their impact in various scenarios. +While there may appear to be limited differences between `#!python Symbol` and `#!python Char` representations of objects, the choice of underlying representation can have an impact on the performance and memory profile of many applications of KDB-X Python. This section will describe a number of these differences and their impact in various scenarios. -Although `#!python Symbol` and `#!python Char` representations of objects might seem similar, the choice between them can significantly affect the performance and memory usage of many PyKX applications. This section exploreS the impact of these differences in various scenarios. +Although `#!python Symbol` and `#!python Char` representations of objects might seem similar, the choice between them can significantly affect the performance and memory usage of many KDB-X Python applications. This section exploreS the impact of these differences in various scenarios. ### Text access and mutability diff --git a/docs_archive/kx_for_databricks/clusterSetups.md b/docs_archive/kx_for_databricks/clusterSetups.md deleted file mode 100644 index 6ffdd07..0000000 --- a/docs_archive/kx_for_databricks/clusterSetups.md +++ /dev/null @@ -1,407 +0,0 @@ ---- -title: KX for Databricks cluster setups -description: How to use KX for Databricks -date: September 2024 -author: KX Systems, Inc., -tags: PyKX, Databricks, single node, multi node, workflows ---- - -# Cluster setups - -_The purpose of this page is to help you set up your single node or multi node clusters._ - -You can run KX for Databricks on both single node and multi-node computes. Choose the compute and RAM requirements depending on your data/pipeline requirements. - -For the best experience, read the [KX for Databricks Overview](introduction.md) and the [Get started](getStarted.md) guide first. - -Now let's explore examples for two types of cluster setups: - -1. [Single node workflow](#1-single-node-workflow) -2. [Multi node workflow](#2-multi-node-workflow) - -!!! info "Pre-requisites" - - The following pre-requisites must be satisfied in order to run the below notebooks: - - - Configuration of single node or multi node compute - - Latest version of PyKX must be installed - - QLIC env variable must point to a valid kc.lic file - - Load Trade, Quote & Execution data from a Databricks Unity Catalogue. - - For more information refer to the [Get Started documentation](getStarted.md). - -## 1. Single node workflow - -A single node compute runs spark locally and the driver works as both master and worker, with no worker nodes. -It is intended for jobs that use small amounts of data or non-distributed workloads such as single-node machine learning libraries. - -Import the required Python libraries: - -```python -import pykx as kx -import numpy as np -import pandas as pd -from datetime import datetime, timedelta -``` - -### Hourly OHLC - Pythonic vs. q magic - -!!! note "OHLC is the open, high, low, and close price for a given period." - -The example below calculates the hourly OHLC in two ways: using regular Python syntax and then the q magic. - -!!! example "" - - === "Pythonic" - - Calculate the OHLC using regular Python syntax: - - ```python - kx.q("{select Open:first price,High:max price,Low:min price,Close:last price by bucket:0D01 xbar time,sym from x}", trade).head() - ``` - - This should produce the following output: - - ``` - bucket sym Open High Low Close - 2024.07.04D04:00:00.000000000 AEF 617f 617f 617f 617f - BKR 93.21 93.21 93.21 93.21 - CLMT 2327.99 2327.99 2327.99 2327.99 - CMG 780f 780f 777.6 780f - F.PRA 333.99 336.79 332.6 336.69 - ``` - - === "q magic" - - Calculate the OHLC using q magic: - - ```python - %%q - select Open:first price,High:max price,Low:min price,Close:last price by bucket:0D01 xbar time,sym from trade - ``` - - This should produce the following output: - - ```python - bucket sym | Open High Low Close - ------------------------------------| ------------------------------- - 2024.07.04D04:00:00.000000000 AEF | 617 617 617 617 - 2024.07.04D04:00:00.000000000 BKR | 93.21 93.21 93.21 93.21 - 2024.07.04D04:00:00.000000000 CLMT | 2327.99 2327.99 2327.99 2327.99 - 2024.07.04D04:00:00.000000000 CMG | 780 780 777.6 780 - 2024.07.04D04:00:00.000000000 F.PRA | 333.99 336.79 332.6 336.69 - ``` - -Notice how both examples produce the same result. - -### VWAP - -!!! note "VWAP - The average price a security has traded at throughout the day, based on both volume and price." - -The example below calculates the volume-weighted average price (VWAP) using q magic: - -```python -%%q -select vwap:size wavg price by sym from trade -``` - -??? example "Output" - - ``` - sym | vwap - ------| -------- - AEF | 625.727 - AGND | 228.8684 - AUY | 251.6101 - AXJL | 494.2899 - AXX | 326.6949 - ``` - -### Volatility - -!!! note "Volatility - A statistical measure of the dispersion of returns for a given security or market index." - -The example below calculates volatility for the stock `AEF`. - -Define the functions used to calculate the volatility: - -```python -get_volatility_pykx = kx.q("{[bid;ask] {r:0^log[x]-log next x;sqrt ema[0.001] r*r } 0.5*ask+bid}") - -def get_volatility(quote_tbl, sym) -> dict: - subset_quotes = quote_tbl.loc[ - (quote_tbl["sym"] == sym) - & (quote_tbl["time"] >= timedelta(hours=9, minutes=25)) - & (quote_tbl["time"] < timedelta(hours=16, minutes=0)) - ] - return { - "datetime": subset_quotes["time"], - "volatility": get_volatility_pykx(subset_quotes["bid"], subset_quotes["ask"]) - } -``` - -Pass the `quote` table and the `AEF` symbol to the `get_volatility` function: - -```python -get_volatility(quote, 'AEF') -``` - -??? example "Output" - - ```python - {'datetime': pykx.TimestampVector(pykx.q('2024.07.04D09:25:03.491000000 2024.07.04D09:25:46.166000000 2024.07.04D09:27:..')), - 'volatility': pykx.FloatVector(pykx.q('0.003473635 0.003496871 0.003534634 0.003532866 0.003612577 0.003686721 0.003..'))} - ``` - -### Spread - -!!! note "Spread - The difference or gap that exists between two prices, rates, or yields." - -The example below calculates the moving average over the spread for the stock `AEF`. - -Define the functions used to calculate the moving average over the spread: - -```python -def get_mavg_of_spread(bid, ask, mavg_size: int): - spreads = ask - bid - return np.convolve(spreads, np.ones(mavg_size), mode="same") / mavg_size - -kx.q["mavg_of_spread"] = get_mavg_of_spread - -def get_spread(quotes: kx.Table, sym: str) -> kx.Table: - return kx.q.qsql.select( - quotes, - columns={"datetime": "time", "spread": "mavg_of_spread[bid; ask; 1000]", "bid":"bid", "ask":"ask"}, - where=[f'sym=`$"{sym}"', "bid>0", "ask>0", "time>=09:25", "time <16:00"] - ) -``` - -Pass the `quote` table and the `AEF` symbol to the `get_spread` function: - -```python -get_spread(quote,'AEF').head() -``` - -??? example "Output" - - ```python - datetime spread bid ask - 0 2024.07.04D09:25:03.491000000 15.07707 580.67 653.61 - 1 2024.07.04D09:25:46.166000000 15.07735 600f 630f - 2 2024.07.04D09:27:01.015000000 15.07766 588.88 625f - 3 2024.07.04D09:27:10.556000000 15.07844 580.67 653.61 - 4 2024.07.04D09:27:10.557000000 15.07873 580.67 653.61 - ``` - -### Slippage - -!!! note "Slippage - The difference between the expected price of a trade and the price at which the trade is executed." - -The example below calculates the Slippage based on the midpoint between the prevailing quoted bid and ask prices. - -Create the `merged` table by performing an as-of-join on the `execs` and `quote` tables: - -```python -merged = kx.q.aj(kx.SymbolVector(["sym", "time"]), execs, quote) -``` - -Calculate the Mid,Delta and Slippage columns: - -```python -merged["mid"] = 0.5 * (merged["bid"] + merged["ask"]) -merged["diff"] = np.where( - merged["side"] == "BUY", merged["mid"] - merged["price"], merged["price"] - merged["mid"] -) -merged["slippage"] = (merged["diff"] / merged["mid"]) * 10000 - -merged.head() -``` - -??? example "Output" - - ```python - date sym time price size side venue bid ask bsize asize mode ex mid diff slippage - 0 2022.03.31 GE.PRA 2024.07.04D09:30:02.676000000 397.6131 12 BUY venue1 396.51 397.96 1 1 R N 397.235 -0.3781 -9.518295 - 1 2022.03.31 MQC 2024.07.04D09:30:02.851000000 254.8359 258 SELL venue3 254.32 254.99 2 1 R V 254.655 0.1809 7.103729 - 2 2022.03.31 BKR 2024.07.04D09:30:03.426000000 93.64192 154 BUY venue2 93.64 93.72 1 1 R N 93.68 0.03808 4.064902 - 3 2022.03.31 GE.PRA 2024.07.04D09:30:04.489000000 397.5469 12 SELL venue2 397.21 398.8 1 2 R T 398.005 -0.4581 -11.50991 - 4 2022.03.31 DXGE 2024.07.04D09:30:06.677000000 570.1041 60 SELL venue3 569.08 571.34 2 3 R P 570.21 -0.1059 -1.857211 - ``` - -## 2. Multi node workflow - -A multi node (distributed) compute consists of one driver node and one or more worker nodes. - -The driver node maintains state information of all notebooks attached to the compute, maintains the SparkContext, interprets all the commands you run from a notebook or a library on the compute, and runs the Apache Spark master that coordinates with the Spark executors. -Worker nodes run the Spark executors and other services required for proper functioning compute. - -Use multi-node compute for larger jobs with distributed workloads. - -Before you start, make sure you meet the pre-requisites at the top of this page. - -### Import required libraries - -Import a PySpark and Python libraries by running: - -```python -import pykx as kx -from pyspark.sql import functions as F -from pyspark.sql import SparkSession -from pyspark.sql.functions import pandas_udf, PandasUDFType, lit -from pyspark.sql.types import StructType, StructField, DoubleType, StringType, DateType, IntegerType -spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", True) -import copy as cp -``` - -### Prepare the data - -!!! note "The below examples will make use of the full dataset while using PySpark and a subset of the data while using the Python Driver." - -Create `trade_sdf_all_syms` and `quote_sdf_all_syms` Spark Dataframes containing trade and quote data respectively using Spark SQL: - -```python -trade_sdf_all_syms = spark.sql('SELECT TTime as time, Symbol as sym, cast(Trade_Volume AS INT) as size, Trade_Price as price FROM default.nyse_trades') -quote_sdf_all_syms = spark.sql('SELECT TTime as time, Symbol as sym, Bid_Size as bsize, Bid_Price as bid, Offer_Size as asize, Offer_Price as ask FROM default.nyse_quotes') -``` - -Filter the data to the a subset of the tickers: - -```python -SYMBOLS = ['GOOG', 'AAPL', 'FB', 'AMZN', 'MSFT'] -trade_sdf_few_syms = trade_sdf_all_syms.where(F.col("Symbol").isin(SYMBOLS)).orderBy(["Symbol","TTime"]) -quote_sdf_few_syms = quote_sdf_all_syms.where(F.col("Symbol").isin(SYMBOLS)).orderBy(["Symbol","TTime"]) -``` - -Convert the Spark Dataframes to Pandas Dataframes: - -```python -trade_pdf = trade_sdf_few_syms.toPandas() -quote_pdf = quote_sdf_few_syms.toPandas() -``` - -### Slippage markouts (Python Driver/Single Node) - -!!! note "Slippage - The difference between the expected price of a trade and the price at which the trade is executed." - -In the below example, we will use PyKX to calculate the execution slippage markouts for a subset of the data (5 tickers) using just the Python Driver (Single node). - -??? example "Define the required functions used to calculate the slippage markouts" - - ```python - # adjust the time column in the past/future by offset e.g 10 20 30 seconds, 1 5 10 minutes - def time_offset(trade, offset): - offset_trade = cp.copy(trade) - offset_trade['time'] = (offset_trade['time'] + offset).sorted() - return offset_trade[['sym', 'time']] - - # compute N markouts and calculate slippage using mid price - def trade_to_mid_markouts(trade, quote): - import os - os.environ['QLIC'] = '/dbfs/tmp/' - import pykx as kx - - trade = kx.toq(trade) - quote = kx.toq(quote) - - trade = trade.sort_values(['time']).grouped('sym') - quote = quote.sort_values(['time']).grouped('sym') - quote['mid'] = (quote['ask'] + quote['bid']) / 2 - - # markouts we are interested in - seconds = [1, 10, 30] - minutes = [1, 5, 10, 30] - - qseconds = kx.toq(seconds).cast(kx.SecondVector) - qminutes = kx.toq(minutes).cast(kx.MinuteVector) - - for s, q in zip(seconds, qseconds): - trade['tp'+str(s)+'s'] = trade['price'] - time_offset(trade, q).merge_asof(quote[['sym', 'time', 'mid']], on=['sym', 'time'])['mid'] - trade['tm'+str(s)+'s'] = trade['price'] - time_offset(trade, -q).merge_asof(quote[['sym', 'time', 'mid']], on=['sym', 'time'])['mid'] - - for s, q in zip(minutes, qminutes): - trade['tp'+str(s)+'m'] = trade['price'] - time_offset(trade, q).merge_asof(quote[['sym', 'time', 'mid']], on=['sym', 'time'])['mid'] - trade['tm'+str(s)+'m'] = trade['price'] - time_offset(trade, q).merge_asof(quote[['sym', 'time', 'mid']], on=['sym', 'time'])['mid'] - - return trade.pd() - ``` - -Pass the `trade_pdf` and `quote_pdf` Pandas Dataframes to the `trade_to_mid_markouts` function in order to calculate the slippage: - -```python -result = trade_to_mid_markouts(trade_pdf, quote_pdf) - -print('Executions input row count - ', f'{trade_pdf.shape[0]:,}') -print('Market quotes input row count - ', f'{quote_pdf.shape[0]:,}') -print('TCA report result row count - ', f'{kx.q.count(result).np():,}') - -result.tail() -``` - -!!! note "Since we have supplied a Pandas dataframe, the workload will be performed by only the Python driver." - -??? example "Output" - - ```python - Executions input row count - 1,851,406 - Market quotes input row count - 27,424,147 - TCA report result row count - 1,851,406 - - time sym size price tp1s tm1s tp10s tm10s tp30s tm30s tp1m tm1m tp5m tm5m tp10m tm10m tp30m tm30m - 1851401 2022-10-03 19:59:55.559408 AAPL 1 142.935 0.000 0.000 -0.025 0.010 -0.025 0.045 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 - 1851402 2022-10-03 19:59:58.444119 MSFT 25 241.700 0.185 0.185 0.185 0.185 0.185 3.200 0.185 0.185 0.185 0.185 0.185 0.185 0.185 0.185 - 1851403 2022-10-03 19:59:59.146947 AAPL 1 142.935 -0.025 0.000 -0.025 0.010 -0.025 0.055 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 - 1851404 2022-10-03 19:59:59.551589 MSFT 1 241.700 0.185 0.185 0.185 0.185 0.185 3.200 0.185 0.185 0.185 0.185 0.185 0.185 0.185 0.185 - 1851405 2022-10-03 19:59:59.668143 MSFT 150 241.700 0.185 0.185 0.185 0.185 0.185 3.200 0.185 0.185 0.185 0.185 0.185 0.185 0.185 0.1815 - ``` - -### Slippage markouts (PySpark/Multi node) - -In the below example, we will use PyKX to calculate the execution slippage markouts for all of the data (11686 tickers) using PySpark (Multi node). - -Prepare the schema for the results dataframe: - -```python -full_schema = "time timestamp, sym string, size int, price double" - -seconds = [1, 10, 30] -minutes = [1, 5, 10, 30] - -for s in seconds: - full_schema += ', tp'+str(s)+'s float' - full_schema += ', tm'+str(s)+'s float' -for m in minutes: - full_schema += ', tp'+str(m)+'m float' - full_schema += ', tm'+str(m)+'m float' -``` - -Pass the `trade_sdf_all_syms` and `quote_sdf_all_syms` Spark Dataframes to the `trade_to_mid_markouts` function in order to calculate the slippage: - -```python -result_pyspark = trade_sdf_all_syms.groupby('sym').cogroup(quote_sdf_all_syms.groupby('sym')).applyInPandas(trade_to_mid_markouts, schema=full_schema) - -print('Executions input row count - ', f'{trade_sdf_all_syms.count():,}') -print('Market quotes input row count - ', f'{quote_sdf_all_syms.count():,}') -print('TCA report result row count - ', f'{result_pyspark.count():,}') - -result_pyspark.show(5) -``` - -!!! note "Since we have supplied a Spark dataframe, the workload will be farmed out to the configured worker nodes for processing before returning the result to the Python driver." - -??? example "Output" - - ```python - Executions input row count - 77,026,358 - Market quotes input row count - 1,541,781,424 - TCA report result row count - 77,026,358 - - +--------------------+------+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------+------+-----+-----+ - | time| sym|size|price| tp1s| tm1s|tp10s|tm10s|tp30s|tm30s| tp1m| tm1m| tp5m| tm5m| tp10m| tm10m|tp30m|tm30m| - +--------------------+------+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------+------+-----+-----+ - |2022-10-03 09:16:...|AKIC U| 85| 9.95| 0.21| 0.21| 0.21| 0.21| 0.21| 0.21| 0.21| 0.21| 0.17| 0.17|-0.265|-0.265|-0.03|-0.03| - |2022-10-03 11:20:...|AKIC U| 9| 9.96| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| - |2022-10-03 11:56:...|AKIC U| 21| 9.96| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -0.02| -0.02|-0.02|-0.02| - |2022-10-03 13:36:...|AKIC U| 21| 9.95|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01| -0.01| -0.01|-0.01|-0.01| - |2022-10-03 13:36:...|AKIC U| 21| 9.95|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01| -0.01| -0.01|-0.01|-0.01| - +--------------------+------+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------+------+-----+-----+ - ``` \ No newline at end of file diff --git a/docs_archive/kx_for_databricks/getStarted.md b/docs_archive/kx_for_databricks/getStarted.md deleted file mode 100644 index 4d0654c..0000000 --- a/docs_archive/kx_for_databricks/getStarted.md +++ /dev/null @@ -1,228 +0,0 @@ ---- -title: Get started with KX for Databricks -description: How to install and use KX for Databricks -date: September 2024 -author: KX Systems, Inc., -tags: PyKX, setup, install, ---- - -# Get started with KX for Databricks - -_This page explains how to deploy and get started with KX for Databricks._ - -1. [Install PyKX](#1-install-pykx) -2. [Licensing](#2-licensing) -3. [Load data using Spark Dataframes](#3-load-data-using-spark-dataframes) -4. [PyKX pythonic vs. q magic (%%)](#4-pykx-pythonic-vs-q-magic-) - - -## 1. Install PyKX - -Install PyKX on the cluster using one of the following methods: - -!!! Note "" - - === "Install PyKX from PyPI" - - Create a new notebook and install using `pip`: - - ```python - pip install pykx - ``` - - === "Install PyKX from Anaconda" - - Create a new notebook and install using `conda`: - - ```python - conda install -c kx pykx - ``` - - === "Install PyKX from the Python Wheel" - - Download the Python Wheel file from the [Download Portal](https://portal.dl.kx.com/assets/pypi/pykx) and store it on the cluster: - - ```shell - > ls /path/to/folder - pykx-2.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - ``` - - Create a new notebook and install from the wheel file: - - ```python - pip install /path/to/folder/pykx-2.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - ``` -!!! tip "Tip: After installing the packages, you may need to restart the kernel by running the following command:" - - - ```python - dbutils.library.restartPython() - ``` - -## 2. Licensing - -To use all PyKX functionalities, you need to download and install a KDB Insights license. - -1. Choose a license type: - - !!! Note "" - - === "Personal License" - - For personal usage, navigate to [Personal License Download](https://kx.com/kdb-insights-personal-edition-license-download/) and complete the form. - - === "Commercial License" - - For commercial usage, contact your KX sales representative, email [sales@kx.com](mailto:sales@kx.com), or apply through [Book demo](https://kx.com/book-demo). - - - !!! info "More on the [license installation process](https://code.kx.com/pykx/2.5/getting-started/installing.html#2-install-a-kdb-insights-license)." - -2. On receipt of an email from KX, download and save the `kc.lic` license file to a secure location on your cluster: - - ```shell - > ls /path/to/folder - kc.lic - ``` - -3. Set an environment variable `QLIC` pointing to the folder containing the license file: - - ```python - import os - os.environ['QLIC'] = '/path/to/folder' - ``` - That's it! To verify if you successfully installed PyKX, run the following: - - ```python - import pykx as kx - print('PyKX Version: ', kx.__version__) - ``` - You should have the following output: - - ```python - PyKX Version: 2.5.1 - ``` - !!! note "Note: The license requires internet connectivity." - - Update the workspace security policies to whitelist the following: - - - The remote servers should treat kdb+ on-demand as a CDN-fronted service and employ DNS-based filtering. - - The current list of hostnames required for kdb+ on-demand operation is {h,g}.kdbod.{com,net,org}. - -## 3. Load data using Spark Dataframes - -**a. From the Databricks Unity Catalogue:** - -```python -dfTrades = spark.table('..') -dfQuotes = spark.table('..') -``` - -**b. Directly from a CSV file:** - -```python -dfTrades = spark.read.format("csv").option("header", "true").option("delimiter", ",").load("/path/to/file") -dfQuotes = spark.read.format("csv").option("header", "true").option("delimiter", ",").load("/path/to/file") -``` - -To convert the Spark Dataframe to a Pandas Dataframe, use the `toPandas()` function: - -```python -pandas_trade = dfTrades.toPandas() -pandas_quote = dfQuotes.toPandas() -``` - -Convert the Pandas Dataframe to a q table with the `toq()` function: - -```python -pykx_trades = kx.toq(pandas_trade) -pykx_quotes = kx.toq(pandas_quote) -``` - -## 4. PyKX pythonic vs. q magic (%%) - -Reference q functions in PyKX with the standard python syntax: - -```python -kx.q('til 10') -``` - -This produces a sequence of numbers from 0 to 9: - -```python -pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) -``` - -Another way to achieve this is by using native q by leveraging q magic (`%%q`) : - -```python -%%q -til 10 -``` - -You will get the same output: - -```python -0 1 2 3 4 5 6 7 8 9 -``` - -??? example "Simple load and query example using PyKX" - - Consider a `samples` catalogue with a `finance` schema that has a `trades` table. Load data from the table into a Spark Dataframe `dfTrade`: - - ```python - dfTrade = spark.table('samples.finance.trades') - ``` - - Next, convert the Spark Dataframe to a Pandas Dataframe `panda_trade` using the `toPandas()` function: - - ```python - panda_trade = dfTrade.toPandas() - ``` - - Using the `toq()` function to convert the Pandas Dataframe to a q table `pykx_trade`: - - ```python - pykx_trade = kx.toq(panda_trade) - ``` - - To perform a simple OHLC query on the data, use regular python syntax: - - ```python - kx.q("{select Open:first price,High:max price,Low:min price,Close:last price by bucket:0D01 xbar time,sym from x}", pykx_trade).head() - ``` - - This should produce the following output: - - ```python - bucket sym Open High Low Close - 2017.08.31D00:00:00.000000000 A 347.8271 349.256 346.0935 348.2853 - AAN 347.4742 349.1463 346.2195 348.3558 - ACA 347.4005 348.4313 346.5526 347.2568 - ACP 346.8897 350.9313 345.8342 347.8243 - ``` - - You can achieve the same result with q magic by storing the table: - - ```python - kx.q['pykx_trade'] = pykx_trade - ``` - - and performing the same query: - - ```python - %%q - select Open:first price,High:max price,Low:min price,Close:last price by bucket:0D01 xbar time,sym from pykx_trade - ``` - - In the end, you get the same output: - - ```python - bucket sym | Open High Low Close - ----------------------------------| ----------------------------------- - 2017.08.31D00:00:00.000000000 A | 347.8271 349.256 346.0935 348.2853 - 2017.08.31D00:00:00.000000000 AAN | 347.4742 349.1463 346.2195 348.3558 - 2017.08.31D00:00:00.000000000 ACA | 347.4005 348.4313 346.5526 347.2568 - 2017.08.31D00:00:00.000000000 ACP | 346.8897 350.9313 345.8342 347.8243 - 2017.08.31D00:00:00.000000000 ACRE| 349.8609 351.621 345.0562 345.9471 - ``` \ No newline at end of file diff --git a/docs_archive/kx_for_databricks/introduction.md b/docs_archive/kx_for_databricks/introduction.md deleted file mode 100644 index f9f1149..0000000 --- a/docs_archive/kx_for_databricks/introduction.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: KX for Databricks -description: Kx, PyKX, and Databricks -date: September 2024 -author: KX Systems, Inc., -tags: kx, PyKX, Databricks ---- - -# KX for Databricks -_This page briefly describes PyKX on Databricks use cases and capabilities._ - -KX for Databricks brings the full capability of KX's time series engine to Databricks. KX's PyKX, the Python-first interface already familiar to data scientists and quants, turbocharges workloads and research on use cases such as: - -- Algorithmic trading strategy development and backtesting​ -- Large-scale pre- and post-trade analytics -- Macro-economic research and analysis using large datasets​ -- Cross-asset analytics with deep historical data​ - -KX for Databricks leverages PyKX directly inside Databricks, for Python and distributed Spark workloads. Use it on existing Delta Lake datasets without external KX dependencies, or choose to flexibly interop with other KX products to leverage the power of your broader KX systems estate. - -![KX for Databricks reference architecture](kx-databricks-refarch.png) - -## Get started and workflows - -Refer to the [Get started](getStarted.md) page if you wish to complete the following actions: - -- [Install](getStarted.md#1-install-pykx) PyKX on Databricks. -- Check what type of [license](getStarted.md#2-licensing) you need. -- [Load data](getStarted.md#3-load-data-using-spark-dataframes) using Spark Dataframes. -- [Use PyKX pythonic vs. q magic (%%)](getStarted.md#4-pykx-pythonic-vs-q-magic-). - -Next, learn how to [set up a cluster](clusterSetups.md), depending on the type of workflow you need: - -- Work with a [single node/driver](clusterSetups.md#1-single-node-workflow) on analytic calculation of OHLC, VWAP pricing, volatility, spread, and trade execution analysis (slippage). -- Work with a [multi/distributed nodes](clusterSetups.md#2-multi-node-workflow) cluster on slippage calculation with PySpark. diff --git a/docs_archive/kx_for_databricks/kx-databricks-refarch.png b/docs_archive/kx_for_databricks/kx-databricks-refarch.png deleted file mode 100644 index f24db3b..0000000 Binary files a/docs_archive/kx_for_databricks/kx-databricks-refarch.png and /dev/null differ diff --git a/examples/README.md b/examples/README.md index acc649e..c9420d8 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,6 @@ # Examples -This folder contains a number of subfolders/files containing example uses of the PyKX interface. These examples are not intended to show fully all that can be achieved by the interface but rather act as inspiration for some of the potential use cases of the interface. +This folder contains a number of subfolders/files containing example uses of the KDB-X Python interface. These examples are not intended to show fully all that can be achieved by the interface but rather act as inspiration for some of the potential use cases of the interface. This `README.md` is intended to act as a guide for the contents within each of the individual examples explaining the use case and how the example should be run @@ -34,7 +34,7 @@ This example is the best centralised representation of the wide ranging function The ipc folder contains any IPC specific examples which may be useful for users. -This IPC example provided at present outlines the use of PyKX as a real-time engine, ingesting data as a subscriber to an external q process, evaluating an analytic written in Python and publishing the result of this analytic to another external q process. +This IPC example provided at present outlines the use of KDB-X Python as a real-time engine, ingesting data as a subscriber to an external q process, evaluating an analytic written in Python and publishing the result of this analytic to another external q process. This example works in the presence or absence of a q license and is fully outlined in `ipc/README.md` @@ -50,7 +50,7 @@ The compress and encrypt demo outlined by the `compress_and_encrypt.py` file and $ python compress_and_encrypt.py ``` -This example provides an example of how a user of PyKX can make use of the supported compression algorithms provided by q +This example provides an example of how a user of KDB-X Python can make use of the supported compression algorithms provided by q * gzip * snappy diff --git a/examples/ipc/README.md b/examples/ipc/README.md index c37fe36..9bb5691 100644 --- a/examples/ipc/README.md +++ b/examples/ipc/README.md @@ -1,6 +1,6 @@ # IPC interface example -The purpose of this example is to provide a quickstart for interfacing with external q processes using PyKX. The example presented should operate in the presence or absence of a `k4.lic`, and as such is intended to show the flexibility of this interface for users who had previously used both PyKX and those who are familiar with qPython. +The purpose of this example is to provide a quickstart for interfacing with external q processes using KDB-X Python. The example presented should operate in the presence or absence of a `k4.lic`, and as such is intended to show the flexibility of this interface for users who had previously used both KDB-X Python and those who are familiar with qPython. ## Quickstart @@ -27,7 +27,7 @@ q) $ q tick/feed.q q) -// Start the q process to receive data from PyKX +// Start the q process to receive data from KDB-X Python $ q -p 5130 q) ``` diff --git a/examples/quick_demo.txt b/examples/quick_demo.txt index 022ddde..dc92242 100644 --- a/examples/quick_demo.txt +++ b/examples/quick_demo.txt @@ -1,4 +1,4 @@ -# PyKX Demo +# KDB-X Python Demo # Note that here "K object" refers to an object for which # `isinstance(obj, pykx.K)` is `True`. "K objects" from q @@ -82,8 +82,8 @@ kx.q.Q.addr('localhost') kx.q.z.h -# By default PyKX allows users to define extensions modules -# which are loaded in automatically from QHOME when PyKX is +# By default KDB-X Python allows users to define extensions modules +# which are loaded in automatically from QHOME when KDB-X Python is # initialised. To run this example add the script # 'extension/extension.q' to QHOME and initialise the interface diff --git a/examples/subscriber/readme.md b/examples/subscriber/readme.md index 292ed81..2078d24 100644 --- a/examples/subscriber/readme.md +++ b/examples/subscriber/readme.md @@ -1,13 +1,13 @@ --- -title: PyKX installation guide +title: KDB-X Python installation guide description: Subscriber Examples date: October 2024 author: KX Systems, Inc., tags: subscriber, synchronous, asynchronous, PyKX --- -# PyKX Subscribing to a `q` Process +# KDB-X Python Subscribing to a `q` Process -_This example demonstrates using `PyKX` to setup a python process as a subscriber to data messages published from a q process._ +_This example demonstrates using `KDB-X Python` to setup a python process as a subscriber to data messages published from a q process._ ## Pre-requisites diff --git a/licenses/QSO.txt b/licenses/QSO.txt index ee31923..9e57305 100644 --- a/licenses/QSO.txt +++ b/licenses/QSO.txt @@ -1,7 +1,7 @@ Software License for q.so -Version Number: 1.0 -Date Last Revised: April 2023 +Version Number: 1.1 +Date Last Revised: June 2026 Software License Agreement for use of q.so (“Agreement”) @@ -13,7 +13,7 @@ This Agreement is made between KX Systems, Inc. (“KX” or “we”) and the E  1. LICENSE GRANTS -1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. +1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX or KDB-X software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX or KDB-X software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. 1.2 Software Use Restrictions. End User may not: (a) modify any part of the Software or create derivative works thereof, (b) sell, lease, license or distribute the Software to any third party, (c) attempt to decompile, disassemble or reverse engineer the Software, (d) copy the Software, except for purposes of installing and executing it within the limitations set out at clause 1.1, (e) use or attempt to use the Software in any way that is unlawful or fraudulent or has any unlawful or fraudulent purpose or effect, (f) use or attempt to use the Software in any way that would breach the license granted herein. diff --git a/mkdocs.yml b/mkdocs.yml index 6e2f759..e70c393 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,7 +1,7 @@ --- -site_name: 'PyKX' +site_name: 'KDB-X Python Documentation' site_author: 'KX' -site_description: 'PyKX Documentation' +site_description: 'KDB-X Python Documentation' site_url: 'https://code.kx.com/pykx' copyright: '© 2026 KX Systems, Inc. KX, KDB-X, and kdb+ are registered trademarks of KX Systems, Inc., a subsidiary of KX Software Limited.' site_dir: 'public' @@ -9,7 +9,7 @@ dev_addr: 'localhost:8080' use_directory_urls: false -# Keep commented out until PyKX is open-source +# Keep commented out until KDB-X Python is open-source repo_url: 'https://github.com/kxsystems/pykx' edit_uri: 'edit/main/docs/' @@ -147,7 +147,56 @@ plugins: - examples/db-management.ipynb - examples/streaming/Evolving System.ipynb - examples/charting.ipynb - + - redirects: + redirect_maps: + 'api/console.md': 'api/pykx-execution/console.md' + 'api/ctx.md': 'api/pykx-execution/ctx.md' + 'api/embedded_q.md': 'api/pykx-execution/embedded_q.md' + 'api/pykx-save-load/read.md': 'api/pykx-save-load/fileio.md' + 'api/pykx-save-load/write.md': 'api/pykx-save-load/fileio.md' + 'api/pykx_under_q.md': 'api/pykx-execution/ctx.md' + 'api/q/q.md': 'api/pykx-execution/q.md' + 'api/read.md': 'api/pykx-save-load/fileio.md' + 'api/toq.md': 'api/pykx-q-data/toq.md' + 'api/type_conversions.md': 'api/pykx-q-data/type_conversions.md' + 'api/wrappers.md': 'api/pykx-q-data/wrappers.md' + 'api/write.md': 'api/pykx-save-load/fileio.md' + 'beta-features/compress-encypt.md': 'user-guide/advanced/compress-encrypt.md' + 'beta-features/db-management.md': 'user-guide/advanced/database/index.md' + 'beta-features/remote-functions.md': 'user-guide/advanced/remote-functions.md' + 'beta-features/streamlit.md': 'user-guide/advanced/streamlit.md' + 'beta-features/threading.md': 'user-guide/advanced/threading.md' + 'changelog.md': 'release-notes/changelog.md' + 'comparisons.md': 'getting-started/what_is_pykx.md' + 'evaluating.md': 'user-guide/fundamentals/evaluating.md' + 'examples/threaded_execution/index.md': 'examples/threaded_execution/threading.md' + 'examples/threaded_execution/threaded_execution.md': 'examples/threaded_execution/threading.md' + 'extras/comparisons.md': 'getting-started/what_is_pykx.md' + 'extras/faq.md': 'help/faq.md' + 'extras/known_issues.md': 'help/issues.md' + 'faq.md': 'help/faq.md' + 'getting-started/interface_overview.md': 'getting-started/what_is_pykx.md' + 'getting-started/q_magic_command.md': 'examples/jupyter-integration.ipynb' + 'indexing.md': 'user-guide/fundamentals/indexing.md' + 'installing.md': 'getting-started/installing.md' + 'interface_overview.md': 'getting-started/what_is_pykx.md' + 'known_issues.md': 'help/issues.md' + 'licenses.md': 'license.md' + 'limitations.md': 'help/issues.md' + 'modes.md': 'user-guide/advanced/modes.md' + 'nulls_and_infinities.md': 'user-guide/fundamentals/nulls_and_infinities.md' + 'parallelization.md': 'user-guide/advanced/performance.md' + 'performance.md': 'user-guide/advanced/performance.md' + 'pykx-under-q/known_issues.md': 'help/issues.md' + 'troubleshooting.md': 'help/troubleshooting.md' + 'user-guide/advanced/environment_variables.md': 'user-guide/configuration.md' + 'user-guide/advanced/limitations.md': 'help/issues.md' + 'user-guide/advanced/running_under_q.md': 'pykx-under-q/intro.md' + 'user-guide/fundamentals/querying.md': 'user-guide/fundamentals/query/index.md' + 'user-guide/index.md': 'index.md' + '1.0/licenses.md': 'license.md' + '1.5/api/type_conversion.md': 'api/pykx-q-data/type_conversions.md' + '3.0/pykx-under-q/type_conversions.md': 'api/pykx-q-data/type_conversions.md' theme: name: material @@ -188,7 +237,7 @@ nav: - Install: getting-started/installing.md - Quickstart guide: getting-started/quickstart.md - Learn: - - What is PyKX?: getting-started/what_is_pykx.md + - What is KDB-X Python?: getting-started/what_is_pykx.md - Objects and attributes: learn/objects.md - Databases: user-guide/advanced/database/index.md - Modes of operation: user-guide/advanced/modes.md @@ -197,7 +246,7 @@ nav: - Glossary: extras/glossary.md - Blogs, articles, videos: blogs.md - How to: - - Configure PyKX: user-guide/configuration.md + - Configure KDB-X Python: user-guide/configuration.md - Interact with data: - Create objects: user-guide/fundamentals/creating.md - Use objects: user-guide/fundamentals/evaluating.md @@ -231,7 +280,7 @@ nav: - Import existing q functions: user-guide/advanced/context_interface.md - Run remote functions: user-guide/advanced/remote-functions.md - Serialize data: user-guide/advanced/serialization.md - - Use PyKX in subprocesses: user-guide/advanced/subprocess.md + - Use KDB-X Python in subprocesses: user-guide/advanced/subprocess.md - Run q code multithreaded: user-guide/advanced/threading.md - Manage your license: user-guide/advanced/license.md - Use Python in a q process: pykx-under-q/intro.md @@ -258,11 +307,12 @@ nav: - Utility functions: api/util.md - Reimporter module: api/reimporting.md - License management: api/license.md - - PyKX under q: pykx-under-q/api.md + - Module importing: api/module.md + - KDB-X Python under q: pykx-under-q/api.md - Data types and conversions: - - Python to PyKX: api/pykx-q-data/toq.md + - Python to KDB-X: api/pykx-q-data/toq.md - Wrappers: api/pykx-q-data/wrappers.md - - PyKX to Python: api/pykx-q-data/type_conversions.md + - KDB-X to Python: api/pykx-q-data/type_conversions.md - Pandas API: user-guide/advanced/Pandas_API.ipynb - Exceptions: api/exceptions.md - Integrations: @@ -271,29 +321,29 @@ nav: - Python charting libraries: examples/charting.ipynb - Jupyter Integration: examples/jupyter-integration.ipynb - Examples: - - PyKX Introduction Notebook: examples/interface-overview.ipynb + - KDB-X Python Introduction Notebook: examples/interface-overview.ipynb - Subscriber: examples/subscriber/readme.md - Compress and encrypt: examples/compress_and_encrypt/readme.md - Database creation and management: examples/db-management.ipynb - IPC: examples/ipc/README.md - - PyKX as a server: examples/server/server.md + - KDB-X Python as a server: examples/server/server.md - Real-time streaming: examples/streaming/index.md - Multithreaded execution: examples/threaded_execution/threading.md - Asynchronous Queries: examples/AsynchronousQueries/async_querying.md - Releases: - Release notes: - - PyKX: release-notes/changelog.md - - PyKX under q: release-notes/underq-changelog.md + - KDB-X Python: release-notes/changelog.md + - KDB-X Python under q: release-notes/underq-changelog.md - Deprecations: release-notes/deprecations.md - - 2.x -> 3.x Upgrade : upgrades/2030.md + - 3.* to 4.* Migration : upgrades/3040.md + - 2.* to 3.* Upgrade : upgrades/2030.md - Beta features: - Introduction: beta-features/index.md - PyTorch Conversions: beta-features/torch.md - Help and Support: - Troubleshooting: help/troubleshooting.md - FAQ: help/faq.md - - PyKX support: help/support.md + - KDB-X Python support: help/support.md - Issues and limitations: help/issues.md - - PyKX license: license.md + - KDB-X Python license: license.md - Contributors: contributors.md - diff --git a/pyproject.toml b/pyproject.toml index 567110c..bc1922a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "pykx" description = "An interface between Python and q" readme = "README.md" -requires-python = ">=3.7" +requires-python = ">=3.9" license = { file = "LICENSE.txt" } authors = [ { name = "KX", email = "pykx@kx.com" }, @@ -31,11 +31,12 @@ classifiers = [ "Programming Language :: Cython", "Programming Language :: Other", "Programming Language :: Other Scripting Engines", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: Implementation :: CPython", # "Programming Language :: Python :: Implementation :: PyPy", @@ -48,17 +49,14 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "numpy~=1.20; python_version=='3.7'", - "numpy~=1.22; python_version=='3.8'", "numpy~=1.22; python_version=='3.9'", "numpy>=1.22; python_version=='3.10'", "numpy>=1.23; python_version=='3.11'", "numpy>=1.26; python_version=='3.12'", "numpy>=1.26; python_version=='3.13'", "numpy>=2.3.0; python_version=='3.14'", - "pandas>=1.2, <2.0.0; python_version=='3.8'", - "pandas>=1.2, <3.0; python_version>'3.8' and python_version<'3.14'", - "pandas>= 2.3.3, <3.0; python_version=='3.14'", + "pandas>=1.2; python_version>'3.8' and python_version<'3.14'", + "pandas>= 2.3.3; python_version=='3.14'", "pytz>=2022.1", "toml~=0.10.2", "dill>=0.2.0", @@ -80,6 +78,7 @@ doc = [ "mkdocs-exclude-search==0.6.6", "mkdocs-jupyter~=0.24", "mkdocs-material~=9.4.5", + "mkdocs-redirects>=1.2.0", "mkdocs-render-swagger-plugin==0.0.3", "mkdocs-spellcheck==0.2.0", "mkdocstrings[python]==0.18.0", @@ -112,7 +111,7 @@ dashboards = [ "ast2json~=0.3", ] streamlit = [ - "streamlit~=1.28; python_version>'3.7'" + "streamlit~=1.28" ] torch = [ "torch>2.1; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version <= '3.12'" @@ -145,21 +144,14 @@ changelog = "https://code.kx.com/pykx/release-notes/changelog.html" [build-system] requires = [ "Cython~=3.0.0", - "numpy~=1.20.0; python_version=='3.7'", # Use numpy version 1.20.x for building the python 3.7 wheel - "numpy~=1.22.0; python_version=='3.8'", # Use the highest patch version of numpy 1.22.x, this will still support a user using numpy version 1.22.0 "numpy~=2.0; python_version=='3.9'", "numpy~=2.0; python_version=='3.10'", "numpy~=2.0; python_version=='3.11'", "numpy~=2.0; python_version=='3.12'", "numpy~=2.0; python_version=='3.13'", "numpy>=2.3.0; python_version=='3.14'", - "numpy~=2.0; python_version=='3.11'", - "numpy~=2.0; python_version=='3.12'", - "numpy~=2.0; python_version=='3.13'", - "setuptools~=68.0.0; python_version=='3.7'", - "setuptools~=69.0.2; python_version!='3.7'", - "setuptools-scm[toml]~=7.1.0; python_version=='3.7'", - "setuptools-scm[toml]~=8.0; python_version!='3.7'", + "setuptools~=69.0.2", + "setuptools-scm[toml]~=8.0", "tomli>=2.0.1", "wheel>=0.42.0", ] diff --git a/setup.py b/setup.py index ffb1742..3f2cd03 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ -Handles building/packaging PyKX. Generally pip should be used instead of +Handles building/packaging KDB-X Python. Generally pip should be used instead of executing this file directly. """ @@ -256,8 +256,7 @@ def ext(name: str, ]), ] - if py_minor_version >= 8: # python 3.8 or higher is required for NEP-49 - exts.append(ext('_numpy', numpy=True, cython=False, libraries=['dl', *windows_libraries])) + exts.append(ext('_numpy', numpy=True, cython=False, libraries=['dl', *windows_libraries])) exts.append(ext('numpy_conversions', numpy=True, cython=False, @@ -268,59 +267,31 @@ def ext(name: str, libraries=['dl', *windows_libraries])) with cd(src_dir/'q.so'/'qk'): - [ - shutil.copy(f, f'../../lib/4-1-libs/{f}') - for f in - [str(f) for f in os.listdir() if os.path.isfile(f) and (str(f) != 'q.k') and ('pykx_init' not in str(f))] # noqa: E501 - ] [ shutil.copy(f, f'../../lib/{f}') for f in - [str(f) for f in os.listdir() if os.path.isfile(f) and (str(f) != 'q.k') and ('pykx_init' not in str(f))] # noqa: E501 + [str(f) for f in os.listdir() if os.path.isfile(f) and ('pykx_init' not in str(f))] ] [ shutil.copy('pykx_init.q_', '../../' + p + 'pykx_init.q_') for p in - ['', 'lib/', 'lib/4-1-libs/'] + ['', 'lib/'] ] for p in ('l64', 'l64arm', 'm64', 'm64arm', 'w64'): - with cd(src_dir/'q.so'/'libs'/'4-1'/p): - [ - shutil.copy(f, f'../../../../lib/4-1-libs/{p}/{f}') - for f in - [str(f) for f in os.listdir() - if str(f) != 'symbols.txt' and not os.path.exists(f'../../../../4-1-libs/{p}/{f}')] - ] - - with cd(src_dir/'q.so'/'libs'/'4-0'/p): + with cd(src_dir/'q.so'/'libs'/'kdbx'/p): [ shutil.copy(f, f'../../../../lib/{p}/{f}') for f in [str(f) for f in os.listdir() - if str(f) != 'symbols.txt' and not os.path.exists(f'../../../../../{p}/{f}')] - ] - - with cd(src_dir/'lib'/p): - [ - shutil.copy(f, f'../4-1-libs/{p}/{f}') - for f in - [str(f) for f in os.listdir() - if str(f) != 'symbols.txt' and not os.path.exists(f'../4-1-libs/{p}/{f}')] - ] - - with cd(src_dir/'lib'): - [ - shutil.copy(f, f'4-1-libs/{f}') - for f in - [str(f) for f in os.listdir() if os.path.isfile(f) and (str(f) != 'q.k')] + if str(f) != 'symbols.txt' and not os.path.exists( + f'../../../../{p}/{f}')] ] with cd(src_dir): shutil.copy('pykx.q', 'lib') - shutil.copy('pykx.q', 'lib/4-1-libs') setup( name=pyproject['name'], diff --git a/src/pykx/__init__.py b/src/pykx/__init__.py index 1d8c1bd..08fd45e 100644 --- a/src/pykx/__init__.py +++ b/src/pykx/__init__.py @@ -21,7 +21,7 @@ pass -# List of beta features available in the current PyKX version +# List of beta features available in the current KDB-X Python version beta_features = [] @@ -49,7 +49,7 @@ if platform.python_version_tuple()[:2] >= ('3', '8'): os.add_dll_directory(pykx_platlib_dir) -# Cache initialised signal values prior to PyKX loading +# Cache initialised signal values prior to KDB-X Python loading _signal_list = [ 'signal.SIGINT', 'signal.SIGTERM', @@ -241,7 +241,7 @@ def _register(self, )]) if " " in str(path) and not suppress_warnings: warn("""Space detected in path being loaded. - PyKX will change directory to the path parent and then load directly before returning to current working directory. + KDB-X Python will change directory to the path parent and then load directly before returning to current working directory. To turn off this warning set PYKX_SUPPRESS_WARNINGS to True.""", exceptions.PyKXWarning) # noqa: E501 if name is None: # defaults to filename at end of path sans extension name = path.stem @@ -293,6 +293,7 @@ def paths(self, paths: List[Union[str, Path]]): from .system import SystemCommands from .write import QWriter from .reimporter import PyKXReimport +from .module import use from . import config from . import console @@ -354,6 +355,9 @@ def paths(self, paths: List[Union[str, Path]]): _help_init(q) qhelp = help.qhelp +from .module import _init as _module_init +_module_init(q) + if k_allocator: from . import _numpy as _pykx_numpy_cext @@ -362,11 +366,9 @@ def merge_asof(left, *args, **kwargs): return left.merge_asof(*args, **kwargs) -if sys.version_info[1] < 8: +if sys.version_info[1] < 9: warn( - 'Python 3.7 has reach its end of life period and is no longer supported.' - 'Please consider upgrading to Python 3.8, as PyKX will no longer support issues for this ' - 'Python version.', + 'Python 3.9 is the minimum support version. Please upgrade.', exceptions.PyKXWarning ) @@ -374,7 +376,7 @@ def merge_asof(left, *args, **kwargs): def install_into_QHOME(overwrite_embedpy=False, to_local_folder=False, cloud_libraries=False) -> None: - """Copies the embedded Python functionality of PyKX into `$QHOME`. + """Copies the embedded Python functionality of KDB-X Python into `$QHOME`. Parameters: overwrite_embedpy: If embedPy had previously been installed replace it otherwise @@ -404,7 +406,7 @@ def install_into_QHOME(overwrite_embedpy=False, return p = Path(dest)/'p.k' - c_files = ['kurl.q_', 'kurl.sidecar.q_', 'objstor.q_', 'qlog.q_', 'rest.q_', 'bq.q_', 's.k_'] + c_files = ['kurl.q_', 'kurl.sidecar.q_', 'objstor.q_', 'qlog.q_', 'rest.q_', 'bq.q_'] if not p.exists() or overwrite_embedpy: shutil.copy(Path(__file__).parent/'p.k', p) @@ -413,6 +415,7 @@ def install_into_QHOME(overwrite_embedpy=False, if cloud_libraries: for i in c_files: shutil.copy(Path(__file__).parent/'lib'/i, Path(dest)/i) + shutil.copy(Path(__file__).parent/'lib'/'sql.k_', Path(dest)/'s.k_') shutil.copy(Path(__file__).parent/'pykx.q', dest) shutil.copy(Path(__file__).parent/'pykx_init.q_', dest) if platform.system() == 'Windows': @@ -422,10 +425,10 @@ def install_into_QHOME(overwrite_embedpy=False, def activate_numpy_allocator() -> None: - """Sets the allocator used for Numpy array data to one optimized for use with PyKX. + """Sets the allocator used for Numpy array data to one optimized for use with KDB-X Python. - This will only change the default allocator if the environment variable `PYKX_ALLOCATOR` is set - to 1 or if the flag `--pykxalloc` is present in the QARGS environment variable. + This will only change the default allocator if the environment variable `PYKX_NO_ALLOCATOR` + is not set to 1 or if the flag `--pykxnoalloc` is not present in the QARGS environment variable. The name of the allocator set by this function is `'pykx_allocator'`. @@ -434,7 +437,8 @@ def activate_numpy_allocator() -> None: Refer to NEP 49 for details about custom Numpy allocators: https://numpy.org/neps/nep-0049.html - The custom allocator set by PyKX allocates Numpy array data into the embedded q memory space. + The custom allocator set by KDB-X Python allocates Numpy array data into the embedded q memory + space. Numpy arrays created with this allocator can be converted into a q vector without copying the data. @@ -452,12 +456,12 @@ def activate_numpy_allocator() -> None: is deallocated, it decrements the refcount of the q vector. If the installed version of Numpy support custom allocators (i.e. is greater than or equal to - version `1.22.0`), and the active Numpy allocator when PyKX is imported is the default - allocator, PyKX will automatically call this function. + version `1.22.0`), and the active Numpy allocator when KDB-X Python is imported is the default + allocator, KDB-X Python will automatically call this function. Note that Numpy arrays created with an allocator other than `'pykx_allocator'` (e.g. before - PyKX has been imported) will not gain the benefits enabled by this allocator. The name of the - allocator used by a given array `x` can be retrieved by running + KDB-X Python has been imported) will not gain the benefits enabled by this allocator. + The name of the allocator used by a given array `x` can be retrieved by running `numpy.core.multiarray.get_handler_name(x)`. """ if k_allocator: @@ -476,11 +480,11 @@ def activate_numpy_allocator() -> None: def deactivate_numpy_allocator(): - """Deactivate the PyKX Numpy allocator. + """Deactivate the KDB-X Python Numpy allocator. - If the PyKX Numpy allocator has not been activated then this function has no effect. Otherwise, - it replaces the PyKX Numpy allocator with the allocator that was in use when it was first - activated. + If the KDB-X Python Numpy allocator has not been activated then this function has no effect. + Otherwise, it replaces the KDB-X Python Numpy allocator with the allocator that was in use + when it was first activated. """ if k_allocator: _pykx_numpy_cext.deactivate_pykx_allocators() @@ -489,7 +493,7 @@ def deactivate_numpy_allocator(): try: # If we are running under IPython/Jupyter... ipython = get_ipython() # noqa - # Load the PyKX extension for Jupyter Notebook. + # Load the KDB-X Python extension for Jupyter Notebook. ipython.extension_manager.load_extension('pykx.nbextension') if config.jupyterq: @@ -503,7 +507,7 @@ def deactivate_numpy_allocator(): if licensed: days_to_expiry = q('"D"$', q.z.l[1]) - q.z.D if days_to_expiry < 10: - logging.warning(f'PyKX license set to expire in {int(days_to_expiry)} days, ' + logging.warning(f'KDB-X Python license set to expire in {int(days_to_expiry)} days, ' 'please consider installing an updated license') __all__ = sorted([ @@ -538,6 +542,7 @@ def deactivate_numpy_allocator(): 'config', 'util', 'q', + 'use', 'shutdown_thread', 'PyKXReimport', 'help', diff --git a/src/pykx/_ipc.pyx b/src/pykx/_ipc.pyx index ecf5d57..2a6aacc 100644 --- a/src/pykx/_ipc.pyx +++ b/src/pykx/_ipc.pyx @@ -74,10 +74,10 @@ def _unlicensed_call(handle: int, query: bytes, parameters: List[K], wait: bool) cpdef ssl_info(): - """View information relating to the TLS settings used by PyKX from your process + """View information relating to the TLS settings used by KDB-X Python from your process Returns: - A dictionary outlining the TLS settings used by PyKX + A dictionary outlining the TLS settings used by KDB-X Python Example: diff --git a/src/pykx/_pyarrow.py b/src/pykx/_pyarrow.py index 703f266..73bd70c 100644 --- a/src/pykx/_pyarrow.py +++ b/src/pykx/_pyarrow.py @@ -1,11 +1,11 @@ -"""A single point in PyKX through which PyArrow can be imported safely. +"""A single point in KDB-X Python through which PyArrow can be imported safely. PyArrow can appear to install without error, but then fail to import with a Python exception, or a segmentation fault. Because it's very hard to know when a segfault may occur when importing PyArrow, we test it out first by importing it in a subprocess. If that works, then we import it here, and provide it for -the rest of PyKX to use under the name `pyarrow`. Otherwise `pyarrow` is set to `None`. +the rest of KDB-X Python to use under the name `pyarrow`. Otherwise `pyarrow` is set to `None`. """ import os diff --git a/src/pykx/_wrappers.pyx b/src/pykx/_wrappers.pyx index 8510220..891998c 100644 --- a/src/pykx/_wrappers.pyx +++ b/src/pykx/_wrappers.pyx @@ -133,7 +133,7 @@ cpdef inline deserialize(x): if 0 == core.okx(buff): core.r0(buff) - raise QError('Failed to deserialize supplied non PyKX IPC serialized format object') + raise QError('Failed to deserialize supplied non pykx IPC serialized format object') cdef core.K kx = core.ee(core.d9(buff)) core.r0(buff) return factory(kx, False) @@ -471,6 +471,17 @@ cdef object q_table_type(core.K val): return wrapper +cdef object q_foreign_type(core.K val): + cdef core.K x + x = core.k(0, '.Q.qO', core.r1(val), NULL) + if x.t == -1 and x.g == 1: # x == 1b + wrapper = wrappers.VirtualTable + else: + wrapper = wrappers.Foreign + core.r0(x) + return wrapper + + cdef extern from 'include/vector_conversion.h': object symbol_vector_to_py(core.K s, int raw) object char_vector_to_py(core.K s) @@ -540,6 +551,8 @@ cdef inline object select_wrapper(core.K k): key_ktype = (k.G0)[0].t value_ktype = (k.G0)[1].t wrapper = wrappers.KeyedTable if key_ktype == 98 and value_ktype == 98 else wrappers.Dictionary + elif wrapper is wrappers._k_foreign_type: + wrapper = q_foreign_type(k) if licensed else wrappers.Foreign return wrapper diff --git a/src/pykx/compress_encrypt.py b/src/pykx/compress_encrypt.py index 66cbfe5..f2d2237 100644 --- a/src/pykx/compress_encrypt.py +++ b/src/pykx/compress_encrypt.py @@ -56,7 +56,7 @@ class CompressionAlgorithm(Enum): class Encrypt(): def __init__(self, path: str = None, password: str = None) -> None: """ - A class for controlling the use of encryption with PyKX. + A class for controlling the use of encryption with KDB-X Python. Parameters: path: Location of a user's encryption key file @@ -111,7 +111,7 @@ def __init__(self, level: int = None ) -> None: """ - A class object for controlling q compression with PyKX. + A class object for controlling q compression with KDB-X Python. Parameters: algo: Compression algorithm to use. This must be one of: @@ -145,8 +145,6 @@ def __init__(self, raise ValueError(f'block_size must be a power of 2, not {block_size}') self.encrypt = False self.block_size = int(log2(block_size)) - if (algo == CompressionAlgorithm.zstd) & q('.z.K<4.1').py(): - raise ValueError("'CompressionAlgorithm.zstd' only supported on PyKX>=4.1") compression_range = _compression_ranges[algo] if level is None: level = compression_range.stop -1 diff --git a/src/pykx/config.py b/src/pykx/config.py index 9bd625b..ea5009c 100644 --- a/src/pykx/config.py +++ b/src/pykx/config.py @@ -11,8 +11,9 @@ import toml import pandas as pd +import numpy as np -from .exceptions import PyKXWarning, QError +from .exceptions import QError system = platform.system() @@ -31,13 +32,13 @@ # Profile information for user defined config # If PYKX_CONFIGURATION_LOCATION is not set it will search '.' -pykx_config_location = Path(os.path.expanduser(os.getenv('PYKX_CONFIGURATION_LOCATION', ''))) +pykx_config_location = os.path.expanduser(os.getenv('PYKX_CONFIGURATION_LOCATION', '')) pykx_config_profile = os.getenv('PYKX_PROFILE', 'default') def _get_config_value(param, default): try: - default = _pykx_profile_content[param] + default = pykx_profile_content[param] except KeyError: pass except NameError: @@ -58,22 +59,26 @@ def _is_set(envvar): return os.getenv(envvar, None) -pykx_config_locs = [Path('.'), pykx_config_location, Path.home()] -pykx_config_locs = [path / '.pykx-config' for path in pykx_config_locs] +pykx_config_locs = [Path.home()/'.kx/config-pykx'] +if pykx_config_location != '': + pykx_config_locs = [Path(pykx_config_location)] + pykx_config_locs +else: + pykx_config_location = None pykx_config_locs = [os.path.abspath(path) for path in pykx_config_locs if os.path.isfile(path)] pykx_config_locs = list(set(pykx_config_locs)) -_pykx_config_content = None -_pykx_config_location = None -_pykx_profile_content = {} +pykx_config_content = None +pykx_profile_content = {} for path in pykx_config_locs: - _pykx_config_content = toml.load(path) + pykx_config_content = toml.load(path) try: - _pykx_profile_content = _pykx_config_content[pykx_config_profile] - _pykx_config_location = path + pykx_profile_content = pykx_config_content[pykx_config_profile] + pykx_config_location = path break except KeyError: + pykx_profile_content = {} + pykx_config_location = None print("Unable to locate specified 'PYKX_PROFILE': '" + pykx_config_profile + "' in file '" + str(path) + "'") # noqa E501 @@ -81,8 +86,9 @@ def _is_set(envvar): os.environ['PYKX_DIR'] = str(pykx_dir) pykx_executable = sys.executable os.environ['PYKX_EXECUTABLE'] = pykx_executable -pykx_4_1 = _is_enabled('PYKX_4_1_ENABLED') -pykx_libs_dir = Path(pykx_dir/'lib'/'4-1-libs') if pykx_4_1 else Path(pykx_dir/'lib') # noqa + +pykx_libs_dir = Path(pykx_dir/'lib') + pykx_lib_dir = Path(_get_config_value('PYKX_Q_LIB_LOCATION', pykx_libs_dir)) pykx_platlib_dir = pykx_lib_dir/q_lib_dir_name lib_prefix = '' if system == 'Windows' else 'lib' @@ -93,33 +99,113 @@ def _is_set(envvar): }[system] +def _remove_outer_doublequotes(s): + if s.startswith('"') and s.endswith('"'): + return s[1:-1] + return s + + +def _get_qcfg_values(): + qcfg_values = {} + qcfg_path = None + + qargs_str = _get_config_value('QARGS', '') + qargs_list = shlex.split(qargs_str) + + for i, arg in enumerate(qargs_list): + if arg == '-v' and i + 1 < len(qargs_list): + qcfg_path = qargs_list[i + 1] + break + + if qcfg_path is None: + qcfg_path = os.getenv('QCFG') + + if qcfg_path is None: + qcfg_path = Path.home()/'.kx/config' + + if qcfg_path: + try: + qcfg_path = Path(qcfg_path).resolve(strict=True) + + with open(qcfg_path, 'r') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + + if '=' in line: + key, value = line.split('=', 1) + key = key.strip() + value = value.strip() + + key = _remove_outer_doublequotes(key) + value = _remove_outer_doublequotes(value) + + if key in ('QHOME', 'QLIC'): + qcfg_values[key] = value + + except (IOError): + # Error or Warning to handle failing QCFG + pass + + return qcfg_values + + +_qcfg_values = _get_qcfg_values() + + def _get_qhome(): - try: - qhome = Path(_get_config_value('QHOME', pykx_lib_dir)).resolve(strict=True) - except FileNotFoundError: # nocov - # If QHOME and its fallback weren't set/valid, then q/Python must be - # running in the same directory as q.k (and presumably other stuff one - # would expect to find in QHOME). - qhome = Path().resolve(strict=True) - return qhome + qhome_value = _get_config_value('QHOME', None) + + if qhome_value is not None: + try: + qhome = Path(qhome_value).resolve(strict=True) + return qhome + except FileNotFoundError: + pass + + kx_path = Path.home()/'.kx' + if kx_path.exists() and kx_path.is_dir(): + return kx_path.resolve(strict=True) -qhome = _get_qhome() + return Path(pykx_lib_dir).resolve(strict=True) + + +def _get_qpath(qhome): + qpath = _get_config_value('QPATH', None) + os.environ['QPATH'] = str(qhome/'mod' if qpath is None else qpath+':'+str(qhome/'mod')) + + +if 'QHOME' in _qcfg_values: + try: + qhome = Path(_qcfg_values['QHOME']).resolve(strict=True) + except FileNotFoundError: + warn(f"QHOME from QCFG does not exist: {_qcfg_values['QHOME']}") + qhome = Path(_qcfg_values['QHOME']).resolve(strict=False) +else: + qhome = _get_qhome() + +qpath = _get_qpath(qhome) # License search -_qlic = _get_config_value('QLIC', '') -if (_qlic != '') and (not os.path.isdir(_qlic)): - warn(f'Configuration value QLIC set to non directory value: {_qlic}') +if 'QLIC' in _qcfg_values: + _qlic = _qcfg_values['QLIC'] + if not os.path.isdir(_qlic): + warn(f'QCFG value QLIC set to non directory value: {_qlic}') +else: + _qlic = _get_config_value('QLIC', '') + if (_qlic != '') and (not os.path.isdir(_qlic)): + warn(f'Configuration value QLIC set to non directory value: {_qlic}') _kc_lic = 'kc.lic' _k4_lic = 'k4.lic' _kx_lic = 'kx.lic' -_pwd = os.getcwd() license_located = False lic_path = '' lic_type = '' -for loc in (_pwd, _qlic, qhome): +for loc in (_qlic, qhome): if loc=='': pass for lic in (_kx_lic, _kc_lic, _k4_lic): @@ -143,8 +229,8 @@ def _get_qhome(): suppress_warnings = _is_enabled('PYKX_SUPPRESS_WARNINGS') or light_load _unsupported_qargs = { - '-p': 'PyKX running without a main loop, setting a port in this way is not supported', - '-t': 'PyKX running without a main loop, setting timers in this way has no effect' + '-p': 'KDB-X Python running without a main loop, setting a port in this way is not supported', + '-t': 'KDB-X Python running without a main loop, setting timers in this way has no effect' } @@ -183,7 +269,7 @@ def _license_install_path(download_location, qlic): valid. Must be one of:{lic_types}") shutil.copy(download_location, qlic) - print(f'\nPyKX license successfully installed to: {qlic / lic_type}\n') + print(f'\nKDB-X Python license successfully installed to: {qlic / lic_type}\n') def _license_install_B64(license, license_type): # pragma: no cover @@ -207,7 +293,7 @@ def _license_check(lic_type, lic_encoding, lic_variable): # pragma: no cover license_content = base64.encodebytes(f.read()).decode('utf-8') license_content = license_content.replace('\n', '') if lic_encoding == license_content: - conflict_message = 'We have been unable to update your license for PyKX using '\ + conflict_message = 'We have been unable to update your license for KDB-X Python using '\ 'the following information:\n'\ f" Environment variable: {lic_variable} \n"\ f' License location: {qlic}/{lic_type}.lic\n'\ @@ -219,9 +305,12 @@ def _license_check(lic_type, lic_encoding, lic_variable): # pragma: no cover def _unlicensed_config(unlicensed_message): - choice = input('\nWould you like us to remember this choice? [Y/n]: ') - if choice in ('y', 'Y', ''): - fpath = Path(os.path.expanduser('~')) / '.pykx-config' + choice = input('\nWould you like us to remember this choice? [N/y]: ') + if choice in ('y', 'Y'): + if pykx_config_location is None: + fpath = Path.home()/'.kx/config-pykx' + else: + fpath = pykx_config_location try: os.access(fpath, os.W_OK) except FileNotFoundError: @@ -232,8 +321,8 @@ def _unlicensed_config(unlicensed_message): with open(fpath, 'r') as file: data = toml.load(file) else: - data = {'default': {}} - data['default']['PYKX_UNLICENSED'] = 'True' + data = {pykx_config_profile: {}} + data[pykx_config_profile]['PYKX_UNLICENSED'] = 'True' with open(fpath, 'w') as file: toml.dump(data, file) print(f"\nConfiguration updated at: {fpath}.\n" @@ -243,7 +332,7 @@ def _unlicensed_config(unlicensed_message): os.environ['PYKX_UNLICENSED']='true' -def _license_install(intro=None, return_value=False, license_check=False, license_error=None): # noqa: +def _license_install(intro=None, return_value=False, license_check=False, license_error=None): # noqa: if not hasattr(sys, 'ps1'): # Exit if running in a non-interactive session return False @@ -271,17 +360,16 @@ def _license_install(intro=None, return_value=False, license_check=False, licens print(install_message) return True - personal_url = "https://kx.com/kdb-insights-sdk-personal-edition-download" - commercial_url = "https://kx.com/book-demo" - unlicensed_message = '\nPyKX unlicensed mode enabled. To set this as your default behavior '\ - "set the following environment variable PYKX_UNLICENSED='true'" - first_user = '\nThank you for installing PyKX!\n\n'\ - 'We have been unable to locate your license for PyKX.\n\n'\ + lic_url = 'https://developer.kx.com/products/kdb-x/install' + lic_type = 'kc.lic' + unlicensed_message = '\nKDB-X Python unlicensed mode enabled. To set this as your default '\ + "behavior set the following environment variable PYKX_UNLICENSED='true'" + first_user = '\nThank you for installing KDB-X Python!\n\n'\ + 'We have been unable to locate your license for KDB-X Python.\n\n'\ 'Paths searched:\n'\ - f' . {_pwd}\n'\ f' QLIC {_qlic if _qlic else "Not Set"}\n'\ f' QHOME {qhome if qhome else "Not Set"}\n\n'\ - 'Running PyKX in unlicensed mode has reduced functionality.\n'\ + 'Running KDB-X Python in unlicensed mode has reduced functionality.\n'\ 'Would you like to install a license? [Y/n]: ' root = 'C:\\path\\to\\' if platform.system() == 'Windows' else '~/path/to/' continue_license = input(first_user if intro is None else intro) @@ -291,34 +379,13 @@ def _license_install(intro=None, return_value=False, license_check=False, licens return False elif continue_license in ('y', 'Y', ''): - existing_license = input('\nDo you have access to an existing license for PyKX ' + existing_license = input('\nDo you have access to an existing license for KDB-X Python ' 'that you would like to use? [N/y]: ') if existing_license not in ('Y', 'y', 'N', 'n', ''): raise Exception('Invalid input provided please try again') if existing_license in ('N', 'n', ''): - commercial = input('\nIs the intended use of this software for:' - '\n [1] Personal use (Default)' - '\n [2] Commercial use' - '\nEnter your choice here [1/2]: ').strip().lower() - if commercial not in ('1', '2', ''): - raise Exception('User provided option was not one of [1/2]') - - personal = commercial in ('1', '') - - lic_url = personal_url if personal else commercial_url - lic_type = _kc_lic if personal else _k4_lic - - if personal: - redirect = input(f'\nTo apply for your PyKX license, navigate to {lic_url}.\n' - 'Shortly after you submit your application, you will receive a ' - 'welcome email containing your license information.\n' - 'Would you like to open this page? [Y/n]: ') - else: - redirect = input('\nTo apply for your PyKX license, contact your ' - 'KX sales representative or sales@kx.com.\n' - f'Alternately apply through {lic_url}.\n' - 'Would you like to open this page? [Y/n]: ') - + redirect = input(f'\nFor instructions and to obtain a license visit: {lic_url}\n' + 'Would you like to open this page? [Y/n]: ') if redirect.lower() in ('y', ''): try: webbrowser.open(lic_url) @@ -327,38 +394,29 @@ def _license_install(intro=None, return_value=False, license_check=False, licens raise Exception('Unable to open web browser') install_type = input('\nPlease select the method you wish to use to activate your ' - 'license:\n [1] Download the license file provided in your ' - 'welcome email and input the file path (Default)' - '\n [2] Input the activation key (base64 encoded string) ' - 'provided in your welcome email' - '\n [3] Proceed with unlicensed mode' - '\nEnter your choice here [1/2/3]: ').strip().lower() + 'license:\n [1] Input the license key (base64 encoded string) ' + 'provided on the KX Developer Center' + '\n [2] Proceed with unlicensed mode' + '\nEnter your choice here [1/2]: ').strip().lower() - if install_type not in ('1', '2', '3', ''): - raise Exception('User provided option was not one of [1/2/3]') + if install_type not in ('1', '2', ''): + raise Exception('User provided option was not one of [1/2]') if install_type in ('1', ''): - license = input('\nProvide the download location of your license ' - f'(for example, {root}{lic_type}) : ').strip() - download_location = os.path.expanduser(Path(license)) - _license_install_path(download_location, qlic) - - elif install_type == '2': - license = input('\nProvide your activation key (base64 encoded string) ' - 'provided with your welcome email : ').strip() - + license = input('\nProvide your license key (base64 encoded string) ' + 'provided on the KX Developer Center: ').strip() _license_install_B64(license, lic_type) - print('\nPyKX license successfully installed to: {qlic / lic_type}\n') # noqa: E501 - elif install_type == '3': + print('\nKDB-X Python license successfully installed to: {qlic / lic_type}\n') # noqa: E501 + elif install_type == '2': _unlicensed_config(unlicensed_message) if return_value: return False else: install_type = input( '\nPlease select the method you wish to use to activate your license:\n' - ' [1] Provide the location of your license\n' - ' [2] Input the activation key\n' + ' [1] Provide the location of your license file\n' + ' [2] Paste the license key\n' 'Enter your choice here [1/2]: ') if install_type not in ('1', '2', ''): @@ -370,20 +428,19 @@ def _license_install(intro=None, return_value=False, license_check=False, licens _license_install_path(download_location, qlic) else: - commercial = input('\nPlease confirm the license type:\n' - f' [1] Personal use ({_kc_lic})\n' - f' [2] Commercial use ({_k4_lic})\n' - 'Enter your choice here [1/2]: ') - if commercial not in ('1', '2', ''): + lic_type_choice = input('\nPlease confirm the license type:\n' + f' [1] {_kc_lic} - The default\n' + f' [2] {_k4_lic} - Used in some scenarios\n' + 'Enter your choice here [1/2]: ') + if lic_type_choice not in ('1', '2', ''): raise Exception('User provided option was not one of [1/2]') - personal = commercial in ('1', '') - lic_type = _kc_lic if personal else _k4_lic - license = input('\nProvide your activation key (base64 encoded string) : ').strip() + lic_type = _k4_lic if lic_type_choice == '2' else _kc_lic + license = input('f\nProvide your {lic_type} license key (base64 encoded string) : ').strip() # noqa: E501 _license_install_B64(license, lic_type) - print(f'\nPyKX license successfully installed to: {qlic / lic_type}\n') # noqa: E501 + print(f'\nKDB-X Python license successfully installed to: {qlic / lic_type}\n') # noqa: E501 else: raise Exception('Invalid input provided please try again') @@ -420,28 +477,20 @@ def _license_install(intro=None, return_value=False, license_check=False, licens keep_local_times = _is_enabled('PYKX_KEEP_LOCAL_TIMES') max_error_length = int(_get_config_value('PYKX_MAX_ERROR_LENGTH', 256)) -allocator = _is_enabled('PYKX_ALLOCATOR', '--pykxalloc') -if allocator: - if sys.version_info[1] <= 7: - raise PyKXWarning('A python version of at least 3.8 is required to use the PyKX allocators') # noqa nocov - k_allocator = False # nocov - else: - k_allocator = True -else: - k_allocator = False - +k_allocator = not _is_enabled('PYKX_NO_ALLOCATOR', '--pykxnoalloc') k_gc = _is_enabled('PYKX_GC', '--pykxgc') release_gil = _is_enabled('PYKX_RELEASE_GIL', '--release-gil') use_q_lock = _get_config_value('PYKX_Q_LOCK', False) skip_under_q = _is_enabled('PYKX_SKIP_UNDERQ', '--skip-under-q') -no_qce = _is_enabled('PYKX_NOQCE', '--no-qce') +qce = _is_enabled('PYKX_QCE', '--qce') beta_features = _is_enabled('PYKX_BETA_FEATURES', '--beta') load_pyarrow_unsafe = _is_enabled('PYKX_LOAD_PYARROW_UNSAFE', '--load-pyarrow-unsafe') pykx_qdebug = _is_enabled('PYKX_QDEBUG', '--q-debug') pykx_debug_insights = _is_enabled('PYKX_DEBUG_INSIGHTS_LIBRARIES') -pandas_2 = pd.__version__.split('.')[0] == '2' - +pandas_gt1 = int(pd.__version__.split('.')[0]) > 1 +pandas_gt2 = int(pd.__version__.split('.')[0]) > 2 +numpy_gt1 = int(np.__version__.split('.')[0]) > 1 jupyterq = _is_enabled('PYKX_JUPYTERQ') @@ -483,6 +532,10 @@ def _check_beta(feature_name, *, status=beta_features): 'q_lib_dir_name', 'pykx_dir', 'pykx_lib_dir', + 'pykx_config_location', + 'pykx_config_content', + 'pykx_profile_content', + 'pykx_platlib_dir', 'lib_prefix', 'lib_ext', @@ -502,12 +555,13 @@ def _check_beta(feature_name, *, status=beta_features): 'release_gil', 'use_q_lock', 'skip_under_q', - 'no_qce', + 'qce', 'load_pyarrow_unsafe', 'find_core_lib', - 'pandas_2', + 'pandas_gt1', + 'pandas_gt2', ] diff --git a/src/pykx/core.pyx b/src/pykx/core.pyx index cfede9e..a052f77 100644 --- a/src/pykx/core.pyx +++ b/src/pykx/core.pyx @@ -11,7 +11,7 @@ import sys from . import beta_features from .util import add_to_config, num_available_cores from .config import (tcore_path_location, _is_enabled, _license_install, pykx_threading, _get_config_value, pykx_lib_dir, - ignore_qhome, _pwd, lic_path, pykx_4_1, _pykx_force_licensed, _pykx_force_unlicensed) + ignore_qhome, lic_path, _pykx_force_licensed, _pykx_force_unlicensed) def _normalize_qargs(user_args: List[str]) -> Tuple[bytes]: @@ -26,7 +26,7 @@ def _normalize_qargs(user_args: List[str]) -> Tuple[bytes]: try: s_index = user_args.index('-s') except ValueError: # -s was not specified - num_cores_as_bytes = str(num_available_cores()).encode() + num_cores_as_bytes = None else: try: num_cores_user_arg = user_args[s_index + 1] @@ -41,7 +41,8 @@ def _normalize_qargs(user_args: List[str]) -> Tuple[bytes]: "Invalid argument for '-s' (number of secondary threads) in $QARGS: " f'{num_cores_user_arg!r}' ) from ex - normalized_args.extend([b'-s', num_cores_as_bytes]) + if num_cores_as_bytes != None: + normalized_args.extend([b'-s', num_cores_as_bytes]) return ( *normalized_args, @@ -71,7 +72,7 @@ cdef void* _q_handle # like a license error, but could also cause a segfault. The subprocess allows us to safely attempt # calling `_qinit`, and return the result for the parent to decide if it should use licensed mode, # or fallback to unlicensed mode. -# Data is passed in via the `PYKX_QINIT_CHECK` env var to reduce PyKX's import time - why do the +# Data is passed in via the `PYKX_QINIT_CHECK` env var to reduce `pykx` import time - why do the # work to gather that data when the parent process has already done it? qinit_check_data = os.environ.get('PYKX_QINIT_CHECK') if qinit_check_data is not None: # nocov @@ -215,7 +216,7 @@ def _link_qhome(): # QHOME has not had any files/directories added/removed since we last updated links. return # Avoid recursion, but allow for the effective merger via symlinks of the directories under the - # lib dir that come with PyKX. + # lib dir that come with KDB-X Python. if not ignore_qhome: for subdir in subdirs: # Remove old symlinks: @@ -227,6 +228,8 @@ def _link_qhome(): try: with os.scandir(qhome/subdir) as dir_iter: for dir_entry in dir_iter: + if dir_entry.name in ['q.k', 's.k_']: + continue try: os.symlink( dir_entry, @@ -234,14 +237,14 @@ def _link_qhome(): target_is_directory=dir_entry.is_dir() ) except FileExistsError: - pass # Skip files/dirs that would overwrite those that come with PyKX. + pass # Skip files/dirs that would overwrite those that come with KDB-X Python. except OSError as ex: #nocov # Making this a warning instead of an error is particularly important for # Windows, which essentially only lets admins create symlinks. - warn('Unable to connect user QHOME to PyKX QHOME via symlinks.\n' # nocov + warn('Unable to connect user QHOME to KDB-X Python QHOME via symlinks.\n' # nocov 'To permanently disable attempts to create symlinks you can\n' # nocov '\t1. Set the environment variable "PYKX_IGNORE_QHOME" = True.\n' # nocov - '\t2. Update the file ".pykx-config" using kx.util.add_to_config({\'PYKX_IGNORE_QHOME\': \'True\'})\n' # nocov + '\t2. Update the file "~/.kx/config-pykx" using kx.util.add_to_config({\'PYKX_IGNORE_QHOME\': \'True\'})\n' # nocov f'Error: {ex}\n', # nocov PyKXWarning) # nocov return # nocov @@ -296,8 +299,7 @@ if not pykx_threading: if _qinit_unsuccessful: # Fallback to unlicensed mode if _qinit_output != ' ': _capout_msg = f'Captured output from initialization attempt:\n{_qinit_output}' - _paths_checked = f' . {_pwd}\n'\ - f' QLIC {_qlic if _qlic else "Not Set"}\n'\ + _paths_checked = f' QLIC {_qlic if _qlic else "Not Set"}\n'\ f' QHOME {qhome if qhome else "Not Set"}' _lic_location = f'License used:\n {lic_path}' else: @@ -306,42 +308,40 @@ if not pykx_threading: _paths_checked = '' # nocov - this additional line is to ensure this code path is covered. if (hasattr(sys, 'ps1') and not _pykx_force_licensed): if re.compile('licen[cs]e error: exp').search(_capout_msg): - _exp_license = 'Your PyKX license has now expired.\n\n'\ + _exp_license = 'Your KDB-X Python license has now expired.\n\n'\ f'{_capout_msg}\n\n'\ f'{_lic_location}\n\n'\ 'Would you like to renew your license? (Selecting no will proceed with unlicensed mode) [Y/n]: ' _license_message = _license_install(_exp_license, True, True, 'exp') elif re.compile('licen[cs]e error: embedq').search(_capout_msg): - _ce_license = 'You appear to be using a non kdb Insights license.\n\n'\ + _ce_license = 'You appear to be using a non KDB-X Python enabled license.\n\n'\ f'{_capout_msg}\n\n'\ f'{_lic_location}\n\n'\ - 'Running PyKX in the absence of a kdb Insights license '\ + 'Running KDB-X Python in the absence of a license '\ 'has reduced functionality.\nWould you like to install '\ - 'a kdb Insights personal license? [Y/n]: ' + 'a license? [Y/n]: ' _license_message = _license_install(_ce_license, True) elif re.compile('licen[cs]e error: upd').search(_capout_msg): _upd_license = 'Your installed license is out of date for this version'\ - ' of PyKX and must be updated.\n\n'\ + ' of KDB-X Python and must be updated.\n\n'\ f'{_capout_msg}\n\n'\ f'{_lic_location}\n\n'\ - 'Would you like to install an updated kdb '\ - 'Insights personal license? [Y/n]: ' + 'Would you like to install an updated license? [Y/n]: ' _license_message = _license_install(_upd_license, True) elif re.compile('licen[cs]e error: k[xc4].lic').search(_capout_msg) or re.compile('licen[cs]e error: badmsg').search(_capout_msg): - _k_license = '\nThe PyKX license found is corrupt or incompatible with'\ - ' kdb+ ' + ('4.1' if pykx_4_1 else '4.0') + '.\n\n'\ + _k_license = '\nThe KDB-X Python license found is corrupt or incompatible.\n\n'\ f'{_capout_msg}\n\n'\ f'{_lic_location}\n\n'\ 'Would you like to install a new license? [Y/n]: ' _license_message = _license_install(_k_license, True) else: - warning = f'Failed to initialize PyKX successfully with the following error: {_capout_msg}\n\n' + warning = f'Failed to initialize KDB-X Python successfully with the following error: {_capout_msg}\n\n' if _lic_location == '': - warning = warning + f'PyKX was unable to locate a license file in:\n{_paths_checked}\n' + warning = warning + f'KDB-X Python was unable to locate a license file in:\n{_paths_checked}\n' else: warning = warning + f'{_lic_location}\n' warn(warning, PyKXWarning) - _missing_license = 'Running PyKX in unlicensed mode has reduced functionality.\n\n'\ + _missing_license = 'Running KDB-X Python in unlicensed mode has reduced functionality.\n\n'\ 'Would you like to install a license? (Selecting no will proceed with unlicensed mode) [Y/n]: ' _license_message = _license_install(_missing_license, True) if (not _license_message) and _qinit_check_proc.returncode: @@ -361,13 +361,13 @@ if not pykx_threading: raise PyKXException(f'Non-zero qinit following license install with configuration: {_qinit_args}\n' f'failed with output: {_qinit_output}') if 'QHOME' in os.environ and not ignore_qhome: - # Only link the user's QHOME to PyKX's QHOME if the user actually set $QHOME. + # Only link the user's QHOME to KDB-X Python's QHOME if the user actually set $QHOME. # Note that `pykx.qhome` has a default value of `./q`, as that is the behavior # employed by q. try: _link_qhome() except BaseException as e: - warn('Failed to link user QHOME directory contents to allow access to PyKX.\n' + warn('Failed to link user QHOME directory contents to allow access to KDB-X Python.\n' 'To suppress this warning please set the configuration option "PYKX_IGNORE_QHOME" as outlined at:\n' 'https://code.kx.com/pykx/user-guide/configuration.html') _libq_path_py = bytes(_core_q_lib_path) diff --git a/src/pykx/ctx.py b/src/pykx/ctx.py index dc8266a..8e464b5 100644 --- a/src/pykx/ctx.py +++ b/src/pykx/ctx.py @@ -210,7 +210,7 @@ class ZContext(QContext): 'po', 'pp', 'pq', 'ps', 'pw', 'vs', 'wc', 'wo', 'ws', 'zd') _has_default = ('a', 'b', 'c', 'D', 'd', 'e', 'f', 'H', 'h', 'i', 'K', 'k', - 'l', 'N', 'n', 'o', 'P', 'p', 'pg', 'q', 'T', 't', 'u', 'W', 'w', + 'l', 'N', 'n', 'o', 'P', 'p', 'pg', 'q', 'T', 't', 'u', 'v', 'W', 'w', 'X', 'x') _unsupported_keys_with_msg = { @@ -221,7 +221,7 @@ class ZContext(QContext): 'ey': '.z.ey is only available during a debugging session, so it is ' 'not exposed through the context interface.', 'ts': '.z.ts is not exposed through the context interface because the ' - 'main loop is inactive in PyKX.', + 'main loop is inactive in KDB-X Python.', 'z': 'The q datetime type is deprecated, and so the datetime ' 'object .z.z is not exposed through the context interface.', 'Z': 'The q datetime type is deprecated, and so the datetime ' diff --git a/src/pykx/db.py b/src/pykx/db.py index a021920..d49df09 100644 --- a/src/pykx/db.py +++ b/src/pykx/db.py @@ -1,10 +1,9 @@ """ -_This page documents the API for managing kdb+ databases using PyKX._ +_This page documents the API for managing kdb+ databases using KDB-X Python._ """ from .exceptions import DBError, QError from . import wrappers as k -from .config import pykx_4_1 from .compress_encrypt import Compress, Encrypt import os @@ -127,11 +126,8 @@ def __init__(self, Parameters: path: The location at which your database is/will be located. change_dir: Should the working directory be changed to the location of the - loaded database, for q 4.0 this is the only supported behavior, please - set `PYKX_4_1_ENABLED` to allow use if this functionality. - load_scripts: Should any q scripts find in the database directory be loaded, - for q 4.0 this is the only supported behavior, please set - `PYKX_4_1_ENABLED` to allow use if this functionality. + loaded database. + load_scripts: Should any q scripts find in the database directory be loaded. Returns: A database class which can be used to interact with a partitioned database. @@ -165,11 +161,6 @@ def __init__(self, """ self._change_dir = change_dir self._load_scripts = load_scripts - if not pykx_4_1: - if not change_dir: - raise QError("'change_dir' behavior only supported with PYKX_4_1_ENABLED") - if not load_scripts: - raise QError("'load_scripts' behavior only supported with PYKX_4_1_ENABLED") if path is not None: try: self.load(path, change_dir=self._change_dir, load_scripts=self._load_scripts) @@ -383,11 +374,8 @@ def load(self, Parameters: path: The file system path at which your database is located change_dir: Should the working directory be changed to the location of the - loaded database, for q 4.0 this is the only supported behavior, please - set `PYKX_4_1_ENABLED` to allow use if this functionality. - load_scripts: Should any q scripts find in the database directory be loaded, - for q 4.0 this is the only supported behavior, please set - `PYKX_4_1_ENABLED` to allow use if this functionality. + loaded database, for q 4.0 this is the only supported behavior. + load_scripts: Should any q scripts find in the database directory be loaded. overwrite: Should loading of the database overwrite any currently loaded databases encrypt: The encryption key object to be loaded prior to database load @@ -479,38 +467,20 @@ def load(self, raise ValueError('Supplied encrypt object not an instance of pykx.Encrypt') if not encrypt.loaded: encrypt.load_key() - if pykx_4_1: - q(''' - {[path;cd;ld] - .[.Q.lo; - (`$1_string path;cd;ld); - {x:$[x like "*.DS_Store"; - "Invalid MacOS metadata file '.DS_Store' stored in Database"; - x]; - '"Failed to load Database with error: ",x - } - ] - } - ''', load_path, change_dir, load_scripts) - else: - if not change_dir: - raise QError("'change_dir' behavior only supported with PYKX_4_1_ENABLED") - if not load_scripts: - raise QError("'load_scripts' behavior only supported with PYKX_4_1_ENABLED") - db_path = load_path.parent - db_name = os.path.basename(load_path) - q(''' - {[dbpath;dbname] - .[.pykx.util.loadfile; - (1_string dbpath;string dbname); - {x:$[x like "*.DS_Store"; - "Invalid MacOS metadata file '.DS_Store' stored in Database"; - x]; - '"Failed to load Database with error: ",x - } - ] + + q(''' + {[path;cd;ld] + .[.Q.lo; + (`$1_string path;cd;ld); + {x:$[x like "*.DS_Store"; + "Invalid MacOS metadata file '.DS_Store' stored in Database"; + x]; + '"Failed to load Database with error: ",x } - ''', db_path, db_name) + ] + } + ''', load_path, change_dir, load_scripts) + self.path = load_path self.loaded = True self.tables = q('{x where {$[-1h=type t:.Q.qp tab:get x;$[t;x in .Q.pt;in[`$last vs["/";-1_string value flip tab]; key y]];0b]}[;y]each x}', q.tables(), load_path).py() # noqa: E501 @@ -892,7 +862,7 @@ def set_column_type(self, table: str, column_name: str, new_type: k.K) -> None: Parameters: table: The name of the table within which a column is to be converted column_name: Name of the column which is to be converted - new_type: PyKX type to which a column is to be converted + new_type: KDB-X Python type to which a column is to be converted Returns: A `#!python None` object on successfully updating the type of the column diff --git a/src/pykx/embedded_q.py b/src/pykx/embedded_q.py index a1ce504..7838858 100644 --- a/src/pykx/embedded_q.py +++ b/src/pykx/embedded_q.py @@ -2,7 +2,6 @@ import os from pathlib import Path import sys -import platform from typing import Any, Optional, Union from warnings import warn @@ -12,7 +11,7 @@ from . import toq from . import wrappers from . import schema -from .config import find_core_lib, licensed, no_qce, pykx_dir, pykx_libs_dir, pykx_qdebug, pykx_threading, qargs, skip_under_q, suppress_warnings, pykx_debug_insights # noqa +from .config import find_core_lib, licensed, pykx_dir, pykx_libs_dir, pykx_qdebug, pykx_threading, qargs, qce, skip_under_q, suppress_warnings, pykx_debug_insights # noqa from .core import keval as _keval from .exceptions import FutureCancelled, LicenseException, NoResults, PyKXException, PyKXWarning, QError # noqa from ._wrappers import _factory as factory @@ -117,6 +116,7 @@ def __call__(self): class EmbeddedQ(Q, metaclass=ABCMetaSingleton): """Interface for using q within the current python process. Call this to execute q code.""" def __init__(self): # noqa + object.__setattr__(self, '_modules', {}) if licensed: kxic_path = pykx_libs_dir.as_posix() kxic_file = 'kxic.k' @@ -139,12 +139,11 @@ def __init__(self): # noqa }; ''' code += f'.pykx.util.loadfile["{kxic_path}";"html.q"];' - if not no_qce: - code += f''' - if[not `comkxic in key `; - .pykx.util.loadfile["{kxic_path}";"{kxic_file}"] - ]; - ''' + code += f''' + if[not `comkxic in key `; + .pykx.util.loadfile["{kxic_path}";"{kxic_file}"] + ]; + ''' if os.getenv('PYKX_UNDER_Q') is None: os.environ['PYKX_UNDER_PYTHON'] = 'true' code += 'setenv[`PYKX_UNDER_PYTHON;"true"];' @@ -158,8 +157,8 @@ def __init__(self): # noqa code += f'`.pykx.modpow set {{((`$"{pykx_qlib_path}q{suffix}") 2: (`k_modpow; 3))["j"$x;"j"$y;$[z~(::);(::);"j"$z]]}};' # noqa: E501 code += '@[get;`.pykx.i.kxic.loadfailed;{()!()}]' kxic_loadfailed = self._call(code, skip_debug=True).py() - if (platform.system() != "Linux") and (not no_qce) and ('--no-sql' not in qargs): - sql = self._call('$[("insights.lib.sql" in " " vs .z.l 4)¬ `s in key`; @[system; "l s.k_";{x}];::]', skip_debug=True).py() # noqa: E501 + if qce or '--sql' in qargs: + sql = self._call('$[(any ("sq";"insights.lib.sql") in " " vs .z.l 4) and not @[{2]*class="[^"]*md-content__inner[^"]*"[^>]*>' - r'(.*?)', - html, re.DOTALL - ) - if not article_match: - return None - - article_html = article_match.group(1) - - # Find the h3 section for this keyword - # Pattern:

...

... content ...

- pattern = rf']*id="{keyword}"[^>]*>.*?

(.*?)(?=]*id="|$)' - section_match = re.search(pattern, article_html, re.DOTALL) - - if not section_match: - return None - - section_html = section_match.group(1) - text = f" • {keyword}\n\n" - - # Extract paragraphs - for para_match in re.finditer(r']*)?>(.*?)

', section_html, re.DOTALL): - para_text = re.sub(r'<[^>]+>', '', para_match.group(1)) - para_text = _decode_html_entities(para_text).strip() - if para_text: - text += para_text + "\n\n" - - # Extract code blocks - for code_match in re.finditer(r']*>]*>(.*?)', - section_html, re.DOTALL): - code_text = _decode_html_entities(code_match.group(1)).strip() - if code_text: - text += "\n" - for line in code_text.split("\n"): - text += " " + line + "\n" - text += "\n" - - return text - - except (URLError, HTTPError, TimeoutError): - return None - except Exception: - return None - - -def _decode_html_entities(text): - """Decode common HTML entities.""" - return (text.replace('<', '<') - .replace('>', '>') - .replace('&', '&') - .replace('"', '"') - .replace(''', "'") - .replace(' ', ' ') - .replace('¶', '') # Remove paragraph symbols - .replace('·', '·')) - - def _load_markdown_section(keyword, file_path=_filepath): """Load and parse markdown section for a keyword.""" try: @@ -174,15 +103,9 @@ def qhelp(keyword): return text - # Fallback to web scraping if local file fails - web_result = _fetch_web_documentation(keyword) - if web_result: - return web_result - # If both local and web sources fail return (f" • {keyword}\n\n" - f"Documentation not available. Please check your internet connection\n" - f"or visit {_base_url}#{keyword} for documentation.") + f"Documentation not available locally. Visit https://code.kx.com for documentation.") def _parse_markdown_table(lines): diff --git a/src/pykx/ipc.py b/src/pykx/ipc.py index fca6b8f..0ec0fa3 100644 --- a/src/pykx/ipc.py +++ b/src/pykx/ipc.py @@ -1,5 +1,5 @@ """ -_This page documents the API functions for using q IPC within PyKX._ +_This page documents the API functions for using q IPC within KDB-X Python._ """ from enum import Enum @@ -460,7 +460,6 @@ def __init__(self, reconnection_delay: float = 0.5, reconnection_function: callable = reconnection_function, connection_timeout: Optional[float] = None, - ): """Interface with a q process using the q IPC protocol. @@ -503,7 +502,6 @@ def __init__(self, connection_timeout: Timeout in seconds for connection to `q` server. If left as default `None`, the socket will not timeout when connecting. - Note: The `#!python username` and `#!python password` parameters are not required. The `#!python username` and `#!python password` parameters are only required if the q server requires authorization. Refer to @@ -825,7 +823,7 @@ async def _recv2(self, locked=False, acceptAsync=False, fut: Optional[QFuture]=N print("WARN: Discarding unexpected sync message from handle: " + str(self.fileno()), file=sys.stderr) try: - self._send(SymbolAtom("PyKX cannot receive queries in client mode"), + self._send(SymbolAtom("KDB-X Python cannot receive queries in client mode"), error=True) except BaseException: pass @@ -866,7 +864,7 @@ def _recv(self, locked=False, acceptAsync=False): print("WARN: Discarding unexpected sync message from handle: " + str(self.fileno()), file=sys.stderr) try: - self._send(SymbolAtom("PyKX cannot receive queries in client mode"), + self._send(SymbolAtom("KDB-X Python cannot receive queries in client mode"), error=True) except BaseException: pass @@ -899,7 +897,7 @@ async def _recv_socket2(self, sock): self.close() except BaseException: self.close() - raise RuntimeError("PyKX attempted to process a message containing less than " + raise RuntimeError("KDB-X Python attempted to process a message containing less than " "the expected number of bytes, connection closed." f"\nReturned bytes: {chunks}.\n" "If you have a reproducible use-case please raise an " @@ -960,7 +958,7 @@ def _recv_socket(self, sock): self.close() except BaseException: self.close() - raise RuntimeError("PyKX attempted to process a message containing less than " + raise RuntimeError("KDB-X Python attempted to process a message containing less than " "the expected number of bytes, connection closed." f"\nReturned bytes: {chunks}.\n" "If you have a reproducible use-case please raise an " @@ -1030,7 +1028,7 @@ def upd(self, table: str, data:Union[list, Table]) -> None: Execute `#!q .u.upd` on a remote q process. This function assumes the definition of `#!q .u.upd` on the remote q process takes the same count and data type of arguments as the default implementation (q keyword `#!q insert`). The `#!python data` argument - will be converted to a list if it is a PyKX `#!python Table`. + will be converted to a list if it is a KDB-X Python `#!python Table`. Parameters: table: The name of the global variable on the q process to update. @@ -1098,11 +1096,11 @@ def file_execute( ``` """ wlist = ['k', 'q', 'p', 'py'] - with open(pykx_lib_dir/'q.k', 'r') as f: + with open(pykx_lib_dir/'pykx.q', 'r') as f: lines = f.readlines() for line in lines: if 'pykxld:' in line: - ld = line[7:].encode() + ld = line[12:].encode() if isinstance(file_path, str): path_stem = Path(file_path).suffix[1:] if not path_stem in wlist: @@ -1115,7 +1113,7 @@ def file_execute( $[any stem~/:("p";"py"); $[`pykx in key `; .pykx.pyexec "\n" sv code; - '"PyKX must be loaded on remote server"]; + '"pykx must be loaded on remote server"]; value (@';last file;enlist[file],/:value[\"k)\",fn]code) ] } @@ -1196,7 +1194,6 @@ def __init__(self, connection_timeout: Timeout in seconds for connection to `q` server. If left as default `None`, the socket will not timeout when connecting. - Note: The `#!python username` and `#!python password` parameters are not required. The `#!python username` and `#!python password` parameters are only required if the q server requires authorization. Refer to @@ -1264,7 +1261,6 @@ def __init__(self, reconnection_delay=reconnection_delay, reconnection_function=reconnection_function, connection_timeout=connection_timeout, - ) super().__init__() @@ -1334,13 +1330,13 @@ def __call__(self, q('{x set y+til z}', 'async_query', 10, 5, wait=True) ``` - Call a PyKX Operator function with supplied parameters + Call a KDB-X Python Operator function with supplied parameters ```python q(kx.q.sum, [1, 2, 3]) ``` - Call a PyKX Keyword function with supplied parameters + Call a KDB-X Python Keyword function with supplied parameters ```python q(kx.q.floor, [5.2, 10.4]) @@ -1472,7 +1468,6 @@ def __init__(self, reconnection_delay: float = 0.5, reconnection_function: callable = reconnection_function, connection_timeout: Optional[float] = None, - ): """Interface with a q process using the q IPC protocol. @@ -1521,7 +1516,6 @@ def __init__(self, connection_timeout: Timeout in seconds for connection to `q` server. If left as default `None`, the socket will not timeout when connecting. - Note: The `#!python username` and `#!python password` parameters are not required. The `#!python username` and `#!python password` parameters are only required if the q server requires authorization. Refer to @@ -1771,13 +1765,13 @@ def __call__(self, await q('{x set y+til z}', 'async_query', 10, 5, wait=True) ``` - Call a PyKX Operator function with supplied parameters + Call a KDB-X Python Operator function with supplied parameters ```python await q(kx.q.sum, [1, 2, 3]) ``` - Call a PyKX Keyword function with supplied parameters + Call a KDB-X Python Keyword function with supplied parameters ```python await q(kx.q.floor, [5.2, 10.4]) @@ -2123,7 +2117,6 @@ def __init__(self, connection_timeout: Timeout in seconds for connection to `q` server. If left as default `None`, the socket will not timeout when connecting. - Note: The `#!python username` and `#!python password` parameters are not required. The `#!python username` and `#!python password` parameters are only required if the q server requires authorization. Refer to @@ -2463,7 +2456,7 @@ def _recv_socket_server(self, sock): # noqa return elif len(chunks) <8: self.close() - raise RuntimeError("PyKX attempted to process a message containing less than " + raise RuntimeError("KDB-X Python attempted to process a message containing less than " "the expected minimum number of bytes, connection closed." f"\nReturned bytes: {chunks}.\n" "If you have a reproducible use-case please raise an " @@ -2866,7 +2859,6 @@ def __init__(self, connection_timeout: Timeout in seconds for connection to `q` server. If left as default `None`, the socket will not timeout when connecting. - Note: The `#!python username` and `#!python password` parameters are not required. The `#!python username` and `#!python password` parameters are only required if the q server requires authorization. Refer to @@ -2988,13 +2980,13 @@ def __call__(self, q('{x set y+til z}', 'async_query', 10, 5, wait=True) ``` - Call a PyKX Operator function with supplied parameters + Call a KDB-X Python Operator function with supplied parameters ```python q(kx.q.sum, [1, 2, 3]) ``` - Call a PyKX Keyword function with supplied parameters + Call a KDB-X Python Keyword function with supplied parameters ```python q(kx.q.floor, [5.2, 10.4]) diff --git a/src/pykx/lib/4-1-libs/q.k b/src/pykx/lib/4-1-libs/q.k deleted file mode 100644 index b48fde4..0000000 --- a/src/pykx/lib/4-1-libs/q.k +++ /dev/null @@ -1,269 +0,0 @@ -\d .q -/each: +-*%&|^<>=$ <= >= <> @ ? in within bin div abs log exp sqrt sin cos tan f' f\: f/: -neg:-:;not:~:;null:^:;string:$:;reciprocal:%:;floor:_:;ceiling:-_-:;signum:{(x>0)-x<0} -mod:{x-y*x div y};xbar:{$[9h=t:abs[@x];x*r+y=x*1+r:(y:9h$y)div x;x*y div x:$[16h=t;"j"$x;x]]};xlog:{log[y]%log x};and:&;or:|;each:{x'y};scan:{x\y};over:{x/y};prior:{x':y} -mmu:$;lsq:!;inv:!:;md5:-15!;ltime:%:;gtime:{t+x-%t:x+x-%x}; /xnull:{$[0>@y;(,y)@~x=y;x=y;y 0N;y]} - -/aggr: last sum prd min max avg wsum wavg f/ /beta:{cov[x;y]%var x} -count:#:;first:*:;svar:{(n*var x)%-1+n:(#x)-+/^x};sdev:{sqrt svar x};scov:{(n*cov[x;y])%-1+n:(#x)-+/^x+y};med:{avg x(@x;'`rank;1_x,,x 0N]}; -rank:{$[0h>@x;'`rank;<@x;'`rank;@x;'`rank;>x]} -asc:{$[99h=@x;(!x)[i]!`s#r i:@x;'`rank;`s#x@r:. x;0h>@x;'`rank;x@>x]} - -msum:{$[99h=@y;(!y)!.z.s[x;. y];y-(-x)_(0i*x#y),y:+\y]};mcount:{msum[x;~^y]};mavg:{msum[x;0.0^y]%mcount[x;y]};mdev:{sqrt mavg[x;y*y]-m*m:mavg[x;y:"f"$y]} -xrank:{$[0h>@y;'`rank;_y*x%#y:<@y;'`rank;y(!#y)-x]};rotate:{$[0h>@y;'`rank;98h<@y;'`type;#y;,/|(0;mod[x;#y])_y;y]};ema:{(*y)(1f-x)\x*y} - -/other: ~,#_ !. getenv exit -distinct:?:;group:=:;where:&:;flip:+:;type:@:;key:!:;til:{$[0>@x;!x;'`type]};value:get:.:;attr:-2!;cut:{$[0h>@x;x*!-_-(#y)%x;x]_y} -set:{$[@x;.[x;();:;y];-19!((,y),x)]};upsert:.[;();,;] / :: ,: files? -raze:,/;union:?,;inter:{x@&x in y};except:{x@&~x in y};cross:{n:#m:&(#x)##y;$[99h=@x;((!x)[m],'n#!y)!(. x)[m],'n#. y;((),x)[m],'n#y]} /extant:{x@&~^x} -sv:{x/:y};vs:{x\:y};sublist:{$[99h=@y;sublist[x;!y]!sublist[x;. y];~0h>@x;$[.Q.qp y;.Q.ind[y];y]i+!"j"$0|x[1]&(#y)-i:*x;abs[x]<#y;x#y;y]} - -/file&comm -read0:0::;read1:1::;hclose:>:;hdel:~:;hsym:"s"$-1!';hcount:-7!;peach:{x':y};system:."\\", - -/string: like ss -ltrim:{$[~t&(77h>t)|99ht)|99ht)|99ht)|99h@z;:[;z];z]]} - -/select insert update delete exec / fkeys[&keys] should be eponymous, e.g. order.customer.nation -/{keys|cols}`t `f's{xasc|xdesc}`t n!`t xcol(prename) xcols(prearrange) FT(xcol xasc xdesc) -view:{$`. .`\:x};tables:{."\\a ",$$[^x;`;x]};views:{."\\b ",$$[^x;`;x]} -cols:{$[.Q.qp x:.Q.v x;.Q.pf,!+x;98h=@x;!+x;11h=@!x;!x;!+0!x]} /cols:{!.Q.V x} -xcols:{(x,f@&~(f:cols y)in x)#y};keys:{$[98h=@x:.Q.v x;0#`;!+!x]};xkey:{(#x)!.[0!y;();xcols x]}; -xcol:{.Q.ft[{+$[99h=@x;@[!y;(!y)?!x;:;. x];x,(#x)_!y]!. y:+y}x]y};xasc:{$[$[#x;~`s=-2!(0!.Q.v y)x;0];.Q.ft[@[;*x;`s#]].Q.ord[<:;x]y;y]};xdesc:{$[#x;.Q.ord[>:;x]y;y]} -fkeys:{(&~^x)#x:.Q.fk'.Q.V x};meta:{([!c].Q.ty't;f:.Q.fk't;a:-2!'t:. c:.Q.V x)} - -/ R uj R(union join) R lj K(left(equi/asof)join) trade asof`sym`time!(`IBM;09:31:00.0) -lj:{.Q.sx[x[;z]]y}{$[$[99h=@y;(98h=@!y)&98h=@. y;()~y];x,\:y;'"type"]} /;la:{$[&/j:z>-1;x,'y z;+.[+ff[x]y;(!+y;j);:;.+y z j:&j]]}{la[x;. y](!y)?(!+!y)#x}[;y]]x} /lj:,\:;aj:{lj[y]`s#xkey[x]z};aj0:{lj[y]`s#(x#z)!z}; /;bn:{@[i;&0>i:x bin y;:;#x]} -ljf:{.Q.sx[x[;z]]y}{$[`s=-2!y;ajf[!+!y;x;0!y];$[&/j:(#y:. y)>i?:(!+i:!y)#x;.Q.fl[x]y i;+.[+x;(f;j);:;.+.Q.fl[((f:!+y)#x:.Q.ff[x]y)j]y i j:&j]]]} -.Q.ajf0:{[f;g;x;y;z]x,:();z:0!z;d:$[g;x_z;z];g:(:;^)f;f:(,;^)f;$[&/j:-1i:(!y)?(!+!y)#x}[;y]]x} -ijf:{.Q.ft[{.Q.fl[x j]y i j:&(#y:. y)>i?:(!+i:!y)#x}[;y]]x} -pj:{.Q.sx[{x+0i^y(!+!y)#x}[;y]]x};asof:{f:!$[99h=@y;y;+y];(f_x)(f#x)bin y} -uj:{$[()~x;y;()~y;x;98h=@x;x,(!+x:.Q.ff[x;y])#.Q.ff[y;x];lj[(?(!x),!y)#x]y]} -ujf:{$[()~x;y;98h=@x;x,(!+x:.Q.ff[x;y])#.Q.ff[y;x];ljf[(?(!x),!y)#x]y]} - -/wj[-1000 2000+\:trade`time;`sym`time;trade;(quote;(max;`ask);(min;`bid))] (given `sym`time xasc quote) -ww:{[a;w;f;y;z]f,:();e:1_z;z:*z;y,'n#+(:/'f)!+{[e;d;a;b]e .'d@\:\:a+!b-a}[*:'e;z f:1_'e]/'$[n:#*w;+$[#g;(g#z)?g#y;0]|/:a+$[#g:-1_f;(f#z)bin@[f#y;*|f;:;]@;z[*f]bin]'w;,0 0]} -wj:{[w;f;y;z].Q.sx[ww[0 1;w;f;;z]]y};wj1:{[w;f;y;z].Q.sx[ww[1;w-1 0;f;;z]]y} - -fby:{$[(#x 1)=#y;@[(#y)#x[0]0#x 1;g;:;x[0]'x[1]g:.=y];'`length]};xgroup:{x,:();a:x#y:0!y;$[#x_:y;+:'x@=a;a!+f!(#f:!+x)#()]};ungroup:{$[#x:0!x;,/+:'x;x]} -ej:{x,:();y[&#:'i],'(x_z)(!0),/i:(=x#z:0!z)x#y:0!y} /{ungroup lj[z]xgroup[x]y} - -/`[:../]t[.{csv|txt}] -save:{$[1=#p:`\:*|`\:x:-1!x;set[x;. *p]; x 0:.h.tx[p 1]@.*p]}' -load:{$[1=#p:`\:*|`\:x:-1!x;set[*p;. x];set[*p].h.xt[p 1]@0:x]}' -rsave:{x:-1!x;.[`/:x,`;();:;.*|`\:x]}' -rload:{x:-1!x;.[*|`\:x;();:;. x]}' -dsave:{.[*x;1_x,y,`;:;@[;*!+a;`p#].Q.en[*x]a:. y];y}/: - -show:{1 .Q.s x;};csv:"," / ";" also \z 1 for "D"$"dd/mm/yyyy" - -parse:{$["\\"=*x;(system;1_x);-5!x]};eval:-6!;reval:-24! -\d .Q /def[`a`b`c!(0;0#0;`)]`b`c!(("23";"24");,"qwe") -k:4.1;K:0Nd;host:-12!;addr:-13!;gc:-20!;ts:{-34!(x;y)};gz:-35!;w:{`used`heap`peak`wmax`mmap`mphy`syms`symw!(."\\w"),."\\w 0"} / used: dpft en par chk ind fs fu fc -res:`abs`acos`asin`atan`avg`bin`binr`cor`cos`cov`delete`dev`div`do`enlist`exec`exit`exp`getenv`hopen`if`in`insert`last`like`log`max`min`prd`select`setenv`sin`sqrt`ss`sum`tan`update`var`wavg`while`within`wsum`xexp`from`by -addmonths:{("d"$m+y)+x-"d"$m:"m"$x} -f:{$[^y;"";y<0;"-",f[x;-y];y<1;1_f[x;10+y];9e15>j:"j"$y*prd x#10f;(x_j),".",(x:-x)#j:$j;$y]} -fmt:{$[x<#y:f[y;z];x#"*";(-x)$y]} -pykxld:{x:("#!"~2#*x)_x:-1!'x;+(1+*:'i;)@"\n"/:'x i:(&|1^\|0N 0 1@"/ "?*:'(v:x i),'"/")_i:&~|':(b?-1)#b:+\-/x~\:/:+,"/\\"} - -/DO NOT USE ROUTINES PAST HERE. SUBJECT TO CHANGE -ff:{$[&/(!+y)in f:!+x;x;x,'(f_y)(#x)#0N]} -fl:{$[98h=t:@x;+fl[+x;+y];99h=t;@[x,y;f;:;x[f]fl'y f@:&(f:!x)in!y];t&t<77h;x^y;@[y;i;:;x i:&(0<#:'x)&~#:'y]]} -opt:{[o]x::$[#i:&o like"-[^0-9]*";i[0]#o;o];((`$1_*:)'o)!1_'o:i_o};def:{x,((!y)#x){$[0h>@x;*:;::](@*x)$y}'y};ld:-39! -qt:{$[99h=@x;(98h=@!x)&98h=@. x;98h=@x]}; v:{$[-11h=@x;.$[":"=*t:$x;`$t,"/";x];x]}; -qp:{$[~98h=@x;0;@x:.+x;~":"=*$x;0]}; V:{$[qp x:v x;((,pf)!,. pf),+$[(~x in!d0[])&(x:.+x)in !*. .Q.vt;.Q.vp x;d0[]@x];0h>@*x:+0!x;x@'!x;x]} -ft:{$[{$[99h=@t:v x;98h=@. t;0]}y;[n:#+!y;n!x 0!y];x y]}; -ord:{if[~&/b:{(!#x)=x?x}c:.q.cols z;'"dup ",$c b?0b];if[~&/b:(y,:())in c;'y b?0b];ft[@[;!+t:0!v z;::]]z;ft[@[;!+t;@[;x@+y!t y]]]z};nv:{$["."~*$x;x;`$"..",$x]} - -tx:{$[(76ht;t;11h=t:@x:. nv@!x;t;98h=t;7h;1=#x:+!x;tx@*x;7h]};tt:0|tx'. V@ -fk:{$[(20h>t)|76h@. nv t:!x;`;t]};t:" bg xhijefcspmdznuvts";ty:{$[0h>x:tx x;.q.upper t@-x;t x]} -nct:{$[`~x;,/nct'.q.tables`.;([]n:x;c:.q.cols x;t:tt x)]} - -fu:{[f;x]$[0h>@x;f x;f[u](u:?x)?x]} /uniques -fc:{$[1@x;`${$[(*x)in n,"_ ";"a",x;x]}x@&(x:$x)in an;ft[{$[98h=@x;+.z.s[+x];[s[i]:`$($s i:&((s:id'!x) in`i,res,!`.q)),'"1";({unm[x]}/s)!. x]]}]x]} -j10:64/:b6?;x10:b6@0x40\: /base64 J from char10 -j12:36/:nA?;x12:nA@0x24\: /base36 J from char12(cusip) -btoa:-32!;atob:-44!;sha1:-33!;prf0:+`name`file`line`col`text`pos!*-37! -objp:{x like"[mgs][s3]://*"}; -lo:{if[$[1>@d:!f:-1!x;1;`.d~*d];:.[$[qt d;*|`\:f;99=@d;`.;'`type];();:;d:. f]];d@:&~d like"*$";p:(d=`par.txt)|d like"[0-9]*"; - if[y;if[objp x;'"no cd to object storage"];."\\cd ",$x;f:`:.];{.q.set[*|`\:x]$[0h>@!x;. x;x`]}'`/:'f,'d@&~(d=`html)|(d like"*#")|p|s:|/d like/:("*.[kq]_";"*.?");if[|/p;L[d@&p;f;~y|objp x]];if[z&~`.=x;(."\\l ",$:)'`/:'f,'d@&s&~p];} -l:lo[;1;1] - -sw:{.q.sublist[_.5**|."\\C"]x};tab:{" "/:'(x*|/#:''y)$/:y};t0:{$[x>#y;((#*y)#'" -"),y;y]} -s1:{-3!x} /{$[0>t:@x;$x;99h76h;s1'x;t;$x;|/(97ht:?@:'x;s1'x;(1<#?#:'x)|$[1=#t;(*t)in 1 4 10h;0];s1'x;tab[1]@s2'sw'x]} -S:{x[1]{$[x<#y;((x-2)#y),"..";y]}'$[t&77h>t:@z;,s1 z;99h#x)|(@x)&~11=@x} - -/ CAN EXIT HERE FOR SMALL Q -/ pt(tables) pf(date/month/year/int) pd(dirs) pv(values) pn(count) pt::0#pf::` -vt:(,`)!,()!(); -bvfp:{g:$[(::)~x;max;min];x:.Q.d;d:$[y~z;{`/:'x,'d@&(d:!x)in`$$y}[;z];{u@&{11h=@!x}'u:`/:'x,'`$$y}[;y@:&y in pv]]'P:$[`par.txt in!x;jp[x]'`$0:`/:x,`par.txt;,x];if[.Q.vt~(,`)!,()!();.Q.vt:P!(#P)#.:.Q.vt]; - t:?,/!:'vt:{(&#:'x)(=,/. x)}'{({("DMJJ"`date`month`year`int?pf)$$last@`\:x}'x)!!:':x}'d; - if[#nt:(exc:.q.except)[t;!vp];d:{`/:'x[(. y)[;0]],'(`$$(. y)[;1]),'!y}[P]@{i:y@&:x=y x:@[x;&x~\:();:;*0#pv];(i;x i)}[;g]'+:nt#/:g''nt#/:vt;.Q.vp,:nt!{(+(,pf)!,0#pv),'+(-2!'.+x)#'+|0#x:?[x;();0b;()]}':d;.Q.vt:.Q.vt,\:nt!(#nt)#,exc[z;y]];.Q.vt:(.Q.vt,''P!exc[y]''vt)exc''vt:(nt,!vp)#/:vt;{.[x;();:;+.q.except[!+vp x;pf]!x]}'exc[!vp;pt];pt::.q.asc@!vp;vpv::z;} -bvi:{$[(`vpv in !.Q);bvfp[x;?.q.except[pv;vpv];?pv];bv x]} -bv:{vt::(,`)!,()!();vp::(0#`)!();vpv::0#pv;dpv:?pv;bvfp[x;dpv;dpv]} - -sp:{$[0>."\\s";x'y;x':y]} -pt:pm:();MAP:{{$[0>@a:.+0!. x;.q.set[x]@.`$-1_$a;]}'a@&~(a:."\\a")in pt;pm::();if[#pt;pm::pt!sp[{(`u#pd,'pv)!sp[p2[(x;();0b;())]/;+(pd;pv)]}]pt]} -pfc:{$[$[#pm;|/{|/0b,{f:{pf~*`\:x};|/$[-11h=t:@x;f x;(t&~11h~t)|2>#:x;0b;11h=t;f'x;.z.s'x]}'$[99h=@x;.:x;x]}'x 1 2 3;0];+((,pf)!,z),+y;y]} -dd:{`/:x,`$$y};d0:{dd[*|pd;*|pv]};p1:{$[#pm;pm[x](y;z);z in vt[y;x];vp x;+(!+. x)!`/:dd[y;z],x]};p2:{0!(?).@[x;0;pfc[x;;z]p1[;y;z]@]} - -p:{$[~#D;sp[p2[x;d]]y;sp[p2[x]/;,/P[i](;)''y]@<,/y@:i:&0<#:'y:D{x@&x in y}\:y]} -view:{pd::PD x:$[(::)~x;x;$[#x:&PV in x;x;'"invalid partition filter"]];u~:?u::..[pf;();:;pv::PV x];.[;();:;]'[pt;sp[{+(x . y,`.d)!y}[x]]pt::!x:d0[]];pn::pt!(#pt)#()} - -jp:{$[$["w"~*$.z.o;u[$[(_u:$y)like"[a-z]:*";2;0]]in"\\/";("/"=*$y)|objp y];-1!y;`/:x,y]};rp:-500!;fobj:{!:'?{-1!`$("/"/:3#"/"\:x),"/_"}'u@&objp'u:1_'$x;x} -L1:{[x;y;z;i;p]D::`$'$D;fobj@,d::$[z;rp y;y];$[x~,`par.txt;if[~#x:,/D::$[i;{?:'D,'{x@&{11h=@!x}'`/:'y,'x:`$$x}[x]'y}[p];{x@&~(x:!x)like"*$"}']@P::fobj@jp[d]'`$0:`/:d,*x;'empty];i;x:`$$?PV,p@&{11h=@!x}'jp[d]'`$$p];if[i;dv:+(PD;PV);{PN[y;x]:0N}[&PV in p]'pt];if[^*PV::x@:."\\p")|."\\_";cn'.:'pt];} -L:{D::();L1[x;y;z;0;()]};li:{L1[$[D~();::;,`par.txt];d;0;1;x,()]} -/L:{P::,`:.;D::,x;pf::`date;pt::!P[0]@**D;T::P!P{z!{x!(y . ,[;`]z,)'x}[x;y]'z}[pt]'D} -cN:{$[~#PN x;PN[x]:sp[#p1 .;+(x;PD;PV)];#i:&^PN x;PN[x;i]:sp[#p1 .;+(x;PD i;PV i)];];PN x};cn:{$[#n:pn x:.+x;n;pn[x]:cN[x]@&PV in pv]};pcnt:{+/cn x};dt:{cn[y]@&pv in x} -ind:{,/i[j]{fp[pf;p]p1[x;pd y;p:pv y]z}[.+x]'(j:&~=':i)_y-n i:(n:+\0,cn x)bin y} -fp:{+((,*x)!,(#z)#$[-7h=@y;y;(*|x)$y]),+z} -foo:{[t;c;b;a;v;d]if[v;g:*|`\:b f:*!b;b:1_b];,/r(::;i)1=#i:&0<#:'r:$[v|~#a;d fp[$[v;f,g;pf]]';::]p[(.+t;c;b;a)]d} - -/ select{u's|a's[by[date,]u's]}from t where[date..],[sym{=|in}..],.. -a2:({(%;(sum;("f"$;x));(sum;(~^:;x)))};{(sum;(*;("f"$;x);y))};{(%;(wsum;x;y);(sum;(*;x;(~^:;y))))};{(cov;x;x)};{(sqrt;(var;x))} - {(-;(avg;(*;("f"$;x);y));(*;(avg;x);(avg;y)))};{(%;(cov;x;y);(*;(dev;x);(dev;y)))};{(.q.scov;x;x)};{(sqrt;(.q.svar;x))};{(*;(%;(#:;`i);(+;-1;(#:;`i)));(cov;x;y))};{'`part}) - -/ x0 translate;x1 aggrs;x2 translate ?terminal (subselect/exec within partition) unknown assumed uniform? -qd:{$[(#:)~*x;(?:)~*x 1;0]};xy:{`$$*&x~/:y}; x1:{$[qb x;();IN[*x;a0];$[qd x;1_x;,x];,/x1'1_x]} -x0:{$[qb x;x;IN[*x;a1];x0 a2[a1?*x]. 1_x;x0'x]};x2:{$[qb x; x;IN[*x;a0];$[qd x;(#:;(?:;(,/;xy[x 1]y)));[y:xy[x]y;$[(?:)~*x;(?:;(,/;y));(#:)~*x;(sum;y);(*x;y)]]];@[x;&~(::)~/:x;x2[;y]]]} -ua:{((`$$!#u)!u;x2[;u:?,/x1'x]'x:x0'x)};q0:{$[~qb x;,/q0'x;-11h=@x;*`\:x;()]};qe:{$[#x;99h=@x;1]} -ps:{[t;c;b;a]if[-11h=@t;t:. t];if[~qe[a]&qe[b]|-1h=@b;'`nyi];d:pv;v:$[q:0>@b;0;~#b;0;-11h=@v:*. b;pf~*`\:v;0] - if[$[~#c;0;@*c;0;-11h=@x:c[0]1;pf~*`\:x;0];d@:&-6!*c;c:1_c] - if[$[#c;0;(g:(. a)~,pf)|(. a)~,(#:;`i)];f:!a;j:dt[d]t;if[q;:+f!,$[g;?d@&0(#.:)'t:.q.tables`.;if[h:@[hopen;h;0];h"\\l .";>h]} - -/loop through text /lc:{+/{+/0xa=1:(x;y*z;y)}[x;m]'!-_-(-7!x)%m:1048576} /line count of big file -fsn:{[f;s;n]>[-7!s]{[f;s;x;n]i:(#r)^1+last@&"\n"=r:1:(s;x;n);f@`\:i#r;x+i}[f;s;;n]/0};fs:fsn[;;131000] -fpn:{{r:.Q.trp[y;h;{>x;'(y;z)}h:hopen":fifo://",1_$x];>h;r}[y - {[f;h;n]b:"x"$();while[#s:1:h;if[~n>#b,:s;v:`\:b,0x0a;if[1<#v;f@-1_v];b:"x"$last v]];if[#b;f@`\:b]}[x;;z]]};fps:fpn[;;131000] -/e.g. p:` sv(d:`:dir;`2007.02.12;`t);@[;`sym;`p#]p{@[x;`;,;`sym xasc .Q.en[d]get y]}/rdb's -dsftg:{[dpt;xom;f;tw;g]d:*dpt;dpt:par . dpt;x:*xom;if[~0m:_a:(m-:o:xom 1)%b:+/tw 10=@*tw;'`length];0N!($dpt),": ",$m&:M;i::0;n::1000000&_1e8%b - do[-_-m%n;@[dpt;`;$[i;,;:];en[d]@+g f!tw 1:(x;o+b*i;b*n&:m-i)];0N!i+:n];dpt};M:0W - -/ fill in empty partitions -chk:{if[x~(::);'"missing dir argument"];f:{`/:'x,'d@&(d:!x)like"[0-9]*"};d@:>.:'$last'`\:'d:$[`par.txt in!x;,/f'-1!'`$0:`/:x,`par.txt;f x] - {[e;u;d]u[i]{.[x;(y;`);:;?[z;();0b;()]]}[d]'e i:&~u in!d}[d[(+u in/:t)?\:1b](0#.)'u,'`;u:?,/t:!:'d]'d} -Ll:-100!;Lp:-101!;Lx:-102!;Lu:-103!;Ls:-104!;fqk:{("<"~*x)|x~*-3#.{}};fql:{x[1;1]~""};btx:{r:,Lp x;while[@x:Lu(0b;x);r,:(~fql v)#,v:Lp x;if[(fqk v[1]1)&".Q.dbg"~*v 1;f:(Lx(x;"k";,"v"))1]];$[~#f;r;@[r;r?f;,;,">>"]]};bt:{1'pl'(2*4<|/#:'v)_v:1_btx Ll`;};sbt:{,/pl'x};trp:{-105!(x;,y;z)};trpd:{-105!(x;y;z)} -dr:{0:0};dw:{@[2@;x;::];};pl0:{.[{[f;v;x;y;z]g:{$[x>0;x&#y;x|-#y]#y};m:1;n:0|(b:+\0,1+#:'s:`\:s)bin y:(0|y)+2*"k)"~2#s:x 3;sf:$[#x 1;x[1],$[0>x 2;"";":",($n+x 2)],": ";""];sn:{$[(#x)&@x;((2*".."~2#x)_x),":";()]}@*x;h:($[4<#v;*|v;" "],5$"[",($z),"]") - if[("locked"~x 3)|$[f;0b;fqk x 1];:h,"(",($[#sn;-1_sn;-2_sf]),")\n\n"];h,:sf,sn;h,`/:((n>m)#,""),g[-m+1;(n+1)#s],(,(" \t"(((#h)*n<1)#0),(y-b n)#s[n]="\t"),"^"),g[m](n+1)_s}[x;y];3#1_y;{dw"pl0\n";}]};pl:pl0 0 -jl8:{$[$[#x;~x[3]in y[;3];0b];,x;()],y};srr:{@[{if["abort"~*x;:""];"'",$[.z.q;($.z.Z)," ";""],(x[0],"\n"),/pl'jl8[x 3;x 2]};x;"srr\n"]};prr:{dw srr x}; -lu:{f:{$[@a:Lu(x;*y);(a;Lp a);()]}x;r:{$[#x;fql x 1;0b]}f/f y;$[#r;$[.Q.dbg~**r 1;y;r];y]} -DL:0;dbg:{pm:{dw" q"[x],$[1<#d:$."\\d";d;""]," )"[x],(1|DL)#")"};dq:{v:$[#*y;(~"k)"~2#y[1;3];*(.**y)3);1<#*y 1;@[*y 1;0;"q"=];(1b;`)];x v 1;*v}ds:{."\\d ",((~"."=*x)#"."),x:$x};de:@[{DL-:1;x y;{."\\x .z.e",x}'"xy"}ds;;::] - @[{DL+:1;.z.ex:*x 1;.z.ey:x[1]1};x;::];b:$[#*v:Lp y;0xff~(*. bf:**v)bi:v[0;1];0b];if[b;dw "#",($bi),"\n"];w:(y0:y;v:Lp y);if[e:@*x;prr x];d0:."\\d" - y:*w:{$[~fqk x[1;1]1;x;lu[0b]x]}/w;q:dq v:w 1;dw pl v;while[*(~(,"\\")~l:dr[];pm[q]);$[l~"\\ ";q:~q;l in+,".`&";[w:$["&"=*l;;lu["."=*l]](y;v);y:*w;v:w 1;if[y~y0;prr x];dw pl0[1]v;q:dq v] - (~e)&l in+,":>";:*($[b;bc[bf;bi];]0 0N(":"=*l);de d0);"'"=*l;:*((1_l;());de d0);c&~#*r:Lx(y;"kq"q;(c:":"=*l)_l);:*(r;de d0);$[#*r;::;dw $[q;.Q.s;{$[(::)~x;"";`/:,-3!x]}]r 1]]];de d0;("abort";())} -err:{(*x;jl8[x 3;$[@y;btx y;()],x 2])}; BP:(,"")!,();bp:{*|(Ls(x;(),y;(#y)#z);(*. x)y)};bs:{BP[$x],:y!bp[x;y,:();0xff];};bu:{bp[x;y;BP[$x;y,:()]]} -bd:{bu[x;y];@[`BP;$x;_;y]};bc:{bu[x;y];dbg::{[d;f;i;r;u;v]dbg::d;bs[f;i];$[#*u;d[u;v];r]}[dbg;x;y;z];-1} - -\d .h / c -htc:{,/("<";x;y;"")};hta:{,/("<";$x;,/" ",'($!y),'"=",'{$[10h=@x;"\"",x,"\"";$x]}'. y;">")};htac:{,/(hta[x]y;z;"")} -ha:{htac[`a;(,`href)!,x]y};hb:{htac[`a;`target`href!`v,,x]y};pre:{htc[`pre]@`/:x};xmp:{htc[`xmp]@`/:x} -d:" ";cd:{.q.csv 0:$[.Q.qt x;![x;();0b;(!t)[c]!,:'.q.sv[d]@/:'$v c:&(~l=-10h)&0>l:.Q.tx'v:. t:+0!x];x]};td:{"\t"0:x};hc:{"<"/:"<"\:x};xs:.q.ssr/[;"&<>";("&";"<";">")] -xd:{g:{(#*y)#'(,,"<",x),y,,,""};(,""),(,/'+g[`r]@,/(!x)g'{,xs'$[11h=@x;$x;t&77h>t:@x;$x;x]}'x:+0!x),,""} - -ex:("";);iso8601:{$[^x;"";@[$"p"$x;4 7 10;:;"--T"]]} -eb:{htac[`Workbook;(`$"xmlns",/:$``:ss`:o`:x`:html)!("urn:schemas-microsoft-com:office:",/:$(2#`spreadsheet),`office`excel),,"http://www.w3.org/TR/REC-html40"]x} -es:{htac[`Worksheet;(,`ss:Name)!,$x]htc[`Table]@,/(htc[`Row]@,/ec')'(,!+y),+.+y:0!y};ed:{ex eb es[`Sheet1]x};edsn:{ex eb@"\r\n"/:(!x)es'. x} -ec:{htc[`Cell]htac[`Data;(,`ss:Type)!,$`String`Number`String`DateTime`DateTime`String i](xs;$:;xs@$:;iso8601;iso8601 1899.12.31+"n"$;xs@$:)[i:-10 1 10 12 16 20h bin -@x]x} - -\d .j /[]{} Cbg*xhijefcspmdznuvt -e:{"c"$$[x<128;x;0b/:'@[;0;(1b,i#1b),]10b,/:((5 10;0 4 10)i:x>2047h)_0b\:"h"$x]}@256/: -q:"\"";s:{q,x,q};es:{s@,/{$[x in r:"\t\n\r\"\\";"\\","tnr\"\\"r?x;x]}'x};J:(($`0`1)!$`false`true;s;{$[#x;x;"null"]};es;{s@[x;&"."=8#x;:;"-"]};s)1 2 5 11 12 16h bin -/j:{$[10=abs t:@x;es x;(::)~x;"null";99=t;"{",(","/:({$[q~*x;x;s x]}'j'!x),'":",'j'. x),"}";-1*x;"F"$x;"n"=*x;0n;"t"=*x]} -/k:{c x@&~v[x]&x in" \t\n\r"};v:{~0(0 1 0;1 0 2;1 1 1)\x*3>x:" \"\\"?x};u:{?[v x;x;" "]};d:{$[1" -ka:-41! -c0:`024C7E;c1:`958600;logo:"kx.com" /logo:,/(c0;c1;c0){htac[`font;(,`color)!,x]y}'("[kx";"systems";"]") /808000 -logo:,/(c0;c1){htac[`font;(,`color)!,x]htc[`b]y}'("KX";".com") -sa:"a{text-decoration:none}",/`link`visited`active{"a:",($x),"{color:",y,"}"}'$(c0;c0;c1) -html:{htc[`html]htc[`head;htc[`style]sa,sb],htc[`body]x};sb:"body{font:10pt verdana;text-align:justify}" -fram:{htc[`html]htc[`head]htc[`title;x],htac[`frameset;(,`cols)!,($116|316&43+(7+"?"~*z)*|/#:'y),",*"]@,/hta[`frame]'((,`src)!,*z;`name`src!`v,,*|z)} -jx:{[j;x]x:val x;$[$[.Q.qt[x];(N:(*."\\C")-4)";hr:{(#x)#"-"};nbr:htc[`nobr];code:{" "/:@[x;1+2*!_.5*#x:"\t"\:x;{htc[`code]nbr hc x}]} -http:{$[#i:ss[x]"http://";(i[0]#x),/{ha[x;x:i#x],(i:&/x?") ")_x}'i_x;x]};text:{`/:{htc[`p]http code x}'x@&0<#:'x} -data:{if[100<#*x:.Q.tab[1]n#'x,\:(n:|/#:'x:"\t"\:'x:1_'x)#,"";0N!#*x;0N!'x];xmp$["- "~2#x 1;@[x;1;hr];x]} -ht:{[t]x:0:`$"src/",(T:t:$t),".txt";if[~"."=**x;T:*x;x:1_x];h:.q.trim(n@:i)_'x i:&0#x;x,1;(y#x),1+x y-:1]}\n),'" ",/:h - x:`/:("Copyright © ",logo;htc[`h5]T),((#x)#(text;data))@'x:(&~=':~(0<#:'x)&" "=*:'x)_x - c:h{hb[x,"#",y;z],br}[a:t,".htm"]'H;c:nbr@`/:$[&/n=:1;c;1_,/(,br),/:(&n)_c] - (`$a)1:fram[T;H]b:"a/",/:("_",a;a);(`$b)1:'html'(($.z.D),htc[`h5;"Contents"],c;x);} - -\d .o /odbc timedec/cs could be 3(23,12) -ex:{$[.Q.qt x:."s)",x;0!x;x]} / char(1+) binary(display2*) -1 longvarchar -4 longvarbinary sql_no_total(-4)SAS/4096 -T:`text`bit```tinyint`smallint`int`bigint`real`float`char`varchar``date`date`datetime``time`time`time -T0: -1 -7 0 0 -6 5 4 -5 7 8 1 12 0 9 9 11 0 10 10 10h -B0:4096 1 0 0 1 2 4 8 4 8 1 255 0 6 6 16 0 6 6 6 -C0:4096 1 0 0 3 5 10 19 7 15 1 255 0 10 10 23 0 8 8 8 -PS: 1 2 0 0 2 2 2 2 2 2 3 3 0 2 2 2 0 2 2 2h /char basic both -t:0 1 4 5 6 7 8 9 10 11 14 15 19;Columns:{$[#x;Cols`$x;,/Cols'.q.tables`.]} -q)TI:`DATA_TYPE xasc([]TYPE_NAME:T t;DATA_TYPE:T0 t;COLUMN_SIZE:C0 t;LITERAL_PREFIX:`;LITERAL_SUFFIX:`;CREATE_PARAMS:`;NULLABLE:1h;CASE_SENSITIVE:1h;SEARCHABLE:PS t;UNSIGNED_ATTRIBUTE:0h;FIXED_PREC_SCALE:0h;AUTO_INCREMENT:0h;LOCAL_TYPE_NAME:`;MINIMUM_SCALE:0h;MAXIMUM_SCALE:0h) -q)TypeInfo:{$[count x;select from TI where DATA_TYPE="H"$x;TI]} /msqry32 uses special e.g. select p.i from p p -q)Special:{([]SCOPE:2h;COLUMN_NAME:(0&0 similar position substring trim {lower|upper} diff --git a/src/pykx/lib/html.q b/src/pykx/lib/html.q index 09dc093..b261d3b 100644 --- a/src/pykx/lib/html.q +++ b/src/pykx/lib/html.q @@ -3,7 +3,7 @@ // @private // @desc // Utility for generating the JSON data to be used for the rendering -// of a HTML version of a PyKX in-memory and splayed table +// of a HTML version of a KDB-X Python in-memory and splayed table util.html.memsplay:{[c;t] n:count t; cls:{x!x}$[c[1]1023;'version];vi:sum/ 2/:'.:'"0"^(r[0],20#"0";-20$r[1],10#"0";-10$r 2),\:"b";"***III"$`K`k`Kl`major`minor`patch!enlist[v],enlist["D"$"2022.03.10"],vi,raw}; .[`.comkxic;();,;.comkxic.Kf "3.0.0"]; .[`.comkxic.libs;();,;`nelf`kurl`objstor`qlog`restserver`bq`sql`l64!("6a93b2b";"7fa6ae8";"32363d0";"a49dc65";"00a5b89";"14d547d";"99f3988";"3fa87f1")]; -if[(.z.o~`l64)&$[0=#.z.x;1b;~"--no-qce"in .z.x]; +if[(.z.o~`l64)&$[0=#.z.x;in[_getenv`$"PYKX_QCE";("true";,"1")];"--qce"in .z.x]; {[] lf:{@[{."\\l ",$[""~x:getenv`QHOME;getenv[`HOME],"/q";x],"/",($x),"_"};x;{.pykx.i.kxic.loadfailed,:enlist[x]!enlist y}[x]]}; - d:`kurl.q`objstor.q`qlog.q`s.k!$`insights.lib.kurl`insights.lib.objstore`insights.lib.qlog`insights.lib.sql; - if[`s in !`;d:d _ `s.k]; + d:`kurl.q`objstor.q`qlog.q`s.k!$(`insights.lib.kurl`krl;`insights.lib.objstore`obj;`insights.lib.qlog`lg;`insights.lib.sql`sql); + if[@[{2<#: .s};(::);{0b}];d:d _ `s.k]; if[~0=#.z.x; - if["--no-kurl"in .z.x;d:d _ `kurl.q]; - if["--no-objstor"in .z.x;d:d _ `objstor.q]; - if["--no-qlog"in .z.x;d:d _ `qlog.q]; - if["--no-sql"in .z.x;d:d _ `s.k]]; - if[~0=#d;lf' (!d) @ & (.:d) in " "\:.z.l 4;]; / only load enabled features + if[~:in[_getenv`$"PYKX_QCE";("true";,"1")]|("--qce"in .z.x); + if[~:"--kurl"in .z.x;d:d _ `kurl.q]; + if[~:"--objstor"in .z.x;d:d _ `objstor.q]; + if[~:"--qlog"in .z.x;d:d _ `qlog.q]; + if[~:"--sql"in .z.x;d:d _ `s.k] + ]; + ]; + if[(`s.k in !d) & |/d[`s.k] in " "\:.z.l 4;@[.s.init;(::);{.pykx.i.kxic.loadfailed,:enlist[x]!enlist y}[`s.k]];d:d _ `s.k]; + if[~0=#d;lf' (!d) @ & |/'(.:d) in " "\:.z.l 4;]; / only load enabled features }[] ]; diff --git a/src/pykx/lib/4-1-libs/l64/.gitkeep b/src/pykx/lib/l64/.gitkeep similarity index 100% rename from src/pykx/lib/4-1-libs/l64/.gitkeep rename to src/pykx/lib/l64/.gitkeep diff --git a/src/pykx/lib/4-1-libs/l64arm/.gitkeep b/src/pykx/lib/l64arm/.gitkeep similarity index 100% rename from src/pykx/lib/4-1-libs/l64arm/.gitkeep rename to src/pykx/lib/l64arm/.gitkeep diff --git a/src/pykx/lib/4-1-libs/m64/.gitkeep b/src/pykx/lib/m64/.gitkeep similarity index 100% rename from src/pykx/lib/4-1-libs/m64/.gitkeep rename to src/pykx/lib/m64/.gitkeep diff --git a/src/pykx/lib/4-1-libs/m64arm/.gitkeep b/src/pykx/lib/m64arm/.gitkeep similarity index 100% rename from src/pykx/lib/4-1-libs/m64arm/.gitkeep rename to src/pykx/lib/m64arm/.gitkeep diff --git a/src/pykx/lib/q.k b/src/pykx/lib/q.k deleted file mode 100644 index 5855490..0000000 --- a/src/pykx/lib/q.k +++ /dev/null @@ -1,267 +0,0 @@ -\d .q -/each: +-*%&|^<>=$ <= >= <> @ ? in within bin div abs log exp sqrt sin cos tan f' f\: f/: -neg:-:;not:~:;null:^:;string:$:;reciprocal:%:;floor:_:;ceiling:-_-:;signum:{(x>0)-x<0} -mod:{x-y*x div y};xbar:{x*y div x:$[16h=abs[@x];"j"$x;x]};xlog:{log[y]%log x};and:&;or:|;each:{x'y};scan:{x\y};over:{x/y};prior:{x':y} -mmu:$;lsq:!;inv:!:;md5:-15!;ltime:%:;gtime:{t+x-%t:x+x-%x}; /xnull:{$[0>@y;(,y)@~x=y;x=y;y 0N;y]} - -/aggr: last sum prd min max avg wsum wavg f/ /beta:{cov[x;y]%var x} -count:#:;first:*:;svar:{(n*var x)%-1+n:(#x)-+/^x};sdev:{sqrt svar x};scov:{(n*cov[x;y])%-1+n:(#x)-+/^x+y};med:{avg x(@x;'`rank;1_x,,x 0N]}; -rank:{$[0h>@x;'`rank;<@x;'`rank;@x;'`rank;>x]} -asc:{$[99h=@x;(!x)[i]!`s#r i:@x;'`rank;`s#x@r:. x;0h>@x;'`rank;x@>x]} - -msum:{$[99h=@y;(!y)!.z.s[x;. y];y-(-x)_(0i*x#y),y:+\y]};mcount:{msum[x;~^y]};mavg:{msum[x;0.0^y]%mcount[x;y]};mdev:{sqrt mavg[x;y*y]-m*m:mavg[x;y:"f"$y]} -xrank:{$[0h>@y;'`rank;_y*x%#y:<@y;'`rank;y(!#y)-x]};rotate:{$[0h>@y;'`rank;98h<@y;'`type;#y;,/|(0;mod[x;#y])_y;y]};ema:{(*y)(1f-x)\x*y} - -/other: ~,#_ !. getenv exit -distinct:?:;group:=:;where:&:;flip:+:;type:@:;key:!:;til:{$[0>@x;!x;'`type]};value:get:.:;attr:-2!;cut:{$[0h>@x;x*!-_-(#y)%x;x]_y} -set:{$[@x;.[x;();:;y];-19!((,y),x)]};upsert:.[;();,;] / :: ,: files? -raze:,/;union:?,;inter:{x@&x in y};except:{x@&~x in y};cross:{n:#m:&(#x)##y;$[99h=@x;((!x)[m],'n#!y)!(. x)[m],'n#. y;((),x)[m],'n#y]} /extant:{x@&~^x} -sv:{x/:y};vs:{x\:y};sublist:{$[99h=@y;sublist[x;!y]!sublist[x;. y];~0h>@x;$[.Q.qp y;.Q.ind[y];y]i+!"j"$0|x[1]&(#y)-i:*x;abs[x]<#y;x#y;y]} - -/file&comm -read0:0::;read1:1::;hclose:>:;hdel:~:;hsym:"s"$-1!';hcount:-7!;peach:{x':y};system:."\\", - -/string: like ss -ltrim:{$[~t&(77h>t)|99ht)|99ht)|99ht)|99h@z;:[;z];z]]} - -/select insert update delete exec / fkeys[&keys] should be eponymous, e.g. order.customer.nation -/{keys|cols}`t `f's{xasc|xdesc}`t n!`t xcol(prename) xcols(prearrange) FT(xcol xasc xdesc) -view:{$`. .`\:x};tables:{."\\a ",$$[^x;`;x]};views:{."\\b ",$$[^x;`;x]} -cols:{$[.Q.qp x:.Q.v x;.Q.pf,!+x;98h=@x;!+x;11h=@!x;!x;!+0!x]} /cols:{!.Q.V x} -xcols:{(x,f@&~(f:cols y)in x)#y};keys:{$[98h=@x:.Q.v x;0#`;!+!x]};xkey:{(#x)!.[0!y;();xcols x]}; -xcol:{.Q.ft[{+$[99h=@x;@[!y;(!y)?!x;:;. x];x,(#x)_!y]!. y:+y}x]y};xasc:{$[$[#x;~`s=-2!(0!.Q.v y)x;0];.Q.ft[@[;*x;`s#]].Q.ord[<:;x]y;y]};xdesc:{$[#x;.Q.ord[>:;x]y;y]} -fkeys:{(&~^x)#x:.Q.fk'.Q.V x};meta:{([!c].Q.ty't;f:.Q.fk't;a:-2!'t:. c:.Q.V x)} - -/ R uj R(union join) R lj K(left(equi/asof)join) trade asof`sym`time!(`IBM;09:31:00.0) -lj:{$[$[99h=@y;(98h=@!y)&98h=@. y;()~y];.Q.ft[,\:[;y];x];'"type"]} /;la:{$[&/j:z>-1;x,'y z;+.[+ff[x]y;(!+y;j);:;.+y z j:&j]]}{la[x;. y](!y)?(!+!y)#x}[;y]]x} /lj:,\:;aj:{lj[y]`s#xkey[x]z};aj0:{lj[y]`s#(x#z)!z}; /;bn:{@[i;&0>i:x bin y;:;#x]} -ljf:{$[`s=-2!y;ajf[!+!y;x;0!y];.Q.ft[{$[&/j:(#y:. y)>i?:(!+i:!y)#x;.Q.fl[x]y i;+.[+x;(f;j);:;.+.Q.fl[((f:!+y)#x:.Q.ff[x]y)j]y i j:&j]]}[;y]]x]} -.Q.ajf0:{[f;g;x;y;z]x,:();z:0!z;d:$[g;x_z;z];g:(:;^)f;f:(,;^)f;$[&/j:-1i:(!y)?(!+!y)#x}[;y]]x} -ijf:{.Q.ft[{.Q.fl[x j]y i j:&(#y:. y)>i?:(!+i:!y)#x}[;y]]x} -pj:{.Q.ft[{x+0i^y(!+!y)#x}[;y]]x};asof:{f:!$[99h=@y;y;+y];(f_x)(f#x)bin y} -uj:{$[()~x;y;()~y;x;98h=@x;x,(!+x:.Q.ff[x;y])#.Q.ff[y;x];lj[(?(!x),!y)#x]y]} -ujf:{$[()~x;y;98h=@x;x,(!+x:.Q.ff[x;y])#.Q.ff[y;x];ljf[(?(!x),!y)#x]y]} - -/wj[-1000 2000+\:trade`time;`sym`time;trade;(quote;(max;`ask);(min;`bid))] (given `sym`time xasc quote) -ww:{[a;w;f;y;z]f,:();e:1_z;z:*z;y,'n#+(:/'f)!+{[e;d;a;b]e .'d@\:\:a+!b-a}[*:'e;z f:1_'e]/'$[n:#*w;+$[#g;(g#z)?g#y;0]|/:a+$[#g:-1_f;(f#z)bin@[f#y;*|f;:;]@;z[*f]bin]'w;,0 0]} -wj:{[w;f;y;z]ww[0 1;w;f;y;z]};wj1:{[w;f;y;z]ww[1;w-1 0;f;y;z]} - -fby:{$[(#x 1)=#y;@[(#y)#x[0]0#x 1;g;:;x[0]'x[1]g:.=y];'`length]};xgroup:{x,:();a:x#y:0!y;$[#x_:y;+:'x@=a;a!+f!(#f:!+x)#()]};ungroup:{$[#x:0!x;,/+:'x;x]} -ej:{x,:();y[&#:'i],'(x_z)(!0),/i:(=x#z:0!z)x#y:0!y} /{ungroup lj[z]xgroup[x]y} - -/`[:../]t[.{csv|txt}] -save:{$[1=#p:`\:*|`\:x:-1!x;set[x;. *p]; x 0:.h.tx[p 1]@.*p]}' -load:{$[1=#p:`\:*|`\:x:-1!x;set[*p;. x];set[*p].h.xt[p 1]@0:x]}' -rsave:{x:-1!x;.[`/:x,`;();:;.*|`\:x]}' -rload:{x:-1!x;.[*|`\:x;();:;. x]}' -dsave:{.[*x;1_x,y,`;:;@[;*!+a;`p#].Q.en[*x]a:. y];y}/: - -show:{1 .Q.s x;};csv:"," / ";" also \z 1 for "D"$"dd/mm/yyyy" - -parse:{$["\\"=*x;(system;1_x);-5!x]};eval:-6!;reval:-24! -\d .Q /def[`a`b`c!(0;0#0;`)]`b`c!(("23";"24");,"qwe") -k:4.0;K:2025.02.18;host:-12!;addr:-13!;gc:-20!;ts:{-34!(x;y)};gz:-35!;w:{`used`heap`peak`wmax`mmap`mphy`syms`symw!(."\\w"),."\\w 0"} / used: dpft en par chk ind fs fu fc -res:`abs`acos`asin`atan`avg`bin`binr`cor`cos`cov`delete`dev`div`do`enlist`exec`exit`exp`getenv`hopen`if`in`insert`last`like`log`max`min`prd`select`setenv`sin`sqrt`ss`sum`tan`update`var`wavg`while`within`wsum`xexp`from`by -addmonths:{("d"$m+y)+x-"d"$m:"m"$x} -Xf:{y 1:0xfe20,("x"$77+@x$()),13#0x00;(`$($y),"#")1:0x};Cf:Xf`char -f:{$[^y;"";y<0;"-",f[x;-y];y<1;1_f[x;10+y];9e15>j:"j"$y*prd x#10f;(x_j),".",(x:-x)#j:$j;$y]} -fmt:{$[x<#y:f[y;z];x#"*";(-x)$y]} -pykxld:{x:("#!"~2#*x)_x:-1!'x;+(1+*:'i;)@"\n"/:'x i:(&|1^\|0N 0 1@"/ "?*:'(v:x i),'"/")_i:&~|':(b?-1)#b:+\-/x~\:/:+,"/\\"} - -/DO NOT USE ROUTINES PAST HERE. SUBJECT TO CHANGE -ff:{$[&/(!+y)in f:!+x;x;x,'(f_y)(#x)#0N]} -fl:{$[98h=t:@x;+fl[+x;+y];99h=t;@[x,y;f;:;x[f]fl'y f@:&(f:!x)in!y];t&t<77h;x^y;@[y;i;:;x i:&(0<#:'x)&~#:'y]]} -opt:{[o]x::$[#i:&o like"-[^0-9]*";i[0]#o;o];((`$1_*:)'o)!1_'o:i_o};def:{x,((!y)#x){$[0h>@x;*:;::](@*x)$y}'y};ld:-39! -qt:{$[99h=@x;(98h=@!x)&98h=@. x;98h=@x]}; v:{$[-11h=@x;.$[":"=*t:$x;`$t,"/";x];x]}; -qp:{$[~98h=@x;0;@x:.+x;~":"=*$x;0]}; V:{$[qp x:v x;((,pf)!,. pf),+$[(~x in!d0[])&(x:.+x)in !*. .Q.vt;.Q.vp x;d0[]@x];0h>@*x:+0!x;x@'!x;x]} -ft:{$[{$[99h=@t:v x;98h=@. t;0]}y;[n:#+!y;n!x 0!y];x y]}; -ord:{if[~&/b:{(!#x)=x?x}c:.q.cols z;'"dup ",$c b?0b];if[~&/b:(y,:())in c;'y b?0b];ft[@[;!+t:0!v z;::]]z;ft[@[;!+t;@[;x@+y!t y]]]z};nv:{$["."~*$x;x;`$"..",$x]} - -tx:{$[(76ht;t;11h=t:@x:. nv@!x;t;98h=t;7h;1=#x:+!x;tx@*x;7h]};tt:0|tx'. V@ -fk:{$[(20h>t)|76h@. nv t:!x;`;t]};t:" bg xhijefcspmdznuvts";ty:{$[0h>x:tx x;.q.upper t@-x;t x]} -nct:{$[`~x;,/nct'.q.tables`.;([]n:x;c:.q.cols x;t:tt x)]} - -fu:{[f;x]$[0h>@x;f x;f[u](u:?x)?x]} /uniques -fc:{$[1@x;`${$[(*x)in n,"_ ";"a",x;x]}x@&(x:$x)in an;ft[{s[i]:`$($s i:&((s:id'!x:+x) in`i,res,!`.q)),'"1";+({unm[x]}/s)!. x}]x]} -j10:64/:b6?;x10:b6@0x40\: /base64 J from char10 -j12:36/:nA?;x12:nA@0x24\: /base36 J from char12(cusip) -btoa:-32!;sha1:-33!;prf0:+`name`file`line`col`text`pos!*-37! -objp:{x like"[mgs][s3]://*"}; -lo:{if[$[1>@d:!f:-1!x;1;`.d~*d];:.[$[qt d;*|`\:f;99=@d;`.;'`type];();:;d:. f]];d@:&~d like"*$";p:(d=`par.txt)|d like"[0-9]*"; - if[y;if[objp x;'"no cd to object storage"];."\\cd ",$x;f:`:.];{.q.set[*|`\:x]$[0h>@!x;. x;x`]}'`/:'f,'d@&~(d=`html)|(d like"*#")|p|s:d like"*.?";if[|/p;L[d@&p;f;~y|objp x]];if[z&~`.=x;(."\\l ",$:)'`/:'f,'d@&s&~p];} -l:lo[;1;1] - -sw:{.q.sublist[_.5**|."\\C"]x};tab:{" "/:'(x*|/#:''y)$/:y};t0:{$[x>#y;((#*y)#'" -"),y;y]} -s1:{-3!x} /{$[0>t:@x;$x;99h76h;s1'x;t;$x;|/(97ht:?@:'x;s1'x;(1<#?#:'x)|$[1=#t;(*t)in 1 4 10h;0];s1'x;tab[1]@s2'sw'x]} -S:{x[1]{$[x<#y;((x-2)#y),"..";y]}'$[t&77h>t:@z;,s1 z;99h@y;y:,y];p:{$[#y;y;x]}/'getenv@+`$_:\("HTTP";"NO"),\:"_PROXY";u:hap@x;t:~(~#*p)||/(*":"\:u 2)like/:{(("."=*x)#"*"),x}'","\:p 1;a:$[t;p:hap@*p;u]1;r:(`$":",,/($[t;p;u]0 2))($*y)," ",$[t;x;u 3]," HTTP/1.1",s,(s/:(~![-35;::])_("Accept-Encoding: gzip";"Connection: close";"Host: ",u 2),((0<#a)#,$[t;"Proxy-";""],"Authorization: Basic ",btoa a),$[#z;("Content-type: ",z 0;"Content-length: ",$#z 1);()],1_y),(d:s,s:"\r\n"),$[#z;z 1;""];(0;n)_$[#ss[(n:4+*r ss d)#r;"content-encoding: gzip\r\n"];-35!r;r]} -hg:{hmb[x;`GET;()]1};hp:{hmb[x;`POST;(y;z)]1} - -a0:(#:;*:;last;sum;prd;min;max;.q.all;.q.any;?:),a1:(avg;wsum;wavg;var;dev;cov;cor),`.q `svar`sdev`scov`med -IN:{$[99h<@x;x in y;0]};qa:{$[qb x;0;IN[*x;a0];1;|/qa'1_x]};qb:{(2>#x)|(@x)&~11=@x} - -/ CAN EXIT HERE FOR SMALL Q -/ pt(tables) pf(date/month/year/int) pd(dirs) pv(values) pn(count) pt::0#pf::` -vt:(,`)!,()!(); -bv:{g:$[(::)~x;max;min];x:.Q.d;d:{`/:'x,'d@&(d:!x)like"[0-9]*"}'P:$[`par.txt in!x;jp[x]'`$0:`/:x,`par.txt;,x]; - t:?,/!:'.Q.vt:{(&#:'x)(=,/. x)}'{({("DMJJ"`date`month`year`int?.Q.pf)$$last@x:`\:x}'x)!!:'x}'d; - d:{`/:'x[(. y)[;0]],'(`$$(. y)[;1]),'!y}[P]@{i:y@&:x=y x:@[x;&x~\:();:;*0#`. pf];(i;x i)}[;g]'+:t#/:g''.Q.vt:t#/:.Q.vt;.Q.vt:P!.q.except[. .Q.pf]''.Q.vt; - .Q.vp:t!{(+(,.Q.pf)!,0#. .Q.pf),'+(-2!'.+x)#'+|0#x:?[x;();0b;()]}'d;.Q.pt,:{.[x;();:;+.q.except[!+.Q.vp x;.Q.pf]!x];x}'.q.except[t;.Q.pt];} - -pt:pm:();MAP:{{$[0>@a:.+0!. x;.q.set[x]@.`$-1_$a;]}'a@&~(a:."\\a")in pt;pm::();if[#pt;pm::pt!{(`u#pd,'pv)!p2[(x;();0b;())]'[pd;pv]}'pt]} -pfc:{$[$[#pm;|/{|/0b,{f:{pf~*`\:x};|/$[-11h=t:@x;f x;(t&~11h~t)|2>#:x;0b;11h=t;f'x;.z.s'x]}'$[99h=@x;.:x;x]}'x 1 2 3;0];+((,pf)!,z),+y;y]} -dd:{`/:x,`$$y};d0:{dd[*|pd;*|pv]};p1:{$[#pm;pm[x](y;z);z in vt[y;x];vp x;+(!+. x)!`/:dd[y;z],x]};p2:{0!(?).@[x;0;pfc[x;;z]p1[;y;z]@]} - -p:{$[~#D;p2[x;d]':y;(,/p2[x]'/':P[i](;)'y)@<,/y@:i:&0<#:'y:D{x@&x in y}\:y]} -view:{pd::PD x:$[(::)~x;x;$[#x:&PV in x;x;'"invalid partition filter"]];u~:?u::..[pf;();:;pv::PV x];.[;();:;]'[pt;{+(x . y,`.d)!y}[x]':pt::!x:d0[]];pn::pt!(#pt)#()} - -jp:{$[$["w"~*$.z.o;u[$[(_u:$y)like"[a-z]:*";2;0]]in"\\/";("/"=*$y)|objp y];-1!y;`/:x,y]};rp:-500! -L:{D::();f:{!:'?{-1!`$("/"/:3#"/"\:x),"/_"}'u@&objp'u:1_'$x;x};f@,d::$[z;rp y;y];if[x~,`par.txt;if[~#x:,/D::{x@&~(x:!x)like"*$"}'P::f@jp[d]'`$0:`/:d,*x;'empty]];if[^*PV::x@:."\\p")|."\\_";cn'.:'pt];} -/L:{P::,`:.;D::,x;pf::`date;pt::!P[0]@**D;T::P!P{z!{x!(y . ,[;`]z,)'x}[x;y]'z}[pt]'D} - -cn:{$[#n:pn x:.+x;n;pn[x]:(#p1 .)':+(x;pd;pv)]};pcnt:{+/cn x};dt:{cn[y]@&pv in x} -ind:{,/i[j]{fp[pf;p]p1[x;pd y;p:pv y]z}[.+x]'(j:&~=':i)_y-n i:(n:+\0,cn x)bin y} -fp:{+((,*x)!,(#z)#$[-7h=@y;y;(*|x)$y]),+z} -foo:{[t;c;b;a;v;d]if[v;g:*|`\:b f:*!b;b:1_b];,/$[v|~#a;d fp[$[v;f,g;pf]]';::]p[(.+t;c;b;a)]d} - -/ select{u's|a's[by[date,]u's]}from t where[date..],[sym{=|in}..],.. -a2:({(%;(sum;("f"$;x));(sum;(~^:;x)))};{(sum;(*;("f"$;x);y))};{(%;(wsum;x;y);(sum;(*;x;(~^:;y))))};{(cov;x;x)};{(sqrt;(var;x))} - {(-;(avg;(*;("f"$;x);y));(*;(avg;x);(avg;y)))};{(%;(cov;x;y);(*;(dev;x);(dev;y)))};{(.q.scov;x;x)};{(sqrt;(.q.svar;x))};{(*;(%;(#:;`i);(+;-1;(#:;`i)));(cov;x;y))};{'`part}) - -/ x0 translate;x1 aggrs;x2 translate ?terminal (subselect/exec within partition) unknown assumed uniform? -qd:{$[(#:)~*x;(?:)~*x 1;0]};xy:{`$$*&x~/:y}; x1:{$[qb x;();IN[*x;a0];$[qd x;1_x;,x];,/x1'1_x]} -x0:{$[qb x;x;IN[*x;a1];x0 a2[a1?*x]. 1_x;x0'x]};x2:{$[qb x; x;IN[*x;a0];$[qd x;(#:;(?:;(,/;xy[x 1]y)));[y:xy[x]y;$[(?:)~*x;(?:;(,/;y));(#:)~*x;(sum;y);(*x;y)]]];@[x;&~(::)~/:x;x2[;y]]]} -ua:{((`$$!#u)!u;x2[;u:?,/x1'x]'x:x0'x)};q0:{$[~qb x;,/q0'x;-11h=@x;*`\:x;()]};qe:{$[#x;99h=@x;1]} -ps:{[t;c;b;a]if[-11h=@t;t:. t];if[~qe[a]&qe[b]|-1h=@b;'`nyi];d:pv;v:$[q:0>@b;0;~#b;0;-11h=@v:*. b;pf~*`\:v;0] - if[$[~#c;0;@*c;0;-11h=@x:c[0]1;pf~*`\:x;0];d@:&-6!*c;c:1_c] - if[$[#c;0;(g:(. a)~,pf)|(. a)~,(#:;`i)];f:!a;j:dt[d]t;if[q;:+f!,$[g;?d@&0(#.:)'t:.q.tables`.;if[h:@[hopen;h;0];h"\\l .";>h]} - -/loop through text /lc:{+/{+/0xa=1:(x;y*z;y)}[x;m]'!-_-(-7!x)%m:1048576} /line count of big file -fsn:{[f;s;n]>[-7!s]{[f;s;x;n]i:(#r)^1+last@&"\n"=r:1:(s;x;n);f@`\:i#r;x+i}[f;s;;n]/0};fs:fsn[;;131000] -fpn:{{r:.Q.trp[y;h;{>x;'(y;z)}h:hopen":fifo://",1_$x];>h;r}[y - {[f;h;n]b:"x"$();while[#s:1:h;if[~n>#b,:s;v:`\:b,0x0a;if[1<#v;f@-1_v];b:"x"$last v]];if[#b;f@`\:b]}[x;;z]]};fps:fpn[;;131000] -/e.g. p:` sv(d:`:dir;`2007.02.12;`t);@[;`sym;`p#]p{@[x;`;,;`sym xasc .Q.en[d]get y]}/rdb's -dsftg:{[dpt;xom;f;tw;g]d:*dpt;dpt:par . dpt;x:*xom;if[~0m:_a:(m-:o:xom 1)%b:+/tw 10=@*tw;'`length];0N!($dpt),": ",$m&:M;i::0;n::1000000&_1e8%b - do[-_-m%n;@[dpt;`;$[i;,;:];en[d]@+g f!tw 1:(x;o+b*i;b*n&:m-i)];0N!i+:n];dpt};M:0W - -/ fill in empty partitions -chk:{if[x~(::);'"missing dir argument"];f:{`/:'x,'d@&(d:!x)like"[0-9]*"};d@:>.:'$last'`\:'d:$[`par.txt in!x;,/f'-1!'`$0:`/:x,`par.txt;f x] - {[e;u;d]u[i]{.[x;(y;`);:;?[z;();0b;()]]}[d]'e i:&~u in!d}[d[(+u in/:t)?\:1b](0#.)'u,'`;u:?,/t:!:'d]'d} -Ll:-100!;Lp:-101!;Lx:-102!;Lu:-103!;Ls:-104!;fqk:{("<"~*x)|x~*-3#.{}};fql:{x[1;1]~""};btx:{r:,Lp x;while[@x:Lu(0b;x);r,:(~fql v)#,v:Lp x;if[(fqk v[1]1)&".Q.dbg"~*v 1;f:(Lx(x;"k";,"v"))1]];$[~#f;r;@[r;r?f;,;,">>"]]};bt:{1'pl'(2*4<|/#:'v)_v:1_btx Ll`;};sbt:{,/pl'x};trp:{-105!(x;,y;z)} -dr:{0:0};dw:{@[2@;x;::];};pl0:{.[{[f;v;x;y;z]g:{$[x>0;x&#y;x|-#y]#y};m:1;n:0|(b:+\0,1+#:'s:`\:s)bin y:(0|y)+2*"k)"~2#s:x 3;sf:$[#x 1;x[1],$[0>x 2;"";":",($n+x 2)],": ";""];sn:{$[(#x)&@x;((2*".."~2#x)_x),":";()]}@*x;h:($[4<#v;*|v;" "],5$"[",($z),"]") - if[("locked"~x 3)|$[f;0b;fqk x 1];:h,"(",($[#sn;-1_sn;-2_sf]),")\n\n"];h,:sf,sn;h,`/:((n>m)#,""),g[-m+1;(n+1)#s],(,(" \t"(((#h)*n<1)#0),(y-b n)#s[n]="\t"),"^"),g[m](n+1)_s}[x;y];3#1_y;{dw"pl0\n";}]};pl:pl0 0 -jl8:{$[$[#x;~x[3]in y[;3];0b];,x;()],y};srr:{@[{if["abort"~*x;:""];"'",$[.z.q;($.z.Z)," ";""],(x[0],"\n"),/pl'jl8[x 3;x 2]};x;"srr\n"]};prr:{dw srr x}; -lu:{f:{$[@a:Lu(x;*y);(a;Lp a);()]}x;r:{$[#x;fql x 1;0b]}f/f y;$[#r;$[.Q.dbg~**r 1;y;r];y]} -DL:0;dbg:{pm:{dw" q"[x],$[1<#d:$."\\d";d;""]," )"[x],(1|DL)#")"};dq:{v:$[#*y;(~"k)"~2#y[1;3];*(.**y)3);1<#*y 1;@[*y 1;0;"q"=];(1b;`)];x v 1;*v}ds:{."\\d ",((~"."=*x)#"."),x:$x};de:@[{DL-:1;x y;{."\\x .z.e",x}'"xy"}ds;;::] - @[{DL+:1;.z.ex:*x 1;.z.ey:x[1]1};x;::];b:$[#*v:Lp y;0xff~(*. bf:**v)bi:v[0;1];0b];if[b;dw "#",($bi),"\n"];w:(y0:y;v:Lp y);if[e:@*x;x[2;;3]+:1+v 3;prr x];d0:."\\d" - y:*w:{$[~fqk x[1;1]1;x;lu[0b]x]}/w;q:dq v:w 1;dw pl v;while[*(~(,"\\")~l:dr[];pm[q]);$[l~"\\ ";q:~q;l in+,".`&";[w:$["&"=*l;;lu["."=*l]](y;v);y:*w;v:w 1;if[y~y0;prr x];dw pl0[1]v;q:dq v] - (~e)&l in+,":>";:*($[b;bc[bf;bi];]0 0N(":"=*l);de d0);"'"=*l;:*((1_l;());de d0);c&~#*r:Lx(y;"kq"q;(c:":"=*l)_l);:*(r;de d0);$[#*r;::;dw $[q;.Q.s;{$[(::)~x;"";`/:,-3!x]}]r 1]]];de d0;("abort";())} -err:{(*x;jl8[x 3;$[@y;btx y;()],x 2])}; BP:(,"")!,();bp:{*|(Ls(x;(),y;(#y)#z);(*. x)y)};bs:{BP[$x],:y!bp[x;y,:();0xff];};bu:{bp[x;y;BP[$x;y,:()]]} -bd:{bu[x;y];@[`BP;$x;_;y]};bc:{bu[x;y];dbg::{[d;f;i;r;u;v]dbg::d;bs[f;i];$[#*u;d[u;v];r]}[dbg;x;y;z];-1} - -\d .h / c -htc:{,/("<";x;y;"")};hta:{,/("<";$x;,/" ",'($!y),'"=",'{$[10h=@x;"\"",x,"\"";$x]}'. y;">")};htac:{,/(hta[x]y;z;"")} -ha:{htac[`a;(,`href)!,x]y};hb:{htac[`a;`target`href!`v,,x]y};pre:{htc[`pre]@`/:x};xmp:{htc[`xmp]@`/:x} -d:" ";cd:{.q.csv 0:$[.Q.qt x;![x;();0b;(!t)[c]!,:'.q.sv[d]@/:'$v c:&(~l=-10h)&0>l:.Q.tx'v:. t:+0!x];x]};td:{"\t"0:x};hc:{"<"/:"<"\:x};xs:.q.ssr/[;"&<>";("&";"<";">")] -xd:{g:{(#*y)#'(,,"<",x),y,,,""};(,""),(,/'+g[`r]@,/(!x)g'{,xs'$[11h=@x;$x;t&77h>t:@x;$x;x]}'x:+0!x),,""} - -ex:("";);iso8601:{$[^x;"";@[$"p"$x;4 7 10;:;"--T"]]} -eb:{htac[`Workbook;(`$"xmlns",/:$``:ss`:o`:x`:html)!("urn:schemas-microsoft-com:office:",/:$(2#`spreadsheet),`office`excel),,"http://www.w3.org/TR/REC-html40"]x} -es:{htac[`Worksheet;(,`ss:Name)!,$x]htc[`Table]@,/(htc[`Row]@,/ec')'(,!+y),+.+y:0!y};ed:{ex eb es[`Sheet1]x};edsn:{ex eb@"\r\n"/:(!x)es'. x} -ec:{htc[`Cell]htac[`Data;(,`ss:Type)!,$`String`Number`String`DateTime`DateTime`String i](xs;$:;xs@$:;iso8601;iso8601 1899.12.31+"n"$;xs@$:)[i:-10 1 10 12 16 20h bin -@x]x} - -\d .j /[]{} Cbg*xhijefcspmdznuvt -e:{"c"$$[x<128;x;0b/:'@[;0;(1b,i#1b),]10b,/:((5 10;0 4 10)i:x>2047h)_0b\:"h"$x]}@256/: -q:"\"";s:{q,x,q};es:{s@,/{$[x in r:"\t\n\r\"\\";"\\","tnr\"\\"r?x;x]}'x};J:(($`0`1)!$`false`true;s;{$[#x;x;"null"]};es;{s@[x;&"."=8#x;:;"-"]};s)1 2 5 11 12 16h bin -/j:{$[10=abs t:@x;es x;(::)~x;"null";99=t;"{",(","/:({$[q~*x;x;s x]}'j'!x),'":",'j'. x),"}";-1*x;"F"$x;"n"=*x;0n;"t"=*x]} -/k:{c x@&~v[x]&x in" \t\n\r"};v:{~0(0 1 0;1 0 2;1 1 1)\x*3>x:" \"\\"?x};u:{?[v x;x;" "]};d:{$[1" - -c0:`024C7E;c1:`958600;logo:"kx.com" /logo:,/(c0;c1;c0){htac[`font;(,`color)!,x]y}'("[kx";"systems";"]") /808000 -logo:,/(c0;c1){htac[`font;(,`color)!,x]htc[`b]y}'("KX";".com") -sa:"a{text-decoration:none}",/`link`visited`active{"a:",($x),"{color:",y,"}"}'$(c0;c0;c1) -html:{htc[`html]htc[`head;htc[`style]sa,sb],htc[`body]x};sb:"body{font:10pt verdana;text-align:justify}" -fram:{htc[`html]htc[`head]htc[`title;x],htac[`frameset;(,`cols)!,($116|316&43+(7+"?"~*z)*|/#:'y),",*"]@,/hta[`frame]'((,`src)!,*z;`name`src!`v,,*|z)} -jx:{[j;x]x:val x;$[$[.Q.qt[x];(N:(*."\\C")-4)";hr:{(#x)#"-"};nbr:htc[`nobr];code:{" "/:@[x;1+2*!_.5*#x:"\t"\:x;{htc[`code]nbr hc x}]} -http:{$[#i:ss[x]"http://";(i[0]#x),/{ha[x;x:i#x],(i:&/x?") ")_x}'i_x;x]};text:{`/:{htc[`p]http code x}'x@&0<#:'x} -data:{if[100<#*x:.Q.tab[1]n#'x,\:(n:|/#:'x:"\t"\:'x:1_'x)#,"";0N!#*x;0N!'x];xmp$["- "~2#x 1;@[x;1;hr];x]} -ht:{[t]x:0:`$"src/",(T:t:$t),".txt";if[~"."=**x;T:*x;x:1_x];h:.q.trim(n@:i)_'x i:&0#x;x,1;(y#x),1+x y-:1]}\n),'" ",/:h - x:`/:("Copyright © ",logo;htc[`h5]T),((#x)#(text;data))@'x:(&~=':~(0<#:'x)&" "=*:'x)_x - c:h{hb[x,"#",y;z],br}[a:t,".htm"]'H;c:nbr@`/:$[&/n=:1;c;1_,/(,br),/:(&n)_c] - (`$a)1:fram[T;H]b:"a/",/:("_",a;a);(`$b)1:'html'(($.z.D),htc[`h5;"Contents"],c;x);} - -\d .o /odbc timedec/cs could be 3(23,12) -ex:{$[.Q.qt x:."s)",x;0!x;x]} / char(1+) binary(display2*) -1 longvarchar -4 longvarbinary sql_no_total(-4)SAS/4096 -T:`text`bit```tinyint`smallint`int`bigint`real`float`char`varchar``date`date`datetime``time`time`time -T0: -1 -7 0 0 -6 5 4 -5 7 8 1 12 0 9 9 11 0 10 10 10h -B0:4096 1 0 0 1 2 4 8 4 8 1 255 0 6 6 16 0 6 6 6 -C0:4096 1 0 0 3 5 10 19 7 15 1 255 0 10 10 23 0 8 8 8 -PS: 1 2 0 0 2 2 2 2 2 2 3 3 0 2 2 2 0 2 2 2h /char basic both -t:0 1 4 5 6 7 8 9 10 11 14 15 19;Columns:{$[#x;Cols`$x;,/Cols'.q.tables`.]} -q)TI:`DATA_TYPE xasc([]TYPE_NAME:T t;DATA_TYPE:T0 t;COLUMN_SIZE:C0 t;LITERAL_PREFIX:`;LITERAL_SUFFIX:`;CREATE_PARAMS:`;NULLABLE:1h;CASE_SENSITIVE:1h;SEARCHABLE:PS t;UNSIGNED_ATTRIBUTE:0h;FIXED_PREC_SCALE:0h;AUTO_INCREMENT:0h;LOCAL_TYPE_NAME:`;MINIMUM_SCALE:0h;MAXIMUM_SCALE:0h) -q)TypeInfo:{$[count x;select from TI where DATA_TYPE="H"$x;TI]} /msqry32 uses special e.g. select p.i from p p -q)Special:{([]SCOPE:2h;COLUMN_NAME:(0&0 similar position substring trim {lower|upper} diff --git a/src/pykx/lib/4-1-libs/w64/.gitkeep b/src/pykx/lib/w64/.gitkeep similarity index 100% rename from src/pykx/lib/4-1-libs/w64/.gitkeep rename to src/pykx/lib/w64/.gitkeep diff --git a/src/pykx/license.py b/src/pykx/license.py index 9a532d1..be26faa 100644 --- a/src/pykx/license.py +++ b/src/pykx/license.py @@ -31,7 +31,7 @@ def check(license: str, ) -> bool: """ Validate the license key information you provided matches the license - saved to disk which is read by PyKX + saved to disk which is read by KDB-X Python Parameters: license: If using "FILE" format this is the location of the file being used for comparison. @@ -39,8 +39,7 @@ def check(license: str, format: Is the license check being completed using a downloaded file or base64 encoded string. Accepted inputs are "FILE"(default) or "STRING". license_type: The license file type/name which is to be checked, by default this - is 'kc.lic' which is the version provided with personal and commercial - evaluation licenses but can be changed to 'k4.lic' or 'kx.lic' if appropriate + is 'kc.lic' but can be changed to 'k4.lic' or 'kx.lic' if appropriate Returns: A boolean indicating if the license is correct or not and a printed message describing @@ -166,8 +165,7 @@ def install(license: str, format: Is the license check being completed using a downloaded file or base64 encoded string. Accepted inputs are "FILE"(default) or "STRING". license_type: The license file type/name which is to be checked, by default this - is 'kc.lic' which is the version provided with personal and commercial - evaluation licenses but can be changed to 'k4.lic' or 'kx.lic' if appropriate + is 'kc.lic' but can be changed to 'k4.lic' or 'kx.lic' if appropriate force: Enforce overwrite without opt-in message for overwrite Returns: @@ -222,5 +220,5 @@ def install(license: str, with open(qlic/license_type, 'wb') as binary_file: binary_file.write(lic) - print("PyKX license successfully installed!") + print("KDB-X Python license successfully installed!") return True diff --git a/src/pykx/module.py b/src/pykx/module.py new file mode 100644 index 0000000..03c4726 --- /dev/null +++ b/src/pykx/module.py @@ -0,0 +1,339 @@ +"""Loading KDB-X modules. + +When `pykx.use` is called KDB-X will attempt to load a module with the given name. +""" +from typing import Optional + +from .wrappers import CharVector, Dictionary, Identity, SymbolAtom +from .exceptions import QError + +__all__ = [ + 'QModule', + 'QNamespaceProxy', + 'SearchPath', + 'use' +] + + +def __dir__(): + return __all__ + + +def _init(_q): + global q + q = _q + + +class QModule: + """ + A loaded KDB-X q Module. + + Example loading the pq module and the pq.t submodule within KDB-X Python. + ```python + >>> import pykx as kx + >>> pq = kx.use('kx.pq') + >>> pq + pykx.QModule(pykx.q(' + pq| pykx.Lambda + op| pykx.Lambda + rd| pykx.Lambda + ')) + >>> pq.t = kx.use('kx.pq.t') + >>> pq + pykx.QModule(pykx.q(' + pq| pykx.Lambda + op| pykx.Lambda + rd| pykx.Lambda + t | pykx.QModule(pykx.q(' + mkT| pykx.Lambda + mkP| pykx.Lambda + tt | pykx.Lambda + mt | pykx.Lambda + fv | pykx.Lambda + ')) + ')) + ``` + """ + def __init__(self, name, qns, qkeys, qdict): + object.__setattr__(self, '_name', name) + object.__setattr__(self, '_qns', qns) + object.__setattr__(self, '_qkeys', qkeys) + object.__setattr__(self, '_qdict', qdict) + object.__setattr__(self, '_submods', {}) + + def __dir__(self): + return list(self._qkeys) + + def __getattribute__(self, name): + if (name in object.__getattribute__(self, "_qkeys") + and name not in object.__getattribute__(self, '_submods').keys()): + ns = object.__getattribute__(self, "_qns") + qd = object.__getattribute__(self, "_qdict") + return _wrap_if_namespace(_fetch(ns, name, fallback=qd), ns, name) + return super().__getattribute__(name) + + def __setattr__(self, name, value): + if (name in object.__getattribute__(self, "_qkeys") + and name not in object.__getattribute__(self, '_submods').keys()): + ns = object.__getattribute__(self, "_qns") + q('set', ns + '.' + name, value) + else: + if isinstance(value, QModule): + sub_m = object.__getattribute__(self, '_submods') + sub_m[name] = value + object.__setattr__(self, '_submods', sub_m) + keys = [k.py() for k in object.__getattribute__(self, '_qdict').keys() if k.py()] + keys.extend(object.__getattribute__(self, '_submods').keys()) + object.__setattr__( + self, + '_qkeys', + frozenset( + keys + ) + ) + super().__setattr__(name, value) + + def __repr__(self): + res = 'pykx.QModule(pykx.q(\'\n' + qd = object.__getattribute__(self, "_qdict") + all_keys = [x.py() for x in list(qd.keys())]\ + + list(object.__getattribute__(self, '_submods').keys()) + if all_keys == []: + return f'pykx.QModule(pykx.q(\'{qd}\'))' + max_len = max([len(x) for x in all_keys]) + + for k in all_keys: + if k in qd.keys(): + res += k + (' ' * (max_len - len(k))) + '| ' +\ + 'pykx.' + str(type(qd[k])).split("'")[-2].split('.')[-1] + '\n' + else: + res += k + (' ' * (max_len - len(k))) + '| pykx.QModule(pykx.q(\'\n' +\ + '\n'.join([ + ' ' * (max_len + 2) + x for x in + object.__getattribute__(self, '_submods')[k].__repr__().split('\n')[1:-1] + ]) + '\n' + ' ' * (max_len + 2) + '\'))\n' + return res + '\'))' + + +def _is_q_namespace(value: Dictionary) -> bool: + # Q namespaces, when fetched as dicts, have an empty-symbol key mapping to ::. + try: + return any(k.py() == "" for k in value.keys()) + except Exception: + return False + + +def _fetch(qns: str, key: str, fallback: Optional[Dictionary] = None): + # Fetch a key from a q namespace. + + # @[ns; key] returns :: when key is a sub-namespace (not a plain value). + # In that case we fall back to the dict returned by `use`, which already + # contains the full sub-namespace contents. + result = q('@', qns, key) + if isinstance(result, Identity) and fallback is not None: + result = fallback[SymbolAtom(key)] + return result + + +def _wrap_if_namespace(result, qns: str, key: str): + # Wrap q namespace dicts in a QNamespaceProxy; leave plain data dicts alone. + if isinstance(result, Dictionary) and _is_q_namespace(result): + return QNamespaceProxy(f"{qns}.{key}", result) + return result + + +class QNamespaceProxy: + # Proxy for a nested q namespace returned as a dictionary attribute. + + def __init__(self, qns: str, value: Dictionary): + object.__setattr__(self, "_qns", qns) + object.__setattr__(self, "_value", value) + + def _qkeys(self): + val = object.__getattribute__(self, "_value") + return frozenset(k.py() for k in val.keys() if k.py()) + + def __dir__(self): + return list(self._qkeys()) + + def __getattribute__(self, name): + if name.startswith("_"): + return object.__getattribute__(self, name) + val = object.__getattribute__(self, "_value") + if name in (k.py() for k in val.keys() if k.py()): + qns = object.__getattribute__(self, "_qns") + return _wrap_if_namespace(val[SymbolAtom(name)], qns, name) + return object.__getattribute__(self, name) + + def __setattr__(self, name, value): + if not name.startswith("_") and name in self._qkeys(): + qns = object.__getattribute__(self, "_qns") + q('set', qns + '.' + name, value) + else: + object.__setattr__(self, name, value) + + def __repr__(self): + qns = object.__getattribute__(self, "_qns") + val = object.__getattribute__(self, "_value") + return f"\n{val}" + + +def _resolve_qns(module_name) -> str: + # Look up the q namespace for this module in .Q.m.M after loading. + + module_map = q(".Q.m.M").py() + for (path_bytes, sub_name), info in module_map.items(): + end_path = path_bytes.decode('utf-8').split('/')[-1].split('\\')[-1] + mod_name = module_name.split('.')[-1] + if end_path == mod_name or end_path == mod_name + '.k'\ + or sub_name == module_name.split(':')[-1]: + ns = info.get('m', '') + if ns: + return ns + return '' + + +def use(module_name: str, export_argument: object = None) -> QModule: + """ + Function for loading KDB-X modules. + + Parameters: + module_name: The name of the KDB-X module to load as a string. + export_argument: Optional argument to pass to the module export function. Default None. + + Returns: + A loaded QModule instance. + + Examples: + + Example loading the pq module and the pq.t submodule within KDB-X Python. + ```python + >>> import pykx as kx + >>> from pathlib import Path + >>> pq = kx.module.use('kx.pq') + >>> pq.t = kx.module.use('kx.pq.t') + >>> tab = pq.pq(Path('../types.parquet')) + >>> tab + pykx.VirtualTable(pykx.q('`T!`f`m`t!(k){[f;t;c;b;a;v]g:$[s:-1h=@b;0;#b];(bf;b1):$[g;df[t;0,0b;b];(::;()..')) + >>> tab.select() + pykx.Table(pykx.q(' + col0 col1 col2 + --------------- + 1 1 "asd" + 1 2 "bsd" + 0 3 "csd" + 1 4 "asd" + 0 5 "bsd" + 0 6 "csd" + 1 7 "asd" + 0 8 "bsd" + 0 9 "csd" + ')) + ``` + """ + mod = q._get_module(module_name) + if mod is not None: + return mod + try: + qdict = q.use(module_name if export_argument is None else [module_name, export_argument]) + qns = _resolve_qns(module_name) + except QError as e: + raise ImportError(f"Failed to load {module_name}") from e + + qkeys = frozenset(k.py() for k in qdict.keys() if k.py()) + module = QModule(module_name, qns, qkeys, qdict) + # Seed __dict__ with the exported names so IPython/jedi can discover + # them via static inspection. __getattribute__ above ensures live q + # values are always fetched on actual attribute access. + module.__dict__.update(dict.fromkeys(qkeys)) + + if q._get_module('.'.join(module_name.split('.')[:-1])) is not None: + q._get_module('.'.join(module_name.split('.')[:-1])).__setattr__( + module_name.split('.')[-1], + module + ) + q._add_module(module_name, module) + return module + + +def _SP(paths: Optional[list[str]] = None): + + if paths is None: + return [b.decode() for b in q('.Q.m.SP').py()] + else: + q('set', '.Q.m.SP', [CharVector(s.encode()) for s in paths]) + + +def SearchPath( + entry: Optional[str] = None, + *, + add: bool = False, + remove: bool = False, + prepend: bool = False, + strict: bool = False, + allow_duplicates: bool = False, +) -> Optional[list[str]]: + """ + Manage the module SearchPath. + + Parameters: + entry: The path string to add or remove. If omitted, the current + SearchPath is returned. + add: If True, add the entry to the SearchPath. Default False. + remove: If True, remove all occurrences of the entry from the SearchPath. Default False. + prepend: If True, insert the entry at the front rather than the end. Default False. + strict: If True, raise a ValueError when attempting to remove an + entry that is not present. Default False. + allow_duplicates: If True, allow the same entry to appear more than + once in the SearchPath. Default False. + + Examples: + + Retrieve the current SearchPath: + + ```python + >>> kx.module.SearchPath() + [] + ``` + + Append and prepend entries: + + ```python + >>> kx.module.SearchPath("/home/user/.kx/mod", add=True) + >>> kx.module.SearchPath("/home/user/mod", add=True) + >>> kx.module.SearchPath("/other/mod", add=True, prepend=True) + >>> kx.module.SearchPath() + ['/other/mod', '/home/user/.kx/mod', '/home/user/mod'] + ``` + + Remove an entry: + + ```python + >>> kx.module.SearchPath("/other/mod", remove=True) + >>> kx.module.SearchPath() + ['/home/user/.kx/mod', '/home/user/mod'] + ``` + """ + sp = _SP() + if entry is None: + return sp + if add: + if not allow_duplicates and entry in sp: + raise ValueError( + f"{entry!r} already on SearchPath. Use allow_duplicates=True to override") + if prepend: + sp.insert(0, entry) + else: + sp.append(entry) + _SP(sp) + return None + if remove: + if entry not in sp: + if strict: + raise ValueError(f"{entry!r} not found in SearchPath.") + return None + sp = [p for p in sp if p != entry] + _SP(sp) + return None + + raise ValueError("Specify add=True or remove=True, or call with no arguments to retrieve.") diff --git a/src/pykx/nbextension.py b/src/pykx/nbextension.py index 0955a43..28b2ec0 100644 --- a/src/pykx/nbextension.py +++ b/src/pykx/nbextension.py @@ -125,11 +125,11 @@ def q(instructions, code): # noqa except kx.QError as err: if '.Q.pykxld' in str(err): # .Q.pykxld is not defined on the server so we pass it as inline code - with open(kx.config.pykx_lib_dir/'q.k', 'r') as f: + with open(kx.config.pykx_dir/'pykx.q', 'r') as f: lines = f.readlines() for line in lines: if 'pykxld:' in line: - ld = _q("k)"+line[7:-1], skip_debug=True) + ld = _q("k)"+line[12:], skip_debug=True) break else: raise err diff --git a/src/pykx/pandas_api/pandas_conversions.py b/src/pykx/pandas_api/pandas_conversions.py index e26b6eb..3fa7a37 100644 --- a/src/pykx/pandas_api/pandas_conversions.py +++ b/src/pykx/pandas_api/pandas_conversions.py @@ -58,7 +58,7 @@ 109: 'EachPrior', 110: 'EachRight', 111: 'EachLeft', - 112: 'Foreign'} + 112: '_k_foreign_type'} kx_type_to_type_number = {v: k for k, v in type_number_to_pykx_k_type.items()} @@ -95,7 +95,7 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 if dict_grab == {}: return self except IndexError: - raise QError('Value passed does not match PyKX wrapper type') + raise QError('Value passed does not match pykx wrapper type') # Check needed to ensure no non string nested cols in table to convert check_mixed_columns = q('''{[tab;dict] @@ -145,20 +145,20 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c1:()!(); if[any b1; dCols1:dictCols where b1; - f1:{(`$';x)}; c1:dCols1!(f1 each dCols1)]; + f1:{({(0b;`$'x)};x)}; c1:dCols1!(f1 each dCols1)]; // Check casting to symbol, run `$string col // (also covers any symbol -> symbol cases) b2:(dictColTypes=11h) & not (b1 or tabColTypes=0h); c2:()!(); if[any b2; dCols2:dictCols where b2; - f2:{(`$string; x)}; c2:dCols2!(f2 each dCols2)]; + f2:{({(0b;`$string x)}; x)}; c2:dCols2!(f2 each dCols2)]; // Casting to string covering all cases except mixed lists b3: (dictColTypes=10h) & not tabColTypes=0h; c3:()!(); if[any b3; dCols3:dictCols where b3; - f3:{(string; x)}; c3:dCols3!(f3 each dCols3)]; + f3:{({(0b;string x)}; x)}; c3:dCols3!(f3 each dCols3)]; // Check mixed lists // if string column then allow cast to symbol // Check at beginning of method @@ -168,31 +168,32 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c4:()!(); if[any b4; dCols4:dictCols where b4; - f4:{(`$; x)}; c4:dCols4!(f4 each dCols4)]; + f4:{({(0b;`$x)}; x)}; c4:dCols4!(f4 each dCols4)]; // Any matches that meet the vanilla case // and don't have additional needs --> not any (bools) b5:not any (b1;b2;b3;b4); - .pykx.i.errorList:(); if[any b5; dCols5:dictCols where b5; dictColTypes5:dictColTypes where b5; f5:{[c;t;tvd] ({[cl;t;tvd] - @[t$;cl; - {[cl;ty;tvd;err].pykx.i.errorList,:enlist + @[{(0b;x$y)}t;cl; + {[cl;ty;tvd;err] (1b; "Not supported: Error casting ", string[tvd 7h$type cl], " to ", string[tvd 7h$ty], - " with q error: ", err;}[cl;t;tvd;]]}[;t;tvd]; + " with q error: ", err)}[cl;t;tvd;]]}[;t;tvd]; c)}; f5:f5[;;typeValDict]; c5:dCols5!(f5\'[dCols5;dictColTypes5])]; // Grab all cols c:c1,c2,c3,c4,c5; - tableOutput:tabColsOrig xcols ![tab;();0b;c]; - $[count .pykx.i.errorList; - .pykx.i.errorList; - tableOutput] + res:?[tab;();();c]; + $[any value[res][;0]; + {x[;1] where x[;0]} value[res]; + tabColsOrig xcols flip res[;1], + $[count cls:cols[tab] except key c; + ?[tab;();();{x!x} cls];()]] }''', self, dict_grab, type_number_to_pykx_k_type) else: @@ -200,7 +201,7 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 dtype_val = abs(kx_type_to_type_number[next(x for x in kx_type_to_type_number.keys() if x in str(dtype))]) except StopIteration: - raise QError('Value passed does not match PyKX wrapper type') + raise QError('Value passed does not match pykx wrapper type') # Check needed to ensure no non string nested cols in table to convert check_mixed_columns = q('''{[tab;dtype] @@ -234,21 +235,21 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c1:()!(); if[any b1; tCols1:tabCols where b1; - f1:{(`$';x)}; c1:tCols1!(f1 each tCols1) + f1:{({(0b;`$'x)};x)}; c1:tCols1!(f1 each tCols1) ]; // Support casting to symbol b2:(dtype=11h) & not (b1 or tabColTypes=0h); c2:()!(); if[any b2; tCols2:tabCols where b2; - f2:{(`$string@; x)}; c2:tCols2!(f2 each tCols2) + f2:{({(0b;`$string x)}; x)}; c2:tCols2!(f2 each tCols2) ]; // Support casting to string except for mixed lists b3:(dtype=10h) & not tabColTypes=0h; c3:()!(); if[any b3; tCols3:tabCols where b3; - f3:{(string; x)}; c3:tCols3!(f3 each tCols3) + f3:{({(0b;string x)}; x)}; c3:tCols3!(f3 each tCols3) ]; // For mixed lists support casting strings to symbols b4:(dtype=11h) & (tabColTypes=0h) @@ -256,30 +257,31 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c4:()!(); if[any b4; tCols4:tabCols where b4; - f4:{(`$; x)}; c4:tCols4!(f4 each tCols4) + f4:{({(0b;`$x)}; x)}; c4:tCols4!(f4 each tCols4) ]; // Any other combination not matching b1-4 b5:not any (b1;b2;b3;b4); - .pykx.i.errorList:(); c5:()!(); if[any b5; tCols5: tabCols where b5; f5:{[c;t;tvd] ({[cl;t;tvd] - @[t$;cl; - {[cl;ty;tvd;err].pykx.i.errorList,:enlist + @[{(0b;x$y)}t;cl; + {[cl;ty;tvd;err] (1b; "Not supported: Error casting ", string[tvd 7h$type cl], " to ", string[tvd 7h$ty], " with q error: ", - err;}[cl;t;tvd;]]}[;t;tvd]; + err)}[cl;t;tvd;]]}[;t;tvd]; c)}; c5:tCols5!(f5[;dtype;typeValDict] each tCols5) ]; c:c1,c2,c3,c4,c5; - tableOutput:tabCols xcols ![tab;();0b;c]; - $[count .pykx.i.errorList; - .pykx.i.errorList; - tableOutput] + res:?[tab;();();c]; + $[any value[res][;0]; + {x[;1] where x[;0]} value[res]; + tabCols xcols flip res[;1], + $[count cls:cols[tab] except key c; + ?[tab;();();{x!x} cls];()]] }''', self, dtype_val, type_number_to_pykx_k_type) diff --git a/src/pykx/pandas_api/pandas_merge.py b/src/pykx/pandas_api/pandas_merge.py index 56e5cbe..c893291 100644 --- a/src/pykx/pandas_api/pandas_merge.py +++ b/src/pykx/pandas_api/pandas_merge.py @@ -383,7 +383,6 @@ def merge_asof( direction != 'backward' or not allow_exact_matches or tolerance is not None - or by is not None or left_by is not None or right_by is not None ): @@ -395,7 +394,9 @@ def merge_asof( ) else: raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() only implemented for direction='backward', \ - allow_exact_matches=True, tolerance=None, by=None, left_by=None, right_by=None.") # noqa: E501 + allow_exact_matches=True, tolerance=None, left_by=None, right_by=None.") # noqa: E501 + if by is not None: + on = ([by] if isinstance(by, str) else by) + [on] (left, right, on, added_idx) = _parse_input( self, right, diff --git a/src/pykx/pandas_api/pandas_meta.py b/src/pykx/pandas_api/pandas_meta.py index 92f2427..e3bad0c 100644 --- a/src/pykx/pandas_api/pandas_meta.py +++ b/src/pykx/pandas_api/pandas_meta.py @@ -437,6 +437,14 @@ def count(self, axis=0, numeric_only=False): def isna(self): return q.null(self) + @api_return + def fills(self): + return q.fills(self) + + @api_return + def ungroup(self): + return q.ungroup(self) + @api_return def isnull(self): return self.isna() diff --git a/src/pykx/pandas_api/pandas_reset_index.py b/src/pykx/pandas_api/pandas_reset_index.py index cb10a10..179d46f 100644 --- a/src/pykx/pandas_api/pandas_reset_index.py +++ b/src/pykx/pandas_api/pandas_reset_index.py @@ -12,7 +12,7 @@ class PandasResetIndex: def reset_index(self, levels=None, *, drop=False, inplace=False, col_level=0, col_fill='', allow_duplicates=False, names=None): - """Reset keys/index of a PyKX Table""" + """Reset keys/index of a `pykx.Table`""" if col_level != 0: raise QError("'col_level' not presently supported") if col_fill != '': diff --git a/src/pykx/pykx.c b/src/pykx/pykx.c index 36ffdbd..9ae3b49 100644 --- a/src/pykx/pykx.c +++ b/src/pykx/pykx.c @@ -550,6 +550,8 @@ EXPORT K get_attr(K f, K attr) { } K res = create_foreign(pres); Py_XDECREF(_attr); + // INCREF is called in create_foreign, decref here to avoid object not being freed + Py_XDECREF(pres); PyGILState_Release(gstate); return res; } diff --git a/src/pykx/pykx.q b/src/pykx/pykx.q index e849e96..86266e7 100644 --- a/src/pykx/pykx.q +++ b/src/pykx/pykx.q @@ -1,4 +1,4 @@ -// pykx.q - PyKX functionality for operation within a q process +// pykx.q - KDB-X Python functionality for operation within a q process // // @namespace .pykx // @category api @@ -6,16 +6,16 @@ if[`e in key`.p; if[not"{.pykx.pyexec x}"~string get `.p.e; - -1"Warning: Detected invalid '.p.e' function definition expected for PyKX.\n", + -1"Warning: Detected invalid '.p.e' function definition expected for KDB-X Python.\n", "Have you loaded another Python integration first?\n\n", - "Please consider full installation of PyKX under q following instructions at:\n", + "Please consider full installation of KDB-X Python under q following instructions at:\n", "https://code.kx.com/pykx/pykx-under-q/intro.html#install.\n"; - '"Unable to load PyKX, see logged output for more information" + '"Unable to load KDB-X Python, see logged output for more information" ] ] // @private -// @desc Process context prior to PyKX initialization +// @desc Process context prior to KDB-X Python initialization .pykx.util.prevCtx:system"d"; \d .pykx @@ -28,7 +28,7 @@ if[`e in key`.p; util.getLoadDir:{@[{"/"sv -1_"/"vs ssr[;"\\";"/"](-3#get .z.s)0};`;""]} // @private -// @desc Operating system within which PyKX under q is loaded +// @desc Operating system within which KDB-X Python under q is loaded // // @type {char} util.os:first string .z.o; @@ -56,7 +56,7 @@ util.loadfile:{[folder;file] } // @private -// @desc Retrieval of PyKX initialization directory on first initialization +// @desc Retrieval of KDB-X Python initialization directory on first initialization if[not "true"~lower getenv`PYKX_LOADED_UNDER_Q; util.whichPython:$[count pykxExecutable:getenv`PYKX_EXECUTABLE;pykxExecutable;()]; util.dirCommand:"-c \"import pykx; print('PYKX_DIR: ' + str(pykx.config.pykx_dir))\""; @@ -70,7 +70,7 @@ if[not "true"~lower getenv`PYKX_LOADED_UNDER_Q; ]; // @private -// @desc Allow a user to force PyKX to use the location of libpython +// @desc Allow a user to force KDB-X Python to use the location of libpython // found by the Python library find_libpython if[(lower getenv`PYKX_USE_FIND_LIBPYTHON) in ("true";enlist"1"); libpython_path:first system util.whichPython," -c\"from find_libpython import find_libpython;print(find_libpython())\""; @@ -79,8 +79,7 @@ if[(lower getenv`PYKX_USE_FIND_LIBPYTHON) in ("true";enlist"1"); // @private // @desc -// Environment variable denoting that pythonic invocations of PyKX should the -// PyKX act as of in a q process, as q symbols are already defined in the process. +// Environment variable denoting that KDB-X loaded in a q process setenv[`PYKX_UNDER_Q;"true"]; // @private @@ -97,7 +96,7 @@ if["true"~getenv`PYKX_UNDER_PYTHON; ]; // @private -// @desc Load PyKX initialization script if not previously initialised +// @desc Load KDB-X Python initialization script if not previously initialised if[not "true"~lower getenv`PYKX_LOADED_UNDER_Q; util.pyEnvInfo:("None"; "None"; ""); if[0=count getenv`PYKX_Q_LOADED_MARKER; @@ -127,7 +126,7 @@ if[not `debug in key `.pykx; preinit[]; // @private -// @desc Validate that PyKX was installed, failover will initialise pykxDir to "/pykx" +// @desc Validate that KDB-X Python was installed, failover will initialise pykxDir to "/pykx" if[pykxDir~"/pykx"; '"Failed to find pykx - ensure your Python environment is properly configured and activated" ]; @@ -151,7 +150,7 @@ util.CFunctions:flip `qname`cname`args!flip ( // @private // @desc -// Load defined C functions to PyKX for later use +// Load defined C functions to KDB-X Python for later use {.Q.dd[`.pykx;x`qname]set util.load x`cname`args}each util.CFunctions; // @private @@ -673,7 +672,7 @@ topt:{x y}(`..torch;;) // @name .pykx.tok // @category api // @overview -// _Tag a q object to be indicate conversion to a Pythonic PyKX object when called in Python_ +// _Tag a q object to be indicate conversion to a Pythonic `pykx` object when called in Python_ // // ```q // .pykx.tok[qObject] @@ -683,16 +682,16 @@ topt:{x y}(`..torch;;) // // name | type | description | // ----------|---------|-------------| -// `qObject` | `any` | A q object which is to be defined as a PyKX object in Python. | +// `qObject` | `any` | A q object which is to be defined as a `pykx` object in Python. | // // **Return:** // // type | description // -------------|------------ -// `projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a PyKX type object. | +// `projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a `pykx` type object. | // // ```q -// // Denote that a q object once passed to Python should be managed as a PyKX object +// // Denote that a q object once passed to Python should be managed as a `pykx` object // q).pykx.tok til 10 // enlist[`..k;;][0 1 2 3 4 5 6 7 8 9] // @@ -700,7 +699,7 @@ topt:{x y}(`..torch;;) // q).pykx.typepy til 10 // "" // -// // Pass a q object to Python treating the Python object as a PyKX object +// // Pass a q object to Python treating the Python object as a `pykx` object // q).pykx.typepy .pykx.tok til 10 // "" // ``` @@ -745,7 +744,7 @@ toraw: {x y}(`..raw;;) // @name .pykx.todefault // @category api // @overview -// _Tag a q object to indicate it should use the PyKX default conversion when called in Python_ +// _Tag a q object to indicate it should use the `pykx` default conversion when called in Python_ // // ```q // .pykx.todefault[qObject] @@ -772,7 +771,7 @@ toraw: {x y}(`..raw;;) // q).pykx.todefault til 10 // enlist[`..numpy;;][0 1 2 3 4 5 6 7 8 9] // -// // Pass a q list to Python treating the Python object as PyKX default +// // Pass a q list to Python treating the Python object as `pykx` default // q).pykx.typepy .pykx.todefault (til 10;til 10) // "" // @@ -929,7 +928,7 @@ setdefault:{ // @name .pykx.toq // @category api // @overview -// _Convert an (un)wrapped `PyKX` foreign object into an analogous q type._ +// _Convert an (un)wrapped `pykx` foreign object into an analogous q type._ // // ```q // .pykx.toq[pythonObject] @@ -948,19 +947,19 @@ setdefault:{ // `any` | A q object converted from Python // // ```q -// // Convert a wrapped PyKX foreign object to q +// // Convert a wrapped `pykx` foreign object to q // q)show a:.pykx.eval["1+1"] // {[f;x].pykx.util.pykx[f;x]}[foreign]enlist // q).pykx.toq a // 2 // -// // Convert an unwrapped PyKX foreign object to q +// // Convert an unwrapped `pykx` foreign object to q // q)show b:a`. // foreign // q).pykx.toq b // 2 // -// // Convert a PyKX conversion object back to q +// // Convert a `pykx` conversion object back to q // q).pykx.toq .pykx.topd ([]5?1f;5?`a`b`c) // // x x1 @@ -984,7 +983,7 @@ py2q:toq:{ // @name .pykx.toq0 // @category api // @overview -// _Convert an (un)wrapped `PyKX` foreign object into an analogous q type._ +// _Convert an (un)wrapped `pykx` foreign object into an analogous q type._ // // ```q // .pykx.toq0[pythonObject;strAsChar] @@ -1004,13 +1003,13 @@ py2q:toq:{ // `any` | A q object converted from Python // // ```q -// // Convert a wrapped PyKX foreign object to q +// // Convert a wrapped `pykx` foreign object to q // q)show a:.pykx.eval["1+1"] // {[f;x].pykx.util.pykx[f;x]}[foreign]enlist // q).pykx.toq0 a // 2 // -// // Convert an unwrapped PyKX foreign object to q +// // Convert an unwrapped `pykx` foreign object to q // q)show b:a`. // foreign // q).pykx.toq0 b @@ -1357,7 +1356,7 @@ print:{ // @name .pykx.version // @category api // @overview -// _Retrieve the version of PyKX presently being used by a q process_ +// _Retrieve the version of KDB-X Python presently being used by a q process_ // // ```q // .pykx.version[] @@ -1367,7 +1366,7 @@ print:{ // // type | description // ---------|------------ -// `string` | The version number of PyKX installed within the users q session +// `string` | The version number of KDB-X Python installed within the users q session // // ```q // q).pykx.version[] @@ -1673,7 +1672,7 @@ qcallable:{$[util.isw x;wrap[unwrap[x]](<);util.isf x;wrap[x](<);'"Could not con // @name .pykx.safeReimport // @category api // @overview -// _Isolated execution of a q function which relies on importing PyKX_ +// _Isolated execution of a q function which relies on importing KDB-X Python_ // // ```q // .pykx.safeReimport[qFunction] @@ -1688,7 +1687,7 @@ qcallable:{$[util.isw x;wrap[unwrap[x]](<);util.isf x;wrap[x](<);'"Could not con // // name | type | description // -------------|------------|------------- -// `qFunction` | `function` | A function which is to be run following unsetting of PyKX environment variables and prior to their reset +// `qFunction` | `function` | A function which is to be run following unsetting of KDB-X Python environment variables and prior to their reset // // **Returns:** // @@ -1698,7 +1697,7 @@ qcallable:{$[util.isw x;wrap[unwrap[x]](<);util.isf x;wrap[x](<);'"Could not con // // **Example:** // -// Initializing a Python process which imports PyKX +// Initializing a Python process which imports KDB-X Python // // ```q // q)\l pykx.q @@ -1812,7 +1811,7 @@ loadPy:{[file] // // ```q // q).pykx.debugInfo[] -// "**** PyKX information ****" +// "**** KDB-X Python under q Information ****" // "pykx.args: ()" // "pykx.qhome: /usr/local/anaconda3/envs/qenv/q" // "pykx.qlic: /usr/local/anaconda3/envs/qenv/q" @@ -1820,7 +1819,7 @@ loadPy:{[file] // .. // ``` debugInfo:{ - pykxQHeader:enlist"**** PyKX under q Information ****"; + pykxQHeader:enlist"**** KDB-X Python under q Information ****"; pykxQInfo :{string[x 0],": ",x 1}each flip(key;value)@\:.pykx.debug; pykxPythonInfo:"\n" vs string .pykx.import[`pykx;`:util.debug_environment][pykwargs enlist[`return_info]!enlist 1b]`; pykxPythonInfo,pykxQHeader,pykxQInfo @@ -1845,7 +1844,7 @@ debugInfo:{ // **Example:** // // ```q -// Enter PyKX console and evaluate Python code +// Enter Python console and evaluate Python code // q).pykx.console[] // >>> 1+1 // 2 @@ -1854,7 +1853,7 @@ debugInfo:{ // >>> quit() // q) // -// // Enter PyKX console setting q objects using PyKX +// // Enter Python console setting q objects // q).pykx.console[] // >>> import pykx as kx // >>> kx.q['table'] = kx.q('([]2?1f;2?0Ng;2?`3)' @@ -1865,7 +1864,7 @@ debugInfo:{ // 0.439081 49f2404d-5aec-f7c8-abba-e2885a580fb6 mil // 0.5759051 656b5e69-d445-417e-bfe7-1994ddb87915 igf // -// // Enter PyKX console setting Python objects using PyKX +// // Enter Python console setting Python objects // q).pykx.console[] // >>> a = list(range(5)) // >>> quit() @@ -1878,18 +1877,22 @@ console:{if[.z.o like "w*";'".pykx.console is not available on Windows"]; "__pykx_console__ = InteractiveConsole(globals())"; "__pykx_console__.push('import sys')"; "__pykx_console__.push('quit = sys.exit')"; - "__pykx_console__.push('exit = sys.exit')"; - "try:"; - " line = __pykx_console__.interact(banner='', exitmsg='')"; - "except SystemExit:"; - " pykx._pykx_helpers.clean_errors()") + "__pykx_console__.push('exit = sys.exit')"); + c:(1b;""); + 1 ">>> "; + while[c 0; + c:@[{(1b;pyexec x)}; + "print('... ', end='') if __pykx_console__.push('",(read0 0),"') else print('>>> ', end='')"; + {pyexec"pykx._pykx_helpers.clean_errors()"; + $[x~"SystemExit()";(0b;"");(1b;x)]}]]; + if[count c 1;'c 1]; }; // @private // @desc // Set the execution function used when loading files with the extension `*.p` // or when using the following syntax `p)` within a q session -.p.e:{.pykx.pyexec x} // If changing this line please ensure you have updated the check used at the beginning of this file to warn users about PyKX being loaded with other Python libraries +.p.e:{.pykx.pyexec x} // If changing this line please ensure you have updated the check used at the beginning of this file to warn users about KDB-X Python being loaded with other Python libraries // @private // @desc @@ -1898,8 +1901,8 @@ setdefault {$[""~c:getenv`PYKX_DEFAULT_CONVERSION;"default";c]}[]; // @private // @desc -// Finalise loading of PyKX functionality setting environment variables -// needed to ensure loading PyKX multiple times does not result in unexpected errors +// Finalise loading of KDB-X Python functionality setting environment variables +// needed to ensure loading KDB-X Python multiple times does not result in unexpected errors finalise[]; // @private @@ -1917,7 +1920,7 @@ finalise[]; // // type | description // -------|------------ -// `list` | A list of strings denoting the available extensions in your version of PyKX +// `list` | A list of strings denoting the available extensions in your version of KDB-X Python // // **Example:** // @@ -1933,7 +1936,7 @@ listExtensions:{-2 _/:lst where like[;"*.q"]lst:string key hsym`$pykxDir,"/exten // @name .pykx.loadExtension // @category api // @overview -// _Loading of a PyKX extension_ +// _Loading of a KDB-X Python extension_ // // ```q // .pykx.loadExtension[ext] @@ -2012,3 +2015,4 @@ loadExtension:{[ext] // @desc Restore context used at initialization of script system"d ",string .pykx.util.prevCtx; +k).Q.pykxld:{x:("#!"~2#*x)_x:-1!'x;+(1+*:'i;)@"\n"/:'x i:(&|1^\|0N 0 1@"/ "?*:'(v:x i),'"/")_i:&~|':(b?-1)#b:+\-/x~\:/:+,"/\\"} diff --git a/src/pykx/pykxq.c b/src/pykx/pykxq.c index 10fad12..50c8d0c 100644 --- a/src/pykx/pykxq.c +++ b/src/pykx/pykxq.c @@ -506,8 +506,11 @@ EXPORT K get_attr(K f, K attr) { int gstate = PyGILState_Ensure(); P p = get_py_ptr(f); P _attr = Py_BuildValue("s", attr->s); - K res = create_foreign(PyObject_GetAttr(p, _attr)); + P __attr = PyObject_GetAttr(p, _attr); + K res = create_foreign(__attr); Py_XDECREF(_attr); + // INCREF is called in create_foreign, decref here to avoid object not being freed + Py_XDECREF(__attr); if ((k = k_py_error())) { PyGILState_Release(gstate); return k; diff --git a/src/pykx/pykxq_m.c b/src/pykx/pykxq_m.c index 48de3bc..7f2d038 100644 --- a/src/pykx/pykxq_m.c +++ b/src/pykx/pykxq_m.c @@ -664,8 +664,11 @@ K _get_attr(K f, K attr) { } P p = get_py_ptr(f); P _attr = Py_BuildValue("s", attr->s); - K res = create_foreign(PyObject_GetAttr(p, _attr)); + P __attr = PyObject_GetAttr(p, _attr); + K res = create_foreign(__attr); Py_XDECREF(_attr); + // INCREF is called in create_foreign, decref here to avoid object not being freed + Py_XDECREF(__attr); if ((k = k_py_error())) { return k; } diff --git a/src/pykx/q.so/libs/4-0/l64/libq.so b/src/pykx/q.so/libs/4-0/l64/libq.so deleted file mode 100755 index 0a1f6b6..0000000 Binary files a/src/pykx/q.so/libs/4-0/l64/libq.so and /dev/null differ diff --git a/src/pykx/q.so/libs/4-0/l64arm/libq.so b/src/pykx/q.so/libs/4-0/l64arm/libq.so deleted file mode 100755 index f077b95..0000000 Binary files a/src/pykx/q.so/libs/4-0/l64arm/libq.so and /dev/null differ diff --git a/src/pykx/q.so/libs/4-0/m64/libq.dylib b/src/pykx/q.so/libs/4-0/m64/libq.dylib deleted file mode 100755 index c4fcc07..0000000 Binary files a/src/pykx/q.so/libs/4-0/m64/libq.dylib and /dev/null differ diff --git a/src/pykx/q.so/libs/4-0/m64arm/libq.dylib b/src/pykx/q.so/libs/4-0/m64arm/libq.dylib deleted file mode 100755 index a8e0f16..0000000 Binary files a/src/pykx/q.so/libs/4-0/m64arm/libq.dylib and /dev/null differ diff --git a/src/pykx/q.so/libs/4-0/w64/q.dll b/src/pykx/q.so/libs/4-0/w64/q.dll deleted file mode 100644 index 6e5c670..0000000 Binary files a/src/pykx/q.so/libs/4-0/w64/q.dll and /dev/null differ diff --git a/src/pykx/q.so/libs/4-0/w64/q.lib b/src/pykx/q.so/libs/4-0/w64/q.lib deleted file mode 100644 index 124e3b7..0000000 Binary files a/src/pykx/q.so/libs/4-0/w64/q.lib and /dev/null differ diff --git a/src/pykx/q.so/libs/4-1/l64/kxreaper b/src/pykx/q.so/libs/4-1/l64/kxreaper deleted file mode 100644 index 79151c2..0000000 Binary files a/src/pykx/q.so/libs/4-1/l64/kxreaper and /dev/null differ diff --git a/src/pykx/q.so/libs/4-1/l64/libkurl.so b/src/pykx/q.so/libs/4-1/l64/libkurl.so deleted file mode 100644 index 6f44fb4..0000000 Binary files a/src/pykx/q.so/libs/4-1/l64/libkurl.so and /dev/null differ diff --git a/src/pykx/q.so/libs/4-1/l64/libobjstor.so b/src/pykx/q.so/libs/4-1/l64/libobjstor.so deleted file mode 100644 index 828d6ad..0000000 Binary files a/src/pykx/q.so/libs/4-1/l64/libobjstor.so and /dev/null differ diff --git a/src/pykx/q.so/libs/4-1/l64/libq.so b/src/pykx/q.so/libs/4-1/l64/libq.so deleted file mode 100755 index 56684db..0000000 Binary files a/src/pykx/q.so/libs/4-1/l64/libq.so and /dev/null differ diff --git a/src/pykx/q.so/libs/4-1/l64/pg b/src/pykx/q.so/libs/4-1/l64/pg deleted file mode 100644 index f5989b0..0000000 Binary files a/src/pykx/q.so/libs/4-1/l64/pg and /dev/null differ diff --git a/src/pykx/q.so/libs/4-1/l64arm/libq.so b/src/pykx/q.so/libs/4-1/l64arm/libq.so deleted file mode 100755 index 3e2e4f9..0000000 Binary files a/src/pykx/q.so/libs/4-1/l64arm/libq.so and /dev/null differ diff --git a/src/pykx/q.so/libs/4-1/m64/libq.dylib b/src/pykx/q.so/libs/4-1/m64/libq.dylib deleted file mode 100755 index 5358dba..0000000 Binary files a/src/pykx/q.so/libs/4-1/m64/libq.dylib and /dev/null differ diff --git a/src/pykx/q.so/libs/4-1/m64arm/libq.dylib b/src/pykx/q.so/libs/4-1/m64arm/libq.dylib deleted file mode 100755 index 21660d4..0000000 Binary files a/src/pykx/q.so/libs/4-1/m64arm/libq.dylib and /dev/null differ diff --git a/src/pykx/q.so/libs/4-1/w64/q.dll b/src/pykx/q.so/libs/4-1/w64/q.dll deleted file mode 100644 index 35c9677..0000000 Binary files a/src/pykx/q.so/libs/4-1/w64/q.dll and /dev/null differ diff --git a/src/pykx/q.so/libs/4-1/w64/q.lib b/src/pykx/q.so/libs/4-1/w64/q.lib deleted file mode 100644 index bc672d4..0000000 Binary files a/src/pykx/q.so/libs/4-1/w64/q.lib and /dev/null differ diff --git a/src/pykx/q.so/libs/4-0/l64/kxreaper b/src/pykx/q.so/libs/kdbx/l64/kxreaper similarity index 100% rename from src/pykx/q.so/libs/4-0/l64/kxreaper rename to src/pykx/q.so/libs/kdbx/l64/kxreaper diff --git a/src/pykx/q.so/libs/4-0/l64/libkurl.so b/src/pykx/q.so/libs/kdbx/l64/libkurl.so similarity index 100% rename from src/pykx/q.so/libs/4-0/l64/libkurl.so rename to src/pykx/q.so/libs/kdbx/l64/libkurl.so diff --git a/src/pykx/q.so/libs/4-0/l64/libobjstor.so b/src/pykx/q.so/libs/kdbx/l64/libobjstor.so similarity index 100% rename from src/pykx/q.so/libs/4-0/l64/libobjstor.so rename to src/pykx/q.so/libs/kdbx/l64/libobjstor.so diff --git a/src/pykx/q.so/libs/kdbx/l64/libq.so b/src/pykx/q.so/libs/kdbx/l64/libq.so new file mode 100755 index 0000000..99d779b Binary files /dev/null and b/src/pykx/q.so/libs/kdbx/l64/libq.so differ diff --git a/src/pykx/q.so/libs/4-0/l64/pg b/src/pykx/q.so/libs/kdbx/l64/pg similarity index 100% rename from src/pykx/q.so/libs/4-0/l64/pg rename to src/pykx/q.so/libs/kdbx/l64/pg diff --git a/src/pykx/q.so/libs/4-0/l64/symbols.txt b/src/pykx/q.so/libs/kdbx/l64/symbols.txt similarity index 100% rename from src/pykx/q.so/libs/4-0/l64/symbols.txt rename to src/pykx/q.so/libs/kdbx/l64/symbols.txt diff --git a/src/pykx/q.so/libs/kdbx/l64arm/libq.so b/src/pykx/q.so/libs/kdbx/l64arm/libq.so new file mode 100755 index 0000000..d9515f0 Binary files /dev/null and b/src/pykx/q.so/libs/kdbx/l64arm/libq.so differ diff --git a/src/pykx/q.so/libs/kdbx/m64/libq.dylib b/src/pykx/q.so/libs/kdbx/m64/libq.dylib new file mode 100755 index 0000000..298295b Binary files /dev/null and b/src/pykx/q.so/libs/kdbx/m64/libq.dylib differ diff --git a/src/pykx/q.so/libs/kdbx/m64arm/libq.dylib b/src/pykx/q.so/libs/kdbx/m64arm/libq.dylib new file mode 100755 index 0000000..e5f2278 Binary files /dev/null and b/src/pykx/q.so/libs/kdbx/m64arm/libq.dylib differ diff --git a/src/pykx/q.so/libs/kdbx/w64/q.dll b/src/pykx/q.so/libs/kdbx/w64/q.dll new file mode 100644 index 0000000..f0ad5d4 Binary files /dev/null and b/src/pykx/q.so/libs/kdbx/w64/q.dll differ diff --git a/src/pykx/q.so/libs/kdbx/w64/q.lib b/src/pykx/q.so/libs/kdbx/w64/q.lib new file mode 100644 index 0000000..a4beb9d Binary files /dev/null and b/src/pykx/q.so/libs/kdbx/w64/q.lib differ diff --git a/src/pykx/q.so/qk/pykx_init.q_ b/src/pykx/q.so/qk/pykx_init.q_ index 7d27e91..f9e96c5 100644 Binary files a/src/pykx/q.so/qk/pykx_init.q_ and b/src/pykx/q.so/qk/pykx_init.q_ differ diff --git a/src/pykx/q.so/qk/s.k_ b/src/pykx/q.so/qk/sql.k_ similarity index 100% rename from src/pykx/q.so/qk/s.k_ rename to src/pykx/q.so/qk/sql.k_ diff --git a/src/pykx/query.py b/src/pykx/query.py index 8ed61f6..3078ba8 100644 --- a/src/pykx/query.py +++ b/src/pykx/query.py @@ -1,4 +1,4 @@ -"""_This page documents query interfaces for querying q tables using PyKX._""" +"""_This page documents query interfaces for querying q tables using KDB-X Python._""" from abc import ABCMeta import asyncio @@ -8,6 +8,7 @@ from . import wrappers as k from .ipc import QFuture from .exceptions import PyKXException, QError +from .config import licensed __all__ = [ 'Insert', @@ -75,7 +76,7 @@ def select(self, See [here](https://code.kx.com/q/basics/qsql/#result-and-side-effects). Returns: - A PyKX Table or KeyedTable object resulting from the executed select query + A `pykx` Table or KeyedTable object resulting from the executed select query Examples: @@ -138,7 +139,7 @@ def exec(self, values are aggregation rules used when q functionally applies group-by. Returns: - A PyKX Vector or Dictionary object resulting from the executed exec query + A `pykx` Vector or Dictionary object resulting from the executed exec query Examples: @@ -224,7 +225,7 @@ def update(self, https://code.kx.com/q/basics/qsql/#result-and-side-effects. Returns: - The updated PyKX Table or KeyedTable object resulting from the executed update query + The updated `pykx` Table or KeyedTable object resulting from the executed update query Examples: @@ -300,7 +301,7 @@ def delete(self, https://code.kx.com/q/basics/qsql/#result-and-side-effects. Returns: - The updated PyKX Table or KeyedTable object resulting from the executed delete query + The updated `pykx` Table or KeyedTable object resulting from the executed delete query Examples: @@ -349,9 +350,9 @@ def _seud(self, table, query_type, columns=None, where=None, by=None, inplace=Fa if not isinstance(table, str): table = k.K(table) - if isinstance(table, (k.SplayedTable, k.PartitionedTable)) and inplace: + if isinstance(table, (k.SplayedTable, k.PartitionedTable, k.VirtualTable)) and inplace: raise QError("Application of 'inplace' updates not " - "supported for splayed/partitioned tables") + "supported for splayed/partitioned/virtual tables") if isinstance(columns, k.Column) and columns._renamed: columns=k.QueryPhrase(columns) if isinstance(by, k.Column) and by._renamed: @@ -362,7 +363,7 @@ def _seud(self, table, query_type, columns=None, where=None, by=None, inplace=Fa get = '' query_char = '!' if query_type in ('delete', 'update') else '?' if isinstance(table, k.K): - if not isinstance(table, (k.Table, k.KeyedTable)): + if not isinstance(table, (k.Table, k.KeyedTable, k.VirtualTable)): raise TypeError("'table' object provided was not a K tabular object or an " "object which could be converted to an appropriate " "representation") @@ -523,6 +524,32 @@ class SQL: def __init__(self, q: Q): self._q = q + self.loaded = None + if '_connection_info' in q.__dict__: + if not q._connection_info['no_ctx'] and 'event_loop' not in q._connection_info.keys(): + try: + self.loaded = self._q('@[{2>> kx.q.sql.loaded + False + >>> kx.q.sql.init() + >>> kx.q.sql.loaded + True + ``` + """ + if not self.loaded: + self._q('s) ') + self.loaded = True def __call__(self, query: str, *args: Any) -> k.Table: """Compile and run a SQL statement using string interpolation. @@ -592,6 +619,7 @@ def __call__(self, query: str, *args: Any) -> k.Table: ')) ``` """ + self.init() return self._q('.s.sp', k.CharVector(query), args) def prepare(self, query: str, *args: Any) -> k.List: @@ -652,6 +680,7 @@ def prepare(self, query: str, *args: Any) -> k.List: _args = [] for a in args: _args.append(a._prototype() if (isinstance(a, type) or isinstance(a, ABCMeta)) else a) + self.init() return self._q('.s.sq', k.CharVector(query), _args) def execute(self, query: k.List, *args: Any) -> k.K: @@ -712,6 +741,7 @@ def execute(self, query: k.List, *args: Any) -> k.K: ')) ``` """ + self.init() return self._q('.s.sx', query, args) def get_input_types(self, prepared_query: k.List) -> List[str]: diff --git a/src/pykx/random.py b/src/pykx/random.py index 2a1d6e7..5822b95 100644 --- a/src/pykx/random.py +++ b/src/pykx/random.py @@ -16,7 +16,7 @@ def _init(_q): def seed(seed: int) -> None: - """Set random seed for PyKX random data generation. + """Set random seed for KDB-X Python random data generation. Parameters: seed: Integer value defining the seed value to be set diff --git a/src/pykx/register.py b/src/pykx/register.py index c79e725..c60c6ae 100644 --- a/src/pykx/register.py +++ b/src/pykx/register.py @@ -1,4 +1,4 @@ -"""Functionality for the registration of conversion functions between PyKX and Python""" +"""Functionality for the registration of conversion functions between KDB-X Python and Python""" from .toq import _converter_from_python_type from .wrappers import Column @@ -26,7 +26,7 @@ def py_toq(py_type: Any, ) -> None: """ Register conversion logic for a specified Python type when converting it to - a PyKX object. + a `pykx` object. !!! Note The return of registered functions should be a valid `pykx` object type @@ -38,10 +38,10 @@ def py_toq(py_type: Any, Parameters: py_type: The `type` signature used for determining when a conversion - should be triggered for PyKX, in particular this will check the + should be triggered for KDB-X Python, in particular this will check the `type(x)` on incoming data to determine this. conversion_function: The function/callable which will be used to convert - the supplied object to a PyKX object specified by the user. + the supplied object to a KDB-X Python object specified by the user. *, overwrite: If a definition for this type already exists should it be overwritten by default this is set to False to avoid accidental overwriting of diff --git a/src/pykx/reimporter.py b/src/pykx/reimporter.py index fbeacd7..4b17e86 100644 --- a/src/pykx/reimporter.py +++ b/src/pykx/reimporter.py @@ -1,11 +1,11 @@ -"""PyKX reimport helper module. +"""KDB-X Python reimport helper module. -PyKX uses various environment variables to monitor the state of various modules initialization. This -is required to manage all of the different modes of operation, however it can cause issues when -attempting to reimport PyKX within a spawned subprocess. +KDB-X Python uses various environment variables to monitor the state of various modules +initialization. This is required to manage all of the different modes of operation, +however it can cause issues when attempting to reimport KDB-X Python within a spawned subprocess. -This module provides a mechanism to allow users to safely reimport PyKX within spawned subprocesses -without having to manually manage any of these internal environment variables. +This module provides a mechanism to allow users to safely reimport KDB-X Python within spawned +subprocesses without having to manually manage any of these internal environment variables. """ import os @@ -13,7 +13,7 @@ class PyKXReimport: - """Helper class to help manage the environment variables around reimporting PyKX in a + """Helper class to help manage the environment variables around reimporting KDB-X Python in a subprocess. It is strongly recommended to use this class by using the python `with` syntax. This will ensure @@ -24,7 +24,7 @@ class PyKXReimport: ``` with kx.PyKXReimport(): - # This process can safely import PyKX + # This process can safely import KDB-X Python subprocess.Popen(f"python other_file.py") ``` """ diff --git a/src/pykx/remote.py b/src/pykx/remote.py index 72718a2..6c3a192 100644 --- a/src/pykx/remote.py +++ b/src/pykx/remote.py @@ -103,8 +103,8 @@ def __init__(self, no_ctx=True) pykx_loaded = self._session('`pykx in key `') if not pykx_loaded: - print("PyKX not loaded on remote server, attempting to load PyKX") - self._session('@[system"l ",;"pykx.q";{\'"Failed to load PyKX with error: ",x}]') + print("KDB-X Python not loaded on remote server, attempting to load KDB-X Python") + self._session('@[system"l ",;"pykx.q";{\'"Failed to load KDB-X Python with error: ",x}]') # noqa: E501 self.valid = True if self._libraries is not None: self.libraries(self._libraries) @@ -169,8 +169,8 @@ def function(remote_session: session, *args) -> None: *args: Arguments that will be passed to the decorated function when it is invoked Returns: - A PyKX converted type of the result returned from the execution of the decorated function - on the remote process + A KDB-X Python converted type of the result returned from the execution of the decorated + function on the remote process Examples: diff --git a/src/pykx/streamlit.py b/src/pykx/streamlit.py index 00c2c2a..c415bfe 100644 --- a/src/pykx/streamlit.py +++ b/src/pykx/streamlit.py @@ -6,7 +6,7 @@ # This class is required to ensure that in the absence -# of the streamlit dependency PyKX can be imported +# of the streamlit dependency KDB-X Python can be imported class _dummy_class(object): def __getattr__(self, item): return self diff --git a/src/pykx/system.py b/src/pykx/system.py index b02d7dc..3a5838a 100644 --- a/src/pykx/system.py +++ b/src/pykx/system.py @@ -1,4 +1,4 @@ -"""System command wrappers for PyKX.""" +"""System command wrappers for KDB-X Python.""" import os from pathlib import Path from warnings import warn diff --git a/src/pykx/tick.py b/src/pykx/tick.py index 89379bd..40be0f5 100644 --- a/src/pykx/tick.py +++ b/src/pykx/tick.py @@ -1,5 +1,5 @@ """ -Functionality for the generation and management of streaming infrastructures using PyKX. +Functionality for the generation and management of streaming infrastructures using KDB-X Python. Fully described [here](../user-guide/advanced/streaming/index.md), this allows users to ingest, persist and query vast amounts of real-time and historical data in a unified data-format. """ @@ -221,7 +221,7 @@ def libraries(self, libs: dict = None) -> None: def register_api(self, api_name: str, function: Callable) -> None: """ Define a registered API to be callable by name on a process, - this API can be a Python function or a PyKX + this API can be a Python function or a KDB-X Python lambda/projection. Parameters: @@ -539,7 +539,7 @@ def set_snap(self, snap_function: Callable) -> None: Tickerplant process. Parameters: - snap_function: A Python function or callable PyKX Lambda which takes + snap_function: A Python function or callable `pykx` Lambda which takes a single argument and returns the expected tabular dataset for display @@ -598,7 +598,7 @@ def __init__(self, historical databases to reload if needed. In a more complex case an RTP will run analytics on data prior to and post data insert - as noted in step 2 above. These analytics can either be Python or q/PyKX functions. + as noted in step 2 above. These analytics can either be Python or q/pykx functions. Additionally users can define 'apis' on the server which can be called explicitly by users. @@ -612,7 +612,7 @@ def __init__(self, subscriptions: A list of tables (str) from which to receive updates, if None the RTP will receive updates from all tables apis: A dictionary mapping the names to be used by users when calling a - defined API to the callable Python functions or PyKX lambdas/projections + defined API to the callable Python functions or `pykx` lambdas/projections which will be called. vanilla: In the case that the RTP is defined as 'vanilla' data received from an upstream tickerplant will be inserted into an in-memory table. @@ -785,7 +785,7 @@ def start(self, config: dict = None) -> None: 3. `database: a string denoting the directory where your current days data will be persisted. This should be the same directory as the `database` keyword for your HDB process should it be used. - By default the location "db" will be used in the directory PyKX was + By default the location "db" will be used in the directory KDB-X Python was imported. Returns: @@ -912,7 +912,7 @@ def pre_processor(self, function: Callable) -> None: The pre-processing function should return Parameters: - function: A callable function or PyKX Lambda taking 2 arguments + function: A callable function or `pykx` Lambda taking 2 arguments the name of the table as a `str` and the message to be processed Returns: @@ -969,7 +969,7 @@ def post_processor(self, function: Callable) -> None: This function can have side-effects and does not expect a return Parameters: - function: A callable function or PyKX Lambda taking 2 arguments + function: A callable function or KDB-X Python Lambda taking 2 arguments the name of the table as a `str` and the message to be processed Returns: @@ -1083,7 +1083,7 @@ def __init__(self, libraries: A dictionary mapping the alias by which a Python library will be referred to the name of library apis: A dictionary mapping the names to be used by users when calling a - defined API to the callable Python functions or PyKX lambdas/projections + defined API to the callable Python functions or KDB-X Python lambdas/projections which will be called. init_args: A list of arguments passed to the initialized q process at startup denoting the command line options to be used for the initialized q process @@ -1294,7 +1294,7 @@ def __init__(self, libraries: A dictionary mapping the alias by which a Python library will be referred to the name of library apis: A dictionary mapping the names to be used by users when calling a - defined API to the callable Python functions or PyKX lambdas/projections + defined API to the callable Python functions or `pykx` lambdas/projections which will be called. connections: A dictionary passed to the sub-process which is used by maps a key denoting the 'name' to be assigned @@ -1331,7 +1331,7 @@ def __init__(self, >>> rdb.start({'tickerplant': 'localhost:5030'}) >>> def gateway_func(x): ... # The 'module' gateway is a populated class - ... # on the PyKX Gateway processes + ... # on the KDB-X Python Gateway processes ... rdb_data = gateway.call_port('rdb', b'{x+1}', x) ... hdb_data = gateway.call_port('hdb', b'{x+2}', x) ... return([rdb_data, hdb_data]) @@ -1555,7 +1555,7 @@ def __init__( database=None, ports=_default_ports): """ - Initialise a configuration for a basic PyKX streaming workflow. + Initialise a configuration for a basic KDB-X Python streaming workflow. This configuration will be used to (by default) start the following processes: diff --git a/src/pykx/toq.pyx b/src/pykx/toq.pyx index ef606cf..d8a2937 100644 --- a/src/pykx/toq.pyx +++ b/src/pykx/toq.pyx @@ -112,7 +112,7 @@ from . import wrappers as k from ._pyarrow import pyarrow as pa from .cast import * from . import config -from .config import beta_features, find_core_lib, k_allocator, licensed, pandas_2, system +from .config import beta_features, find_core_lib, k_allocator, licensed, pandas_gt1, system from .constants import NULL_INT16, NULL_INT32, NULL_INT64 from .constants import INF_INT16, INF_INT32, INF_INT64, INF_NEG_INT16, INF_NEG_INT32, INF_NEG_INT64 from .exceptions import LicenseException, PyArrowUnavailable, PyKXException, QError @@ -666,8 +666,8 @@ def from_int(x: Any, range is the null value for that type, one larger than that is the negative infinity for that type, and the upper bound of the range is positive infinity for that type. An exception will not be raised if `int` objects with these values are provided. Refer to - the nulls and infinities PyKX documentation for more information about q nulls and - infinities, and how to handle them with PyKX. + the nulls and infinities documentation for more information about q nulls and + infinities, and how to handle them with KDB-X Python. Returns: An instance of a `pykx.IntegralNumericAtom` subclass. @@ -1207,7 +1207,7 @@ def from_numpy_ndarray(x: np.ndarray, ) -> k.Vector: """Converts a `numpy.ndarray` into a `pykx.Vector`. - Warning: Data is always copied when converting from Numpy to q if not using the PyKX Allocator. + Warning: Data is always copied when converting from Numpy to q if not using the KDB-X Python Allocator. While many conversions from q to Python types can be performed without incurring a copy, the reverse is not true. As mentioned in [the q C API documentation](https://code.kx.com/q/interfaces/capiref/#k-object), vectors in @@ -1215,9 +1215,8 @@ def from_numpy_ndarray(x: np.ndarray, an array in Python's memory cannot be shared by q, since q would have to store the metadata in a memory location it has no ownership of. - If the `pykx.k_allocator` is enabled with the environment variable `PYKX_ALLOCATOR` or if the - `QARGS` environment variable contains `--pykxalloc`, then 0 copy numpy array conversions - will be enabled for certain `numpy.dtypes`s. This will allow for 0 copy conversions from `numpy` + 0 copy numpy array conversions are be enabled for certain `numpy.dtypes`s. + This will allow for 0 copy conversions from `numpy` arrays of a corresponding `numpy.dtype` to `pykx.NumericVector`s, and `pykx.TimespanVector`s. You can find the corresponding `numpy.dtype` of a `pykx.Vector` type by checking the `pykx.Vector._np_dtype` property of the type you want to create. @@ -1501,6 +1500,8 @@ def from_numpy_ndarray(x: np.ndarray, mask = None if handle_nulls: + if not x.flags.writeable: + x = x.copy() mask = (x != NULL_INT64) x[~mask] = NULL_INT32 @@ -1579,10 +1580,14 @@ def _to_numpy_or_categorical(x, col_name=None, df=None): if isinstance(x.values, pd.Categorical): return from_pandas_categorical( x.values, - name=col_name if pandas_2 and col_name is not None else x.name + name=col_name if pandas_gt1 and col_name is not None else x.name ) elif isinstance(x.values, pd.core.arrays.ExtensionArray): - if x.dtype.kind != 'f' and hasattr(x, 'isnull') and x.isnull().values.any(): + attrnull = hasattr(x, 'isnull') + hasnull = x.isnull().values.any() if attrnull else False + if isinstance(x.dtype, pd.StringDtype) and attrnull and hasnull: + return np.array(x.to_numpy(copy=False, na_value=None)) + elif x.dtype.kind != 'f' and attrnull and hasnull: if not x.dtype.kind in ['i', 'M', 'm']: raise TypeError( 'Non-integral masked array conversions to q are not yet implemented' @@ -1596,7 +1601,7 @@ def _to_numpy_or_categorical(x, col_name=None, df=None): if k_allocator: return np.array(x.to_numpy(copy=False, na_value=_size_to_nan[x.dtype.itemsize], dtype=dtype)) return x.to_numpy(copy=False, na_value=_size_to_nan[x.dtype.itemsize], dtype=dtype) - elif x.dtype.kind == 'f' and hasattr(x, 'isnull') and x.isnull().values.any(): + elif x.dtype.kind == 'f' and attrnull and hasnull: float_class = _float_size_to_class[x.dtype.itemsize] if k_allocator: return np.array(x.to_numpy(copy=False, dtype=float_class)) @@ -1660,7 +1665,7 @@ def from_pandas_dataframe(x: pd.DataFrame, """ cdef core.K kx - if not x.columns.values.dtype == np.object_: + if not isinstance(x.columns.values.dtype, (np.dtypes.ObjectDType, pd.StringDtype)): col_names = ['x'] cols_length = len(x.columns)-1 for i in range(cols_length): @@ -1747,7 +1752,7 @@ def from_pandas_series(x: pd.Series, return arr -if not pandas_2: +if not pandas_gt1: _supported_pandas_index_types_via_numpy = ( pd.core.indexes.base.Index, pd.core.indexes.numeric.NumericIndex, @@ -1948,7 +1953,7 @@ def from_arrow(x: Union['pa.Array', 'pa.Table'], strings_as_char: bool = False, no_allocator: bool = False, ) -> Union[k.Vector, k.Table]: - """Converts PyArrow arrays/tables into PyKX vectors/tables, respectively. + """Converts PyArrow arrays/tables into KDB-X Python vectors/tables, respectively. Conversions from PyArrow to q are performed by converting the PyArrow array/table to pandas first, which avoids copying data when possible, then converting the resulting Pandas @@ -1996,7 +2001,7 @@ def from_arrow_py(x, strings_as_char: bool = False, no_allocator: bool = False, ) -> Union[k.Vector, k.Table]: - """Converts PyArrow scalars into PyKX objects. + """Converts PyArrow scalars into KDB-X Python objects. Conversions from PyArrow to q are performed by converting the PyArrow object to python first. @@ -2548,7 +2553,7 @@ def from_ellipsis(x: Ellipsis, If you aren't sure you know what you are doing, then you probably don't need to use a projection null. - PyKX uses Python's `...` singleton to represent the q projection null, which is similar to + KDB-X Python uses Python's `...` singleton to represent the q projection null, which is similar to generic null (see [`from_none`][pykx.toq.from_none]), but indicates to q not just that there is missing data, but that it should be filled in with the next q object that is applied onto it. @@ -2557,7 +2562,7 @@ def from_ellipsis(x: Ellipsis, the q function projection `{x+y}[;7]` has an argument list of 2 q objects: a projection null, followed by a long atom with a value of 7. - Because PyKX treats `...` as projection null, q projections can be created in Python like so: + Because KDB-X Python treats `...` as projection null, q projections can be created in Python like so: ```python >>> f = pykx.K(lambda x, y: x + y) @@ -2669,7 +2674,7 @@ def from_callable(x: Callable, `*args` parameters and a `**kwargs` each count as a single parameter towards this limit. Note: Inspection of the callable must be possible. - In order to determine how many parameters a provided callable has, PyKX calls + In order to determine how many parameters a provided callable has, KDB-X Python calls `inspect.signature` on it. Some callables may not be introspectable in certain implementations of Python. For example, in CPython, some built-in functions defined in C provide no metadata about their arguments. @@ -2933,7 +2938,7 @@ _converter_from_python_type = { } -if not pandas_2: +if not pandas_gt1: _converter_from_python_type[pd.core.indexes.numeric.Int64Index] = from_pandas_index _converter_from_python_type[pd.core.indexes.numeric.Float64Index] = from_pandas_index diff --git a/src/pykx/util.py b/src/pykx/util.py index 8d4560b..b9f6a7d 100644 --- a/src/pykx/util.py +++ b/src/pykx/util.py @@ -16,17 +16,19 @@ import pandas as pd from pandas.core.internals import BlockManager, make_block +import numpy as np import requests import toml from .config import ( - _executable, _get_qexecutable, _get_qhome, _pykx_config_location, _pykx_profile_content, - allocator, beta_features, ignore_qhome, jupyterq, k_gc, keep_local_times, - load_pyarrow_unsafe, max_error_length, no_pykx_signal, no_qce, pykx_4_1, - pykx_config_location, pykx_config_profile, pykx_debug_insights, pykx_dir, pykx_lib_dir, - pykx_qdebug, pykx_threading, q_executable, qargs, qhome, qlic, release_gil, + _executable, _get_qexecutable, _get_qhome, + beta_features, ignore_qhome, jupyterq, k_allocator, k_gc, keep_local_times, + load_pyarrow_unsafe, max_error_length, no_pykx_signal, numpy_gt1, pandas_gt2, pykx_config_location, # noqa E501 + pykx_config_profile, pykx_debug_insights, pykx_dir, pykx_lib_dir, pykx_profile_content, + pykx_qdebug, pykx_threading, q_executable, qargs, qce, qhome, qlic, release_gil, skip_under_q, suppress_warnings, use_q_lock) from ._version import version as __version__ + from .exceptions import PyKXException from .reimporter import PyKXReimport @@ -250,7 +252,7 @@ def __reduce__(self): # return (BlockManager, super().__reduce__()[1]) # Manually reassigning the class attribute to the pandas BlockManager fixes this function # in those situations. - c = self.copy() + c = self.copy(deep=False) c.__class__ = BlockManager return c.__reduce__() @@ -263,23 +265,28 @@ def _consolidate(self): # nocov def df_from_arrays(columns, arrays, index): """Create a DataFrame from Numpy arrays without copying.""" - blocks = tuple( - make_block(values=a.reshape((1, len(a))), placement=(i,)) - for i, a in enumerate(arrays) - ) - return pd.DataFrame( - BlockManagerUnconsolidated(axes=[columns, index], blocks=blocks), - copy=False - ) - - # TODO: The following code probably ought to work, but some parts of Pandas still rely on the - # assumption that a BlockManager is being used. Next time we update the minimum Pandas version - # required by PyKX we should check if we can use an ArrayManager (KXI-9722). - - # return pd.DataFrame( - # ArrayManager(arrays=arrays, axes=[index, columns]), - # copy=False - # ) + if pandas_gt2 and numpy_gt1: + stringColumns = [i for i, a in enumerate(arrays) if + isinstance(a.dtype, np.dtypes.StringDType)] + else: + stringColumns = [] + if pandas_gt2 and any(type(a).__name__ == 'PandasUUIDArray' for a in arrays): + df = pd.DataFrame(index=index) + for col, arr in zip(columns, arrays): + df[col] = arr + else: + blocks = tuple( + make_block(values=a.reshape((1, len(a))), placement=(i,)) + for i, a in enumerate(arrays) + ) + df= pd.DataFrame( + BlockManagerUnconsolidated(axes=[columns, index], blocks=blocks), + copy=False + ) + if stringColumns and pandas_gt2: + for col in df.columns[stringColumns]: + df[col] = df[col].astype("str") + return df def get_default_args(f: Callable) -> Dict[str, Any]: @@ -317,7 +324,7 @@ def debug_environment(detailed: bool = False, return_info: bool = False) -> Unio ```python >>> import pykx as kx >>> kx.util.debug_environment() - **** PyKX information **** + **** KDB-X Python information **** pykx.args: () pykx.qhome: /usr/local/anaconda3/envs/qenv/q pykx.qlic: /usr/local/anaconda3/envs/qenv/q @@ -337,26 +344,25 @@ def debug_environment(detailed: bool = False, return_info: bool = False) -> Unio **** Platform information **** platform.platform: macOS-13.0.1-x86_64-i386-64bit - **** PyKX Configuration File **** - File location: /usr/local/.pykx-config + **** KDB-X Python Configuration File **** + File location: /usr/local/.kx/config-pykx Used profile: default Profile content: {'PYKX_Q_EXECUTABLE': '/usr/local/anaconda3/envs/qenv/q/m64/q'} - **** PyKX Configuration Variables **** + **** KDB-X Python Configuration Variables **** PYKX_IGNORE_QHOME: False PYKX_KEEP_LOCAL_TIMES: False - PYKX_ALLOCATOR: False + PYKX_NO_ALLOCATOR: False PYKX_GC: False PYKX_LOAD_PYARROW_UNSAFE: False PYKX_MAX_ERROR_LENGTH: 256 - PYKX_NOQCE: False + PYKX_QCE: False PYKX_RELEASE_GIL: False PYKX_Q_LIB_LOCATION: /Library/Versions/3.12/lib/python3.12/site-packages/pykx/lib PYKX_Q_LOCK: False PYKX_SKIP_UNDERQ: False PYKX_Q_EXECUTABLE: /usr/local/anaconda3/envs/qenv/q/m64/q PYKX_THREADING: False - PYKX_4_1_ENABLED: False PYKX_QDEBUG: False PYKX_DEBUG_INSIGHTS_LIBRARIES: False PYKX_CONFIGURATION_LOCATION: . @@ -374,13 +380,13 @@ def debug_environment(detailed: bool = False, return_info: bool = False) -> Unio PYKX_USE_FIND_LIBPYTHON: PYKX_UNLICENSED: PYKX_LICENSED: - PYKX_4_1_ENABLED: **** q Environment Variables **** QARGS: QHOME: /Library/Versions/3.12/lib/python3.12/site-packages/pykx/lib QLIC: /usr/local/anaconda3/envs/qenv/q QINIT: + QCFG: **** License information **** pykx.qlic directory: True @@ -415,7 +421,7 @@ def debug_environment(detailed: bool = False, return_info: bool = False) -> Unio def pykx_information(): from .core import _is_licensed - pykx_info = "**** PyKX information ****\n" + pykx_info = "**** KDB-X Python information ****\n" pykx_info += f"pykx.args: {qargs}\n" pykx_info += f"pykx.qhome: {qhome}\n" pykx_info += f"pykx.qlic: {qlic}\n" @@ -456,24 +462,25 @@ def platform_information(): def config_information(): - config_info = '\n**** PyKX Configuration File ****\n' - config_info += f"File location: {_pykx_config_location}\n" + config_info = '\n**** KDB-X Python Configuration File ****\n' + config_info += f"File location: {pykx_config_location}\n" config_info += f"Used profile: {pykx_config_profile}\n" - config_info += f"Profile content: {_pykx_profile_content}\n" + config_info += f"Profile content: {pykx_profile_content}\n" return config_info def env_information(): - env_info = '\n**** PyKX Configuration Variables ****\n' + env_info = '\n**** KDB-X Python Configuration Variables ****\n' global_config = {'PYKX_IGNORE_QHOME': ignore_qhome, 'PYKX_KEEP_LOCAL_TIMES': keep_local_times, - 'PYKX_ALLOCATOR': allocator, 'PYKX_GC': k_gc, + 'PYKX_NO_ALLOCATOR': k_allocator, 'PYKX_GC': k_gc, 'PYKX_LOAD_PYARROW_UNSAFE': load_pyarrow_unsafe, - 'PYKX_MAX_ERROR_LENGTH': max_error_length, 'PYKX_NOQCE': no_qce, + 'PYKX_MAX_ERROR_LENGTH': max_error_length, + 'PYKX_QCE': qce, 'PYKX_RELEASE_GIL': release_gil, 'PYKX_Q_LIB_LOCATION': pykx_lib_dir, 'PYKX_Q_LOCK': use_q_lock, 'PYKX_SKIP_UNDERQ': skip_under_q, 'PYKX_Q_EXECUTABLE': q_executable, 'PYKX_THREADING': pykx_threading, - 'PYKX_4_1_ENABLED': pykx_4_1, 'PYKX_QDEBUG': pykx_qdebug, + 'PYKX_QDEBUG': pykx_qdebug, 'PYKX_DEBUG_INSIGHTS_LIBRARIES': pykx_debug_insights, 'PYKX_CONFIGURATION_LOCATION': pykx_config_location, 'PYKX_NO_SIGNAL': no_pykx_signal, @@ -484,8 +491,7 @@ def env_information(): env_only = ['PYKX_DEFAULT_CONVERSION', 'PYKX_EXECUTABLE', 'PYKX_PYTHON_LIB_PATH', 'PYKX_PYTHON_BASE_PATH', 'PYKX_PYTHON_HOME_PATH', 'PYKX_DIR', - 'PYKX_USE_FIND_LIBPYTHON', 'PYKX_UNLICENSED', 'PYKX_LICENSED', - 'PYKX_4_1_ENABLED' + 'PYKX_USE_FIND_LIBPYTHON', 'PYKX_UNLICENSED', 'PYKX_LICENSED' ] for k, v in global_config.items(): @@ -495,7 +501,7 @@ def env_information(): env_info += f"{x}: {os.getenv(x, '')}\n" env_info += '\n**** q Environment Variables ****\n' - qenv = ['QARGS', 'QHOME', 'QLIC', 'QINIT'] + qenv = ['QARGS', 'QHOME', 'QLIC', 'QINIT', 'QCFG'] for x in qenv: env_info += f"{x}: {os.getenv(x, '')}\n" @@ -557,7 +563,7 @@ def pykx_startup_information(): sec_threads = q('string system"s"') pykx_start_info += f"secondary threads: {sec_threads}\n" else: - pykx_start_info += "Gathering PyKX startup information only available in licensed mode\n" + pykx_start_info += "Gathering KDB-X Python startup information only available in licensed mode\n" # noqa: E501 return pykx_start_info @@ -583,32 +589,33 @@ def qfirst_modify(state): ipython = get_ipython() if _run_all_cell_with_magics in ipython.input_transformers_cleanup and state == "python": ipython.input_transformers_cleanup.remove(_run_all_cell_with_magics) - print("""PyKX now running in 'python' mode (default). All cells by default will be run as python code. -Include '%%q' at the beginning of each cell to run as q code. """) # noqa + print("""KDB-X Python now running in 'python' mode (default). All cells by default will be run as python code. +Include '%%q' at the beginning of each cell to run as q code. """) # noqa elif _run_all_cell_with_magics not in ipython.input_transformers_cleanup and state == "q": ipython.input_transformers_cleanup.append(_run_all_cell_with_magics) - print("""PyKX now running in 'jupyter_qfirst' mode. All cells by default will be run as q code. -Include '%%py' at the beginning of each cell to run as python code. """) # noqa + print("""KDB-X Python now running in 'jupyter_qfirst' mode. All cells by default will be run as q code. +Include '%%py' at the beginning of each cell to run as python code. """) # noqa else: - print(f"PyKX already running in '{state}' mode") + print(f"KDB-X Python already running in '{state}' mode") except NameError: print("Not running under IPython/Jupyter") -def add_to_config(config, folder='~'): +def add_to_config(config, file=pykx_config_location, profile=pykx_config_profile): """ - Add configuration options to the file '.pykx-config' in a specified folder + Add configuration options to the file 'config-pykx' in a specified folder Parameters: config: A dictionary mapping the configuration options to their associated value - folder: The folder where the users '.pykx-config' file is to be updated + file: The file a user wishes to use. Default inherits from kx.config.pykx_config_location + profile: The profile to store in. Default inherits from kx.config.pykx_config_profile Examples: ```python >>> import pykx as kx >>> kx.util.add_to_config({'PYKX_GC': 'True', 'PYKX_BETA_FEATURES': 'True'}) - Configuration updated at: /usr/local/.pykx-config. + Configuration updated at: /usr/local/.kx/config-pykx. Profile updated: default. Successfully added: - PYKX_GC = True @@ -617,28 +624,30 @@ def add_to_config(config, folder='~'): """ if not isinstance(config, dict): raise TypeError(f'Supplied config must be of type dict, supplied type: {type(config)}') - fpath = str(Path(os.path.expanduser(folder)) / '.pykx-config') + if file is None: + file = str(Path.home()/'.kx/config-pykx') try: - os.access(fpath, os.W_OK) + os.access(file, os.W_OK) except FileNotFoundError: pass except PermissionError: - raise PermissionError(f"You do not have sufficient permissions to write to: {fpath}") - print_config = f"\nConfiguration updated at: {fpath}.\nProfile updated: "\ - f"{pykx_config_profile}.\nSuccessfully added:\n" - if os.path.exists(fpath): - with open(fpath, 'r') as file: - data = toml.load(file) + raise PermissionError(f"You do not have sufficient permissions to write to: {file}") + file = os.path.abspath(os.path.expanduser(file)) + print_config = f"\nConfiguration updated at: {file}.\nProfile updated: "\ + f"{profile}.\nSuccessfully added:\n" + if os.path.exists(file): + with open(file, 'r') as f: + data = toml.load(f) else: - data = {pykx_config_profile: {}} + data = {profile: {}} + os.makedirs(os.path.dirname(file), exist_ok=True) for k, v in config.items(): - data[pykx_config_profile][k] = v + data[profile][k] = v print_config += f'\t- {k} = {v}\n' if isinstance(v, (int, bool)): v = str(v) - os.environ[k] = v - with open(fpath, 'w') as file: - toml.dump(data, file) + with open(file, 'w') as f: + toml.dump(data, f) print(print_config) @@ -650,7 +659,9 @@ def add_to_config(config, folder='~'): def install_q(location: str = '~/q', - date: str = '2024.07.08'): + date: str = '2024.07.08', + config_file=pykx_config_location, + config_profile=pykx_config_profile): """ Install q to a specified location. @@ -676,11 +687,10 @@ def install_q(location: str = '~/q', except BaseException: raise RuntimeError(f"Unable to set execute permissions on file: {executable_loc}") shutil.copy(pykx_dir/'pykx.q', location/'pykx.q') - shutil.copy(pykx_dir/'lib/s.k_', location/'s.k_') add_to_config({ 'PYKX_Q_EXECUTABLE': str(executable_loc), - 'QHOME': str(location)}) + 'QHOME': str(location)}, file=config_file, profile=config_profile) print('Please restart your process to use this executable.') diff --git a/src/pykx/wrappers.py b/src/pykx/wrappers.py index bd1020d..fd1cb4e 100644 --- a/src/pykx/wrappers.py +++ b/src/pykx/wrappers.py @@ -1,8 +1,8 @@ """ Wrappers for q data structures, with conversion functions to Python/Numpy/Pandas/Arrow. -Under PyKX, q has its own memory space in which it stores q data structures in the same way it is -stored within a regular q process. PyKX provides Pythonic wrappers around these objects in q +Under Python, q has its own memory space in which it stores q data structures in the same way it is +stored within a regular q process. KDB-X Python provides Pythonic wrappers around these objects in q memory. In general, these wrappers consist of a pointer to a location in q memory, and a collection @@ -183,7 +183,7 @@ from . import _wrappers, beta_features, help from ._pyarrow import pyarrow as pa -from .config import _check_beta, k_gc, licensed, pandas_2, suppress_warnings +from .config import _check_beta, k_gc, licensed, numpy_gt1, pandas_gt1, pandas_gt2, suppress_warnings # noqa E501 from .core import keval as _keval from .constants import INF_INT16, INF_INT32, INF_INT64, INF_NEG_INT16, INF_NEG_INT32, INF_NEG_INT64 from .constants import NULL_INT16, NULL_INT32, NULL_INT64 @@ -910,7 +910,7 @@ class DatetimeAtom(TemporalFixedAtom): """Wrapper for q datetime atoms. Warning: The q datetime type is deprecated. - PyKX does not provide a rich interface for the q datetime type, as it is deprecated. Avoid + `pykx` does not provide a rich interface for the q datetime type, as it is deprecated. Avoid using it whenever possible. """ t = -15 @@ -1790,7 +1790,7 @@ def __getitem__(self, key): return q('@', self, _idx_to_k(key, _wrappers.k_n(self))) -if pandas_2 and pa is not None: +if pandas_gt1 and pa is not None: _as_arrow_map = { 'List': 'object', 'BooleanVector': 'bool[pyarrow]', @@ -1906,7 +1906,7 @@ def pd( if 0 != len(null_inds): res[null_inds] = pd.NA if as_arrow: - if not pandas_2: + if not pandas_gt1: raise RuntimeError('Pandas Version must be at least 2.0 to use as_arrow=True') if pa is None: raise PyArrowUnavailable # nocov @@ -1989,6 +1989,9 @@ def mean(self): def median(self): return q.med(self) + def fills(self): + return q.fills(self) + def mode(self): return q('{where max[c]=c:count each d:group x}', self) @@ -1996,7 +1999,7 @@ def append(self, data): """Append object to the end of a vector. Parameters: - self: PyKX Vector/List object + self: `pykx` Vector/List object data: Data to be used when appending to a list/vector, when appending to a typed list this must be an object with a type which converts to an equivalent vector type. @@ -2071,9 +2074,9 @@ def extend(self, data): """Extend a vector by appending supplied values to the vector. Parameters: - self: PyKX Vector/List object + self: KDB-X Python Vector/List object data: Data to be used when extending the a list/vector, this can - be data of any type which can be converted to a PyKX object. + be data of any type which can be converted to a KDB-X Python object. Raises: PyKXException: When dealing with typed vectors extending this vector @@ -2271,7 +2274,7 @@ def __ufunc_output__(self, res, out, dtype): def __array_function__(self, func, types, args, kwargs): if not suppress_warnings: warn('Warning: Attempting to call numpy __array_function__ on a ' - f'PyKX Vector type. __array_function__: {func}. Support for this method ' + f'KDB-X Python Vector type. __array_function__: {func}. Support for this method ' 'is on a best effort basis. To suppress this warning please set the ' 'configuration/environment variable PYKX_SUPPRESS_WARNINGS=True') a = [] @@ -2367,7 +2370,7 @@ class List(Vector): vector. By contrast, q lists are a a vector of pointers to K objects, so they are structured in-memory as a K object containing metadata, followed immediately by pointers. As a result, the base data "contained" by the list is located elsewhere in memory. This has performance - and ownership implications in q, which carry over to PyKX. + and ownership implications in q, which carry over to KDB-X Python. """ t = 0 _np_dtype = object @@ -2455,7 +2458,7 @@ def pd( as_arrow: Optional[bool] = False, ): if as_arrow: - if not pandas_2: + if not pandas_gt1: raise RuntimeError('Pandas Version must be at least 2.0 to use as_arrow=True') if pa is None: raise PyArrowUnavailable # nocov @@ -2546,7 +2549,7 @@ def _from_sequence(cls, seq): @classmethod def _from_factorized(cls, fac): - raise NotImplementedError(f"{__class__.__name__}.{inspect.stack()[0][3]}() is not implemented. This function doesn't support PyKX UUID objects") # noqa: E501 + raise NotImplementedError(f"{__class__.__name__}.{inspect.stack()[0][3]}() is not implemented. This function doesn't support KDB-X Python UUID objects") # noqa: E501 def __getitem__(self, key): return self.array[key] @@ -2577,7 +2580,7 @@ def copy(self): return PandasUUIDArray(np.array(self).copy()) def _concat_same_type(self): - raise NotImplementedError(f"{__class__.__name__}.{inspect.stack()[0][3]}() is not implemented. This function doesn't support PyKX UUID objects") # noqa: E501 + raise NotImplementedError(f"{__class__.__name__}.{inspect.stack()[0][3]}() is not implemented. This function doesn't support KDB-X Python UUID objects") # noqa: E501 if pa is not None: @@ -2981,8 +2984,8 @@ class DatetimeVector(TemporalFixedVector): """Wrapper for q datetime vectors. Warning: The q datetime type is deprecated. - PyKX does not provide a rich interface for the q datetime type, as it is depreceated. Avoid - using it whenever possible. + KDB-X Python does not provide a rich interface for the q datetime type, + as it is depreceated. Avoid using it whenever possible. """ t = 15 _np_dtype = np.float64 @@ -3110,7 +3113,7 @@ def pd( if raw: res = super().pd(raw=raw, has_nulls=has_nulls) if as_arrow: - if not pandas_2: + if not pandas_gt1: raise RuntimeError('Pandas Version must be at least 2.0 to use as_arrow=True') if pa is None: raise PyArrowUnavailable # nocov @@ -3221,6 +3224,15 @@ def _col_name_generator(): yield name + str(i) +def _to_np(vec, raw, has_nulls, reshape=False): + arr = vec.np(raw=raw, has_nulls=has_nulls) + if reshape: + arr = arr.reshape(-1) + if pandas_gt2 and numpy_gt1 and isinstance(vec, SymbolVector): + arr = np.array(arr, np.dtypes.StringDType) + return arr + + class Table(PandasAPI, Mapping): """Wrapper for q tables, including in-memory tables, splayed tables, and partitioned tables. @@ -3306,11 +3318,11 @@ def pd( as_arrow: Optional[bool] = False, ): if raw_guids and not raw: - v = [x.np(raw=isinstance(x, GUIDVector), has_nulls=has_nulls) for x in self._values] + v = [_to_np(x, isinstance(x, GUIDVector), has_nulls) for x in self._values] v = [PandasUUIDArray(x) if x.dtype == complex else x for x in v] else: - v = [x.np(raw=raw, has_nulls=has_nulls) for x in self._values] - if pandas_2: + v = [_to_np(x, raw, has_nulls) for x in self._values] + if pandas_gt1: # The current behavior is a bug and will raise an error in the future, this change # proactively fixes that for us for i in range(len(v)): @@ -3326,7 +3338,7 @@ def pd( _pykx_base_types[self._keys.py()[i]] = str(type(v).__name__) df.attrs['_PyKX_base_types'] = _pykx_base_types if as_arrow: - if not pandas_2: + if not pandas_gt1: raise RuntimeError('Pandas Version must be at least 2.0 to use as_arrow=True') if pa is None: raise PyArrowUnavailable # nocov @@ -3453,7 +3465,7 @@ def sorted(self, cols: Union[List, str] = ''): raise e def sql(self, query, *args): - """Execute an SQL query against the supplied PyKX tabular object. + """Execute an SQL query against the supplied KDB-X Python tabular object. This function expects the table object to be supplied as a parameter to the query, additional parameters can be supplied as positional @@ -3561,7 +3573,7 @@ def select(self, columns=None, where=None, by=None, inplace=False): def exec(self, columns=None, where=None, by=None): """ - Apply a q style exec statement on the supplied PyKX Table. + Apply a q style exec statement on the supplied KDB-X Python Table. This implementation follows the q functional exec syntax with limitations on structures supported for the various clauses a result of this. @@ -3577,7 +3589,7 @@ def exec(self, columns=None, where=None, by=None): Examples: - Define a PyKX Table + Define a KDB-X Python Table ```python >>> qtab = kx.Table(data = { @@ -3700,7 +3712,7 @@ def update(self, columns=None, where=None, by=None, inplace=False): def delete(self, columns=None, where=None, inplace=False): """ - Apply a q style delete statement a PyKX table defined. + Apply a q style delete statement a KDB-X Python table defined. This implementation follows the q functional delete syntax with limitations on structures supported for the various clauses a result of this. @@ -3715,7 +3727,7 @@ def delete(self, columns=None, where=None, inplace=False): Examples: - Define a PyKX Table against which to run the examples + Define a KDB-X Python Table against which to run the examples ```python >>> qtab = kx.Table(data = { @@ -4290,6 +4302,14 @@ def xbar(self, values): raise AttributeError("Operation 'xbar' not supported for PartitionedTable type") +class VirtualTable(Atom): + """Wrapper for q virtual tables.""" + t = 112 + + select = Table.select + exec = Table.exec + + class Dictionary(Mapping): """Wrapper for q dictionaries, including regular dictionaries, and keyed tables.""" t = 99 @@ -4320,7 +4340,7 @@ def __getitem__(self, key): def __setitem__(self, key, val): if isinstance(key, tuple): raise NotImplementedError(f"{__class__.__name__}.{inspect.stack()[0][3]}() is not implemented, .pykx.Dictionary objects do not support tuple key assignment") # noqa: E501 - self.__dict__.update(q('{x,((),y)!((),z)}', self, key, val).__dict__) + self.__dict__.update(q('{ye:(),y;x,ye!$[y~ye;z;enlist z]}', self, key, val).__dict__) def _repr_html_(self): if not licensed: @@ -4514,16 +4534,14 @@ def pd( df = pd.DataFrame(columns=kk.py() + vk.py()) df = df.set_index(kk.py()) if as_arrow: - if not pandas_2: + if not pandas_gt1: raise RuntimeError('Pandas Version must be at least 2.0 to use as_arrow=True') if pa is None: raise PyArrowUnavailable # nocov df = df.convert_dtypes(dtype_backend='pyarrow') return df - idx = [kvg(i).np(raw=raw, has_nulls=has_nulls).reshape(-1) - for i in range(len(kk))] - cols = [vvg(i).np(raw=raw, has_nulls=has_nulls) - for i in range(len(vk))] + idx = [_to_np(kvg(i), raw, has_nulls, reshape=True) for i in range(len(kk))] + cols = [_to_np(vvg(i), raw, has_nulls) for i in range(len(vk))] column_names = pd.Index(kk.py() + vk.py()) columns = idx + cols index = pd.Index(np.arange(len(self))) @@ -4539,7 +4557,7 @@ def pd( df[col] = df[col].astype('category') _pykx_base_types[col] = str(type(vvg(i)).__name__) if as_arrow: - if not pandas_2: + if not pandas_gt1: raise RuntimeError('Pandas Version must be at least 2.0 to use as_arrow=True') if pa is None: raise PyArrowUnavailable # nocov @@ -4725,7 +4743,7 @@ def grouped(self, cols: Union[List, str] = ''): raise e def sql(self, query, *args): - """Execute an SQL query against the supplied PyKX KeyedTable object. + """Execute an SQL query against the supplied KDB-X Python KeyedTable object. This function expects the keyed table object to be supplied as a parameter to the query, additional parameters can be supplied as positional @@ -4838,7 +4856,7 @@ def select(self, columns=None, where=None, by=None, inplace=False): def exec(self, columns=None, where=None, by=None): """ - Apply a q style exec statement on the supplied PyKX KeyedTable. + Apply a q style exec statement on the supplied KDB-X Python KeyedTable. This implementation follows the q functional exec syntax with limitations on structures supported for the various clauses a result of this. @@ -4854,7 +4872,7 @@ def exec(self, columns=None, where=None, by=None): Examples: - Define a PyKX KeyedTable + Define a KDB-X Python KeyedTable ```python >>> qtab = kx.Table(data = { @@ -4909,7 +4927,7 @@ def exec(self, columns=None, where=None, by=None): def update(self, columns=None, where=None, by=None, inplace=False): """ - Apply a q style update statement on a PyKX KeyedTable. + Apply a q style update statement on a KDB-X Python KeyedTable. This implementation follows the q functional update syntax with limitations on structures supported for the various clauses a result of this. @@ -4978,7 +4996,7 @@ def update(self, columns=None, where=None, by=None, inplace=False): def delete(self, columns=None, where=None, inplace=False): """ - Apply a q style delete statement a PyKX keyed table defined. + Apply a q style delete statement a KDB-X Python keyed table defined. This implementation follows the q functional delete syntax with limitations on structures supported for the various clauses a result of this. @@ -4993,7 +5011,7 @@ def delete(self, columns=None, where=None, inplace=False): Examples: - Define a PyKX Table against which to run the examples + Define a KDB-X Python Table against which to run the examples ```python >>> qtab = kx.Table(data = { @@ -5197,9 +5215,6 @@ def each_left(self): vs = each_left -Function.scan.__doc__ = help.qhelp('scan') - - class Lambda(Function): """Wrapper for q lambda functions. @@ -5410,7 +5425,7 @@ class Projection(Function): If the original function had `n` parameters, and it had `m` of them provided, the result would be a function (projection) that has `m` parameters. - In PyKX, the special Python singleton `...` is used to represent + In KDB-X Python, the special Python singleton `...` is used to represent [projection null][pykx.ProjectionNull] """ t = 104 @@ -5894,6 +5909,10 @@ def __init__(self, column=None, name=None, data=None, is_tree=False, value=None) warn("The kx.Column 'value' keyword is deprecated, please use 'data' instead", DeprecationWarning) elif data is not None: + if isinstance(data, (SymbolAtom, SymbolVector, str)): + data = [data] + elif isinstance(data, list) and all(isinstance(x, str) for x in data): + data = [data] self._data = data self._renamed = True else: @@ -5909,8 +5928,45 @@ def __repr__(self): preamble = f'pykx.{type(self).__name__}' return f"{preamble}(name='{self._name}', data={type(self._data)})" - """Function for building up a function call off a Column""" + def __getitem__(self, key): + res = copy.deepcopy(self) + if isinstance(key, (SymbolAtom, SymbolVector, str)): + key = [key] + elif isinstance(key, list) and all(isinstance(x, str) for x in key): + key = [key] + res._data = [res._data, key] + return res + def call(self, op, *other, iterator=None, col_arg_ind=0, project_args=None): + """ + Used to makes function calls off a Column object. + + Parameters: + op: The function to apply. + *other: The other column(s) or variable(s) (Python/q) to be used + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Examples: + + Find the maximum values in nested lists. + + ```python + >>> t = kx.q('([] n:(1 2;4 3;10 10))') + >>> t.select(kx.Column('n').call('max', iterator='each')) + pykx.Table(pykx.q(' + n + -- + 2 + 4 + 10 + ')) + ``` + """ params = [] for param in other: if isinstance(param, Column): @@ -6164,7 +6220,7 @@ def _and(self, other, iterator=None, col_arg_ind=0, project_args=None): Use `&` or `[]` in general to build queries. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -6485,10 +6541,10 @@ def cor(self, other, iterator=None, col_arg_ind=0, project_args=None): - Another column - A vector of equal length to the column - - A PyKX variable in q memory + - A `pykx` variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -6629,10 +6685,10 @@ def cov(self, other, sample=False, iterator=None, col_arg_ind=0, project_args=No - Another column - A vector of equal length to the column - - A PyKX variable in q memory + - A `pykx` variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. sample: Should calculations of covariance return the sample covariance (set True) covariance (set False {default}) iterator: What iterator to use when operating on the column @@ -6691,10 +6747,10 @@ def cross(self, other, iterator=None, col_arg_ind=0, project_args=None): - Another column - A vector of items - - A PyKX variable in q memory + - A `pykx` variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7015,10 +7071,10 @@ def div(self, other, iterator=None, col_arg_ind=0, project_args=None): - Another column - An integer - A vector of items equal in length to the column - - A PyKX variable in q memory + - A `pykx` variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7464,10 +7520,10 @@ def inter(self, other, iterator=None, col_arg_ind=0, project_args=None): - Another column - A Python list/numpy array - - A PyKX variable in q memory + - A `pykx` variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7511,12 +7567,12 @@ def isin(self, other, iterator=None, col_arg_ind=0, project_args=None): - Column - Python list/numpy array - - A PyKX variable in q memory + - A `pykx` variable in q memory Most commonly this function is used in where clauses to filter data Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7619,7 +7675,7 @@ def like(self, other, iterator=None, col_arg_ind=0, project_args=None): clauses to filter data. Parameters: - other: A string/byte array defining a regex pattern to be used for query + other: A string/byte array defining a regex pattern to be used for query. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7767,7 +7823,7 @@ def mavg(self, other, iterator=None, col_arg_ind=1, project_args=None): The results are returned as a floating point. Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving average iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -7905,7 +7961,7 @@ def mcount(self, other, iterator=None, col_arg_ind=1, project_args=None): so far, thereafter the result is the moving average Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving count iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -7978,7 +8034,7 @@ def mdev(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving standard deviation Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving standard deviation iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8152,7 +8208,7 @@ def mmax(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving maximum Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving maximum iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8185,7 +8241,7 @@ def mmin(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving minimum Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving minimum iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8216,11 +8272,11 @@ def mod(self, other, iterator=None, col_arg_ind=0, project_args=None): Calculate the modulus of items in a column for a given value. Parameters: - other: An integer denoting the divisor to be used when calculating the modulus + other: An integer denoting the divisor to be used when calculating the modulus. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 1. + where the column parameter will be used. Default 0. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -8248,7 +8304,7 @@ def msum(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving sum Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving sum iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8341,7 +8397,7 @@ def _or(self, other, iterator=None, col_arg_ind=0, project_args=None): Return the larger of the underlying boolean values between two columns: Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -8617,7 +8673,7 @@ def rotate(self, other, iterator=None, col_arg_ind=1, project_args=None): by the parameter other. Parameters: - other: An integer denoting the number of elements left(positive) or right(negative) + other: An integer denoting the number of elements left(positive) or right(negative). which the column list will be shifted iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8932,7 +8988,7 @@ def sqrt(self, iterator=None): def string(self, iterator=None): """ - Convert all elements of a column to a PyKX string (CharVector) + Convert all elements of a column to a KDB-X Python string (CharVector) Parameters: iterator: What iterator to use when operating on the column @@ -9186,10 +9242,10 @@ def union(self, other, iterator=None, col_arg_ind=0, project_args=None): - Another column - A Python list/numpy array - - A PyKX variable in q memory + - A `pykx` variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9307,16 +9363,16 @@ def var(self, iterator=None, sample=False): fn = 'svar' if sample else 'var' return self.call(fn, iterator=iterator) - def wavg(self, other, iterator=None, col_arg_ind=0, project_args=None): + def wavg(self, weight, iterator=None, col_arg_ind=0, project_args=None): """ Return the weighted average between a column and: - Another column - A Python list/numpy array - - A PyKX variable in q memory + - A `pykx` variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + weight: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9351,7 +9407,7 @@ def wavg(self, other, iterator=None, col_arg_ind=0, project_args=None): pykx.FloatAtom(pykx.q('2.431111')) ``` """ - return self.call('wavg', other, iterator=iterator, col_arg_ind=col_arg_ind, + return self.call('wavg', weight, iterator=iterator, col_arg_ind=col_arg_ind, project_args=project_args) def within(self, lower, upper, iterator=None, col_arg_ind=0, project_args=None): @@ -9414,10 +9470,10 @@ def wsum(self, other, iterator=None, col_arg_ind=0, project_args=None): - Another column - A Python list/numpy array - - A PyKX variable in q memory + - A `pykx` variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9461,7 +9517,7 @@ def xbar(self, other, iterator=None, col_arg_ind=1, project_args=None): parameter other. Parameters: - other: An integer denoting the multiple to which all values will be rounded + other: An integer denoting the multiple to which all values will be rounded. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9519,7 +9575,7 @@ def xexp(self, other, iterator=None, col_arg_ind=1, project_args=None): the parameter other. Parameters: - other: An integer denoting the power to which all values will be raised + other: An integer denoting the power to which all values will be raised. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9559,7 +9615,7 @@ def xlog(self, other, iterator=None, col_arg_ind=1, project_args=None): by the parameter other. Parameters: - other: An integer denoting the logarithmic base to which all values will be set + other: An integer denoting the logarithmic base to which all values will be set. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9598,7 +9654,7 @@ def xprev(self, other, iterator=None, col_arg_ind=1, project_args=None): before it. Where N is specified by the parameter other. Parameters: - other: An integer denoting the number of indices before elements in the list + other: An integer denoting the number of indices before elements in the list. to retrieve the value of iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -9829,7 +9885,7 @@ def add(self, other, iterator=None, col_arg_ind=0, project_args=None): - Another column - A vector of equal length to the column - - A PyKX variable in q memory + - A `pykx` variable in q memory Note in it's most basic usage this is equivalent to @@ -9841,7 +9897,7 @@ def add(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9974,6 +10030,10 @@ def data(self, data): ``` """ cpy = copy.deepcopy(self) + if isinstance(data, (SymbolAtom, SymbolVector, str)): + data = [data] + elif isinstance(data, list) and all(isinstance(x, str) for x in data): + data = [data] cpy._data = data cpy._renamed=True return cpy @@ -10026,22 +10086,22 @@ def average(self, iterator=None): def cast(self, other, iterator=None, col_arg_ind=1, project_args=None): """ - Convert the content of a column to another PyKX type + Convert the content of a column to another KDB-X Python type Parameters: - other: The name of the type to which your column should be cast + other: The name of the type to which your column should be cast. or the lower case letter used to define it in q, for more information see [here](https://code.kx.com/q/ref/cast/). iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. Example: - Cast a column containing PyKX long objects to float objects + Cast a column containing KDB-X Python long objects to float objects ```python >>> import pykx as kx @@ -10071,6 +10131,8 @@ def cast(self, other, iterator=None, col_arg_ind=1, project_args=None): """ if not isinstance(other, str): raise QError('Supplied value other must be a str') + if other == 'symbol': + other = '' if 1 == len(other): other = other.encode('UTF-8') return self.call('$', other, iterator=iterator, col_arg_ind=col_arg_ind, @@ -10082,10 +10144,10 @@ def correlation(self, other, iterator=None, col_arg_ind=0, project_args=None): - Another column - A vector of equal length to the column - - A PyKX variable in q memory + - A `pykx` variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10142,10 +10204,10 @@ def covariance(self, other, sample=False, iterator=None, col_arg_ind=0, project_ - Another column - A vector of equal length to the column - - A PyKX variable in q memory + - A `pykx` variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. sample: Should calculations of covariance return the sample covariance (set True) covariance (set False {default}) iterator: What iterator to use when operating on the column @@ -10204,7 +10266,7 @@ def divide(self, other, iterator=None, col_arg_ind=0, project_args=None): - Another column - Python/Numpy list/item - - A PyKX variable in q memory + - A `pykx` variable in q memory Note in it's most basic usage this is equivalent to @@ -10216,7 +10278,7 @@ def divide(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10296,11 +10358,11 @@ def drop(self, other, iterator=None, col_arg_ind=1, project_args=None): an iterator. Where N is specified by the other parameter. Parameters: - other: An integer defining the number of elements to drop + other: An integer defining the number of elements to drop. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -10334,11 +10396,11 @@ def fill(self, other, iterator=None, col_arg_ind=1, project_args=None): Replace all null values in a column with a specified 'other' parameter Parameters: - other: The value which should replace nulls within a column + other: The value which should replace nulls within a column. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -10427,7 +10489,7 @@ def join(self, other, iterator=None, col_arg_ind=0, project_args=None): - A variable in q memory Parameters: - other: The Column, list, item or variable to be joined to the + other: The Column, list, item or variable to be joined to the. original column iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -10514,7 +10576,7 @@ def modulus(self, other, iterator=None, col_arg_ind=1, project_args=None): Calculate the modulus of items in a column for a given value. Parameters: - other: An integer denoting the divisor to be used when calculating the modulus + other: An integer denoting the divisor to be used when calculating the modulus. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10545,7 +10607,7 @@ def multiply(self, other, iterator=None, col_arg_ind=0, project_args=None): - Another column - Python/Numpy list - - A PyKX variable in q memory + - A `pykx` variable in q memory Note in it's most basic usage this is equivalent to @@ -10557,7 +10619,7 @@ def multiply(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10852,7 +10914,7 @@ def subtract(self, other, iterator=None, col_arg_ind=0, project_args=None): - The values of another column - Python/Numpy list/value - - A PyKX variable in q memory + - A `pykx` variable in q memory Note in it's most basic usage this is equivalent to @@ -10864,7 +10926,7 @@ def subtract(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10944,11 +11006,11 @@ def take(self, other, iterator=None, col_arg_ind=1, project_args=None): from a column using an iterator. Where N is specified by the other parameter. Parameters: - other: An integer defining the number of elements to retrieve + other: An integer defining the number of elements to retrieve. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -11072,10 +11134,416 @@ def bin(self, other, iterator=None, col_arg_ind=1, project_args=None): 0 1 ')) + ``` """ return self.call('bin', other, iterator=iterator, col_arg_ind=col_arg_ind, project_args=project_args) + def enlist(self, iterator=None): + """ + Return column values as a list. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 2 4)') + >>> t.exec(kx.Column('n').enlist(iterator='each')) + pykx.List(pykx.q(' + 2 + 4 + ')) + ``` + """ + return self.call('enlist', iterator=iterator) + + def all(self, iterator=None): + """ + Returns a boolean atom 1b if all items are nonzero, else retuns 0b. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 5#1b)') + >>> t.exec(kx.Column('n').all()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + """ + return self.call('all', iterator=iterator) + + def any(self, iterator=None): + """ + Returns a boolean atom 1b if at least one item is nonzero, else retuns 0b. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 0010b)') + >>> t.exec(kx.Column('n').any()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + """ + return self.call('any', iterator=iterator) + + def _in(self, other, iterator=None, col_arg_ind=0, project_args=None): + """ + Returns a boolean atom 1b if at least one item is nonzero, else retuns 0b. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:1 2 3 4 5)') + >>> t.select(kx.Column('n')._in([0,2,4], iterator='\\:')) + pykx.Table(pykx.q(' + n + - + 0 + 1 + 0 + 1 + 0 + ')) + ``` + """ + return self.call('in', other, iterator=iterator, col_arg_ind=col_arg_ind, + project_args=project_args) + + def _except(self, other, iterator=None, col_arg_ind=0, project_args=None): + """ + Returns a list of all items of column that are not items of `other`. + + Parameters: + other: A list or atom to be excepted from the column. + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: til 5)') + >>> t.exec(kx.Column('n')._except(3)) + pykx.LongVector(pykx.q('0 1 2 4')) + ``` + """ + return self.call('except', other, iterator=iterator, col_arg_ind=col_arg_ind, + project_args=project_args) + + def hsym(self, iterator=None): + """ + Returns a list of all items of column prefixed with a colon. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: `a`b`c)') + >>> t.exec(kx.Column('n').hsym()) + pykx.SymbolVector(pykx.q('`:a`:b`:c')) + ``` + """ + return self.call('hsym', iterator=iterator) + + def raze(self, iterator=None): + """ + Returns the collapsed items of the column. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').raze()) + pykx.List(pykx.q(' + 1 + 2 + 3 4 + 5 6 + 7 + 8 + ')) + ``` + + If you want to flatten all levels use the `over` or `\\` option for iterator. + ```python + >>> t.exec(kx.Column('n').raze(iterator='over')) + pykx.LongVector(pykx.q('1 2 3 4 5 6 7 8')) + ``` + """ + return self.call('raze', iterator=iterator) + + def type(self, iterator=None): + """ + Returns the type of the column. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').type(iterator='each')) + pykx.ShortVector(pykx.q('7 0 -7 -7h')) + ``` + """ + return self.call('type', iterator=iterator) + + def key(self, iterator=None): + """ + Where a column is made of dictionaries, return the keys. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> d1 = kx.q('`a`b`c ! 1 2 3') + >>> d2 = kx.q('`d`e`f ! 4 5 6') + >>> t = kx.Table(data={'n':[d1, d2]}) + >>> t.exec(kx.Column('n').type(iterator='each')) + pykx.Table(pykx.q(' + n + ----- + a b c + d e f + ')) + ``` + """ + return self.call('key', iterator=iterator) + + def get(self, iterator=None): + """ + Read value from a column of named global variables or file names. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> kx.q('`:tb1 set ([] n: `a`b`c)') + pykx.SymbolAtom(pykx.q('`:tb1')) + >>> kx.q('`:tb2 set ([] n: 1 2 3)') + pykx.SymbolAtom(pykx.q('`:tb2')) + >>> t = kx.q('([] n: `:tb1`:tb2)') + >>> t.exec(kx.Column('n').get(iterator='each')) + pykx.List(pykx.q(' + +(,`n)!,`a`b`c + +(,`n)!,1 2 3 + ')) + ``` + """ + return self.call('get', iterator=iterator) + + def read0(self, iterator=None): + """ + Read text from a file or process handle. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read0(iterator='each')) + pykx.List(pykx.q(' + "hello" + "goodbye" + ')) + ``` + """ + return self.call('read0', iterator=iterator) + + def read1(self, iterator=None): + """ + Read bytes from a file or named pipe. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read1(iterator='each')) + pykx.List(pykx.q(' + 0x68656c6c6f0a + 0x676f6f646279650a + ')) + ``` + """ + return self.call('read1', iterator=iterator) + + def next(self, iterator=None): + """ + For each item in the column, return the next item. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:3 6 9 12)') + >>> t.exec(kx.Column('n').next())) + pykx.LongVector(pykx.q('6 9 12 0N')) + ``` + """ + return self.call('next', iterator=iterator) + + def prior(self, other, iterator=None, col_arg_ind=1, project_args=None): + """ + Applies passed function to value in column and the item preceding it. + + Parameters: + other: A binary function. + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 1. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 1 2 3)') + >>> t.select(kx.Column('n').prior(kx.q('+'))) + pykx.Table(pykx.q(' + n + - + 1 + 3 + 5 + ')) + ``` + """ + return self.call('prior', other, iterator=iterator, col_arg_ind=col_arg_ind, + project_args=project_args) + + def where(self, iterator=None): + """ + Return indexes of True values. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each` + + Examples: + + Find indexes where values are True + + ```python + >>> import pykx as kx + >>> tab = kx.q('([] a:101010b)') + >>> tab.exec(kx.Column('a').where()) + pykx.LongVector(pykx.q('0 2 4')) + ``` + """ + return self.call('where', iterator=iterator) + class QueryPhrase: """Special wrapper for a list which will be treated as a QueryPhrase. @@ -11205,6 +11673,7 @@ def _internal_k_dict_to_py(addr: int): _k_table_type = object() _k_dictionary_type = object() _k_unary_primitive = object() +_k_foreign_type = object() type_number_to_pykx_k_type = { @@ -11263,7 +11732,7 @@ def _internal_k_dict_to_py(addr: int): 109: EachPrior, 110: EachRight, 111: EachLeft, - 112: Foreign + 112: _k_foreign_type } diff --git a/tests/conftest.py b/tests/conftest.py index 8ac6ef7..cb34822 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,7 +30,7 @@ def pa(kx): @pytest.fixture def pd(kx): - """Fixture that safely imports Pandas after PyKX has potentially blocked PyArrow imports.""" + """Fixture that safely imports Pandas after pykx has potentially blocked PyArrow imports.""" import pandas return pandas @@ -39,7 +39,7 @@ def pd(kx): def kx(request): markers = {x.name: x.kwargs for x in request.node.own_markers} if 'isolate' in markers and 'pykx' in sys.modules: - raise Exception('PyKX has been imported - test isolation has failed') + raise Exception('KDB-X Python has been imported - test isolation has failed') # Use the seed set by the `pytest-randomly` plugin as q's random seed seed = int(request.config.getoption('randomly_seed')) % (2**31 - 1) os.environ['QARGS'] = f'-S {seed} --testflag {request.param}' @@ -88,8 +88,8 @@ def random_free_port(): return port -# Need to know what $QHOME to use for the q subprocesses before PyKX changes it. -original_QHOME = os.environ['QHOME'] +# Need to know what $QHOME to use for the q subprocesses before KDB-X Python changes it. +original_QHOME = os.getenv('QHOME', '') @contextmanager @@ -239,9 +239,9 @@ def pytest_generate_tests(metafunc): # noqa 'indirect': True, 'ids': lambda argvalue: argvalue.lstrip('-') if argvalue else 'licensed', } - if 'nep49' in markers and py_minor_version >= 8: + if 'nep49' in markers: kx_fixture_kwargs['argvalues'].append( - pytest.param(' --pykxalloc --pykxgc', + pytest.param('--pykxgc', marks=[pytest.mark.licensed, pytest.mark.nep49] ) ) @@ -256,14 +256,14 @@ def pytest_generate_tests(metafunc): # noqa if unlicensed or (ipc and not markers['ipc'].get('licensed_only', False)): if 'unlicensed' in markers and markers['unlicensed'].get('unlicensed_only', False): kx_fixture_kwargs['argvalues'].pop() - if 'nep49' in markers and py_minor_version >= 8: + if 'nep49' in markers: kx_fixture_kwargs['argvalues'].pop() kx_fixture_kwargs['argvalues'].append( pytest.param('--unlicensed', marks=[pytest.mark.isolate, pytest.mark.unlicensed]) ) - if 'nep49' in markers and py_minor_version >= 8: + if 'nep49' in markers: kx_fixture_kwargs['argvalues'].append( - pytest.param('--unlicensed --pykxalloc', + pytest.param('--unlicensed', marks=[pytest.mark.isolate, pytest.mark.unlicensed, pytest.mark.nep49] @@ -280,7 +280,7 @@ def pytest_generate_tests(metafunc): # noqa def pytest_runtest_protocol(item): # FIXME: Because fork is being used instead of spawn, as soon as the first licensed test runs, # all tests afterwards must be licensed. To work around this issue, we run every test within - # a fork that has not imported PyKX, unless we are on Windows. + # a fork that has not imported KDB-X Python, unless we are on Windows. if (system() != 'Windows') and (item.get_closest_marker('isolate') or True): item.ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location) reports = [_pytest.runner.TestReport(**x) for x in marshal.loads(isolate_test(item))] diff --git a/tests/parse_tests.py b/tests/parse_tests.py index c65a45d..7df3809 100644 --- a/tests/parse_tests.py +++ b/tests/parse_tests.py @@ -8,6 +8,7 @@ import test_ctx import test_exceptions import test_ipc +import test_module import test_pandas_api import test_pykx import test_q_foreign @@ -182,21 +183,21 @@ def make_tests(self): # noqa if not any('(unlicensed_only' in x for x in func.decorators): ipc_licensed.append(func) embedded.append(func) - if any(['nep' in x for x in func.decorators]) and py_minor_version >= 8: + if any(['nep' in x for x in func.decorators]): nep_licensed.append(func) added = True if not any('(licensed_only' in x for x in func.decorators): ipc_unlicensed.append(func) - if any(['nep' in x for x in func.decorators]) and py_minor_version >= 8: + if any(['nep' in x for x in func.decorators]): nep_unlicensed.append(func) added = True if any(['.unlicensed' in x for x in func.decorators]): if not any('(unlicensed_only' in x for x in func.decorators): - if any(['nep' in x for x in func.decorators]) and py_minor_version >= 8: + if any(['nep' in x for x in func.decorators]): nep_licensed.append(func) licensed.append(func) - if any(['nep' in x for x in func.decorators]) and py_minor_version >= 8: + if any(['nep' in x for x in func.decorators]): nep_unlicensed.append(func) unlicensed.append(func) added = True @@ -204,7 +205,7 @@ def make_tests(self): # noqa pandas_licensed.append(func) added = True if not added: - if any(['nep' in x for x in func.decorators]) and py_minor_version >= 8: + if any(['nep' in x for x in func.decorators]): nep_licensed.append(func) embedded.append(func) @@ -227,6 +228,7 @@ def make_tests(self): # noqa test_console, test_exceptions, test_ipc, + test_module, test_q_future, test_query, test_util, @@ -246,11 +248,11 @@ def make_tests(self): # noqa ('win_tests/embedded/embedded_tests.py', 5, ''), ('win_tests/pandas_lic/pandas_licensed_tests.py', 8, '--pandas-api') ] -if py_minor_version >= 8: - files.extend([ - ('win_tests/nep_lic/nep_licensed_tests.py', 6, '--pykxalloc --pykxgc'), - ('win_tests/nep_unlic/nep_unlicensed_tests.py', 7, '--unlicensed --pykxalloc')] - ) + +files.extend([ + ('win_tests/nep_lic/nep_licensed_tests.py', 6, '--pykxgc'), + ('win_tests/nep_unlic/nep_unlicensed_tests.py', 7, '--unlicensed')] +) if not os.path.exists('win_tests'): os.makedirs('win_tests') diff --git a/tests/qcumber_tests/examples.quke b/tests/qcumber_tests/examples.quke index 41caa8b..eceb4cf 100644 --- a/tests/qcumber_tests/examples.quke +++ b/tests/qcumber_tests/examples.quke @@ -29,7 +29,7 @@ feature Documentation examples/walkthroughs obj:.pykx.eval"obj(2, 3)"; all(2~obj[`:x]`;3~obj[`:y]`) - expect indexing of PyKX objects to be supported + expect indexing of KDB-X Python objects to be supported lst:.pykx.eval"[True, 2, 3.0, 'four']"; all( 1b~lst[@;0]`; @@ -37,7 +37,7 @@ feature Documentation examples/walkthroughs (3f;2;1b;`four)~lst'[@;;`]2 1 0 3 ) - expect setting of PyKX objects in Python lists to be supported + expect setting of KDB-X Python objects in Python lists to be supported lst:.pykx.eval"[True, 2, 3.0, 'four']"; lst[=;0;0b]; lst[=;-1;`last]; @@ -88,7 +88,7 @@ feature Documentation examples/walkthroughs b:.pykx.toq a; .qu.compare[b;2] - expect the conversion of unwrapped PyKX foreign objects into q + expect the conversion of unwrapped KDB-X Python foreign objects into q a:.pykx.eval["1+1"]; b:a`.; c:.pykx.toq b; diff --git a/tests/qcumber_tests/extra_functions.quke b/tests/qcumber_tests/extra_functions.quke index 46c9411..666a961 100644 --- a/tests/qcumber_tests/extra_functions.quke +++ b/tests/qcumber_tests/extra_functions.quke @@ -83,9 +83,9 @@ feature .pykx.version feature .pykx.debug should return system information when executed - expect a general list with PyKX information as the first element + expect a general list with KDB-X Python information as the first element ret:.pykx.debugInfo[]; - all(0h~type ret;ret[0]like"*PyKX information*") + all(0h~type ret;ret[0]like"*KDB-X Python information*") feature .pykx.print should call print appropriately on python type objects diff --git a/tests/qcumber_tests/list.quke b/tests/qcumber_tests/list.quke index 034c498..9f5e5bb 100644 --- a/tests/qcumber_tests/list.quke +++ b/tests/qcumber_tests/list.quke @@ -1,4 +1,4 @@ -feature PyKX list assignment, retrieval and indexing +feature KDB-X Python list assignment, retrieval and indexing before numpyList ::.pykx.import[`numpy;`:arange]10; pythonList ::.pykx.eval"[True,2,3.0,'four']"; diff --git a/tests/qcumber_tests/memory.quke b/tests/qcumber_tests/memory.quke index 0abf799..e798a44 100644 --- a/tests/qcumber_tests/memory.quke +++ b/tests/qcumber_tests/memory.quke @@ -6,7 +6,7 @@ feature Memory Allocation tests a::til 100; .pykx.set[`bbb;a]; - should not increase memory by above expected amount on usage of PyKX functions + should not increase memory by above expected amount on usage of KDB-X Python functions expect repeated setting of an object to not increase memory initmem:pymem[]; do[10000;.pykx.set[`bbb;a]]; @@ -23,3 +23,11 @@ feature Memory Allocation tests initmem:pymem[]; do[100000;.pykx.eval"bbb(10, 100)"]; 10000 > abs initmem - pymem[] + + expect repeated getting of an objects attribute to not increase memory + .pykx.pyexec"class Foo:\n def __init__(self):\n self.a = 1\n self.b = list(range(10000))"; + f: .pykx.eval"Foo()"; + initmem:pymem[]; + do[10000;f[`:a]]; + do[10000;f[`:b]]; + 10000 > abs initmem-pymem[] diff --git a/tests/qcumber_tests/pykx.quke b/tests/qcumber_tests/pykx.quke index 838d0ff..30a7c90 100644 --- a/tests/qcumber_tests/pykx.quke +++ b/tests/qcumber_tests/pykx.quke @@ -132,12 +132,6 @@ feature Qlog functions to not be defined in pykx namespace @[{get x;0b};`.pykx.i.endpoint;1b] expect namespace to not have functions in root provided by qlog @[{get x;0b};`.pykx.setCorrelator;1b] - - should have logging namespace populated when running on linux - expect logging to be defined if running on linux in root namespace - $[.z.o~`l64;@[{get x;1b};`.com_kx_log.setCorrelator;0b];1b] - expect logging functionality to have loaded in the .i namespace - $[.z.o~`l64;@[{get x;1b};`.com_kx_log.i.endpoint;0b];1b] feature toq conversions to support compositions should pass through compositions diff --git a/tests/qcumber_tests/reimport.quke b/tests/qcumber_tests/reimport.quke index 7daa6f1..9097e8e 100644 --- a/tests/qcumber_tests/reimport.quke +++ b/tests/qcumber_tests/reimport.quke @@ -10,7 +10,7 @@ feature pykx.q expect an error if embedPy is already loaded .p.e:{"not the pykx version"}; err:@[{system"l pykx.q"};`;{x}]; - .qu.compare["Unable to load PyKX, see logged output for more information";err]; + .qu.compare["Unable to load KDB-X Python, see logged output for more information";err]; feature .pykx.safeReimport diff --git a/tests/test_cloud_edition.py b/tests/test_cloud_edition.py index bbdc363..a98b14e 100644 --- a/tests/test_cloud_edition.py +++ b/tests/test_cloud_edition.py @@ -8,7 +8,7 @@ if platform.system() != 'Linux' or platform.machine() == 'aarch64': pytest.skip( - 'PyKX Cloud Edition functionality only supports' + 'KDB-X Cloud Edition functionality only supports' ' x86_64 Linux', allow_module_level=True ) @@ -268,6 +268,7 @@ def test_objstor_aws_read1(q, kx): ) def test_qlog_fd_stdout_endpoint(q): """Test STDOUT endpoint creation.""" + q('\\l qlog.q_') q('.com_kx_log.fd.i.write:{.fd.cache,:enlist(x;y)}') q('.fd.cache:()') @@ -284,6 +285,7 @@ def test_qlog_fd_stdout_endpoint(q): ) def test_qlog_fd_stderr_setup(q): """Tests creation of stderr endpoint.""" + q('\\l qlog.q_') q('id:.com_kx_log.lopen[`:fd://stderr]') q('data:.com_kx_log.i.endpoint[id; `data]') assert q('-2i~data`handle') @@ -296,6 +298,7 @@ def test_qlog_fd_stderr_setup(q): ) def test_qlog_fd_stdout_log_string(q): """Test default string logging.""" + q('\\l qlog.q_') q('.com_kx_log.fd.i.write:{.fd.cache,:enlist(x;y)}') q('.fd.cache:()') @@ -318,6 +321,7 @@ def test_qlog_fd_stdout_log_string(q): ) def test_qlog_fd_stdout_custom_log(q): """Tests custom string logging.""" + q('\\l qlog.q_') q('.com_kx_log.fd.i.write:{.fd.cache,:enlist(x;y)}') q('.fd.cache:()') @@ -338,6 +342,7 @@ def test_qlog_fd_stdout_custom_log(q): ) def test_qlog_fd_stdout_dict_log(q): """Tests dictionary message logging.""" + q('\\l qlog.q_') q('.com_kx_log.fd.i.write:{.fd.cache,:enlist(x;y)}') q('.fd.cache:()') @@ -356,6 +361,7 @@ def test_qlog_fd_stdout_dict_log(q): ) def test_qlog_fd_file_publish(q): """Tests logging message to file.""" + q('\\l qlog.q_') q('.com_kx_log.fd.i.write:{.fd.cache,:enlist(x;y)}') q('.fd.cache:()') diff --git a/tests/test_compress_encrypt.py b/tests/test_compress_encrypt.py index ddec740..e37a4f9 100644 --- a/tests/test_compress_encrypt.py +++ b/tests/test_compress_encrypt.py @@ -1,4 +1,3 @@ -import os from pathlib import Path # Do not import pykx here - use the `kx` fixture instead! @@ -27,11 +26,6 @@ def test_compress_encrypt_errors(kx): kx.Compress(block_size=24) assert 'block_size must be a power of 2' in str(err.value) - if os.getenv('PYKX_4_1_ENABLED') is None: - with pytest.raises(ValueError) as err: - kx.Compress(algo=kx.CompressionAlgorithm.zstd) - assert "'CompressionAlgorithm.zstd' only supported on" in str(err.value) - with pytest.raises(ValueError) as err: kx.Compress(algo=kx.CompressionAlgorithm.gzip, level=100) assert 'Invalid level 100 for CompressionAlgorithm.gzip' in str(err.value) @@ -52,11 +46,6 @@ def test_compression(): compress.global_init() assert kx.q.z.zd.py() == [17, 4, 10] - -@pytest.mark.isolate -def test_compression_4_1(): - os.environ['PYKX_4_1_ENABLED'] = 'True' - import pykx as kx compress = kx.Compress(kx.CompressionAlgorithm.zstd, level=0) compress.global_init() assert kx.q.z.zd.py() == [17, 5, 0] diff --git a/tests/test_config.py b/tests/test_config.py index 6f425a5..4352af0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,17 +1,11 @@ -from pathlib import Path import os +from pathlib import Path from tempfile import TemporaryDirectory import warnings import pytest -@pytest.mark.unlicensed -def test_QHOME(kx): - assert isinstance(kx.config.qhome, Path) - assert (kx.config.qhome/'q.k').exists() - - @pytest.mark.unlicensed def test_dir(kx): assert isinstance(dir(kx.config), list) @@ -22,10 +16,12 @@ def test_dir(kx): def test_missing_profile(capsys): with TemporaryDirectory() as tmp_dir: os.chdir(tmp_dir) - open('.pykx-config', 'a').close() + open('my-config-pykx', 'a').close() + os.environ['PYKX_CONFIGURATION_LOCATION'] = str(Path(tmp_dir)/'my-config-pykx') import pykx as kx # noqa out, _ = capsys.readouterr() assert "Unable to locate specified 'PYKX_PROFILE': 'default' in file" in out + del os.environ['PYKX_CONFIGURATION_LOCATION'] @pytest.mark.isolate @@ -37,11 +33,13 @@ def test_boolean_config(): ''' with TemporaryDirectory() as tmp_dir: os.chdir(tmp_dir) - with open('.pykx-config', 'w+') as f: + with open('my-config-pykx', 'w+') as f: f.writelines(config) + os.environ['PYKX_CONFIGURATION_LOCATION'] = str(Path(tmp_dir)/'my-config-pykx') import pykx as kx assert kx.config.suppress_warnings assert kx.config.pykx_qdebug + del os.environ['PYKX_CONFIGURATION_LOCATION'] @pytest.mark.isolate diff --git a/tests/test_ctx.py b/tests/test_ctx.py index 09c995a..14258c3 100644 --- a/tests/test_ctx.py +++ b/tests/test_ctx.py @@ -49,7 +49,7 @@ def cd(newdir): def test_dir(q, kx): assert isinstance(dir(kx.ctx), list) assert sorted(dir(kx.ctx)) == dir(kx.ctx) - default_contexts = ('h', 'j', 'o', 'Q', 'q', 'z') + default_contexts = ('h', 'j', 'Q', 'q', 'z') assert all(x in dir(q) for x in default_contexts) @@ -162,7 +162,7 @@ def test_namespace_switch(q): @pytest.mark.ipc def test_iter(q): - assert set() == {'Q', 'h', 'j', 'o', 'q', 'z'} - set(q.ctx) + assert set() == {'Q', 'h', 'j', 'q', 'z'} - set(q.ctx) @pytest.mark.ipc @@ -326,3 +326,7 @@ def test_operator_retrieval(kx): def test_context_loadfile(kx): assert isinstance(kx.q.csvutil, kx.ctx.QContext) + + +def test_z_v(kx): + assert isinstance(kx.q.z.v, kx.Dictionary) diff --git a/tests/test_db.py b/tests/test_db.py index d18014e..546940d 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -342,34 +342,18 @@ def test_subview(kx): @pytest.mark.isolate def test_q_lo_move_dir(): - os.environ['PYKX_4_1_ENABLED'] = 'True' curr_dir = os.getcwd() import pykx as kx kx.DB(path='db') assert curr_dir != os.getcwd() - os.unsetenv('PYKX_4_1_ENABLED') - os.unsetenv('PYKX_BETA_FEATURES') @pytest.mark.isolate def test_q_lo_keep_dir(): - os.environ['PYKX_4_1_ENABLED'] = 'True' curr_dir = os.getcwd() import pykx as kx kx.DB(path='db', change_dir=False) assert curr_dir == os.getcwd() - os.unsetenv('PYKX_4_1_ENABLED') - os.unsetenv('PYKX_BETA_FEATURES') - - -def test_q_lo_40(kx): - if os.getenv('PYKX_4_1_ENABLED') is None: - with pytest.raises(kx.QError) as err: - db = kx.DB(path='db', change_dir=False) # noqa: F841 - assert 'behavior only supported with PYKX_4_1_ENABLED' in str(err.value) - with pytest.raises(kx.QError) as err: - db = kx.DB(path='db', load_scripts=False) # noqa: F841 - assert 'behavior only supported with PYKX_4_1_ENABLED' in str(err.value) @pytest.mark.isolate diff --git a/tests/test_ipc.py b/tests/test_ipc.py index c61c8ef..5f8157d 100644 --- a/tests/test_ipc.py +++ b/tests/test_ipc.py @@ -339,7 +339,7 @@ def test_py_file_execution(kx): assert "Provided file type 'l' unsupported" == str(err.value) with pytest.raises(kx.QError) as err: q.file_execute('./tests/qscripts/pyfile.py', return_all=True) - assert "PyKX must be loaded on remote server" == str(err.value) + assert "pykx must be loaded on remote server" == str(err.value) q('\\l pykx.q') q.file_execute('./tests/qscripts/pyfile.py') assert q('.pykx.get[`pyfunc;<][2;3]') == 6 diff --git a/tests/test_license.py b/tests/test_license.py index db56286..86a9060 100644 --- a/tests/test_license.py +++ b/tests/test_license.py @@ -33,7 +33,7 @@ def test_initialization_using_unlicensed_mode(tmp_path, q): def test_fallback_to_unlicensed_mode_error(tmp_path): os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) os.environ['QARGS'] = '--licensed' - # Can't use PyKXException here because we have to import PyKX after entering the with-block + # Can't use PyKXException here, we have to import KDB-X Python after entering the with-block with pytest.raises(Exception, match='(?i)Failed to initialize embedded q'): import pykx # noqa: F401 @@ -292,7 +292,7 @@ def test_envvar_init(): ) def test_use_both_licensed_and_unlicensed_flags(QARGS): os.environ['QARGS'] = QARGS - # Can't use PyKXException here because we have to import PyKX after entering the with-block + # Can't use PyKXException here, we have to import KDB-X Python after entering the with-block with pytest.raises(Exception, match='(?i)mutually exclusive'): import pykx # noqa: F401 diff --git a/tests/test_module.py b/tests/test_module.py new file mode 100644 index 0000000..f02bb93 --- /dev/null +++ b/tests/test_module.py @@ -0,0 +1,149 @@ +"""Tests for the modules.""" + +import pytest + + +class TestSearchPath: + DEFAULT_PATH = "/home/user/.kx/mod" + + # ── Helpers ─────────────────────────────────────────────────────────────────── + + def reset_sp(self, kx): + """Reset kx.module.SearchPath to the default state before each test.""" + kx.module._SP([self.DEFAULT_PATH]) + + # ── Retrieve ────────────────────────────────────────────────────────────────── + + def test_retrieve_returns_list(self, kx): + self.reset_sp(kx) + result = kx.module.SearchPath() + assert isinstance(result, list) + + def test_retrieve_contains_default(self, kx): + self.reset_sp(kx) + assert self.DEFAULT_PATH in kx.module.SearchPath() + + # ── Add ─────────────────────────────────────────────────────────────────────── + + def test_add_appends_to_end(self, kx): + self.reset_sp(kx) + kx.module.SearchPath("/opt/mod", add=True) + sp = kx.module.SearchPath() + assert sp[-1] == "/opt/mod" + + def test_add_prepends_to_front(self, kx): + self.reset_sp(kx) + kx.module.SearchPath("/opt/mod", add=True, prepend=True) + sp = kx.module.SearchPath() + assert sp[0] == "/opt/mod" + + def test_add_duplicate_raises(self, kx): + self.reset_sp(kx) + with pytest.raises(ValueError, match="already on SearchPath"): + kx.module.SearchPath(self.DEFAULT_PATH, add=True) + + def test_add_duplicate_allowed_with_flag(self, kx): + self.reset_sp(kx) + kx.module.SearchPath(self.DEFAULT_PATH, add=True, allow_duplicates=True) + assert kx.module.SearchPath().count(self.DEFAULT_PATH) == 2 + + def test_add_duplicate_allowed_prepend(self, kx): + self.reset_sp(kx) + kx.module.SearchPath(self.DEFAULT_PATH, add=True, prepend=True, allow_duplicates=True) + sp = kx.module.SearchPath() + assert sp[0] == self.DEFAULT_PATH + assert sp.count(self.DEFAULT_PATH) == 2 + + # ── Remove ──────────────────────────────────────────────────────────────────── + + def test_remove_entry(self, kx): + self.reset_sp(kx) + kx.module.SearchPath("/opt/mod", add=True) + kx.module.SearchPath("/opt/mod", remove=True) + assert "/opt/mod" not in kx.module.SearchPath() + + def test_remove_all_occurrences(self, kx): + self.reset_sp(kx) + kx.module.SearchPath(self.DEFAULT_PATH, add=True, allow_duplicates=True) + assert kx.module.SearchPath().count(self.DEFAULT_PATH) == 2 + kx.module.SearchPath(self.DEFAULT_PATH, remove=True) + assert self.DEFAULT_PATH not in kx.module.SearchPath() + + def test_remove_missing_silent(self, kx): + self.reset_sp(kx) + result = kx.module.SearchPath("/nonexistent", remove=True) + assert result is None # no error, silent return + + def test_remove_missing_strict_raises(self, kx): + self.reset_sp(kx) + with pytest.raises(ValueError, match="not found in SearchPath"): + kx.module.SearchPath("/nonexistent", remove=True, strict=True) + + def test_remove_preserves_other_entries(self, kx): + self.reset_sp(kx) + kx.module.SearchPath("/opt/mod", add=True) + kx.module.SearchPath("/other/mod", add=True) + kx.module.SearchPath("/opt/mod", remove=True) + sp = kx.module.SearchPath() + assert "/opt/mod" not in sp + assert "/other/mod" in sp + assert self.DEFAULT_PATH in sp + + # ── No keyword raises ───────────────────────────────────────────────────────── + + def test_no_keyword_raises(self, kx): + self.reset_sp(kx) + with pytest.raises(ValueError, match="Specify add=True or remove=True"): + kx.module.SearchPath("/opt/mod") + + # ── Order integrity ─────────────────────────────────────────────────────────── + + def test_order_append(self, kx): + self.reset_sp(kx) + kx.module.SearchPath("/a", add=True) + kx.module.SearchPath("/b", add=True) + kx.module.SearchPath("/c", add=True) + assert kx.module.SearchPath() == [self.DEFAULT_PATH, "/a", "/b", "/c"] + + def test_order_prepend(self, kx): + self.reset_sp(kx) + kx.module.SearchPath("/a", add=True, prepend=True) + kx.module.SearchPath("/b", add=True, prepend=True) + assert kx.module.SearchPath() == ["/b", "/a", self.DEFAULT_PATH] + + +# TODO: Once Kdb-X is fully installed for testing in CI re-enable these tests. +""" +@pytest.mark.isolate +def test_q_module(): + import pykx as kx + pq = kx.use('kx.pq') + assert isinstance(pq, kx.module.QModule) + pq.t = kx.use('kx.pq.t') + assert isinstance(pq, kx.module.QModule) + assert isinstance(pq.t, kx.module.QModule) + assert isinstance(pq.pq, kx.wrappers.Lambda) + assert isinstance(pq.t.mkT, kx.wrappers.Lambda) + + # assert module is singleton + a = kx.use('kx.pq') + assert isinstance(a, kx.module.QModule) + assert isinstance(a.t, kx.module.QModule) + + +@pytest.mark.isolate +def test_q_module_auto(): + import pykx as kx + pq = kx.use('kx.pq') + assert isinstance(pq, kx.module.QModule) + kx.use('kx.pq.t') + assert isinstance(pq, kx.module.QModule) + assert isinstance(pq.t, kx.module.QModule) + assert isinstance(pq.pq, kx.wrappers.Lambda) + assert isinstance(pq.t.mkT, kx.wrappers.Lambda) + + # assert module is singleton + a = kx.use('kx.pq') + assert isinstance(a, kx.module.QModule) + assert isinstance(a.t, kx.module.QModule) +""" diff --git a/tests/test_pandas_api.py b/tests/test_pandas_api.py index 1d8c3a7..30c9250 100644 --- a/tests/test_pandas_api.py +++ b/tests/test_pandas_api.py @@ -1,6 +1,5 @@ """Tests for the Pandas API.""" -import sys import os import numpy as np @@ -559,401 +558,398 @@ def test_table_inner_merge(kx, q): def test_table_left_merge(kx, q): - if sys.version_info.minor > 7: - # Merge on keys - df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) - df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) - assert df1.merge( - df2, + # Merge on keys + df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) + df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) + assert df1.merge( + df2, + left_on='lkey', + right_on='rkey', + how='left' + ).equals( + tab1.merge( + tab2, left_on='lkey', right_on='rkey', how='left' - ).equals( - tab1.merge( - tab2, - left_on='lkey', - right_on='rkey', - how='left' - ).pd() - ) + ).pd() + ) - # Merge on keys KeyedTable - df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) - df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) - tab1 = q('{1!x}', kx.toq(df1)) - tab2 = q('{1!x}', kx.toq(df2)) - assert df1.merge( - df2, + # Merge on keys KeyedTable + df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) + df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) + tab1 = q('{1!x}', kx.toq(df1)) + tab2 = q('{1!x}', kx.toq(df2)) + assert df1.merge( + df2, + left_on='lkey', + right_on='rkey', + how='left' + ).equals( + q('{0!x}', tab1.merge( + tab2, left_on='lkey', right_on='rkey', how='left' - ).equals( - q('{0!x}', tab1.merge( - tab2, - left_on='lkey', - right_on='rkey', - how='left' - )).pd() - ) + )).pd() + ) + + # Merge on differing keys + df1 = pd.DataFrame({'a': ['foo', 'bar'], 'b': [1, 2]}) + df2 = pd.DataFrame({'a': ['foo', 'baz'], 'c': [3, 4]}) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) + tab_res = tab1.merge(tab2, on='a', how='left').pd() + assert str(tab_res.at[1, 'c']) == '--' + tab_res.at[1, 'c'] = np.nan + assert df1.merge(df2, on='a', how='left').equals(tab_res) - # Merge on differing keys - df1 = pd.DataFrame({'a': ['foo', 'bar'], 'b': [1, 2]}) - df2 = pd.DataFrame({'a': ['foo', 'baz'], 'c': [3, 4]}) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) - tab_res = tab1.merge(tab2, on='a', how='left').pd() - assert str(tab_res.at[1, 'c']) == '--' - tab_res.at[1, 'c'] = np.nan - assert df1.merge(df2, on='a', how='left').equals(tab_res) - - # Merge on same indexes - df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) - df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) - assert df1.merge( - df2, + # Merge on same indexes + df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) + df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) + assert df1.merge( + df2, + left_index=True, + right_index=True, + how='left' + ).equals( + tab1.merge( + tab2, left_index=True, right_index=True, how='left' - ).equals( - tab1.merge( - tab2, - left_index=True, - right_index=True, - how='left' - ).pd() - ) + ).pd() + ) - # Merge on different indexes - df1 = pd.DataFrame( - { - 'lkey': ['foo', 'bar', 'baz', 'foo'], - 'value': [1, 2, 3, 5] - }, - index=[4, 3, 2, 1] - ) - df2 = pd.DataFrame( - { - 'rkey': ['foo', 'bar', 'baz', 'foo'], - 'value': [5, 6, 7, 8] - }, - index=[0, 1, 2, 3] - ) - tab1 = q('{`idx xcols update idx: reverse 1 + til count x from x}', tab1) - tab1 = q('{1!x}', tab1) - tab2 = kx.q.qsql.update(tab2, {'idx': 'til count i'}, inplace=True) - tab2 = tab2.set_index('idx') - res = tab1.merge(tab2, left_index=True, right_index=True, how='left') - assert isinstance(res, kx.KeyedTable) - df_res = df1.merge(df2, left_index=True, right_index=True, how='left') - # assert our index does match properly before removing it - assert q('0!', res)['idx'].py() == list(df_res.index) - # We have idx as a column so we have to remove it to be equal as it won't convert - # to the pandas index column automatically - res = q('{(enlist `idx)_(0!x)}', res).pd() - df_res = df_res.reset_index() # Reset pandas index to default, we already checked it - df_res.pop('index') - res.at[0, 'rkey'] = np.nan - res.at[0, 'value_y'] = np.nan - assert df_res.equals(res) - - df1 = pd.DataFrame( - {'key': ['foo', 'bar', 'baz', 'foo', 'quz'], 'value': [1, 2, 3, 5, None]} - ) - df2 = pd.DataFrame({'key': ['foo', 'bar', 'baz', 'foo', None], 'value': [5, 6, 7, 8, 99]}) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) + # Merge on different indexes + df1 = pd.DataFrame( + { + 'lkey': ['foo', 'bar', 'baz', 'foo'], + 'value': [1, 2, 3, 5] + }, + index=[4, 3, 2, 1] + ) + df2 = pd.DataFrame( + { + 'rkey': ['foo', 'bar', 'baz', 'foo'], + 'value': [5, 6, 7, 8] + }, + index=[0, 1, 2, 3] + ) + tab1 = q('{`idx xcols update idx: reverse 1 + til count x from x}', tab1) + tab1 = q('{1!x}', tab1) + tab2 = kx.q.qsql.update(tab2, {'idx': 'til count i'}, inplace=True) + tab2 = tab2.set_index('idx') + res = tab1.merge(tab2, left_index=True, right_index=True, how='left') + assert isinstance(res, kx.KeyedTable) + df_res = df1.merge(df2, left_index=True, right_index=True, how='left') + # assert our index does match properly before removing it + assert q('0!', res)['idx'].py() == list(df_res.index) + # We have idx as a column so we have to remove it to be equal as it won't convert + # to the pandas index column automatically + res = q('{(enlist `idx)_(0!x)}', res).pd() + df_res = df_res.reset_index() # Reset pandas index to default, we already checked it + df_res.pop('index') + res.at[0, 'rkey'] = np.nan + res.at[0, 'value_y'] = np.nan + assert df_res.equals(res) + + df1 = pd.DataFrame( + {'key': ['foo', 'bar', 'baz', 'foo', 'quz'], 'value': [1, 2, 3, 5, None]} + ) + df2 = pd.DataFrame({'key': ['foo', 'bar', 'baz', 'foo', None], 'value': [5, 6, 7, 8, 99]}) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) - df_res = df1.merge(df2, on='key', how='left') - res = tab1.merge(tab2, on='key', how='left').pd() - assert str(res.at[6, 'value_y']) == '--' - res.at[6, 'value_y'] = np.nan - assert df_res.equals(res) + df_res = df1.merge(df2, on='key', how='left') + res = tab1.merge(tab2, on='key', how='left').pd() + assert str(res.at[6, 'value_y']) == '--' + res.at[6, 'value_y'] = np.nan + assert df_res.equals(res) def test_table_right_merge(kx, q): - if sys.version_info.minor > 7: - # Merge on keys - df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) - df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) - assert df1.merge( - df2, + # Merge on keys + df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) + df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) + assert df1.merge( + df2, + left_on='lkey', + right_on='rkey', + how='right' + ).equals( + tab1.merge( + tab2, left_on='lkey', right_on='rkey', how='right' - ).equals( - tab1.merge( - tab2, - left_on='lkey', - right_on='rkey', - how='right' - ).pd() - ) + ).pd() + ) - # Merge on keys KeyedTable - df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) - df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) - tab1 = q('{1!x}', kx.toq(df1)) - tab2 = q('{1!x}', kx.toq(df2)) - assert df1.merge( - df2, + # Merge on keys KeyedTable + df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) + df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) + tab1 = q('{1!x}', kx.toq(df1)) + tab2 = q('{1!x}', kx.toq(df2)) + assert df1.merge( + df2, + left_on='lkey', + right_on='rkey', + how='right' + ).equals( + q('{0!x}', tab1.merge( + tab2, left_on='lkey', right_on='rkey', how='right' - ).equals( - q('{0!x}', tab1.merge( - tab2, - left_on='lkey', - right_on='rkey', - how='right' - )).pd() - ) + )).pd() + ) + + # Merge on differing keys + df1 = pd.DataFrame({'a': ['foo', 'bar'], 'b': [1, 2]}) + df2 = pd.DataFrame({'a': ['foo', 'baz'], 'c': [3, 4]}) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) + tab_res = tab1.merge(tab2, on='a', how='right').pd() + assert str(tab_res.at[1, 'b']) == '--' + tab_res.at[1, 'b'] = np.nan + assert df1.merge(df2, on='a', how='right').equals(tab_res) - # Merge on differing keys - df1 = pd.DataFrame({'a': ['foo', 'bar'], 'b': [1, 2]}) - df2 = pd.DataFrame({'a': ['foo', 'baz'], 'c': [3, 4]}) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) - tab_res = tab1.merge(tab2, on='a', how='right').pd() - assert str(tab_res.at[1, 'b']) == '--' - tab_res.at[1, 'b'] = np.nan - assert df1.merge(df2, on='a', how='right').equals(tab_res) - - # Merge on same indexes - df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) - df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) - assert df1.merge( - df2, + # Merge on same indexes + df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) + df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) + assert df1.merge( + df2, + left_index=True, + right_index=True, + how='right' + ).equals( + tab1.merge( + tab2, left_index=True, right_index=True, how='right' - ).equals( - tab1.merge( - tab2, - left_index=True, - right_index=True, - how='right' - ).pd() - ) + ).pd() + ) - # Merge on different indexes - df1 = pd.DataFrame( - { - 'lkey': ['foo', 'bar', 'baz', 'foo'], - 'value': [1, 2, 3, 5] - }, - index=[4, 3, 2, 1] - ) - df2 = pd.DataFrame( - { - 'rkey': ['foo', 'bar', 'baz', 'foo'], - 'value': [5, 6, 7, 8] - }, - index=[0, 1, 2, 3] - ) - tab1 = q('{`idx xcols update idx: reverse 1 + til count x from x}', tab1) - tab1 = q('{1!x}', tab1) - tab2 = kx.q.qsql.update(tab2, {'idx': 'til count i'}, inplace=True) - tab2 = tab2.set_index('idx') - res = tab1.merge(tab2, left_index=True, right_index=True, how='right') - assert isinstance(res, kx.KeyedTable) - df_res = df1.merge(df2, left_index=True, right_index=True, how='right') - # assert our index does match properly before removing it - assert q('0!', res)['idx'].py() == list(df_res.index) - # We have idx as a column so we have to remove it to be equal as it won't convert - # to the pandas index column automatically - res = q('{(enlist `idx)_(0!x)}', res).pd() - df_res = df_res.reset_index() # Reset pandas index to default, we already checked it - df_res.pop('index') - res.at[0, 'lkey'] = np.nan - res.at[0, 'value_x'] = np.nan - assert df_res.equals(res) - - df1 = pd.DataFrame( - {'key': ['foo', 'bar', 'baz', 'foo', 'quz'], 'value': [1, 2, 3, 5, None]} - ) - df2 = pd.DataFrame({'key': ['foo', 'bar', 'baz', 'foo', None], 'value': [5, 6, 7, 8, 99]}) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) + # Merge on different indexes + df1 = pd.DataFrame( + { + 'lkey': ['foo', 'bar', 'baz', 'foo'], + 'value': [1, 2, 3, 5] + }, + index=[4, 3, 2, 1] + ) + df2 = pd.DataFrame( + { + 'rkey': ['foo', 'bar', 'baz', 'foo'], + 'value': [5, 6, 7, 8] + }, + index=[0, 1, 2, 3] + ) + tab1 = q('{`idx xcols update idx: reverse 1 + til count x from x}', tab1) + tab1 = q('{1!x}', tab1) + tab2 = kx.q.qsql.update(tab2, {'idx': 'til count i'}, inplace=True) + tab2 = tab2.set_index('idx') + res = tab1.merge(tab2, left_index=True, right_index=True, how='right') + assert isinstance(res, kx.KeyedTable) + df_res = df1.merge(df2, left_index=True, right_index=True, how='right') + # assert our index does match properly before removing it + assert q('0!', res)['idx'].py() == list(df_res.index) + # We have idx as a column so we have to remove it to be equal as it won't convert + # to the pandas index column automatically + res = q('{(enlist `idx)_(0!x)}', res).pd() + df_res = df_res.reset_index() # Reset pandas index to default, we already checked it + df_res.pop('index') + res.at[0, 'lkey'] = np.nan + res.at[0, 'value_x'] = np.nan + assert df_res.equals(res) - df_res = df1.merge(df2, on='key', how='right') - res = tab1.merge(tab2, on='key', how='right').pd() - assert str(res.at[6, 'key']) == '' - res.at[6, 'key'] = None - assert df_res.equals(res) + df1 = pd.DataFrame( + {'key': ['foo', 'bar', 'baz', 'foo', 'quz'], 'value': [1, 2, 3, 5, None]} + ) + df2 = pd.DataFrame({'key': ['foo', 'bar', 'baz', 'foo', None], 'value': [5, 6, 7, 8, 99]}) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) + + df_res = df1.merge(df2, on='key', how='right') + res = tab1.merge(tab2, on='key', how='right').pd() + assert str(res.at[6, 'key']) == '' + res.at[6, 'key'] = None + assert df_res.equals(res) def test_table_outer_merge(kx, q): - if sys.version_info.minor > 7: - # Merge on keys - df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) - df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) - assert df1.merge( - df2, + # Merge on keys + df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) + df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) + assert df1.merge( + df2, + left_on='lkey', + right_on='rkey', + how='outer', + sort=True + ).equals( + tab1.merge( + tab2, left_on='lkey', right_on='rkey', how='outer', sort=True - ).equals( - tab1.merge( - tab2, - left_on='lkey', - right_on='rkey', - how='outer', - sort=True - ).pd() - ) + ).pd() + ) - # Merge on keys KeyedTable - df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) - df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) - tab1 = q('{1!x}', kx.toq(df1)) - tab2 = q('{1!x}', kx.toq(df2)) - assert df1.merge( - df2, + # Merge on keys KeyedTable + df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) + df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) + tab1 = q('{1!x}', kx.toq(df1)) + tab2 = q('{1!x}', kx.toq(df2)) + assert df1.merge( + df2, + left_on='lkey', + right_on='rkey', + how='outer', + sort=True + ).equals( + q('{0!x}', tab1.merge( + tab2, left_on='lkey', right_on='rkey', how='outer', sort=True - ).equals( - q('{0!x}', tab1.merge( - tab2, - left_on='lkey', - right_on='rkey', - how='outer', - sort=True - )).pd() - ) + )).pd() + ) - # Merge on differing keys - df1 = pd.DataFrame({'a': ['foo', 'bar'], 'b': [1, 2]}) - df2 = pd.DataFrame({'a': ['foo', 'baz'], 'c': [3, 4]}) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) - tab_res = tab1.merge(tab2, on='a', how='outer', sort=True).pd() - assert str(tab_res.at[0, 'c']) == '--' - tab_res.at[0, 'c'] = np.nan - assert str(tab_res.at[1, 'b']) == '--' - tab_res.at[1, 'b'] = np.nan - assert df1.merge(df2, on='a', how='outer', sort=True).equals(tab_res) - - # Merge on same indexes - df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) - df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) - assert df1.merge( - df2, + # Merge on differing keys + df1 = pd.DataFrame({'a': ['foo', 'bar'], 'b': [1, 2]}) + df2 = pd.DataFrame({'a': ['foo', 'baz'], 'c': [3, 4]}) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) + tab_res = tab1.merge(tab2, on='a', how='outer', sort=True).pd() + assert str(tab_res.at[0, 'c']) == '--' + tab_res.at[0, 'c'] = np.nan + assert str(tab_res.at[1, 'b']) == '--' + tab_res.at[1, 'b'] = np.nan + assert df1.merge(df2, on='a', how='outer', sort=True).equals(tab_res) + + # Merge on same indexes + df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) + df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) + assert df1.merge( + df2, + left_index=True, + right_index=True, + how='outer' + ).equals( + tab1.merge( + tab2, left_index=True, right_index=True, how='outer' - ).equals( - tab1.merge( - tab2, - left_index=True, - right_index=True, - how='outer' - ).pd() - ) - assert df1.merge( - df2, + ).pd() + ) + assert df1.merge( + df2, + left_index=True, + right_index=True, + how='outer', + sort=True + ).equals( + tab1.merge( + tab2, left_index=True, right_index=True, how='outer', sort=True - ).equals( - tab1.merge( - tab2, - left_index=True, - right_index=True, - how='outer', - sort=True - ).pd() - ) + ).pd() + ) + + # Merge on different indexes + df1 = pd.DataFrame( + { + 'lkey': ['foo', 'bar', 'baz', 'foo'], + 'value': [1, 2, 3, 5] + }, + index=[4, 3, 2, 1] + ) + df2 = pd.DataFrame( + { + 'rkey': ['foo', 'bar', 'baz', 'foo'], + 'value': [5, 6, 7, 8] + }, + index=[0, 1, 2, 3] + ) + tab1 = q('{`idx xcols update idx: reverse 1 + til count x from x}', tab1) + tab1 = q('{1!x}', tab1) + tab2 = q('{`idx xcols update idx: til count x from x}', tab2) + tab2 = q('{1!x}', tab2) + res = tab1.merge(tab2, left_index=True, right_index=True, how='outer') + assert isinstance(res, kx.KeyedTable) + df_res = df1.merge(df2, left_index=True, right_index=True, how='outer') + # assert our index does match properly before removing it + assert q('0!', res)['idx'].py() == list(df_res.index) + # We have idx as a column so we have to remove it to be equal as it won't convert + # to the pandas index column automatically + res = q('{(enlist `idx)_(0!x)}', res).pd() + df_res = df_res.reset_index() # Reset pandas index to default, we already checked it + df_res.pop('index') + res.at[0, 'lkey'] = np.nan + res.at[0, 'value_x'] = np.nan + res.at[4, 'rkey'] = np.nan + res.at[4, 'value_y'] = np.nan + assert df_res.equals(res) + + df1 = pd.DataFrame( + {'key': ['foo', 'bar', 'baz', 'foo', 'quz'], 'value': [1, 2, 3, 5, None]} + ) + df2 = pd.DataFrame( + { + 'key': ['foo', 'bar', 'baz', 'foo', None], + 'value': [5.0, 6.0, 7.0, 8.0, 99.0] + } + ) + tab1 = kx.toq(df1) + tab2 = kx.toq(df2) + df_res = df1.merge(df2, on='key', how='outer') + res = tab1.merge(tab2, on='key', how='outer').pd() + assert res.at[7, 'key'] == '' + res.at[7, 'key'] = None + res.sort_values(['key'], inplace=True, ignore_index=True) + df_res.sort_values(['key'], inplace=True, ignore_index=True) + assert df_res.equals(res) + + +def test_merge_sort(kx, q): + d = kx.q.z.D + a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) + b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) + res = a.merge(b, how="outer", on="date", sort=True) + assert all(res[0]['date'] == kx.q('enlist', d-3)) + + +def test_merge_sort_single_key(kx, q): + d = kx.q.z.D + a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) + b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) + a = kx.q.xkey('date', a) + res = a.merge(b, how="outer", on="date", sort=True) + assert all(res.keys().columns == ['date']) + res = kx.q('0!', res) + assert all(res[0]['date'] == kx.q('enlist', d-3)) + - # Merge on different indexes - df1 = pd.DataFrame( - { - 'lkey': ['foo', 'bar', 'baz', 'foo'], - 'value': [1, 2, 3, 5] - }, - index=[4, 3, 2, 1] - ) - df2 = pd.DataFrame( - { - 'rkey': ['foo', 'bar', 'baz', 'foo'], - 'value': [5, 6, 7, 8] - }, - index=[0, 1, 2, 3] - ) - tab1 = q('{`idx xcols update idx: reverse 1 + til count x from x}', tab1) - tab1 = q('{1!x}', tab1) - tab2 = q('{`idx xcols update idx: til count x from x}', tab2) - tab2 = q('{1!x}', tab2) - res = tab1.merge(tab2, left_index=True, right_index=True, how='outer') - assert isinstance(res, kx.KeyedTable) - df_res = df1.merge(df2, left_index=True, right_index=True, how='outer') - # assert our index does match properly before removing it - assert q('0!', res)['idx'].py() == list(df_res.index) - # We have idx as a column so we have to remove it to be equal as it won't convert - # to the pandas index column automatically - res = q('{(enlist `idx)_(0!x)}', res).pd() - df_res = df_res.reset_index() # Reset pandas index to default, we already checked it - df_res.pop('index') - res.at[0, 'lkey'] = np.nan - res.at[0, 'value_x'] = np.nan - res.at[4, 'rkey'] = np.nan - res.at[4, 'value_y'] = np.nan - assert df_res.equals(res) - - df1 = pd.DataFrame( - {'key': ['foo', 'bar', 'baz', 'foo', 'quz'], 'value': [1, 2, 3, 5, None]} - ) - df2 = pd.DataFrame( - { - 'key': ['foo', 'bar', 'baz', 'foo', None], - 'value': [5.0, 6.0, 7.0, 8.0, 99.0] - } - ) - tab1 = kx.toq(df1) - tab2 = kx.toq(df2) - df_res = df1.merge(df2, on='key', how='outer') - res = tab1.merge(tab2, on='key', how='outer').pd() - assert res.at[7, 'key'] == '' - res.at[7, 'key'] = None - res.sort_values(['key'], inplace=True, ignore_index=True) - df_res.sort_values(['key'], inplace=True, ignore_index=True) - assert df_res.equals(res) - - -def test_merge_sort(kx, q): - d = kx.q.z.D - a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) - b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) - res = a.merge(b, how="outer", on="date", sort=True) - assert all(res[0]['date'] == kx.q('enlist', d-3)) - - -def test_merge_sort_single_key(kx, q): - d = kx.q.z.D - a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) - b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) - a = kx.q.xkey('date', a) - res = a.merge(b, how="outer", on="date", sort=True) - assert all(res.keys().columns == ['date']) - res = kx.q('0!', res) - assert all(res[0]['date'] == kx.q('enlist', d-3)) - - def test_cross_merge(kx, q): df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) @@ -1113,10 +1109,10 @@ def test_df_astype_value_errors(kx, q): match=r"Column name passed in dictionary not present in df table"): raise df.astype({'c100': kx.ShortVector}) with pytest.raises(kx.QError, - match=r'Value passed does not match PyKX wrapper type'): + match=r'Value passed does not match pykx wrapper type'): raise df.astype({'c1': 'nomatchvalue'}) with pytest.raises(kx.QError, - match=r'Value passed does not match PyKX wrapper type'): + match=r'Value passed does not match pykx wrapper type'): raise df.astype('nomatchvalue') df = q('''([] c1:("abc";"def";"ghi");c2:(1 2 3;4 5 6;7 8 9))''') with pytest.raises(ValueError, @@ -1878,113 +1874,115 @@ def test_median(kx, q): def test_mode(kx, q): # noqa - if sys.version_info.minor > 7: - def compare_q_to_pd(tab, df): - if 'idx' in q.cols(tab): - tab.pop('idx') - tab = tab.pd() - for i in range(len(tab)): - for c in tab.columns: - df_c = c - try: - df_c = int(c) - except BaseException: - pass - if str(tab.at[i, c]) == '--': - tab.at[i, c] = np.nan - if str(tab.at[i, c]) == '': - tab.at[i, c] = 'nan' - if str(tab.at[i, c]) == 'nan' and str(df.at[i, df_c]) == 'nan': - continue - if tab.at[i, c] != df.at[i, df_c]: - return False - return True + def compare_q_to_pd(tab, df): + if 'idx' in q.cols(tab): + tab.pop('idx') + tab = tab.pd() + for i in range(len(tab)): + for c in tab.columns: + df_c = c + try: + df_c = int(c) + except BaseException: + pass + if str(tab.at[i, c]) == '--': + tab.at[i, c] = np.nan + if str(tab.at[i, c]) == '': + tab.at[i, c] = 'nan' + if str(tab.at[i, c]) == 'nan' and str(df.at[i, df_c]) == 'nan': + continue + if tab.at[i, c] != df.at[i, df_c]: + return False + return True - df = pd.DataFrame( - { - 'a': [1, 2, 2, 4], - 'b': [1, 2, 6, 7], - 'c': [7, 8, 9, 10], - 'd': [7, 11, 14, 14] - } - ) - tab = kx.toq(df) - p_m = df.mode() - q_m = tab.mode() - assert compare_q_to_pd(q_m, p_m) - - p_m = df.mode(axis=1) - q_m = tab.mode(axis=1) - assert compare_q_to_pd(q_m, p_m) - - q['tab'] = kx.toq(df) - tab = q('1!`idx xcols update idx: til count tab from tab') - - p_m = df.mode() - q_m = tab.mode() - assert compare_q_to_pd(q_m, p_m) - - p_m = df.mode(axis=1) - q_m = tab.mode(axis=1) - assert compare_q_to_pd(q_m, p_m) - - df = pd.DataFrame( - { - 'a': [1, 2, 2, 4], - 'b': [1, 2, 6, 7], - 'c': [7, 8, 9, 10], - 'd': ['foo', 'bar', 'baz', 'foo'] - } - ) - tab = kx.toq(df) - p_m = df.mode() - q_m = tab.mode() - assert compare_q_to_pd(q_m, p_m) - - p_m = df.mode(axis=1, numeric_only=True) - q_m = tab.mode(axis=1, numeric_only=True) - assert compare_q_to_pd(q_m, p_m) - - df = pd.DataFrame({ - 'x': [0, 1, 2, 3, 4, 5, 6, 7, np.nan, np.nan], - 'y': [10, 11, 12, 13, 14, 15, 16, 17, 18, np.nan], - 'z': ['a', 'b', 'c', 'd', 'd', 'e', 'e', 'f', 'g', 'h'] - }) - tab = kx.toq(df) + df = pd.DataFrame( + { + 'a': [1, 2, 2, 4], + 'b': [1, 2, 6, 7], + 'c': [7, 8, 9, 10], + 'd': [7, 11, 14, 14] + } + ) + tab = kx.toq(df) + p_m = df.mode() + q_m = tab.mode() + assert compare_q_to_pd(q_m, p_m) + + p_m = df.mode(axis=1) + q_m = tab.mode(axis=1) + assert compare_q_to_pd(q_m, p_m) - p_m = df.mode() - q_m = tab.mode() - assert compare_q_to_pd(q_m, p_m) + q['tab'] = kx.toq(df) + tab = q('1!`idx xcols update idx: til count tab from tab') - p_m = df.mode(axis=1, numeric_only=True) - q_m = tab.mode(axis=1, numeric_only=True) - assert compare_q_to_pd(q_m, p_m) + p_m = df.mode() + q_m = tab.mode() + assert compare_q_to_pd(q_m, p_m) - p_m = df.mode(numeric_only=True) - q_m = tab.mode(numeric_only=True) - assert compare_q_to_pd(q_m, p_m) + p_m = df.mode(axis=1) + q_m = tab.mode(axis=1) + assert compare_q_to_pd(q_m, p_m) - p_m = df.mode(axis=1, numeric_only=True) - q_m = tab.mode(axis=1, numeric_only=True) - assert compare_q_to_pd(q_m, p_m) + df = pd.DataFrame( + { + 'a': [1, 2, 2, 4], + 'b': [1, 2, 6, 7], + 'c': [7, 8, 9, 10], + 'd': ['foo', 'bar', 'baz', 'foo'] + } + ) + tab = kx.toq(df) + p_m = df.mode() + q_m = tab.mode() + assert compare_q_to_pd(q_m, p_m) + + p_m = df.mode(axis=1, numeric_only=True) + q_m = tab.mode(axis=1, numeric_only=True) + assert compare_q_to_pd(q_m, p_m) + + df = pd.DataFrame({ + 'x': [0, 1, 2, 3, 4, 5, 6, 7, np.nan, np.nan], + 'y': [10, 11, 12, 13, 14, 15, 16, 17, 18, np.nan], + 'z': ['a', 'b', 'c', 'd', 'd', 'e', 'e', 'f', 'g', 'h'] + }) + tab = kx.toq(df) - p_m = df.mode(dropna=False) - q_m = tab.mode(dropna=False) - assert compare_q_to_pd(q_m, p_m) + p_m = df.mode() + q_m = tab.mode() + assert compare_q_to_pd(q_m, p_m) - p_m = df.mode(axis=1, dropna=False, numeric_only=True) - q_m = tab.mode(axis=1, dropna=False, numeric_only=True) - assert compare_q_to_pd(q_m, p_m) + p_m = df.mode(axis=1, numeric_only=True) + q_m = tab.mode(axis=1, numeric_only=True) + assert compare_q_to_pd(q_m, p_m) + + p_m = df.mode(numeric_only=True) + q_m = tab.mode(numeric_only=True) + assert compare_q_to_pd(q_m, p_m) + + p_m = df.mode(axis=1, numeric_only=True) + q_m = tab.mode(axis=1, numeric_only=True) + assert compare_q_to_pd(q_m, p_m) + + p_m = df.mode(dropna=False) + q_m = tab.mode(dropna=False) + assert compare_q_to_pd(q_m, p_m) + + p_m = df.mode(axis=1, dropna=False, numeric_only=True) + q_m = tab.mode(axis=1, dropna=False, numeric_only=True) + assert compare_q_to_pd(q_m, p_m) def test_table_merge_asof(kx, q): - left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]}) - right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "right_val": [1, 2, 3, 6, 7]}) + left = pd.DataFrame({"a": [1, 5, 10], "b": ["a", "a", "b"], "left_val": ["a", "b", "c"]}) + right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "b": ["a", "a", "b", "b", "a"], + "right_val": [1, 2, 3, 6, 7]}) qleft = kx.toq(left) qright = kx.toq(right) assert (pd.merge_asof(left, right, on='a') == kx.merge_asof(qleft, qright, on='a').pd()).all().all() + assert (pd.merge_asof(left, right, on='a', by="b") + == kx.merge_asof(qleft, qright, on='a', by="b").pd()).all().all() assert (pd.merge_asof(left, right, on='a') == qleft.merge_asof(qright, on='a').pd()).all().all() assert (pd.merge_asof(left, right, on='a') @@ -2347,28 +2345,34 @@ def test_pandas_groupby(kx, q): df.groupby(['Animal']).mean() == tab.groupby(['Animal']).mean().pd() ) - assert all( - df.groupby(['Animal'], as_index=False).mean() - == tab.groupby(['Animal'], as_index=False).mean().pd() - ) + # Grouping used not in columns - dropped in Pandas 3.0 + if not kx.config.pandas_gt2: + assert all( + df.groupby(['Animal'], as_index=False).mean() + == tab.groupby(['Animal'], as_index=False).mean().pd() + ) assert all( df.groupby(level=[1]).mean() == tab.groupby(level=[1]).mean().pd() ) - assert all( - df.groupby(level=1, as_index=False).mean() - == tab.groupby(level=1, as_index=False).mean().pd() - ) + # Grouping used not in columns - dropped in Pandas 3.0 + if not kx.config.pandas_gt2: + assert all( + df.groupby(level=1, as_index=False).mean() + == tab.groupby(level=1, as_index=False).mean().pd() + ) - assert all( - df.groupby(level=[0, 1]).mean() - == tab.groupby(level=[0, 1]).mean().pd() - ) - assert all( - df.groupby(level=[0, 1], as_index=False).mean() - == tab.groupby(level=[0, 1], as_index=False).mean().pd() - ) + # Grouping used not in columns - dropped in Pandas 3.0 + if not kx.config.pandas_gt2: + assert all( + df.groupby(level=[0, 1]).mean() + == tab.groupby(level=[0, 1]).mean().pd() + ) + assert all( + df.groupby(level=[0, 1], as_index=False).mean() + == tab.groupby(level=[0, 1], as_index=False).mean().pd() + ) def test_keyed_loc_fixes(q): @@ -2658,3 +2662,13 @@ def test_merge_qjoin_errors(kx): match=r"Right Join requires a keyed table" " for the left dataset."): assert tab1.merge(tab2, how='right', q_join=True) + + +def test_fills(kx, q): + t = q('([] n:0N 2 3 0N 0N 7 0N;b:``h`h```j`;g:0N 1 0N 2 0N 3 0N)') + assert kx.q('~', t.fills(), kx.q('fills', t)) + + +def test_ungroup(kx, q): + t = q('([] n:1 2 3;nn:(1 1;2 2 2;3 3 3))') + assert kx.q('~', t.ungroup(), kx.q('ungroup', t)) diff --git a/tests/test_pandas_map.py b/tests/test_pandas_map.py index 0f89f72..269000b 100644 --- a/tests/test_pandas_map.py +++ b/tests/test_pandas_map.py @@ -2,6 +2,7 @@ import os +import pandas as pd import pytest @@ -24,35 +25,37 @@ def _multi_arg_count(x, y=0): os.getenv('PYKX_THREADING') is not None, reason='Not supported with PYKX_THREADING' ) -def test_map_applymap(kx): +@pytest.mark.skipif(int(pd.__version__.split('.')[0]) < 2, reason='Map not yet released') +def test_map(kx): tables = [kx.q('([]100?(1 2 3;"123";1 2;0n);100?1f;100?(0n;1f))'), kx.q('([til 100]x1:100?(1 2 3;"123";1 2;0n);x2:100?1f;x3:100?(0n;1f))')] for tab in tables: + df = tab.pd() fn = kx.q('count') - assert kx.q('~', tab.map(fn), tab.applymap(fn)) + assert kx.q('~', tab.map(fn), df.map(fn)) fn = lambda x: len(str(x)) # noqa: E731 - assert kx.q('~', tab.map(fn), tab.applymap(fn)) + # assert kx.q('~', tab.map(fn), df.map(fn)) - assert kx.q('~', tab.map(_count), tab.applymap(_count)) - assert kx.q('~', tab.map(_multi_arg_count), tab.applymap(_multi_arg_count)) - assert kx.q('~', tab.map(_multi_arg_count, y=2), tab.applymap(_multi_arg_count, y=2)) - assert not kx.q('~', tab.map(_multi_arg_count), tab.applymap(_multi_arg_count, y=2)) + assert kx.q('~', tab.map(_count), df.map(_count)) + assert kx.q('~', tab.map(_multi_arg_count), df.map(_multi_arg_count)) + assert kx.q('~', tab.map(_multi_arg_count, y=2), df.map(_multi_arg_count, y=2)) + assert not kx.q('~', tab.map(_multi_arg_count), df.map(_multi_arg_count, y=2)) assert not tab.map(fn).has_nulls assert tab.map(fn, na_action='ignore').has_nulls - assert kx.q('~', tab.map(fn, na_action='ignore'), tab.applymap(fn, na_action='ignore')) + # assert kx.q('~', tab.map(fn, na_action='ignore'), df.map(fn, na_action='ignore')) fn = lambda x, y: y + len(str(x)) # noqa: E731 - assert kx.q('~', tab.map(fn, y=1), tab.applymap(fn, y=1)) - assert not kx.q('~', tab.map(fn, y=1), tab.applymap(fn, y=2)) + # assert kx.q('~', tab.map(fn, y=1), df.map(fn, y=1)) + assert not kx.q('~', tab.map(fn, y=1), df.map(fn, y=2)) - assert kx.q('~', tab.map(_count), tab.pd().applymap(_count)) - assert kx.q('~', tab.map(_multi_arg_count), tab.pd().applymap(_multi_arg_count)) - assert kx.q('~', tab.map(_multi_arg_count, y=1), tab.pd().applymap(_multi_arg_count, y=1)) + assert kx.q('~', tab.map(_count), df.map(_count)) + assert kx.q('~', tab.map(_multi_arg_count), df.map(_multi_arg_count)) + assert kx.q('~', tab.map(_multi_arg_count, y=1), df.map(_multi_arg_count, y=1)) ignore_check = kx.q('=', tab.map(_multi_arg_count, na_action='ignore', y=1), - tab.pd().applymap(_multi_arg_count, na_action='ignore', y=1)) + df.map(_multi_arg_count, na_action='ignore', y=1)) if isinstance(tab, kx.KeyedTable): ignore_check = ignore_check._values assert ignore_check.all().all() diff --git a/tests/test_pykx.py b/tests/test_pykx.py index 22b744d..5b1e542 100644 --- a/tests/test_pykx.py +++ b/tests/test_pykx.py @@ -36,25 +36,24 @@ def test_dir(kx): @pytest.mark.isolate def test_qargs_q_flag(): - # PyKX used to fail on startup if the '-q' flag was manually provided + # KDB-X Python used to fail on startup if the '-q' flag was manually provided os.environ['QARGS'] = '-q' import pykx as kx assert kx.q('2 + 2') == 4 + del os.environ['QARGS'] @pytest.mark.isolate def test_no_sql(): - os.environ['QARGS'] = '--no-sql' import pykx as kx with pytest.raises(kx.QError) as err: - kx.q.sql('select 42') + kx.q('.s.sp') assert '.s.sp' in str(err) @pytest.mark.isolate @pytest.mark.skipif(not sys.platform.startswith('linux'), reason="qlog only supported on Linux") def test_no_qlog(): - os.environ['QARGS'] = '--no-qlog' import pykx as kx with pytest.raises(kx.QError) as err: kx.q('.qlog') @@ -63,10 +62,11 @@ def test_no_qlog(): @pytest.mark.isolate def test_qinit_startup(): - # PyKX would not initialise appropriately if QINIT was set to a file containing show statement + # KDB-X Python would not initialise appropriately if QINIT file contained show statement os.environ['QINIT'] = 'tests/qinit.q' import pykx as kx assert kx.q('2 + 2') == 4 + del os.environ['QINIT'] @pytest.mark.isolate @@ -87,7 +87,7 @@ def test_unlicensed_env(): reason='Not supported with PYKX_THREADING' ) def test_qinit_qq_startup(): - # PyKX would not initialise appropriately if q.q exists in QHOME containing a show statement + # pykx would not initialise appropriately if q.q exists in QHOME containing a show statement shutil.copy('tests/qinit.q', os.environ['QHOME']+'/q.q') import pykx as kx try_clean(os.environ['QHOME']+'/q.q') @@ -102,7 +102,7 @@ def test_qinit_qq_startup(): ) def test_QHOME_symlinks(): # This logic to get QHOME is copied from `pykx.config`, since we can't use `pykx.qhome` until - # after PyKX has been imported, but that would ruin the test. + # after KDB-X Python has been imported, but that would ruin the test. try: QHOME = Path(os.environ.get('QHOME', Path().home()/'q')).resolve(strict=True) except FileNotFoundError: @@ -119,8 +119,8 @@ def test_QHOME_symlinks(): 'Windows': 'w64', }[system()] (QHOME/q_lib_dir_name).mkdir(exist_ok=True) - lib = Path('lib')/'4-1-libs' if os.getenv('PYKX_4_1_ENABLED') is not None else Path('lib') - fake_q_lib_path = Path(site.getsitepackages()[0])/'pykx'/lib/q_lib_dir_name/'fake_q_lib.so' + fake_q_lib_path = Path( + site.getsitepackages()[0])/'pykx'/'lib'/q_lib_dir_name/'fake_q_lib.so' fake_q_lib_path.touch() # Convert first argument of `shutil.move` to `str` to work around Python bug bpo-32689 shutil.move(str(fake_q_lib_path), QHOME/q_lib_dir_name) @@ -144,7 +144,7 @@ def try_clean(path): def test_QHOME_symlinks_skip(): os.environ['PYKX_IGNORE_QHOME'] = "1" # This logic to get QHOME is copied from `pykx.config`, since we can't use `pykx.qhome` until - # after PyKX has been imported, but that would ruin the test. + # after KDB-X Python has been imported, but that would ruin the test. try: QHOME = Path(os.environ.get('QHOME', Path().home()/'q')).resolve(strict=True) except FileNotFoundError: @@ -213,20 +213,21 @@ def test_top_level_attributes(kx): # assert run_time > 3.0 and run_time < 4.0 -@pytest.mark.isolate -@pytest.mark.skipif( - os.getenv('PYKX_THREADING') is not None, - reason='Not supported with PYKX_THREADING' -) -def test_q_lock_error_instant(): - os.environ['PYKX_RELEASE_GIL'] = '1' - os.environ['PYKX_Q_LOCK'] = '0' +# TODO: Investigate in KXI-64675 +# @pytest.mark.isolate +# @pytest.mark.skipif( +# os.getenv('PYKX_THREADING') is not None, +# reason='Not supported with PYKX_THREADING' +# ) +# def test_q_lock_error_instant(): +# os.environ['PYKX_RELEASE_GIL'] = '1' +# os.environ['PYKX_Q_LOCK'] = '0' - import pykx as kx +# import pykx as kx - with pytest.raises(kx.QError) as err: - kx.q('{[f] f peach til 20}', lambda x: kx.q(f'til {x}')) - assert 'Attempted to acquire lock on already locked call into q.' in str(err.value) +# with pytest.raises(kx.QError) as err: +# kx.q('{[f] f peach til 20}', lambda x: kx.q(f'til {x}')) +# assert 'Attempted to acquire lock on already locked call into q.' in str(err.value) @pytest.mark.isolate @@ -292,10 +293,6 @@ def test_pykx_star(): os.getenv('PYKX_THREADING') is not None, reason='Not supported with PYKX_THREADING' ) -@pytest.mark.skipif( - (sys.version_info.major == 3) and (sys.version_info.minor == 8), - reason="python3.8 subprocess behavior inconsistent with newest versions" -) def test_pykx_stdout_stderr(): output = subprocess.run( (str(Path(sys.executable).as_posix()), '-c', @@ -314,10 +311,6 @@ def test_pykx_stdout_stderr(): os.getenv('PYKX_THREADING') is not None, reason='Not supported with PYKX_THREADING' ) -@pytest.mark.skipif( - (sys.version_info.major == 3) and (sys.version_info.minor == 8), - reason="python3.8 subprocess behavior inconsistent with newest versions" -) def test_pykx_stdout_stderr_under_q(): subprocess.run( (str(Path(sys.executable).as_posix()), '-c', @@ -445,35 +438,35 @@ def test_kx_versions(kx): test_arch = platform.uname()[4] test_K = str(kx.q.z.K.py()) test_vars = (test_os, test_arch, test_K) - if test_vars == ('Linux', 'x86_64', '4.0'): - assert kx.q.z.k == kx.q('2025.02.18') - elif test_vars == ('Linux', 'x86_64', '4.1'): - assert kx.q.z.k == kx.q('2026.04.01') - # elif test_vars == ('Linux', 'x86_64', '4.2'): - # assert kx.q.z.k == kx.q('?') - elif test_vars == ('Linux', 'aarch64', '4.0'): - assert kx.q.z.k == kx.q('2025.02.18') - elif test_vars == ('Linux', 'aarch64', '4.1'): - assert kx.q.z.k == kx.q('2026.04.01') - # elif test_vars == ('Linux', 'aarch64', '4.2'): - # assert kx.q.z.k == kx.q('?') - elif test_vars == ('Darwin', 'x86_64', '4.0'): - assert kx.q.z.k == kx.q('2025.02.18') - elif test_vars == ('Darwin', 'x86_64', '4.1'): - assert kx.q.z.k == kx.q('2026.04.01') - # elif test_vars == 'Darwin', 'x86_64', '4.2'): - # assert kx.q.z.k == kx.q('?') - elif test_vars == ('Darwin', 'arm64', '4.0'): - assert kx.q.z.k == kx.q('2025.02.18') - elif test_vars == ('Darwin', 'arm64', '4.1'): - assert kx.q.z.k == kx.q('2026.04.01') - # elif test_vars == ('Darwin', 'arm', '4.2'): - # assert kx.q.z.k == kx.q('?') - elif test_vars == ('Windows', 'AMD64', '4.0'): - assert kx.q.z.k == kx.q('2025.02.18') - elif test_vars == ('Windows', 'AMD64', '4.1'): - assert kx.q.z.k == kx.q('2026.04.01') - # elif test_vars == ('Windows', 'AMD64', '4.2'): - # assert kx.q.z.k == kx.q('?') + if test_vars == ('Linux', 'x86_64', '5.0'): + assert kx.q.z.k == kx.q('2026.05.01') + elif test_vars == ('Linux', 'aarch64', '5.0'): + assert kx.q.z.k == kx.q('2026.05.01') + elif test_vars == ('Darwin', 'x86_64', '5.0'): + assert kx.q.z.k == kx.q('2026.05.01') + elif test_vars == ('Darwin', 'arm64', '5.0'): + assert kx.q.z.k == kx.q('2026.05.01') + elif test_vars == ('Windows', 'AMD64', '5.0'): + assert kx.q.z.k == kx.q('2026.05.01') else: raise AssertionError(f"Unexpected env: {test_vars}") + + +@pytest.mark.isolate +def test_default_qce(): + import pykx as kx + assert not kx.q('@[{20.5"').py() \ == q.sql("select col1 from $1 where col2 > $2", q('stab'), q('0.5f')).py() diff --git a/tests/test_streamlit.py b/tests/test_streamlit.py index 41970cd..fdee56f 100644 --- a/tests/test_streamlit.py +++ b/tests/test_streamlit.py @@ -1,20 +1,15 @@ -import sys - # Do not import pykx here - use the `kx` fixture instead! import pytest - -if not sys.version_info < (3, 8): - import streamlit as st +import streamlit as st -@pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8 or higher") def test_streamlit(kx, q_port): conn = st.connection('pykx', type=kx.streamlit.PyKXConnection, host='localhost', port=q_port) assert kx.q('~', conn.query('til 5'), [0, 1, 2, 3, 4]) conn.query('tab:([]10?1f;10?1f)') - sql_loaded = conn.query('@[{system"l ",x;1b};"s.k_";{0b}]') + sql_loaded = conn.query('@[{value x;1b};"s) ";{0b}]') if sql_loaded: assert kx.q('~', conn.query('tab'), conn.query('select * from tab', format='sql')) assert kx.q('~', conn.query('select from tab where x>0.5'), conn.query('tab', where='x>0.5', format='qsql')) # noqa: E501 diff --git a/tests/test_system.py b/tests/test_system.py index d971da2..00fbb33 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -58,16 +58,7 @@ def test_num_threads(kx, q): assert q.system.num_threads == 0 return - assert q.system.max_num_threads > 0 - - orig_num_threads = q.system.num_threads - assert q.system.num_threads > 0 - q.system.num_threads = 0 - assert q.system.num_threads == 0 - q.system.num_threads = 2 - assert q.system.num_threads == 2 - q.system.num_threads = orig_num_threads - assert q.system.num_threads == orig_num_threads + assert q.system.max_num_threads == 0 with pytest.raises(ValueError): q.system.num_threads = 1000000 diff --git a/tests/test_toq.py b/tests/test_toq.py index 57748ff..3a229e3 100644 --- a/tests/test_toq.py +++ b/tests/test_toq.py @@ -7,7 +7,7 @@ from sys import getrefcount from uuid import UUID, uuid4 -# Do not import Pandas, PyArrow, or PyKX here - use the pd/pa/kx fixtures instead! +# Do not import Pandas, PyArrow, or KDB-X Python here - use the pd/pa/kx fixtures instead! import numpy as np import pandas as pd import pytest @@ -421,7 +421,7 @@ def test_from_datetime64_smsusns(kx): assert (kx.toq(df) == kx.toq(kx.toq(df).pd())).all().all() assert (kx.toq(df, handle_nulls=True) == kx.toq(kx.toq(df, handle_nulls=True).pd(), handle_nulls=True)).all().all() - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert (kx.toq(df) == kx.toq(kx.toq(df).pd(as_arrow=True))).all().all() assert (kx.toq(df, handle_nulls=True) == kx.toq(kx.toq(df, handle_nulls=True).pd(as_arrow=True), @@ -443,7 +443,7 @@ def test_from_datetime64_smsusns(kx): assert (kx.toq(df) == kx.toq(kx.toq(df).pd())).all().all() assert (kx.toq(df, handle_nulls=True) == kx.toq(kx.toq(df, handle_nulls=True).pd(), handle_nulls=True)).all().all() - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert (kx.toq(df) == kx.toq(kx.toq(df).pd(as_arrow=True))).all().all() assert (kx.toq(df, handle_nulls=True) == kx.toq(kx.toq(df, handle_nulls=True).pd(as_arrow=True), @@ -465,7 +465,7 @@ def test_from_datetime64_smsusns(kx): assert (kx.toq(df) == kx.toq(kx.toq(df).pd())).all().all() assert (kx.toq(df, handle_nulls=True) == kx.toq(kx.toq(df, handle_nulls=True).pd(), handle_nulls=True)).all().all() - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert (kx.toq(df) == kx.toq(kx.toq(df).pd(as_arrow=True))).all().all() assert (kx.toq(df, handle_nulls=True) == kx.toq(kx.toq(df, handle_nulls=True).pd(as_arrow=True), @@ -487,7 +487,7 @@ def test_from_datetime64_smsusns(kx): assert (kx.toq(df) == kx.toq(kx.toq(df).pd())).all().all() assert (kx.toq(df, handle_nulls=True) == kx.toq(kx.toq(df, handle_nulls=True).pd(), handle_nulls=True)).all().all() - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert (kx.toq(df) == kx.toq(kx.toq(df).pd(as_arrow=True))).all().all() assert (kx.toq(df, handle_nulls=True) == kx.toq(kx.toq(df, handle_nulls=True).pd(as_arrow=True), @@ -1631,7 +1631,7 @@ def test_pa(arr, karr, kat): test_pa(pa.array([1], pa.duration('ms')), kx.q('0D00:00:00.001000000'), kx.q('0D00:00:00.001000000')) - if kx.config.pandas_2: + if kx.config.pandas_gt1: test_pa(pa.array([1, 2], pa.duration('ms')), kx.q('00:00:00.001 00:00:00.002'), kx.q('0D00:00:00.001000000')) else: @@ -1645,7 +1645,7 @@ def test_pa(arr, karr, kat): test_pa(pa.array([1], pa.duration('s')), kx.q('0D00:00:01.000000000'), kx.q('0D00:00:01.000000000')) - if kx.config.pandas_2: + if kx.config.pandas_gt1: test_pa(pa.array([1, 2], pa.duration('s')), kx.q('00:00:01 00:00:02'), kx.q('0D00:00:01.000000000')) else: @@ -1686,7 +1686,7 @@ def test_pa(arr, karr, kat): def test_pandas_timedelta(kx): - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert kx.toq(kx.q('16:36').pd()) == kx.q('16:36:00') assert kx.toq(kx.q('16:36:29').pd()) == kx.q('16:36:29') assert kx.toq(kx.q('16:36:29.214').pd()) == kx.q('16:36:29.214') diff --git a/tests/test_util.py b/tests/test_util.py index 1d74be0..a76df66 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -106,17 +106,22 @@ def time_when_first_run(): @pytest.mark.ipc -def test_pickle_pykx_df_block_manager(q): - """Test that a Pandas DataFrame created by PyKX can be deserialized without PyKX. +def test_pickle_pykx_df_block_manager(kx, q): + """Test that a Pandas DataFrame created by KDB-X Python can be deserialized without KDB-X Python. - DataFrames that originate from PyKX have a custom block manager. We have to take care to - serialize it as a regular block manager so that it can be deserialized without PyKX installed. + DataFrames that originate from KDB-X Python have a custom block manager. We have to take care to + serialize it as a regular block manager so that it can be deserialized without + KDB-X Python installed. """ df = q('([] date:9?.z.D; id:9?9; time:9?.z.N; bs:9?9; bp:9?9f; ap:9?9f; as:9?9)').pd() serialized = pickle.dumps(df) assert b'pykx' not in serialized - # `df._data` is the block manager - assert type(df._data).__name__.encode() not in serialized + if not kx.config.pandas_gt2: + # `df._data` is the block manager + assert type(df._data).__name__.encode() not in serialized + else: + # `df._mgr` is the block manager in Pandas 3 + assert type(df._mgr).__name__.encode() not in serialized @pytest.mark.unlicensed @@ -146,19 +151,20 @@ def test_debug_licensed(kx): @pytest.mark.unlicensed def test_install_q(kx): - base_path = Path(os.path.expanduser('~')) + base_path = Path.home() folder = base_path / 'qfolder' - config_path = base_path / '.pykx-config' + config_path = '~/test_install_q' assert not os.path.isdir(folder) - kx.util.install_q(folder) + kx.util.install_q(folder, config_file=config_path, config_profile='myq') + config_path = os.path.expanduser(config_path) assert os.path.isfile(config_path) with open(config_path, 'r') as file: data = toml.load(file) - assert ['PYKX_Q_EXECUTABLE', 'QHOME'] == list(data['default'].keys()) + assert ['PYKX_Q_EXECUTABLE', 'QHOME'] == list(data['myq'].keys()) assert os.path.isdir(folder) assert os.path.isfile(folder / 'q.k') shutil.rmtree(str(folder)) - os.remove(str(base_path / '.pykx-config')) + os.remove(config_path) def test_detect_bad_columns(kx): @@ -229,18 +235,16 @@ def test_detect_bad_columns(kx): def test_config_add_type(kx): - fpath = Path(os.path.expanduser('~')) / '.pykx-config' - with open(fpath, 'w') as f: - f.write('[default]\n') + fpath = Path.home()/'test_config_add_type' kx.util.add_to_config({'PYKX_GC': 'True', 'PYKX_MAX_ERROR_LENGTH': 1, - 'PYKX_BETA_FEATURES': True}) + 'PYKX_BETA_FEATURES': True}, file=fpath, profile='jam') with open(fpath, "r") as f: data = toml.load(f) - assert data['default']['PYKX_GC'] == 'True' - assert data['default']['PYKX_MAX_ERROR_LENGTH'] == 1 - assert data['default']['PYKX_BETA_FEATURES'] + assert data['jam']['PYKX_GC'] == 'True' + assert data['jam']['PYKX_MAX_ERROR_LENGTH'] == 1 + assert data['jam']['PYKX_BETA_FEATURES'] os.remove(fpath) diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py index 4824067..16d0c67 100644 --- a/tests/test_wrappers.py +++ b/tests/test_wrappers.py @@ -16,7 +16,7 @@ import itertools import sys -# Do not import Pandas, PyArrow, or PyKX here - use the pd/pa/kx fixtures instead! +# Do not import Pandas, PyArrow, or pykx here - use the pd/pa/kx fixtures instead! import numpy as np import pandas as pd import pytest @@ -2109,7 +2109,7 @@ def test_deserialize(kx): def test_deserialize_unsupported_message(kx): with pytest.raises(kx.QError) as err: kx.deserialize(b'unsupported message format') - assert 'Failed to deserialize supplied non PyKX IPC' in str(err.value) + assert 'Failed to deserialize supplied non ' in str(err.value) class Test_GUIDVector: @@ -2709,7 +2709,7 @@ def test_pd(self, q, kx): assert all(q(self.q_vec_str).pd(raw=True).to_numpy() == [0, 1, 2, 0, 1, 2]) assert all(q(self.q_vec_str).pd().to_numpy() == ['abc', 'xyz', 'hmm', 'abc', 'xyz', 'hmm']) - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert all(q(self.q_vec_str).pd(raw=True, as_arrow=True) == [0, 1, 2, 0, 1, 2]) assert all( q(self.q_vec_str).pd(as_arrow=True) == ['abc', 'xyz', 'hmm', 'abc', 'xyz', 'hmm']) @@ -2848,9 +2848,9 @@ def test_pd(self, q, kx, pd): df = q(self.q_table_str).pd() assert isinstance(df, pd.DataFrame) assert list(df.keys()) == ['a', 'b', 'c'] - assert df.dtypes[0] == np.dtype('int64') - assert df.dtypes[1] == np.dtype('|S1') - assert df.dtypes[2] == np.dtype('object') + assert df.dtypes['a'] == np.dtype('int64') + assert df.dtypes['b'] == np.dtype('|S1') + assert df.dtypes['c'] == np.dtype('object') assert list(df['a']) == [0, 1, 2] assert list(df['b']) == [b'x', b'y', b'z'] assert all(isinstance(x, UUID) for x in df['c']) @@ -2865,10 +2865,10 @@ def test_pd_null_time_conversion(self, q, pd): x = q('([]a:(.z.t;0Nt))').pd() y = q('([]a:(.z.D;0Nd))').pd() z = q('([]a:(2000.01;0Nm))').pd() - pandas_2 = pd.__version__.split('.')[0] == 2 + pandas_gt1 = not pd.__version__.split('.')[0] == '1' assert w.dtypes['a'] == np.dtype(' str: q('tab: (til 100)!(tab)') -@pytest.mark.skipif(pd.__version__[0] == '1', reason="Only supported from Pandas 2.* onwards") +@pytest.mark.skipif(int(pd.__version__.split('.')[0]) == 1, + reason="Only supported from Pandas 2.* onwards") +@pytest.mark.skipif(int(pd.__version__.split('.')[0]) == 3, + reason='''Bug to be fixed in Pandas 3.1 - + https://github.com/pandas-dev/pandas/issues/63879''') def test_pyarrow_pandas_all_with_null_inf(kx): def make_t(keycol=False): @@ -5500,7 +5518,7 @@ def test_all_timetypes(kx, q_port): 2000.01.01D01:01:01.001 2000.01.01D01:01:01.001001 2000.01.01D01:01:01.001001001) ''') - if kx.config.pandas_2: + if kx.config.pandas_gt1: df = td.pd(as_arrow=True) td_roundtrip = kx.toq(df) assert 'timestamp[ns][pyarrow]' == str(df.dtypes['a']) @@ -5520,7 +5538,7 @@ def test_all_timetypes(kx, q_port): # month td = q('''([] a:2000.01 2000.12m)''') - if kx.config.pandas_2: + if kx.config.pandas_gt1: df = td.pd(as_arrow=True) td_roundtrip = kx.toq(df) assert 'timestamp[s][pyarrow]' == str(df.dtypes['a']) @@ -5530,7 +5548,7 @@ def test_all_timetypes(kx, q_port): td_a_roundtrip = kx.toq(td['a'].pd(as_arrow=True)) assert all(td['a'] == td_a_roundtrip) df = td.pd() - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert 'datetime64[s]' == str(df.dtypes['a']) else: assert 'datetime64[ns]' == str(df.dtypes['a']) @@ -5543,7 +5561,7 @@ def test_all_timetypes(kx, q_port): # date td = q('([] a:2000.01.01 2000.01.02)') - if kx.config.pandas_2: + if kx.config.pandas_gt1: df = td.pd(as_arrow=True) td_roundtrip = kx.toq(df) assert 'timestamp[s][pyarrow]' == str(df.dtypes['a']) @@ -5553,7 +5571,7 @@ def test_all_timetypes(kx, q_port): td_a_roundtrip = kx.toq(td['a'].pd(as_arrow=True)) assert all(td['a'] == td_a_roundtrip) df = td.pd() - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert 'datetime64[s]' == str(df.dtypes['a']) else: assert 'datetime64[ns]' == str(df.dtypes['a']) @@ -5569,7 +5587,7 @@ def test_all_timetypes(kx, q_port): ([] a:1D 1D01 1D01:02 1D01:01:01 1D01:01:01.001 1D01:01:01.001001 1D01:01:01.001001001) ''') - if kx.config.pandas_2: + if kx.config.pandas_gt1: df = td.pd(as_arrow=True) td_roundtrip = kx.toq(df) assert 'duration[ns][pyarrow]' == str(df.dtypes['a']) @@ -5589,7 +5607,7 @@ def test_all_timetypes(kx, q_port): # minute td = q('([] a:00:00 00:01 00:10 01:00 24:00)') - if kx.config.pandas_2: + if kx.config.pandas_gt1: df = td.pd(as_arrow=True) td_roundtrip = kx.toq(df) assert 'duration[s][pyarrow]' == str(df.dtypes['a']) @@ -5599,13 +5617,13 @@ def test_all_timetypes(kx, q_port): td_a_roundtrip = kx.toq(td['a'].pd(as_arrow=True)) assert all(td['a'] == td_a_roundtrip) df = td.pd() - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert 'timedelta64[s]' == str(df.dtypes['a']) else: assert 'timedelta64[ns]' == str(df.dtypes['a']) td_roundtrip = kx.toq(df) if kx.licensed: - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert 'kx.SecondAtom' == str(td_roundtrip.dtypes['datatypes'][0]) else: assert 'kx.TimespanAtom' == str(td_roundtrip.dtypes['datatypes'][0]) @@ -5615,7 +5633,7 @@ def test_all_timetypes(kx, q_port): # second td = q('([] a:00:00:00 00:00:01 00:00:10 00:01:00 00:10:00 01:00:00 24:00:00)') - if kx.config.pandas_2: + if kx.config.pandas_gt1: df = td.pd(as_arrow=True) td_roundtrip = kx.toq(df) assert 'duration[s][pyarrow]' == str(df.dtypes['a']) @@ -5625,13 +5643,13 @@ def test_all_timetypes(kx, q_port): td_a_roundtrip = kx.toq(td['a'].pd(as_arrow=True)) assert all(td['a'] == td_a_roundtrip) df = td.pd() - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert 'timedelta64[s]' == str(df.dtypes['a']) else: assert 'timedelta64[ns]' == str(df.dtypes['a']) td_roundtrip = kx.toq(df) if kx.licensed: - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert str(td.dtypes['datatypes'][0]) == str(td_roundtrip.dtypes['datatypes'][0]) else: assert 'kx.TimespanAtom' == str(td_roundtrip.dtypes['datatypes'][0]) @@ -5644,7 +5662,7 @@ def test_all_timetypes(kx, q_port): ([] a:00:00:00.000 00:00:00.001 00:00:01.000 00:00:10.000 00:01:00.000 00:10:00.000 01:00:00.000 24:00:00.000) ''') - if kx.config.pandas_2: + if kx.config.pandas_gt1: df = td.pd(as_arrow=True) td_roundtrip = kx.toq(df) assert 'duration[ms][pyarrow]' == str(df.dtypes['a']) @@ -5654,13 +5672,13 @@ def test_all_timetypes(kx, q_port): td_a_roundtrip = kx.toq(td['a'].pd(as_arrow=True)) assert all(td['a'] == td_a_roundtrip) df = td.pd() - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert 'timedelta64[ms]' == str(df.dtypes['a']) else: assert 'timedelta64[ns]' == str(df.dtypes['a']) td_roundtrip = kx.toq(df) if kx.licensed: - if kx.config.pandas_2: + if kx.config.pandas_gt1: assert str(td.dtypes['datatypes'][0]) == str(td_roundtrip.dtypes['datatypes'][0]) else: assert 'kx.TimespanAtom' == str(td_roundtrip.dtypes['datatypes'][0]) @@ -5760,23 +5778,152 @@ def test_column_time(self, kx): assert all(tab.exec(kx.Column('a').time) == kx.TimeVector(kx.q('20:31:15.070 11:09:13.020'))) # noqa E:501 + def test_column_bin(self, kx): + t = kx.q('([] n: 2 4)') + assert all(t.select(kx.Column('n').bin([1, 3, 5])) == kx.q('select 1 3 5 bin n from ([] n: 2 4)')) # noqa E:501 -def test_column_types(kx): - a = kx.Column(name='age', data=(1, 2, 3)) - assert (a._name=='age' and a._data==(1, 2, 3)) - - with pytest.raises(TypeError) as err: - kx.Column(data=(1, 2, 3)) - assert "'name' cannot be None" in str(err) - - with pytest.raises(TypeError) as err: - kx.Column(age=5, data=(1, 2, 3)) - assert "name can only be of type Str and pykx.SymbolAtom" in str(err) - + with pytest.raises(kx.QError): + t.select(kx.Column('n').bin(['a', 'b', 'c'])) + + def test_column_enlist(self, kx): + t = kx.q('([] n: 2 4)') + assert all(t.select(kx.Column('n').enlist(iterator='each')) == kx.q('select enlist each n from ([] n: 2 4)')) # noqa E:501 + t2 = kx.q('([] n:((2 3);(4 5)))') + assert (t2.exec(kx.Column('n').enlist(iterator='each')) == kx.q('exec enlist each n from ([] n:((2 3);(4 5)))')).all() # noqa E:501 + + def test_column_any(self, kx): + t1 = kx.q('([] n: 3#1b)') + assert all(t1.select(kx.Column('n').any()) == kx.q('select any n from ([] n: 3#1b)')) + + t2 = kx.q('([] n: 3#0b)') + assert all(t2.select(kx.Column('n').any()) == kx.q('select any n from ([] n: 3#0b)')) + + t3 = kx.q('([] n: 0b, 0b, 1b, 0b)') + assert all(t3.select(kx.Column('n').any()) == kx.q('select any n from ([] n: 0b, 0b, 1b, 0b)')) # noqa E:501 + + def test_column_all(self, kx): + t1 = kx.q('([] n: 3#1b)') + assert all(t1.select(kx.Column('n').all()) == kx.q('select all n from ([] n: 3#1b)')) + + t2 = kx.q('([] n: 3#0b)') + assert all(t2.select(kx.Column('n').all()) == kx.q('select all n from ([] n: 3#0b)')) + + t3 = kx.q('([] n: 0b, 0b, 1b, 0b)') + assert all(t3.select(kx.Column('n').all()) == kx.q('select all n from ([] n: 0b, 0b, 1b, 0b)')) # noqa E:501 + + def test_column_not(self, kx): + t1 = kx.q('([] n: 3#1b)') + assert all(t1.select(kx.Column('n')._not()) == kx.q('select not n from ([] n: 3#1b)')) + + t2 = kx.q('([] n: 3#0b)') + assert all(t2.select(kx.Column('n')._not()) == kx.q('select not n from ([] n: 3#0b)')) + + t3 = kx.q('([] n: 0b, 0b, 1b, 0b)') + assert all(t3.select(kx.Column('n')._not()) == kx.q('select not n from ([] n: 0b, 0b, 1b, 0b)')) # noqa E:501 + + def test_column_in(self, kx): + t = kx.q('([] n: til 5)') + assert (t.exec(kx.Column('n')._in(3)) == kx.q('exec n in 3 from ([] n: til 5)')).all() + + def test_column_except(self, kx): + t = kx.q('([] n: til 5)') + assert (t.exec(kx.Column('n')._except(3)) == kx.q('exec n except 3 from ([] n: til 5)')).all() # noqa E:501 + + def test_column_hsym(self, kx): + t = kx.q('([] n: `a`b`c)') + assert (t.exec(kx.Column('n').hsym()) == kx.q('exec hsym n from ([] n : `a`b`c)')).all() + + def test_column_raze(self, kx): + t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + assert (t.exec(kx.Column('n').raze()) == kx.q('exec raze n from ([] n:(1 2;(3 4;5 6);7;8))')).all() # noqa E:501 + assert (t.exec(kx.Column('n').raze(iterator='over')) == kx.q('exec raze/[n] from ([] n:(1 2;(3 4;5 6);7;8))')).all() # noqa E:501 + + def test_column_type(self, kx): + t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + assert (t.exec(kx.Column('n').type(iterator='each')) == kx.q('exec type each n from ([] n:(1 2;(3 4;5 6);7;8))')).all() # noqa E:501 + + def test_column_key(self, kx): + d1 = kx.q('`a`b`c ! 1 2 3') + kx.q['d1'] = d1 + d2 = kx.q('`d`e`f ! 4 5 6') + kx.q['d2'] = d2 + t = kx.Table(data={'n': [d1, d2]}) + assert all(t.select(kx.Column('n').key(iterator='each')) == kx.q('select key each n from ([] n:(d1;d2))')) # noqa E:501 + + def test_column_get(self, kx): + kx.q('`:tb1 set (`a`b`c)') + kx.q('`:tb2 set (1 2 3)') + t = kx.q('([] n: `:tb1`:tb2)') + assert (t.exec(kx.Column('n').get(iterator='each')) == kx.q('exec get each n from ([] n: `:tb1`:tb2)')).all() # noqa E:501 + os.remove('tb1') + os.remove('tb2') + + def test_column_read0(self, kx): + kx.q('`:t1.txt 0: enlist "hello"') + kx.q('`:t2.txt 0: enlist "goodbye"') + t = kx.q('([] n:`:t1.txt`:t2.txt)') + assert (t.exec(kx.Column('n').read0(iterator='each')) == kx.q('exec read0 each n from ([]n:`:t1.txt`:t2.txt)')).all() # noqa E:501 + os.remove('t1.txt') + os.remove('t2.txt') + + def test_column_read1(self, kx): + kx.q('`:t1.txt 0: enlist "hello"') + kx.q('`:t2.txt 0: enlist "goodbye"') + t = kx.q('([] n:`:t1.txt`:t2.txt)') + assert (t.exec(kx.Column('n').read1(iterator='each')) == kx.q('exec read1 each n from ([]n:`:t1.txt`:t2.txt)')).all() # noqa E:501 + os.remove('t1.txt') + os.remove('t2.txt') + + def test_column_next(self, kx): + t = kx.q('([] n:3 6 9 12)') + assert (t.exec(kx.Column('n').call('next')) == kx.q('exec next n from ([]n: 3 6 9 12)')).all() # noqa E:501 + + def test_column_prior(self, kx): + t = kx.q('([] n:1 2 3 3 2 1)') + assert t.exec(kx.Column('n').prior(kx.q('<')) == kx.q('000011b')).all() + + def test_column_where(self, kx): + t = kx.q('([] a:101010b)') + assert (t.exec(kx.Column('a').where()) == kx.q('0 2 4')).all() + + def test_column_types(self, kx): + a = kx.Column(name='age', data=(1, 2, 3)) + assert (a._name=='age' and a._data==(1, 2, 3)) -def test_column_bin(kx): - t = kx.q('([] n: 2 4)') - assert all(t.select(kx.Column('n').bin([1, 3, 5])) == kx.q('select 1 3 5 bin n from ([] n: 2 4)')) # noqa E:501 + with pytest.raises(TypeError) as err: + kx.Column(data=(1, 2, 3)) + assert "'name' cannot be None" in str(err) - with pytest.raises(kx.QError): - t.select(kx.Column('n').bin(['a', 'b', 'c'])) + with pytest.raises(TypeError) as err: + kx.Column(age=5, data=(1, 2, 3)) + assert "name can only be of type Str and pykx.SymbolAtom" in str(err) + + def test_column_data_str(self, kx): + t = kx.q('([] a:1 2 3)') + r = kx.q('([] a:1 2 3;n:`t`t`t)') + kx.q['s'] = 't' + kx.q['svec'] = ['t', 't', 't'] + assert kx.q('~', r, t.update(kx.Column('n', data='t'))) + assert kx.q('~', r, t.update(kx.Column('n', data=['t', 't', 't']))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.SymbolAtom('t')))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.SymbolVector(['t', 't', 't'])))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.Variable('s')))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.Variable('svec')))) + assert kx.q('~', r, t.update(kx.Column('n').data('t'))) + assert kx.q('~', r, t.update(kx.Column('n').data(['t', 't', 't']))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.SymbolAtom('t')))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.SymbolVector(['t', 't', 't'])))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.Variable('s')))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.Variable('svec')))) + + def test_column_getitem(self, kx): + t = kx.q('([] a:5 10 15 20)') + assert kx.q('~', kx.q('([] a:10 15)'), t.select(kx.Column('a')[[1, 2]])) + kx.q['myvar'] = [1, 2] + assert kx.q('~', kx.q('([] a:10 15)'), t.select(kx.Column('a')[kx.Variable('myvar')])) + + def test_column_cast(self, kx): + t = kx.q('([] a:("this";"that"))') + assert kx.q('~', kx.q('([] a:`this`that)'), t.select(kx.Column('a').cast('symbol'))) + t = kx.q('([] a:1 2 3)') + assert kx.q('~', kx.q('([] a:1 2 3i)'), t.select(kx.Column('a').cast('int')))