From 3075376b065418aa1cfd35479a3ac1a58d1cde83 Mon Sep 17 00:00:00 2001 From: Stephan Schultchen Date: Fri, 26 Jun 2026 13:11:03 +0200 Subject: [PATCH 1/2] Restructure codebase to one class per file, implement async counterparts, and update API documentation --- docs/api/client/async_client.md | 3 + docs/api/client/async_cluster_client.md | 3 + docs/api/client/async_hash_client.md | 3 + docs/api/client/async_pubsub_client.md | 3 + docs/api/client/async_sentinel_client.md | 3 + docs/api/client/client.md | 3 + docs/api/client/cluster_client.md | 3 + docs/api/client/hash_client.md | 3 + docs/api/client/pubsub_client.md | 3 + docs/api/client/sentinel_client.md | 3 + docs/api/commands/connection.md | 3 + docs/api/commands/geo.md | 3 + docs/api/commands/hash.md | 3 + docs/api/commands/hyperloglog.md | 3 + docs/api/commands/key.md | 3 + docs/api/commands/list.md | 3 + docs/api/commands/publish.md | 3 + docs/api/commands/scripting.md | 3 + docs/api/commands/set.md | 3 + docs/api/commands/sset.md | 3 + docs/api/commands/string.md | 3 + docs/api/commands/subscribe.md | 3 + docs/api/commands/transaction.md | 3 + docs/api/connection/async_connection.md | 3 + docs/api/connection/connection.md | 3 + docs/api/pool/async_base_pool.md | 3 + docs/api/pool/async_cluster_pool.md | 3 + docs/api/pool/async_hash_pool.md | 3 + docs/api/pool/async_pool.md | 3 + docs/api/pool/async_sentinel_hash_pool.md | 3 + docs/api/pool/async_sentinel_pool.md | 3 + docs/api/pool/base_pool.md | 3 + docs/api/pool/cluster_pool.md | 3 + docs/api/pool/hash_pool.md | 3 + docs/api/pool/pool.md | 3 + docs/api/pool/sentinel_hash_pool.md | 3 + docs/api/pool/sentinel_pool.md | 3 + docs/example.md | 156 ++ docs/pyredis.md | 107 -- mkdocs.yml | 43 +- pyproject.toml | 2 +- pyredis/__init__.py | 65 +- pyredis/async_helper.py | 110 ++ pyredis/client.py | 752 ---------- pyredis/client/__init__.py | 33 + pyredis/client/async_client.py | 32 + pyredis/client/async_cluster.py | 159 +++ pyredis/client/async_hash.py | 164 +++ pyredis/client/async_pubsub.py | 20 + pyredis/client/async_sentinel.py | 83 ++ pyredis/client/client.py | 85 ++ pyredis/client/cluster.py | 159 +++ pyredis/client/hash.py | 164 +++ pyredis/client/pubsub.py | 20 + pyredis/client/sentinel.py | 84 ++ pyredis/commands.py | 1566 --------------------- pyredis/commands/__init__.py | 31 + pyredis/commands/base.py | 6 + pyredis/commands/connection.py | 26 + pyredis/commands/geo.py | 66 + pyredis/commands/hash.py | 156 ++ pyredis/commands/hyperloglog.py | 36 + pyredis/commands/key.py | 224 +++ pyredis/commands/list.py | 176 +++ pyredis/commands/publish.py | 13 + pyredis/commands/scripting.py | 83 ++ pyredis/commands/set.py | 156 ++ pyredis/commands/sset.py | 216 +++ pyredis/commands/string.py | 246 ++++ pyredis/commands/subscribe.py | 23 + pyredis/commands/transaction.py | 60 + pyredis/connection/__init__.py | 19 + pyredis/connection/async_connection.py | 186 +++ pyredis/{ => connection}/connection.py | 140 +- pyredis/helper.py | 7 +- pyredis/pool.py | 709 ---------- pyredis/pool/__init__.py | 49 + pyredis/pool/async_base.py | 134 ++ pyredis/pool/async_cluster.py | 53 + pyredis/pool/async_hash.py | 38 + pyredis/pool/async_pool.py | 58 + pyredis/pool/async_sentinel.py | 101 ++ pyredis/pool/async_sentinel_hash.py | 123 ++ pyredis/pool/base.py | 129 ++ pyredis/pool/cluster.py | 53 + pyredis/pool/hash.py | 38 + pyredis/pool/pool.py | 52 + pyredis/pool/sentinel.py | 101 ++ pyredis/pool/sentinel_hash.py | 123 ++ pyredis/protocol.py | 29 +- tests/unit/test_async_client.py | 468 ++++++ tests/unit/test_pool.py | 10 +- 92 files changed, 4796 insertions(+), 3257 deletions(-) create mode 100644 docs/api/client/async_client.md create mode 100644 docs/api/client/async_cluster_client.md create mode 100644 docs/api/client/async_hash_client.md create mode 100644 docs/api/client/async_pubsub_client.md create mode 100644 docs/api/client/async_sentinel_client.md create mode 100644 docs/api/client/client.md create mode 100644 docs/api/client/cluster_client.md create mode 100644 docs/api/client/hash_client.md create mode 100644 docs/api/client/pubsub_client.md create mode 100644 docs/api/client/sentinel_client.md create mode 100644 docs/api/commands/connection.md create mode 100644 docs/api/commands/geo.md create mode 100644 docs/api/commands/hash.md create mode 100644 docs/api/commands/hyperloglog.md create mode 100644 docs/api/commands/key.md create mode 100644 docs/api/commands/list.md create mode 100644 docs/api/commands/publish.md create mode 100644 docs/api/commands/scripting.md create mode 100644 docs/api/commands/set.md create mode 100644 docs/api/commands/sset.md create mode 100644 docs/api/commands/string.md create mode 100644 docs/api/commands/subscribe.md create mode 100644 docs/api/commands/transaction.md create mode 100644 docs/api/connection/async_connection.md create mode 100644 docs/api/connection/connection.md create mode 100644 docs/api/pool/async_base_pool.md create mode 100644 docs/api/pool/async_cluster_pool.md create mode 100644 docs/api/pool/async_hash_pool.md create mode 100644 docs/api/pool/async_pool.md create mode 100644 docs/api/pool/async_sentinel_hash_pool.md create mode 100644 docs/api/pool/async_sentinel_pool.md create mode 100644 docs/api/pool/base_pool.md create mode 100644 docs/api/pool/cluster_pool.md create mode 100644 docs/api/pool/hash_pool.md create mode 100644 docs/api/pool/pool.md create mode 100644 docs/api/pool/sentinel_hash_pool.md create mode 100644 docs/api/pool/sentinel_pool.md delete mode 100644 docs/pyredis.md create mode 100644 pyredis/async_helper.py delete mode 100644 pyredis/client.py create mode 100644 pyredis/client/__init__.py create mode 100644 pyredis/client/async_client.py create mode 100644 pyredis/client/async_cluster.py create mode 100644 pyredis/client/async_hash.py create mode 100644 pyredis/client/async_pubsub.py create mode 100644 pyredis/client/async_sentinel.py create mode 100644 pyredis/client/client.py create mode 100644 pyredis/client/cluster.py create mode 100644 pyredis/client/hash.py create mode 100644 pyredis/client/pubsub.py create mode 100644 pyredis/client/sentinel.py delete mode 100644 pyredis/commands.py create mode 100644 pyredis/commands/__init__.py create mode 100644 pyredis/commands/base.py create mode 100644 pyredis/commands/connection.py create mode 100644 pyredis/commands/geo.py create mode 100644 pyredis/commands/hash.py create mode 100644 pyredis/commands/hyperloglog.py create mode 100644 pyredis/commands/key.py create mode 100644 pyredis/commands/list.py create mode 100644 pyredis/commands/publish.py create mode 100644 pyredis/commands/scripting.py create mode 100644 pyredis/commands/set.py create mode 100644 pyredis/commands/sset.py create mode 100644 pyredis/commands/string.py create mode 100644 pyredis/commands/subscribe.py create mode 100644 pyredis/commands/transaction.py create mode 100644 pyredis/connection/__init__.py create mode 100644 pyredis/connection/async_connection.py rename pyredis/{ => connection}/connection.py (57%) delete mode 100644 pyredis/pool.py create mode 100644 pyredis/pool/__init__.py create mode 100644 pyredis/pool/async_base.py create mode 100644 pyredis/pool/async_cluster.py create mode 100644 pyredis/pool/async_hash.py create mode 100644 pyredis/pool/async_pool.py create mode 100644 pyredis/pool/async_sentinel.py create mode 100644 pyredis/pool/async_sentinel_hash.py create mode 100644 pyredis/pool/base.py create mode 100644 pyredis/pool/cluster.py create mode 100644 pyredis/pool/hash.py create mode 100644 pyredis/pool/pool.py create mode 100644 pyredis/pool/sentinel.py create mode 100644 pyredis/pool/sentinel_hash.py create mode 100644 tests/unit/test_async_client.py diff --git a/docs/api/client/async_client.md b/docs/api/client/async_client.md new file mode 100644 index 0000000..af359ea --- /dev/null +++ b/docs/api/client/async_client.md @@ -0,0 +1,3 @@ +# AsyncClient + +::: pyredis.AsyncClient diff --git a/docs/api/client/async_cluster_client.md b/docs/api/client/async_cluster_client.md new file mode 100644 index 0000000..8d20b11 --- /dev/null +++ b/docs/api/client/async_cluster_client.md @@ -0,0 +1,3 @@ +# AsyncClusterClient + +::: pyredis.AsyncClusterClient diff --git a/docs/api/client/async_hash_client.md b/docs/api/client/async_hash_client.md new file mode 100644 index 0000000..89c3d76 --- /dev/null +++ b/docs/api/client/async_hash_client.md @@ -0,0 +1,3 @@ +# AsyncHashClient + +::: pyredis.AsyncHashClient diff --git a/docs/api/client/async_pubsub_client.md b/docs/api/client/async_pubsub_client.md new file mode 100644 index 0000000..39ef908 --- /dev/null +++ b/docs/api/client/async_pubsub_client.md @@ -0,0 +1,3 @@ +# AsyncPubSubClient + +::: pyredis.AsyncPubSubClient diff --git a/docs/api/client/async_sentinel_client.md b/docs/api/client/async_sentinel_client.md new file mode 100644 index 0000000..6bfa469 --- /dev/null +++ b/docs/api/client/async_sentinel_client.md @@ -0,0 +1,3 @@ +# AsyncSentinelClient + +::: pyredis.AsyncSentinelClient diff --git a/docs/api/client/client.md b/docs/api/client/client.md new file mode 100644 index 0000000..e3916d1 --- /dev/null +++ b/docs/api/client/client.md @@ -0,0 +1,3 @@ +# Client + +::: pyredis.Client diff --git a/docs/api/client/cluster_client.md b/docs/api/client/cluster_client.md new file mode 100644 index 0000000..bf6fd5d --- /dev/null +++ b/docs/api/client/cluster_client.md @@ -0,0 +1,3 @@ +# ClusterClient + +::: pyredis.ClusterClient diff --git a/docs/api/client/hash_client.md b/docs/api/client/hash_client.md new file mode 100644 index 0000000..6e173a1 --- /dev/null +++ b/docs/api/client/hash_client.md @@ -0,0 +1,3 @@ +# HashClient + +::: pyredis.HashClient diff --git a/docs/api/client/pubsub_client.md b/docs/api/client/pubsub_client.md new file mode 100644 index 0000000..83e3250 --- /dev/null +++ b/docs/api/client/pubsub_client.md @@ -0,0 +1,3 @@ +# PubSubClient + +::: pyredis.PubSubClient diff --git a/docs/api/client/sentinel_client.md b/docs/api/client/sentinel_client.md new file mode 100644 index 0000000..e34938f --- /dev/null +++ b/docs/api/client/sentinel_client.md @@ -0,0 +1,3 @@ +# SentinelClient + +::: pyredis.SentinelClient diff --git a/docs/api/commands/connection.md b/docs/api/commands/connection.md new file mode 100644 index 0000000..b19c2df --- /dev/null +++ b/docs/api/commands/connection.md @@ -0,0 +1,3 @@ +# Connection Commands + +::: pyredis.commands.Connection diff --git a/docs/api/commands/geo.md b/docs/api/commands/geo.md new file mode 100644 index 0000000..2de58bc --- /dev/null +++ b/docs/api/commands/geo.md @@ -0,0 +1,3 @@ +# Geo Commands + +::: pyredis.commands.Geo diff --git a/docs/api/commands/hash.md b/docs/api/commands/hash.md new file mode 100644 index 0000000..31df000 --- /dev/null +++ b/docs/api/commands/hash.md @@ -0,0 +1,3 @@ +# Hash Commands + +::: pyredis.commands.Hash diff --git a/docs/api/commands/hyperloglog.md b/docs/api/commands/hyperloglog.md new file mode 100644 index 0000000..663c74c --- /dev/null +++ b/docs/api/commands/hyperloglog.md @@ -0,0 +1,3 @@ +# HyperLogLog Commands + +::: pyredis.commands.HyperLogLog diff --git a/docs/api/commands/key.md b/docs/api/commands/key.md new file mode 100644 index 0000000..2730e2c --- /dev/null +++ b/docs/api/commands/key.md @@ -0,0 +1,3 @@ +# Key Commands + +::: pyredis.commands.Key diff --git a/docs/api/commands/list.md b/docs/api/commands/list.md new file mode 100644 index 0000000..9e1ab78 --- /dev/null +++ b/docs/api/commands/list.md @@ -0,0 +1,3 @@ +# List Commands + +::: pyredis.commands.List diff --git a/docs/api/commands/publish.md b/docs/api/commands/publish.md new file mode 100644 index 0000000..cd3bfca --- /dev/null +++ b/docs/api/commands/publish.md @@ -0,0 +1,3 @@ +# Publish Commands + +::: pyredis.commands.Publish diff --git a/docs/api/commands/scripting.md b/docs/api/commands/scripting.md new file mode 100644 index 0000000..7725494 --- /dev/null +++ b/docs/api/commands/scripting.md @@ -0,0 +1,3 @@ +# Scripting Commands + +::: pyredis.commands.Scripting diff --git a/docs/api/commands/set.md b/docs/api/commands/set.md new file mode 100644 index 0000000..1e95d5c --- /dev/null +++ b/docs/api/commands/set.md @@ -0,0 +1,3 @@ +# Set Commands + +::: pyredis.commands.Set diff --git a/docs/api/commands/sset.md b/docs/api/commands/sset.md new file mode 100644 index 0000000..398bb94 --- /dev/null +++ b/docs/api/commands/sset.md @@ -0,0 +1,3 @@ +# SSet Commands + +::: pyredis.commands.SSet diff --git a/docs/api/commands/string.md b/docs/api/commands/string.md new file mode 100644 index 0000000..16cc46e --- /dev/null +++ b/docs/api/commands/string.md @@ -0,0 +1,3 @@ +# String Commands + +::: pyredis.commands.String diff --git a/docs/api/commands/subscribe.md b/docs/api/commands/subscribe.md new file mode 100644 index 0000000..8bc5a71 --- /dev/null +++ b/docs/api/commands/subscribe.md @@ -0,0 +1,3 @@ +# Subscribe Commands + +::: pyredis.commands.Subscribe diff --git a/docs/api/commands/transaction.md b/docs/api/commands/transaction.md new file mode 100644 index 0000000..534c29b --- /dev/null +++ b/docs/api/commands/transaction.md @@ -0,0 +1,3 @@ +# Transaction Commands + +::: pyredis.commands.Transaction diff --git a/docs/api/connection/async_connection.md b/docs/api/connection/async_connection.md new file mode 100644 index 0000000..02cfa34 --- /dev/null +++ b/docs/api/connection/async_connection.md @@ -0,0 +1,3 @@ +# AsyncConnection + +::: pyredis.connection.AsyncConnection diff --git a/docs/api/connection/connection.md b/docs/api/connection/connection.md new file mode 100644 index 0000000..dacb3f0 --- /dev/null +++ b/docs/api/connection/connection.md @@ -0,0 +1,3 @@ +# Connection + +::: pyredis.connection.Connection diff --git a/docs/api/pool/async_base_pool.md b/docs/api/pool/async_base_pool.md new file mode 100644 index 0000000..6b16565 --- /dev/null +++ b/docs/api/pool/async_base_pool.md @@ -0,0 +1,3 @@ +# AsyncBasePool + +::: pyredis.pool.AsyncBasePool diff --git a/docs/api/pool/async_cluster_pool.md b/docs/api/pool/async_cluster_pool.md new file mode 100644 index 0000000..f3fa89c --- /dev/null +++ b/docs/api/pool/async_cluster_pool.md @@ -0,0 +1,3 @@ +# AsyncClusterPool + +::: pyredis.AsyncClusterPool diff --git a/docs/api/pool/async_hash_pool.md b/docs/api/pool/async_hash_pool.md new file mode 100644 index 0000000..ab51d03 --- /dev/null +++ b/docs/api/pool/async_hash_pool.md @@ -0,0 +1,3 @@ +# AsyncHashPool + +::: pyredis.AsyncHashPool diff --git a/docs/api/pool/async_pool.md b/docs/api/pool/async_pool.md new file mode 100644 index 0000000..6a2e751 --- /dev/null +++ b/docs/api/pool/async_pool.md @@ -0,0 +1,3 @@ +# AsyncPool + +::: pyredis.AsyncPool diff --git a/docs/api/pool/async_sentinel_hash_pool.md b/docs/api/pool/async_sentinel_hash_pool.md new file mode 100644 index 0000000..8f36117 --- /dev/null +++ b/docs/api/pool/async_sentinel_hash_pool.md @@ -0,0 +1,3 @@ +# AsyncSentinelHashPool + +::: pyredis.AsyncSentinelHashPool diff --git a/docs/api/pool/async_sentinel_pool.md b/docs/api/pool/async_sentinel_pool.md new file mode 100644 index 0000000..63a6468 --- /dev/null +++ b/docs/api/pool/async_sentinel_pool.md @@ -0,0 +1,3 @@ +# AsyncSentinelPool + +::: pyredis.AsyncSentinelPool diff --git a/docs/api/pool/base_pool.md b/docs/api/pool/base_pool.md new file mode 100644 index 0000000..024bf96 --- /dev/null +++ b/docs/api/pool/base_pool.md @@ -0,0 +1,3 @@ +# BasePool + +::: pyredis.pool.BasePool diff --git a/docs/api/pool/cluster_pool.md b/docs/api/pool/cluster_pool.md new file mode 100644 index 0000000..d78facf --- /dev/null +++ b/docs/api/pool/cluster_pool.md @@ -0,0 +1,3 @@ +# ClusterPool + +::: pyredis.ClusterPool diff --git a/docs/api/pool/hash_pool.md b/docs/api/pool/hash_pool.md new file mode 100644 index 0000000..1c832d9 --- /dev/null +++ b/docs/api/pool/hash_pool.md @@ -0,0 +1,3 @@ +# HashPool + +::: pyredis.HashPool diff --git a/docs/api/pool/pool.md b/docs/api/pool/pool.md new file mode 100644 index 0000000..ad47f5d --- /dev/null +++ b/docs/api/pool/pool.md @@ -0,0 +1,3 @@ +# Pool + +::: pyredis.Pool diff --git a/docs/api/pool/sentinel_hash_pool.md b/docs/api/pool/sentinel_hash_pool.md new file mode 100644 index 0000000..65b5e58 --- /dev/null +++ b/docs/api/pool/sentinel_hash_pool.md @@ -0,0 +1,3 @@ +# SentinelHashPool + +::: pyredis.SentinelHashPool diff --git a/docs/api/pool/sentinel_pool.md b/docs/api/pool/sentinel_pool.md new file mode 100644 index 0000000..407c13b --- /dev/null +++ b/docs/api/pool/sentinel_pool.md @@ -0,0 +1,3 @@ +# SentinelPool + +::: pyredis.SentinelPool diff --git a/docs/example.md b/docs/example.md index ef63803..a50c1bb 100644 --- a/docs/example.md +++ b/docs/example.md @@ -140,3 +140,159 @@ client.publish('/blub', 'test') subscribe.get() [b'message', b'/blub', b'test'] ``` + +## Asynchronous Client and Pool Usage + +### Simple Async Client Usage + +```python +import asyncio +from pyredis import AsyncClient + +async def main(): + client = AsyncClient(host="localhost") + await client.ping() + await client.close() + +asyncio.run(main()) +``` + +### Async Bulk Mode + +```python +import asyncio +from pyredis import AsyncClient + +async def main(): + client = AsyncClient(host="localhost") + await client.bulk_start() + await client.set("key1", "value1") + await client.set("key2", "value2") + await client.set("key3", "value3") + results = await client.bulk_stop() + print(results) + await client.close() + +asyncio.run(main()) +``` + +### Using an Async Connection Pool + +```python +import asyncio +from pyredis import AsyncPool + +async def main(): + pool = AsyncPool(host="localhost") + client = await pool.acquire() + await client.ping() + await pool.release(client) + +asyncio.run(main()) +``` + +### Using an Async Cluster Connection Pool + +```python +import asyncio +from pyredis import AsyncClusterPool + +async def main(): + pool = AsyncClusterPool( + seeds=[ + ("seed1", 6379), + ("seed2", 6379) + ] + ) + client = await pool.acquire() + await client.ping(shard_key="test") + await pool.release(client) + +asyncio.run(main()) +``` + +### Using an Async Hash Connection Pool + +```python +import asyncio +from pyredis import AsyncHashPool + +async def main(): + pool = AsyncHashPool( + buckets=[ + ("host1", 6379), + ("host2", 6379) + ] + ) + client = await pool.acquire() + await client.ping(shard_key="test") + await pool.release(client) + +asyncio.run(main()) +``` + +### Using an Async Sentinel backed Connection Pool + +```python +import asyncio +from pyredis import AsyncSentinelPool + +async def main(): + pool = AsyncSentinelPool( + sentinels=[ + ("sentinel1", 26379), + ("sentinel2", 26379) + ], + name="mymaster" + ) + client = await pool.acquire() + await client.ping() + await pool.release(client) + +asyncio.run(main()) +``` + +### Getting Async Pools/Clients by URL + +```python +from pyredis import get_by_url + +pool = get_by_url( + url="redis://localhost?password=topsecret", + async_client=True +) + +pubsub = get_by_url( + url="pubsub://localhost?password=topsecret", + async_client=True +) +``` + +### Async Publish Subscribe + +```python +import asyncio +from pyredis import AsyncClient +from pyredis import AsyncPubSubClient + +async def main(): + client = AsyncClient(host="localhost") + subscribe = AsyncPubSubClient(host="localhost") + + await subscribe.subscribe("/blub") + res1 = await subscribe.get() + print(res1) + + await client.publish( + "/blub", + "test" + ) + res2 = await subscribe.get() + print(res2) + + await client.close() + await subscribe.close() + +asyncio.run(main()) +``` + diff --git a/docs/pyredis.md b/docs/pyredis.md deleted file mode 100644 index ae0ed36..0000000 --- a/docs/pyredis.md +++ /dev/null @@ -1,107 +0,0 @@ -# API Documentation - -## Connection - -::: pyredis.connection.Connection - -## Client - -### Client - -::: pyredis.Client - -### ClusterClient - -::: pyredis.ClusterClient - -### HashClient - -::: pyredis.HashClient - -### PubSubClient - -::: pyredis.PubSubClient - -### SentinelClient - -::: pyredis.SentinelClient - -## Pool - -### BasePool - -::: pyredis.pool.BasePool - -### ClusterPool - -::: pyredis.ClusterPool - -### HashPool - -::: pyredis.HashPool - -### Pool - -::: pyredis.Pool - -### SentinelHashPool - -::: pyredis.SentinelHashPool - -### SentinelPool - -::: pyredis.SentinelPool - -## Commands - -### Connection - -::: pyredis.commands.Connection - -### Hash - -::: pyredis.commands.Hash - -### HyperLogLog - -::: pyredis.commands.HyperLogLog - -### Geo - -::: pyredis.commands.Geo - -### Key - -::: pyredis.commands.Key - -### List - -::: pyredis.commands.List - -### Publish - -::: pyredis.commands.Publish - -### Scripting - -::: pyredis.commands.Scripting - -### Set - -::: pyredis.commands.Set - -### SSet - -::: pyredis.commands.SSet - -### String - -::: pyredis.commands.String - -### Subscribe - -::: pyredis.commands.Subscribe - -### Transaction - -::: pyredis.commands.Transaction diff --git a/mkdocs.yml b/mkdocs.yml index 892dc06..31ae384 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -18,7 +18,48 @@ theme: nav: - Introduction: index.md - Usage Example: example.md - - API Documentation: pyredis.md + - API Reference: + - Connection: + - Connection: api/connection/connection.md + - AsyncConnection: api/connection/async_connection.md + - Clients: + - Client: api/client/client.md + - AsyncClient: api/client/async_client.md + - ClusterClient: api/client/cluster_client.md + - AsyncClusterClient: api/client/async_cluster_client.md + - HashClient: api/client/hash_client.md + - AsyncHashClient: api/client/async_hash_client.md + - PubSubClient: api/client/pubsub_client.md + - AsyncPubSubClient: api/client/async_pubsub_client.md + - SentinelClient: api/client/sentinel_client.md + - AsyncSentinelClient: api/client/async_sentinel_client.md + - Pools: + - BasePool: api/pool/base_pool.md + - AsyncBasePool: api/pool/async_base_pool.md + - Pool: api/pool/pool.md + - AsyncPool: api/pool/async_pool.md + - ClusterPool: api/pool/cluster_pool.md + - AsyncClusterPool: api/pool/async_cluster_pool.md + - HashPool: api/pool/hash_pool.md + - AsyncHashPool: api/pool/async_hash_pool.md + - SentinelPool: api/pool/sentinel_pool.md + - AsyncSentinelPool: api/pool/async_sentinel_pool.md + - SentinelHashPool: api/pool/sentinel_hash_pool.md + - AsyncSentinelHashPool: api/pool/async_sentinel_hash_pool.md + - Commands: + - Connection: api/commands/connection.md + - Hash: api/commands/hash.md + - HyperLogLog: api/commands/hyperloglog.md + - Geo: api/commands/geo.md + - Key: api/commands/key.md + - List: api/commands/list.md + - Publish: api/commands/publish.md + - Scripting: api/commands/scripting.md + - Set: api/commands/set.md + - SSet: api/commands/sset.md + - String: api/commands/string.md + - Subscribe: api/commands/subscribe.md + - Transaction: api/commands/transaction.md plugins: - search diff --git a/pyproject.toml b/pyproject.toml index cc963b7..20483c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "python_redis" -version = "0.4.1" +version = "1.0.0" requires-python = ">=3.9" authors = [ {name = "Stephan Schultchen", email = "stephan.schultchen@gmail.com"}, diff --git a/pyredis/__init__.py b/pyredis/__init__.py index f547cc1..5044f23 100644 --- a/pyredis/__init__.py +++ b/pyredis/__init__.py @@ -8,30 +8,56 @@ License: MIT (see LICENSE for details) """ -from pyredis.exceptions import * +from pyredis.exceptions import PyRedisConnError +from pyredis.exceptions import PyRedisConnReadTimeout +from pyredis.exceptions import PyRedisConnClosed +from pyredis.exceptions import PyRedisError +from pyredis.exceptions import ProtocolError +from pyredis.exceptions import ReplyError +from pyredis.exceptions import PyRedisURLError from pyredis.client import Client +from pyredis.client import AsyncClient from pyredis.client import ClusterClient +from pyredis.client import AsyncClusterClient from pyredis.client import HashClient +from pyredis.client import AsyncHashClient from pyredis.client import PubSubClient +from pyredis.client import AsyncPubSubClient from pyredis.client import SentinelClient +from pyredis.client import AsyncSentinelClient from pyredis.pool import ClusterPool +from pyredis.pool import AsyncClusterPool from pyredis.pool import HashPool +from pyredis.pool import AsyncHashPool from pyredis.pool import Pool +from pyredis.pool import AsyncPool from pyredis.pool import SentinelHashPool +from pyredis.pool import AsyncSentinelHashPool from pyredis.pool import SentinelPool +from pyredis.pool import AsyncSentinelPool __all__ = [ "get_by_url", "Client", + "AsyncClient", "ClusterClient", + "AsyncClusterClient", "ClusterPool", + "AsyncClusterPool", "HashClient", + "AsyncHashClient", "PubSubClient", + "AsyncPubSubClient", "SentinelClient", + "AsyncSentinelClient", "HashPool", + "AsyncHashPool", "Pool", + "AsyncPool", "SentinelPool", + "AsyncSentinelPool", "SentinelHashPool", + "AsyncSentinelHashPool", "PyRedisConnError", "PyRedisConnReadTimeout", "PyRedisConnClosed", @@ -41,7 +67,10 @@ ] -def get_by_url(url): +def get_by_url( + url, + async_client=False +): scheme, rest = url.split("://", 1) conns = list() kwargs = dict() @@ -62,6 +91,8 @@ def get_by_url(url): kwargs[key] = _opts_type_helper(key, value) try: if scheme == "cluster": + if async_client: + return AsyncClusterPool(seeds=conns, **kwargs) return ClusterPool(seeds=conns, **kwargs) elif scheme == "redis": host = conns[0][0] @@ -69,8 +100,24 @@ def get_by_url(url): port = conns[0][1] except IndexError: port = 6379 - return Pool(host=host, port=port, **kwargs) + if async_client: + return AsyncPool( + host=host, + port=port, + **kwargs + ) + return Pool( + host=host, + port=port, + **kwargs + ) elif scheme == "sentinel": + if "buckets" in kwargs: + if async_client: + return AsyncSentinelHashPool(sentinels=conns, **kwargs) + return SentinelHashPool(sentinels=conns, **kwargs) + if async_client: + return AsyncSentinelPool(sentinels=conns, **kwargs) return SentinelPool(sentinels=conns, **kwargs) elif scheme == "pubsub": host = conns[0][0] @@ -78,7 +125,17 @@ def get_by_url(url): port = conns[0][1] except IndexError: port = 6379 - return PubSubClient(host=host, port=port, **kwargs) + if async_client: + return AsyncPubSubClient( + host=host, + port=port, + **kwargs + ) + return PubSubClient( + host=host, + port=port, + **kwargs + ) else: raise PyRedisURLError("invalid schema: {0}") except TypeError as err: diff --git a/pyredis/async_helper.py b/pyredis/async_helper.py new file mode 100644 index 0000000..d4f6d96 --- /dev/null +++ b/pyredis/async_helper.py @@ -0,0 +1,110 @@ +import asyncio +import random +from collections import deque +from uuid import uuid4 + +from pyredis.connection import AsyncConnection +from pyredis.exceptions import PyRedisError +from pyredis.helper import slot_from_key + + +class AsyncClusterMap(object): + def __init__( + self, + seeds, + password=None, + lock=None, + username=None, + ): + self._id = uuid4() + if lock is None: + self._lock = asyncio.Lock() + else: + self._lock = lock + self._map = {} + self._seeds = deque(seeds) + self._password = password + self._username = username + + @property + def id(self): + return self._id + + @staticmethod + def _make_str(endpoint): + return str(endpoint[0]) + "_" + str(endpoint[1]) + + async def _fetch_map(self): + for seed in self._seeds: + conn = AsyncConnection( + host=seed[0], + port=seed[1], + encoding="utf-8", + password=self._password, + username=self._username, + ) + try: + await conn.write( + *["CLUSTER", "SLOTS"] + ) + return await conn.read() + except PyRedisError: + pass + finally: + await conn.close() + raise PyRedisError( + "Could not get cluster info from any seed node: " + "{0}".format(self._seeds) + ) + + def _update_slot( + self, + slot, + master, + slaves + ): + self._map[slot] = { + "master": self._make_str(master), + "slave": self._make_str(random.choice(slaves)), + } + + def get_slot( + self, + shard_key, + slave=None + ): + if not slave: + return self._map[slot_from_key(shard_key)]["master"] + else: + return self._map[slot_from_key(shard_key)]["slave"] + + def hosts( + self, + slave=None + ): + result = set() + if not slave: + selector = "master" + else: + selector = "slave" + for host in self._map.values(): + result.add(host[selector]) + return result + + async def update(self, map_id): + async with self._lock: + if map_id != self.id: + return self.id + slots = await self._fetch_map() + for entry in slots: + for slot in range( + entry[0], + entry[1] + 1 + ): + self._update_slot( + slot=slot, + master=entry[2], + slaves=entry[3:] + ) + self._id = uuid4() + return self.id diff --git a/pyredis/client.py b/pyredis/client.py deleted file mode 100644 index 964c7fe..0000000 --- a/pyredis/client.py +++ /dev/null @@ -1,752 +0,0 @@ -from collections import deque - -from pyredis import commands -from pyredis.connection import Connection -from pyredis.exceptions import PyRedisError -from pyredis.exceptions import PyRedisConnError -from pyredis.exceptions import PyRedisConnReadTimeout -from pyredis.exceptions import ReplyError -from pyredis.helper import dict_from_list -from pyredis.helper import ClusterMap -from pyredis.helper import slot_from_key - - -class Client( - commands.Connection, - commands.Geo, - commands.Hash, - commands.HyperLogLog, - commands.Key, - commands.List, - commands.Publish, - commands.Scripting, - commands.Set, - commands.SSet, - commands.String, - commands.Transaction, -): - """Base Client for Talking to Redis. - - Inherits the following Command classes: - - commands.Connection, - - commands.Geo, - - commands.Hash, - - commands.HyperLogLog, - - commands.Key, - - commands.List, - - commands.Publish, - - commands.Scripting, - - commands.Set, - - commands.SSet, - - commands.String, - - commands.Transaction - - - :param kwargs: - pyredis.Client takes the same arguments as pyredis.connection.Connection. - """ - - def __init__(self, **kwargs): - super().__init__() - self._conn = Connection(**kwargs) - self._bulk = False - self._bulk_keep = False - self._bulk_results = None - self._bulk_size = None - self._bulk_size_current = None - - def _bulk_fetch(self): - while self._bulk_size_current != 0: - result = self._conn.read(raise_on_result_err=False) - self._bulk_size_current -= 1 - if self._bulk_keep: - self._bulk_results.append(result) - - def _execute_basic(self, *args): - self._conn.write(*args) - return self._conn.read() - - def _execute_bulk(self, *args): - self._conn.write(*args) - self._bulk_size_current += 1 - if self._bulk_size_current == self._bulk_size: - self._bulk_fetch() - - @property - def bulk(self): - """True if bulk mode is enabled. - - :return: bool - """ - return self._bulk - - def bulk_start(self, bulk_size=5000, keep_results=True): - """Enable bulk mode - - Put the client into bulk mode. Instead of executing a command & waiting for - the reply, all commands are send to Redis without fetching the result. - The results get fetched whenever $bulk_size commands have been executed, - which will also resets the counter, or of bulk_stop() is called. - - :param bulk_size: - Number of commands to execute, before fetching results. - :type bulk_size: int - - :param keep_results: - If True, keep the results. The Results will be returned when calling bulk_stop. - :type keep_results: bool - - :return: None - """ - if self.bulk: - raise PyRedisError("Already in bulk mode") - self._bulk = True - self._bulk_size = bulk_size - self._bulk_size_current = 0 - if keep_results: - self._bulk_results = [] - self._bulk_keep = True - - def bulk_stop(self): - """Stop bulk mode. - - All outstanding results from previous commands get fetched. - If bulk_start was called with keep_results=True, return a list with all - results from the executed commands in order. The list of results can also contain - Exceptions, hat you should check for. - - :return: None, list - """ - if not self.bulk: - raise PyRedisError("Not in bulk mode") - self._bulk_fetch() - results = self._bulk_results - self._bulk = False - self._bulk_keep = False - self._bulk_results = None - self._bulk_size = None - self._bulk_size_current = None - return results - - def close(self): - """Close client. - - :return: None - """ - self._conn.close() - - @property - def closed(self): - """Check if client is closed. - - :return: bool - """ - return self._conn.closed - - def execute(self, *args): - """Execute arbitrary redis command. - - :param args: - :type args: list, int, float - - :return: result, exception - """ - if not self._bulk: - return self._execute_basic(*args) - else: - self._execute_bulk(*args) - - -class ClusterClient( - commands.Connection, - commands.Geo, - commands.Hash, - commands.HyperLogLog, - commands.Key, - commands.List, - commands.Scripting, - commands.Set, - commands.SSet, - commands.String, - commands.Transaction, -): - """Base Client for Talking to Redis Cluster. - - Inherits the following Commmand classes: - - commands.Connection, - - commands.Geo, - - commands.Hash, - - commands.HyperLogLog, - - commands.Key, - - commands.List, - - commands.Scripting, - - commands.Set, - - commands.SSet, - - commands.String, - - commands.Transaction - - :param seeds: - Accepts a list of seed nodes in this form: [('seed1', 6379), ('seed2', 6379), ('seed3', 6379)] - :type seeds: list - - :param slave_ok: - Set to True if this Client should use slave nodes. - :type slave_ok: bool - - :param database: - Select which db should be used for this connection - :type database: int - - :param password: - Password used for authentication. If None, no authentication is done - :type password: str - - :param encoding: - Convert result strings with this encoding. If None, no encoding is done. - :type encoding: str - - :param conn_timeout: - Connect Timeout. - :type conn_timeout: float - - :param read_timeout: - Read Timeout. - :type read_timeout: float - - :param username: - Username used for acl scl authentication. If not set, fall back use legacy auth. - :type username: str - """ - - def __init__( - self, - seeds=None, - database=0, - password=None, - encoding=None, - slave_ok=False, - conn_timeout=2, - read_timeout=2, - cluster_map=None, - username=None, - ): - super().__init__() - if not bool(seeds) != bool(cluster_map): - raise PyRedisError("Ether seeds or cluster_map has to be provided") - self._cluster = True - self._conns = dict() - self._conn_timeout = conn_timeout - self._read_timeout = read_timeout - self._encoding = encoding - self._password = password - self._database = database - self._slave_ok = slave_ok - if cluster_map: - self._map = cluster_map - else: - self._map = ClusterMap(seeds=seeds) - self._map_id = self._map.id - self._username = username - - def _cleanup_conns(self): - hosts = self._map.hosts(slave=self._slave_ok) - wipe = set() - for conn in self._conns.keys(): - if conn not in hosts: - wipe.add(conn) - - for conn in wipe: - self._conns[conn].close() - del self._conns[conn] - - def _connect(self, sock): - host, port = sock.split("_") - client = Connection( - host=host, - port=int(port), - conn_timeout=self._conn_timeout, - read_timeout=self._read_timeout, - read_only=self._slave_ok, - encoding=self._encoding, - password=self._password, - database=self._database, - username=self._username, - ) - self._conns[sock] = client - - def _get_slot_info(self, shard_key): - if self._map_id != self._map.id: - self._map_id = self._map.id - self._cleanup_conns() - try: - return self._map.get_slot(shard_key, self._slave_ok) - except KeyError: - self._map_id = self._map.update(self._map_id) - self._cleanup_conns() - return self._map.get_slot(shard_key, self._slave_ok) - - @property - def closed(self): - return False - - def execute(self, *args, shard_key=None, sock=None, asking=False, retries=3): - """Execute arbitrary redis command. - - :param args: - :type args: list, int, float - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - :type sock: string - - :return: result, exception - """ - if not bool(shard_key) != bool(sock): - raise PyRedisError("Ether shard_key or sock has to be provided") - if not sock: - sock = self._get_slot_info(shard_key) - if sock not in self._conns.keys(): - self._connect(sock) - try: - if asking: - self._conns[sock].write("ASKING", *args) - else: - self._conns[sock].write(*args) - return self._conns[sock].read() - except ReplyError as err: - errstr = str(err) - if retries <= 1 and ( - errstr.startswith("MOVED") or errstr.startswith("ASK") - ): - raise PyRedisError("Slot moved to often or wrong shard_key, giving up,") - if errstr.startswith("MOVED"): - if not shard_key: - raise ReplyError( - "Explicitly set socket, but key does not belong to this redis: {0}".format( - sock - ) - ) - self._map_id = self._map.update(self._map_id) - self._cleanup_conns() - return self.execute(*args, shard_key=shard_key, retries=retries - 1) - elif errstr.startswith("ASK"): - sock = errstr.split()[2].replace(":", "_") - return self.execute(*args, sock=sock, retries=retries - 1, asking=True) - else: - raise err - except (PyRedisConnError, PyRedisConnReadTimeout) as err: - self._conns[sock].close() - del self._conns[sock] - self._map.update(self._map_id) - raise err - - -class HashClient( - commands.Connection, - commands.Geo, - commands.Hash, - commands.HyperLogLog, - commands.Key, - commands.List, - commands.Publish, - commands.Scripting, - commands.Set, - commands.SSet, - commands.String, - commands.Transaction, -): - """Client for Talking to Static Hashed Redis Cluster. - - The Client will calculate a crc16 hash using the shard_key, - which is be default the first Key in case the command supports multiple keys. - If the Key is using the TAG annotation "bla{tag}blarg", - then only the tag portion is used, in this case "tag". - The key space is split into 16384 buckets, so in theory you could provide - a list with 16384 ('host', port) pairs to the "buckets" parameter. - If you have less then 16384 ('host', port) pairs, the client will try to - distribute the key spaces evenly between available pairs. - - --- Warning --- - Since this is static hashing, the order of pairs has to match on each client you use! - Also changing the number of pairs will change the mapping between buckets and pairs, - rendering your data inaccessible! - - Inherits the following Commmand classes: - - commands.Connection, - - commands.Geo, - - commands.Hash, - - commands.HyperLogLog, - - commands.Key, - - commands.List, - - commands.Publish, - - commands.Scripting, - - commands.Set, - - commands.SSet, - - commands.String, - - commands.Transaction - """ - - def __init__( - self, - buckets, - database=None, - password=None, - encoding=None, - conn_timeout=2, - read_timeout=2, - username=None, - ): - super().__init__() - self._conns = dict() - self._conn_names = list() - self._bulk = False - self._bulk_keep = False - self._bulk_results = None - self._bulk_size = None - self._bulk_size_current = None - self._bulk_bucket_order = list() - self._closed = False - self._cluster = True - self._map = dict() - self._init_conns( - buckets=buckets, - database=database, - password=password, - encoding=encoding, - conn_timeout=conn_timeout, - read_timeout=read_timeout, - username=username, - ) - self._init_map() - - def _bulk_fetch(self): - for conn in self._bulk_bucket_order: - result = conn.read(raise_on_result_err=False) - if self._bulk_keep: - self._bulk_results.append(result) - self._bulk_bucket_order = list() - self._bulk_size_current = 0 - - @staticmethod - def _execute_basic(*args, conn): - conn.write(*args) - return conn.read() - - def _execute_bulk(self, *args, conn): - conn.write(*args) - self._bulk_size_current += 1 - self._bulk_bucket_order.append(conn) - if self._bulk_size_current == self._bulk_size: - self._bulk_fetch() - - def _init_conns( - self, - buckets, - database, - password, - encoding, - conn_timeout, - read_timeout, - username, - ): - for bucket in buckets: - host, port = bucket - bucketname = "{0}_{1}".format(host, port) - self._conn_names.append(bucketname) - self._conns[bucketname] = Connection( - host=host, - port=port, - database=database, - password=password, - encoding=encoding, - conn_timeout=conn_timeout, - read_timeout=read_timeout, - username=username, - ) - - def _init_map(self): - num_buckets = len(self._conn_names) - 1 - cur_bucket = 0 - for slot in range(16384): - self._map[slot] = self._conn_names[cur_bucket] - if cur_bucket == num_buckets: - cur_bucket = 0 - else: - cur_bucket += 1 - - @property - def bulk(self): - """True if bulk mode is enabled. - - :return: bool - """ - return self._bulk - - def bulk_start(self, bulk_size=5000, keep_results=True): - """Enable bulk mode - - Put the client into bulk mode. Instead of executing a command & waiting for - the reply, all commands are send to Redis without fetching the result. - The results get fetched whenever $bulk_size commands have been executed, - which will also resets the counter, or of bulk_stop() is called. - - :param bulk_size: - Number of commands to execute, before fetching results. - :type bulk_size: int - - :param keep_results: - If True, keep the results. The Results will be returned when calling bulk_stop. - :type keep_results: bool - - :return: None - """ - if self.bulk: - raise PyRedisError("Already in bulk mode") - self._bulk = True - self._bulk_size = bulk_size - self._bulk_size_current = 0 - if keep_results: - self._bulk_results = [] - self._bulk_keep = True - - def bulk_stop(self): - """Stop bulk mode. - - All outstanding results from previous commands get fetched. - If bulk_start was called with keep_results=True, return a list with all - results from the executed commands in order. The list of results can also contain - Exceptions, hat you should check for. - - :return: None, list - """ - if not self.bulk: - raise PyRedisError("Not in bulk mode") - self._bulk_fetch() - results = self._bulk_results - self._bulk = False - self._bulk_keep = False - self._bulk_results = None - self._bulk_size = None - self._bulk_size_current = None - return results - - def close(self): - """Close client. - - :return: None - """ - for conn in self._conns.values(): - conn.close() - self._closed = True - - @property - def closed(self): - """Check if client is closed. - - :return: bool - """ - return self._closed - - def execute(self, *args, shard_key=None, sock=None): - """Execute arbitrary redis command. - - :param args: - :type args: list, int, float - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - :type sock: string - - :return: result, exception - """ - if not bool(shard_key) != bool(sock): - raise PyRedisError("Ether shard_key or sock has to be provided") - if not sock: - sock = self._map[slot_from_key(shard_key)] - conn = self._conns[sock] - try: - if not self._bulk: - return self._execute_basic(conn=conn, *args) - else: - self._execute_bulk(conn=conn, *args) - except PyRedisConnError as err: - self.close() - raise err - - -class PubSubClient(commands.Subscribe): - """Pub/Sub Client. - - Subscribe part of the Redis Pub/Sub System. - - :param kwargs: - pyredis.PubSubClient takes the same arguments as pyredis.connection.Connection. - """ - - def __init__(self, **kwargs): - self._conn = Connection(**kwargs) - - def close(self): - """Close Client - - :return: None - """ - self._conn.close() - - @property - def closed(self): - """Check if Client is closed. - - :return: bool - """ - return self._conn.closed - - def write(self, *args): - return self._conn.write(*args) - - def get(self): - """Fetch published item from Redis. - - :return: list - """ - return self._conn.read(close_on_timeout=False) - - -class SentinelClient(object): - """Redis Sentinel Client. - - :param sentinels: - Accepts a list of sentinels in this form: [('sentinel1', 26379), ('sentinel2', 26379), ('sentinel3', 26379)] - :type sentinels: list - - :param password: - Password used for authentication of Sentinel instance itself. If None, no authentication is done. - Only available starting with Redis 5.0.1. - :type password: str - - :param username: - Username used for acl scl authentication. If not set, fall back use legacy auth. - :type username: str - """ - - def __init__(self, sentinels, password=None, username=None): - self._conn = None - self._sentinels = deque(sentinels) - self._password = password - self._username = username - - def _sentinel_connect(self, sentinel): - host, port = sentinel - self._conn = Connection( - host=host, - port=port, - conn_timeout=0.1, - sentinel=True, - password=self._password, - username=self._username, - ) - try: - self.execute("PING") - return True - except PyRedisConnError: - self.close() - return False - - def _sentinel_get(self): - for sentinel in range(len(self._sentinels)): - if self._sentinel_connect(self._sentinels[0]): - return True - else: - self._sentinels.rotate(-1) - raise PyRedisConnError("Could not connect to any sentinel") - - def close(self): - """Close Connection. - - :return: None - """ - if self._conn: - self._conn.close() - self._conn = None - - @property - def sentinels(self): - """Return configured sentinels. - - :return: deque - """ - return self._sentinels - - def execute(self, *args): - """Execute sentinel command. - - :param args: - :type args: string, int, float - - :return: result, exception - """ - if not self._conn: - self._sentinel_get() - self._conn.write(*args) - return self._conn.read() - - def get_master(self, name): - """Get Master Info. - - Return dictionary with master details. - - :param name: Name of Redis service - :type name: str - - :return: dict - """ - return dict_from_list(self.execute("SENTINEL", "master", name)) - - def get_masters(self): - """Get list of masters. - - :return: list of dicts - """ - masters = self.execute("SENTINEL", "masters") - result = [] - for master in masters: - result.append(dict_from_list(master)) - return result - - def get_slaves(self, name): - """Get slaves. - - Return a list of dictionaries, with slave details. - - :param name: Name of Redis service - :type name: str - - :return: - """ - slaves = self.execute("SENTINEL", "slaves", name) - result = [] - for slave in slaves: - result.append(dict_from_list(slave)) - return result - - def next_sentinel(self): - """Switch to the Next Sentinel. - - :return: None - """ - self.close() - self._sentinels.rotate(-1) diff --git a/pyredis/client/__init__.py b/pyredis/client/__init__.py new file mode 100644 index 0000000..3258f95 --- /dev/null +++ b/pyredis/client/__init__.py @@ -0,0 +1,33 @@ +from pyredis.connection import Connection +from pyredis.connection import AsyncConnection +from pyredis.helper import dict_from_list +from pyredis.helper import ClusterMap +from pyredis.async_helper import AsyncClusterMap +from pyredis.client.client import Client +from pyredis.client.async_client import AsyncClient +from pyredis.client.cluster import ClusterClient +from pyredis.client.async_cluster import AsyncClusterClient +from pyredis.client.hash import HashClient +from pyredis.client.async_hash import AsyncHashClient +from pyredis.client.pubsub import PubSubClient +from pyredis.client.async_pubsub import AsyncPubSubClient +from pyredis.client.sentinel import SentinelClient +from pyredis.client.async_sentinel import AsyncSentinelClient + +__all__ = [ + "Client", + "AsyncClient", + "ClusterClient", + "AsyncClusterClient", + "HashClient", + "AsyncHashClient", + "PubSubClient", + "AsyncPubSubClient", + "SentinelClient", + "AsyncSentinelClient", + "Connection", + "AsyncConnection", + "dict_from_list", + "ClusterMap", + "AsyncClusterMap", +] diff --git a/pyredis/client/async_client.py b/pyredis/client/async_client.py new file mode 100644 index 0000000..961750e --- /dev/null +++ b/pyredis/client/async_client.py @@ -0,0 +1,32 @@ +from pyredis import commands +from pyredis.connection import AsyncConnection + + +class AsyncClient( + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, + commands.Transaction, +): + def __init__(self, **kwargs): + super().__init__() + self._conn = AsyncConnection(**kwargs) + + async def execute(self, *args): + await self._conn.write(*args) + return await self._conn.read() + + async def close(self): + await self._conn.close() + + @property + def closed(self): + return self._conn.closed diff --git a/pyredis/client/async_cluster.py b/pyredis/client/async_cluster.py new file mode 100644 index 0000000..9e53c04 --- /dev/null +++ b/pyredis/client/async_cluster.py @@ -0,0 +1,159 @@ +from pyredis import commands +import pyredis.client +from pyredis.exceptions import PyRedisConnError +from pyredis.exceptions import PyRedisConnReadTimeout +from pyredis.exceptions import PyRedisError +from pyredis.exceptions import ReplyError + + +class AsyncClusterClient( + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, + commands.Transaction, +): + def __init__( + self, + seeds=None, + database=0, + password=None, + encoding=None, + slave_ok=False, + conn_timeout=2, + read_timeout=2, + cluster_map=None, + username=None, + ): + super().__init__() + if not bool(seeds) != bool(cluster_map): + raise PyRedisError("Ether seeds or cluster_map has to be provided") + self._cluster = True + self._conns = dict() + self._conn_timeout = conn_timeout + self._read_timeout = read_timeout + self._encoding = encoding + self._password = password + self._database = database + self._slave_ok = slave_ok + if cluster_map: + self._map = cluster_map + else: + self._map = pyredis.client.AsyncClusterMap(seeds=seeds) + self._map_id = self._map.id + self._username = username + + async def _cleanup_conns(self): + hosts = self._map.hosts(slave=self._slave_ok) + wipe = set() + for conn in self._conns.keys(): + if conn not in hosts: + wipe.add(conn) + + for conn in wipe: + await self._conns[conn].close() + del self._conns[conn] + + async def _connect(self, sock): + host, port = sock.split("_") + client = pyredis.client.AsyncConnection( + host=host, + port=int(port), + conn_timeout=self._conn_timeout, + read_timeout=self._read_timeout, + read_only=self._slave_ok, + encoding=self._encoding, + password=self._password, + database=self._database, + username=self._username, + ) + self._conns[sock] = client + + async def _get_slot_info(self, shard_key): + if self._map_id != self._map.id: + self._map_id = self._map.id + await self._cleanup_conns() + try: + return self._map.get_slot( + shard_key=shard_key, + slave=self._slave_ok + ) + except KeyError: + self._map_id = await self._map.update(self._map_id) + await self._cleanup_conns() + return self._map.get_slot( + shard_key=shard_key, + slave=self._slave_ok + ) + + @property + def closed(self): + return False + + async def execute( + self, + *args, + shard_key=None, + sock=None, + asking=False, + retries=3 + ): + if not bool(shard_key) != bool(sock): + raise PyRedisError("Ether shard_key or sock has to be provided") + if not sock: + sock = await self._get_slot_info(shard_key) + if sock not in self._conns.keys(): + await self._connect(sock) + try: + if asking: + await self._conns[sock].write( + *["ASKING", *args] + ) + else: + await self._conns[sock].write(*args) + return await self._conns[sock].read() + except ReplyError as err: + errstr = str(err) + if retries <= 1 and ( + errstr.startswith("MOVED") or errstr.startswith("ASK") + ): + raise PyRedisError( + "Slot moved to often or wrong shard_key, giving up," + ) + if errstr.startswith("MOVED"): + if not shard_key: + raise ReplyError( + f"Explicitly set socket, but key does " + f"not belong to this redis: {sock}" + ) + self._map_id = await self._map.update(self._map_id) + await self._cleanup_conns() + return await self.execute( + *args, + shard_key=shard_key, + retries=retries - 1 + ) + elif errstr.startswith("ASK"): + sock = errstr.split()[2].replace( + ":", + "_" + ) + return await self.execute( + *args, + sock=sock, + retries=retries - 1, + asking=True + ) + else: + raise err + except (PyRedisConnError, PyRedisConnReadTimeout) as err: + await self._conns[sock].close() + del self._conns[sock] + await self._map.update(self._map_id) + raise err diff --git a/pyredis/client/async_hash.py b/pyredis/client/async_hash.py new file mode 100644 index 0000000..bbf6b2f --- /dev/null +++ b/pyredis/client/async_hash.py @@ -0,0 +1,164 @@ +from pyredis import commands +import pyredis.client +from pyredis.exceptions import PyRedisConnError +from pyredis.exceptions import PyRedisError +from pyredis.helper import slot_from_key + + +class AsyncHashClient( + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, + commands.Transaction, +): + def __init__( + self, + buckets, + database=None, + password=None, + encoding=None, + conn_timeout=2, + read_timeout=2, + username=None, + ): + super().__init__() + self._conns = dict() + self._conn_names = list() + self._bulk = False + self._bulk_keep = False + self._bulk_results = None + self._bulk_size = None + self._bulk_size_current = None + self._bulk_bucket_order = list() + self._closed = False + self._cluster = True + self._map = dict() + self._init_conns( + buckets=buckets, + database=database, + password=password, + encoding=encoding, + conn_timeout=conn_timeout, + read_timeout=read_timeout, + username=username, + ) + self._init_map() + + async def _bulk_fetch(self): + for conn in self._bulk_bucket_order: + result = await conn.read(raise_on_result_err=False) + if self._bulk_keep: + self._bulk_results.append(result) + self._bulk_bucket_order = list() + self._bulk_size_current = 0 + + @staticmethod + async def _execute_basic(*args, conn): + await conn.write(*args) + return await conn.read() + + async def _execute_bulk(self, *args, conn): + await conn.write(*args) + self._bulk_size_current += 1 + self._bulk_bucket_order.append(conn) + if self._bulk_size_current == self._bulk_size: + await self._bulk_fetch() + + def _init_conns( + self, + buckets, + database, + password, + encoding, + conn_timeout, + read_timeout, + username, + ): + for bucket in buckets: + host, port = bucket + bucketname = f"{host}_{port}" + self._conn_names.append(bucketname) + self._conns[bucketname] = pyredis.client.AsyncConnection( + host=host, + port=port, + database=database, + password=password, + encoding=encoding, + conn_timeout=conn_timeout, + read_timeout=read_timeout, + username=username, + ) + + def _init_map(self): + num_buckets = len(self._conn_names) - 1 + cur_bucket = 0 + for slot in range(16384): + self._map[slot] = self._conn_names[cur_bucket] + if cur_bucket == num_buckets: + cur_bucket = 0 + else: + cur_bucket += 1 + + @property + def bulk(self): + return self._bulk + + def bulk_start(self, bulk_size=5000, keep_results=True): + if self.bulk: + raise PyRedisError("Already in bulk mode") + self._bulk = True + self._bulk_size = bulk_size + self._bulk_size_current = 0 + if keep_results: + self._bulk_results = [] + self._bulk_keep = True + + async def bulk_stop(self): + if not self.bulk: + raise PyRedisError("Not in bulk mode") + await self._bulk_fetch() + results = self._bulk_results + self._bulk = False + self._bulk_keep = False + self._bulk_results = None + self._bulk_size = None + self._bulk_size_current = None + return results + + async def close(self): + for conn in self._conns.values(): + await conn.close() + self._closed = True + + @property + def closed(self): + return self._closed + + async def execute(self, *args, shard_key=None, sock=None): + if not bool(shard_key) != bool(sock): + raise PyRedisError("Ether shard_key or sock has to be provided") + if not sock: + sock = self._map[slot_from_key(shard_key)] + conn = self._conns[sock] + try: + if not self._bulk: + return await self._execute_basic( + *args, + conn=conn + ) + else: + await self._execute_bulk( + *args, + conn=conn + ) + except PyRedisConnError as err: + await self.close() + raise err diff --git a/pyredis/client/async_pubsub.py b/pyredis/client/async_pubsub.py new file mode 100644 index 0000000..c28f3c1 --- /dev/null +++ b/pyredis/client/async_pubsub.py @@ -0,0 +1,20 @@ +from pyredis import commands +import pyredis.client + + +class AsyncPubSubClient(commands.Subscribe): + def __init__(self, **kwargs): + self._conn = pyredis.client.AsyncConnection(**kwargs) + + async def close(self): + await self._conn.close() + + @property + def closed(self): + return self._conn.closed + + async def write(self, *args): + return await self._conn.write(*args) + + async def get(self): + return await self._conn.read(close_on_timeout=False) diff --git a/pyredis/client/async_sentinel.py b/pyredis/client/async_sentinel.py new file mode 100644 index 0000000..4c3b95c --- /dev/null +++ b/pyredis/client/async_sentinel.py @@ -0,0 +1,83 @@ +from collections import deque +import pyredis.client +from pyredis.exceptions import PyRedisConnError + + +class AsyncSentinelClient(object): + def __init__(self, sentinels, password=None, username=None): + self._conn = None + self._sentinels = deque(sentinels) + self._password = password + self._username = username + + async def _sentinel_connect(self, sentinel): + host, port = sentinel + self._conn = pyredis.client.AsyncConnection( + host=host, + port=port, + conn_timeout=0.1, + sentinel=True, + password=self._password, + username=self._username, + ) + try: + await self.execute("PING") + return True + except PyRedisConnError: + await self.close() + return False + + async def _sentinel_get(self): + for sentinel in range(len(self._sentinels)): + if await self._sentinel_connect(self._sentinels[0]): + return True + else: + self._sentinels.rotate(-1) + raise PyRedisConnError("Could not connect to any sentinel") + + async def close(self): + if self._conn: + await self._conn.close() + self._conn = None + + @property + def sentinels(self): + return self._sentinels + + async def execute(self, *args): + if not self._conn: + await self._sentinel_get() + await self._conn.write(*args) + return await self._conn.read() + + async def get_master(self, name): + result = await self.execute( + *["SENTINEL", "master", name] + ) + return pyredis.client.dict_from_list(result) + + async def get_masters(self): + masters = await self.execute( + *["SENTINEL", "masters"] + ) + result = [] + for master in masters: + result.append( + pyredis.client.dict_from_list(master) + ) + return result + + async def get_slaves(self, name): + slaves = await self.execute( + *["SENTINEL", "slaves", name] + ) + result = [] + for slave in slaves: + result.append( + pyredis.client.dict_from_list(slave) + ) + return result + + async def next_sentinel(self): + await self.close() + self._sentinels.rotate(-1) diff --git a/pyredis/client/client.py b/pyredis/client/client.py new file mode 100644 index 0000000..2220a8d --- /dev/null +++ b/pyredis/client/client.py @@ -0,0 +1,85 @@ +from pyredis import commands +import pyredis.client +from pyredis.exceptions import PyRedisError + + +class Client( + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, + commands.Transaction, +): + """Base Client for Talking to Redis.""" + + def __init__(self, **kwargs): + super().__init__() + self._conn = pyredis.client.Connection(**kwargs) + self._bulk = False + self._bulk_keep = False + self._bulk_results = None + self._bulk_size = None + self._bulk_size_current = None + + def _bulk_fetch(self): + while self._bulk_size_current != 0: + result = self._conn.read(raise_on_result_err=False) + self._bulk_size_current -= 1 + if self._bulk_keep: + self._bulk_results.append(result) + + def _execute_basic(self, *args): + self._conn.write(*args) + return self._conn.read() + + def _execute_bulk(self, *args): + self._conn.write(*args) + self._bulk_size_current += 1 + if self._bulk_size_current == self._bulk_size: + self._bulk_fetch() + + @property + def bulk(self): + return self._bulk + + def bulk_start(self, bulk_size=5000, keep_results=True): + if self.bulk: + raise PyRedisError("Already in bulk mode") + self._bulk = True + self._bulk_size = bulk_size + self._bulk_size_current = 0 + if keep_results: + self._bulk_results = [] + self._bulk_keep = True + + def bulk_stop(self): + if not self.bulk: + raise PyRedisError("Not in bulk mode") + self._bulk_fetch() + results = self._bulk_results + self._bulk = False + self._bulk_keep = False + self._bulk_results = None + self._bulk_size = None + self._bulk_size_current = None + return results + + def close(self): + self._conn.close() + + @property + def closed(self): + return self._conn.closed + + def execute(self, *args): + if not self._bulk: + return self._execute_basic(*args) + else: + self._execute_bulk(*args) diff --git a/pyredis/client/cluster.py b/pyredis/client/cluster.py new file mode 100644 index 0000000..a7735f5 --- /dev/null +++ b/pyredis/client/cluster.py @@ -0,0 +1,159 @@ +from pyredis import commands +import pyredis.client +from pyredis.exceptions import PyRedisConnError +from pyredis.exceptions import PyRedisConnReadTimeout +from pyredis.exceptions import PyRedisError +from pyredis.exceptions import ReplyError + + +class ClusterClient( + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, + commands.Transaction, +): + def __init__( + self, + seeds=None, + database=0, + password=None, + encoding=None, + slave_ok=False, + conn_timeout=2, + read_timeout=2, + cluster_map=None, + username=None, + ): + super().__init__() + if not bool(seeds) != bool(cluster_map): + raise PyRedisError("Ether seeds or cluster_map has to be provided") + self._cluster = True + self._conns = dict() + self._conn_timeout = conn_timeout + self._read_timeout = read_timeout + self._encoding = encoding + self._password = password + self._database = database + self._slave_ok = slave_ok + if cluster_map: + self._map = cluster_map + else: + self._map = pyredis.client.ClusterMap(seeds=seeds) + self._map_id = self._map.id + self._username = username + + def _cleanup_conns(self): + hosts = self._map.hosts(slave=self._slave_ok) + wipe = set() + for conn in self._conns.keys(): + if conn not in hosts: + wipe.add(conn) + + for conn in wipe: + self._conns[conn].close() + del self._conns[conn] + + def _connect(self, sock): + host, port = sock.split("_") + client = pyredis.client.Connection( + host=host, + port=int(port), + conn_timeout=self._conn_timeout, + read_timeout=self._read_timeout, + read_only=self._slave_ok, + encoding=self._encoding, + password=self._password, + database=self._database, + username=self._username, + ) + self._conns[sock] = client + + def _get_slot_info(self, shard_key): + if self._map_id != self._map.id: + self._map_id = self._map.id + self._cleanup_conns() + try: + return self._map.get_slot( + shard_key=shard_key, + slave=self._slave_ok + ) + except KeyError: + self._map_id = self._map.update(self._map_id) + self._cleanup_conns() + return self._map.get_slot( + shard_key=shard_key, + slave=self._slave_ok + ) + + @property + def closed(self): + return False + + def execute( + self, + *args, + shard_key=None, + sock=None, + asking=False, + retries=3 + ): + if not bool(shard_key) != bool(sock): + raise PyRedisError("Ether shard_key or sock has to be provided") + if not sock: + sock = self._get_slot_info(shard_key) + if sock not in self._conns.keys(): + self._connect(sock) + try: + if asking: + self._conns[sock].write( + *["ASKING", *args] + ) + else: + self._conns[sock].write(*args) + return self._conns[sock].read() + except ReplyError as err: + errstr = str(err) + if retries <= 1 and ( + errstr.startswith("MOVED") or errstr.startswith("ASK") + ): + raise PyRedisError( + "Slot moved to often or wrong shard_key, giving up," + ) + if errstr.startswith("MOVED"): + if not shard_key: + raise ReplyError( + f"Explicitly set socket, but key does " + f"not belong to this redis: {sock}" + ) + self._map_id = self._map.update(self._map_id) + self._cleanup_conns() + return self.execute( + *args, + shard_key=shard_key, + retries=retries - 1 + ) + elif errstr.startswith("ASK"): + sock = errstr.split()[2].replace( + ":", + "_" + ) + return self.execute( + *args, + sock=sock, + retries=retries - 1, + asking=True + ) + else: + raise err + except (PyRedisConnError, PyRedisConnReadTimeout) as err: + self._conns[sock].close() + del self._conns[sock] + self._map.update(self._map_id) + raise err diff --git a/pyredis/client/hash.py b/pyredis/client/hash.py new file mode 100644 index 0000000..0bcd625 --- /dev/null +++ b/pyredis/client/hash.py @@ -0,0 +1,164 @@ +from pyredis import commands +import pyredis.client +from pyredis.exceptions import PyRedisConnError +from pyredis.exceptions import PyRedisError +from pyredis.helper import slot_from_key + + +class HashClient( + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, + commands.Transaction, +): + def __init__( + self, + buckets, + database=None, + password=None, + encoding=None, + conn_timeout=2, + read_timeout=2, + username=None, + ): + super().__init__() + self._conns = dict() + self._conn_names = list() + self._bulk = False + self._bulk_keep = False + self._bulk_results = None + self._bulk_size = None + self._bulk_size_current = None + self._bulk_bucket_order = list() + self._closed = False + self._cluster = True + self._map = dict() + self._init_conns( + buckets=buckets, + database=database, + password=password, + encoding=encoding, + conn_timeout=conn_timeout, + read_timeout=read_timeout, + username=username, + ) + self._init_map() + + def _bulk_fetch(self): + for conn in self._bulk_bucket_order: + result = conn.read(raise_on_result_err=False) + if self._bulk_keep: + self._bulk_results.append(result) + self._bulk_bucket_order = list() + self._bulk_size_current = 0 + + @staticmethod + def _execute_basic(*args, conn): + conn.write(*args) + return conn.read() + + def _execute_bulk(self, *args, conn): + conn.write(*args) + self._bulk_size_current += 1 + self._bulk_bucket_order.append(conn) + if self._bulk_size_current == self._bulk_size: + self._bulk_fetch() + + def _init_conns( + self, + buckets, + database, + password, + encoding, + conn_timeout, + read_timeout, + username, + ): + for bucket in buckets: + host, port = bucket + bucketname = f"{host}_{port}" + self._conn_names.append(bucketname) + self._conns[bucketname] = pyredis.client.Connection( + host=host, + port=port, + database=database, + password=password, + encoding=encoding, + conn_timeout=conn_timeout, + read_timeout=read_timeout, + username=username, + ) + + def _init_map(self): + num_buckets = len(self._conn_names) - 1 + cur_bucket = 0 + for slot in range(16384): + self._map[slot] = self._conn_names[cur_bucket] + if cur_bucket == num_buckets: + cur_bucket = 0 + else: + cur_bucket += 1 + + @property + def bulk(self): + return self._bulk + + def bulk_start(self, bulk_size=5000, keep_results=True): + if self.bulk: + raise PyRedisError("Already in bulk mode") + self._bulk = True + self._bulk_size = bulk_size + self._bulk_size_current = 0 + if keep_results: + self._bulk_results = [] + self._bulk_keep = True + + def bulk_stop(self): + if not self.bulk: + raise PyRedisError("Not in bulk mode") + self._bulk_fetch() + results = self._bulk_results + self._bulk = False + self._bulk_keep = False + self._bulk_results = None + self._bulk_size = None + self._bulk_size_current = None + return results + + def close(self): + for conn in self._conns.values(): + conn.close() + self._closed = True + + @property + def closed(self): + return self._closed + + def execute(self, *args, shard_key=None, sock=None): + if not bool(shard_key) != bool(sock): + raise PyRedisError("Ether shard_key or sock has to be provided") + if not sock: + sock = self._map[slot_from_key(shard_key)] + conn = self._conns[sock] + try: + if not self._bulk: + return self._execute_basic( + *args, + conn=conn + ) + else: + self._execute_bulk( + *args, + conn=conn + ) + except PyRedisConnError as err: + self.close() + raise err diff --git a/pyredis/client/pubsub.py b/pyredis/client/pubsub.py new file mode 100644 index 0000000..79f25a0 --- /dev/null +++ b/pyredis/client/pubsub.py @@ -0,0 +1,20 @@ +from pyredis import commands +import pyredis.client + + +class PubSubClient(commands.Subscribe): + def __init__(self, **kwargs): + self._conn = pyredis.client.Connection(**kwargs) + + def close(self): + self._conn.close() + + @property + def closed(self): + return self._conn.closed + + def write(self, *args): + return self._conn.write(*args) + + def get(self): + return self._conn.read(close_on_timeout=False) diff --git a/pyredis/client/sentinel.py b/pyredis/client/sentinel.py new file mode 100644 index 0000000..55d0998 --- /dev/null +++ b/pyredis/client/sentinel.py @@ -0,0 +1,84 @@ +from collections import deque +import pyredis.client +from pyredis.exceptions import PyRedisConnError + + +class SentinelClient(object): + def __init__(self, sentinels, password=None, username=None): + self._conn = None + self._sentinels = deque(sentinels) + self._password = password + self._username = username + + def _sentinel_connect(self, sentinel): + host, port = sentinel + self._conn = pyredis.client.Connection( + host=host, + port=port, + conn_timeout=0.1, + sentinel=True, + password=self._password, + username=self._username, + ) + try: + self.execute("PING") + return True + except PyRedisConnError: + self.close() + return False + + def _sentinel_get(self): + for sentinel in range(len(self._sentinels)): + if self._sentinel_connect(self._sentinels[0]): + return True + else: + self._sentinels.rotate(-1) + raise PyRedisConnError("Could not connect to any sentinel") + + def close(self): + if self._conn: + self._conn.close() + self._conn = None + + @property + def sentinels(self): + return self._sentinels + + def execute(self, *args): + if not self._conn: + self._sentinel_get() + self._conn.write(*args) + return self._conn.read() + + def get_master(self, name): + return pyredis.client.dict_from_list( + self.execute( + *["SENTINEL", "master", name] + ) + ) + + def get_masters(self): + masters = self.execute( + *["SENTINEL", "masters"] + ) + result = [] + for master in masters: + result.append( + pyredis.client.dict_from_list(master) + ) + return result + + def get_slaves(self, name): + slaves = self.execute( + *["SENTINEL", "slaves", name] + ) + result = [] + for slave in slaves: + result.append( + pyredis.client.dict_from_list(slave) + ) + return result + + def next_sentinel(self): + self.close() + self._sentinels.rotate(-1) diff --git a/pyredis/commands.py b/pyredis/commands.py deleted file mode 100644 index 64b5a23..0000000 --- a/pyredis/commands.py +++ /dev/null @@ -1,1566 +0,0 @@ -__author__ = "schlitzer" - -__all__ = [ - "Connection", - "Geo", - "Hash", - "HyperLogLog", - "Key", - "List", - "Publish", - "Scripting", - "Set", - "SSet", - "String", - "Subscribe", - "Transaction", -] - - - -class BaseCommand(object): - def __init__(self): - self._cluster = False - - def execute(self, *args, **kwargs): - raise NotImplemented - - -class Connection(BaseCommand): - def __init__(self): - super().__init__() - - def echo(self, *args, shard_key=None, sock=None): - """Execute ECHO Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ECHO", *args, shard_key=shard_key, sock=sock) - return self.execute(b"ECHO", *args) - - def ping(self, shard_key=None, sock=None): - """Execute PING Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result,exception - """ - if self._cluster: - return self.execute(b"PING", shard_key=shard_key, sock=sock) - return self.execute(b"PING") - - -class Geo(BaseCommand): - def __init__(self): - super().__init__() - - def geoadd(self, *args): - """Execute GEOADD Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"GEOADD", *args, shard_key=args[0]) - return self.execute(b"GEOADD", *args) - - def geodist(self, *args): - """Execute GEODIST Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"GEODIST", *args, shard_key=args[0]) - return self.execute(b"GEODIST", *args) - - def geohash(self, *args): - """Execute GEOHASH Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"GEOHASH", *args, shard_key=args[0]) - return self.execute(b"GEOHASH", *args) - - def georadius(self, *args): - """Execute GEORADIUS Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"GEORADIUS", *args, shard_key=args[0]) - return self.execute(b"GEORADIUS", *args) - - def geopos(self, *args): - """Execute GEOPOS Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"GEOPOS", *args, shard_key=args[0]) - return self.execute(b"GEOPOS", *args) - - def georadiusbymember(self, *args): - """Execute GEORADIUSBYMEMBER Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"GEORADIUSBYMEMBER", *args, shard_key=args[0]) - return self.execute(b"GEORADIUSBYMEMBER", *args) - - -class Key(BaseCommand): - def __init__(self): - super().__init__() - - def delete(self, *args): - """Execute DEL Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"DEL", *args, shard_key=args[0]) - return self.execute(b"DEL", *args) - - def dump(self, *args): - """Execute DUMP Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"DUMP", *args, shard_key=args[0]) - return self.execute(b"DUMP", *args) - - def exists(self, *args): - """Execute EXISTS Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"EXISTS", *args, shard_key=args[0]) - return self.execute(b"EXISTS", *args) - - def expire(self, *args): - """Execute EXPIRE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"EXPIRE", *args, shard_key=args[0]) - return self.execute(b"EXPIRE", *args) - - def expireat(self, *args): - """Execute EXPIREAT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"EXPIREAT") - return self.execute(b"EXPIREAT", *args) - - def keys(self, *args, shard_key=None, sock=None): - """Execute KEYS Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute(b"KEYS", *args, shard_key=shard_key, sock=sock) - return self.execute(b"KEYS", *args) - - def migrate(self, *args): - """Execute MIGRATE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - raise NotImplemented - return self.execute(b"MIGRATE", *args) - - def move(self, *args): - """Execute MOVE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"MOVE", *args, shard_key=args[0]) - return self.execute(b"MOVE", *args) - - def object(self, *args, shard_key=None, sock=None): - """Execute OBJECT Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute(b"DEL", *args, shard_key=shard_key, sock=sock) - return self.execute(b"OBJECT", *args) - - def persist(self, *args): - """Execute PERSIST Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"PERSIST", *args, shard_key=args[0]) - return self.execute(b"PERSIST", *args) - - def pexpire(self, *args): - """Execute PEXPIRE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"PEXPIRE", *args, shard_key=args[0]) - return self.execute(b"PEXPIRE", *args) - - def pexpireat(self, *args): - """Execute PEXPIREAT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"PEXPIREAT", *args, shard_key=args[0]) - return self.execute(b"PEXPIREAT", *args) - - def pttl(self, *args): - """Execute PTTL Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"PTTL", *args, shard_key=args[0]) - return self.execute(b"PTTL", *args) - - def randomkey(self, *args, shard_key=None, sock=None): - """Execute RANDOMKEY Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute(b"RANDOMKEY", *args, shard_key=shard_key, sock=sock) - return self.execute(b"RANDOMKEY", *args) - - def rename(self, *args): - """Execute RENAME Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"RENAME", *args, shard_key=args[0]) - return self.execute(b"RENAME", *args) - - def renamenx(self, *args): - """Execute RENAMENX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"RENAMENX", *args, shard_key=args[0]) - return self.execute(b"RENAMENX", *args) - - def restore(self, *args): - """Execute RESTORE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"RESTORE", *args, shard_key=args[0]) - return self.execute(b"RESTORE", *args) - - def scan(self, *args, shard_key=None, sock=None): - """Execute SCAN Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SCAN", *args, shard_key=shard_key, sock=sock) - return self.execute(b"SCAN", *args) - - def sort(self, *args): - """Execute SORT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SORT", *args, shard_key=args[0]) - return self.execute(b"SORT", *args) - - def ttl(self, *args): - """Execute TTL Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"TTL", *args, shard_key=args[0]) - return self.execute(b"TTL", *args) - - def type(self, *args): - """Execute TYPE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"TYPE", *args, shard_key=args[0]) - return self.execute(b"TYPE", *args) - - def wait(self, *args): - """Execute WAIT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"WAIT", *args, shard_key=args[0]) - return self.execute(b"WAIT", *args) - - -class String(BaseCommand): - def __init__(self): - super().__init__() - - def append(self, *args): - """Execute APPEND Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"APPEND", *args, shard_key=args[0]) - return self.execute(b"APPEND", *args) - - def bitcount(self, *args): - """Execute BITCOUNT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"BITCOUNT", *args, shard_key=args[0]) - return self.execute(b"BITCOUNT", *args) - - def bitfield(self, *args): - """Execute BITFIELD Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"BITFIELD", *args, shard_key=args[0]) - return self.execute(b"BITFIELD", *args) - - def bitop(self, *args): - """Execute BITOP Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"BITOP", *args, shard_key=args[1]) - return self.execute(b"BITOP", *args) - - def bitpos(self, *args): - """Execute BITPOS Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"BITPOS", *args, shard_key=args[0]) - return self.execute(b"BITPOS", *args) - - def decr(self, *args): - """Execute DECR Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"DECR", *args, shard_key=args[0]) - return self.execute(b"DECR", *args) - - def decrby(self, *args): - """Execute DECRBY Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"DECRBY", *args, shard_key=args[0]) - return self.execute(b"DECRBY", *args) - - def get(self, *args): - """Execute GET Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"GET", *args, shard_key=args[0]) - return self.execute(b"GET", *args) - - def getbit(self, *args): - """Execute GETBIT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"GETBIT", *args, shard_key=args[0]) - return self.execute(b"GETBIT", *args) - - def getrange(self, *args): - """Execute GETRANGE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"GETRANGE", *args, shard_key=args[0]) - return self.execute(b"GETRANGE", *args) - - def getset(self, *args): - """Execute GETSET Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"GETSET", *args, shard_key=args[0]) - return self.execute(b"GETSET", *args) - - def incr(self, *args): - """Execute INCR Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"INCR", *args, shard_key=args[0]) - return self.execute(b"INCR", *args) - - def incrby(self, *args): - """Execute INCRBY Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"INCRBY", *args, shard_key=args[0]) - return self.execute(b"INCRBY", *args) - - def incrbyfloat(self, *args): - """Execute INCRBYFLOAT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"INCRBYFLOAT", *args, shard_key=args[0]) - return self.execute(b"INCRBYFLOAT", *args) - - def mget(self, *args): - """Execute MGET Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"MGET", *args, shard_key=args[0]) - return self.execute(b"MGET", *args) - - def mset(self, *args): - """Execute MSET Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"MSET", *args, shard_key=args[0]) - return self.execute(b"MSET", *args) - - def msetnx(self, *args): - """Execute MSETNX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"MSETNX", *args, shard_key=args[0]) - return self.execute(b"MSETNX", *args) - - def psetex(self, *args): - """Execute PSETEX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"PSETEX", *args, shard_key=args[0]) - return self.execute(b"PSETEX", *args) - - def set(self, *args): - """Execute SET Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SET", *args, shard_key=args[0]) - return self.execute(b"SET", *args) - - def setbit(self, *args): - """Execute SETBIT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SETBIT", *args, shard_key=args[0]) - return self.execute(b"SETBIT", *args) - - def setex(self, *args): - """Execute SETEX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SETEX", *args, shard_key=args[0]) - return self.execute(b"SETEX", *args) - - def setnx(self, *args): - """Execute SETNX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SETNX", *args, shard_key=args[0]) - return self.execute(b"SETNX", *args) - - def setrange(self, *args): - """Execute SETRANGE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SETRANGE", *args, shard_key=args[0]) - return self.execute(b"SETRANGE", *args) - - def strlen(self, *args): - """Execute STRLEN Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"STRLEN", *args, shard_key=args[0]) - return self.execute(b"STRLEN", *args) - - -class Hash(BaseCommand): - def __init__(self): - super().__init__() - - def hdel(self, *args): - """Execute HDEL Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HDEL", *args, shard_key=args[0]) - return self.execute(b"HDEL", *args) - - def hexists(self, *args): - """Execute HEXISTS Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HEXISTS", *args, shard_key=args[0]) - return self.execute(b"HEXISTS", *args) - - def hget(self, *args): - """Execute HGET Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HGET", *args, shard_key=args[0]) - return self.execute(b"HGET", *args) - - def hgetall(self, *args): - """Execute HGETALL Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HGETALL", *args, shard_key=args[0]) - return self.execute(b"HGETALL", *args) - - def hincrby(self, *args): - """Execute HINCRBY Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HINCRBY", *args, shard_key=args[0]) - return self.execute(b"HINCRBY", *args) - - def hincrbyfloat(self, *args): - """Execute HINCRBYFLOAT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HINCRBYFLOAT", *args, shard_key=args[0]) - return self.execute(b"HINCRBYFLOAT", *args) - - def hkeys(self, *args): - """Execute HKEYS Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HKEYS", *args, shard_key=args[0]) - return self.execute(b"HKEYS", *args) - - def hlen(self, *args): - """Execute HLEN Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HLEN", *args, shard_key=args[0]) - return self.execute(b"HLEN", *args) - - def hmget(self, *args): - """Execute HMGET Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HMGET", *args, shard_key=args[0]) - return self.execute(b"HMGET", *args) - - def hmset(self, *args): - """Execute HMSET Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HMSET", *args, shard_key=args[0]) - return self.execute(b"HMSET", *args) - - def hset(self, *args): - """Execute HSET Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HSET", *args, shard_key=args[0]) - return self.execute(b"HSET", *args) - - def hsetnx(self, *args): - """Execute HSETNX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HSETNX", *args, shard_key=args[0]) - return self.execute(b"HSETNX", *args) - - def hstrlen(self, *args): - """Execute HSTRLEN Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HSTRLEN", *args, shard_key=args[0]) - return self.execute(b"HSTRLEN", *args) - - def hvals(self, *args): - """Execute HVALS Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HVALS", *args, shard_key=args[0]) - return self.execute(b"HVALS", *args) - - def hscan(self, *args): - """Execute HSCAN Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"HSCAN", *args, shard_key=args[0]) - return self.execute(b"HSCAN", *args) - - -class List(BaseCommand): - def __init__(self): - super().__init__() - - def blpop(self, *args): - """Execute BLPOP Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"BLPOP", *args, shard_key=args[0]) - return self.execute(b"BLPOP", *args) - - def brpop(self, *args): - """Execute BRPOP Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"BRPOP", *args, shard_key=args[0]) - return self.execute(b"BRPOP", *args) - - def brpoplpush(self, *args): - """Execute BRPOPPUSH Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"BRPOPPUSH", *args, shard_key=args[0]) - return self.execute(b"BRPOPPUSH", *args) - - def lindex(self, *args): - """Execute LINDEX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"LINDEX", *args, shard_key=args[0]) - return self.execute(b"LINDEX", *args) - - def linsert(self, *args): - """Execute LINSERT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"LINSERT", *args, shard_key=args[0]) - return self.execute(b"LINSERT", *args) - - def llen(self, *args): - """Execute LLEN Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"LLEN", *args, shard_key=args[0]) - return self.execute(b"LLEN", *args) - - def lpop(self, *args): - """Execute LPOP Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"LPOP", *args, shard_key=args[0]) - return self.execute(b"LPOP", *args) - - def lpush(self, *args): - """Execute LPUSH Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"LPUSH", *args, shard_key=args[0]) - return self.execute(b"LPUSH", *args) - - def lpushx(self, *args): - """Execute LPUSHX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"LPUSHX", *args, shard_key=args[0]) - return self.execute(b"LPUSHX", *args) - - def lrange(self, *args): - """Execute LRANGE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"LRANGE", *args, shard_key=args[0]) - return self.execute(b"LRANGE", *args) - - def lrem(self, *args): - """Execute LREM Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"LREM", *args, shard_key=args[0]) - return self.execute(b"LREM", *args) - - def lset(self, *args): - """Execute LSET Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"LSET", *args, shard_key=args[0]) - return self.execute(b"LSET", *args) - - def ltrim(self, *args): - """Execute LTRIM Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"LTRIM", *args, shard_key=args[0]) - return self.execute(b"LTRIM", *args) - - def rpop(self, *args): - """Execute RPOP Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"RPOP", *args, shard_key=args[0]) - return self.execute(b"RPOP", *args) - - def rpoplpush(self, *args): - """Execute RPOPLPUSH Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"RPOPLPUSH", *args, shard_key=args[0]) - return self.execute(b"RPOPLPUSH", *args) - - def rpush(self, *args): - """Execute RPUSH Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"RPUSH", *args, shard_key=args[0]) - return self.execute(b"RPUSH", *args) - - def rpushx(self, *args): - """Execute RPUSHX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"RPUSHX", *args, shard_key=args[0]) - return self.execute(b"RPUSHX", *args) - - -class Set(BaseCommand): - def __init__(self): - super().__init__() - - def sadd(self, *args): - """Execute SADD Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SADD", *args, shard_key=args[0]) - return self.execute(b"SADD", *args) - - def scard(self, *args): - """Execute SCARD Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SCARD", *args, shard_key=args[0]) - return self.execute(b"SCARD", *args) - - def sdiff(self, *args): - """Execute SDIFF Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SDIFF", *args, shard_key=args[0]) - return self.execute(b"SDIFF", *args) - - def sdiffstore(self, *args): - """Execute SDIFFSTORE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SDIFFSTORE", *args, shard_key=args[0]) - return self.execute(b"SDIFFSTORE", *args) - - def sinter(self, *args): - """Execute SINTER Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SINTER", *args, shard_key=args[0]) - return self.execute(b"SINTER", *args) - - def sinterstore(self, *args): - """Execute SINTERSTORE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SINTERSTORE", *args, shard_key=args[0]) - return self.execute(b"SINTERSTORE", *args) - - def sismember(self, *args): - """Execute SISMEMBER Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SISMEMBER", *args, shard_key=args[0]) - return self.execute(b"SISMEMBER", *args) - - def smembers(self, *args): - """Execute SMEMBERS Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SMEMBERS", *args, shard_key=args[0]) - return self.execute(b"SMEMBERS", *args) - - def smove(self, *args): - """Execute SMOVE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SMOVE", *args, shard_key=args[0]) - return self.execute(b"SMOVE", *args) - - def spop(self, *args): - """Execute SPOP Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SPOP", *args, shard_key=args[0]) - return self.execute(b"SPOP", *args) - - def srandmember(self, *args): - """Execute SRANDMEMBER Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SRANDMEMBER", *args, shard_key=args[0]) - return self.execute(b"SRANDMEMBER", *args) - - def srem(self, *args): - """Execute SREM Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SREM", *args, shard_key=args[0]) - return self.execute(b"SREM", *args) - - def sunion(self, *args): - """Execute SUNION Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SUNION", *args, shard_key=args[0]) - return self.execute(b"SUNION", *args) - - def sunoinstore(self, *args): - """Execute SUNIONSTORE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SUNIONSTORE", *args, shard_key=args[0]) - return self.execute(b"SUNIONSTORE", *args) - - def sscan(self, *args): - """Execute SSCAN Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"SSCAN", *args, shard_key=args[0]) - return self.execute(b"SSCAN", *args) - - -class SSet(BaseCommand): - def __init__(self): - super().__init__() - - def zadd(self, *args): - """Execute ZADD Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZADD", *args, shard_key=args[0]) - return self.execute(b"ZADD", *args) - - def zcard(self, *args): - """Execute ZCARD Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZCARD", *args, shard_key=args[0]) - return self.execute(b"ZCARD", *args) - - def zcount(self, *args): - """Execute ZCOUNT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZCOUNT", *args, shard_key=args[0]) - return self.execute(b"ZCOUNT", *args) - - def zincrby(self, *args): - """Execute ZINCRBY Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZINCRBY", *args, shard_key=args[0]) - return self.execute(b"ZINCRBY", *args) - - def zinterstore(self, *args): - """Execute ZINTERSTORE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZINTERSTORE", *args, shard_key=args[0]) - return self.execute(b"ZINTERSTORE", *args) - - def zlexcount(self, *args): - """Execute ZLEXCOUNT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZLEXCOUNT", *args, shard_key=args[0]) - return self.execute(b"ZLEXCOUNT", *args) - - def zrange(self, *args): - """Execute ZRANGE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZRANGE", *args, shard_key=args[0]) - return self.execute(b"ZRANGE", *args) - - def zrangebylex(self, *args): - """Execute ZRANGEBYLEX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZRANGEBYLEX", *args, shard_key=args[0]) - return self.execute(b"ZRANGEBYLEX", *args) - - def zrangebyscore(self, *args): - """Execute ZRANGEBYSCORE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZRANGEBYSCORE", *args, shard_key=args[0]) - return self.execute(b"ZRANGEBYSCORE", *args) - - def zrank(self, *args): - """Execute ZRANK Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZRANK", *args, shard_key=args[0]) - return self.execute(b"ZRANK", *args) - - def zrem(self, *args): - """Execute ZREM Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZREM", *args, shard_key=args[0]) - return self.execute(b"ZREM", *args) - - def zremrangebylex(self, *args): - """Execute ZREMRANGEBYLEX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZREMRANGEBYLEX", *args, shard_key=args[0]) - return self.execute(b"ZREMRANGEBYLEX", *args) - - def zremrangebyrank(self, *args): - """Execute ZREMRANGEBYRANK Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZREMRANGEBYRANK", *args, shard_key=args[0]) - return self.execute(b"ZREMRANGEBYRANK", *args) - - def zremrangebyscrore(self, *args): - """Execute ZREMRANGEBYSCORE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZREMRANGEBYSCORE", *args, shard_key=args[0]) - return self.execute(b"ZREMRANGEBYSCORE", *args) - - def zrevrange(self, *args): - """Execute ZREVRANGE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZREVRANGE", *args, shard_key=args[0]) - return self.execute(b"ZREVRANGE", *args) - - def zrevrangebylex(self, *args): - """Execute ZREVRANGEBYLEX Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZREVRANGEBYLEX", *args, shard_key=args[0]) - return self.execute(b"ZREVRANGEBYLEX", *args) - - def zrevrangebyscore(self, *args): - """Execute ZREVRANGEBYSCORE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZREVRANGEBYSCORE", *args, shard_key=args[0]) - return self.execute(b"ZREVRANGEBYSCORE", *args) - - def zrevrank(self, *args): - """Execute ZREVRANK Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZREVRANK", *args, shard_key=args[0]) - return self.execute(b"ZREVRANK", *args) - - def zscore(self, *args): - """Execute ZSCORE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZSCORE", *args, shard_key=args[0]) - return self.execute(b"ZSCORE", *args) - - def zunionstore(self, *args): - """Execute ZUNIONSTORE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZUNIONSTORE", *args, shard_key=args[0]) - return self.execute(b"ZUNIONSTORE", *args) - - def zscan(self, *args): - """Execute ZSCAN Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"ZSCAN", *args, shard_key=args[0]) - return self.execute(b"ZSCAN", *args) - - -class HyperLogLog(BaseCommand): - def __init__(self): - super().__init__() - - def pfadd(self, *args): - """Execute PFADD Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"PFADD", *args, shard_key=args[0]) - return self.execute(b"PFADD", *args) - - def pfcount(self, *args): - """Execute PFCOUNT Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"PFCOUNT", *args, shard_key=args[0]) - return self.execute(b"PFCOUNT", *args) - - def pfmerge(self, *args): - """Execute PFMERGE Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"PFMERGE", *args, shard_key=args[0]) - return self.execute(b"PFMERGE", *args) - - -class Publish(BaseCommand): - def __init__(self): - super().__init__() - - def publish(self, *args): - """Execute PUBLISH Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - raise NotImplemented - return self.execute(b"PUBLISH", *args) - - -class Subscribe(object): - def write(self, *args): - raise NotImplemented - - def psubscribe(self, *args): - """Execute PSUBSCRIBE Command, consult Redis documentation for details. - - :return: result, exception - """ - return self.write(b"PSUBSCRIBE", *args) - - def punsubscribe(self, *args): - """Execute PUNSUBSCRIBE Command, consult Redis documentation for details. - - :return: result, exception - """ - return self.write(b"PUNSUBSCRIBE", *args) - - def subscribe(self, *args): - """Execute SUBSCRIBE Command, consult Redis documentation for details. - - :return: result, exception - """ - return self.write(b"SUBSCRIBE", *args) - - def unsubscribe(self, *args): - """Execute UNSUBSCRIBE Command, consult Redis documentation for details. - - :return: result, exception - """ - return self.write(b"UNSUBSCRIBE", *args) - - -class Transaction(BaseCommand): - def __init__(self): - super().__init__() - - def discard(self, *args, shard_key=None, sock=None): - """Execute DISCARD Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"DISCARD", *args, shard_key=shard_key, sock=sock) - return self.execute(b"DISCARD", *args) - - def exec(self, *args, shard_key=None, sock=None): - """Execute EXEC Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"EXEC", *args, shard_key=shard_key, sock=sock) - return self.execute(b"EXEC", *args) - - def multi(self, *args, shard_key=None, sock=None): - """Execute MULTI Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"MULTI", *args, shard_key=shard_key, sock=sock) - return self.execute(b"MULTI", *args) - - def unwatch(self, *args, shard_key=None, sock=None): - """Execute UNWATCH Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"UNWATCH", *args, shard_key=shard_key, sock=sock) - return self.execute(b"UNWATCH", *args) - - def watch(self, *args): - """Execute WATCH Command, consult Redis documentation for details. - - :return: result, exception - """ - if self._cluster: - return self.execute(b"WATCH", *args, shard_key=args[0]) - return self.execute(b"WATCH", *args) - - -class Scripting(BaseCommand): - def __init__(self): - super().__init__() - - def eval(self, *args, shard_key=None, sock=None): - """Execute EVAL Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute(b"EVAL", *args, shard_key=shard_key, sock=sock) - return self.execute(b"EVAL", *args) - - def evalsha(self, *args, shard_key=None, sock=None): - """Execute EVALSHA Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute(b"EVALSHA", *args, shard_key=shard_key, sock=sock) - return self.execute(b"EVALSHA", *args) - - def script_debug(self, *args, shard_key=None, sock=None): - """Execute SCRIPT DEBUG Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute( - b"SCRIPT", b"DEBUG", *args, shard_key=shard_key, sock=sock - ) - return self.execute(b"SCRIPT", b"DEBUG", *args) - - def script_exists(self, *args, shard_key=None, sock=None): - """Execute SCRIPT EXISTS Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute( - b"SCRIPT", b"EXISTS", *args, shard_key=shard_key, sock=sock - ) - return self.execute(b"SCRIPT", b"EXISTS", *args) - - def script_flush(self, *args, shard_key=None, sock=None): - """Execute SCRIPT FLUSH Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute( - b"SCRIPT", b"FLUSH", *args, shard_key=shard_key, sock=sock - ) - return self.execute(b"SCRIPT", b"FLUSH", *args) - - def script_kill(self, *args, shard_key=None, sock=None): - """Execute SCRIPT KILL Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute( - b"SCRIPT", b"KILL", *args, shard_key=shard_key, sock=sock - ) - return self.execute(b"SCRIPT", b"KILL", *args) - - def script_load(self, *args, shard_key=None, sock=None): - """Execute SCRIPT LOAD Command, consult Redis documentation for details. - - :param shard_key: (optional) - Should be set to the key name you try to work with. - Can not be used if sock is set. - - Only used if used with a Cluster Client - :type shard_key: string - - :param sock: (optional) - The string representation of a socket, the command should be executed against. - For example: "testhost_6379" - Can not be used if shard_key is set. - - Only used if used with a Cluster Client - :type sock: string - - :return: result, exception - """ - if self._cluster: - return self.execute( - b"SCRIPT", b"LOAD", *args, shard_key=shard_key, sock=sock - ) - return self.execute(b"SCRIPT", b"LOAD", *args) diff --git a/pyredis/commands/__init__.py b/pyredis/commands/__init__.py new file mode 100644 index 0000000..33db4b8 --- /dev/null +++ b/pyredis/commands/__init__.py @@ -0,0 +1,31 @@ +from pyredis.commands.base import BaseCommand +from pyredis.commands.connection import Connection +from pyredis.commands.geo import Geo +from pyredis.commands.hash import Hash +from pyredis.commands.hyperloglog import HyperLogLog +from pyredis.commands.key import Key +from pyredis.commands.list import List +from pyredis.commands.publish import Publish +from pyredis.commands.scripting import Scripting +from pyredis.commands.set import Set +from pyredis.commands.sset import SSet +from pyredis.commands.string import String +from pyredis.commands.subscribe import Subscribe +from pyredis.commands.transaction import Transaction + +__all__ = [ + "BaseCommand", + "Connection", + "Geo", + "Hash", + "HyperLogLog", + "Key", + "List", + "Publish", + "Scripting", + "Set", + "SSet", + "String", + "Subscribe", + "Transaction", +] diff --git a/pyredis/commands/base.py b/pyredis/commands/base.py new file mode 100644 index 0000000..c771e56 --- /dev/null +++ b/pyredis/commands/base.py @@ -0,0 +1,6 @@ +class BaseCommand(object): + def __init__(self): + self._cluster = False + + def execute(self, *args, **kwargs): + raise NotImplementedError diff --git a/pyredis/commands/connection.py b/pyredis/commands/connection.py new file mode 100644 index 0000000..9de8e85 --- /dev/null +++ b/pyredis/commands/connection.py @@ -0,0 +1,26 @@ +from pyredis.commands.base import BaseCommand + + +class Connection(BaseCommand): + def __init__(self): + super().__init__() + + def echo(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"ECHO", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"ECHO", *args] + ) + + def ping(self, shard_key=None, sock=None): + if self._cluster: + return self.execute( + b"PING", + shard_key=shard_key, + sock=sock + ) + return self.execute(b"PING") diff --git a/pyredis/commands/geo.py b/pyredis/commands/geo.py new file mode 100644 index 0000000..7ecc271 --- /dev/null +++ b/pyredis/commands/geo.py @@ -0,0 +1,66 @@ +from pyredis.commands.base import BaseCommand + + +class Geo(BaseCommand): + def __init__(self): + super().__init__() + + def geoadd(self, *args): + if self._cluster: + return self.execute( + *[b"GEOADD", *args], + shard_key=args[0] + ) + return self.execute( + *[b"GEOADD", *args] + ) + + def geodist(self, *args): + if self._cluster: + return self.execute( + *[b"GEODIST", *args], + shard_key=args[0] + ) + return self.execute( + *[b"GEODIST", *args] + ) + + def geohash(self, *args): + if self._cluster: + return self.execute( + *[b"GEOHASH", *args], + shard_key=args[0] + ) + return self.execute( + *[b"GEOHASH", *args] + ) + + def georadius(self, *args): + if self._cluster: + return self.execute( + *[b"GEORADIUS", *args], + shard_key=args[0] + ) + return self.execute( + *[b"GEORADIUS", *args] + ) + + def geopos(self, *args): + if self._cluster: + return self.execute( + *[b"GEOPOS", *args], + shard_key=args[0] + ) + return self.execute( + *[b"GEOPOS", *args] + ) + + def georadiusbymember(self, *args): + if self._cluster: + return self.execute( + *[b"GEORADIUSBYMEMBER", *args], + shard_key=args[0] + ) + return self.execute( + *[b"GEORADIUSBYMEMBER", *args] + ) diff --git a/pyredis/commands/hash.py b/pyredis/commands/hash.py new file mode 100644 index 0000000..43f2c03 --- /dev/null +++ b/pyredis/commands/hash.py @@ -0,0 +1,156 @@ +from pyredis.commands.base import BaseCommand + + +class Hash(BaseCommand): + def __init__(self): + super().__init__() + + def hdel(self, *args): + if self._cluster: + return self.execute( + *[b"HDEL", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HDEL", *args] + ) + + def hexists(self, *args): + if self._cluster: + return self.execute( + *[b"HEXISTS", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HEXISTS", *args] + ) + + def hget(self, *args): + if self._cluster: + return self.execute( + *[b"HGET", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HGET", *args] + ) + + def hgetall(self, *args): + if self._cluster: + return self.execute( + *[b"HGETALL", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HGETALL", *args] + ) + + def hincrby(self, *args): + if self._cluster: + return self.execute( + *[b"HINCRBY", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HINCRBY", *args] + ) + + def hincrbyfloat(self, *args): + if self._cluster: + return self.execute( + *[b"HINCRBYFLOAT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HINCRBYFLOAT", *args] + ) + + def hkeys(self, *args): + if self._cluster: + return self.execute( + *[b"HKEYS", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HKEYS", *args] + ) + + def hlen(self, *args): + if self._cluster: + return self.execute( + *[b"HLEN", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HLEN", *args] + ) + + def hmget(self, *args): + if self._cluster: + return self.execute( + *[b"HMGET", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HMGET", *args] + ) + + def hmset(self, *args): + if self._cluster: + return self.execute( + *[b"HMSET", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HMSET", *args] + ) + + def hset(self, *args): + if self._cluster: + return self.execute( + *[b"HSET", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HSET", *args] + ) + + def hsetnx(self, *args): + if self._cluster: + return self.execute( + *[b"HSETNX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HSETNX", *args] + ) + + def hstrlen(self, *args): + if self._cluster: + return self.execute( + *[b"HSTRLEN", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HSTRLEN", *args] + ) + + def hvals(self, *args): + if self._cluster: + return self.execute( + *[b"HVALS", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HVALS", *args] + ) + + def hscan(self, *args): + if self._cluster: + return self.execute( + *[b"HSCAN", *args], + shard_key=args[0] + ) + return self.execute( + *[b"HSCAN", *args] + ) diff --git a/pyredis/commands/hyperloglog.py b/pyredis/commands/hyperloglog.py new file mode 100644 index 0000000..3d941eb --- /dev/null +++ b/pyredis/commands/hyperloglog.py @@ -0,0 +1,36 @@ +from pyredis.commands.base import BaseCommand + + +class HyperLogLog(BaseCommand): + def __init__(self): + super().__init__() + + def pfadd(self, *args): + if self._cluster: + return self.execute( + *[b"PFADD", *args], + shard_key=args[0] + ) + return self.execute( + *[b"PFADD", *args] + ) + + def pfcount(self, *args): + if self._cluster: + return self.execute( + *[b"PFCOUNT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"PFCOUNT", *args] + ) + + def pfmerge(self, *args): + if self._cluster: + return self.execute( + *[b"PFMERGE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"PFMERGE", *args] + ) diff --git a/pyredis/commands/key.py b/pyredis/commands/key.py new file mode 100644 index 0000000..22caced --- /dev/null +++ b/pyredis/commands/key.py @@ -0,0 +1,224 @@ +from pyredis.commands.base import BaseCommand + + +class Key(BaseCommand): + def __init__(self): + super().__init__() + + def delete(self, *args): + if self._cluster: + return self.execute( + *[b"DEL", *args], + shard_key=args[0] + ) + return self.execute( + *[b"DEL", *args] + ) + + def dump(self, *args): + if self._cluster: + return self.execute( + *[b"DUMP", *args], + shard_key=args[0] + ) + return self.execute( + *[b"DUMP", *args] + ) + + def exists(self, *args): + if self._cluster: + return self.execute( + *[b"EXISTS", *args], + shard_key=args[0] + ) + return self.execute( + *[b"EXISTS", *args] + ) + + def expire(self, *args): + if self._cluster: + return self.execute( + *[b"EXPIRE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"EXPIRE", *args] + ) + + def expireat(self, *args): + if self._cluster: + return self.execute(b"EXPIREAT") + return self.execute( + *[b"EXPIREAT", *args] + ) + + def keys(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"KEYS", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"KEYS", *args] + ) + + def migrate(self, *args): + if self._cluster: + raise NotImplementedError + return self.execute( + *[b"MIGRATE", *args] + ) + + def move(self, *args): + if self._cluster: + return self.execute( + *[b"MOVE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"MOVE", *args] + ) + + def object(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"DEL", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"OBJECT", *args] + ) + + def persist(self, *args): + if self._cluster: + return self.execute( + *[b"PERSIST", *args], + shard_key=args[0] + ) + return self.execute( + *[b"PERSIST", *args] + ) + + def pexpire(self, *args): + if self._cluster: + return self.execute( + *[b"PEXPIRE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"PEXPIRE", *args] + ) + + def pexpireat(self, *args): + if self._cluster: + return self.execute( + *[b"PEXPIREAT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"PEXPIREAT", *args] + ) + + def pttl(self, *args): + if self._cluster: + return self.execute( + *[b"PTTL", *args], + shard_key=args[0] + ) + return self.execute( + *[b"PTTL", *args] + ) + + def randomkey(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"RANDOMKEY", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"RANDOMKEY", *args] + ) + + def rename(self, *args): + if self._cluster: + return self.execute( + *[b"RENAME", *args], + shard_key=args[0] + ) + return self.execute( + *[b"RENAME", *args] + ) + + def renamenx(self, *args): + if self._cluster: + return self.execute( + *[b"RENAMENX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"RENAMENX", *args] + ) + + def restore(self, *args): + if self._cluster: + return self.execute( + *[b"RESTORE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"RESTORE", *args] + ) + + def scan(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"SCAN", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"SCAN", *args] + ) + + def sort(self, *args): + if self._cluster: + return self.execute( + *[b"SORT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SORT", *args] + ) + + def ttl(self, *args): + if self._cluster: + return self.execute( + *[b"TTL", *args], + shard_key=args[0] + ) + return self.execute( + *[b"TTL", *args] + ) + + def type(self, *args): + if self._cluster: + return self.execute( + *[b"TYPE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"TYPE", *args] + ) + + def wait(self, *args): + if self._cluster: + return self.execute( + *[b"WAIT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"WAIT", *args] + ) diff --git a/pyredis/commands/list.py b/pyredis/commands/list.py new file mode 100644 index 0000000..f8da3d6 --- /dev/null +++ b/pyredis/commands/list.py @@ -0,0 +1,176 @@ +from pyredis.commands.base import BaseCommand + + +class List(BaseCommand): + def __init__(self): + super().__init__() + + def blpop(self, *args): + if self._cluster: + return self.execute( + *[b"BLPOP", *args], + shard_key=args[0] + ) + return self.execute( + *[b"BLPOP", *args] + ) + + def brpop(self, *args): + if self._cluster: + return self.execute( + *[b"BRPOP", *args], + shard_key=args[0] + ) + return self.execute( + *[b"BRPOP", *args] + ) + + def brpoplpush(self, *args): + if self._cluster: + return self.execute( + *[b"BRPOPPUSH", *args], + shard_key=args[0] + ) + return self.execute( + *[b"BRPOPPUSH", *args] + ) + + def lindex(self, *args): + if self._cluster: + return self.execute( + *[b"LINDEX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"LINDEX", *args] + ) + + def linsert(self, *args): + if self._cluster: + return self.execute( + *[b"LINSERT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"LINSERT", *args] + ) + + def llen(self, *args): + if self._cluster: + return self.execute( + *[b"LLEN", *args], + shard_key=args[0] + ) + return self.execute( + *[b"LLEN", *args] + ) + + def lpop(self, *args): + if self._cluster: + return self.execute( + *[b"LPOP", *args], + shard_key=args[0] + ) + return self.execute( + *[b"LPOP", *args] + ) + + def lpush(self, *args): + if self._cluster: + return self.execute( + *[b"LPUSH", *args], + shard_key=args[0] + ) + return self.execute( + *[b"LPUSH", *args] + ) + + def lpushx(self, *args): + if self._cluster: + return self.execute( + *[b"LPUSHX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"LPUSHX", *args] + ) + + def lrange(self, *args): + if self._cluster: + return self.execute( + *[b"LRANGE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"LRANGE", *args] + ) + + def lrem(self, *args): + if self._cluster: + return self.execute( + *[b"LREM", *args], + shard_key=args[0] + ) + return self.execute( + *[b"LREM", *args] + ) + + def lset(self, *args): + if self._cluster: + return self.execute( + *[b"LSET", *args], + shard_key=args[0] + ) + return self.execute( + *[b"LSET", *args] + ) + + def ltrim(self, *args): + if self._cluster: + return self.execute( + *[b"LTRIM", *args], + shard_key=args[0] + ) + return self.execute( + *[b"LTRIM", *args] + ) + + def rpop(self, *args): + if self._cluster: + return self.execute( + *[b"RPOP", *args], + shard_key=args[0] + ) + return self.execute( + *[b"RPOP", *args] + ) + + def rpoplpush(self, *args): + if self._cluster: + return self.execute( + *[b"RPOPLPUSH", *args], + shard_key=args[0] + ) + return self.execute( + *[b"RPOPLPUSH", *args] + ) + + def rpush(self, *args): + if self._cluster: + return self.execute( + *[b"RPUSH", *args], + shard_key=args[0] + ) + return self.execute( + *[b"RPUSH", *args] + ) + + def rpushx(self, *args): + if self._cluster: + return self.execute( + *[b"RPUSHX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"RPUSHX", *args] + ) diff --git a/pyredis/commands/publish.py b/pyredis/commands/publish.py new file mode 100644 index 0000000..793dcdd --- /dev/null +++ b/pyredis/commands/publish.py @@ -0,0 +1,13 @@ +from pyredis.commands.base import BaseCommand + + +class Publish(BaseCommand): + def __init__(self): + super().__init__() + + def publish(self, *args): + if self._cluster: + raise NotImplementedError + return self.execute( + *[b"PUBLISH", *args] + ) diff --git a/pyredis/commands/scripting.py b/pyredis/commands/scripting.py new file mode 100644 index 0000000..353ee14 --- /dev/null +++ b/pyredis/commands/scripting.py @@ -0,0 +1,83 @@ +from pyredis.commands.base import BaseCommand + + +class Scripting(BaseCommand): + def __init__(self): + super().__init__() + + def eval(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"EVAL", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"EVAL", *args] + ) + + def evalsha(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"EVALSHA", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"EVALSHA", *args] + ) + + def script_debug(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"SCRIPT", b"DEBUG", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"SCRIPT", b"DEBUG", *args] + ) + + def script_exists(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"SCRIPT", b"EXISTS", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"SCRIPT", b"EXISTS", *args] + ) + + def script_flush(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"SCRIPT", b"FLUSH", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"SCRIPT", b"FLUSH", *args] + ) + + def script_kill(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"SCRIPT", b"KILL", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"SCRIPT", b"KILL", *args] + ) + + def script_load(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"SCRIPT", b"LOAD", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"SCRIPT", b"LOAD", *args] + ) diff --git a/pyredis/commands/set.py b/pyredis/commands/set.py new file mode 100644 index 0000000..b58d779 --- /dev/null +++ b/pyredis/commands/set.py @@ -0,0 +1,156 @@ +from pyredis.commands.base import BaseCommand + + +class Set(BaseCommand): + def __init__(self): + super().__init__() + + def sadd(self, *args): + if self._cluster: + return self.execute( + *[b"SADD", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SADD", *args] + ) + + def scard(self, *args): + if self._cluster: + return self.execute( + *[b"SCARD", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SCARD", *args] + ) + + def sdiff(self, *args): + if self._cluster: + return self.execute( + *[b"SDIFF", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SDIFF", *args] + ) + + def sdiffstore(self, *args): + if self._cluster: + return self.execute( + *[b"SDIFFSTORE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SDIFFSTORE", *args] + ) + + def sinter(self, *args): + if self._cluster: + return self.execute( + *[b"SINTER", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SINTER", *args] + ) + + def sinterstore(self, *args): + if self._cluster: + return self.execute( + *[b"SINTERSTORE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SINTERSTORE", *args] + ) + + def sismember(self, *args): + if self._cluster: + return self.execute( + *[b"SISMEMBER", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SISMEMBER", *args] + ) + + def smembers(self, *args): + if self._cluster: + return self.execute( + *[b"SMEMBERS", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SMEMBERS", *args] + ) + + def smove(self, *args): + if self._cluster: + return self.execute( + *[b"SMOVE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SMOVE", *args] + ) + + def spop(self, *args): + if self._cluster: + return self.execute( + *[b"SPOP", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SPOP", *args] + ) + + def srandmember(self, *args): + if self._cluster: + return self.execute( + *[b"SRANDMEMBER", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SRANDMEMBER", *args] + ) + + def srem(self, *args): + if self._cluster: + return self.execute( + *[b"SREM", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SREM", *args] + ) + + def sunion(self, *args): + if self._cluster: + return self.execute( + *[b"SUNION", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SUNION", *args] + ) + + def sunoinstore(self, *args): + if self._cluster: + return self.execute( + *[b"SUNIONSTORE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SUNIONSTORE", *args] + ) + + def sscan(self, *args): + if self._cluster: + return self.execute( + *[b"SSCAN", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SSCAN", *args] + ) diff --git a/pyredis/commands/sset.py b/pyredis/commands/sset.py new file mode 100644 index 0000000..25d2708 --- /dev/null +++ b/pyredis/commands/sset.py @@ -0,0 +1,216 @@ +from pyredis.commands.base import BaseCommand + + +class SSet(BaseCommand): + def __init__(self): + super().__init__() + + def zadd(self, *args): + if self._cluster: + return self.execute( + *[b"ZADD", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZADD", *args] + ) + + def zcard(self, *args): + if self._cluster: + return self.execute( + *[b"ZCARD", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZCARD", *args] + ) + + def zcount(self, *args): + if self._cluster: + return self.execute( + *[b"ZCOUNT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZCOUNT", *args] + ) + + def zincrby(self, *args): + if self._cluster: + return self.execute( + *[b"ZINCRBY", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZINCRBY", *args] + ) + + def zinterstore(self, *args): + if self._cluster: + return self.execute( + *[b"ZINTERSTORE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZINTERSTORE", *args] + ) + + def zlexcount(self, *args): + if self._cluster: + return self.execute( + *[b"ZLEXCOUNT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZLEXCOUNT", *args] + ) + + def zrange(self, *args): + if self._cluster: + return self.execute( + *[b"ZRANGE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZRANGE", *args] + ) + + def zrangebylex(self, *args): + if self._cluster: + return self.execute( + *[b"ZRANGEBYLEX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZRANGEBYLEX", *args] + ) + + def zrangebyscore(self, *args): + if self._cluster: + return self.execute( + *[b"ZRANGEBYSCORE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZRANGEBYSCORE", *args] + ) + + def zrank(self, *args): + if self._cluster: + return self.execute( + *[b"ZRANK", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZRANK", *args] + ) + + def zrem(self, *args): + if self._cluster: + return self.execute( + *[b"ZREM", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZREM", *args] + ) + + def zremrangebylex(self, *args): + if self._cluster: + return self.execute( + *[b"ZREMRANGEBYLEX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZREMRANGEBYLEX", *args] + ) + + def zremrangebyrank(self, *args): + if self._cluster: + return self.execute( + *[b"ZREMRANGEBYRANK", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZREMRANGEBYRANK", *args] + ) + + def zremrangebyscrore(self, *args): + if self._cluster: + return self.execute( + *[b"ZREMRANGEBYSCORE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZREMRANGEBYSCORE", *args] + ) + + def zrevrange(self, *args): + if self._cluster: + return self.execute( + *[b"ZREVRANGE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZREVRANGE", *args] + ) + + def zrevrangebylex(self, *args): + if self._cluster: + return self.execute( + *[b"ZREVRANGEBYLEX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZREVRANGEBYLEX", *args] + ) + + def zrevrangebyscore(self, *args): + if self._cluster: + return self.execute( + *[b"ZREVRANGEBYSCORE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZREVRANGEBYSCORE", *args] + ) + + def zrevrank(self, *args): + if self._cluster: + return self.execute( + *[b"ZREVRANK", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZREVRANK", *args] + ) + + def zscore(self, *args): + if self._cluster: + return self.execute( + *[b"ZSCORE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZSCORE", *args] + ) + + def zunionstore(self, *args): + if self._cluster: + return self.execute( + *[b"ZUNIONSTORE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZUNIONSTORE", *args] + ) + + def zscan(self, *args): + if self._cluster: + return self.execute( + *[b"ZSCAN", *args], + shard_key=args[0] + ) + return self.execute( + *[b"ZSCAN", *args] + ) diff --git a/pyredis/commands/string.py b/pyredis/commands/string.py new file mode 100644 index 0000000..767c22d --- /dev/null +++ b/pyredis/commands/string.py @@ -0,0 +1,246 @@ +from pyredis.commands.base import BaseCommand + + +class String(BaseCommand): + def __init__(self): + super().__init__() + + def append(self, *args): + if self._cluster: + return self.execute( + *[b"APPEND", *args], + shard_key=args[0] + ) + return self.execute( + *[b"APPEND", *args] + ) + + def bitcount(self, *args): + if self._cluster: + return self.execute( + *[b"BITCOUNT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"BITCOUNT", *args] + ) + + def bitfield(self, *args): + if self._cluster: + return self.execute( + *[b"BITFIELD", *args], + shard_key=args[0] + ) + return self.execute( + *[b"BITFIELD", *args] + ) + + def bitop(self, *args): + if self._cluster: + return self.execute( + *[b"BITOP", *args], + shard_key=args[1] + ) + return self.execute( + *[b"BITOP", *args] + ) + + def bitpos(self, *args): + if self._cluster: + return self.execute( + *[b"BITPOS", *args], + shard_key=args[0] + ) + return self.execute( + *[b"BITPOS", *args] + ) + + def decr(self, *args): + if self._cluster: + return self.execute( + *[b"DECR", *args], + shard_key=args[0] + ) + return self.execute( + *[b"DECR", *args] + ) + + def decrby(self, *args): + if self._cluster: + return self.execute( + *[b"DECRBY", *args], + shard_key=args[0] + ) + return self.execute( + *[b"DECRBY", *args] + ) + + def get(self, *args): + if self._cluster: + return self.execute( + *[b"GET", *args], + shard_key=args[0] + ) + return self.execute( + *[b"GET", *args] + ) + + def getbit(self, *args): + if self._cluster: + return self.execute( + *[b"GETBIT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"GETBIT", *args] + ) + + def getrange(self, *args): + if self._cluster: + return self.execute( + *[b"GETRANGE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"GETRANGE", *args] + ) + + def getset(self, *args): + if self._cluster: + return self.execute( + *[b"GETSET", *args], + shard_key=args[0] + ) + return self.execute( + *[b"GETSET", *args] + ) + + def incr(self, *args): + if self._cluster: + return self.execute( + *[b"INCR", *args], + shard_key=args[0] + ) + return self.execute( + *[b"INCR", *args] + ) + + def incrby(self, *args): + if self._cluster: + return self.execute( + *[b"INCRBY", *args], + shard_key=args[0] + ) + return self.execute( + *[b"INCRBY", *args] + ) + + def incrbyfloat(self, *args): + if self._cluster: + return self.execute( + *[b"INCRBYFLOAT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"INCRBYFLOAT", *args] + ) + + def mget(self, *args): + if self._cluster: + return self.execute( + *[b"MGET", *args], + shard_key=args[0] + ) + return self.execute( + *[b"MGET", *args] + ) + + def mset(self, *args): + if self._cluster: + return self.execute( + *[b"MSET", *args], + shard_key=args[0] + ) + return self.execute( + *[b"MSET", *args] + ) + + def msetnx(self, *args): + if self._cluster: + return self.execute( + *[b"MSETNX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"MSETNX", *args] + ) + + def psetex(self, *args): + if self._cluster: + return self.execute( + *[b"PSETEX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"PSETEX", *args] + ) + + def set(self, *args): + if self._cluster: + return self.execute( + *[b"SET", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SET", *args] + ) + + def setbit(self, *args): + if self._cluster: + return self.execute( + *[b"SETBIT", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SETBIT", *args] + ) + + def setex(self, *args): + if self._cluster: + return self.execute( + *[b"SETEX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SETEX", *args] + ) + + def setnx(self, *args): + if self._cluster: + return self.execute( + *[b"SETNX", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SETNX", *args] + ) + + def setrange(self, *args): + if self._cluster: + return self.execute( + *[b"SETRANGE", *args], + shard_key=args[0] + ) + return self.execute( + *[b"SETRANGE", *args] + ) + + def strlen(self, *args): + if self._cluster: + return self.execute( + *[b"STRLEN", *args], + shard_key=args[0] + ) + return self.execute( + *[b"STRLEN", *args] + ) diff --git a/pyredis/commands/subscribe.py b/pyredis/commands/subscribe.py new file mode 100644 index 0000000..2d79ecf --- /dev/null +++ b/pyredis/commands/subscribe.py @@ -0,0 +1,23 @@ +class Subscribe(object): + def write(self, *args): + raise NotImplementedError + + def psubscribe(self, *args): + return self.write( + *[b"PSUBSCRIBE", *args] + ) + + def punsubscribe(self, *args): + return self.write( + *[b"PUNSUBSCRIBE", *args] + ) + + def subscribe(self, *args): + return self.write( + *[b"SUBSCRIBE", *args] + ) + + def unsubscribe(self, *args): + return self.write( + *[b"UNSUBSCRIBE", *args] + ) diff --git a/pyredis/commands/transaction.py b/pyredis/commands/transaction.py new file mode 100644 index 0000000..c90a5d3 --- /dev/null +++ b/pyredis/commands/transaction.py @@ -0,0 +1,60 @@ +from pyredis.commands.base import BaseCommand + + +class Transaction(BaseCommand): + def __init__(self): + super().__init__() + + def discard(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"DISCARD", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"DISCARD", *args] + ) + + def exec(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"EXEC", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"EXEC", *args] + ) + + def multi(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"MULTI", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"MULTI", *args] + ) + + def unwatch(self, *args, shard_key=None, sock=None): + if self._cluster: + return self.execute( + *[b"UNWATCH", *args], + shard_key=shard_key, + sock=sock + ) + return self.execute( + *[b"UNWATCH", *args] + ) + + def watch(self, *args): + if self._cluster: + return self.execute( + *[b"WATCH", *args], + shard_key=args[0] + ) + return self.execute( + *[b"WATCH", *args] + ) diff --git a/pyredis/connection/__init__.py b/pyredis/connection/__init__.py new file mode 100644 index 0000000..40837ef --- /dev/null +++ b/pyredis/connection/__init__.py @@ -0,0 +1,19 @@ +import socket + +from pyredis.protocol import writer + +try: + from hiredis import Reader +except ImportError: + from pyredis.protocol import Reader + +from pyredis.connection.connection import Connection +from pyredis.connection.async_connection import AsyncConnection + +__all__ = [ + "Connection", + "AsyncConnection", + "socket", + "writer", + "Reader", +] diff --git a/pyredis/connection/async_connection.py b/pyredis/connection/async_connection.py new file mode 100644 index 0000000..27dc907 --- /dev/null +++ b/pyredis/connection/async_connection.py @@ -0,0 +1,186 @@ +import asyncio +from pyredis.exceptions import PyRedisConnClosed +from pyredis.exceptions import PyRedisConnError +from pyredis.exceptions import PyRedisConnReadTimeout +from pyredis.exceptions import PyRedisError +from pyredis.exceptions import ReplyError +import pyredis.connection + + +class AsyncConnection(object): + def __init__( + self, + host=None, + port=6379, + unix_sock=None, + database=None, + password=None, + encoding=None, + conn_timeout=2, + read_only=False, + read_timeout=2, + sentinel=False, + username=None, + ): + if not bool(host) != bool(unix_sock): + raise PyRedisError("Ether host or unix_sock has to be provided") + self._closed = False + self._conn_timeout = conn_timeout + self._read_only = read_only + self._read_timeout = read_timeout + self._encoding = encoding + self._reader_parser = None + self._sentinel = sentinel + self._writer_func = pyredis.connection.writer + self._reader = None + self._writer = None + self.host = host + self.port = port + self.unix_sock = unix_sock + self.password = password + self.username = username + self.database = database + + async def _authenticate(self): + if self.username and self.password: + await self.write( + *["AUTH", self.username, self.password] + ) + try: + await self.read() + except ReplyError as err: + await self.close() + raise err + elif self.password: + await self.write( + *["AUTH", self.password] + ) + try: + await self.read() + except ReplyError as err: + await self.close() + raise err + + async def _connect(self): + if self._closed: + raise PyRedisConnError("Connection Gone") + try: + if self.host: + reader, writer = await asyncio.wait_for( + asyncio.open_connection( + host=self.host, + port=self.port + ), + timeout=self._conn_timeout + ) + else: + reader, writer = await asyncio.wait_for( + asyncio.open_unix_connection( + path=self.unix_sock + ), + timeout=self._conn_timeout + ) + except ( + ConnectionAbortedError, + ConnectionRefusedError, + OverflowError, + asyncio.TimeoutError, + OSError, + ) as err: + await self.close() + raise PyRedisConnError( + f"Could not Connect to {self.host}:{self.port}: {err}" + ) + self._reader = reader + self._writer = writer + if self._encoding: + self._reader_parser = pyredis.connection.Reader( + encoding=self._encoding + ) + else: + self._reader_parser = pyredis.connection.Reader() + await self._authenticate() + if not self._sentinel: + await self._setdb() + await self._set_read_only() + + async def _setdb(self): + if self._sentinel: + return + if self.database is None: + return + await self.write( + *["SELECT", self.database] + ) + try: + await self.read() + except ReplyError as err: + await self.close() + raise err + + async def _set_read_only(self): + if self._read_only: + await self.write("READONLY") + try: + await self.read() + except ReplyError as err: + await self.close() + raise err + + async def close(self): + if self._writer: + self._writer.close() + try: + await self._writer.wait_closed() + except Exception: + pass + self._reader = None + self._writer = None + self._reader_parser = None + self._closed = True + + @property + def closed(self): + return self._closed + + async def read(self, close_on_timeout=True, raise_on_result_err=True): + if not self._writer: + await self._connect() + while True: + result = self._reader_parser.gets() + if result is not False: + if raise_on_result_err: + if isinstance(result, Exception): + raise result + return result + try: + data = await asyncio.wait_for( + self._reader.read(1500), + timeout=self._read_timeout + ) + except asyncio.TimeoutError: + if close_on_timeout: + await self.close() + raise PyRedisConnReadTimeout( + "Connection timeout while reading" + ) + except ConnectionResetError: + await self.close() + raise PyRedisConnError("Connection reset by peer") + if not data: + await self.close() + raise PyRedisConnClosed("Connection went away while reading") + self._reader_parser.feed(data) + + async def write(self, *args): + if not self._writer: + await self._connect() + data = self._writer_func(*args) + try: + self._writer.write(data) + await self._writer.drain() + except BrokenPipeError as err: + await self.close() + raise PyRedisConnError( + f"Connection lost while writing: {err}" + ) diff --git a/pyredis/connection.py b/pyredis/connection/connection.py similarity index 57% rename from pyredis/connection.py rename to pyredis/connection/connection.py index 0117d18..97592f2 100644 --- a/pyredis/connection.py +++ b/pyredis/connection/connection.py @@ -1,65 +1,13 @@ -from pyredis.exceptions import * -from pyredis.protocol import writer -import socket - -try: - from hiredis import Reader -except ImportError: - from pyredis.protocol import Reader - -__all__ = ["Connection"] +import pyredis.connection +from pyredis.exceptions import PyRedisConnClosed +from pyredis.exceptions import PyRedisConnError +from pyredis.exceptions import PyRedisConnReadTimeout +from pyredis.exceptions import PyRedisError +from pyredis.exceptions import ReplyError class Connection(object): - """Low level client for talking to a Redis Server. - - This class is should not be used directly to talk to a Redis server, - unless you know what you are doing. In most cases it should be - sufficient to use one of the Client classes, or one of the Connection Pools. - - :param host: - Host IP or Name to connect, - can only be set when unix_sock is None. - :type host: str - - :param port: - Port to connect, only used when host is also set. - :type port: int - - :param unix_sock: - Unix Socket to connect, - can only be set when host is None. - :type unix_sock: str - - :param database: - Select which db should be used for this connection, defaults to None, so we use the default database 0 - :type database: int - - :param password: - Password used for authentication. If None, no authentication is done - :type password: str - - :param encoding: - Convert result strings with this encoding. If None, no encoding is done. - :type encoding: str - - :param conn_timeout: - Connect Timeout. - :type conn_timeout: float - - :param read_timeout: - Read Timeout. - :type read_timeout: float - - :param sentinel: - If True, authentication and database selection is skipped. - :type sentinel: bool - - :param username: - Username used for acl scl authentication. If not set, fall back use legacy auth. - :type username: str - - """ + """Low level client for talking to a Redis Server.""" def __init__( self, @@ -84,7 +32,7 @@ def __init__( self._encoding = encoding self._reader = None self._sentinel = sentinel - self._writer = writer + self._writer = pyredis.connection.writer self._sock = None self.host = host self.port = port @@ -95,14 +43,18 @@ def __init__( def _authenticate(self): if self.username and self.password: - self.write("AUTH", self.username, self.password) + self.write( + *["AUTH", self.username, self.password] + ) try: self.read() except ReplyError as err: self.close() raise err elif self.password: - self.write("AUTH", self.password) + self.write( + *["AUTH", self.password] + ) try: self.read() except ReplyError as err: @@ -118,9 +70,9 @@ def _connect(self): sock = self._connect_unix() self._sock = sock if self._encoding: - self._reader = Reader(encoding=self._encoding) + self._reader = pyredis.connection.Reader(encoding=self._encoding) else: - self._reader = Reader() + self._reader = pyredis.connection.Reader() self._authenticate() if not self._sentinel: self._setdb() @@ -129,7 +81,7 @@ def _connect(self): def _connect_inet46(self): try: - sock = socket.create_connection( + sock = pyredis.connection.socket.create_connection( address=(self.host, self.port), timeout=self._conn_timeout ) @@ -137,35 +89,33 @@ def _connect_inet46(self): ConnectionAbortedError, ConnectionRefusedError, OverflowError, - socket.timeout, + pyredis.connection.socket.timeout, OSError, ) as err: self.close() raise PyRedisConnError( - "Could not Connect to {0}:{1}: {2}".format( - self.host, - self.port, - err - ) + f"Could not Connect to {self.host}:{self.port}: {err}" ) return sock - def _connect_unix(self): try: - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock = pyredis.connection.socket.socket( + family=pyredis.connection.socket.AF_UNIX, + type=pyredis.connection.socket.SOCK_STREAM, + ) sock.settimeout(self._conn_timeout) sock.connect(self.unix_sock) except ( ConnectionAbortedError, ConnectionRefusedError, FileNotFoundError, - socket.timeout, + pyredis.connection.socket.timeout, OSError, ) as err: self.close() raise PyRedisConnError( - "Could not Connect to {0}: {1}".format(self.host, err) + f"Could not Connect to {self.host}: {err}" ) return sock @@ -175,7 +125,9 @@ def _setdb(self): if self.database is None: return self._sock.settimeout(0.1) - self.write("SELECT", self.database) + self.write( + *["SELECT", self.database] + ) try: self.read() except ReplyError as err: @@ -192,12 +144,6 @@ def _set_read_only(self): raise err def close(self): - """Close Client Connection. - - This closes the underlying socket, and mark the connection as closed. - - :return: None - """ if self._sock: self._sock.close() self._sock = None @@ -209,18 +155,6 @@ def closed(self): return self._closed def read(self, close_on_timeout=True, raise_on_result_err=True): - """Read result from the socket. - - :param close_on_timeout: - Close the connection after a read timeout - :type close_on_timeout: book - - :param raise_on_result_err: - Raise exception on protocol errors - :type raise_on_result_err: bool - - :return: result, exception - """ if not self._sock: self._connect() while True: @@ -232,10 +166,12 @@ def read(self, close_on_timeout=True, raise_on_result_err=True): return result try: data = self._sock.recv(1500) - except socket.timeout: + except pyredis.connection.socket.timeout: if close_on_timeout: self.close() - raise PyRedisConnReadTimeout("Connection timeout while reading") + raise PyRedisConnReadTimeout( + "Connection timeout while reading" + ) except ConnectionResetError: self.close() raise PyRedisConnError("Connection reset by peer") @@ -245,14 +181,6 @@ def read(self, close_on_timeout=True, raise_on_result_err=True): self._reader.feed(data) def write(self, *args): - """Write commands to socket. - - :param args: - Accepts a variable number of arguments - :type args: str, int, float - - :return: None - """ if not self._sock: self._connect() data = self._writer(*args) @@ -260,4 +188,6 @@ def write(self, *args): self._sock.sendall(data) except BrokenPipeError as err: self.close() - raise PyRedisConnError("Connection lost while writing: {0}".format(err)) + raise PyRedisConnError( + f"Connection lost while writing: {err}" + ) diff --git a/pyredis/helper.py b/pyredis/helper.py index dd1e170..0529078 100644 --- a/pyredis/helper.py +++ b/pyredis/helper.py @@ -9,8 +9,6 @@ from pyredis.protocol import to_bytes - - def dict_from_list(source): return dict(zip(*[iter(source)] * 2)) @@ -32,7 +30,7 @@ def tag_from_key(key): if not (lcb >= 0 and rcb >= 0): return key else: - return key[lcb + 1 : rcb] + return key[lcb + 1: rcb] def slot_from_key(key): @@ -82,7 +80,8 @@ def _fetch_map(self): finally: conn.close() raise PyRedisError( - "Could not get cluster info from any seed node: {0}".format(self._seeds) + "Could not get cluster info from any seed node: " + "{0}".format(self._seeds) ) def _update_slot(self, slot, master, slaves): diff --git a/pyredis/pool.py b/pyredis/pool.py deleted file mode 100644 index dd6799c..0000000 --- a/pyredis/pool.py +++ /dev/null @@ -1,709 +0,0 @@ -from random import shuffle -import threading -from pyredis import commands -from pyredis.client import Client, ClusterClient, HashClient, SentinelClient -from pyredis.exceptions import * -from pyredis.helper import ClusterMap - - -class BasePool(object): - """Base Class for all other pools. - - All other pools inherit from this base class. - This class itself, cannot be used directly. - - :param database: - Select which db should be used for this pool - :type database: int - - :param password: - Password used for authentication. If None, no authentication is done - :type password: str - - :param encoding: - Convert result strings with this encoding. If None, no encoding is done. - :type encoding: str - - :param conn_timeout: - Connect Timeout. - :type conn_timeout: float - - :param read_timeout: - Read Timeout. - :type read_timeout: float - - :param pool_size: - Upper limit of connections this pool can handle. - :type pool_size: int - - :param lock: - Class implementing a Lock. - :type lock: _lock object, defaults to threading.Lock - - :param username: - Username used for acl scl authentication. If not set, fall back use legacy auth. - :type username: str - """ - - def __init__( - self, - database=0, - password=None, - encoding=None, - conn_timeout=2, - read_timeout=2, - pool_size=16, - lock=threading.Lock(), - username=None, - ): - self._conn_timeout = conn_timeout - self._read_timeout = read_timeout - self._lock = lock - self._pool_free = set() - self._pool_used = set() - self._database = database - self._password = password - self._encoding = encoding - self._pool_size = pool_size - self._close_on_err = False - self._cluster = False - self._username = username - - @property - def conn_timeout(self): - """Return configured connection timeout - - :return: float - """ - return self._conn_timeout - - @property - def read_timeout(self): - """Return configured read timeout - - :return: float - """ - return self._read_timeout - - @property - def database(self): - """Return configured database. - - :return: int - """ - return self._database - - @property - def password(self): - """Return configured password for this pool. - - :return: str, None - """ - return self._password - - @property - def encoding(self): - """Return configured encoding - - :return: str, None - """ - return self._encoding - - @property - def pool_size(self): - """Return, or adjust the current pool size. - - shrinking is implemented via closing unused connections. - if there not enough unused connections to fulfil the shrink request, - connections returned via pool.release are closed. - - :return: int, None - """ - return self._pool_size - - @pool_size.setter - def pool_size(self, size): - try: - self._lock.acquire() - self._pool_size = size - current_size = len(self._pool_free) + len(self._pool_used) - while current_size > size: - try: - client = self._pool_free.pop() - client.close() - current_size -= 1 - except KeyError: - break - finally: - self._lock.release() - - @property - def close_on_err(self): - return self._close_on_err - - @property - def username(self): - return self._username - - def _connect(self): - raise NotImplemented - - def acquire(self): - """Acquire a client connection from the pool. - - :return: redis.Client, exception - """ - try: - self._lock.acquire() - client = self._pool_free.pop() - self._pool_used.add(client) - except KeyError: - if len(self._pool_used) < self.pool_size: - client = self._connect() - self._pool_used.add(client) - else: - raise PyRedisError( - "Max connections {0} exhausted".format(self.pool_size) - ) - finally: - self._lock.release() - return client - - def release(self, conn): - """Return a client connection to the pool. - - :param conn: - redis.Client instance, managed by this pool. - :return: None - """ - try: - self._lock.acquire() - current_size = len(self._pool_free) + len(self._pool_used) - self._pool_used.remove(conn) - if conn.closed and self.close_on_err: - for conn in self._pool_free: - conn.close() - self._pool_free = set() - self._pool_used = set() - elif not conn.closed: - if current_size > self.pool_size: - conn.close() - else: - self._pool_free.add(conn) - except KeyError: - conn.close() - finally: - self._lock.release() - - def execute(self, *args, **kwargs): - conn = self.acquire() - try: - return conn.execute( - *args, - **kwargs - ) - finally: - self.release(conn) - - -class ClusterPool( - BasePool, - commands.Connection, - commands.Geo, - commands.Hash, - commands.HyperLogLog, - commands.Key, - commands.List, - commands.Scripting, - commands.Set, - commands.SSet, - commands.String, -): - """Redis Cluster Pool. - - Inherits all the arguments, methods and attributes from BasePool. - - :param seeds: - Accepts a list of seed nodes in this form: [('host1', 6379), ('host2', 6379), ('host3', 6379)] - :type sentinels: list - - :param slave_ok: - Defaults to False. If True, this pool will return connections to slave instances. - :type slave_ok: bool - - :param retries: - In case there is a chunk move ongoing, while executing a command, how many times should - we try to find the right node, before giving up. - :type retries: int - """ - - def __init__(self, seeds, slave_ok=False, password=None, username=None, **kwargs): - super().__init__(password=password, **kwargs) - self._map = ClusterMap(seeds=seeds, password=password, username=username) - self._slave_ok = slave_ok - self._cluster = True - - @property - def slave_ok(self): - """True if this pool will return slave connections - - :return: bool - """ - return self._slave_ok - - def _connect(self): - return ClusterClient( - database=self.database, - password=self.password, - encoding=self.encoding, - slave_ok=self.slave_ok, - conn_timeout=self.conn_timeout, - read_timeout=self.read_timeout, - cluster_map=self._map, - username=self.username, - ) - - -class HashPool( - BasePool, - commands.Connection, - commands.Geo, - commands.Hash, - commands.HyperLogLog, - commands.Key, - commands.List, - commands.Publish, - commands.Scripting, - commands.Set, - commands.SSet, - commands.String, -): - """Pool for straight connections to Redis - - Inherits all the arguments, methods and attributes from BasePool. - - The Client will calculate a crc16 hash using the shard_key, - which is be default the first Key in case the command supports multiple keys. - If the Key is using the TAG annotation "bla{tag}blarg", - then only the tag portion is used, in this case "tag". - The key space is split into 16384 buckets, so in theory you could provide - a list with 16384 ('host', port) pairs to the "buckets" parameter. - If you have less then 16384 ('host', port) pairs, the client will try to - distribute the key spaces evenly between available pairs. - - --- Warning --- - Since this is static hashing, the the order of pairs has to match on each client you use! - Also changing the number of pairs will change the mapping between buckets and pairs, - rendering your data inaccessible! - - - :param buckets: - list of ('host', port) pairs, where each pair represents a bucket - example: [('localhost', 7001), ('localhost', 7002), ('localhost', 7003)] - :type port: list - """ - - def __init__(self, buckets, **kwargs): - super().__init__(**kwargs) - self._buckets = buckets - self._cluster = True - - @property - def buckets(self): - """Return configured buckets. - - :return: list - """ - return self._buckets - - def _connect(self): - return HashClient( - buckets=self.buckets, - database=self.database, - password=self.password, - encoding=self.encoding, - conn_timeout=self.conn_timeout, - read_timeout=self.read_timeout, - username=self.username, - ) - - -class Pool( - BasePool, - commands.Connection, - commands.Geo, - commands.Hash, - commands.HyperLogLog, - commands.Key, - commands.List, - commands.Publish, - commands.Scripting, - commands.Set, - commands.SSet, - commands.String, -): - """Pool for straight connections to Redis - - Inherits all the arguments, methods and attributes from BasePool. - - :param host: - Host IP or Name to connect, - can only be set when unix_sock is None. - :type host: str - - :param port: - Port to connect, only used when host is also set. - :type port: int - - :param unix_sock: - Unix Socket to connect, - can only be set when host is None. - :type unix_sock: str - - """ - - def __init__(self, host=None, port=6379, unix_sock=None, **kwargs): - if not bool(host) != bool(unix_sock): - raise PyRedisError("Ether host or unix_sock has to be provided") - super().__init__(**kwargs) - self._host = host - self._port = port - self._unix_sock = unix_sock - - @property - def host(self): - """Return configured host. - - :return: str, None - """ - return self._host - - @property - def port(self): - """Return configured port. - - :return: int - """ - return self._port - - @property - def unix_sock(self): - """Return configured Unix socket. - - :return: str, None - """ - return self._unix_sock - - def _connect(self): - return Client( - host=self.host, - port=self.port, - unix_sock=self.unix_sock, - database=self.database, - password=self.password, - encoding=self.encoding, - conn_timeout=self.conn_timeout, - read_timeout=self.read_timeout, - username=self.username, - ) - - -class SentinelHashPool( - BasePool, - commands.Connection, - commands.Geo, - commands.Hash, - commands.HyperLogLog, - commands.Key, - commands.List, - commands.Publish, - commands.Scripting, - commands.Set, - commands.SSet, - commands.String, -): - """Sentinel backed Pool. - - Inherits all the arguments, methods and attributes from BasePool. - - :param sentinels: - Accepts a list of sentinels in this form: [('sentinel1', 26379), ('sentinel2', 26379), ('sentinel3', 26379)] - :type sentinels: list - - :param buckets: - list of Sentinel managed replications sets which make up this HashPool - :type name: list - - :param slave_ok: - Defaults to False. If True, this pool will return connections to slave instances. - :type slave_ok: bool - - :param retries: - In case a sentinel delivers stale data, how many other sentinels should be tried. - :type retries: int - - :param sentinel_password: - Password used for authentication of Sentinel instance itself. If None, no authentication is done. - Only available starting with Redis 5.0.1. - :type sentinel_password: str - - :param sentinel_username: - Username used for acl style authentication of Sentinel instance itself. If None, no authentication is done. - Only available starting with Redis 5.0.1. - :type sentinel_username: str - """ - - def __init__( - self, - sentinels, - buckets, - slave_ok=False, - retries=3, - sentinel_password=None, - sentinel_username=None, - **kwargs - ): - super().__init__(**kwargs) - self._sentinel = SentinelClient( - sentinels=sentinels, password=sentinel_password, username=sentinel_username - ) - self._buckets = buckets - self._slave_ok = slave_ok - self._retries = retries - self._close_on_err = True - self._cluster = True - - @property - def slave_ok(self): - """True if this pool return slave connections - - :return: bool - """ - return self._slave_ok - - @property - def buckets(self): - """Name of the configured Sentinel managed cluster. - - :return: str - """ - return self._buckets - - @property - def retries(self): - """Number of retries in case of stale sentinel. - - :return: int - """ - return self._retries - - @property - def sentinels(self): - """Deque with configured sentinels. - - :return: deque - """ - return self._sentinel.sentinels - - def _connect(self): - if self.slave_ok: - client = self._get_slaves() - else: - client = self._get_masters() - if client: - return client - - def _get_hash_client(self, buckets): - return HashClient( - buckets=buckets, - database=self.database, - password=self.password, - encoding=self.encoding, - conn_timeout=self.conn_timeout, - read_timeout=self.read_timeout, - username=self.username, - ) - - def _get_master(self, bucket): - candidate = self._sentinel.get_master(bucket) - host = candidate[b"ip"].decode("utf8") - port = int(candidate[b"port"]) - return host, port - - def _get_masters(self): - buckets = list() - for bucket in self.buckets: - _counter = self.retries - while _counter >= 0: - _counter -= 1 - _bucket = self._get_master(bucket) - if _bucket: - buckets.append(_bucket) - break - if _counter == 0: - raise PyRedisConnError( - "Could not connect to bucket {0}".format(bucket) - ) - return self._get_hash_client(buckets=buckets) - - def _get_slave(self, bucket): - candidates = [] - for candidate in self._sentinel.get_slaves(bucket): - candidates.append((candidate[b"ip"], int(candidate[b"port"]))) - shuffle(candidates) - host = candidates[0][0].decode("utf8") - port = int(candidates[0][1]) - return host, port - - def _get_slaves(self): - buckets = list() - for bucket in self.buckets: - _counter = self.retries - while _counter >= 0: - _counter -= 1 - _bucket = self._get_slave(bucket) - if _bucket: - buckets.append(_bucket) - break - if _counter == 0: - raise PyRedisConnError( - "Could not connect to bucket {0}".format(bucket) - ) - return self._get_hash_client(buckets=buckets) - - -class SentinelPool( - BasePool, - commands.Connection, - commands.Geo, - commands.Hash, - commands.HyperLogLog, - commands.Key, - commands.List, - commands.Publish, - commands.Scripting, - commands.Set, - commands.SSet, - commands.String, -): - """Sentinel backed Pool. - - Inherits all the arguments, methods and attributes from BasePool. - - :param sentinels: - Accepts a list of sentinels in this form: [('sentinel1', 26379), ('sentinel2', 26379), ('sentinel3', 26379)] - :type sentinels: list - - :param name: - Name of the cluster managed by sentinel, that this pool should manage. - :type name: str - - :param slave_ok: - Defaults to False. If True, this pool will return connections to slave instances. - :type slave_ok: bool - - :param retries: - In case a sentinel delivers stale data, how many other sentinels should be tried. - :type retries: int - - :param sentinel_password: - Password used for authentication of Sentinel instance itself. If None, no authentication is done. - Only available starting with Redis 5.0.1. - :type sentinel_password: str - - :param sentinel_username: - Username used for acl style authentication of Sentinel instance itself. If None, no authentication is done. - Only available starting with Redis 5.0.1. - :type sentinel_username: str - """ - - def __init__( - self, - sentinels, - name, - slave_ok=False, - retries=3, - sentinel_password=None, - sentinel_username=None, - **kwargs - ): - super().__init__(**kwargs) - self._sentinel = SentinelClient( - sentinels=sentinels, password=sentinel_password, username=sentinel_username - ) - self._name = name - self._slave_ok = slave_ok - self._retries = retries - self._close_on_err = True - - @property - def slave_ok(self): - """True if this pool return slave connections - - :return: bool - """ - return self._slave_ok - - @property - def name(self): - """Name of the configured Sentinel managed cluster. - - :return: str - """ - return self._name - - @property - def retries(self): - """Number of retries in case of stale sentinel. - - :return: int - """ - return self._retries - - @property - def sentinels(self): - """Deque with configured sentinels. - - :return: deque - """ - return self._sentinel.sentinels - - def _connect(self): - for _ in range(self.retries): - if self.slave_ok: - client = self._get_slave() - else: - client = self._get_master() - if client: - return client - raise PyRedisConnError("Could not connect to Redis") - - def _get_client(self, host, port): - return Client( - host=host, - port=port, - database=self.database, - password=self.password, - encoding=self.encoding, - conn_timeout=self.conn_timeout, - read_timeout=self.read_timeout, - username=self.username, - ) - - def _get_master(self): - candidate = self._sentinel.get_master(self.name) - host = candidate[b"ip"] - port = int(candidate[b"port"]) - client = self._get_client(host, port) - return client - - def _get_slave(self): - candidates = [] - for candidate in self._sentinel.get_slaves(self.name): - candidates.append((candidate[b"ip"], int(candidate[b"port"]))) - shuffle(candidates) - host = candidates[0][0] - port = int(candidates[0][1]) - client = self._get_client(host, port) - return client - diff --git a/pyredis/pool/__init__.py b/pyredis/pool/__init__.py new file mode 100644 index 0000000..bc26c89 --- /dev/null +++ b/pyredis/pool/__init__.py @@ -0,0 +1,49 @@ +from random import shuffle +from pyredis.client import Client +from pyredis.client import AsyncClient +from pyredis.client import ClusterClient +from pyredis.client import AsyncClusterClient +from pyredis.client import HashClient +from pyredis.client import AsyncHashClient +from pyredis.client import SentinelClient +from pyredis.client import AsyncSentinelClient +from pyredis.helper import ClusterMap +from pyredis.async_helper import AsyncClusterMap +from pyredis.pool.base import BasePool +from pyredis.pool.async_base import AsyncBasePool +from pyredis.pool.pool import Pool +from pyredis.pool.async_pool import AsyncPool +from pyredis.pool.cluster import ClusterPool +from pyredis.pool.async_cluster import AsyncClusterPool +from pyredis.pool.hash import HashPool +from pyredis.pool.async_hash import AsyncHashPool +from pyredis.pool.sentinel import SentinelPool +from pyredis.pool.async_sentinel import AsyncSentinelPool +from pyredis.pool.sentinel_hash import SentinelHashPool +from pyredis.pool.async_sentinel_hash import AsyncSentinelHashPool + +__all__ = [ + "BasePool", + "AsyncBasePool", + "Pool", + "AsyncPool", + "ClusterPool", + "AsyncClusterPool", + "HashPool", + "AsyncHashPool", + "SentinelPool", + "AsyncSentinelPool", + "SentinelHashPool", + "AsyncSentinelHashPool", + "shuffle", + "Client", + "AsyncClient", + "ClusterClient", + "AsyncClusterClient", + "HashClient", + "AsyncHashClient", + "SentinelClient", + "AsyncSentinelClient", + "ClusterMap", + "AsyncClusterMap", +] diff --git a/pyredis/pool/async_base.py b/pyredis/pool/async_base.py new file mode 100644 index 0000000..1520e8a --- /dev/null +++ b/pyredis/pool/async_base.py @@ -0,0 +1,134 @@ +import asyncio +from pyredis.exceptions import PyRedisError + + +class AsyncBasePool(object): + def __init__( + self, + database=0, + password=None, + encoding=None, + conn_timeout=2, + read_timeout=2, + pool_size=16, + lock=None, + username=None, + ): + self._conn_timeout = conn_timeout + self._read_timeout = read_timeout + if lock is None: + self._lock = asyncio.Lock() + else: + self._lock = lock + self._pool_free = set() + self._pool_used = set() + self._database = database + self._password = password + self._encoding = encoding + self._pool_size = pool_size + self._close_on_err = False + self._cluster = False + self._username = username + + @property + def conn_timeout(self): + return self._conn_timeout + + @property + def read_timeout(self): + return self._read_timeout + + @property + def database(self): + return self._database + + @property + def password(self): + return self._password + + @property + def encoding(self): + return self._encoding + + @property + def pool_size(self): + return self._pool_size + + @pool_size.setter + def pool_size(self, size): + self._pool_size = size + current_size = len(self._pool_free) + len(self._pool_used) + while current_size > size: + try: + client = self._pool_free.pop() + asyncio.create_task( + client.close() + ) + current_size -= 1 + except KeyError: + break + + @property + def close_on_err(self): + return self._close_on_err + + @property + def username(self): + return self._username + + def _connect(self): + raise NotImplementedError + + async def acquire(self): + async with self._lock: + try: + client = self._pool_free.pop() + self._pool_used.add(client) + except KeyError: + if len(self._pool_used) < self.pool_size: + client = self._connect() + if asyncio.iscoroutine(client): + client = await client + self._pool_used.add(client) + else: + raise PyRedisError( + f"Max connections {self.pool_size} exhausted" + ) + return client + + async def release( + self, + conn + ): + async with self._lock: + try: + current_size = len(self._pool_free) + len(self._pool_used) + self._pool_used.remove(conn) + if conn.closed and self.close_on_err: + for c in self._pool_free: + await c.close() + self._pool_free = set() + self._pool_used = set() + elif not conn.closed: + if current_size > self.pool_size: + await conn.close() + else: + self._pool_free.add(conn) + except KeyError: + await conn.close() + + async def execute( + self, + *args, + **kwargs + ): + conn = await self.acquire() + try: + return await conn.execute( + *args, + **kwargs + ) + finally: + await self.release( + conn=conn + ) diff --git a/pyredis/pool/async_cluster.py b/pyredis/pool/async_cluster.py new file mode 100644 index 0000000..f804f1e --- /dev/null +++ b/pyredis/pool/async_cluster.py @@ -0,0 +1,53 @@ +import pyredis.pool +from pyredis import commands +from pyredis.pool.async_base import AsyncBasePool + + +class AsyncClusterPool( + AsyncBasePool, + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, +): + def __init__( + self, + seeds, + slave_ok=False, + password=None, + username=None, + **kwargs + ): + super().__init__( + password=password, + **kwargs + ) + self._map = pyredis.pool.AsyncClusterMap( + seeds=seeds, + password=password, + username=username + ) + self._slave_ok = slave_ok + self._cluster = True + + @property + def slave_ok(self): + return self._slave_ok + + def _connect(self): + return pyredis.pool.AsyncClusterClient( + database=self.database, + password=self.password, + encoding=self.encoding, + slave_ok=self.slave_ok, + conn_timeout=self.conn_timeout, + read_timeout=self.read_timeout, + cluster_map=self._map, + username=self.username, + ) diff --git a/pyredis/pool/async_hash.py b/pyredis/pool/async_hash.py new file mode 100644 index 0000000..82bcc11 --- /dev/null +++ b/pyredis/pool/async_hash.py @@ -0,0 +1,38 @@ +import pyredis.pool +from pyredis import commands +from pyredis.pool.async_base import AsyncBasePool + + +class AsyncHashPool( + AsyncBasePool, + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, +): + def __init__(self, buckets, **kwargs): + super().__init__(**kwargs) + self._buckets = buckets + self._cluster = True + + @property + def buckets(self): + return self._buckets + + def _connect(self): + return pyredis.pool.AsyncHashClient( + buckets=self.buckets, + database=self.database, + password=self.password, + encoding=self.encoding, + conn_timeout=self.conn_timeout, + read_timeout=self.read_timeout, + username=self.username, + ) diff --git a/pyredis/pool/async_pool.py b/pyredis/pool/async_pool.py new file mode 100644 index 0000000..4e29d8d --- /dev/null +++ b/pyredis/pool/async_pool.py @@ -0,0 +1,58 @@ +from pyredis import commands +from pyredis.client import AsyncClient +from pyredis.exceptions import PyRedisError +from pyredis.pool.async_base import AsyncBasePool + + +class AsyncPool( + AsyncBasePool, + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, +): + def __init__( + self, + host=None, + port=6379, + unix_sock=None, + **kwargs + ): + if not bool(host) != bool(unix_sock): + raise PyRedisError("Ether host or unix_sock has to be provided") + super().__init__(**kwargs) + self._host = host + self._port = port + self._unix_sock = unix_sock + + @property + def host(self): + return self._host + + @property + def port(self): + return self._port + + @property + def unix_sock(self): + return self._unix_sock + + def _connect(self): + return AsyncClient( + host=self.host, + port=self.port, + unix_sock=self.unix_sock, + database=self.database, + password=self.password, + encoding=self.encoding, + conn_timeout=self.conn_timeout, + read_timeout=self.read_timeout, + username=self.username, + ) diff --git a/pyredis/pool/async_sentinel.py b/pyredis/pool/async_sentinel.py new file mode 100644 index 0000000..9ac966a --- /dev/null +++ b/pyredis/pool/async_sentinel.py @@ -0,0 +1,101 @@ +import pyredis.pool +from pyredis import commands +from pyredis.exceptions import PyRedisConnError +from pyredis.pool.async_base import AsyncBasePool + + +class AsyncSentinelPool( + AsyncBasePool, + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, +): + def __init__( + self, + sentinels, + name, + slave_ok=False, + retries=3, + sentinel_password=None, + sentinel_username=None, + **kwargs + ): + super().__init__(**kwargs) + self._sentinel = pyredis.pool.AsyncSentinelClient( + sentinels=sentinels, + password=sentinel_password, + username=sentinel_username + ) + self._name = name + self._slave_ok = slave_ok + self._retries = retries + self._close_on_err = True + + @property + def slave_ok(self): + return self._slave_ok + + @property + def name(self): + return self._name + + @property + def retries(self): + return self._retries + + @property + def sentinels(self): + return self._sentinel.sentinels + + async def _connect(self): + for _ in range(self.retries): + if self.slave_ok: + client = await self._get_slave() + else: + client = await self._get_master() + if client: + return client + raise PyRedisConnError("Could not connect to Redis") + + def _get_client(self, host, port): + return pyredis.pool.AsyncClient( + host=host, + port=port, + database=self.database, + password=self.password, + encoding=self.encoding, + conn_timeout=self.conn_timeout, + read_timeout=self.read_timeout, + username=self.username, + ) + + async def _get_master(self): + candidate = await self._sentinel.get_master(self.name) + host = candidate[b"ip"] + port = int(candidate[b"port"]) + client = self._get_client( + host=host, + port=port + ) + return client + + async def _get_slave(self): + candidates = [] + for candidate in await self._sentinel.get_slaves(self.name): + candidates.append((candidate[b"ip"], int(candidate[b"port"]))) + pyredis.pool.shuffle(candidates) + host = candidates[0][0] + port = int(candidates[0][1]) + client = self._get_client( + host=host, + port=port + ) + return client diff --git a/pyredis/pool/async_sentinel_hash.py b/pyredis/pool/async_sentinel_hash.py new file mode 100644 index 0000000..0cb1c3b --- /dev/null +++ b/pyredis/pool/async_sentinel_hash.py @@ -0,0 +1,123 @@ +import pyredis.pool +from pyredis import commands +from pyredis.exceptions import PyRedisConnError +from pyredis.pool.async_base import AsyncBasePool + + +class AsyncSentinelHashPool( + AsyncBasePool, + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, +): + def __init__( + self, + sentinels, + buckets, + slave_ok=False, + retries=3, + sentinel_password=None, + sentinel_username=None, + **kwargs + ): + super().__init__(**kwargs) + self._sentinel = pyredis.pool.AsyncSentinelClient( + sentinels=sentinels, + password=sentinel_password, + username=sentinel_username + ) + self._buckets = buckets + self._slave_ok = slave_ok + self._retries = retries + self._close_on_err = True + self._cluster = True + + @property + def slave_ok(self): + return self._slave_ok + + @property + def buckets(self): + return self._buckets + + @property + def retries(self): + return self._retries + + @property + def sentinels(self): + return self._sentinel.sentinels + + async def _connect(self): + if self.slave_ok: + client = await self._get_slaves() + else: + client = await self._get_masters() + if client: + return client + + def _get_hash_client(self, buckets): + return pyredis.pool.AsyncHashClient( + buckets=buckets, + database=self.database, + password=self.password, + encoding=self.encoding, + conn_timeout=self.conn_timeout, + read_timeout=self.read_timeout, + username=self.username, + ) + + async def _get_master(self, bucket): + candidate = await self._sentinel.get_master(bucket) + host = candidate[b"ip"].decode("utf8") + port = int(candidate[b"port"]) + return host, port + + async def _get_masters(self): + buckets = list() + for bucket in self.buckets: + _counter = self.retries + while _counter >= 0: + _counter -= 1 + _bucket = await self._get_master(bucket) + if _bucket: + buckets.append(_bucket) + break + if _counter == 0: + raise PyRedisConnError( + f"Could not connect to bucket {bucket}" + ) + return self._get_hash_client(buckets=buckets) + + async def _get_slave(self, bucket): + candidates = [] + for candidate in await self._sentinel.get_slaves(bucket): + candidates.append((candidate[b"ip"], int(candidate[b"port"]))) + pyredis.pool.shuffle(candidates) + host = candidates[0][0].decode("utf8") + port = int(candidates[0][1]) + return host, port + + async def _get_slaves(self): + buckets = list() + for bucket in self.buckets: + _counter = self.retries + while _counter >= 0: + _counter -= 1 + _bucket = await self._get_slave(bucket) + if _bucket: + buckets.append(_bucket) + break + if _counter == 0: + raise PyRedisConnError( + f"Could not connect to bucket {bucket}" + ) + return self._get_hash_client(buckets=buckets) diff --git a/pyredis/pool/base.py b/pyredis/pool/base.py new file mode 100644 index 0000000..25ba0ac --- /dev/null +++ b/pyredis/pool/base.py @@ -0,0 +1,129 @@ +import threading +from pyredis.exceptions import PyRedisError + + +class BasePool(object): + def __init__( + self, + database=0, + password=None, + encoding=None, + conn_timeout=2, + read_timeout=2, + pool_size=16, + lock=None, + username=None, + ): + self._conn_timeout = conn_timeout + self._read_timeout = read_timeout + if lock is None: + self._lock = threading.Lock() + else: + self._lock = lock + self._pool_free = set() + self._pool_used = set() + self._database = database + self._password = password + self._encoding = encoding + self._pool_size = pool_size + self._close_on_err = False + self._cluster = False + self._username = username + + @property + def conn_timeout(self): + return self._conn_timeout + + @property + def read_timeout(self): + return self._read_timeout + + @property + def database(self): + return self._database + + @property + def password(self): + return self._password + + @property + def encoding(self): + return self._encoding + + @property + def pool_size(self): + return self._pool_size + + @pool_size.setter + def pool_size(self, size): + try: + self._lock.acquire() + self._pool_size = size + current_size = len(self._pool_free) + len(self._pool_used) + while current_size > size: + try: + client = self._pool_free.pop() + client.close() + current_size -= 1 + except KeyError: + break + finally: + self._lock.release() + + @property + def close_on_err(self): + return self._close_on_err + + @property + def username(self): + return self._username + + def _connect(self): + raise NotImplementedError + + def acquire(self): + try: + self._lock.acquire() + client = self._pool_free.pop() + self._pool_used.add(client) + except KeyError: + if len(self._pool_used) < self.pool_size: + client = self._connect() + self._pool_used.add(client) + else: + raise PyRedisError( + f"Max connections {self.pool_size} exhausted" + ) + finally: + self._lock.release() + return client + + def release(self, conn): + try: + self._lock.acquire() + current_size = len(self._pool_free) + len(self._pool_used) + self._pool_used.remove(conn) + if conn.closed and self.close_on_err: + for c in self._pool_free: + c.close() + self._pool_free = set() + self._pool_used = set() + elif not conn.closed: + if current_size > self.pool_size: + conn.close() + else: + self._pool_free.add(conn) + except KeyError: + conn.close() + finally: + self._lock.release() + + def execute(self, *args, **kwargs): + conn = self.acquire() + try: + return conn.execute( + *args, + **kwargs + ) + finally: + self.release(conn) diff --git a/pyredis/pool/cluster.py b/pyredis/pool/cluster.py new file mode 100644 index 0000000..e1336f8 --- /dev/null +++ b/pyredis/pool/cluster.py @@ -0,0 +1,53 @@ +import pyredis.pool +from pyredis import commands +from pyredis.pool.base import BasePool + + +class ClusterPool( + BasePool, + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, +): + def __init__( + self, + seeds, + slave_ok=False, + password=None, + username=None, + **kwargs + ): + super().__init__( + password=password, + **kwargs + ) + self._map = pyredis.pool.ClusterMap( + seeds=seeds, + password=password, + username=username + ) + self._slave_ok = slave_ok + self._cluster = True + + @property + def slave_ok(self): + return self._slave_ok + + def _connect(self): + return pyredis.pool.ClusterClient( + database=self.database, + password=self.password, + encoding=self.encoding, + slave_ok=self.slave_ok, + conn_timeout=self.conn_timeout, + read_timeout=self.read_timeout, + cluster_map=self._map, + username=self.username, + ) diff --git a/pyredis/pool/hash.py b/pyredis/pool/hash.py new file mode 100644 index 0000000..3960903 --- /dev/null +++ b/pyredis/pool/hash.py @@ -0,0 +1,38 @@ +import pyredis.pool +from pyredis import commands +from pyredis.pool.base import BasePool + + +class HashPool( + BasePool, + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, +): + def __init__(self, buckets, **kwargs): + super().__init__(**kwargs) + self._buckets = buckets + self._cluster = True + + @property + def buckets(self): + return self._buckets + + def _connect(self): + return pyredis.pool.HashClient( + buckets=self.buckets, + database=self.database, + password=self.password, + encoding=self.encoding, + conn_timeout=self.conn_timeout, + read_timeout=self.read_timeout, + username=self.username, + ) diff --git a/pyredis/pool/pool.py b/pyredis/pool/pool.py new file mode 100644 index 0000000..6f24b7b --- /dev/null +++ b/pyredis/pool/pool.py @@ -0,0 +1,52 @@ +import pyredis.pool +from pyredis import commands +from pyredis.exceptions import PyRedisError +from pyredis.pool.base import BasePool + + +class Pool( + BasePool, + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, +): + def __init__(self, host=None, port=6379, unix_sock=None, **kwargs): + if not bool(host) != bool(unix_sock): + raise PyRedisError("Ether host or unix_sock has to be provided") + super().__init__(**kwargs) + self._host = host + self._port = port + self._unix_sock = unix_sock + + @property + def host(self): + return self._host + + @property + def port(self): + return self._port + + @property + def unix_sock(self): + return self._unix_sock + + def _connect(self): + return pyredis.pool.Client( + host=self.host, + port=self.port, + unix_sock=self.unix_sock, + database=self.database, + password=self.password, + encoding=self.encoding, + conn_timeout=self.conn_timeout, + read_timeout=self.read_timeout, + username=self.username, + ) diff --git a/pyredis/pool/sentinel.py b/pyredis/pool/sentinel.py new file mode 100644 index 0000000..85799ca --- /dev/null +++ b/pyredis/pool/sentinel.py @@ -0,0 +1,101 @@ +import pyredis.pool +from pyredis import commands +from pyredis.exceptions import PyRedisConnError +from pyredis.pool.base import BasePool + + +class SentinelPool( + BasePool, + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, +): + def __init__( + self, + sentinels, + name, + slave_ok=False, + retries=3, + sentinel_password=None, + sentinel_username=None, + **kwargs + ): + super().__init__(**kwargs) + self._sentinel = pyredis.pool.SentinelClient( + sentinels=sentinels, + password=sentinel_password, + username=sentinel_username + ) + self._name = name + self._slave_ok = slave_ok + self._retries = retries + self._close_on_err = True + + @property + def slave_ok(self): + return self._slave_ok + + @property + def name(self): + return self._name + + @property + def retries(self): + return self._retries + + @property + def sentinels(self): + return self._sentinel.sentinels + + def _connect(self): + for _ in range(self.retries): + if self.slave_ok: + client = self._get_slave() + else: + client = self._get_master() + if client: + return client + raise PyRedisConnError("Could not connect to Redis") + + def _get_client(self, host, port): + return pyredis.pool.Client( + host=host, + port=port, + database=self.database, + password=self.password, + encoding=self.encoding, + conn_timeout=self.conn_timeout, + read_timeout=self.read_timeout, + username=self.username, + ) + + def _get_master(self): + candidate = self._sentinel.get_master(self.name) + host = candidate[b"ip"] + port = int(candidate[b"port"]) + client = self._get_client( + host=host, + port=port + ) + return client + + def _get_slave(self): + candidates = [] + for candidate in self._sentinel.get_slaves(self.name): + candidates.append((candidate[b"ip"], int(candidate[b"port"]))) + pyredis.pool.shuffle(candidates) + host = candidates[0][0] + port = int(candidates[0][1]) + client = self._get_client( + host=host, + port=port + ) + return client diff --git a/pyredis/pool/sentinel_hash.py b/pyredis/pool/sentinel_hash.py new file mode 100644 index 0000000..56f23af --- /dev/null +++ b/pyredis/pool/sentinel_hash.py @@ -0,0 +1,123 @@ +import pyredis.pool +from pyredis import commands +from pyredis.exceptions import PyRedisConnError +from pyredis.pool.base import BasePool + + +class SentinelHashPool( + BasePool, + commands.Connection, + commands.Geo, + commands.Hash, + commands.HyperLogLog, + commands.Key, + commands.List, + commands.Publish, + commands.Scripting, + commands.Set, + commands.SSet, + commands.String, +): + def __init__( + self, + sentinels, + buckets, + slave_ok=False, + retries=3, + sentinel_password=None, + sentinel_username=None, + **kwargs + ): + super().__init__(**kwargs) + self._sentinel = pyredis.pool.SentinelClient( + sentinels=sentinels, + password=sentinel_password, + username=sentinel_username + ) + self._buckets = buckets + self._slave_ok = slave_ok + self._retries = retries + self._close_on_err = True + self._cluster = True + + @property + def slave_ok(self): + return self._slave_ok + + @property + def buckets(self): + return self._buckets + + @property + def retries(self): + return self._retries + + @property + def sentinels(self): + return self._sentinel.sentinels + + def _connect(self): + if self.slave_ok: + client = self._get_slaves() + else: + client = self._get_masters() + if client: + return client + + def _get_hash_client(self, buckets): + return pyredis.pool.HashClient( + buckets=buckets, + database=self.database, + password=self.password, + encoding=self.encoding, + conn_timeout=self.conn_timeout, + read_timeout=self.read_timeout, + username=self.username, + ) + + def _get_master(self, bucket): + candidate = self._sentinel.get_master(bucket) + host = candidate[b"ip"].decode("utf8") + port = int(candidate[b"port"]) + return host, port + + def _get_masters(self): + buckets = list() + for bucket in self.buckets: + _counter = self.retries + while _counter >= 0: + _counter -= 1 + _bucket = self._get_master(bucket) + if _bucket: + buckets.append(_bucket) + break + if _counter == 0: + raise PyRedisConnError( + f"Could not connect to bucket {bucket}" + ) + return self._get_hash_client(buckets=buckets) + + def _get_slave(self, bucket): + candidates = [] + for candidate in self._sentinel.get_slaves(bucket): + candidates.append((candidate[b"ip"], int(candidate[b"port"]))) + pyredis.pool.shuffle(candidates) + host = candidates[0][0].decode("utf8") + port = int(candidates[0][1]) + return host, port + + def _get_slaves(self): + buckets = list() + for bucket in self.buckets: + _counter = self.retries + while _counter >= 0: + _counter -= 1 + _bucket = self._get_slave(bucket) + if _bucket: + buckets.append(_bucket) + break + if _counter == 0: + raise PyRedisConnError( + f"Could not connect to bucket {bucket}" + ) + return self._get_hash_client(buckets=buckets) diff --git a/pyredis/protocol.py b/pyredis/protocol.py index 0f9a067..59f5cc7 100644 --- a/pyredis/protocol.py +++ b/pyredis/protocol.py @@ -33,10 +33,13 @@ def is_exception(inst, classinfo): ) - class ReplyParser(object): def __init__( - self, encoding, source, protocol_error=ProtocolError, reply_error=ReplyError + self, + encoding, + source, + protocol_error=ProtocolError, + reply_error=ReplyError ): self._len = 0 self._nested_parser = None @@ -137,7 +140,9 @@ def parse_error(self): result = self.readline() if result: self.complete = True - self.result = self._reply_error(result.decode(sys.getdefaultencoding())) + self.result = self._reply_error( + result.decode(sys.getdefaultencoding()) + ) return True def parse_int(self): @@ -181,7 +186,10 @@ def __init__( if is_exception(replyError, Exception): self._reply_error = replyError self._replyparser = ReplyParser( - self._encoding, self._buffer, self._protocol_error, self._reply_error + encoding=self._encoding, + source=self._buffer, + protocol_error=self._protocol_error, + reply_error=self._reply_error, ) def _truncate(self): @@ -230,7 +238,8 @@ def to_bytes(value): return str(value).encode() else: raise ValueError( - "Unsupported value, has to be a instance of bytes, str, int or float" + "Unsupported value, has to be a instance of " + "bytes, str, int or float" ) @@ -241,6 +250,14 @@ def writer(*args): extend((TYPE_ARRAY, str(len(args)).encode(), SYM_CRLF)) for member in map(to_bytes, args): - extend((TYPE_BULK, str(len(member)).encode(), SYM_CRLF, member, SYM_CRLF)) + extend( + ( + TYPE_BULK, + str(len(member)).encode(), + SYM_CRLF, + member, + SYM_CRLF, + ) + ) return b"".join(buf) diff --git a/tests/unit/test_async_client.py b/tests/unit/test_async_client.py new file mode 100644 index 0000000..7910352 --- /dev/null +++ b/tests/unit/test_async_client.py @@ -0,0 +1,468 @@ +import asyncio +from unittest import IsolatedAsyncioTestCase +from unittest.mock import AsyncMock +from unittest.mock import MagicMock +from unittest.mock import Mock +from unittest.mock import patch + +from pyredis import get_by_url +from pyredis.client import AsyncClient +from pyredis.client import AsyncClusterClient +from pyredis.client import AsyncHashClient +from pyredis.client import AsyncPubSubClient +from pyredis.client import AsyncSentinelClient +from pyredis.connection import AsyncConnection +from pyredis.exceptions import PyRedisConnClosed +from pyredis.exceptions import PyRedisConnError +from pyredis.exceptions import PyRedisConnReadTimeout +from pyredis.exceptions import PyRedisError +from pyredis.pool import AsyncPool +from pyredis.pool import AsyncClusterPool +from pyredis.pool import AsyncHashPool +from pyredis.pool import AsyncSentinelPool +from pyredis.pool import AsyncSentinelHashPool + + +class TestAsyncConnection(IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.addCleanup( + patch.stopall + ) + self.mock_reader = AsyncMock() + self.mock_writer = MagicMock() + self.mock_writer.drain = AsyncMock() + + self.patch_open = patch( + target="asyncio.open_connection", + new_callable=AsyncMock + ) + self.mock_open = self.patch_open.start() + self.mock_open.return_value = ( + self.mock_reader, + self.mock_writer + ) + + self.patch_open_unix = patch( + target="asyncio.open_unix_connection", + new_callable=AsyncMock + ) + self.mock_open_unix = self.patch_open_unix.start() + self.mock_open_unix.return_value = ( + self.mock_reader, + self.mock_writer + ) + + self.patch_reader_parser = patch( + target="pyredis.connection.Reader", + autospec=True + ) + self.mock_reader_parser_class = self.patch_reader_parser.start() + self.mock_reader_parser = self.mock_reader_parser_class.return_value + + def test_init_default(self): + conn = AsyncConnection( + host="127.0.0.1" + ) + self.assertEqual( + first=conn.host, + second="127.0.0.1" + ) + self.assertEqual( + first=conn.port, + second=6379 + ) + + def test_init_invalid(self): + with self.assertRaises( + expected_exception=PyRedisError + ): + AsyncConnection() + + async def test_connect_and_authenticate(self): + conn = AsyncConnection( + host="127.0.0.1", + password="testpass" + ) + self.mock_reader_parser.gets.side_effect = [ + False, + b"OK" + ] + self.mock_reader.read.return_value = b"+OK\r\n" + + await conn.write( + *["PING"] + ) + + self.assertEqual( + first=self.mock_open.call_args[1], + second={ + "host": "127.0.0.1", + "port": 6379 + } + ) + + async def test_read_timeout(self): + conn = AsyncConnection( + host="127.0.0.1" + ) + self.mock_reader_parser.gets.return_value = False + self.mock_reader.read.side_effect = asyncio.TimeoutError + + with self.assertRaises( + expected_exception=PyRedisConnReadTimeout + ): + await conn.read() + + async def test_read_closed(self): + conn = AsyncConnection( + host="127.0.0.1" + ) + self.mock_reader_parser.gets.return_value = False + self.mock_reader.read.return_value = b"" + + with self.assertRaises( + expected_exception=PyRedisConnClosed + ): + await conn.read() + + +class TestAsyncClient(IsolatedAsyncioTestCase): + async def test_execute(self): + client = AsyncClient( + host="127.0.0.1" + ) + client._conn = AsyncMock() + client._conn.read.return_value = b"PONG" + + res = await client.execute( + *["PING"] + ) + + self.assertEqual( + first=res, + second=b"PONG" + ) + self.assertEqual( + first=client._conn.write.call_args[0], + second=( + "PING", + ) + ) + + async def test_close(self): + client = AsyncClient( + host="127.0.0.1" + ) + client._conn = AsyncMock() + await client.close() + + self.assertTrue( + expr=client._conn.close.called + ) + + +class TestAsyncPool(IsolatedAsyncioTestCase): + async def test_pool_acquire_release(self): + pool = AsyncPool( + host="127.0.0.1", + pool_size=2 + ) + mock_client = AsyncMock() + mock_client.closed = False + pool._connect = Mock() + pool._connect.return_value = mock_client + + c1 = await pool.acquire() + self.assertEqual( + first=c1, + second=mock_client + ) + self.assertEqual( + first=len(pool._pool_used), + second=1 + ) + + await pool.release( + conn=c1 + ) + self.assertEqual( + first=len(pool._pool_used), + second=0 + ) + self.assertEqual( + first=len(pool._pool_free), + second=1 + ) + + async def test_pool_execute(self): + pool = AsyncPool( + host="127.0.0.1" + ) + mock_client = AsyncMock() + mock_client.closed = False + mock_client.execute.return_value = b"OK" + pool._connect = Mock() + pool._connect.return_value = mock_client + + res = await pool.execute( + *["SET", "foo", "bar"] + ) + self.assertEqual( + first=res, + second=b"OK" + ) + + +class TestAsyncClusterClient(IsolatedAsyncioTestCase): + async def test_async_cluster_client(self): + with patch( + target="pyredis.client.AsyncClusterMap", + autospec=True + ) as mock_map_class: + mock_map = mock_map_class.return_value + mock_map.id = "mapid" + mock_map.get_slot.return_value = "127.0.0.1_6379" + + client = AsyncClusterClient( + seeds=[("127.0.0.1", 6379)] + ) + client._conns["127.0.0.1_6379"] = AsyncMock() + client._conns["127.0.0.1_6379"].read.return_value = b"OK" + + res = await client.execute( + *["SET", "foo", "bar"], + shard_key="foo" + ) + self.assertEqual( + first=res, + second=b"OK" + ) + + async def test_async_cluster_pool(self): + with patch( + target="pyredis.pool.AsyncClusterMap", + autospec=True + ): + pool = AsyncClusterPool( + seeds=[("127.0.0.1", 6379)] + ) + mock_client = AsyncMock() + mock_client.closed = False + mock_client.execute.return_value = b"OK" + pool._connect = Mock() + pool._connect.return_value = mock_client + + res = await pool.execute( + *["SET", "foo", "bar"] + ) + self.assertEqual( + first=res, + second=b"OK" + ) + + +class TestAsyncHashClient(IsolatedAsyncioTestCase): + async def test_async_hash_client(self): + client = AsyncHashClient( + buckets=[("127.0.0.1", 6379)] + ) + mock_conn = AsyncMock() + client._conns["127.0.0.1_6379"] = mock_conn + mock_conn.read.return_value = b"OK" + + res = await client.execute( + *["SET", "foo", "bar"], + shard_key="foo" + ) + self.assertEqual( + first=res, + second=b"OK" + ) + self.assertTrue( + expr=mock_conn.write.called + ) + + async def test_async_hash_pool(self): + pool = AsyncHashPool( + buckets=[("127.0.0.1", 6379)] + ) + mock_client = AsyncMock() + mock_client.closed = False + mock_client.execute.return_value = b"OK" + pool._connect = Mock() + pool._connect.return_value = mock_client + + res = await pool.execute( + *["SET", "foo", "bar"] + ) + self.assertEqual( + first=res, + second=b"OK" + ) + + +class TestAsyncPubSubClient(IsolatedAsyncioTestCase): + async def test_async_pubsub_client(self): + with patch( + target="pyredis.client.AsyncConnection", + autospec=True + ) as mock_conn_class: + mock_conn = mock_conn_class.return_value + mock_conn.read.return_value = b"message" + + client = AsyncPubSubClient( + host="127.0.0.1" + ) + res = await client.get() + self.assertEqual( + first=res, + second=b"message" + ) + self.assertEqual( + first=mock_conn.read.call_args[1], + second={ + "close_on_timeout": False + } + ) + + await client.write( + *["SUBSCRIBE", "channel"] + ) + self.assertTrue( + expr=mock_conn.write.called + ) + + +class TestAsyncSentinel(IsolatedAsyncioTestCase): + async def test_async_sentinel_client(self): + client = AsyncSentinelClient( + sentinels=[("127.0.0.1", 26379)] + ) + client._conn = AsyncMock() + client._conn.read.return_value = [ + b"ip", + b"127.0.0.1", + b"port", + b"6379", + ] + + master = await client.get_master( + name="mymaster" + ) + self.assertEqual( + first=master, + second={ + b"ip": b"127.0.0.1", + b"port": b"6379" + } + ) + + async def test_async_sentinel_pool(self): + with patch( + target="pyredis.pool.AsyncSentinelClient", + autospec=True + ) as mock_sentinel_class: + mock_sentinel = mock_sentinel_class.return_value + mock_sentinel.get_master.return_value = { + b"ip": b"127.0.0.1", + b"port": b"6379" + } + + pool = AsyncSentinelPool( + sentinels=[("127.0.0.1", 26379)], + name="mymaster" + ) + mock_client = AsyncMock() + pool._get_client = Mock() + pool._get_client.return_value = mock_client + + client = await pool._get_master() + self.assertEqual( + first=client, + second=mock_client + ) + self.assertEqual( + first=pool._get_client.call_args[1], + second={ + "host": b"127.0.0.1", + "port": 6379 + } + ) + + async def test_async_sentinel_hash_pool(self): + with patch( + target="pyredis.pool.AsyncSentinelClient", + autospec=True + ) as mock_sentinel_class: + mock_sentinel = mock_sentinel_class.return_value + mock_sentinel.get_master.return_value = { + b"ip": b"127.0.0.1", + b"port": b"6379" + } + + pool = AsyncSentinelHashPool( + sentinels=[("127.0.0.1", 26379)], + buckets=[("127.0.0.1", 6379)] + ) + pool._get_hash_client = Mock() + mock_hash_client = AsyncMock() + pool._get_hash_client.return_value = mock_hash_client + + client = await pool._get_masters() + self.assertEqual( + first=client, + second=mock_hash_client + ) + self.assertEqual( + first=pool._get_hash_client.call_args[1], + second={ + "buckets": [ + ( + "127.0.0.1", + 6379 + ) + ] + } + ) + + +class TestGetByUrl(IsolatedAsyncioTestCase): + def test_get_by_url_async(self): + pool = get_by_url( + url="redis://127.0.0.1:6379", + async_client=True + ) + self.assertIsInstance( + obj=pool, + cls=AsyncPool + ) + + def test_get_by_url_cluster_async(self): + with patch( + target="pyredis.pool.AsyncClusterMap", + autospec=True + ): + pool = get_by_url( + url="cluster://127.0.0.1:6379", + async_client=True + ) + self.assertIsInstance( + obj=pool, + cls=AsyncClusterPool + ) + + def test_get_by_url_sentinel_async(self): + with patch( + target="pyredis.pool.AsyncSentinelClient", + autospec=True + ): + pool = get_by_url( + url="sentinel://127.0.0.1:26379?name=mymaster", + async_client=True + ) + self.assertIsInstance( + obj=pool, + cls=AsyncSentinelPool + ) + + diff --git a/tests/unit/test_pool.py b/tests/unit/test_pool.py index cb9c7bb..317188b 100644 --- a/tests/unit/test_pool.py +++ b/tests/unit/test_pool.py @@ -464,7 +464,10 @@ def test__get_master(self): pool._get_client.return_value = client_mock client = pool._get_master() pool._sentinel.get_master.assert_called_with('mymaster') - pool._get_client.assert_called_with(b'127.0.0.1', 12345) + pool._get_client.assert_called_with( + host=b'127.0.0.1', + port=12345 + ) self.assertEqual(client, client_mock) def test_get_slave(self): @@ -500,5 +503,8 @@ def test_get_slave(self): (b'127.0.0.3', 12345) ] ) - pool._get_client.assert_called_with(b'127.0.0.1', 12345) + pool._get_client.assert_called_with( + host=b'127.0.0.1', + port=12345 + ) self.assertEqual(client, client_mock1) From 7e4f0e9b5f208bc1ea14118f03373c2c98eda60d Mon Sep 17 00:00:00 2001 From: Stephan Schultchen Date: Fri, 26 Jun 2026 13:12:03 +0200 Subject: [PATCH 2/2] Update README.md features list to reflect async support --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1ae2163..52aabde 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,16 @@ Redis Client implementation for Python 3. ## Features -- Base Redis Client -- Publish Subscribe Client -- Sentinel Client -- Connection Pool -- Sentinel Backed Connection Pool -- Client & Pool for Redis Cluster -- Bulk Mode (Not supported with Redis Cluster) -- Client & Pool with Static Hash Cluster (Supports Bulk Mode) -- Sentinel Backed Pool with Static Hash Cluster (Supports Bulk Mode) +- Complete Synchronous and Asynchronous (asyncio) counterparts for all client and pool classes. +- Base Redis Client & AsyncClient +- Publish Subscribe Client & AsyncPubSubClient +- Sentinel Client & AsyncSentinelClient +- Connection Pool & AsyncPool +- Sentinel Backed Connection Pool & AsyncSentinelPool +- Client & Pool for Redis Cluster & AsyncClusterPool +- Bulk Mode (Sync & Async, not supported with Redis Cluster) +- Client & Pool with Static Hash Cluster & AsyncHashPool +- Sentinel Backed Pool with Static Hash Cluster & AsyncSentinelHashPool ## Documentation Documentation can be found on GitHub Pages: