From fac0f04dd4f8432a1eb2555cc1528fd1ff5e02d1 Mon Sep 17 00:00:00 2001 From: Thibault Meunier Date: Mon, 15 Sep 2025 11:14:43 +0200 Subject: [PATCH 1/4] Optional signature agent and jwk thumprint This commit aligns the implementation to be closer to [draft-meunier-web-bot-auth-architecture](https://datatracker.ietf.org/doc/html/draft-meunier-web-bot-auth-architecture#name-generating-http-message-sig). Namely: 1. It removes the default signature-agent which is an artifact from previous tests. This makes for a saner ecosystem by not providing a default tied to one company 2. It replaces the placeholder "compute-jwk-thumbprint" with the actual thumbprint This commit also updates the readme, and remove legacy copyright which have been moved to pyproject.toml. --- README.md | 2 +- src/bot_auth/__init__.py | 43 ++++++++++++++++++++++------ uv.lock | 62 ++++++++++++++++++++-------------------- 3 files changed, 66 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index bd121ef..b25b030 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Web Bot Auth Python -![GitHub License](https://img.shields.io/github/license/cloudflareresearch/web-bot-auth) +![GitHub License](https://img.shields.io/github/license/cyberstormdotmu/bot-authentication) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) Implementation of Web Bot Auth in Python. diff --git a/src/bot_auth/__init__.py b/src/bot_auth/__init__.py index d1130a7..a25e1db 100644 --- a/src/bot_auth/__init__.py +++ b/src/bot_auth/__init__.py @@ -7,6 +7,8 @@ __version__ = "0.1.0" import base64 +import hashlib +import json import requests import time from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey @@ -49,15 +51,12 @@ class BotAuth: - get_local_keys() -> -> list[dict[str, str]] - get_remote_keys() -> list[dict[str, str]] - get_header()-> dict[str, str] - - @info: - © 2025 Atish Joottun """ def __init__( self, localKeys, - signAgent="http-message-signatures-example.research.cloudflare.com", + signAgent=None, ): self.localKeys = localKeys self.signAgent = signAgent @@ -118,6 +117,24 @@ def _base64_encode_bytes(self, val): def _jwt_to_private_key(self, jwk): return Ed25519PrivateKey.from_private_bytes(self._base64url_decode(jwk["d"])) + def _public_key_to_jwk_thumbprint(self, public_key): + """ + Compute the base64url JWK SHA-256 Thumbprint for an Ed25519 public key. + """ + # JWK Thumbprint according to RFC 7638, base64url with padding and sha256 + + jwk_dict = { + "crv": "Ed25519", + "kty": "OKP", + "x": self._base64_encode_bytes(public_key.public_bytes_raw()), + } + + jwk_json = json.dumps(jwk_dict, separators=(",", ":"), sort_keys=True) + sha256_hash = hashlib.sha256(jwk_json.encode("utf-8")).digest() + thumbprint = base64.urlsafe_b64encode(sha256_hash).decode("ascii") + + return thumbprint + # def _jwk_to_public_key_bytes(self, jwk): # private_key = self.jwk_to_private_key(jwk) # public_key = private_key.public_key() @@ -133,7 +150,8 @@ def get_bot_signature_header(self, url) -> dict[str, str]: ## Get similar key in local repo selected_key = local_keys[0] - resolver = SingleKeyResolver(self._jwt_to_private_key(selected_key)) + private_key = self._jwt_to_private_key(selected_key) + resolver = SingleKeyResolver(private_key) signer = HTTPMessageSigner( signature_algorithm=algorithms.ED25519, key_resolver=resolver ) @@ -141,26 +159,33 @@ def get_bot_signature_header(self, url) -> dict[str, str]: created = datetime.fromtimestamp(time.time()) expires = created + timedelta(minutes=5) + headers = {"Signature-Agent": self.signAgent} if self.signAgent else {} request = requests.Request( "GET", url, headers={ - "Signature-Agent": self.signAgent, + **headers, }, ) + + key_id = self._public_key_to_jwk_thumbprint(private_key.public_key()) + covered_components = ( + ("@authority", "signature-agent") if self.signAgent else ["@authority"] + ) signer.sign( request, - key_id="compute-jwk-thumbprint", - covered_component_ids=("@authority", "signature-agent"), + key_id=key_id, + covered_component_ids=covered_components, created=created, expires=expires, tag="web-bot-auth", + label="sig1", ) header = { - "Signature-Agent": request.headers["Signature-Agent"], "Signature-Input": request.headers["Signature-Input"], "Signature": request.headers["Signature"], + **headers, } return header diff --git a/uv.lock b/uv.lock index a9d6415..0c72137 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.12'", @@ -8,9 +8,9 @@ resolution-markers = [ [manifest] members = [ + "bot-auth", "crawl4ai-hook", "scrapy-spider", - "web-bot-auth", ] [[package]] @@ -225,6 +225,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/09/71/54e999902aed72baf26bca0d50781b01838251a462612966e9fc4891eadd/black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717", size = 207646, upload-time = "2025-01-29T04:15:38.082Z" }, ] +[[package]] +name = "bot-auth" +version = "0.2.2" +source = { editable = "." } +dependencies = [ + { name = "cryptography" }, + { name = "http-message-signatures" }, + { name = "requests" }, +] + +[package.dev-dependencies] +dev = [ + { name = "black" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "cryptography", specifier = ">=45.0.0" }, + { name = "http-message-signatures", specifier = ">=0.7.0" }, + { name = "requests", specifier = ">=2.25.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "black", specifier = ">=25.1.0" }, + { name = "ruff", specifier = ">=0.12.4" }, +] + [[package]] name = "brotli" version = "1.1.0" @@ -2776,35 +2805,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/dd/56f0d8af71e475ed194d702f8b4cf9cea812c95e82ad823d239023c6558c/w3lib-2.3.1-py3-none-any.whl", hash = "sha256:9ccd2ae10c8c41c7279cd8ad4fe65f834be894fe7bfdd7304b991fd69325847b", size = 21751, upload-time = "2025-01-27T14:22:09.421Z" }, ] -[[package]] -name = "web-bot-auth" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "cryptography" }, - { name = "http-message-signatures" }, - { name = "requests" }, -] - -[package.dev-dependencies] -dev = [ - { name = "black" }, - { name = "ruff" }, -] - -[package.metadata] -requires-dist = [ - { name = "cryptography", specifier = ">=45.0.0" }, - { name = "http-message-signatures", specifier = ">=0.7.0" }, - { name = "requests", specifier = ">=2.25.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "black", specifier = ">=25.1.0" }, - { name = "ruff", specifier = ">=0.12.4" }, -] - [[package]] name = "xxhash" version = "3.5.0" From c25eb70cd0de53fbce4b28eabc1cae24039a7a1f Mon Sep 17 00:00:00 2001 From: Thibault Meunier Date: Mon, 15 Sep 2025 11:25:40 +0200 Subject: [PATCH 2/4] Release 0.3.0 --- pyproject.toml | 2 +- src/bot_auth/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bee8668..1d1ea22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bot-auth" -version = "0.2.2" +version = "0.3.0" description = "A library to check for AI Bot Authentication using the latest HTTP header Signature." readme = "README.md" requires-python = ">=3.11" diff --git a/src/bot_auth/__init__.py b/src/bot_auth/__init__.py index a25e1db..ece3bc0 100644 --- a/src/bot_auth/__init__.py +++ b/src/bot_auth/__init__.py @@ -4,7 +4,7 @@ A library to check for AI Bot Authentication using the latest HTTP header Signature. """ -__version__ = "0.1.0" +__version__ = "0.3.0" import base64 import hashlib From 269a4c014d2298bdb1bf6aa5b64f4b53a2981f9e Mon Sep 17 00:00:00 2001 From: Thibault Meunier Date: Fri, 26 Sep 2025 11:43:27 +0200 Subject: [PATCH 3/4] Fix thumbprint calculation base64url was not encoded properly --- pyproject.toml | 2 +- src/bot_auth/__init__.py | 9 ++++++--- uv.lock | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1d1ea22..860c081 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bot-auth" -version = "0.3.0" +version = "0.3.1" description = "A library to check for AI Bot Authentication using the latest HTTP header Signature." readme = "README.md" requires-python = ">=3.11" diff --git a/src/bot_auth/__init__.py b/src/bot_auth/__init__.py index ece3bc0..06c9d7e 100644 --- a/src/bot_auth/__init__.py +++ b/src/bot_auth/__init__.py @@ -4,7 +4,7 @@ A library to check for AI Bot Authentication using the latest HTTP header Signature. """ -__version__ = "0.3.0" +__version__ = "0.3.1" import base64 import hashlib @@ -114,6 +114,9 @@ def _base64url_decode(self, val): def _base64_encode_bytes(self, val): return base64.b64encode(val).decode("ascii") + def _base64url_nopad_encode_bytes(self, val): + return base64.urlsafe_b64encode(val).decode("ascii").strip("=") + def _jwt_to_private_key(self, jwk): return Ed25519PrivateKey.from_private_bytes(self._base64url_decode(jwk["d"])) @@ -126,12 +129,12 @@ def _public_key_to_jwk_thumbprint(self, public_key): jwk_dict = { "crv": "Ed25519", "kty": "OKP", - "x": self._base64_encode_bytes(public_key.public_bytes_raw()), + "x": self._base64url_nopad_encode_bytes(public_key.public_bytes_raw()), } jwk_json = json.dumps(jwk_dict, separators=(",", ":"), sort_keys=True) sha256_hash = hashlib.sha256(jwk_json.encode("utf-8")).digest() - thumbprint = base64.urlsafe_b64encode(sha256_hash).decode("ascii") + thumbprint = self._base64url_nopad_encode_bytes(sha256_hash) return thumbprint diff --git a/uv.lock b/uv.lock index 0c72137..1f39203 100644 --- a/uv.lock +++ b/uv.lock @@ -227,7 +227,7 @@ wheels = [ [[package]] name = "bot-auth" -version = "0.2.2" +version = "0.3.1" source = { editable = "." } dependencies = [ { name = "cryptography" }, From d474507e722eb07c9058000c73d69bd7fddce486 Mon Sep 17 00:00:00 2001 From: Thibault Meunier Date: Thu, 6 Nov 2025 12:28:36 -0500 Subject: [PATCH 4/4] Update package metadata Add homepage, source, and issues urls as metadata Bumps the package version as well to allow for publication on pypi --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 860c081..f1793bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bot-auth" -version = "0.3.1" +version = "0.3.2" description = "A library to check for AI Bot Authentication using the latest HTTP header Signature." readme = "README.md" requires-python = ">=3.11" @@ -15,6 +15,10 @@ dependencies = [ "http-message-signatures>=0.7.0", "requests>=2.25.0", ] +[project.urls] +homepage = "https://github.com/cyberstormdotmu/Bot-Authentication" +source = "https://github.com/cyberstormdotmu/Bot-Authentication.git" +issues = "https://github.com/cyberstormdotmu/Bot-Authentication/issues" [build-system] requires = ["uv_build>=0.8.0,<0.9"]