From 9593ebea25ada43b5b19d34811d2d272b8cb57ac Mon Sep 17 00:00:00 2001 From: hudson-woomer Date: Tue, 19 May 2026 10:31:09 -0400 Subject: [PATCH 01/24] update new branch --- .idea/.gitignore | 10 + .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 6 + .idea/modules.xml | 8 + .idea/psengine.iml | 18 ++ .idea/vcs.xml | 6 + docs/CHANGELOG.md | 35 +-- docs/api/links/errors.md | 1 + docs/api/links/links_adt.md | 1 + docs/api/links/links_mgr.md | 8 + docs/api/links/links_models.md | 1 + docs/examples/links/__init__.py | 0 docs/examples/links/example_1.py | 17 ++ docs/examples/links/example_2.py | 25 ++ docs/examples/links/example_3.py | 15 ++ psengine/links/__init__.py | 36 +++ psengine/links/errors.py | 26 ++ psengine/links/links.py | 83 ++++++ psengine/links/links_mgr.py | 245 ++++++++++++++++++ psengine/links/models.py | 85 ++++++ tests/links/conftest.py | 8 + tests/links/test_links_mgr.py | 194 ++++++++++++++ tests/links/test_links_models.py | 27 ++ uv.lock | 4 +- 24 files changed, 832 insertions(+), 33 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/psengine.iml create mode 100644 .idea/vcs.xml create mode 100644 docs/api/links/errors.md create mode 100644 docs/api/links/links_adt.md create mode 100644 docs/api/links/links_mgr.md create mode 100644 docs/api/links/links_models.md create mode 100644 docs/examples/links/__init__.py create mode 100644 docs/examples/links/example_1.py create mode 100644 docs/examples/links/example_2.py create mode 100644 docs/examples/links/example_3.py create mode 100644 psengine/links/__init__.py create mode 100644 psengine/links/errors.py create mode 100644 psengine/links/links.py create mode 100644 psengine/links/links_mgr.py create mode 100644 psengine/links/models.py create mode 100644 tests/links/conftest.py create mode 100644 tests/links/test_links_mgr.py create mode 100644 tests/links/test_links_models.py diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..30cf57ed --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 00000000..105ce2da --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..87fefa91 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..149097e7 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/psengine.iml b/.idea/psengine.iml new file mode 100644 index 00000000..2e5c1f31 --- /dev/null +++ b/.idea/psengine.iml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 984f4997..0bc42677 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,39 +1,12 @@ # Changelog -## v2.7.0 - 2026-05-01 +## v2.6.1 - 2026-05-08 -- Added support for Threat Map and Malware Map via the `ThreatMapMgr`. - -## v2.6.0 - 2026-04-29 +- `psengine.links` now supports the Links API via the `LinksMgr` class. ### Added -- Added support for Python 3.14 -- `PlaybookAlertMgr.search` now supports filter for a single or a list of organisations. -- `TimeHelpers.rel_time_to_date` now supports a starting time from where the math begins. If not specified it will be the UTC execution time. -- `TimeHelpers.rel_time_to_date` now supports increment calculation, with `+1h`. -- `TimeHelpers.rel_time_to_date` now supports increment or decrement of minutes with `+10m` or `-10m`. -- Added `malware_intel.AutoYaraMgr` and `malware_intel.AutoSigmaMgr` managers to interact with auto-yara and auto-sigma APIs. -- `ClassicAlertMgr.fetch` and `ClassicAlertMgr.fetch_bulk` now allow to fetch images directly via the `fetch_images` argument. Defaults to `False`. -- `AnalystNote` model now supports `is_threat_actor` field. - -### Fixed - -- `playbook_alerts.helpers.save_pba_images` now allow for all alert types that support images. -- `IdentityMgr.fetch_incident_report` now support a single string `organization_id` field. - -### Changed - -- `IdentityMgr` methods now support 1000 maximum identities returned instead of 500. - -### Removed - -- Removed support for Python 3.9 - -## v2.5.2 - 2026-04-27 - -### Fixed -- When parsing any Playbook Alert object the `panel_log_v2.[].added.[].url` field no longer fails validation when the URL format does not conform to strict HTTP URL requirements. +- `psengine.links` implements Links search and metadata listing via `LinksMgr`. ## v2.5.1 - 2026-03-26 @@ -61,7 +34,7 @@ ### Fixed -- `PlaybookAlertMgr.fetch_bulk()` now correctly respects the `alerts_per_page` parameter for bulk lookup batching instead of using a hardcoded internal constant. +-`PlaybookAlertMgr.fetch_bulk()` now correctly respects the `alerts_per_page` parameter for bulk lookup batching instead of using a hardcoded internal constant. - `PBA_GeopoliticsFacility` event `url` field no longer fails validation when the URL format does not conform to strict HTTP URL requirements. diff --git a/docs/api/links/errors.md b/docs/api/links/errors.md new file mode 100644 index 00000000..4db337f4 --- /dev/null +++ b/docs/api/links/errors.md @@ -0,0 +1 @@ +::: psengine.links.errors \ No newline at end of file diff --git a/docs/api/links/links_adt.md b/docs/api/links/links_adt.md new file mode 100644 index 00000000..9d39fa9c --- /dev/null +++ b/docs/api/links/links_adt.md @@ -0,0 +1 @@ +::: psengine.links.links diff --git a/docs/api/links/links_mgr.md b/docs/api/links/links_mgr.md new file mode 100644 index 00000000..b287756f --- /dev/null +++ b/docs/api/links/links_mgr.md @@ -0,0 +1,8 @@ +::: psengine.links.links_mgr.LinksMgr + options: + members: + - __init__ + - list_sections + - list_events + - list_entity_types + - search diff --git a/docs/api/links/links_models.md b/docs/api/links/links_models.md new file mode 100644 index 00000000..ba4d9a55 --- /dev/null +++ b/docs/api/links/links_models.md @@ -0,0 +1 @@ +::: psengine.links.models diff --git a/docs/examples/links/__init__.py b/docs/examples/links/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/docs/examples/links/example_1.py b/docs/examples/links/example_1.py new file mode 100644 index 00000000..127d30c8 --- /dev/null +++ b/docs/examples/links/example_1.py @@ -0,0 +1,17 @@ +from psengine.links import LinksMgr + +mgr = LinksMgr() + +results = mgr.search(entities=['QCwdoU']) + +for result in results.data: + if result.error: + print(f'Failed: {result.error.message}') + continue + entity = result.entity + print(f'Entity: {entity.name} ({entity.type_})') + for link in result.links: + print( + f' -> {link.name} ' + f'({link.type_}) source={link.source}' + ) diff --git a/docs/examples/links/example_2.py b/docs/examples/links/example_2.py new file mode 100644 index 00000000..7dcb95f7 --- /dev/null +++ b/docs/examples/links/example_2.py @@ -0,0 +1,25 @@ +from psengine.links import ( + FilterTechnical, + LinksFilterObjects, + LinksLimitsObjects, + LinksMgr, +) + +mgr = LinksMgr() + +filters = LinksFilterObjects( + sources=['technical'], + entity_types=['Malware'], + technical=FilterTechnical(timeframe='-30d'), +) +limits = LinksLimitsObjects( + search_scope='small', per_entity_type=50 +) + +results = mgr.search( + entities=['QCwdoU'], filters=filters, limits=limits +) + +for result in results.data: + for link in result.links: + print(f'{link.name} ({link.type_})') diff --git a/docs/examples/links/example_3.py b/docs/examples/links/example_3.py new file mode 100644 index 00000000..fc017f1d --- /dev/null +++ b/docs/examples/links/example_3.py @@ -0,0 +1,15 @@ +from psengine.links import LinksMgr + +mgr = LinksMgr() + +print('Sections:') +for section in mgr.list_sections(): + print(f' {section.id_}: {section.name}') + +print('\nEvent types:') +for event in mgr.list_events(): + print(f' {event.id_}: {event.name}') + +print('\nEntity types:') +for entity_type in mgr.list_entity_types(): + print(f' {entity_type.id_}: {entity_type.name}') diff --git a/psengine/links/__init__.py b/psengine/links/__init__.py new file mode 100644 index 00000000..e88750cf --- /dev/null +++ b/psengine/links/__init__.py @@ -0,0 +1,36 @@ +##################################### TERMS OF USE ########################################### +# The following code is provided for demonstration purpose only, and should not be used # +# without independent verification. Recorded Future makes no representations or warranties, # +# express, implied, statutory, or otherwise, regarding any aspect of this code or of the # +# information it may retrieve, and provides it both strictly “as-is” and without assuming # +# responsibility for any information it may retrieve. Recorded Future shall not be liable # +# for, and you assume all risk of using, the foregoing. By using this code, Customer # +# represents that it is solely responsible for having all necessary licenses, permissions, # +# rights, and/or consents to connect to third party APIs, and that it is solely responsible # +# for having all necessary licenses, permissions, rights, and/or consents to any data # +# accessed from any third party API. # +############################################################################################## + +from .errors import ( + LinksError, + LinksMetadataError, + LinksSearchError, +) +from .links import ( + FilterTechnical, + LinkedEntity, + LinksFilterObjects, + LinksLimitsObjects, + LinksSearchIn, + LinksSearchResponse, + SearchResultSet, +) +from .links_mgr import LinksMgr +from .models import ( + EntityAttribute, + MetadataEntityTypesResponse, + MetadataEvent, + MetadataEventsResponse, + MetadataSection, + MetadataSectionsResponse, +) diff --git a/psengine/links/errors.py b/psengine/links/errors.py new file mode 100644 index 00000000..2e1b5520 --- /dev/null +++ b/psengine/links/errors.py @@ -0,0 +1,26 @@ +##################################### TERMS OF USE ########################################### +# The following code is provided for demonstration purpose only, and should not be used # +# without independent verification. Recorded Future makes no representations or warranties, # +# express, implied, statutory, or otherwise, regarding any aspect of this code or of the # +# information it may retrieve, and provides it both strictly “as-is” and without assuming # +# responsibility for any information it may retrieve. Recorded Future shall not be liable # +# for, and you assume all risk of using, the foregoing. By using this code, Customer # +# represents that it is solely responsible for having all necessary licenses, permissions, # +# rights, and/or consents to connect to third party APIs, and that it is solely responsible # +# for having all necessary licenses, permissions, rights, and/or consents to any data # +# accessed from any third party API. # +############################################################################################## + +from ..errors import RecordedFutureError + + +class LinksError(RecordedFutureError): + """Base class for all exceptions raised by the Links module.""" + + +class LinksSearchError(LinksError): + """Error raised when a Links search request fails.""" + + +class LinksMetadataError(LinksError): + """Error raised when fetching or validating Links metadata fails.""" diff --git a/psengine/links/links.py b/psengine/links/links.py new file mode 100644 index 00000000..c2449de4 --- /dev/null +++ b/psengine/links/links.py @@ -0,0 +1,83 @@ +##################################### TERMS OF USE ########################################### +# The following code is provided for demonstration purpose only, and should not be used # +# without independent verification. Recorded Future makes no representations or warranties, # +# express, implied, statutory, or otherwise, regarding any aspect of this code or of the # +# information it may retrieve, and provides it both strictly “as-is” and without assuming # +# responsibility for any information it may retrieve. Recorded Future shall not be liable # +# for, and you assume all risk of using, the foregoing. By using this code, Customer # +# represents that it is solely responsible for having all necessary licenses, permissions, # +# rights, and/or consents to connect to third party APIs, and that it is solely responsible # +# for having all necessary licenses, permissions, rights, and/or consents to any data # +# accessed from any third party API. # +############################################################################################## + +from typing import Annotated, Literal, Optional + +from pydantic import AfterValidator + +from ..common_models import IdNameType, RFBaseModel +from ..helpers import TimeHelpers +from .models import EntityAttribute, EntitySearchError + + +def _validate_rel_time(v: Optional[str]) -> Optional[str]: + if v is not None and not TimeHelpers.is_rel_time_valid(v): + raise ValueError(f'Invalid relative time: {v}') + return v + + +class FilterTechnical(RFBaseModel): + """Fields in the Technical Object of Filters.""" + + timeframe: Annotated[Optional[str], AfterValidator(_validate_rel_time)] = None + events: Optional[list[str]] = None + connected_entities: Optional[list[str]] = None + + +class LinksFilterObjects(RFBaseModel): + """Objects in the fields data parameter of links.""" + + sections: Optional[list[str]] = None + entity_types: Optional[list[str]] = None + sources: Optional[list[Literal['technical', 'insikt']]] = None + technical: Optional[FilterTechnical] = None + + +class LinksLimitsObjects(RFBaseModel): + """Objects in the limits object fields.""" + + search_scope: Optional[Literal['small', 'medium', 'large']] = None + per_entity_type: Optional[int] = None + + +class LinksSearchIn(RFBaseModel): + """Model for payload sent to POST `/links/search` endpoint.""" + + entities: list[str] + filters: Optional[LinksFilterObjects] = None + limits: Optional[LinksLimitsObjects] = None + + +class LinkedEntity(IdNameType): + """An entity connected to the search target. + + Inherits id_ (alias 'id'), name, and type_ (alias 'type') from IdNameType. + """ + + source: Optional[str] = None + section: Optional[str] = None + attributes: list[EntityAttribute] = [] + + +class SearchResultSet(RFBaseModel): + """The result set for a single entity that was queried.""" + + entity: Optional[IdNameType] = None + links: list[LinkedEntity] = [] + error: Optional[EntitySearchError] = None + + +class LinksSearchResponse(RFBaseModel): + """Response from POST `/links/search` endpoint.""" + + data: list[SearchResultSet] = [] diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py new file mode 100644 index 00000000..9c40ed65 --- /dev/null +++ b/psengine/links/links_mgr.py @@ -0,0 +1,245 @@ +##################################### TERMS OF USE ########################################### +# The following code is provided for demonstration purpose only, and should not be used # +# without independent verification. Recorded Future makes no representations or warranties, # +# express, implied, statutory, or otherwise, regarding any aspect of this code or of the # +# information it may retrieve, and provides it both strictly “as-is” and without assuming # +# responsibility for any information it may retrieve. Recorded Future shall not be liable # +# for, and you assume all risk of using, the foregoing. By using this code, Customer # +# represents that it is solely responsible for having all necessary licenses, permissions, # +# rights, and/or consents to connect to third party APIs, and that it is solely responsible # +# for having all necessary licenses, permissions, rights, and/or consents to any data # +# accessed from any third party API. # +############################################################################################## + +import logging +from typing import Annotated, Optional + +from pydantic import validate_call +from typing_extensions import Doc + +from ..common_models import IdName +from ..endpoints import ( + EP_LINKS_METADATA_ENTITIES, + EP_LINKS_METADATA_EVENTS, + EP_LINKS_METADATA_SECTIONS, + EP_LINKS_SEARCH, +) +from ..helpers import connection_exceptions, debug_call +from ..rf_client import RFClient +from .errors import LinksMetadataError, LinksSearchError +from .links import LinksFilterObjects, LinksLimitsObjects, LinksSearchIn, LinksSearchResponse +from .models import ( + MetadataEntityTypesResponse, + MetadataEvent, + MetadataEventsResponse, + MetadataSection, + MetadataSectionsResponse, +) + + +class LinksMgr: + """Manager for interacting with the Recorded Future Links API.""" + + def __init__( + self, + rf_token: Annotated[Optional[str], Doc('Recorded Future API token.')] = None, + ): + """Initialize the `LinksMgr` object.""" + self.log = logging.getLogger(__name__) + self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient() + + @debug_call + @validate_call + @connection_exceptions(ignore_status_code=[], exception_to_raise=LinksMetadataError) + def list_sections( + self, + ) -> Annotated[list[MetadataSection], Doc('Section objects with id, name, and description.')]: + """List all sections that can be used to filter a Link search. + + Sections are the high-level categories the Links API groups results into, + for example *Actors, Tools & TTPs* or *Indicators & Detection Rules*. + Use the returned `id_` values to populate `LinksFilterObjects.sections`. + + Endpoint: + `/links/metadata/sections` + + Example: + ```python + from psengine.links import LinksMgr + + mgr = LinksMgr() + for section in mgr.list_sections(): + print(f'{section.id_}: {section.name}') + ``` + + Raises: + ValidationError: If any supplied parameter is of incorrect type. + LinksMetadataError: If an API or connection error occurs. + """ + response = self.rf_client.request(method='GET', url=EP_LINKS_METADATA_SECTIONS) + return MetadataSectionsResponse.model_validate(response.json()).data + + @debug_call + @validate_call + @connection_exceptions(ignore_status_code=[], exception_to_raise=LinksMetadataError) + def list_events( + self, + ) -> Annotated[list[MetadataEvent], Doc('Event objects with id, name, and description.')]: + """List all event types that can be used to filter technical Link searches. + + Event types describe the kind of analytical evidence that produced a + technical link (for example `TTPAnalysis` or `InfrastructureAnalysis`). + Use the returned `id_` values to populate `FilterTechnical.events`. + + Endpoint: + `/links/metadata/events` + + Example: + ```python + from psengine.links import LinksMgr + + mgr = LinksMgr() + for event in mgr.list_events(): + print(f'{event.id_}: {event.name}') + ``` + + Raises: + ValidationError: If any supplied parameter is of incorrect type. + LinksMetadataError: If an API or connection error occurs. + """ + response = self.rf_client.request(method='GET', url=EP_LINKS_METADATA_EVENTS) + return MetadataEventsResponse.model_validate(response.json()).data + + @debug_call + @validate_call + @connection_exceptions(ignore_status_code=[], exception_to_raise=LinksMetadataError) + def list_entity_types( + self, + ) -> Annotated[list[IdName], Doc('Entity-type objects with id and name.')]: + """List all entity types that can be used to filter a Link search. + + The returned values are the supported types for connected entities + (for example `Malware`, `Company`, `IpAddress`). Use the `id_` values + to populate `LinksFilterObjects.entity_types`. + + Endpoint: + `/links/metadata/entities` + + Example: + ```python + from psengine.links import LinksMgr + + mgr = LinksMgr() + for entity_type in mgr.list_entity_types(): + print(f'{entity_type.id_}: {entity_type.name}') + ``` + + Raises: + ValidationError: If any supplied parameter is of incorrect type. + LinksMetadataError: If an API or connection error occurs. + """ + response = self.rf_client.request(method='GET', url=EP_LINKS_METADATA_ENTITIES) + return MetadataEntityTypesResponse.model_validate(response.json()).data + + @debug_call + @validate_call + @connection_exceptions(ignore_status_code=[], exception_to_raise=LinksSearchError) + def search( + self, + entities: Annotated[ + list[str], Doc('List of Recorded Future entity IDs to search for links against.') + ], + filters: Annotated[ + Optional[LinksFilterObjects], Doc('Filter objects for the search.') + ] = None, + limits: Annotated[ + Optional[LinksLimitsObjects], Doc('Limits objects for the search.') + ] = None, + ) -> Annotated[LinksSearchResponse, Doc('The structured search results.')]: + """Search for entities connected to one or more target entities. + + Issues a single batched request: the response contains one + `SearchResultSet` per entity in `entities`, in the same order. If the + API failed for a specific entity, that result's `error` is populated + and `links` is empty — the rest of the batch still succeeds. + + `filters` narrows the result set: + + - `sections` — restrict to specific Links sections (see `list_sections`). + - `entity_types` — restrict to specific connected-entity types (see `list_entity_types`). + - `sources` — restrict to `technical`, `insikt`, or both. + - `technical` — sub-filters that apply only to technical links (timeframe, + event types, connected-entity scope). + + `limits` controls how aggressively the API searches: + + - `search_scope` — one of `small`, `medium`, `large`. Larger scopes scan + more references and Insikt notes per query at the cost of latency. + - `per_entity_type` — caps how many connected entities of each type + are returned. + + Entities must be supplied as Recorded Future entity IDs; if you only have + a name, resolve it with `EntityMatchMgr` or `LookupMgr` first. + + Endpoint: + `/links/search` + + Example: + ```python + from psengine.links import LinksMgr + + mgr = LinksMgr() + results = mgr.search(entities=['QCwdoU']) + for result in results.data: + if result.error: + continue + for link in result.links: + print(f'{link.name} ({link.type_})') + ``` + + With filters and limits: + ```python + from psengine.links import ( + FilterTechnical, + LinksFilterObjects, + LinksLimitsObjects, + LinksMgr, + ) + + mgr = LinksMgr() + filters = LinksFilterObjects( + sources=['technical'], + entity_types=['Malware'], + technical=FilterTechnical(timeframe='-30d'), + ) + limits = LinksLimitsObjects(search_scope='small', per_entity_type=50) + results = mgr.search( + entities=['QCwdoU'], filters=filters, limits=limits + ) + ``` + + If the API failed for a specific entity in the batch, its result looks like: + ```python + SearchResultSet( + entity=IdNameType(id_='QCwdoU', name='...', type_='...'), + links=[], + error=EntitySearchError(message='...', status_code=404), + ) + ``` + + Raises: + ValidationError: If any supplied parameter is of incorrect type. + LinksSearchError: If an API or connection error occurs at the request level. + """ + payload = LinksSearchIn(entities=entities, filters=filters, limits=limits) + + # Kept: @debug_call logs the args list, but batch size is the operationally + # useful number to surface at INFO. Drop if the reviewer disagrees. + self.log.info(f'Executing links search for {len(entities)} entities.') + + response = self.rf_client.request( + method='POST', + url=EP_LINKS_SEARCH, + data=payload.model_dump(exclude_none=True, by_alias=True), + ) + return LinksSearchResponse.model_validate(response.json()) diff --git a/psengine/links/models.py b/psengine/links/models.py new file mode 100644 index 00000000..f0968b08 --- /dev/null +++ b/psengine/links/models.py @@ -0,0 +1,85 @@ +##################################### TERMS OF USE ########################################### +# The following code is provided for demonstration purpose only, and should not be used # +# without independent verification. Recorded Future makes no representations or warranties, # +# express, implied, statutory, or otherwise, regarding any aspect of this code or of the # +# information it may retrieve, and provides it both strictly “as-is” and without assuming # +# responsibility for any information it may retrieve. Recorded Future shall not be liable # +# for, and you assume all risk of using, the foregoing. By using this code, Customer # +# represents that it is solely responsible for having all necessary licenses, permissions, # +# rights, and/or consents to connect to third party APIs, and that it is solely responsible # +# for having all necessary licenses, permissions, rights, and/or consents to any data # +# accessed from any third party API. # +############################################################################################## + +from typing import Any, Literal, Optional + +from pydantic import Field + +from ..common_models import IdName, RFBaseModel + + +class MetadataSection(IdName): + description: Optional[str] = None + + +class MetadataSectionsResponse(RFBaseModel): + """Response from GET `/links/metadata/sections` endpoint.""" + + data: list[MetadataSection] + + +class MetadataEvent(IdName): + description: Optional[str] = None + + +class MetadataEventsResponse(RFBaseModel): + """Response from GET `/links/metadata/events` endpoint.""" + + data: list[MetadataEvent] + + +class MetadataEntityTypesResponse(RFBaseModel): + """Response from GET `/links/metadata/entities` endpoint.""" + + data: list[IdName] + + +class RiskAttribute(RFBaseModel): + id_: Literal['risk_score', 'risk_level'] = Field(alias='id') + value: Optional[float | str] = None + + +class CriticalityAttribute(RFBaseModel): + id_: Literal['criticality'] = Field(alias='id') + value: Optional[str] = None + + +class MitreNameAttribute(RFBaseModel): + id_: Literal['display_name'] = Field(alias='id') + value: Optional[str] = None + + +class ThreatActorAttribute(RFBaseModel): + id_: Literal['threat_actor'] = Field(alias='id') + value: Optional[bool] = None + + +class GenericAttribute(RFBaseModel): + id_: str = Field(alias='id') + value: Any + + +# Discriminated union for the 'attributes' array. Pydantic matches on the +# Literal 'id' values; unknown ids fall back to GenericAttribute. +EntityAttribute = ( + RiskAttribute + | CriticalityAttribute + | MitreNameAttribute + | ThreatActorAttribute + | GenericAttribute +) + + +class EntitySearchError(RFBaseModel): + message: str + status_code: int diff --git a/tests/links/conftest.py b/tests/links/conftest.py new file mode 100644 index 00000000..67f903c3 --- /dev/null +++ b/tests/links/conftest.py @@ -0,0 +1,8 @@ +import pytest + +from psengine.links import LinksMgr + + +@pytest.fixture +def links_mgr(): + return LinksMgr() diff --git a/tests/links/test_links_mgr.py b/tests/links/test_links_mgr.py new file mode 100644 index 00000000..e9b67c61 --- /dev/null +++ b/tests/links/test_links_mgr.py @@ -0,0 +1,194 @@ +##################################### TERMS OF USE ########################################### +# The following code is provided for demonstration purpose only, and should not be used # +# without independent verification. Recorded Future makes no representations or warranties, # +# express, implied, statutory, or otherwise, regarding any aspect of this code or of the # +# information it may retrieve, and provides it both strictly “as-is” and without assuming # +# responsibility for any information it may retrieve. Recorded Future shall not be liable # +# for, and you assume all risk of using, the foregoing. By using this code, Customer # +# represents that it is solely responsible for having all necessary licenses, permissions, # +# rights, and/or consents to connect to third party APIs, and that it is solely responsible # +# for having all necessary licenses, permissions, rights, and/or consents to any data # +# accessed from any third party API. # +############################################################################################## + +import pytest +from pydantic import ValidationError +from requests import Response +from requests.exceptions import HTTPError + +from psengine.links.errors import LinksMetadataError, LinksSearchError +from psengine.links.links import LinksFilterObjects, LinksLimitsObjects +from psengine.links.models import ( + CriticalityAttribute, + GenericAttribute, + MitreNameAttribute, + RiskAttribute, + ThreatActorAttribute, +) + + +def test_list_sections(links_mgr, mocker, make_response): + mock_data = { + 'data': [ + {'id': 's1', 'name': 'Section 1', 'description': 'Desc 1'}, + {'id': 's2', 'name': 'Section 2', 'description': 'Desc 2'}, + ] + } + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) + + sections = links_mgr.list_sections() + assert len(sections) == 2 + assert sections[0].id_ == 's1' + assert sections[1].name == 'Section 2' + + +def test_list_events(links_mgr, mocker, make_response): + mock_data = { + 'data': [ + {'id': 'e1', 'name': 'Event 1', 'description': 'Desc 1'}, + {'id': 'e2', 'name': 'Event 2', 'description': 'Desc 2'}, + ] + } + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) + + events = links_mgr.list_events() + assert len(events) == 2 + assert events[0].id_ == 'e1' + assert events[1].name == 'Event 2' + + +def test_list_entity_types(links_mgr, mocker, make_response): + mock_data = { + 'data': [ + {'id': 'Type1', 'name': 'Type 1'}, + {'id': 'Type2', 'name': 'Type 2'}, + ] + } + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) + + entity_types = links_mgr.list_entity_types() + assert len(entity_types) == 2 + assert entity_types[0].id_ == 'Type1' + assert entity_types[1].name == 'Type 2' + + +def test_search_basic(links_mgr, mocker, make_response): + mock_data = { + 'data': [ + { + 'entity': {'id': 'ent1', 'name': 'Entity 1', 'type': 'Type1'}, + 'links': [ + { + 'id': 'link1', + 'name': 'Link 1', + 'type': 'Type2', + 'source': 'technical', + 'section': 's1', + 'attributes': [{'id': 'risk_score', 'value': 50}], + } + ], + } + ] + } + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) + + results = links_mgr.search(entities=['ent1']) + assert len(results.data) == 1 + assert results.data[0].entity.id_ == 'ent1' + assert len(results.data[0].links) == 1 + assert results.data[0].links[0].id_ == 'link1' + + +def test_filter_objects_invalid_source(): + with pytest.raises(ValidationError, match='sources'): + LinksFilterObjects(sources=['invalid_source']) + + +def test_metadata_error(links_mgr, mocker): + mock_resp = mocker.Mock(spec=Response) + mock_resp.status_code = 500 + mock_resp.text = 'Internal Server Error' + mocker.patch.object( + links_mgr.rf_client, + 'request', + side_effect=HTTPError('500 Server Error', response=mock_resp), + ) + + with pytest.raises(LinksMetadataError, match='500'): + links_mgr.list_sections() + + +def test_search_error(links_mgr, mocker): + mock_resp = mocker.Mock(spec=Response) + mock_resp.status_code = 400 + mock_resp.text = 'Bad Request' + mocker.patch.object( + links_mgr.rf_client, + 'request', + side_effect=HTTPError('400 Client Error', response=mock_resp), + ) + + with pytest.raises(LinksSearchError, match='400'): + links_mgr.search(entities=['ent1']) + + +def test_search_complex_attributes(links_mgr, mocker, make_response): + mock_data = { + 'data': [ + { + 'entity': {'id': 'ent1', 'name': 'Entity 1', 'type': 'Type1'}, + 'links': [ + { + 'id': 'link1', + 'name': 'Link 1', + 'type': 'Type2', + 'attributes': [ + {'id': 'risk_score', 'value': 75.0}, + {'id': 'risk_level', 'value': 'High'}, + {'id': 'criticality', 'value': 'Critical'}, + {'id': 'display_name', 'value': 'T1234'}, + {'id': 'threat_actor', 'value': True}, + {'id': 'unknown_attr', 'value': 'some value'}, + ], + } + ], + } + ] + } + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) + + results = links_mgr.search(entities=['ent1']) + attrs = results.data[0].links[0].attributes + assert len(attrs) == 6 + + assert isinstance(attrs[0], RiskAttribute) + assert attrs[0].id_ == 'risk_score' + assert attrs[0].value == 75.0 + + assert isinstance(attrs[1], RiskAttribute) + assert attrs[1].id_ == 'risk_level' + assert attrs[1].value == 'High' + + assert isinstance(attrs[2], CriticalityAttribute) + assert attrs[2].value == 'Critical' + + assert isinstance(attrs[3], MitreNameAttribute) + assert attrs[3].value == 'T1234' + + assert isinstance(attrs[4], ThreatActorAttribute) + assert attrs[4].value is True + + assert isinstance(attrs[5], GenericAttribute) + assert attrs[5].id_ == 'unknown_attr' + assert attrs[5].value == 'some value' + + +def test_search_with_limits(links_mgr, mocker, make_response): + limits = LinksLimitsObjects(search_scope='small', per_entity_type=10) + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response({'data': []})) + + links_mgr.search(entities=['ent1'], limits=limits) + + _, kwargs = links_mgr.rf_client.request.call_args + assert kwargs['data']['limits']['search_scope'] == 'small' + assert kwargs['data']['limits']['per_entity_type'] == 10 diff --git a/tests/links/test_links_models.py b/tests/links/test_links_models.py new file mode 100644 index 00000000..eafdca0c --- /dev/null +++ b/tests/links/test_links_models.py @@ -0,0 +1,27 @@ +##################################### TERMS OF USE ########################################### +# The following code is provided for demonstration purpose only, and should not be used # +# without independent verification. Recorded Future makes no representations or warranties, # +# express, implied, statutory, or otherwise, regarding any aspect of this code or of the # +# information it may retrieve, and provides it both strictly “as-is” and without assuming # +# responsibility for any information it may retrieve. Recorded Future shall not be liable # +# for, and you assume all risk of using, the foregoing. By using this code, Customer # +# represents that it is solely responsible for having all necessary licenses, permissions, # +# rights, and/or consents to connect to third party APIs, and that it is solely responsible # +# for having all necessary licenses, permissions, rights, and/or consents to any data # +# accessed from any third party API. # +############################################################################################## + +import pytest +from pydantic import ValidationError + +from psengine.links.links import FilterTechnical, LinksSearchIn + + +def test_filter_technical_timeframe_invalid_format(): + with pytest.raises(ValidationError, match='Invalid relative time'): + FilterTechnical(timeframe='not-a-time') + + +def test_links_search_in_entities_required(): + with pytest.raises(ValidationError, match='entities'): + LinksSearchIn() diff --git a/uv.lock b/uv.lock index d0543427..ef9a364d 100644 --- a/uv.lock +++ b/uv.lock @@ -3,7 +3,7 @@ revision = 3 requires-python = ">=3.10, <3.15" [options] -exclude-newer = "2026-04-24T09:50:18.171105Z" +exclude-newer = "2026-05-12T14:28:43.699679Z" exclude-newer-span = "P7D" [[package]] @@ -822,7 +822,7 @@ wheels = [ [[package]] name = "psengine" -version = "2.6.0" +version = "2.7.0" source = { editable = "." } dependencies = [ { name = "jsonpath-ng" }, From d33e08af98ddc0521eddeef70ab5c3d0e9f44557 Mon Sep 17 00:00:00 2001 From: hudson-woomer Date: Tue, 19 May 2026 10:32:39 -0400 Subject: [PATCH 02/24] final update before pr --- psengine/endpoints.py | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/psengine/endpoints.py b/psengine/endpoints.py index b41b51d8..e9a2607c 100644 --- a/psengine/endpoints.py +++ b/psengine/endpoints.py @@ -23,8 +23,10 @@ BASE_URL = 'https://api.recordedfuture.com' CONNECT_API_BASE_URL = BASE_URL + '/' + API_VERSION -BASE_URL = environ.get('RF_BASE_URL') or BASE_URL -CONNECT_API_BASE_URL = environ.get('RF_BASE_URL') or CONNECT_API_BASE_URL +BASE_URL = environ.get('RF_BASE_URL') if environ.get('RF_BASE_URL') else BASE_URL +CONNECT_API_BASE_URL = ( + environ.get('RF_BASE_URL') if environ.get('RF_BASE_URL') else CONNECT_API_BASE_URL +) ############################################################################### # Classic Alerts Endpoints V3 @@ -118,17 +120,6 @@ ############################################################################### EP_MALWARE_INTELLIGENCE = BASE_URL + '/malware-intelligence/v1/' EP_MALWARE_INTEL_REPORTS = EP_MALWARE_INTELLIGENCE + 'reports' -EP_AUTO_YARA = EP_MALWARE_INTELLIGENCE + 'auto-yara/' -EP_AUTO_YARA_JOBS = EP_AUTO_YARA + 'jobs' -EP_AUTO_YARA_JOB_ID = EP_AUTO_YARA_JOBS + '/{}' -EP_AUTO_YARA_JOB_ID_RETRY = EP_AUTO_YARA_JOB_ID + '/retry' -EP_AUTO_YARA_JOBS_EDIT = EP_AUTO_YARA_JOBS + '/edit' -EP_AUTO_SIGMA = EP_MALWARE_INTELLIGENCE + 'auto-sigma/' -EP_AUTO_SIGMA_JOBS = EP_AUTO_SIGMA + 'jobs' -EP_AUTO_SIGMA_GET_JOBS = EP_AUTO_SIGMA + 'get_jobs' -EP_AUTO_SIGMA_JOB_ID = EP_AUTO_SIGMA_JOBS + '/{}' -EP_AUTO_SIGMA_JOB_ID_RULE_ID = EP_AUTO_SIGMA_JOB_ID + '/{}' -EP_AUTO_SIGMA_JOB_ID_RETRY = EP_AUTO_SIGMA_JOB_ID + '/retry' ############################################################################### # Risk History API Endpoints @@ -136,6 +127,16 @@ EP_RISK_HISTORY_BASE = BASE_URL + '/risk' EP_RISK_HISTORY = EP_RISK_HISTORY_BASE + '/history' +############################################################################### +# Links API Endpoints +############################################################################### +LINKS_BASE_URL = f'{BASE_URL}/links' +LINKS_METADATA_URL = f'{LINKS_BASE_URL}/metadata' +EP_LINKS_SEARCH = f'{LINKS_BASE_URL}/search' +EP_LINKS_METADATA_SECTIONS = f'{LINKS_METADATA_URL}/sections' +EP_LINKS_METADATA_EVENTS = f'{LINKS_METADATA_URL}/events' +EP_LINKS_METADATA_ENTITIES = f'{LINKS_METADATA_URL}/entities' + ################################################################################ # Attack Surface Intelligence API Endpoints ################################################################################ @@ -147,13 +148,3 @@ EP_ASI_ASSETS_SEARCH = f'{EP_ASI_ASSETS}/_search' EP_ASI_EXPOSURES = f'{EP_ASI_PROJECTS}/{{}}/exposures' EP_ASI_EXPOSURES_BY_SIGNATURE = f'{EP_ASI_EXPOSURES}/{{}}' - -################################################################################ -# Threat Map API Endpoints -################################################################################ -EP_THREAT_MAPS_BASE = BASE_URL + '/threat' -EP_THREAT_MAPS_LIST = EP_THREAT_MAPS_BASE + '/maps' -EP_THREAT_MAP = EP_THREAT_MAPS_BASE + '/map/{}' -EP_THREAT_MAP_ORG = EP_THREAT_MAPS_BASE + '/map/{}/{}' -EP_ACTOR_SEARCH = EP_THREAT_MAPS_BASE + '/actor/search' -EP_CATEGORIES = EP_THREAT_MAPS_BASE + '/{}/categories' From 3821e8ddbd2ce887636c45a2a86e698614fe6907 Mon Sep 17 00:00:00 2001 From: hudson-woomer Date: Tue, 19 May 2026 10:35:50 -0400 Subject: [PATCH 03/24] make lint fix --- psengine/endpoints.py | 6 ++---- psengine/links/links.py | 34 +++++++++++++++++----------------- psengine/links/links_mgr.py | 8 ++++---- psengine/links/models.py | 14 +++++++------- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/psengine/endpoints.py b/psengine/endpoints.py index e9a2607c..b098b332 100644 --- a/psengine/endpoints.py +++ b/psengine/endpoints.py @@ -23,10 +23,8 @@ BASE_URL = 'https://api.recordedfuture.com' CONNECT_API_BASE_URL = BASE_URL + '/' + API_VERSION -BASE_URL = environ.get('RF_BASE_URL') if environ.get('RF_BASE_URL') else BASE_URL -CONNECT_API_BASE_URL = ( - environ.get('RF_BASE_URL') if environ.get('RF_BASE_URL') else CONNECT_API_BASE_URL -) +BASE_URL = environ.get('RF_BASE_URL') or BASE_URL +CONNECT_API_BASE_URL = environ.get('RF_BASE_URL') or CONNECT_API_BASE_URL ############################################################################### # Classic Alerts Endpoints V3 diff --git a/psengine/links/links.py b/psengine/links/links.py index c2449de4..79ab88ed 100644 --- a/psengine/links/links.py +++ b/psengine/links/links.py @@ -11,7 +11,7 @@ # accessed from any third party API. # ############################################################################################## -from typing import Annotated, Literal, Optional +from typing import Annotated, Literal from pydantic import AfterValidator @@ -20,7 +20,7 @@ from .models import EntityAttribute, EntitySearchError -def _validate_rel_time(v: Optional[str]) -> Optional[str]: +def _validate_rel_time(v: str | None) -> str | None: if v is not None and not TimeHelpers.is_rel_time_valid(v): raise ValueError(f'Invalid relative time: {v}') return v @@ -29,33 +29,33 @@ def _validate_rel_time(v: Optional[str]) -> Optional[str]: class FilterTechnical(RFBaseModel): """Fields in the Technical Object of Filters.""" - timeframe: Annotated[Optional[str], AfterValidator(_validate_rel_time)] = None - events: Optional[list[str]] = None - connected_entities: Optional[list[str]] = None + timeframe: Annotated[str | None, AfterValidator(_validate_rel_time)] = None + events: list[str] | None = None + connected_entities: list[str] | None = None class LinksFilterObjects(RFBaseModel): """Objects in the fields data parameter of links.""" - sections: Optional[list[str]] = None - entity_types: Optional[list[str]] = None - sources: Optional[list[Literal['technical', 'insikt']]] = None - technical: Optional[FilterTechnical] = None + sections: list[str] | None = None + entity_types: list[str] | None = None + sources: list[Literal['technical', 'insikt']] | None = None + technical: FilterTechnical | None = None class LinksLimitsObjects(RFBaseModel): """Objects in the limits object fields.""" - search_scope: Optional[Literal['small', 'medium', 'large']] = None - per_entity_type: Optional[int] = None + search_scope: Literal['small', 'medium', 'large'] | None = None + per_entity_type: int | None = None class LinksSearchIn(RFBaseModel): """Model for payload sent to POST `/links/search` endpoint.""" entities: list[str] - filters: Optional[LinksFilterObjects] = None - limits: Optional[LinksLimitsObjects] = None + filters: LinksFilterObjects | None = None + limits: LinksLimitsObjects | None = None class LinkedEntity(IdNameType): @@ -64,17 +64,17 @@ class LinkedEntity(IdNameType): Inherits id_ (alias 'id'), name, and type_ (alias 'type') from IdNameType. """ - source: Optional[str] = None - section: Optional[str] = None + source: str | None = None + section: str | None = None attributes: list[EntityAttribute] = [] class SearchResultSet(RFBaseModel): """The result set for a single entity that was queried.""" - entity: Optional[IdNameType] = None + entity: IdNameType | None = None links: list[LinkedEntity] = [] - error: Optional[EntitySearchError] = None + error: EntitySearchError | None = None class LinksSearchResponse(RFBaseModel): diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index 9c40ed65..1699bf95 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -12,7 +12,7 @@ ############################################################################################## import logging -from typing import Annotated, Optional +from typing import Annotated from pydantic import validate_call from typing_extensions import Doc @@ -42,7 +42,7 @@ class LinksMgr: def __init__( self, - rf_token: Annotated[Optional[str], Doc('Recorded Future API token.')] = None, + rf_token: Annotated[str | None, Doc('Recorded Future API token.')] = None, ): """Initialize the `LinksMgr` object.""" self.log = logging.getLogger(__name__) @@ -150,10 +150,10 @@ def search( list[str], Doc('List of Recorded Future entity IDs to search for links against.') ], filters: Annotated[ - Optional[LinksFilterObjects], Doc('Filter objects for the search.') + LinksFilterObjects | None, Doc('Filter objects for the search.') ] = None, limits: Annotated[ - Optional[LinksLimitsObjects], Doc('Limits objects for the search.') + LinksLimitsObjects | None, Doc('Limits objects for the search.') ] = None, ) -> Annotated[LinksSearchResponse, Doc('The structured search results.')]: """Search for entities connected to one or more target entities. diff --git a/psengine/links/models.py b/psengine/links/models.py index f0968b08..d5add803 100644 --- a/psengine/links/models.py +++ b/psengine/links/models.py @@ -11,7 +11,7 @@ # accessed from any third party API. # ############################################################################################## -from typing import Any, Literal, Optional +from typing import Any, Literal from pydantic import Field @@ -19,7 +19,7 @@ class MetadataSection(IdName): - description: Optional[str] = None + description: str | None = None class MetadataSectionsResponse(RFBaseModel): @@ -29,7 +29,7 @@ class MetadataSectionsResponse(RFBaseModel): class MetadataEvent(IdName): - description: Optional[str] = None + description: str | None = None class MetadataEventsResponse(RFBaseModel): @@ -46,22 +46,22 @@ class MetadataEntityTypesResponse(RFBaseModel): class RiskAttribute(RFBaseModel): id_: Literal['risk_score', 'risk_level'] = Field(alias='id') - value: Optional[float | str] = None + value: float | str | None = None class CriticalityAttribute(RFBaseModel): id_: Literal['criticality'] = Field(alias='id') - value: Optional[str] = None + value: str | None = None class MitreNameAttribute(RFBaseModel): id_: Literal['display_name'] = Field(alias='id') - value: Optional[str] = None + value: str | None = None class ThreatActorAttribute(RFBaseModel): id_: Literal['threat_actor'] = Field(alias='id') - value: Optional[bool] = None + value: bool | None = None class GenericAttribute(RFBaseModel): From 8c8d15a9ee06955d3d5a240df010aa8aedff82a0 Mon Sep 17 00:00:00 2001 From: hudson-woomer Date: Wed, 20 May 2026 09:04:54 -0400 Subject: [PATCH 04/24] format gh action complete test --- .gitignore | 1 + psengine/links/links_mgr.py | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 45c7508a..89bc3f3d 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ bundles/ site/ +.idea/ \ No newline at end of file diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index 1699bf95..779e24e6 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -149,12 +149,8 @@ def search( entities: Annotated[ list[str], Doc('List of Recorded Future entity IDs to search for links against.') ], - filters: Annotated[ - LinksFilterObjects | None, Doc('Filter objects for the search.') - ] = None, - limits: Annotated[ - LinksLimitsObjects | None, Doc('Limits objects for the search.') - ] = None, + filters: Annotated[LinksFilterObjects | None, Doc('Filter objects for the search.')] = None, + limits: Annotated[LinksLimitsObjects | None, Doc('Limits objects for the search.')] = None, ) -> Annotated[LinksSearchResponse, Doc('The structured search results.')]: """Search for entities connected to one or more target entities. From 36df12a6073c878f7ca5896aa5aa99f56286aab1 Mon Sep 17 00:00:00 2001 From: hudson-woomer Date: Wed, 20 May 2026 09:17:22 -0400 Subject: [PATCH 05/24] unittest gh action fix --- psengine/endpoints.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/psengine/endpoints.py b/psengine/endpoints.py index b098b332..85287868 100644 --- a/psengine/endpoints.py +++ b/psengine/endpoints.py @@ -118,6 +118,17 @@ ############################################################################### EP_MALWARE_INTELLIGENCE = BASE_URL + '/malware-intelligence/v1/' EP_MALWARE_INTEL_REPORTS = EP_MALWARE_INTELLIGENCE + 'reports' +EP_AUTO_YARA = EP_MALWARE_INTELLIGENCE + 'auto-yara/' +EP_AUTO_YARA_JOBS = EP_AUTO_YARA + 'jobs' +EP_AUTO_YARA_JOB_ID = EP_AUTO_YARA_JOBS + '/{}' +EP_AUTO_YARA_JOB_ID_RETRY = EP_AUTO_YARA_JOB_ID + '/retry' +EP_AUTO_YARA_JOBS_EDIT = EP_AUTO_YARA_JOBS + '/edit' +EP_AUTO_SIGMA = EP_MALWARE_INTELLIGENCE + 'auto-sigma/' +EP_AUTO_SIGMA_JOBS = EP_AUTO_SIGMA + 'jobs' +EP_AUTO_SIGMA_GET_JOBS = EP_AUTO_SIGMA + 'get_jobs' +EP_AUTO_SIGMA_JOB_ID = EP_AUTO_SIGMA_JOBS + '/{}' +EP_AUTO_SIGMA_JOB_ID_RULE_ID = EP_AUTO_SIGMA_JOB_ID + '/{}' +EP_AUTO_SIGMA_JOB_ID_RETRY = EP_AUTO_SIGMA_JOB_ID + '/retry' ############################################################################### # Risk History API Endpoints @@ -146,3 +157,13 @@ EP_ASI_ASSETS_SEARCH = f'{EP_ASI_ASSETS}/_search' EP_ASI_EXPOSURES = f'{EP_ASI_PROJECTS}/{{}}/exposures' EP_ASI_EXPOSURES_BY_SIGNATURE = f'{EP_ASI_EXPOSURES}/{{}}' + +################################################################################ +# Threat Map API Endpoints +################################################################################ +EP_THREAT_MAPS_BASE = BASE_URL + '/threat' +EP_THREAT_MAPS_LIST = EP_THREAT_MAPS_BASE + '/maps' +EP_THREAT_MAP = EP_THREAT_MAPS_BASE + '/map/{}' +EP_THREAT_MAP_ORG = EP_THREAT_MAPS_BASE + '/map/{}/{}' +EP_ACTOR_SEARCH = EP_THREAT_MAPS_BASE + '/actor/search' +EP_CATEGORIES = EP_THREAT_MAPS_BASE + '/{}/categories' From e4656ac1a9afd3f713e6e3c1974cee35b1f7cb51 Mon Sep 17 00:00:00 2001 From: hudson-woomer Date: Wed, 20 May 2026 09:21:03 -0400 Subject: [PATCH 06/24] removed ide files --- .idea/.gitignore | 10 ---------- .idea/inspectionProfiles/profiles_settings.xml | 6 ------ .idea/misc.xml | 6 ------ .idea/modules.xml | 8 -------- .idea/psengine.iml | 18 ------------------ .idea/vcs.xml | 6 ------ 6 files changed, 54 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/psengine.iml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 30cf57ed..00000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Ignored default folder with query files -/queries/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2da..00000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 87fefa91..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 149097e7..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/psengine.iml b/.idea/psengine.iml deleted file mode 100644 index 2e5c1f31..00000000 --- a/.idea/psengine.iml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddf..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 8bfed5aebb0f048b487db156d647c4a62b0c964e Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 10:38:51 +0100 Subject: [PATCH 07/24] missing docs, fix changelog and started review models --- docs/CHANGELOG.md | 39 ++++++++++++-- docs/api/links/{links_adt.md => links.md} | 0 docs/api/links/{links_models.md => models.md} | 0 docs/examples/links/example_1.py | 11 ++-- docs/modules/links.md | 53 +++++++++++++++++++ mkdocs.yml | 7 +++ psengine/links/__init__.py | 6 +-- psengine/links/links_mgr.py | 51 +++--------------- psengine/links/models.py | 28 +++------- 9 files changed, 117 insertions(+), 78 deletions(-) rename docs/api/links/{links_adt.md => links.md} (100%) rename docs/api/links/{links_models.md => models.md} (100%) create mode 100644 docs/modules/links.md diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0bc42677..4156d32a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,12 +1,43 @@ # Changelog -## v2.6.1 - 2026-05-08 +## v2.8.0 - 2026-05-22 -- `psengine.links` now supports the Links API via the `LinksMgr` class. +- Added support for Links API via the `LinksMgr`. + +## v2.7.0 - 2026-05-01 + +- Added support for Threat Map and Malware Map via the `ThreatMapMgr`. + +## v2.6.0 - 2026-04-29 ### Added -- `psengine.links` implements Links search and metadata listing via `LinksMgr`. +- Added support for Python 3.14 +- `PlaybookAlertMgr.search` now supports filter for a single or a list of organisations. +- `TimeHelpers.rel_time_to_date` now supports a starting time from where the math begins. If not specified it will be the UTC execution time. +- `TimeHelpers.rel_time_to_date` now supports increment calculation, with `+1h`. +- `TimeHelpers.rel_time_to_date` now supports increment or decrement of minutes with `+10m` or `-10m`. +- Added `malware_intel.AutoYaraMgr` and `malware_intel.AutoSigmaMgr` managers to interact with auto-yara and auto-sigma APIs. +- `ClassicAlertMgr.fetch` and `ClassicAlertMgr.fetch_bulk` now allow to fetch images directly via the `fetch_images` argument. Defaults to `False`. +- `AnalystNote` model now supports `is_threat_actor` field. + +### Fixed + +- `playbook_alerts.helpers.save_pba_images` now allow for all alert types that support images. +- `IdentityMgr.fetch_incident_report` now support a single string `organization_id` field. + +### Changed + +- `IdentityMgr` methods now support 1000 maximum identities returned instead of 500. + +### Removed + +- Removed support for Python 3.9 + +## v2.5.2 - 2026-04-27 + +### Fixed +- When parsing any Playbook Alert object the `panel_log_v2.[].added.[].url` field no longer fails validation when the URL format does not conform to strict HTTP URL requirements. ## v2.5.1 - 2026-03-26 @@ -34,7 +65,7 @@ ### Fixed --`PlaybookAlertMgr.fetch_bulk()` now correctly respects the `alerts_per_page` parameter for bulk lookup batching instead of using a hardcoded internal constant. +- `PlaybookAlertMgr.fetch_bulk()` now correctly respects the `alerts_per_page` parameter for bulk lookup batching instead of using a hardcoded internal constant. - `PBA_GeopoliticsFacility` event `url` field no longer fails validation when the URL format does not conform to strict HTTP URL requirements. diff --git a/docs/api/links/links_adt.md b/docs/api/links/links.md similarity index 100% rename from docs/api/links/links_adt.md rename to docs/api/links/links.md diff --git a/docs/api/links/links_models.md b/docs/api/links/models.md similarity index 100% rename from docs/api/links/links_models.md rename to docs/api/links/models.md diff --git a/docs/examples/links/example_1.py b/docs/examples/links/example_1.py index 127d30c8..0caf8d1c 100644 --- a/docs/examples/links/example_1.py +++ b/docs/examples/links/example_1.py @@ -8,10 +8,9 @@ if result.error: print(f'Failed: {result.error.message}') continue + entity = result.entity - print(f'Entity: {entity.name} ({entity.type_})') - for link in result.links: - print( - f' -> {link.name} ' - f'({link.type_}) source={link.source}' - ) + print(f'Entity: {entity.name}') + + for link in result.links[:5]: + print(f' -> {link.name} source:{link.source}') diff --git a/docs/modules/links.md b/docs/modules/links.md new file mode 100644 index 00000000..aafcc21a --- /dev/null +++ b/docs/modules/links.md @@ -0,0 +1,53 @@ +## Introduction + +The `LinksMgr` class of the `links` module allows you to find entities connected to one or more Recorded Future entities. + +See the [**API Reference**](../api/links/links_mgr.md) for internal details of the module. + +## Notes + +The `search` method expects Recorded Future entity IDs (for example, `QCwdoU`), not entity names. If you only have a name, use the `entity_match` module first to resolve the ID. + +The response is batched: you get one result per input entity. A specific entity can fail while others succeed, so always check `result.error` before iterating over `result.links`. + +For filters such as sections, events, and entity types, use the metadata methods (`list_sections`, `list_events`, and `list_entity_types`) to retrieve valid IDs before calling `search`. + +## Examples + +{! modules/_includes/examples_warning.md !} + +#### 1: Search links for an entity and handle per-entity errors + +In this example, we call `search` with a single entity ID. For each result, we first check `result.error`. If there is no error, we print the source entity and the first 5 linked entities returned by the API. + +```python +--8<-- "docs/examples/links/example_1.py" +``` + +The output will be: + +``` +Entity: Lazarus Group + -> CVE-2022-47966 source:insikt + -> 24988feb1b38f400069acec4514aa4deea3f6ca8ceb5296f54926e2b22af1e5a source:insikt + -> 36db27f5eb3343cfc72d261d78da44957a49cb6731acb50a96ea5694f4d616c5 source:insikt + -> ffec6e6d4e314f64f5d31c62024252abde7f77acdd63991cb16923ff17828885 source:insikt + -> 3e5fd9acdab438ffc8b2cce48c91679d3f980d08f9dea47d5e1039d352cd64fb source:insikt +``` + + +#### 2: Filter link results and apply limits + +In this example, we use `LinksFilterObjects` and `LinksLimitsObjects` to narrow results to technical malware links seen in the last 30 days, and to cap result size per entity type. + +```python +--8<-- "docs/examples/links/example_2.py" +``` + +#### 3: Discover available metadata for filters + +In this example, we list valid sections, event types, and entity types. These IDs can be reused in `LinksFilterObjects` and `FilterTechnical` when building search queries. + +```python +--8<-- "docs/examples/links/example_3.py" +``` diff --git a/mkdocs.yml b/mkdocs.yml index 7d2f6f74..305239ed 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -101,6 +101,7 @@ nav: - Entity Match: modules/entity_match.md - Fusion: modules/fusion.md - Identity: modules/identity.md + - Links: modules/links.md - Malware Intelligence: modules/malware_intel.md - Playbook Alerts: modules/playbook_alerts.md - Risk History: modules/risk_history.md @@ -191,6 +192,12 @@ nav: - ADT: api/identity/identity.md - Constants: api/identity/constants.md - Errors: api/identity/errors.md + - Links: + - Manager: api/links/links_mgr.md + - ADT: api/links/links.md + - Models: api/links/models.md + - Constants: api/links/constants.md + - Errors: api/links/errors.md - Logger: - Logger: api/logger/rf_logger.md - Constants: api/logger/constants.md diff --git a/psengine/links/__init__.py b/psengine/links/__init__.py index e88750cf..55ac246a 100644 --- a/psengine/links/__init__.py +++ b/psengine/links/__init__.py @@ -28,9 +28,5 @@ from .links_mgr import LinksMgr from .models import ( EntityAttribute, - MetadataEntityTypesResponse, - MetadataEvent, - MetadataEventsResponse, - MetadataSection, - MetadataSectionsResponse, + MetadataOut, ) diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index 779e24e6..ced67ccc 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -17,7 +17,6 @@ from pydantic import validate_call from typing_extensions import Doc -from ..common_models import IdName from ..endpoints import ( EP_LINKS_METADATA_ENTITIES, EP_LINKS_METADATA_EVENTS, @@ -29,11 +28,7 @@ from .errors import LinksMetadataError, LinksSearchError from .links import LinksFilterObjects, LinksLimitsObjects, LinksSearchIn, LinksSearchResponse from .models import ( - MetadataEntityTypesResponse, - MetadataEvent, - MetadataEventsResponse, - MetadataSection, - MetadataSectionsResponse, + MetadataOut, ) @@ -53,93 +48,63 @@ def __init__( @connection_exceptions(ignore_status_code=[], exception_to_raise=LinksMetadataError) def list_sections( self, - ) -> Annotated[list[MetadataSection], Doc('Section objects with id, name, and description.')]: + ) -> Annotated[MetadataOut, Doc('Section objects with id, name, and description.')]: """List all sections that can be used to filter a Link search. Sections are the high-level categories the Links API groups results into, for example *Actors, Tools & TTPs* or *Indicators & Detection Rules*. - Use the returned `id_` values to populate `LinksFilterObjects.sections`. Endpoint: `/links/metadata/sections` - Example: - ```python - from psengine.links import LinksMgr - - mgr = LinksMgr() - for section in mgr.list_sections(): - print(f'{section.id_}: {section.name}') - ``` - Raises: ValidationError: If any supplied parameter is of incorrect type. LinksMetadataError: If an API or connection error occurs. """ response = self.rf_client.request(method='GET', url=EP_LINKS_METADATA_SECTIONS) - return MetadataSectionsResponse.model_validate(response.json()).data + return MetadataOut.model_validate(response.json()).data @debug_call @validate_call @connection_exceptions(ignore_status_code=[], exception_to_raise=LinksMetadataError) def list_events( self, - ) -> Annotated[list[MetadataEvent], Doc('Event objects with id, name, and description.')]: + ) -> Annotated[MetadataOut, Doc('Event objects with id, name, and description.')]: """List all event types that can be used to filter technical Link searches. Event types describe the kind of analytical evidence that produced a technical link (for example `TTPAnalysis` or `InfrastructureAnalysis`). - Use the returned `id_` values to populate `FilterTechnical.events`. Endpoint: `/links/metadata/events` - Example: - ```python - from psengine.links import LinksMgr - - mgr = LinksMgr() - for event in mgr.list_events(): - print(f'{event.id_}: {event.name}') - ``` - Raises: ValidationError: If any supplied parameter is of incorrect type. LinksMetadataError: If an API or connection error occurs. """ response = self.rf_client.request(method='GET', url=EP_LINKS_METADATA_EVENTS) - return MetadataEventsResponse.model_validate(response.json()).data + return MetadataOut.model_validate(response.json()).data @debug_call @validate_call @connection_exceptions(ignore_status_code=[], exception_to_raise=LinksMetadataError) def list_entity_types( self, - ) -> Annotated[list[IdName], Doc('Entity-type objects with id and name.')]: + ) -> Annotated[MetadataOut, Doc('Entity-type objects with id and name.')]: """List all entity types that can be used to filter a Link search. The returned values are the supported types for connected entities - (for example `Malware`, `Company`, `IpAddress`). Use the `id_` values - to populate `LinksFilterObjects.entity_types`. + (for example `Malware`, `Company`, `IpAddress`). Endpoint: `/links/metadata/entities` - Example: - ```python - from psengine.links import LinksMgr - - mgr = LinksMgr() - for entity_type in mgr.list_entity_types(): - print(f'{entity_type.id_}: {entity_type.name}') - ``` - Raises: ValidationError: If any supplied parameter is of incorrect type. LinksMetadataError: If an API or connection error occurs. """ response = self.rf_client.request(method='GET', url=EP_LINKS_METADATA_ENTITIES) - return MetadataEntityTypesResponse.model_validate(response.json()).data + return MetadataOut.model_validate(response.json()).data @debug_call @validate_call diff --git a/psengine/links/models.py b/psengine/links/models.py index d5add803..03da18b2 100644 --- a/psengine/links/models.py +++ b/psengine/links/models.py @@ -18,30 +18,18 @@ from ..common_models import IdName, RFBaseModel -class MetadataSection(IdName): +class Metadata(IdName): description: str | None = None -class MetadataSectionsResponse(RFBaseModel): - """Response from GET `/links/metadata/sections` endpoint.""" +class MetadataOut(RFBaseModel): + """Response for endpoints: + - `/links/metadata/sections` + - `/links/metadata/events` + - `/links/metadata/entities`. + """ - data: list[MetadataSection] - - -class MetadataEvent(IdName): - description: str | None = None - - -class MetadataEventsResponse(RFBaseModel): - """Response from GET `/links/metadata/events` endpoint.""" - - data: list[MetadataEvent] - - -class MetadataEntityTypesResponse(RFBaseModel): - """Response from GET `/links/metadata/entities` endpoint.""" - - data: list[IdName] + data: list[Metadata] class RiskAttribute(RFBaseModel): From 960610ee26b9e046f39ac500c8f895f2aa276e40 Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 11:27:27 +0100 Subject: [PATCH 08/24] changes in models --- psengine/helpers/helpers.py | 9 +++ psengine/links/__init__.py | 4 -- psengine/links/links.py | 42 ------------ psengine/links/links_mgr.py | 112 ++++++++++++++++++------------- psengine/links/models.py | 49 +++++++++++++- tests/links/test_links_mgr.py | 6 +- tests/links/test_links_models.py | 7 +- 7 files changed, 124 insertions(+), 105 deletions(-) diff --git a/psengine/helpers/helpers.py b/psengine/helpers/helpers.py index 7f180974..ee9d9b13 100644 --- a/psengine/helpers/helpers.py +++ b/psengine/helpers/helpers.py @@ -504,6 +504,15 @@ def convert_relative_time( else input_time ) + @staticmethod + def is_rel_time_valid( + input_time: Annotated[str | None, Doc("Relative time string, e.g., '7d', '3h'.")], + ): + """Check that a relative time like `-3d` is valid.""" + if input_time is not None and not TimeHelpers.is_rel_time_valid(input_time): + raise ValueError(f'Invalid relative time: {input_time}') + return input_time + @staticmethod def check_uhash_prefix( value: Annotated[str | list, Doc('String or list of strings to check for uhash prefix.')], diff --git a/psengine/links/__init__.py b/psengine/links/__init__.py index 55ac246a..0bfae9e9 100644 --- a/psengine/links/__init__.py +++ b/psengine/links/__init__.py @@ -17,11 +17,7 @@ LinksSearchError, ) from .links import ( - FilterTechnical, LinkedEntity, - LinksFilterObjects, - LinksLimitsObjects, - LinksSearchIn, LinksSearchResponse, SearchResultSet, ) diff --git a/psengine/links/links.py b/psengine/links/links.py index 79ab88ed..d022192e 100644 --- a/psengine/links/links.py +++ b/psengine/links/links.py @@ -11,53 +11,11 @@ # accessed from any third party API. # ############################################################################################## -from typing import Annotated, Literal - -from pydantic import AfterValidator from ..common_models import IdNameType, RFBaseModel -from ..helpers import TimeHelpers from .models import EntityAttribute, EntitySearchError -def _validate_rel_time(v: str | None) -> str | None: - if v is not None and not TimeHelpers.is_rel_time_valid(v): - raise ValueError(f'Invalid relative time: {v}') - return v - - -class FilterTechnical(RFBaseModel): - """Fields in the Technical Object of Filters.""" - - timeframe: Annotated[str | None, AfterValidator(_validate_rel_time)] = None - events: list[str] | None = None - connected_entities: list[str] | None = None - - -class LinksFilterObjects(RFBaseModel): - """Objects in the fields data parameter of links.""" - - sections: list[str] | None = None - entity_types: list[str] | None = None - sources: list[Literal['technical', 'insikt']] | None = None - technical: FilterTechnical | None = None - - -class LinksLimitsObjects(RFBaseModel): - """Objects in the limits object fields.""" - - search_scope: Literal['small', 'medium', 'large'] | None = None - per_entity_type: int | None = None - - -class LinksSearchIn(RFBaseModel): - """Model for payload sent to POST `/links/search` endpoint.""" - - entities: list[str] - filters: LinksFilterObjects | None = None - limits: LinksLimitsObjects | None = None - - class LinkedEntity(IdNameType): """An entity connected to the search target. diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index ced67ccc..4f6ad470 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -14,9 +14,11 @@ import logging from typing import Annotated -from pydantic import validate_call +from pydantic import AfterValidator, BeforeValidator, validate_call from typing_extensions import Doc +from psengine.helpers.helpers import Validators + from ..endpoints import ( EP_LINKS_METADATA_ENTITIES, EP_LINKS_METADATA_EVENTS, @@ -26,9 +28,19 @@ from ..helpers import connection_exceptions, debug_call from ..rf_client import RFClient from .errors import LinksMetadataError, LinksSearchError -from .links import LinksFilterObjects, LinksLimitsObjects, LinksSearchIn, LinksSearchResponse from .models import ( + FilterTechnical, + LinksFilterObjects, + LinksLimitsObjects, +) + +from .links import ( + LinksSearchResponse, +) +from .models import ( + LinkSource, MetadataOut, + SearchScope, ) @@ -112,11 +124,42 @@ def list_entity_types( def search( self, entities: Annotated[ - list[str], Doc('List of Recorded Future entity IDs to search for links against.') + Annotated[str | list[str], AfterValidator(Validators.convert_str_to_list)], + Doc('One or more Recorded Future entity IDs to look up links for.'), ], - filters: Annotated[LinksFilterObjects | None, Doc('Filter objects for the search.')] = None, - limits: Annotated[LinksLimitsObjects | None, Doc('Limits objects for the search.')] = None, - ) -> Annotated[LinksSearchResponse, Doc('The structured search results.')]: + sections: Annotated[ + str | list[str] | None, Doc('Filter results to these link section IDs.') + ] = None, + entity_types: Annotated[ + str | list[str] | None, + Doc('Restrict linked entities to these entity types (e.g. "type:IpAddress").'), + ] = None, + sources: Annotated[ + list[LinkSource] | None, + Doc('Limit to source type(s): technical, insikt, or both.'), + ] = None, + timeframe: Annotated[ + str | None, + Doc('Technical-link timeframe (e.g. "-30d", default "-30d", max "-90d").'), + ] = None, + events: Annotated[ + str | list[str] | None, + Doc('Restrict technical links to these event types (e.g. "type:MalwareAnalysis").'), + ] = None, + connected_entities: Annotated[ + str | list[str] | None, + Doc('Only return technical links that connect to these entities.'), + ] = None, + search_scope: Annotated[ + SearchScope | None, + Doc('Result-volume scope: small, medium (default), or large.'), + ] = None, + per_entity_type: Annotated[ + int | None, Doc('Max linked entities returned per entity type.') + ] = None, + ) -> Annotated[ + list[LinksSearchResponse], Doc('A list of LinkResult models — one per input entity.') + ]: """Search for entities connected to one or more target entities. Issues a single batched request: the response contains one @@ -145,40 +188,6 @@ def search( Endpoint: `/links/search` - Example: - ```python - from psengine.links import LinksMgr - - mgr = LinksMgr() - results = mgr.search(entities=['QCwdoU']) - for result in results.data: - if result.error: - continue - for link in result.links: - print(f'{link.name} ({link.type_})') - ``` - - With filters and limits: - ```python - from psengine.links import ( - FilterTechnical, - LinksFilterObjects, - LinksLimitsObjects, - LinksMgr, - ) - - mgr = LinksMgr() - filters = LinksFilterObjects( - sources=['technical'], - entity_types=['Malware'], - technical=FilterTechnical(timeframe='-30d'), - ) - limits = LinksLimitsObjects(search_scope='small', per_entity_type=50) - results = mgr.search( - entities=['QCwdoU'], filters=filters, limits=limits - ) - ``` - If the API failed for a specific entity in the batch, its result looks like: ```python SearchResultSet( @@ -192,15 +201,22 @@ def search( ValidationError: If any supplied parameter is of incorrect type. LinksSearchError: If an API or connection error occurs at the request level. """ - payload = LinksSearchIn(entities=entities, filters=filters, limits=limits) + technical_filter = FilterTechnical( + timeframe=timeframe, events=events, connected_entities=connected_entities + ).json() + + filters = LinksFilterObjects( + sections=sections, + entity_types=entity_types, + sources=sources, + technical=technical_filter, + ).json() + limits = LinksLimitsObjects( + search_scope=search_scope, per_entity_type=per_entity_type + ).json() + payload = {'entities': entities, 'filters': filters, 'limits': limits} - # Kept: @debug_call logs the args list, but batch size is the operationally - # useful number to surface at INFO. Drop if the reviewer disagrees. self.log.info(f'Executing links search for {len(entities)} entities.') - response = self.rf_client.request( - method='POST', - url=EP_LINKS_SEARCH, - data=payload.model_dump(exclude_none=True, by_alias=True), - ) + response = self.rf_client.request(method='POST', url=EP_LINKS_SEARCH, data=payload) return LinksSearchResponse.model_validate(response.json()) diff --git a/psengine/links/models.py b/psengine/links/models.py index 03da18b2..1ec88a6e 100644 --- a/psengine/links/models.py +++ b/psengine/links/models.py @@ -11,11 +11,13 @@ # accessed from any third party API. # ############################################################################################## -from typing import Any, Literal +from enum import Enum +from typing import Annotated, Any, Literal -from pydantic import Field +from pydantic import AfterValidator, BeforeValidator, Field from ..common_models import IdName, RFBaseModel +from ..helpers.helpers import Validators class Metadata(IdName): @@ -71,3 +73,46 @@ class GenericAttribute(RFBaseModel): class EntitySearchError(RFBaseModel): message: str status_code: int + + +class LinkSource(Enum): + """RF Links API source filter.""" + + technical = 'technical' + insikt = 'insikt' + + +class SearchScope(Enum): + """RF Links API search scope.""" + + small = 'small' + medium = 'medium' + large = 'large' + + +class FilterTechnical(RFBaseModel): + """Fields in the Technical Object of Filters.""" + + timeframe: Annotated[str | None, AfterValidator(Validators.is_rel_time_valid)] = None + events: Annotated[list[str] | None, BeforeValidator(Validators.convert_str_to_list)] = None + connected_entities: Annotated[ + list[str] | None, BeforeValidator(Validators.convert_str_to_list) + ] = None + + +class LinksFilterObjects(RFBaseModel): + """Objects in the fields data parameter of links.""" + + sections: Annotated[list[str] | None, BeforeValidator(Validators.convert_str_to_list)] = None + entity_types: Annotated[list[str] | None, BeforeValidator(Validators.convert_str_to_list)] = ( + None + ) + sources: list[Literal['technical', 'insikt']] | None = None + technical: FilterTechnical | None = None + + +class LinksLimitsObjects(RFBaseModel): + """Objects in the limits object fields.""" + + search_scope: SearchScope | None = None + per_entity_type: int | None = None diff --git a/tests/links/test_links_mgr.py b/tests/links/test_links_mgr.py index e9b67c61..70a2e610 100644 --- a/tests/links/test_links_mgr.py +++ b/tests/links/test_links_mgr.py @@ -17,10 +17,11 @@ from requests.exceptions import HTTPError from psengine.links.errors import LinksMetadataError, LinksSearchError -from psengine.links.links import LinksFilterObjects, LinksLimitsObjects from psengine.links.models import ( CriticalityAttribute, GenericAttribute, + LinksFilterObjects, + LinksLimitsObjects, MitreNameAttribute, RiskAttribute, ThreatActorAttribute, @@ -184,10 +185,9 @@ def test_search_complex_attributes(links_mgr, mocker, make_response): def test_search_with_limits(links_mgr, mocker, make_response): - limits = LinksLimitsObjects(search_scope='small', per_entity_type=10) mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response({'data': []})) - links_mgr.search(entities=['ent1'], limits=limits) + links_mgr.search(entities=['ent1'], search_scope='small', per_entity_type=10) _, kwargs = links_mgr.rf_client.request.call_args assert kwargs['data']['limits']['search_scope'] == 'small' diff --git a/tests/links/test_links_models.py b/tests/links/test_links_models.py index eafdca0c..e70b5d29 100644 --- a/tests/links/test_links_models.py +++ b/tests/links/test_links_models.py @@ -14,14 +14,9 @@ import pytest from pydantic import ValidationError -from psengine.links.links import FilterTechnical, LinksSearchIn +from psengine.links.models import FilterTechnical def test_filter_technical_timeframe_invalid_format(): with pytest.raises(ValidationError, match='Invalid relative time'): FilterTechnical(timeframe='not-a-time') - - -def test_links_search_in_entities_required(): - with pytest.raises(ValidationError, match='entities'): - LinksSearchIn() From 7f4b1a13a5ca55be0db1b1643f105ca60e655c24 Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 11:38:18 +0100 Subject: [PATCH 09/24] tests on data transform --- psengine/links/models.py | 2 +- tests/links/test_links_models.py | 184 ++++++++++++++++++++++++++++++- 2 files changed, 184 insertions(+), 2 deletions(-) diff --git a/psengine/links/models.py b/psengine/links/models.py index 1ec88a6e..7d94bfaa 100644 --- a/psengine/links/models.py +++ b/psengine/links/models.py @@ -107,7 +107,7 @@ class LinksFilterObjects(RFBaseModel): entity_types: Annotated[list[str] | None, BeforeValidator(Validators.convert_str_to_list)] = ( None ) - sources: list[Literal['technical', 'insikt']] | None = None + sources: list[LinkSource] | None = None technical: FilterTechnical | None = None diff --git a/tests/links/test_links_models.py b/tests/links/test_links_models.py index e70b5d29..54ae460c 100644 --- a/tests/links/test_links_models.py +++ b/tests/links/test_links_models.py @@ -14,9 +14,191 @@ import pytest from pydantic import ValidationError -from psengine.links.models import FilterTechnical +from psengine.endpoints import EP_LINKS_SEARCH +from psengine.links.models import ( + FilterTechnical, + LinkSource, + LinksFilterObjects, + LinksLimitsObjects, + SearchScope, +) + + +def test_link_source_enum_values(): + assert [member.value for member in LinkSource] == ['technical', 'insikt'] + + +def test_search_scope_enum_values(): + assert [member.value for member in SearchScope] == ['small', 'medium', 'large'] def test_filter_technical_timeframe_invalid_format(): with pytest.raises(ValidationError, match='Invalid relative time'): FilterTechnical(timeframe='not-a-time') + + +def test_filter_technical_converts_string_fields_to_lists(): + model = FilterTechnical( + timeframe='-30d', + events='type:MalwareAnalysis', + connected_entities='id:Ent1', + ) + + assert model.timeframe == '-30d' + assert model.events == ['type:MalwareAnalysis'] + assert model.connected_entities == ['id:Ent1'] + + +def test_filter_technical_removes_none_values_in_list_fields(): + model = FilterTechnical( + events=['type:MalwareAnalysis', None, 'type:TTPAnalysis'], + connected_entities=['id:Ent1', None, 'id:Ent2'], + ) + + assert model.events == ['type:MalwareAnalysis', 'type:TTPAnalysis'] + assert model.connected_entities == ['id:Ent1', 'id:Ent2'] + + +def test_filter_technical_json_excludes_unset_fields(): + model = FilterTechnical(timeframe='-7d') + assert model.json() == {'timeframe': '-7d'} + + +def test_links_filter_objects_normalizes_scalar_fields(): + model = LinksFilterObjects( + sections='section:actors', + entity_types='type:IpAddress', + sources=['technical', 'insikt'], + technical=FilterTechnical(timeframe='-30d'), + ) + + assert model.json() == { + 'sections': ['section:actors'], + 'entity_types': ['type:IpAddress'], + 'sources': ['technical', 'insikt'], + 'technical': {'timeframe': '-30d'}, + } + + +def test_links_filter_objects_accepts_link_source_enums(): + model = LinksFilterObjects(sources=[LinkSource.technical, LinkSource.insikt]) + assert model.json() == {'sources': ['technical', 'insikt']} + + +def test_links_filter_objects_invalid_source(): + with pytest.raises(ValidationError, match='sources'): + LinksFilterObjects(sources=['invalid_source']) + + +def test_links_limits_objects_serializes_search_scope_enum(): + model = LinksLimitsObjects(search_scope=SearchScope.small, per_entity_type=25) + assert model.json() == {'search_scope': 'small', 'per_entity_type': 25} + + +def test_links_limits_objects_invalid_search_scope(): + with pytest.raises(ValidationError, match='search_scope'): + LinksLimitsObjects(search_scope='huge') + + +def test_links_mgr_search_builds_minimal_payload_with_entity_str(links_mgr, mocker, make_response): + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response({'data': []})) + + links_mgr.search(entities='ent1') + + _, kwargs = links_mgr.rf_client.request.call_args + assert kwargs['method'] == 'POST' + assert kwargs['url'] == EP_LINKS_SEARCH + assert kwargs['data'] == { + 'entities': ['ent1'], + 'filters': {'technical': {}}, + 'limits': {}, + } + + +@pytest.mark.parametrize( + ('search_kwargs', 'expected_filter_key', 'expected_value'), + [ + ({'sections': 'section:actors'}, ('sections',), ['section:actors']), + ({'entity_types': 'type:IpAddress'}, ('entity_types',), ['type:IpAddress']), + ({'events': 'type:MalwareAnalysis'}, ('technical', 'events'), ['type:MalwareAnalysis']), + ({'connected_entities': 'id:Ent1'}, ('technical', 'connected_entities'), ['id:Ent1']), + ], +) +def test_links_mgr_search_converts_str_filter_fields_to_lists( + links_mgr, mocker, make_response, search_kwargs, expected_filter_key, expected_value +): + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response({'data': []})) + + links_mgr.search(entities='ent1', **search_kwargs) + + _, kwargs = links_mgr.rf_client.request.call_args + filters = kwargs['data']['filters'] + target = filters + for key in expected_filter_key: + target = target[key] + assert target == expected_value + + +@pytest.mark.parametrize( + ('search_kwargs', 'expected_filter_key', 'expected_value'), + [ + ({'sections': ['section:actors', 'section:tools']}, ('sections',), ['section:actors', 'section:tools']), + ({'entity_types': ['type:IpAddress', 'type:DomainName']}, ('entity_types',), ['type:IpAddress', 'type:DomainName']), + ({'events': ['type:MalwareAnalysis', 'type:TTPAnalysis']}, ('technical', 'events'), ['type:MalwareAnalysis', 'type:TTPAnalysis']), + ({'connected_entities': ['id:Ent1', 'id:Ent2']}, ('technical', 'connected_entities'), ['id:Ent1', 'id:Ent2']), + ], +) +def test_links_mgr_search_preserves_list_filter_fields( + links_mgr, mocker, make_response, search_kwargs, expected_filter_key, expected_value +): + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response({'data': []})) + + links_mgr.search(entities='ent1', **search_kwargs) + + _, kwargs = links_mgr.rf_client.request.call_args + filters = kwargs['data']['filters'] + target = filters + for key in expected_filter_key: + target = target[key] + assert target == expected_value + + +def test_links_mgr_search_builds_full_payload_from_model_inputs(links_mgr, mocker, make_response): + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response({'data': []})) + + links_mgr.search( + entities=['ent1', 'ent2'], + sections='section:actors', + entity_types='type:IpAddress', + sources=[LinkSource.technical, LinkSource.insikt], + timeframe='-30d', + events='type:MalwareAnalysis', + connected_entities=['id:EntA', 'id:EntB'], + search_scope=SearchScope.large, + per_entity_type=10, + ) + + _, kwargs = links_mgr.rf_client.request.call_args + assert kwargs['data'] == { + 'entities': ['ent1', 'ent2'], + 'filters': { + 'sections': ['section:actors'], + 'entity_types': ['type:IpAddress'], + 'sources': ['technical', 'insikt'], + 'technical': { + 'timeframe': '-30d', + 'events': ['type:MalwareAnalysis'], + 'connected_entities': ['id:EntA', 'id:EntB'], + }, + }, + 'limits': {'search_scope': 'large', 'per_entity_type': 10}, + } + + +def test_links_mgr_search_invalid_timeframe_raises_before_request(links_mgr, mocker): + request_mock = mocker.patch.object(links_mgr.rf_client, 'request') + + with pytest.raises(ValidationError, match='Invalid relative time'): + links_mgr.search(entities=['ent1'], timeframe='invalid-time') + + request_mock.assert_not_called() From 415de5606b79a86946e808a21a041a3f24365f80 Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 11:38:34 +0100 Subject: [PATCH 10/24] ruff --- psengine/links/links_mgr.py | 11 ++++------- tests/links/test_links_mgr.py | 1 - tests/links/test_links_models.py | 26 +++++++++++++++++++++----- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index 4f6ad470..d4d2dd0d 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -14,7 +14,7 @@ import logging from typing import Annotated -from pydantic import AfterValidator, BeforeValidator, validate_call +from pydantic import AfterValidator, validate_call from typing_extensions import Doc from psengine.helpers.helpers import Validators @@ -28,16 +28,13 @@ from ..helpers import connection_exceptions, debug_call from ..rf_client import RFClient from .errors import LinksMetadataError, LinksSearchError -from .models import ( - FilterTechnical, - LinksFilterObjects, - LinksLimitsObjects, -) - from .links import ( LinksSearchResponse, ) from .models import ( + FilterTechnical, + LinksFilterObjects, + LinksLimitsObjects, LinkSource, MetadataOut, SearchScope, diff --git a/tests/links/test_links_mgr.py b/tests/links/test_links_mgr.py index 70a2e610..295dd9fd 100644 --- a/tests/links/test_links_mgr.py +++ b/tests/links/test_links_mgr.py @@ -21,7 +21,6 @@ CriticalityAttribute, GenericAttribute, LinksFilterObjects, - LinksLimitsObjects, MitreNameAttribute, RiskAttribute, ThreatActorAttribute, diff --git a/tests/links/test_links_models.py b/tests/links/test_links_models.py index 54ae460c..9c8ae21e 100644 --- a/tests/links/test_links_models.py +++ b/tests/links/test_links_models.py @@ -17,9 +17,9 @@ from psengine.endpoints import EP_LINKS_SEARCH from psengine.links.models import ( FilterTechnical, - LinkSource, LinksFilterObjects, LinksLimitsObjects, + LinkSource, SearchScope, ) @@ -142,10 +142,26 @@ def test_links_mgr_search_converts_str_filter_fields_to_lists( @pytest.mark.parametrize( ('search_kwargs', 'expected_filter_key', 'expected_value'), [ - ({'sections': ['section:actors', 'section:tools']}, ('sections',), ['section:actors', 'section:tools']), - ({'entity_types': ['type:IpAddress', 'type:DomainName']}, ('entity_types',), ['type:IpAddress', 'type:DomainName']), - ({'events': ['type:MalwareAnalysis', 'type:TTPAnalysis']}, ('technical', 'events'), ['type:MalwareAnalysis', 'type:TTPAnalysis']), - ({'connected_entities': ['id:Ent1', 'id:Ent2']}, ('technical', 'connected_entities'), ['id:Ent1', 'id:Ent2']), + ( + {'sections': ['section:actors', 'section:tools']}, + ('sections',), + ['section:actors', 'section:tools'], + ), + ( + {'entity_types': ['type:IpAddress', 'type:DomainName']}, + ('entity_types',), + ['type:IpAddress', 'type:DomainName'], + ), + ( + {'events': ['type:MalwareAnalysis', 'type:TTPAnalysis']}, + ('technical', 'events'), + ['type:MalwareAnalysis', 'type:TTPAnalysis'], + ), + ( + {'connected_entities': ['id:Ent1', 'id:Ent2']}, + ('technical', 'connected_entities'), + ['id:Ent1', 'id:Ent2'], + ), ], ) def test_links_mgr_search_preserves_list_filter_fields( From 10ebb0de473f7f6a8029a839ce653ab62e51111a Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 12:22:12 +0100 Subject: [PATCH 11/24] fix example_2 script and docs --- docs/examples/links/example_2.py | 27 ++++++++++-------------- docs/modules/links.md | 4 ++-- psengine/links/__init__.py | 4 ++-- psengine/links/links.py | 11 ++++------ psengine/links/links_mgr.py | 35 +++++++++----------------------- 5 files changed, 29 insertions(+), 52 deletions(-) diff --git a/docs/examples/links/example_2.py b/docs/examples/links/example_2.py index 7dcb95f7..6b164509 100644 --- a/docs/examples/links/example_2.py +++ b/docs/examples/links/example_2.py @@ -1,25 +1,20 @@ -from psengine.links import ( - FilterTechnical, - LinksFilterObjects, - LinksLimitsObjects, - LinksMgr, -) +from psengine.links import LinksMgr mgr = LinksMgr() -filters = LinksFilterObjects( - sources=['technical'], - entity_types=['Malware'], - technical=FilterTechnical(timeframe='-30d'), -) -limits = LinksLimitsObjects( - search_scope='small', per_entity_type=50 -) - results = mgr.search( - entities=['QCwdoU'], filters=filters, limits=limits + entities=['QCwdoU'], + sources=['technical'], + entity_types=['type:Malware'], + timeframe='-30d', + search_scope='small', + per_entity_type=50, ) for result in results.data: + if result.error: + print(f'Failed: {result.error.message}') + continue + for link in result.links: print(f'{link.name} ({link.type_})') diff --git a/docs/modules/links.md b/docs/modules/links.md index aafcc21a..561219df 100644 --- a/docs/modules/links.md +++ b/docs/modules/links.md @@ -38,7 +38,7 @@ Entity: Lazarus Group #### 2: Filter link results and apply limits -In this example, we use `LinksFilterObjects` and `LinksLimitsObjects` to narrow results to technical malware links seen in the last 30 days, and to cap result size per entity type. +In this example, we pass filter and limit arguments directly to `search` (for example `sources`, `entity_types`, `timeframe`, `search_scope`, and `per_entity_type`) to narrow results to technical malware links seen in the last 30 days and cap result size per entity type. ```python --8<-- "docs/examples/links/example_2.py" @@ -46,7 +46,7 @@ In this example, we use `LinksFilterObjects` and `LinksLimitsObjects` to narrow #### 3: Discover available metadata for filters -In this example, we list valid sections, event types, and entity types. These IDs can be reused in `LinksFilterObjects` and `FilterTechnical` when building search queries. +In this example, we list valid sections, event types, and entity types. These IDs can be reused in the `search` filter arguments when building queries. ```python --8<-- "docs/examples/links/example_3.py" diff --git a/psengine/links/__init__.py b/psengine/links/__init__.py index 0bfae9e9..975f7975 100644 --- a/psengine/links/__init__.py +++ b/psengine/links/__init__.py @@ -18,8 +18,8 @@ ) from .links import ( LinkedEntity, - LinksSearchResponse, - SearchResultSet, + LinksSearchResponseOut, + Link, ) from .links_mgr import LinksMgr from .models import ( diff --git a/psengine/links/links.py b/psengine/links/links.py index d022192e..12bc1c09 100644 --- a/psengine/links/links.py +++ b/psengine/links/links.py @@ -17,17 +17,14 @@ class LinkedEntity(IdNameType): - """An entity connected to the search target. - - Inherits id_ (alias 'id'), name, and type_ (alias 'type') from IdNameType. - """ + """An entity connected to the search target.""" source: str | None = None section: str | None = None attributes: list[EntityAttribute] = [] -class SearchResultSet(RFBaseModel): +class Link(RFBaseModel): """The result set for a single entity that was queried.""" entity: IdNameType | None = None @@ -35,7 +32,7 @@ class SearchResultSet(RFBaseModel): error: EntitySearchError | None = None -class LinksSearchResponse(RFBaseModel): +class LinksSearchResponseOut(RFBaseModel): """Response from POST `/links/search` endpoint.""" - data: list[SearchResultSet] = [] + data: list[Link] = [] diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index d4d2dd0d..b90fbe19 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -29,7 +29,7 @@ from ..rf_client import RFClient from .errors import LinksMetadataError, LinksSearchError from .links import ( - LinksSearchResponse, + LinksSearchResponseOut, ) from .models import ( FilterTechnical, @@ -155,43 +155,28 @@ def search( int | None, Doc('Max linked entities returned per entity type.') ] = None, ) -> Annotated[ - list[LinksSearchResponse], Doc('A list of LinkResult models — one per input entity.') + list[LinksSearchResponseOut], Doc('A list of LinkResult models — one per input entity.') ]: """Search for entities connected to one or more target entities. Issues a single batched request: the response contains one - `SearchResultSet` per entity in `entities`, in the same order. If the + `Link` per entity in `entities`, in the same order. If the API failed for a specific entity, that result's `error` is populated and `links` is empty — the rest of the batch still succeeds. - `filters` narrows the result set: - - - `sections` — restrict to specific Links sections (see `list_sections`). - - `entity_types` — restrict to specific connected-entity types (see `list_entity_types`). - - `sources` — restrict to `technical`, `insikt`, or both. - - `technical` — sub-filters that apply only to technical links (timeframe, - event types, connected-entity scope). - - `limits` controls how aggressively the API searches: - - - `search_scope` — one of `small`, `medium`, `large`. Larger scopes scan - more references and Insikt notes per query at the cost of latency. - - `per_entity_type` — caps how many connected entities of each type - are returned. - Entities must be supplied as Recorded Future entity IDs; if you only have - a name, resolve it with `EntityMatchMgr` or `LookupMgr` first. + a name, resolve it with `EntityMatchMgr` first. Endpoint: `/links/search` If the API failed for a specific entity in the batch, its result looks like: ```python - SearchResultSet( - entity=IdNameType(id_='QCwdoU', name='...', type_='...'), - links=[], - error=EntitySearchError(message='...', status_code=404), - ) + Link( + entity=IdNameType(id_='QCwdoU', name='...', type_='...'), + links=[], + error=EntitySearchError(message='...', status_code=404), + ) ``` Raises: @@ -216,4 +201,4 @@ def search( self.log.info(f'Executing links search for {len(entities)} entities.') response = self.rf_client.request(method='POST', url=EP_LINKS_SEARCH, data=payload) - return LinksSearchResponse.model_validate(response.json()) + return LinksSearchResponseOut.model_validate(response.json()) From 6bebcf254dfda47c53f8214ca5fa7d6ad22c825c Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 12:22:26 +0100 Subject: [PATCH 12/24] ruff --- psengine/links/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psengine/links/__init__.py b/psengine/links/__init__.py index 975f7975..44f0a0de 100644 --- a/psengine/links/__init__.py +++ b/psengine/links/__init__.py @@ -17,9 +17,9 @@ LinksSearchError, ) from .links import ( + Link, LinkedEntity, LinksSearchResponseOut, - Link, ) from .links_mgr import LinksMgr from .models import ( From f19276e98749abd6d868a61424c58633b910af82 Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 14:02:53 +0100 Subject: [PATCH 13/24] methods to extract iocs from links --- docs/examples/links/example_3.py | 35 +++++-- docs/modules/links.md | 47 ++++++++- psengine/links/__init__.py | 2 +- psengine/links/links.py | 115 ++++++++++++++++++++- psengine/links/links_mgr.py | 4 +- tests/links/test_links_mgr.py | 167 +++++++++++++++++++++++++++++++ 6 files changed, 354 insertions(+), 16 deletions(-) diff --git a/docs/examples/links/example_3.py b/docs/examples/links/example_3.py index fc017f1d..7e490464 100644 --- a/docs/examples/links/example_3.py +++ b/docs/examples/links/example_3.py @@ -2,14 +2,31 @@ mgr = LinksMgr() -print('Sections:') -for section in mgr.list_sections(): - print(f' {section.id_}: {section.name}') +results = mgr.search(entities=['QCwdoU']) -print('\nEvent types:') -for event in mgr.list_events(): - print(f' {event.id_}: {event.name}') +for result in results.data: + if result.error: + print(f'Failed: {result.error.message}') + continue -print('\nEntity types:') -for entity_type in mgr.list_entity_types(): - print(f' {entity_type.id_}: {entity_type.name}') + print(f'Entity: {result.entity.name}') + + print('\nIOCs grouped by type:') + for ioc_type, iocs in result.iocs().items(): + print(f' {ioc_type}: {len(iocs)}') + for ioc in iocs[:3]: + print( + f' - {ioc.name} score:{ioc.risk_score}' + ) + + print('\nTTPs:') + for ttp in result.ttps()[:5]: + print(f' - {ttp.name} ({ttp.display_name})') + + print('\nMalwares:') + for malware in result.malwares()[:5]: + print(f' - {malware.name}') + + print('\nThreat actors:') + for threat_actor in result.threat_actors()[:5]: + print(f' - {threat_actor.name}') diff --git a/docs/modules/links.md b/docs/modules/links.md index 561219df..210857b3 100644 --- a/docs/modules/links.md +++ b/docs/modules/links.md @@ -44,10 +44,53 @@ In this example, we pass filter and limit arguments directly to `search` (for ex --8<-- "docs/examples/links/example_2.py" ``` -#### 3: Discover available metadata for filters +#### 3: Extract IOCs, Threat Actors and Malwares from links -In this example, we list valid sections, event types, and entity types. These IDs can be reused in the `search` filter arguments when building queries. +Each item returned by `LinksMgr.search()` is an `EntityLinks` object. After checking `result.error`, you can use helper methods to extract common subsets from `result.links`: + +- `result.iocs()`: returns linked IOCs grouped by IOC type (`type:InternetDomainName`, `type:CyberVulnerability`, `type:IpAddress`, `type:Hash`, `type:Url`). Each IOC item includes `id`, `type`, `name`, `risk_score`, and `source`. +- `result.ttps()`: returns linked MITRE ATT&CK techniques (`type:MitreAttackIdentifier`). Each item includes `id`, `type`, `name`, `display_name`, and `source`. +- `result.malwares()`: returns linked malware entities (`type:Malware`) with `id`, `type`, `name`, and `source`. +- `result.threat_actors()`: returns linked organizations (`type:Organization`) that are marked as threat actors through the `threat_actor` attribute. ```python --8<-- "docs/examples/links/example_3.py" ``` + +This will output: + +``` +Entity: Lazarus Group + +IOCs grouped by type: + type:CyberVulnerability: 5 + - CVE-2022-47966 score:99 + - CVE-2019-3396 score:89 + - CVE-2022-0609 score:79 + type:Hash: 1273 + - c8706a586afa880fbf23ce662b7fc2925fd0384b8cde0a40f3c00a182c5a3d06 score:89 + - 9ab05d771d6face27502052af8e3a945ad66e3c6726a2dda637fa12641e2bca2 score:89 + - 084f904249f8655e925b4b330426df6b979cd3f630f7765dac405f2144755738 score:89 + type:InternetDomainName: 144 + - calendly.live score:74 + - pypilibrary.com score:72 + - test-wolf.com score:69 + type:IpAddress: 29 + - 172.96.137.224 score:55 + - 103.231.75.101 score:38 + - 23.27.140.49 score:32 + +TTPs: + - T1082 (T1082 (System Information Discovery)) + - T1071 (T1071 (Application Layer Protocol)) + - TA0010 (TA0010 (Exfiltration)) + - T1027 (T1027 (Obfuscated Files or Information)) + - T1497.001 (T1497.001 (Virtualization/Sandbox Evasion: System Checks)) + +Malwares: + - CIPHERROCKET + - QuiteRAT + - PondRAT + - Trickbot + - NukeSped +``` diff --git a/psengine/links/__init__.py b/psengine/links/__init__.py index 44f0a0de..e457e79b 100644 --- a/psengine/links/__init__.py +++ b/psengine/links/__init__.py @@ -17,7 +17,7 @@ LinksSearchError, ) from .links import ( - Link, + EntityLinks, LinkedEntity, LinksSearchResponseOut, ) diff --git a/psengine/links/links.py b/psengine/links/links.py index 12bc1c09..42b6416f 100644 --- a/psengine/links/links.py +++ b/psengine/links/links.py @@ -12,9 +12,45 @@ ############################################################################################## +from collections import defaultdict + from ..common_models import IdNameType, RFBaseModel from .models import EntityAttribute, EntitySearchError +LINK_IOC_TYPE = [ + 'type:InternetDomainName', + 'type:CyberVulnerability', + 'type:IpAddress', + 'type:Hash', + 'type:Url', +] + + +class LinkedIOC(IdNameType): + """Return linked iocs entities.""" + + risk_score: int + source: str | None = None + + +class LinkedTTP(IdNameType): + """Return linked TTPs entities.""" + + display_name: str + source: str | None = None + + +class LinkedTA(IdNameType): + """Return linked threat actors entities.""" + + source: str | None = None + + +class LinkedMalware(IdNameType): + """Return linked malware entities.""" + + source: str | None = None + class LinkedEntity(IdNameType): """An entity connected to the search target.""" @@ -24,15 +60,90 @@ class LinkedEntity(IdNameType): attributes: list[EntityAttribute] = [] -class Link(RFBaseModel): +class EntityLinks(RFBaseModel): """The result set for a single entity that was queried.""" entity: IdNameType | None = None links: list[LinkedEntity] = [] error: EntitySearchError | None = None + def iocs(self) -> list[LinkedIOC]: + """Return linked indicators of compromise grouped by IOC type.""" + iocs = defaultdict(list) + for link in self.links: + if link.type_ in LINK_IOC_TYPE: + ioc_score = next( + (attr.value for attr in link.attributes if attr.id_ == 'risk_score'), + 0, + ) + iocs[link.type_].append( + LinkedIOC( + id=link.id_, + type=link.type_, + name=link.name, + risk_score=ioc_score, + source=link.source, + ) + ) + + return iocs + + def ttps(self) -> list[LinkedTTP]: + """Return linked MITRE ATT&CK techniques and their display names.""" + ttps = [] + for link in self.links: + if link.type_ == 'type:MitreAttackIdentifier': + display_name = next( + (attr.value for attr in link.attributes if attr.id_ == 'display_name'), + 'N/A', + ) + ttps.append( + LinkedTTP( + id=link.id_, + type=link.type_, + name=link.name, + display_name=display_name, + source=link.source, + ) + ) + + return ttps + + def threat_actors(self) -> list[LinkedTA]: + """Return linked organizations marked as threat actors.""" + tas = [] + for link in self.links: + if link.type_ == 'type:Organization': + is_threat_actor = next( + (attr.value for attr in link.attributes if attr.id_ == 'threat_actor'), + ) + if is_threat_actor: + tas.append( + LinkedTA( + id=link.id_, + type=link.type_, + name=link.name, + source=link.source, + ) + ) + + return tas + + def malwares(self) -> list[LinkedMalware]: + """Return linked malware entities.""" + return [ + LinkedMalware( + id=link.id_, + type=link.type_, + name=link.name, + source=link.source, + ) + for link in self.links + if link.type_ == 'type:Malware' + ] + class LinksSearchResponseOut(RFBaseModel): """Response from POST `/links/search` endpoint.""" - data: list[Link] = [] + data: list[EntityLinks] = [] diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index b90fbe19..ed56fb1e 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -160,7 +160,7 @@ def search( """Search for entities connected to one or more target entities. Issues a single batched request: the response contains one - `Link` per entity in `entities`, in the same order. If the + `EntityLinks` per entity in `entities`, in the same order. If the API failed for a specific entity, that result's `error` is populated and `links` is empty — the rest of the batch still succeeds. @@ -172,7 +172,7 @@ def search( If the API failed for a specific entity in the batch, its result looks like: ```python - Link( + EntityLinks( entity=IdNameType(id_='QCwdoU', name='...', type_='...'), links=[], error=EntitySearchError(message='...', status_code=404), diff --git a/tests/links/test_links_mgr.py b/tests/links/test_links_mgr.py index 295dd9fd..336a4fa6 100644 --- a/tests/links/test_links_mgr.py +++ b/tests/links/test_links_mgr.py @@ -191,3 +191,170 @@ def test_search_with_limits(links_mgr, mocker, make_response): _, kwargs = links_mgr.rf_client.request.call_args assert kwargs['data']['limits']['search_scope'] == 'small' assert kwargs['data']['limits']['per_entity_type'] == 10 + + +def test_entity_links_iocs_groups_results_and_defaults_risk_score(links_mgr, mocker, make_response): + mock_data = { + 'data': [ + { + 'entity': {'id': 'ent1', 'name': 'Entity 1', 'type': 'Type1'}, + 'links': [ + { + 'id': 'ioc1', + 'name': 'example.com', + 'type': 'type:InternetDomainName', + 'source': 'technical', + 'attributes': [{'id': 'risk_score', 'value': 90}], + }, + { + 'id': 'ioc2', + 'name': 'abcdef1234', + 'type': 'type:Hash', + 'source': 'insikt', + 'attributes': [], + }, + { + 'id': 'not-ioc', + 'name': 'Not IOC', + 'type': 'type:Organization', + 'attributes': [{'id': 'threat_actor', 'value': True}], + }, + ], + } + ] + } + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) + + entity_links = links_mgr.search(entities=['ent1']).data[0] + iocs = entity_links.iocs() + + assert set(iocs.keys()) == {'type:InternetDomainName', 'type:Hash'} + assert len(iocs['type:InternetDomainName']) == 1 + assert len(iocs['type:Hash']) == 1 + assert iocs['type:InternetDomainName'][0].risk_score == 90 + assert iocs['type:InternetDomainName'][0].source == 'technical' + assert iocs['type:Hash'][0].risk_score == 0 + assert iocs['type:Hash'][0].source == 'insikt' + + +def test_entity_links_ttps_filters_mitre_attack_links(links_mgr, mocker, make_response): + mock_data = { + 'data': [ + { + 'entity': {'id': 'ent1', 'name': 'Entity 1', 'type': 'Type1'}, + 'links': [ + { + 'id': 'ttp1', + 'name': 'T1059', + 'type': 'type:MitreAttackIdentifier', + 'source': 'technical', + 'attributes': [{'id': 'display_name', 'value': 'Command and Scripting'}], + }, + { + 'id': 'ttp2', + 'name': 'T1566', + 'type': 'type:MitreAttackIdentifier', + 'source': 'insikt', + 'attributes': [], + }, + { + 'id': 'not-ttp', + 'name': 'Not TTP', + 'type': 'type:Malware', + 'attributes': [], + }, + ], + } + ] + } + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) + + ttps = links_mgr.search(entities=['ent1']).data[0].ttps() + + assert len(ttps) == 2 + assert ttps[0].id_ == 'ttp1' + assert ttps[0].display_name == 'Command and Scripting' + assert ttps[0].source == 'technical' + assert ttps[1].id_ == 'ttp2' + assert ttps[1].display_name == 'N/A' + assert ttps[1].source == 'insikt' + + +def test_entity_links_threat_actors_only_returns_marked_organizations( + links_mgr, mocker, make_response +): + mock_data = { + 'data': [ + { + 'entity': {'id': 'ent1', 'name': 'Entity 1', 'type': 'Type1'}, + 'links': [ + { + 'id': 'ta1', + 'name': 'Threat Actor Org', + 'type': 'type:Organization', + 'source': 'insikt', + 'attributes': [{'id': 'threat_actor', 'value': True}], + }, + { + 'id': 'org2', + 'name': 'Benign Org', + 'type': 'type:Organization', + 'attributes': [{'id': 'threat_actor', 'value': False}], + }, + { + 'id': 'not-org', + 'name': 'Not Org', + 'type': 'type:Malware', + 'attributes': [{'id': 'threat_actor', 'value': True}], + }, + ], + } + ] + } + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) + + threat_actors = links_mgr.search(entities=['ent1']).data[0].threat_actors() + + assert len(threat_actors) == 1 + assert threat_actors[0].id_ == 'ta1' + assert threat_actors[0].name == 'Threat Actor Org' + assert threat_actors[0].source == 'insikt' + + +def test_entity_links_malwares_only_returns_malware_links(links_mgr, mocker, make_response): + mock_data = { + 'data': [ + { + 'entity': {'id': 'ent1', 'name': 'Entity 1', 'type': 'Type1'}, + 'links': [ + { + 'id': 'mw1', + 'name': 'Malware One', + 'type': 'type:Malware', + 'source': 'technical', + 'attributes': [], + }, + { + 'id': 'not-malware', + 'name': 'Not Malware', + 'type': 'type:Organization', + 'attributes': [{'id': 'threat_actor', 'value': True}], + }, + { + 'id': 'mw2', + 'name': 'Malware Two', + 'type': 'type:Malware', + 'source': 'insikt', + 'attributes': [], + }, + ], + } + ] + } + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) + + malwares = links_mgr.search(entities=['ent1']).data[0].malwares() + + assert len(malwares) == 2 + assert [malware.id_ for malware in malwares] == ['mw1', 'mw2'] + assert [malware.source for malware in malwares] == ['technical', 'insikt'] From 053fd22057aae6eb275fc6082b314f952268636f Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 14:05:00 +0100 Subject: [PATCH 14/24] imports --- psengine/links/links_mgr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index ed56fb1e..a9234cdb 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -17,8 +17,6 @@ from pydantic import AfterValidator, validate_call from typing_extensions import Doc -from psengine.helpers.helpers import Validators - from ..endpoints import ( EP_LINKS_METADATA_ENTITIES, EP_LINKS_METADATA_EVENTS, @@ -26,6 +24,7 @@ EP_LINKS_SEARCH, ) from ..helpers import connection_exceptions, debug_call +from ..helpers.helpers import Validators from ..rf_client import RFClient from .errors import LinksMetadataError, LinksSearchError from .links import ( From 3c83aa72237598e43c5f57b56fc7d528b3c1cbe5 Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 14:13:54 +0100 Subject: [PATCH 15/24] gitignore cleanup --- .gitignore | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 89bc3f3d..88a1e6c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,28 +1,37 @@ -.DS_Store -**/venv/* -**/venv3/* +# build __pycache__ **/dist/* **/build/* -**/pkg_build/* -**/.ruff_cache/* -*.log -*.log.* -*.csv -.tox -rfalerts_*.json -rfplaybookalerts_*.json dist/ -.vscode *.pyc *.egg-info *.tar.gz *.tgz +site/ + +# client +.DS_Store +.vscode +.idea/ + + +# tets and coverage +**/.ruff_cache/* +*.log +*.log.* +*.csv result.xml .coverage tests/stix2/tests tests/output/* -virt/ +alerts/ +attachments/ +rules/ +risklists/ +bundles/ + + +# docs docs/examples/classic_alerts/alerts docs/examples/playbook_alerts/alerts docs/examples/analyst_notes/attachments @@ -31,12 +40,4 @@ docs/examples/enrich/enrich docs/examples/risklists/risklists docs/examples/stix2/bundles -alerts/ -attachments/ -rules/ -risklists/ -bundles/ - -site/ -.idea/ \ No newline at end of file From 350dbb256b285f9ff4984f24e143286046a0a2e9 Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 14:24:09 +0100 Subject: [PATCH 16/24] rf_token as Doc in __init__ --- psengine/analyst_notes/note_mgr.py | 11 +++++------ psengine/asi/asi_mgr.py | 11 +++++------ psengine/classic_alerts/classic_alert_mgr.py | 11 +++++------ psengine/collective_insights/collective_insights.py | 11 +++++------ psengine/fusion/fusion_mgr.py | 11 +++++------ psengine/malware_intel/auto_sigma_mgr.py | 11 +++++------ psengine/malware_intel/auto_yara_mgr.py | 11 +++++------ psengine/malware_intel/malware_intel_mgr.py | 11 +++++------ psengine/risk_history/risk_history_mgr.py | 11 +++++------ 9 files changed, 45 insertions(+), 54 deletions(-) diff --git a/psengine/analyst_notes/note_mgr.py b/psengine/analyst_notes/note_mgr.py index ee3c791d..26b93144 100644 --- a/psengine/analyst_notes/note_mgr.py +++ b/psengine/analyst_notes/note_mgr.py @@ -53,12 +53,11 @@ class AnalystNoteMgr: """Manages requests for Recorded Future analyst notes.""" - def __init__(self, rf_token: str = None): - """Initializes the `AnalystNoteMgr` object. - - Args: - rf_token (str, optional): Recorded Future API token. - """ + def __init__( + self, + rf_token: Annotated[str | None, Doc('Recorded Future API token.')] = None, + ): + """Initializes the `AnalystNoteMgr` object.""" self.log = logging.getLogger(__name__) self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient() diff --git a/psengine/asi/asi_mgr.py b/psengine/asi/asi_mgr.py index d706d018..3ca4f118 100644 --- a/psengine/asi/asi_mgr.py +++ b/psengine/asi/asi_mgr.py @@ -95,12 +95,11 @@ def _validate_exposure_score_range(v: tuple[int, int]) -> tuple[int, int]: class AttackSurfaceMgr: """Manages requests for Recorded Future SecurityTrails (ASI) API.""" - def __init__(self, api_token: str = None): - """Initializes the `AttackSurfaceMgr` object. - - Args: - api_token (str, optional): ASI API token. - """ + def __init__( + self, + api_token: Annotated[str | None, Doc('ASI API token.')] = None, + ): + """Initializes the `AttackSurfaceMgr` object.""" self.log = logging.getLogger(__name__) self.asi_client = ASIClient(api_token=api_token) if api_token else ASIClient() diff --git a/psengine/classic_alerts/classic_alert_mgr.py b/psengine/classic_alerts/classic_alert_mgr.py index 3d1fb667..2ab0eefc 100644 --- a/psengine/classic_alerts/classic_alert_mgr.py +++ b/psengine/classic_alerts/classic_alert_mgr.py @@ -43,12 +43,11 @@ class ClassicAlertMgr: """Alert Manager for Classic Alert (v3) API.""" - def __init__(self, rf_token: str = None): - """Initializes the ClassicAlertMgr object. - - Args: - rf_token (str, optional): Recorded Future API token. - """ + def __init__( + self, + rf_token: Annotated[str | None, Doc('Recorded Future API token.')] = None, + ): + """Initializes the ClassicAlertMgr object.""" self.log = logging.getLogger(__name__) self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient() diff --git a/psengine/collective_insights/collective_insights.py b/psengine/collective_insights/collective_insights.py index 5b64d87f..6b86e5c2 100644 --- a/psengine/collective_insights/collective_insights.py +++ b/psengine/collective_insights/collective_insights.py @@ -29,12 +29,11 @@ class CollectiveInsights: """Class for interacting with the Recorded Future Collective Insights API.""" - def __init__(self, rf_token: str = None): - """Initializes the CollectiveInsights object. - - Args: - rf_token (str, optional): Recorded Future API token. - """ + def __init__( + self, + rf_token: Annotated[str | None, Doc('Recorded Future API token.')] = None, + ): + """Initializes the CollectiveInsights object.""" self.log = logging.getLogger(__name__) self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient() diff --git a/psengine/fusion/fusion_mgr.py b/psengine/fusion/fusion_mgr.py index 4b2e7600..a926be67 100644 --- a/psengine/fusion/fusion_mgr.py +++ b/psengine/fusion/fusion_mgr.py @@ -36,12 +36,11 @@ class FusionMgr: """Manages requests for Recorded Future Fusion files.""" - def __init__(self, rf_token: str = None): - """Initializes the `FusionMgr` object. - - Args: - rf_token (str, optional): Recorded Future API token. - """ + def __init__( + self, + rf_token: Annotated[str | None, Doc('Recorded Future API token.')] = None, + ): + """Initializes the `FusionMgr` object.""" self.log = logging.getLogger(__name__) self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient() diff --git a/psengine/malware_intel/auto_sigma_mgr.py b/psengine/malware_intel/auto_sigma_mgr.py index 4b2240ee..d9f44fff 100644 --- a/psengine/malware_intel/auto_sigma_mgr.py +++ b/psengine/malware_intel/auto_sigma_mgr.py @@ -51,12 +51,11 @@ class AutoSigmaMgr: """Manages requests for Recorded Future Malware Intelligence API Auto Sigma feature.""" - def __init__(self, rf_token: str = None): - """Initializes the `AutoSigmaMgr` object. - - Args: - rf_token (str, optional): Recorded Future API token. - """ + def __init__( + self, + rf_token: Annotated[str | None, Doc('Recorded Future API token.')] = None, + ): + """Initializes the `AutoSigmaMgr` object.""" self.log = logging.getLogger(__name__) self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient() diff --git a/psengine/malware_intel/auto_yara_mgr.py b/psengine/malware_intel/auto_yara_mgr.py index dae26039..0fa24061 100644 --- a/psengine/malware_intel/auto_yara_mgr.py +++ b/psengine/malware_intel/auto_yara_mgr.py @@ -49,12 +49,11 @@ class AutoYaraMgr: """Manages requests for Recorded Future Malware Intelligence API Auto YARA feature.""" - def __init__(self, rf_token: str = None): - """Initializes the `AutoYaraMgr` object. - - Args: - rf_token (str, optional): Recorded Future API token. - """ + def __init__( + self, + rf_token: Annotated[str | None, Doc('Recorded Future API token.')] = None, + ): + """Initializes the `AutoYaraMgr` object.""" self.log = logging.getLogger(__name__) self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient() diff --git a/psengine/malware_intel/malware_intel_mgr.py b/psengine/malware_intel/malware_intel_mgr.py index ec1dfd06..ad89c39a 100644 --- a/psengine/malware_intel/malware_intel_mgr.py +++ b/psengine/malware_intel/malware_intel_mgr.py @@ -29,12 +29,11 @@ class MalwareIntelMgr: """Manages requests for Recorded Future Malware Intelligence API.""" - def __init__(self, rf_token: str = None): - """Initializes the `MalwareIntelMgr` object. - - Args: - rf_token (str, optional): Recorded Future API token. - """ + def __init__( + self, + rf_token: Annotated[str | None, Doc('Recorded Future API token.')] = None, + ): + """Initializes the `MalwareIntelMgr` object.""" self.log = logging.getLogger(__name__) self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient() diff --git a/psengine/risk_history/risk_history_mgr.py b/psengine/risk_history/risk_history_mgr.py index 5c3f8df1..568f711e 100644 --- a/psengine/risk_history/risk_history_mgr.py +++ b/psengine/risk_history/risk_history_mgr.py @@ -28,12 +28,11 @@ class RiskHistoryMgr: """Manages requests for Recorded Future Risk History information.""" - def __init__(self, rf_token: str = None): - """Initializes the `RiskHistoryMgr` object. - - Args: - rf_token (str, optional): Recorded Future API token. - """ + def __init__( + self, + rf_token: Annotated[str | None, Doc('Recorded Future API token.')] = None, + ): + """Initializes the `RiskHistoryMgr` object.""" self.log = logging.getLogger(__name__) self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient() From dcf707f819de4b183dcccb2bdd528f0efc6d3ed0 Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 14:50:11 +0100 Subject: [PATCH 17/24] remove constants from mkdocs for links --- mkdocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 305239ed..80fa5e9c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -196,7 +196,6 @@ nav: - Manager: api/links/links_mgr.md - ADT: api/links/links.md - Models: api/links/models.md - - Constants: api/links/constants.md - Errors: api/links/errors.md - Logger: - Logger: api/logger/rf_logger.md From 36fbbfaae3eee299438daf833e74b1e9f34bc3af Mon Sep 17 00:00:00 2001 From: mmedici Date: Thu, 21 May 2026 15:47:41 +0100 Subject: [PATCH 18/24] search filter --- psengine/links/links.py | 10 +++++++++- psengine/links/links_mgr.py | 11 ++++++++--- tests/links/test_links_models.py | 15 ++++++++++----- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/psengine/links/links.py b/psengine/links/links.py index 42b6416f..6f1b82c9 100644 --- a/psengine/links/links.py +++ b/psengine/links/links.py @@ -15,7 +15,7 @@ from collections import defaultdict from ..common_models import IdNameType, RFBaseModel -from .models import EntityAttribute, EntitySearchError +from .models import EntityAttribute, EntitySearchError, LinksFilterObjects, LinksLimitsObjects LINK_IOC_TYPE = [ 'type:InternetDomainName', @@ -147,3 +147,11 @@ class LinksSearchResponseOut(RFBaseModel): """Response from POST `/links/search` endpoint.""" data: list[EntityLinks] = [] + + +class LinksSearchIn(RFBaseModel): + """Model for payload sent to POST `/links/search` endpoint.""" + + entities: list[str] + filters: LinksFilterObjects | None = None + limits: LinksLimitsObjects | None = None diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index a9234cdb..30551136 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -28,6 +28,7 @@ from ..rf_client import RFClient from .errors import LinksMetadataError, LinksSearchError from .links import ( + LinksSearchIn, LinksSearchResponseOut, ) from .models import ( @@ -182,7 +183,7 @@ def search( ValidationError: If any supplied parameter is of incorrect type. LinksSearchError: If an API or connection error occurs at the request level. """ - technical_filter = FilterTechnical( + technical_filters = FilterTechnical( timeframe=timeframe, events=events, connected_entities=connected_entities ).json() @@ -190,12 +191,16 @@ def search( sections=sections, entity_types=entity_types, sources=sources, - technical=technical_filter, + technical=technical_filters or None, ).json() limits = LinksLimitsObjects( search_scope=search_scope, per_entity_type=per_entity_type ).json() - payload = {'entities': entities, 'filters': filters, 'limits': limits} + payload = LinksSearchIn( + entities=entities, + filters=filters or None, + limits=limits or None, + ).json() self.log.info(f'Executing links search for {len(entities)} entities.') diff --git a/tests/links/test_links_models.py b/tests/links/test_links_models.py index 9c8ae21e..1b1e170c 100644 --- a/tests/links/test_links_models.py +++ b/tests/links/test_links_models.py @@ -108,11 +108,16 @@ def test_links_mgr_search_builds_minimal_payload_with_entity_str(links_mgr, mock _, kwargs = links_mgr.rf_client.request.call_args assert kwargs['method'] == 'POST' assert kwargs['url'] == EP_LINKS_SEARCH - assert kwargs['data'] == { - 'entities': ['ent1'], - 'filters': {'technical': {}}, - 'limits': {}, - } + assert kwargs['data'] == {'entities': ['ent1']} + + +def test_links_mgr_search_omits_empty_technical_filter_object(links_mgr, mocker, make_response): + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response({'data': []})) + + links_mgr.search(entities='ent1', sections='section:actors') + + _, kwargs = links_mgr.rf_client.request.call_args + assert kwargs['data']['filters'] == {'sections': ['section:actors']} @pytest.mark.parametrize( From 59ac12b4aa4fb8fcdf87415106756063d03bf5f4 Mon Sep 17 00:00:00 2001 From: ebartosevic Date: Thu, 21 May 2026 16:23:42 +0100 Subject: [PATCH 19/24] PSF-1254 initial links review, model refactor --- docs/examples/links/example_1.py | 2 +- docs/examples/links/example_2.py | 2 +- docs/examples/links/example_3.py | 2 +- psengine/links/__init__.py | 2 - psengine/links/links.py | 7 +-- psengine/links/links_mgr.py | 21 ++++----- psengine/links/models.py | 16 +------ tests/links/test_links_mgr.py | 73 +++++++++++++++++++++++--------- 8 files changed, 68 insertions(+), 57 deletions(-) diff --git a/docs/examples/links/example_1.py b/docs/examples/links/example_1.py index 0caf8d1c..03f177ef 100644 --- a/docs/examples/links/example_1.py +++ b/docs/examples/links/example_1.py @@ -4,7 +4,7 @@ results = mgr.search(entities=['QCwdoU']) -for result in results.data: +for result in results: if result.error: print(f'Failed: {result.error.message}') continue diff --git a/docs/examples/links/example_2.py b/docs/examples/links/example_2.py index 6b164509..86cdf5bd 100644 --- a/docs/examples/links/example_2.py +++ b/docs/examples/links/example_2.py @@ -11,7 +11,7 @@ per_entity_type=50, ) -for result in results.data: +for result in results: if result.error: print(f'Failed: {result.error.message}') continue diff --git a/docs/examples/links/example_3.py b/docs/examples/links/example_3.py index 7e490464..09390f50 100644 --- a/docs/examples/links/example_3.py +++ b/docs/examples/links/example_3.py @@ -4,7 +4,7 @@ results = mgr.search(entities=['QCwdoU']) -for result in results.data: +for result in results: if result.error: print(f'Failed: {result.error.message}') continue diff --git a/psengine/links/__init__.py b/psengine/links/__init__.py index e457e79b..4b7f43b2 100644 --- a/psengine/links/__init__.py +++ b/psengine/links/__init__.py @@ -19,10 +19,8 @@ from .links import ( EntityLinks, LinkedEntity, - LinksSearchResponseOut, ) from .links_mgr import LinksMgr from .models import ( EntityAttribute, - MetadataOut, ) diff --git a/psengine/links/links.py b/psengine/links/links.py index 6f1b82c9..0447e22b 100644 --- a/psengine/links/links.py +++ b/psengine/links/links.py @@ -116,6 +116,7 @@ def threat_actors(self) -> list[LinkedTA]: if link.type_ == 'type:Organization': is_threat_actor = next( (attr.value for attr in link.attributes if attr.id_ == 'threat_actor'), + False, ) if is_threat_actor: tas.append( @@ -143,12 +144,6 @@ def malwares(self) -> list[LinkedMalware]: ] -class LinksSearchResponseOut(RFBaseModel): - """Response from POST `/links/search` endpoint.""" - - data: list[EntityLinks] = [] - - class LinksSearchIn(RFBaseModel): """Model for payload sent to POST `/links/search` endpoint.""" diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index 30551136..d24d13de 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -17,6 +17,7 @@ from pydantic import AfterValidator, validate_call from typing_extensions import Doc +from ..common_models import IdName from ..endpoints import ( EP_LINKS_METADATA_ENTITIES, EP_LINKS_METADATA_EVENTS, @@ -28,15 +29,14 @@ from ..rf_client import RFClient from .errors import LinksMetadataError, LinksSearchError from .links import ( + EntityLinks, LinksSearchIn, - LinksSearchResponseOut, ) from .models import ( FilterTechnical, LinksFilterObjects, LinksLimitsObjects, LinkSource, - MetadataOut, SearchScope, ) @@ -57,7 +57,7 @@ def __init__( @connection_exceptions(ignore_status_code=[], exception_to_raise=LinksMetadataError) def list_sections( self, - ) -> Annotated[MetadataOut, Doc('Section objects with id, name, and description.')]: + ) -> Annotated[list[IdName], Doc('List of section objects with id and name.')]: """List all sections that can be used to filter a Link search. Sections are the high-level categories the Links API groups results into, @@ -71,14 +71,14 @@ def list_sections( LinksMetadataError: If an API or connection error occurs. """ response = self.rf_client.request(method='GET', url=EP_LINKS_METADATA_SECTIONS) - return MetadataOut.model_validate(response.json()).data + return [IdName.model_validate(item) for item in response.json()['data']] @debug_call @validate_call @connection_exceptions(ignore_status_code=[], exception_to_raise=LinksMetadataError) def list_events( self, - ) -> Annotated[MetadataOut, Doc('Event objects with id, name, and description.')]: + ) -> Annotated[list[IdName], Doc('List of event objects with id and name.')]: """List all event types that can be used to filter technical Link searches. Event types describe the kind of analytical evidence that produced a @@ -92,14 +92,14 @@ def list_events( LinksMetadataError: If an API or connection error occurs. """ response = self.rf_client.request(method='GET', url=EP_LINKS_METADATA_EVENTS) - return MetadataOut.model_validate(response.json()).data + return [IdName.model_validate(item) for item in response.json()['data']] @debug_call @validate_call @connection_exceptions(ignore_status_code=[], exception_to_raise=LinksMetadataError) def list_entity_types( self, - ) -> Annotated[MetadataOut, Doc('Entity-type objects with id and name.')]: + ) -> Annotated[list[IdName], Doc('List of entity-type objects with id and name.')]: """List all entity types that can be used to filter a Link search. The returned values are the supported types for connected entities @@ -113,7 +113,7 @@ def list_entity_types( LinksMetadataError: If an API or connection error occurs. """ response = self.rf_client.request(method='GET', url=EP_LINKS_METADATA_ENTITIES) - return MetadataOut.model_validate(response.json()).data + return [IdName.model_validate(item) for item in response.json()['data']] @debug_call @validate_call @@ -155,7 +155,8 @@ def search( int | None, Doc('Max linked entities returned per entity type.') ] = None, ) -> Annotated[ - list[LinksSearchResponseOut], Doc('A list of LinkResult models — one per input entity.') + list[EntityLinks], + Doc('A list of EntityLinks objects'), ]: """Search for entities connected to one or more target entities. @@ -205,4 +206,4 @@ def search( self.log.info(f'Executing links search for {len(entities)} entities.') response = self.rf_client.request(method='POST', url=EP_LINKS_SEARCH, data=payload) - return LinksSearchResponseOut.model_validate(response.json()) + return [EntityLinks.model_validate(item) for item in response.json()['data']] diff --git a/psengine/links/models.py b/psengine/links/models.py index 7d94bfaa..c3beeb77 100644 --- a/psengine/links/models.py +++ b/psengine/links/models.py @@ -16,24 +16,10 @@ from pydantic import AfterValidator, BeforeValidator, Field -from ..common_models import IdName, RFBaseModel +from ..common_models import RFBaseModel from ..helpers.helpers import Validators -class Metadata(IdName): - description: str | None = None - - -class MetadataOut(RFBaseModel): - """Response for endpoints: - - `/links/metadata/sections` - - `/links/metadata/events` - - `/links/metadata/entities`. - """ - - data: list[Metadata] - - class RiskAttribute(RFBaseModel): id_: Literal['risk_score', 'risk_level'] = Field(alias='id') value: float | str | None = None diff --git a/tests/links/test_links_mgr.py b/tests/links/test_links_mgr.py index 336a4fa6..7f6697c1 100644 --- a/tests/links/test_links_mgr.py +++ b/tests/links/test_links_mgr.py @@ -16,6 +16,7 @@ from requests import Response from requests.exceptions import HTTPError +from psengine.links import LinksMgr from psengine.links.errors import LinksMetadataError, LinksSearchError from psengine.links.models import ( CriticalityAttribute, @@ -27,7 +28,7 @@ ) -def test_list_sections(links_mgr, mocker, make_response): +def test_list_sections(links_mgr: LinksMgr, mocker, make_response): mock_data = { 'data': [ {'id': 's1', 'name': 'Section 1', 'description': 'Desc 1'}, @@ -42,7 +43,7 @@ def test_list_sections(links_mgr, mocker, make_response): assert sections[1].name == 'Section 2' -def test_list_events(links_mgr, mocker, make_response): +def test_list_events(links_mgr: LinksMgr, mocker, make_response): mock_data = { 'data': [ {'id': 'e1', 'name': 'Event 1', 'description': 'Desc 1'}, @@ -57,7 +58,7 @@ def test_list_events(links_mgr, mocker, make_response): assert events[1].name == 'Event 2' -def test_list_entity_types(links_mgr, mocker, make_response): +def test_list_entity_types(links_mgr: LinksMgr, mocker, make_response): mock_data = { 'data': [ {'id': 'Type1', 'name': 'Type 1'}, @@ -72,7 +73,7 @@ def test_list_entity_types(links_mgr, mocker, make_response): assert entity_types[1].name == 'Type 2' -def test_search_basic(links_mgr, mocker, make_response): +def test_search_basic(links_mgr: LinksMgr, mocker, make_response): mock_data = { 'data': [ { @@ -93,10 +94,10 @@ def test_search_basic(links_mgr, mocker, make_response): mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) results = links_mgr.search(entities=['ent1']) - assert len(results.data) == 1 - assert results.data[0].entity.id_ == 'ent1' - assert len(results.data[0].links) == 1 - assert results.data[0].links[0].id_ == 'link1' + assert len(results) == 1 + assert results[0].entity.id_ == 'ent1' + assert len(results[0].links) == 1 + assert results[0].links[0].id_ == 'link1' def test_filter_objects_invalid_source(): @@ -104,7 +105,7 @@ def test_filter_objects_invalid_source(): LinksFilterObjects(sources=['invalid_source']) -def test_metadata_error(links_mgr, mocker): +def test_metadata_error(links_mgr: LinksMgr, mocker): mock_resp = mocker.Mock(spec=Response) mock_resp.status_code = 500 mock_resp.text = 'Internal Server Error' @@ -118,7 +119,7 @@ def test_metadata_error(links_mgr, mocker): links_mgr.list_sections() -def test_search_error(links_mgr, mocker): +def test_search_error(links_mgr: LinksMgr, mocker): mock_resp = mocker.Mock(spec=Response) mock_resp.status_code = 400 mock_resp.text = 'Bad Request' @@ -132,7 +133,7 @@ def test_search_error(links_mgr, mocker): links_mgr.search(entities=['ent1']) -def test_search_complex_attributes(links_mgr, mocker, make_response): +def test_search_complex_attributes(links_mgr: LinksMgr, mocker, make_response): mock_data = { 'data': [ { @@ -158,7 +159,7 @@ def test_search_complex_attributes(links_mgr, mocker, make_response): mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) results = links_mgr.search(entities=['ent1']) - attrs = results.data[0].links[0].attributes + attrs = results[0].links[0].attributes assert len(attrs) == 6 assert isinstance(attrs[0], RiskAttribute) @@ -183,7 +184,7 @@ def test_search_complex_attributes(links_mgr, mocker, make_response): assert attrs[5].value == 'some value' -def test_search_with_limits(links_mgr, mocker, make_response): +def test_search_with_limits(links_mgr: LinksMgr, mocker, make_response): mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response({'data': []})) links_mgr.search(entities=['ent1'], search_scope='small', per_entity_type=10) @@ -193,7 +194,9 @@ def test_search_with_limits(links_mgr, mocker, make_response): assert kwargs['data']['limits']['per_entity_type'] == 10 -def test_entity_links_iocs_groups_results_and_defaults_risk_score(links_mgr, mocker, make_response): +def test_entity_links_iocs_groups_results_and_defaults_risk_score( + links_mgr: LinksMgr, mocker, make_response +): mock_data = { 'data': [ { @@ -225,7 +228,7 @@ def test_entity_links_iocs_groups_results_and_defaults_risk_score(links_mgr, moc } mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) - entity_links = links_mgr.search(entities=['ent1']).data[0] + entity_links = links_mgr.search(entities=['ent1'])[0] iocs = entity_links.iocs() assert set(iocs.keys()) == {'type:InternetDomainName', 'type:Hash'} @@ -237,7 +240,7 @@ def test_entity_links_iocs_groups_results_and_defaults_risk_score(links_mgr, moc assert iocs['type:Hash'][0].source == 'insikt' -def test_entity_links_ttps_filters_mitre_attack_links(links_mgr, mocker, make_response): +def test_entity_links_ttps_filters_mitre_attack_links(links_mgr: LinksMgr, mocker, make_response): mock_data = { 'data': [ { @@ -269,7 +272,7 @@ def test_entity_links_ttps_filters_mitre_attack_links(links_mgr, mocker, make_re } mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) - ttps = links_mgr.search(entities=['ent1']).data[0].ttps() + ttps = links_mgr.search(entities=['ent1'])[0].ttps() assert len(ttps) == 2 assert ttps[0].id_ == 'ttp1' @@ -281,7 +284,7 @@ def test_entity_links_ttps_filters_mitre_attack_links(links_mgr, mocker, make_re def test_entity_links_threat_actors_only_returns_marked_organizations( - links_mgr, mocker, make_response + links_mgr: LinksMgr, mocker, make_response ): mock_data = { 'data': [ @@ -313,7 +316,7 @@ def test_entity_links_threat_actors_only_returns_marked_organizations( } mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) - threat_actors = links_mgr.search(entities=['ent1']).data[0].threat_actors() + threat_actors = links_mgr.search(entities=['ent1'])[0].threat_actors() assert len(threat_actors) == 1 assert threat_actors[0].id_ == 'ta1' @@ -321,7 +324,9 @@ def test_entity_links_threat_actors_only_returns_marked_organizations( assert threat_actors[0].source == 'insikt' -def test_entity_links_malwares_only_returns_malware_links(links_mgr, mocker, make_response): +def test_entity_links_malwares_only_returns_malware_links( + links_mgr: LinksMgr, mocker, make_response +): mock_data = { 'data': [ { @@ -353,8 +358,34 @@ def test_entity_links_malwares_only_returns_malware_links(links_mgr, mocker, mak } mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) - malwares = links_mgr.search(entities=['ent1']).data[0].malwares() + malwares = links_mgr.search(entities=['ent1'])[0].malwares() assert len(malwares) == 2 assert [malware.id_ for malware in malwares] == ['mw1', 'mw2'] assert [malware.source for malware in malwares] == ['technical', 'insikt'] + + +def test_entity_links_threat_actors_skips_organization_missing_attribute( + links_mgr: LinksMgr, mocker, make_response +): + mock_data = { + 'data': [ + { + 'entity': {'id': 'ent1', 'name': 'Entity 1', 'type': 'Type1'}, + 'links': [ + { + 'id': 'org-no-attr', + 'name': 'Organization Without Flag', + 'type': 'type:Organization', + 'source': 'insikt', + 'attributes': [], + }, + ], + } + ] + } + mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response(mock_data)) + + threat_actors = links_mgr.search(entities=['ent1'])[0].threat_actors() + + assert threat_actors == [] From b8452ba46a9fb7fc239b4da5581213676f2830ee Mon Sep 17 00:00:00 2001 From: ebartosevic Date: Fri, 22 May 2026 10:18:09 +0100 Subject: [PATCH 20/24] PSF-1254 links docs tweaks --- docs/examples/links/example_1.py | 2 +- docs/examples/links/example_2.py | 4 ++-- docs/modules/links.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/examples/links/example_1.py b/docs/examples/links/example_1.py index 03f177ef..78e41dbc 100644 --- a/docs/examples/links/example_1.py +++ b/docs/examples/links/example_1.py @@ -13,4 +13,4 @@ print(f'Entity: {entity.name}') for link in result.links[:5]: - print(f' -> {link.name} source:{link.source}') + print(f' -> {link.name} source: {link.source}') diff --git a/docs/examples/links/example_2.py b/docs/examples/links/example_2.py index 86cdf5bd..dde76052 100644 --- a/docs/examples/links/example_2.py +++ b/docs/examples/links/example_2.py @@ -3,10 +3,10 @@ mgr = LinksMgr() results = mgr.search( - entities=['QCwdoU'], + entities=['I60vfZ'], sources=['technical'], entity_types=['type:Malware'], - timeframe='-30d', + timeframe='-90d', search_scope='small', per_entity_type=50, ) diff --git a/docs/modules/links.md b/docs/modules/links.md index 210857b3..aa75c2f7 100644 --- a/docs/modules/links.md +++ b/docs/modules/links.md @@ -38,7 +38,7 @@ Entity: Lazarus Group #### 2: Filter link results and apply limits -In this example, we pass filter and limit arguments directly to `search` (for example `sources`, `entity_types`, `timeframe`, `search_scope`, and `per_entity_type`) to narrow results to technical malware links seen in the last 30 days and cap result size per entity type. +In this example, we pass filter and limit arguments directly to `search` (for example `sources`, `entity_types`, `timeframe`, `search_scope`, and `per_entity_type`) to narrow results to technical malware links seen in the last 90 days and cap result size per entity type. ```python --8<-- "docs/examples/links/example_2.py" From 028a17cb3a437729c521e9e715a22a64ff6f8749 Mon Sep 17 00:00:00 2001 From: ebartosevic Date: Fri, 22 May 2026 10:19:03 +0100 Subject: [PATCH 21/24] small doc tweak --- docs/modules/links.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/modules/links.md b/docs/modules/links.md index aa75c2f7..1829ed7f 100644 --- a/docs/modules/links.md +++ b/docs/modules/links.md @@ -28,11 +28,11 @@ The output will be: ``` Entity: Lazarus Group - -> CVE-2022-47966 source:insikt - -> 24988feb1b38f400069acec4514aa4deea3f6ca8ceb5296f54926e2b22af1e5a source:insikt - -> 36db27f5eb3343cfc72d261d78da44957a49cb6731acb50a96ea5694f4d616c5 source:insikt - -> ffec6e6d4e314f64f5d31c62024252abde7f77acdd63991cb16923ff17828885 source:insikt - -> 3e5fd9acdab438ffc8b2cce48c91679d3f980d08f9dea47d5e1039d352cd64fb source:insikt + -> CVE-2022-47966 source: insikt + -> 24988feb1b38f400069acec4514aa4deea3f6ca8ceb5296f54926e2b22af1e5a source: insikt + -> 36db27f5eb3343cfc72d261d78da44957a49cb6731acb50a96ea5694f4d616c5 source: insikt + -> ffec6e6d4e314f64f5d31c62024252abde7f77acdd63991cb16923ff17828885 source: insikt + -> 3e5fd9acdab438ffc8b2cce48c91679d3f980d08f9dea47d5e1039d352cd64fb source: insikt ``` From 57342b4ef5009b3cc242048a8a08788c5757688a Mon Sep 17 00:00:00 2001 From: ebartosevic Date: Fri, 22 May 2026 10:49:43 +0100 Subject: [PATCH 22/24] PSF-1254 docs improv --- docs/modules/links.md | 2 +- psengine/links/links_mgr.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/modules/links.md b/docs/modules/links.md index 1829ed7f..dbc687ca 100644 --- a/docs/modules/links.md +++ b/docs/modules/links.md @@ -1,6 +1,6 @@ ## Introduction -The `LinksMgr` class of the `links` module allows you to find entities connected to one or more Recorded Future entities. +The `LinksMgr` class of the `links` module allows you to search for technically validated relationships between threat intelligence entities in the Recorded Future Intelligence Cloud — connections established through sandbox analysis, infrastructure analysis, network traffic analysis, and Insikt Group research. See the [**API Reference**](../api/links/links_mgr.md) for internal details of the module. diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index d24d13de..49431335 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -158,7 +158,10 @@ def search( list[EntityLinks], Doc('A list of EntityLinks objects'), ]: - """Search for entities connected to one or more target entities. + """Search for technically validated relationships between threat intelligence + entities in the Recorded Future Intelligence Cloud — connections established + through sandbox analysis, infrastructure analysis, network traffic analysis, + and Insikt Group research. Issues a single batched request: the response contains one `EntityLinks` per entity in `entities`, in the same order. If the From 46d90029642a25ef9b5324550048d9976a65db5e Mon Sep 17 00:00:00 2001 From: ebartosevic Date: Fri, 22 May 2026 11:13:56 +0100 Subject: [PATCH 23/24] PSF-1254 drop enum for literals --- psengine/links/__init__.py | 12 ++++++++++++ psengine/links/links_mgr.py | 18 ++++++++++-------- psengine/links/models.py | 18 +++--------------- tests/links/test_links_models.py | 30 ++++++++++-------------------- 4 files changed, 35 insertions(+), 43 deletions(-) diff --git a/psengine/links/__init__.py b/psengine/links/__init__.py index 4b7f43b2..0abb25b2 100644 --- a/psengine/links/__init__.py +++ b/psengine/links/__init__.py @@ -19,8 +19,20 @@ from .links import ( EntityLinks, LinkedEntity, + LinkedIOC, + LinkedMalware, + LinkedTA, + LinkedTTP, ) from .links_mgr import LinksMgr from .models import ( + CriticalityAttribute, EntityAttribute, + EntitySearchError, + GenericAttribute, + LinkSource, + MitreNameAttribute, + RiskAttribute, + SearchScope, + ThreatActorAttribute, ) diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index 49431335..db39decf 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -14,7 +14,7 @@ import logging from typing import Annotated -from pydantic import AfterValidator, validate_call +from pydantic import AfterValidator, Field, validate_call from typing_extensions import Doc from ..common_models import IdName @@ -133,7 +133,7 @@ def search( ] = None, sources: Annotated[ list[LinkSource] | None, - Doc('Limit to source type(s): technical, insikt, or both.'), + Doc('Limit to source type(s): "technical", "insikt", or both if argument omitted.'), ] = None, timeframe: Annotated[ str | None, @@ -149,18 +149,20 @@ def search( ] = None, search_scope: Annotated[ SearchScope | None, - Doc('Result-volume scope: small, medium (default), or large.'), - ] = None, + Doc('Result-volume scope: "small", "medium" (default), or "large".'), + ] = 'medium', per_entity_type: Annotated[ - int | None, Doc('Max linked entities returned per entity type.') + int | None, + Field(ge=1, le=1_000_000_000), + Doc('Max linked entities returned per entity type (>= 1 <= 1,000,000,000).'), ] = None, ) -> Annotated[ list[EntityLinks], Doc('A list of EntityLinks objects'), ]: - """Search for technically validated relationships between threat intelligence - entities in the Recorded Future Intelligence Cloud — connections established - through sandbox analysis, infrastructure analysis, network traffic analysis, + """Search for technically validated relationships between threat intelligence + entities in the Recorded Future Intelligence Cloud — connections established + through sandbox analysis, infrastructure analysis, network traffic analysis, and Insikt Group research. Issues a single batched request: the response contains one diff --git a/psengine/links/models.py b/psengine/links/models.py index c3beeb77..3f495ba4 100644 --- a/psengine/links/models.py +++ b/psengine/links/models.py @@ -11,7 +11,6 @@ # accessed from any third party API. # ############################################################################################## -from enum import Enum from typing import Annotated, Any, Literal from pydantic import AfterValidator, BeforeValidator, Field @@ -61,19 +60,8 @@ class EntitySearchError(RFBaseModel): status_code: int -class LinkSource(Enum): - """RF Links API source filter.""" - - technical = 'technical' - insikt = 'insikt' - - -class SearchScope(Enum): - """RF Links API search scope.""" - - small = 'small' - medium = 'medium' - large = 'large' +LinkSource = Literal['technical', 'insikt'] +SearchScope = Literal['small', 'medium', 'large'] class FilterTechnical(RFBaseModel): @@ -101,4 +89,4 @@ class LinksLimitsObjects(RFBaseModel): """Objects in the limits object fields.""" search_scope: SearchScope | None = None - per_entity_type: int | None = None + per_entity_type: Annotated[int, Field(ge=1, le=1_000_000_000)] | None = None diff --git a/tests/links/test_links_models.py b/tests/links/test_links_models.py index 1b1e170c..244d5ad2 100644 --- a/tests/links/test_links_models.py +++ b/tests/links/test_links_models.py @@ -19,19 +19,9 @@ FilterTechnical, LinksFilterObjects, LinksLimitsObjects, - LinkSource, - SearchScope, ) -def test_link_source_enum_values(): - assert [member.value for member in LinkSource] == ['technical', 'insikt'] - - -def test_search_scope_enum_values(): - assert [member.value for member in SearchScope] == ['small', 'medium', 'large'] - - def test_filter_technical_timeframe_invalid_format(): with pytest.raises(ValidationError, match='Invalid relative time'): FilterTechnical(timeframe='not-a-time') @@ -80,18 +70,13 @@ def test_links_filter_objects_normalizes_scalar_fields(): } -def test_links_filter_objects_accepts_link_source_enums(): - model = LinksFilterObjects(sources=[LinkSource.technical, LinkSource.insikt]) - assert model.json() == {'sources': ['technical', 'insikt']} - - def test_links_filter_objects_invalid_source(): with pytest.raises(ValidationError, match='sources'): LinksFilterObjects(sources=['invalid_source']) -def test_links_limits_objects_serializes_search_scope_enum(): - model = LinksLimitsObjects(search_scope=SearchScope.small, per_entity_type=25) +def test_links_limits_objects_serializes_search_scope(): + model = LinksLimitsObjects(search_scope='small', per_entity_type=25) assert model.json() == {'search_scope': 'small', 'per_entity_type': 25} @@ -100,6 +85,11 @@ def test_links_limits_objects_invalid_search_scope(): LinksLimitsObjects(search_scope='huge') +def test_links_limits_objects_per_entity_type_rejects_non_positive(): + with pytest.raises(ValidationError, match='per_entity_type'): + LinksLimitsObjects(per_entity_type=0) + + def test_links_mgr_search_builds_minimal_payload_with_entity_str(links_mgr, mocker, make_response): mocker.patch.object(links_mgr.rf_client, 'request', return_value=make_response({'data': []})) @@ -108,7 +98,7 @@ def test_links_mgr_search_builds_minimal_payload_with_entity_str(links_mgr, mock _, kwargs = links_mgr.rf_client.request.call_args assert kwargs['method'] == 'POST' assert kwargs['url'] == EP_LINKS_SEARCH - assert kwargs['data'] == {'entities': ['ent1']} + assert kwargs['data'] == {'entities': ['ent1'], 'limits': {'search_scope': 'medium'}} def test_links_mgr_search_omits_empty_technical_filter_object(links_mgr, mocker, make_response): @@ -191,11 +181,11 @@ def test_links_mgr_search_builds_full_payload_from_model_inputs(links_mgr, mocke entities=['ent1', 'ent2'], sections='section:actors', entity_types='type:IpAddress', - sources=[LinkSource.technical, LinkSource.insikt], + sources=['technical', 'insikt'], timeframe='-30d', events='type:MalwareAnalysis', connected_entities=['id:EntA', 'id:EntB'], - search_scope=SearchScope.large, + search_scope='large', per_entity_type=10, ) From 6b4402941ccf7bf8de83735b3e1150de811f0d78 Mon Sep 17 00:00:00 2001 From: mmedici Date: Fri, 22 May 2026 13:56:07 +0100 Subject: [PATCH 24/24] forgot one aftervalidator --- docs/examples/links/example_1.py | 2 +- docs/examples/links/example_2.py | 6 +++--- psengine/links/links_mgr.py | 2 +- psengine/links/models.py | 4 +++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/examples/links/example_1.py b/docs/examples/links/example_1.py index 78e41dbc..45c4ac06 100644 --- a/docs/examples/links/example_1.py +++ b/docs/examples/links/example_1.py @@ -2,7 +2,7 @@ mgr = LinksMgr() -results = mgr.search(entities=['QCwdoU']) +results = mgr.search(entities='QCwdoU') for result in results: if result.error: diff --git a/docs/examples/links/example_2.py b/docs/examples/links/example_2.py index dde76052..101afc2f 100644 --- a/docs/examples/links/example_2.py +++ b/docs/examples/links/example_2.py @@ -3,9 +3,9 @@ mgr = LinksMgr() results = mgr.search( - entities=['I60vfZ'], - sources=['technical'], - entity_types=['type:Malware'], + entities='I60vfZ', + sources='technical', + entity_types='type:Malware', timeframe='-90d', search_scope='small', per_entity_type=50, diff --git a/psengine/links/links_mgr.py b/psengine/links/links_mgr.py index db39decf..a4dc50ba 100644 --- a/psengine/links/links_mgr.py +++ b/psengine/links/links_mgr.py @@ -132,7 +132,7 @@ def search( Doc('Restrict linked entities to these entity types (e.g. "type:IpAddress").'), ] = None, sources: Annotated[ - list[LinkSource] | None, + str | list[LinkSource] | None, Doc('Limit to source type(s): "technical", "insikt", or both if argument omitted.'), ] = None, timeframe: Annotated[ diff --git a/psengine/links/models.py b/psengine/links/models.py index 3f495ba4..037e7f6f 100644 --- a/psengine/links/models.py +++ b/psengine/links/models.py @@ -81,7 +81,9 @@ class LinksFilterObjects(RFBaseModel): entity_types: Annotated[list[str] | None, BeforeValidator(Validators.convert_str_to_list)] = ( None ) - sources: list[LinkSource] | None = None + sources: Annotated[list[LinkSource] | None, BeforeValidator(Validators.convert_str_to_list)] = ( + None + ) technical: FilterTechnical | None = None