Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion sdp/processors/toloka/accept_if.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@
import toloka.client.project.template_builder
TOLOKA_AVAILABLE = True
except ImportError:
logger.warning("Toloka is currently not supported. AcceptIf processor functionality will be limited.")
TOLOKA_AVAILABLE = False
toloka = None


from tqdm import tqdm

Expand Down Expand Up @@ -82,6 +82,7 @@ def __init__(
self.pool_id = pool_id
if self.config_file:
self.load_config()
self.toloka_available = TOLOKA_AVAILABLE

def load_config(self):
"""
Expand All @@ -107,6 +108,9 @@ def prepare(self):

This method loads necessary configurations and initializes the Toloka client to interact with Toloka's API.
"""
if self.toloka_available != True:
logger.warning("Toloka is currently not supported. AcceptIf processor functionality will be limited.")

if not self.API_KEY or not self.platform or not self.pool_id:
try:
with open(self.input_data_file, 'r') as file:
Expand Down
7 changes: 6 additions & 1 deletion sdp/processors/toloka/create_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@
import toloka.client.project.template_builder
TOLOKA_AVAILABLE = True
except ImportError:
logger.warning("Toloka is currently not supported. CreatePool processor functionality will be limited.")
TOLOKA_AVAILABLE = False
toloka = None



class CreateTolokaPool(BaseParallelProcessor):
Expand Down Expand Up @@ -67,6 +67,7 @@ def __init__(
# Project ID will be read from the input manifest file in process_dataset_entry
self.project_id = None
self.lang = lang
self.toloka_available = TOLOKA_AVAILABLE

def process_dataset_entry(self, data_entry):
"""
Expand All @@ -85,6 +86,10 @@ def process_dataset_entry(self, data_entry):
list
A list containing a DataEntry object with the new pool ID if successful, or an empty list if failed.
"""

if self.toloka_available != True:
logger.warning("Toloka is currently not supported. CreatePool processor functionality will be limited.")

# Get project_id from the data entry
project_id = data_entry.get("project_id")
if not project_id:
Expand Down
5 changes: 4 additions & 1 deletion sdp/processors/toloka/create_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@
import toloka.client.project.template_builder
TOLOKA_AVAILABLE = True
except ImportError:
logger.warning("Toloka is currently not supported. CreateTolokaProject processor functionality will be limited.")
TOLOKA_AVAILABLE = False
toloka = None



class CreateTolokaProject(BaseParallelProcessor):
Expand Down Expand Up @@ -63,6 +63,7 @@ def __init__(
self.project_name = project_name
self.project_description = project_description
self.project_instructions = project_instructions
self.toloka_availabe = TOLOKA_AVAILABLE

def process(self):
"""
Expand All @@ -75,6 +76,8 @@ def process(self):
After creating the project, it saves the project details (including the project ID) to a specified file.
"""
logger.info("Processing Toloka project creation...")
if self.toloka_availabe != True:
logger.warning("Toloka is currently not supported. CreateTolokaProject processor functionality will be limited.")

toloka_client = toloka.client.TolokaClient(self.API_KEY, self.platform)

Expand Down
8 changes: 7 additions & 1 deletion sdp/processors/toloka/create_task_set.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,12 @@
import toloka.client.project.template_builder
TOLOKA_AVAILABLE = True
except ImportError:
logger.warning("Toloka is currently not supported. CreateTaskSet processor functionality will be limited.")
TOLOKA_AVAILABLE = False
toloka = None




class CreateTolokaTaskSet(BaseParallelProcessor):
"""Creates a set of tasks in a Toloka pool based on user-provided configurations and input data.

Expand All @@ -57,6 +57,10 @@ def __init__(
self.input_pool_file = input_pool_file
self.limit = limit
self.pool_id = None
self.toloka_available = TOLOKA_AVAILABLE




# Get API key and platform from environment variables
self.API_KEY = os.getenv('TOLOKA_API_KEY')
Expand All @@ -76,6 +80,8 @@ def prepare(self):
This method sets up the necessary components for task creation, including loading the
pool configuration and initializing the Toloka client.
"""
if self.toloka_available != True:
logger.warning("Toloka is currently not supported. CreateTaskSet processor functionality will be limited.")
self.load_pool_config()
self.toloka_client = toloka.client.TolokaClient(self.API_KEY, self.platform)

Expand Down
7 changes: 6 additions & 1 deletion sdp/processors/toloka/download_responses.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@
import toloka.client
TOLOKA_AVAILABLE = True
except ImportError:
logger.warning("Toloka is currently not supported. DownloadResponses processor functionality will be limited.")
TOLOKA_AVAILABLE = False
toloka = None




Expand Down Expand Up @@ -92,6 +92,7 @@ def __init__(
self.pool_id = pool_id
if self.config_file:
self.load_config()
self.toloka_available = TOLOKA_AVAILABLE

def load_config(self):
"""
Expand All @@ -100,6 +101,7 @@ def load_config(self):
This method attempts to read configuration details such as API key, platform, and pool ID from a JSON file.
If the file is missing or improperly formatted, an appropriate error is logged.
"""

try:
with open(self.config_file, 'r') as file:
config = json.load(file)
Expand All @@ -117,6 +119,9 @@ def prepare(self):

This method loads necessary configurations and initializes the Toloka client to interact with Toloka's API.
"""
if self.toloka_available != True:
logger.warning("Toloka is currently not supported. DownloadResponses processor functionality will be limited.")

if not self.API_KEY or not self.platform or not self.pool_id:
try:
with open(self.input_data_file, 'r') as file:
Expand Down
5 changes: 4 additions & 1 deletion sdp/processors/toloka/reject_if.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@
import toloka.client.project.template_builder
TOLOKA_AVAILABLE = True
except ImportError:
logger.warning("Toloka is currently not supported. RejectIf processor functionality will be limited.")
TOLOKA_AVAILABLE = False
toloka = None


from docx import Document
from tqdm import tqdm
Expand Down Expand Up @@ -83,6 +83,7 @@ def __init__(
self.API_KEY = API_KEY or os.getenv('TOLOKA_API_KEY')
self.platform = platform or os.getenv('TOLOKA_PLATFORM')
self.pool_id = pool_id
self.toloka_available = TOLOKA_AVAILABLE
if self.config_file:
self.load_config()

Expand Down Expand Up @@ -110,6 +111,8 @@ def prepare(self):

This method loads necessary configurations and initializes the Toloka client to interact with Toloka's API.
"""
if self.toloka_available != True:
logger.warning("Toloka is currently not supported. RejectIf processor functionality will be limited.")
if not self.API_KEY or not self.platform or not self.pool_id:
try:
with open(self.input_data_file, 'r') as file:
Expand Down
Loading