Repository navigation
Expand file tree
/
Copy pathtempCodeRunnerFile.py
More file actions
57 lines (47 loc) · 2.33 KB
/
Copy pathtempCodeRunnerFile.py
File metadata and controls
57 lines (47 loc) · 2.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
'min_samples_split': [2, 5, 10],
'min_samples_leaf': [1, 2, 4]
}
dt = DecisionTreeClassifier(random_state=42)
grid_search = GridSearchCV(dt, param_grid, cv=5, scoring='accuracy')
grid_search.fit(X_train, y_train)
self.model = grid_search.best_estimator_
y_pred = self.model.predict(X_test)
# Log results
logging.info(f"Best Parameters: {grid_search.best_params_}")
logging.info(f"\nClassification Report:\n{classification_report(y_test, y_pred)}")
logging.info(f"Accuracy Score: {accuracy_score(y_test, y_pred):.4f}")
# Save confusion matrix
self.plot_confusion_matrix(y_test, y_pred)
except Exception as e:
logging.error(f"Error during training: {str(e)}")
raise
def predict(self, texts: List[str]) -> List[str]:
"""Predict classifications for new texts"""
try:
cleaned_texts = [self.preprocessor.clean_text(text) for text in texts]
vectors = self.tfidf.transform(cleaned_texts)
predictions = self.model.predict(vectors)
return predictions
except Exception as e:
logging.error(f"Error during prediction: {str(e)}")
raise
def plot_confusion_matrix(self, y_true: np.ndarray, y_pred: np.ndarray) -> None:
"""Plot and save confusion matrix"""
plt.figure(figsize=(10,8))
cm = confusion_matrix(y_true, y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap="YlGnBu")
plt.title('Confusion Matrix')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.savefig('confusion_matrix.png')
plt.close()
def save_model(self) -> None:
"""Save the trained model and vectorizer"""
if not os.path.exists(self.model_path):
os.makedirs(self.model_path)
joblib.dump(self.model, f"{self.model_path}/model.joblib")
joblib.dump(self.tfidf, f"{self.model_path}/tfidf.joblib")
def load_model(self) -> None:
"""Load a trained model and vectorizer"""
self.model = joblib.load(f"{self.model_path}/model.joblib")
self.tfidf = joblib.load(f"{self.model_path}/tfidf.joblib")