Source code for immuneML.reports.ml_reports.TrainingPerformance

import warnings
from pathlib import Path

import numpy as np
import pandas as pd
import plotly.graph_objs as go
from sklearn import metrics as sklearn_metrics
from sklearn.preprocessing import label_binarize

from immuneML.data_model.dataset.Dataset import Dataset
from immuneML.environment.Constants import Constants
from immuneML.hyperparameter_optimization import HPSetting
from immuneML.ml_methods.MLMethod import MLMethod
from immuneML.ml_metrics import ml_metrics
from immuneML.ml_metrics.Metric import Metric
from immuneML.ml_metrics.MetricUtil import MetricUtil
from immuneML.reports.ReportOutput import ReportOutput
from immuneML.reports.ReportResult import ReportResult
from immuneML.reports.ml_reports.MLReport import MLReport
from immuneML.util.ParameterValidator import ParameterValidator
from immuneML.util.PathBuilder import PathBuilder

[docs] class TrainingPerformance(MLReport): """ A report that plots the evaluation metrics for the performance given machine learning model and training dataset. The available metrics are accuracy, balanced_accuracy, confusion_matrix, f1_micro, f1_macro, f1_weighted, precision, recall, auc and log_loss (see :py:obj:`immuneML.environment.Metric.Metric`). Arguments: metrics (list): A list of metrics used to evaluate training performance. See :py:obj:`immuneML.environment.Metric.Metric` for available options. YAML specification: .. indent with spaces .. code-block:: yaml my_performance_report: TrainingPerformance: metrics: - accuracy - balanced_accuracy - confusion_matrix - f1_micro - f1_macro - f1_weighted - precision - recall - auc - log_loss """
[docs] @classmethod def build_object(cls, **kwargs): location = "TrainingPerformance" valid_metrics = [ for m in Metric] name = kwargs["name"] if "name" in kwargs else None metrics = kwargs["metrics"] if "metrics" in kwargs else valid_metrics metrics = [m.upper() for m in metrics] ParameterValidator.assert_all_in_valid_list(metrics, valid_metrics, location, 'metrics') return TrainingPerformance(set(metrics), name=name)
def __init__(self, metrics: set, train_dataset: Dataset = None, test_dataset: Dataset = None, method: MLMethod = None, result_path: Path = None, name: str = None, hp_setting: HPSetting = None, label=None, number_of_processes: int = 1): super().__init__(train_dataset=train_dataset, test_dataset=test_dataset, method=method, result_path=result_path, name=name, hp_setting=hp_setting, label=label, number_of_processes=number_of_processes) self.metrics_set = set(metrics) def _generate(self) -> ReportResult: X = self.train_dataset.encoded_data predicted_y = self.method.predict(X, self.label)[] predicted_proba_y = self.method.predict_proba(X, self.label)[][self.label.positive_class] true_y = self.train_dataset.encoded_data.labels[] classes = self.method.get_classes() scores = {} output = { 'tables': [], 'figures': [] } for metric in self.metrics_set: _score = MetricUtil.score_for_metric(metric=Metric.get_metric(metric), predicted_y=predicted_y, predicted_proba_y=predicted_proba_y, true_y=true_y, classes=classes) if metric == 'CONFUSION_MATRIX': self._generate_heatmap(classes, classes, _score, metric, output) else: scores[metric] = _score scores_df = pd.DataFrame.from_dict(scores, orient='index') scores_df.columns = [] self._generate_barplot(scores_df, output) return ReportResult(, info="Plots the evaluation metrics for the performance given machine learning model and training dataset.", output_tables=output['tables'], output_figures=output['figures']) def _generate_barplot(self, df, output): import as px path_csv = self.result_path / f"{}.csv" path_html = self.result_path / f"{}.html" df.to_csv(path_csv) figure =, x=df.index,, labels={'index': "metrics"}, template='plotly_white', color_discrete_sequence=px.colors.diverging.Tealrose, title=f"Evaluation metrics ({self.label})") figure.write_html(str(path_html)) output['tables'].append(ReportOutput(path_csv, "training performance in csv")) output['figures'].append(ReportOutput(path_html, "training performance on selected metrics")) return def _generate_heatmap(self, x, y, z, metric, output, xlabel='Prediction', ylabel='Ground Truth', zlabel='Count'): path_csv = self.result_path / f"{}_{metric.lower()}.csv" path_html = self.result_path / f"{}_{metric.lower()}.html" z_flip = np.flipud(z) hovertext = [] for yi, yy in enumerate(y): hovertext.append(list()) for xi, xx in enumerate(x): hovertext[-1].append(f"{xlabel}: {xx}<br />{ylabel}: {yy}<br />{zlabel}: {z_flip[yi][xi]}") layout = go.Layout( title=f'Evaluation: {metric} ({self.label})', xaxis=dict(title=xlabel), yaxis=dict(title=ylabel) ) trace = go.Heatmap( z=z_flip, x=x, y=y, hoverongaps = False, colorscale = 'burgyl', hoverinfo='text', text=hovertext ) fig = go.Figure(data=[trace], layout=layout) fig.write_html(str(path_html)) z_df = pd.DataFrame(z) z_df.columns = f'{xlabel} (' + pd.Index(map(str, x)) + ')' z_df.index = f'{ylabel} (' + pd.Index(map(str, y)) + ')' z_df.to_csv(path_csv) output['tables'].append(ReportOutput(path_csv, f"TrainingPerformance table ({metric.lower()})")) output['figures'].append(ReportOutput(path_html, f"TrainingPerformance html ({metric.lower()})")) return
[docs] def check_prerequisites(self) -> bool: if not hasattr(self, "result_path") or self.result_path is None: warnings.warn(f"{self.__class__.__name__} requires an output 'path' to be set. {self.__class__.__name__}" f" report will not be created.") return False if self.train_dataset is None or self.train_dataset.encoded_data is None: warnings.warn( f"{self.__class__.__name__}: train dataset is not encoded and can not be run." f"{self.__class__.__name__} report will not be created.") return False if self.method is None: warnings.warn( f"{self.__class__.__name__}: method is not defined and can not be run." f"{self.__class__.__name__} report will not be created.") return False return True