Test and Score: Add comparison of models

Orange/widgets/evaluate/owtestlearners.py

@@ -12,11 +12,14 @@
 from typing import Any, Optional, List, Dict, Callable
 
 import numpy as np
+import baycomp
 
 from AnyQt import QtGui
-from AnyQt.QtGui import QStandardItem
 from AnyQt.QtCore import Qt, QSize, QThread
 from AnyQt.QtCore import pyqtSlot as Slot
+from AnyQt.QtGui import QStandardItem, QDoubleValidator
+from AnyQt.QtWidgets import QHeaderView, QTableWidgetItem, QTableWidget, \
+    QLabel, QSizePolicy
 
 from Orange.base import Learner
 import Orange.classification
@@ -175,6 +178,10 @@ class Outputs:
     fold_feature = settings.ContextSetting(None)
     fold_feature_selected = settings.ContextSetting(False)
 
+    use_rope = settings.Setting(False)
+    rope = settings.Setting(0.1)
+    comparison_criterion = settings.Setting(0)
+
     TARGET_AVERAGE = "(Average over classes)"
     class_selection = settings.ContextSetting(TARGET_AVERAGE)
 
@@ -275,13 +282,108 @@ def __init__(self):
             callback=self._on_target_class_changed,
             contentsLength=8)
 
+        box = gui.vBox(self.controlArea, "Model Comparison")
+        gui.comboBox(
+            box, self, "comparison_criterion", callback=self.score_changed)
+
+        hbox = gui.hBox(box)
+        gui.checkBox(hbox, self, "use_rope",
+                     "Negligible difference: ",
+                     callback=self.score_changed)
+        gui.lineEdit(hbox, self, "rope", validator=QDoubleValidator(),
+                     controlWidth=70, callback=self.score_changed,
+                     alignment=Qt.AlignRight)
+
         gui.rubber(self.controlArea)
         self.score_table = ScoreTable(self)
         self.score_table.shownScoresChanged.connect(self.update_stats_model)
+        view = self.score_table.view
+        view.horizontalHeader().sortIndicatorChanged.connect(self.score_changed)
+        view.setSizeAdjustPolicy(view.AdjustToContents)
 
         box = gui.vBox(self.mainArea, "Evaluation Results")
         box.layout().addWidget(self.score_table.view)
 
+        self.compbox = box = gui.vBox(self.mainArea, box="Model comparison")
+        table = self.comparison_table = QTableWidget(
+            wordWrap=False, editTriggers=QTableWidget.NoEditTriggers,
+            selectionMode=QTableWidget.NoSelection)
+#        table.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
+ #       table.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
+        table.setSizeAdjustPolicy(table.AdjustToContents)
+
+        header = table.horizontalHeader()
+        header.setSectionResizeMode(QHeaderView.ResizeToContents)
+        header.setDefaultAlignment(Qt.AlignLeft)
+        header.setStretchLastSection(False)
+        box.layout().addWidget(table)
+        box.layout().addWidget(QLabel(
+            "<small>Table shows probabilities that the score for the model in "
+            "the row is higher than that of the model in the column. "
+            "Small numbers show the probability that the difference is"
+            "negligible.</small>", wordWrap=True))
+
+    def score_changed(self):
+        model = self.score_table.model
+        proxy = self.score_table.sorted_model
+
+        keys = (model.data(proxy.mapToSource(proxy.index(row, 0)), Qt.UserRole)
+                for row in range(proxy.rowCount()))
+        slots = [slot for slot in (self.learners[key] for key in keys)
+                 if slot.results if not None and slot.results.success]
+        if len(slots) < 2:
+            return
+
+        scorer = self.scorers[self.comparison_criterion]()
+        self.compbox.setTitle(f"Model comparison by {scorer.name}")
+        if scorer.is_binary:
+            if self.class_selection != self.TARGET_AVERAGE:
+                class_var = self.data.domain.class_var
+                target_index = class_var.values.index(self.class_selection)
+                kw = dict(target=target_index)
+            else:
+                kw = dict(average='weighted')
+        else:
+            kw = {}
+
+        table = self.comparison_table
+        table.setRowCount(len(slots))
+        table.setColumnCount(len(slots))
+
+        names = [learner_name(slot.learner) for slot in slots]
+        table.setVerticalHeaderLabels(names)
+        table.setHorizontalHeaderLabels(names)
+
+        for row, row_slot in enumerate(slots):
+            rowname = names[row]
+            row_values = row_slot.results.value
+            row_folds = scorer.scores_by_folds(row_values, **kw).flatten()
+            for col, col_slot in zip(range(row), slots):
+                colname = names[col]
+                col_values = col_slot.results.value
+                col_folds = scorer.scores_by_folds(col_values, **kw).flatten()
+                if self.use_rope and self.rope:
+                    p0, rope, p1 = baycomp.two_on_single(
+                        row_folds, col_folds, self.rope)
+                    ropeleft = f"\np({rowname} = {colname}) = {rope:.3f}"
+                    ropeup = f"\np({rowname} = {colname}) = {rope:.3f}"
+                    left = QLabel(f"{p0:.3f}<br/><small>{rope:.3f})</small>")
+                    up = QLabel(f"{p1:.3f}<br/><small>{rope:.3f}</small>")
+                else:
+                    p0, p1 = baycomp.two_on_single(row_folds, col_folds)
+                    left = QLabel(f"{p0:.3f}")
+                    up = QLabel(f"{p1:.3f}")
+                    ropeleft = ropeup = ""
+                tooltipleft = f"p({rowname} > {colname}) = {p0:.3f}" + ropeleft
+                tooltipup = f"p({rowname} > {colname}) = {p1:.3f}" + ropeup
+                left.setToolTip(tooltipleft)
+                left.setAlignment(Qt.AlignCenter)
+                up.setAlignment(Qt.AlignCenter)
+                up.setToolTip(tooltipup)
+                table.setCellWidget(row, col, left)
+                table.setCellWidget(col, row, up)
+                table.setItem(col, row, QTableWidgetItem())
+
     @staticmethod
     def sizeHint():
         return QSize(780, 1)
@@ -440,6 +542,8 @@ def _update_scorers(self):
             self.scorers = []
             return
         self.scorers = usable_scorers(self.data.domain.class_var)
+        self.controls.comparison_criterion.addItems(
+            [scorer.long_name or scorer.name for scorer in self.scorers])
 
     @Inputs.preprocessor
     def set_preprocessor(self, preproc):
@@ -611,6 +715,8 @@ def _invalidate(self, which=None):
                         item.setData(None, Qt.DisplayRole)
                         item.setData(None, Qt.ToolTipRole)
 
+            self.comparison_table.clearContents()
+
         self.__needupdate = True
 
     def commit(self):
@@ -866,6 +972,7 @@ def __task_complete(self, f: 'Future[Results]'):
 
         self.score_table.update_header(self.scorers)
         self.update_stats_model()
+        self.score_changed()
 
         self.commit()
 

requirements-core.txt

@@ -19,5 +19,6 @@ networkx
 python-louvain>=0.13
 requests
 openTSNE>=0.3.11
+baycomp>=1.0.2
 pandas
 pyyaml