Test and Score: Warn about transformation, raise error if all is nan

Orange/base.py

@@ -9,6 +9,7 @@
 
 from Orange.data import Table, Storage, Instance, Value
 from Orange.data.filter import HasClass
+from Orange.data.table import DomainTransformationError
 from Orange.data.util import one_hot
 from Orange.misc.wrapper_meta import WrapperMeta
 from Orange.preprocess import Continuize, RemoveNaNColumns, SklImpute, Normalize
@@ -243,10 +244,14 @@ def __call__(self, data, ret=Value):
         elif isinstance(data, scipy.sparse.csr.csr_matrix):
             prediction = self.predict(data)
         elif isinstance(data, (Table, Instance)):
+            orig_data = data
             if isinstance(data, Instance):
                 data = Table(data.domain, [data])
             if data.domain != self.domain:
                 data = data.transform(self.domain)
+            if data is not orig_data and np.all(np.isnan(data.X)):
+                raise DomainTransformationError(
+                    "domain transformation produced no defined values")
             prediction = self.predict_storage(data)
         elif isinstance(data, (list, tuple)):
             if not isinstance(data[0], (list, tuple)):

Orange/data/table.py

@@ -44,6 +44,10 @@ def get_sample_datasets_dir():
 _conversion_cache_lock = RLock()
 
 
+class DomainTransformationError(Exception):
+    pass
+
+
 class RowInstance(Instance):
     sparse_x = None
     sparse_y = None

Orange/widgets/evaluate/owtestlearners.py

@@ -14,6 +14,8 @@
 from concurrent.futures import Future
 from collections import OrderedDict, namedtuple
 
+from Orange.data.table import DomainTransformationError
+
 try:
     # only used in type hinting
     # pylint: disable=unused-import
@@ -209,6 +211,8 @@ class Error(OWWidget.Error):
         memory_error = Msg("Not enough memory.")
         no_class_values = Msg("Target variable has no values.")
         only_one_class_var_value = Msg("Target variable has only one value.")
+        test_data_incompatible = Msg(
+            "Test data may be incompatible with train data.")
 
     class Warning(OWWidget.Warning):
         missing_data = \
@@ -221,6 +225,8 @@ class Warning(OWWidget.Warning):
     class Information(OWWidget.Information):
         data_sampled = Msg("Train data has been sampled")
         test_data_sampled = Msg("Test data has been sampled")
+        test_data_transformed = Msg(
+            "Test data has been transformed to match the train data.")
 
     def __init__(self):
         super().__init__()
@@ -551,14 +557,20 @@ def _update_stats_model(self):
             name = learner_name(slot.learner)
             head = QStandardItem(name)
             head.setData(key, Qt.UserRole)
-            if isinstance(slot.results, Try.Fail):
-                head.setToolTip(str(slot.results.exception))
+            results = slot.results
+            if isinstance(results, Try.Fail):
+                head.setToolTip(str(results.exception))
                 head.setText("{} (error)".format(name))
                 head.setForeground(QtGui.QBrush(Qt.red))
-                errors.append("{name} failed with error:\n"
-                              "{exc.__class__.__name__}: {exc!s}"
-                              .format(name=name, exc=slot.results.exception))
-
+                if isinstance(results.exception, DomainTransformationError) \
+                        and self.resampling == self.TestOnTest:
+                    self.Error.test_data_incompatible()
+                    self.Information.test_data_transformed.clear()
+                else:
+                    errors.append("{name} failed with error:\n"
+                                  "{exc.__class__.__name__}: {exc!s}"
+                                  .format(name=name, exc=slot.results.exception)
+                                  )
             row = [head]
 
             if class_var is not None and class_var.is_discrete and \
@@ -744,7 +756,13 @@ def __update(self):
             self.cancel()
 
         self.Warning.test_data_unused.clear()
+        self.Error.test_data_incompatible.clear()
         self.Warning.test_data_missing.clear()
+        self.Information.test_data_transformed(
+            shown=self.resampling == self.TestOnTest
+            and self.data is not None
+            and self.test_data is not None
+            and self.data.domain.attributes != self.test_data.domain.attributes)
         self.warning()
         self.Error.class_inconsistent.clear()
         self.Error.too_many_folds.clear()