From 72657931519c7b4af84124a6e172b9f1e6a46a67 Mon Sep 17 00:00:00 2001 From: Andreja Date: Thu, 29 Nov 2018 15:28:20 +0100 Subject: [PATCH 1/8] removed header types and flags from .cvs and .tab --- Orange/data/io.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Orange/data/io.py b/Orange/data/io.py index 5125fe2fc7a..37f4b55d608 100644 --- a/Orange/data/io.py +++ b/Orange/data/io.py @@ -922,6 +922,10 @@ def write_file(cls, filename, data): cls.write_data(writer.writerow, data) cls.write_table_metadata(filename, data) + @classmethod + def write_headers(cls, write, data): + write(cls.header_names(data)) + class TabReader(CSVReader): """Reader for tab separated files""" From fc349313d27d1f5ccdfbc9b93cbbe8c6c5bef8e6 Mon Sep 17 00:00:00 2001 From: Andreja Date: Thu, 10 Jan 2019 17:23:55 +0100 Subject: [PATCH 2/8] enable no annotations save for .csv and .tab --- Orange/data/io.py | 26 ++++++++++++-------------- Orange/widgets/data/owsave.py | 11 ++++++++++- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/Orange/data/io.py b/Orange/data/io.py index 37f4b55d608..ecee59ef680 100644 --- a/Orange/data/io.py +++ b/Orange/data/io.py @@ -431,8 +431,8 @@ def get_reader(cls, filename): raise IOError('No readers for file "{}"'.format(filename)) @classmethod - def write(cls, filename, data): - return cls.write_file(filename, data) + def write(cls, filename, data, with_annotations=True): + return cls.write_file(filename, data, with_annotations) @classmethod def write_table_metadata(cls, filename, data): @@ -798,11 +798,12 @@ def header_flags(data): zip(repeat('meta'), data.domain.metas))))) @classmethod - def write_headers(cls, write, data): + def write_headers(cls, write, data, with_annotations): """`write` is a callback that accepts an iterable""" write(cls.header_names(data)) - write(cls.header_types(data)) - write(cls.header_flags(data)) + if with_annotations: + write(cls.header_types(data)) + write(cls.header_flags(data)) @classmethod def formatter(cls, var): @@ -915,16 +916,13 @@ def read(self): raise ValueError('Cannot parse dataset {}: {}'.format(self.filename, error)) from error @classmethod - def write_file(cls, filename, data): + def write_file(cls, filename, data, with_annotations=True): with cls.open(filename, mode='wt', newline='', encoding='utf-8') as file: writer = csv.writer(file, delimiter=cls.DELIMITERS[0]) - cls.write_headers(writer.writerow, data) + cls.write_headers(writer.writerow, data, with_annotations) cls.write_data(writer.writerow, data) - cls.write_table_metadata(filename, data) - - @classmethod - def write_headers(cls, write, data): - write(cls.header_names(data)) + if with_annotations: + cls.write_table_metadata(filename, data) class TabReader(CSVReader): @@ -951,7 +949,7 @@ def read(self): return table @classmethod - def write_file(cls, filename, data): + def write_file(cls, filename, data, with_annotations=True): with cls.open(filename, 'wb') as f: pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) @@ -1062,7 +1060,7 @@ def write_graph(cls, filename, graph): tree.export_graphviz(graph, out_file=cls.open(filename, 'wt')) @classmethod - def write(cls, filename, tree): + def write(cls, filename, tree, with_annotations): if type(tree) == dict: tree = tree['tree'] cls.write_graph(filename, tree) diff --git a/Orange/widgets/data/owsave.py b/Orange/widgets/data/owsave.py index 437121286d0..3984c996ee4 100644 --- a/Orange/widgets/data/owsave.py +++ b/Orange/widgets/data/owsave.py @@ -48,6 +48,7 @@ class Error(widget.OWWidget.Error): filetype = Setting(FILE_TYPES[0][0]) compression = Setting(COMPRESSIONS[0][0]) compress = Setting(False) + with_annotations = Setting(True) def __init__(self): super().__init__() @@ -89,6 +90,10 @@ def __init__(self): box.layout().addLayout(form) + self.annotations_cb = gui.checkBox( + self.controlArea, self, "with_annotations", label="Save with Orange annotations", + ) + self.save = gui.auto_commit( self.controlArea, self, "auto_save", "Save", box=False, commit=self.save_file, callback=self.adjust_label, @@ -175,7 +180,8 @@ def save_file(self): os.path.join( self.last_dir, self.basename + self.type_ext + self.compress_ext), - self.data) + self.data, self.with_annotations, + ) except Exception as err_value: self.error(str(err_value)) else: @@ -183,6 +189,9 @@ def save_file(self): def update_extension(self): self.type_ext = [ext for name, ext, _ in FILE_TYPES if name == self.filetype][0] + self.annotations_cb.setEnabled(True) + if self.type_ext == '.pkl': + self.annotations_cb.setEnabled(False) self.compress_ext = dict(COMPRESSIONS)[self.compression] if self.compress else '' def _update_text(self): From aea622d534f640d71d548d00d3603ce25b8f2388 Mon Sep 17 00:00:00 2001 From: Andreja Date: Fri, 11 Jan 2019 00:31:01 +0100 Subject: [PATCH 3/8] added support for .xlsx files --- Orange/data/io.py | 2 +- Orange/widgets/data/owsave.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Orange/data/io.py b/Orange/data/io.py index ecee59ef680..f7ade356951 100644 --- a/Orange/data/io.py +++ b/Orange/data/io.py @@ -1026,7 +1026,7 @@ def read(self): return table @classmethod - def write_file(cls, filename, data): + def write_file(cls, filename, data, with_annotations=True): vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (), data.domain.attributes, data.domain.class_vars, diff --git a/Orange/widgets/data/owsave.py b/Orange/widgets/data/owsave.py index 3984c996ee4..db052418b51 100644 --- a/Orange/widgets/data/owsave.py +++ b/Orange/widgets/data/owsave.py @@ -190,7 +190,7 @@ def save_file(self): def update_extension(self): self.type_ext = [ext for name, ext, _ in FILE_TYPES if name == self.filetype][0] self.annotations_cb.setEnabled(True) - if self.type_ext == '.pkl': + if self.type_ext in ['.pkl', '.xlsx']: self.annotations_cb.setEnabled(False) self.compress_ext = dict(COMPRESSIONS)[self.compression] if self.compress else '' From e547593db47c0d9fd82ac7c070a4e8d82a9c191f Mon Sep 17 00:00:00 2001 From: Andreja Date: Fri, 11 Jan 2019 14:24:23 +0100 Subject: [PATCH 4/8] added class flag --- Orange/data/io.py | 6 ++++-- Orange/widgets/data/owsave.py | 29 +++++++++++++++++++---------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Orange/data/io.py b/Orange/data/io.py index f7ade356951..12d8ad3e9fd 100644 --- a/Orange/data/io.py +++ b/Orange/data/io.py @@ -377,6 +377,7 @@ def write_file(cls, filename, data): # Priority when multiple formats support the same extension. Also # the sort order in file open/save combo boxes. Lower is better. PRIORITY = 10000 + OPTIONAL_TYPE_ANNOTATIONS = False def __init__(self, filename): """ @@ -798,7 +799,7 @@ def header_flags(data): zip(repeat('meta'), data.domain.metas))))) @classmethod - def write_headers(cls, write, data, with_annotations): + def write_headers(cls, write, data, with_annotations=True): """`write` is a callback that accepts an iterable""" write(cls.header_names(data)) if with_annotations: @@ -857,6 +858,7 @@ class CSVReader(FileFormat): SUPPORT_COMPRESSED = True SUPPORT_SPARSE_DATA = False PRIORITY = 20 + OPTIONAL_TYPE_ANNOTATIONS = True def read(self): for encoding in (lambda: ('us-ascii', None), # fast @@ -1060,7 +1062,7 @@ def write_graph(cls, filename, graph): tree.export_graphviz(graph, out_file=cls.open(filename, 'wt')) @classmethod - def write(cls, filename, tree, with_annotations): + def write(cls, filename, tree, with_annotations=True): if type(tree) == dict: tree = tree['tree'] cls.write_graph(filename, tree) diff --git a/Orange/widgets/data/owsave.py b/Orange/widgets/data/owsave.py index db052418b51..12e9ee14f46 100644 --- a/Orange/widgets/data/owsave.py +++ b/Orange/widgets/data/owsave.py @@ -91,7 +91,7 @@ def __init__(self): box.layout().addLayout(form) self.annotations_cb = gui.checkBox( - self.controlArea, self, "with_annotations", label="Save with Orange annotations", + self.controlArea, self, "with_annotations", label="Add type annotations", ) self.save = gui.auto_commit( @@ -176,12 +176,21 @@ def save_file(self): self.save_file_as() else: try: - self.get_writer_selected().write( - os.path.join( - self.last_dir, - self.basename + self.type_ext + self.compress_ext), - self.data, self.with_annotations, - ) + if self.get_writer_selected().OPTIONAL_TYPE_ANNOTATIONS: + self.get_writer_selected().write( + os.path.join( + self.last_dir, + self.basename + self.type_ext + self.compress_ext), + self.data, self.with_annotations, + ) + else: + self.get_writer_selected().write( + os.path.join( + self.last_dir, + self.basename + self.type_ext + self.compress_ext), + self.data, + ) + except Exception as err_value: self.error(str(err_value)) else: @@ -189,9 +198,9 @@ def save_file(self): def update_extension(self): self.type_ext = [ext for name, ext, _ in FILE_TYPES if name == self.filetype][0] - self.annotations_cb.setEnabled(True) - if self.type_ext in ['.pkl', '.xlsx']: - self.annotations_cb.setEnabled(False) + self.annotations_cb.setEnabled(False) + if self.get_writer_selected().OPTIONAL_TYPE_ANNOTATIONS: + self.annotations_cb.setEnabled(True) self.compress_ext = dict(COMPRESSIONS)[self.compression] if self.compress else '' def _update_text(self): From a8f6517598e0f2ab52f892421cce261a003a9021 Mon Sep 17 00:00:00 2001 From: Andreja Date: Sun, 13 Jan 2019 20:42:57 +0100 Subject: [PATCH 5/8] renamed vars, improved modularity --- Orange/data/io.py | 11 +++++++---- Orange/widgets/data/owsave.py | 16 ++++------------ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/Orange/data/io.py b/Orange/data/io.py index 12d8ad3e9fd..b2ad948926a 100644 --- a/Orange/data/io.py +++ b/Orange/data/io.py @@ -433,7 +433,10 @@ def get_reader(cls, filename): @classmethod def write(cls, filename, data, with_annotations=True): - return cls.write_file(filename, data, with_annotations) + if cls.OPTIONAL_TYPE_ANNOTATIONS: + return cls.write_file(filename, data, with_annotations) + else: + return cls.write_file(filename, data) @classmethod def write_table_metadata(cls, filename, data): @@ -951,7 +954,7 @@ def read(self): return table @classmethod - def write_file(cls, filename, data, with_annotations=True): + def write_file(cls, filename, data): with cls.open(filename, 'wb') as f: pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) @@ -1028,7 +1031,7 @@ def read(self): return table @classmethod - def write_file(cls, filename, data, with_annotations=True): + def write_file(cls, filename, data): vars = list(chain((ContinuousVariable('_w'),) if data.has_weights() else (), data.domain.attributes, data.domain.class_vars, @@ -1062,7 +1065,7 @@ def write_graph(cls, filename, graph): tree.export_graphviz(graph, out_file=cls.open(filename, 'wt')) @classmethod - def write(cls, filename, tree, with_annotations=True): + def write(cls, filename, tree): if type(tree) == dict: tree = tree['tree'] cls.write_graph(filename, tree) diff --git a/Orange/widgets/data/owsave.py b/Orange/widgets/data/owsave.py index 12e9ee14f46..6b4ea053153 100644 --- a/Orange/widgets/data/owsave.py +++ b/Orange/widgets/data/owsave.py @@ -48,7 +48,7 @@ class Error(widget.OWWidget.Error): filetype = Setting(FILE_TYPES[0][0]) compression = Setting(COMPRESSIONS[0][0]) compress = Setting(False) - with_annotations = Setting(True) + add_type_annotations = Setting(True) def __init__(self): super().__init__() @@ -91,7 +91,7 @@ def __init__(self): box.layout().addLayout(form) self.annotations_cb = gui.checkBox( - self.controlArea, self, "with_annotations", label="Add type annotations", + self.controlArea, self, "add_type_annotations", label="Add type annotations", ) self.save = gui.auto_commit( @@ -176,19 +176,11 @@ def save_file(self): self.save_file_as() else: try: - if self.get_writer_selected().OPTIONAL_TYPE_ANNOTATIONS: - self.get_writer_selected().write( + self.get_writer_selected().write( os.path.join( self.last_dir, self.basename + self.type_ext + self.compress_ext), - self.data, self.with_annotations, - ) - else: - self.get_writer_selected().write( - os.path.join( - self.last_dir, - self.basename + self.type_ext + self.compress_ext), - self.data, + self.data, self.add_type_annotations, ) except Exception as err_value: From 863af5991477adb5ce678eccd830bfbc23bde20d Mon Sep 17 00:00:00 2001 From: Andreja Date: Sun, 13 Jan 2019 22:44:50 +0100 Subject: [PATCH 6/8] lint --- Orange/widgets/data/owsave.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Orange/widgets/data/owsave.py b/Orange/widgets/data/owsave.py index 6b4ea053153..7f664677ff4 100644 --- a/Orange/widgets/data/owsave.py +++ b/Orange/widgets/data/owsave.py @@ -177,11 +177,10 @@ def save_file(self): else: try: self.get_writer_selected().write( - os.path.join( - self.last_dir, - self.basename + self.type_ext + self.compress_ext), - self.data, self.add_type_annotations, - ) + os.path.join( + self.last_dir, + self.basename + self.type_ext + self.compress_ext), + self.data, self.add_type_annotations) except Exception as err_value: self.error(str(err_value)) From 7d0d3f41c4ab9ac51008c7adabbc3730f26189a1 Mon Sep 17 00:00:00 2001 From: Andreja Date: Fri, 18 Jan 2019 11:28:39 +0100 Subject: [PATCH 7/8] added tests --- Orange/data/io.py | 1 - Orange/tests/test_io.py | 31 ++++++++++++++++++++++++ Orange/widgets/data/owsave.py | 3 ++- Orange/widgets/data/tests/test_owsave.py | 28 +++++++++++++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/Orange/data/io.py b/Orange/data/io.py index b2ad948926a..8236b13722e 100644 --- a/Orange/data/io.py +++ b/Orange/data/io.py @@ -926,7 +926,6 @@ def write_file(cls, filename, data, with_annotations=True): writer = csv.writer(file, delimiter=cls.DELIMITERS[0]) cls.write_headers(writer.writerow, data, with_annotations) cls.write_data(writer.writerow, data) - if with_annotations: cls.write_table_metadata(filename, data) diff --git a/Orange/tests/test_io.py b/Orange/tests/test_io.py index 5df274fde8f..27853d23196 100644 --- a/Orange/tests/test_io.py +++ b/Orange/tests/test_io.py @@ -2,13 +2,17 @@ # pylint: disable=missing-docstring import unittest +from unittest.mock import Mock, patch import os import tempfile import shutil import io +import csv from Orange.data.io import FileFormat, TabReader, CSVReader, PickleReader from Orange.data.table import get_sample_datasets_dir +from Orange.data import Table + class WildcardReader(FileFormat): @@ -115,3 +119,30 @@ def test_empty_columns(self): self.assertEqual(len(table.domain.attributes), 2) self.assertEqual(cm.warning.args[0], "Columns with no headers were removed.") + + def test_type_annotations(self): + class FooFormat(FileFormat): + write_file = Mock() + + FooFormat.write('test_file', None) + FooFormat.write_file.assert_called_with('test_file', None) + + FooFormat.OPTIONAL_TYPE_ANNOTATIONS = True + FooFormat.write('test_file', None) + FooFormat.write_file.assert_called_with('test_file', None, True) + + FooFormat.write('test_file', None, False) + FooFormat.write_file.assert_called_with('test_file', None, False) + + FooFormat.OPTIONAL_TYPE_ANNOTATIONS = False + FooFormat.write('test_file', None) + FooFormat.write_file.assert_called_with('test_file', None) + + @patch('csv.DictWriter.writerow') + def test_header_call(self, writer): + CSVReader.write_headers(writer, Table("iris"), False) + self.assertEquals(len(writer.call_args_list), 1) + + writer.reset_mock() + CSVReader.write_headers(writer, Table("iris"), True) + self.assertEquals(len(writer.call_args_list), 3) diff --git a/Orange/widgets/data/owsave.py b/Orange/widgets/data/owsave.py index 7f664677ff4..5976f2b265c 100644 --- a/Orange/widgets/data/owsave.py +++ b/Orange/widgets/data/owsave.py @@ -91,8 +91,9 @@ def __init__(self): box.layout().addLayout(form) self.annotations_cb = gui.checkBox( - self.controlArea, self, "add_type_annotations", label="Add type annotations", + None, self, "add_type_annotations", label="Add type annotations", ) + form.addRow(self.annotations_cb, None) self.save = gui.auto_commit( self.controlArea, self, "auto_save", "Save", box=False, diff --git a/Orange/widgets/data/tests/test_owsave.py b/Orange/widgets/data/tests/test_owsave.py index 49f0c6f6321..bf528f1a3cb 100644 --- a/Orange/widgets/data/tests/test_owsave.py +++ b/Orange/widgets/data/tests/test_owsave.py @@ -66,6 +66,34 @@ def choose_file(a, b, c, d, e, fn=filename, w=writer): self.widget.save_file_as() self.assertEqual(len(Table(filename)), 150) + @patch('Orange.data.io.FileFormat.write') + def test_annotations(self, write): + widget = self.widget + + self.send_signal(widget.Inputs.data, Table("iris")) + widget.filetype = FILE_TYPES[1][0] + widget.filename = 'foo.csv' + widget.update_extension() + + widget.add_type_annotations = False + widget.unconditional_save_file() + write.assert_called() + self.assertFalse(write.call_args[0][2]) + + widget.add_type_annotations = True + widget.unconditional_save_file() + self.assertTrue(write.call_args[0][2]) + + def test_disable_checkbox(self): + widget = self.widget + for type_ in FILE_TYPES: + widget.filetype=type_[0] + widget.update_extension() + if widget.get_writer_selected().OPTIONAL_TYPE_ANNOTATIONS: + self.assertTrue(widget.annotations_cb.isEnabled()) + else: + self.assertFalse(widget.annotations_cb.isEnabled()) + def test_compression(self): self.send_signal(self.widget.Inputs.data, Table("iris")) From 1bd47c7e3994080abfd832bc7124707306a77c2a Mon Sep 17 00:00:00 2001 From: Andreja Date: Fri, 18 Jan 2019 12:25:06 +0100 Subject: [PATCH 8/8] lint --- Orange/tests/test_io.py | 14 ++++++-------- Orange/widgets/data/tests/test_owsave.py | 6 +++--- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Orange/tests/test_io.py b/Orange/tests/test_io.py index 27853d23196..71b8b84ca23 100644 --- a/Orange/tests/test_io.py +++ b/Orange/tests/test_io.py @@ -7,14 +7,12 @@ import tempfile import shutil import io -import csv from Orange.data.io import FileFormat, TabReader, CSVReader, PickleReader from Orange.data.table import get_sample_datasets_dir from Orange.data import Table - class WildcardReader(FileFormat): EXTENSIONS = ('.wild', '.wild[0-9]') DESCRIPTION = "Dummy reader for testing extensions" @@ -86,11 +84,11 @@ def test_locate_wildcard_extension(self): fn = os.path.join(tempdir, "t.wild8") with open(fn, "wt") as f: f.write("\n") - l = FileFormat.locate("t.wild8", search_dirs=[tempdir]) - self.assertEqual(l, fn) + location = FileFormat.locate("t.wild8", search_dirs=[tempdir]) + self.assertEqual(location, fn) # test extension adding - l = FileFormat.locate("t", search_dirs=[tempdir]) - self.assertEqual(l, fn) + location = FileFormat.locate("t", search_dirs=[tempdir]) + self.assertEqual(location, fn) shutil.rmtree(tempdir) @@ -141,8 +139,8 @@ class FooFormat(FileFormat): @patch('csv.DictWriter.writerow') def test_header_call(self, writer): CSVReader.write_headers(writer, Table("iris"), False) - self.assertEquals(len(writer.call_args_list), 1) + self.assertEqual(len(writer.call_args_list), 1) writer.reset_mock() CSVReader.write_headers(writer, Table("iris"), True) - self.assertEquals(len(writer.call_args_list), 3) + self.assertEqual(len(writer.call_args_list), 3) diff --git a/Orange/widgets/data/tests/test_owsave.py b/Orange/widgets/data/tests/test_owsave.py index bf528f1a3cb..d37a3239e28 100644 --- a/Orange/widgets/data/tests/test_owsave.py +++ b/Orange/widgets/data/tests/test_owsave.py @@ -76,7 +76,7 @@ def test_annotations(self, write): widget.update_extension() widget.add_type_annotations = False - widget.unconditional_save_file() + widget.unconditional_save_file() write.assert_called() self.assertFalse(write.call_args[0][2]) @@ -87,9 +87,9 @@ def test_annotations(self, write): def test_disable_checkbox(self): widget = self.widget for type_ in FILE_TYPES: - widget.filetype=type_[0] + widget.filetype = type_[0] widget.update_extension() - if widget.get_writer_selected().OPTIONAL_TYPE_ANNOTATIONS: + if widget.get_writer_selected().OPTIONAL_TYPE_ANNOTATIONS: self.assertTrue(widget.annotations_cb.isEnabled()) else: self.assertFalse(widget.annotations_cb.isEnabled())