From 2772741c236d50109ba0c06b33d0fcdd142ad31e Mon Sep 17 00:00:00 2001 From: Matt Riley Date: Tue, 27 Aug 2024 16:32:20 -0400 Subject: [PATCH 1/8] Do not import system case properties by default --- .../case_importer/static/case_importer/js/excel_fields.js | 7 ++++--- corehq/apps/case_importer/static/case_importer/js/main.js | 3 ++- corehq/apps/case_importer/suggested_fields.py | 5 +++++ .../templates/case_importer/excel_fields.html | 1 + corehq/apps/case_importer/views.py | 2 ++ 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/corehq/apps/case_importer/static/case_importer/js/excel_fields.js b/corehq/apps/case_importer/static/case_importer/js/excel_fields.js index 9d5f7f1e1605..0ccc4a8a4bd4 100644 --- a/corehq/apps/case_importer/static/case_importer/js/excel_fields.js +++ b/corehq/apps/case_importer/static/case_importer/js/excel_fields.js @@ -10,7 +10,7 @@ hqDefine('case_importer/js/excel_fields', [ _, levenshtein ) { - function excelFieldRows(excelFields, caseFieldSpecs) { + function excelFieldRows(excelFields, caseFieldSpecs, systemFields) { var self = { excelFields: excelFields, caseFieldSpecs: caseFieldSpecs, @@ -65,7 +65,7 @@ hqDefine('case_importer/js/excel_fields', [ }); row.createNewChecked = ko.computed({ read: function () { - return _.isEmpty(row.caseFieldSpec()); + return row.isCustom(); }, write: row.isCustom, }); @@ -81,7 +81,8 @@ hqDefine('case_importer/js/excel_fields', [ row.isCustom(false); row.selectedCaseField(field); } else { - row.isCustom(true); + var spec = _(caseFieldSpecs).findWhere({field}); + row.isCustom(_.isEmpty(spec) && !_(systemFields).contains(field)); row.selectedCaseField(null); } }; diff --git a/corehq/apps/case_importer/static/case_importer/js/main.js b/corehq/apps/case_importer/static/case_importer/js/main.js index c8c903761f8c..3576e748cff1 100644 --- a/corehq/apps/case_importer/static/case_importer/js/main.js +++ b/corehq/apps/case_importer/static/case_importer/js/main.js @@ -61,6 +61,7 @@ hqDefine("case_importer/js/main", [ var behaviorForExcelMappingPage = function () { var excelFields = initialPageData.get('excel_fields'); var caseFieldSpecs = initialPageData.get('case_field_specs'); + var systemFields = initialPageData.get('system_fields'); if (!excelFields && !caseFieldSpecs) { // We're not on the excel mapping page return; @@ -78,7 +79,7 @@ hqDefine("case_importer/js/main", [ return; } - var excelFieldRows = excelFieldsModule.excelFieldRowsModel(excelFields, caseFieldSpecs); + var excelFieldRows = excelFieldsModule.excelFieldRowsModel(excelFields, caseFieldSpecs, systemFields); $('#excel-field-rows').koApplyBindings(excelFieldRows); $('#js-add-mapping').click(function (e) { diff --git a/corehq/apps/case_importer/suggested_fields.py b/corehq/apps/case_importer/suggested_fields.py index 7db372bec54f..bf55b132b774 100644 --- a/corehq/apps/case_importer/suggested_fields.py +++ b/corehq/apps/case_importer/suggested_fields.py @@ -9,6 +9,7 @@ ) from corehq.apps.case_importer.util import RESERVED_FIELDS from corehq.apps.data_dictionary.util import get_values_hints_dict, get_deprecated_fields +from corehq.apps.export.system_properties import MAIN_CASE_TABLE_PROPERTIES from corehq.toggles import BULK_UPLOAD_DATE_OPENED @@ -115,3 +116,7 @@ def get_special_fields(domain=None): ) ) return special_fields + + +def get_system_fields(): + return [x.label for x in MAIN_CASE_TABLE_PROPERTIES] diff --git a/corehq/apps/case_importer/templates/case_importer/excel_fields.html b/corehq/apps/case_importer/templates/case_importer/excel_fields.html index f8caf4bd0d7a..7ecd2538eb03 100644 --- a/corehq/apps/case_importer/templates/case_importer/excel_fields.html +++ b/corehq/apps/case_importer/templates/case_importer/excel_fields.html @@ -21,6 +21,7 @@ {% initial_page_data 'excel_fields' excel_fields %} {% initial_page_data 'case_field_specs' case_field_specs %} + {% initial_page_data 'system_fields' system_fields %} {% initial_page_data 'is_bulk_import' is_bulk_import %}
Date: Thu, 29 Aug 2024 06:53:04 -0400 Subject: [PATCH 2/8] Hide arrow on fields that will not be imported --- .../apps/case_importer/static/case_importer/js/excel_fields.js | 3 +++ .../templates/case_importer/partials/excel_field_rows.html | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/corehq/apps/case_importer/static/case_importer/js/excel_fields.js b/corehq/apps/case_importer/static/case_importer/js/excel_fields.js index 0ccc4a8a4bd4..0f8929644f0c 100644 --- a/corehq/apps/case_importer/static/case_importer/js/excel_fields.js +++ b/corehq/apps/case_importer/static/case_importer/js/excel_fields.js @@ -30,6 +30,9 @@ hqDefine('case_importer/js/excel_fields', [ customCaseField: ko.observable(excelField), isCustom: ko.observable(false), }; + row.hasValue = ko.computed(function () { + return row.isCustom() || row.selectedCaseField(); + }); row.selectedCaseFieldOrBlank = ko.computed({ read: function () { return row.isCustom() ? '' : row.selectedCaseField(); diff --git a/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html b/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html index 4f7f05538c68..9db7bf293432 100644 --- a/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html +++ b/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html @@ -18,7 +18,7 @@ - + From e2847b7152f092a0c9767dd4fa64acc774863258 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Thu, 29 Aug 2024 06:53:59 -0400 Subject: [PATCH 3/8] Select "caseid" as default search column ...if present --- .../case_importer/templates/case_importer/excel_config.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corehq/apps/case_importer/templates/case_importer/excel_config.html b/corehq/apps/case_importer/templates/case_importer/excel_config.html index 5e4246d928b5..8fee156c1731 100644 --- a/corehq/apps/case_importer/templates/case_importer/excel_config.html +++ b/corehq/apps/case_importer/templates/case_importer/excel_config.html @@ -84,7 +84,7 @@
+ {% if request|toggle_enabled:"CASE_IMPORT_DATA_DICTIONARY_VALIDATION" %} From f0a6d699b5c3eebcc2630fd292e10e1b70ef1654 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Thu, 29 Aug 2024 08:19:00 -0400 Subject: [PATCH 5/8] Change case field suggestion from warning to info --- .../templates/case_importer/partials/excel_field_rows.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html b/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html index 3f7e3ed1c6db..9113de64e096 100644 --- a/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html +++ b/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html @@ -58,7 +58,7 @@

-

+

{% blocktrans with ', ' as suggestion %} Did you mean "{{ suggestion }}" instead? From 6145e2f11786dc1c7e37277f1d7877f2cad2efb8 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Thu, 29 Aug 2024 08:19:45 -0400 Subject: [PATCH 6/8] Warn on import system property --- .../static/case_importer/js/excel_fields.js | 6 +++++- .../case_importer/partials/excel_field_rows.html | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/corehq/apps/case_importer/static/case_importer/js/excel_fields.js b/corehq/apps/case_importer/static/case_importer/js/excel_fields.js index 3468ba5c40c8..395ee3dfb6f6 100644 --- a/corehq/apps/case_importer/static/case_importer/js/excel_fields.js +++ b/corehq/apps/case_importer/static/case_importer/js/excel_fields.js @@ -11,6 +11,7 @@ hqDefine('case_importer/js/excel_fields', [ levenshtein ) { function excelFieldRows(excelFields, caseFieldSpecs, systemFields) { + systemFields = _(systemFields); var self = { excelFields: excelFields, caseFieldSpecs: caseFieldSpecs, @@ -79,7 +80,7 @@ hqDefine('case_importer/js/excel_fields', [ row.selectedCaseField(field); } else { var spec = _(caseFieldSpecs).findWhere({field}); - row.isCustom(_.isEmpty(spec) && !_(systemFields).contains(field)); + row.isCustom(_.isEmpty(spec) && !systemFields.contains(field)); row.selectedCaseField(null); } }; @@ -100,6 +101,9 @@ hqDefine('case_importer/js/excel_fields', [ }); return _.chain(suggestions).sortBy('distance').pluck('field').value(); }); + row.isSystemProperty = ko.computed(function () { + return row.isCustom() && systemFields.contains(row.customCaseField()); + }); self.mappingRows.push(row); }; diff --git a/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html b/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html index 9113de64e096..96289e7c9b71 100644 --- a/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html +++ b/corehq/apps/case_importer/templates/case_importer/partials/excel_field_rows.html @@ -65,6 +65,16 @@ {% endblocktrans %}

+ +

+ + {% blocktrans %} + Importing a system property may have unexpected results such as + adding a duplicate field to case exports. Consider using a + different property name. + {% endblocktrans %} +

+ From 9dc3ca7cf7d315bc881eb8f6b960ab31a69cfaa5 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Thu, 29 Aug 2024 10:40:24 -0400 Subject: [PATCH 7/8] Do not warn about discoverable system properties --- corehq/apps/case_importer/suggested_fields.py | 5 ++-- .../tests/test_suggested_fields.py | 24 +++++++++++++++++++ corehq/apps/case_importer/views.py | 4 ++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/corehq/apps/case_importer/suggested_fields.py b/corehq/apps/case_importer/suggested_fields.py index bf55b132b774..5b8c3c8c6032 100644 --- a/corehq/apps/case_importer/suggested_fields.py +++ b/corehq/apps/case_importer/suggested_fields.py @@ -118,5 +118,6 @@ def get_special_fields(domain=None): return special_fields -def get_system_fields(): - return [x.label for x in MAIN_CASE_TABLE_PROPERTIES] +def get_non_discoverable_system_properties(): + discoverable = {s.field for s in get_special_fields() if s.description and s.discoverable} + return [p.label for p in MAIN_CASE_TABLE_PROPERTIES if p.label not in discoverable] diff --git a/corehq/apps/case_importer/tests/test_suggested_fields.py b/corehq/apps/case_importer/tests/test_suggested_fields.py index 0c5c652acdce..a493f1a8319c 100644 --- a/corehq/apps/case_importer/tests/test_suggested_fields.py +++ b/corehq/apps/case_importer/tests/test_suggested_fields.py @@ -6,6 +6,7 @@ from corehq.apps.case_importer.suggested_fields import ( FieldSpec, get_suggested_case_fields, + get_non_discoverable_system_properties, ) from corehq.util.test_utils import DocTestMixin @@ -68,3 +69,26 @@ def test_basic_inclusion_and_sorting(self): FieldSpec(field='weight', show_in_menu=True), ] ) + + +class TestSystemProperties(SimpleTestCase): + + def test_get_non_discoverable_system_properties(self): + self.assertEqual(get_non_discoverable_system_properties(), [ + 'number', + 'caseid', + 'case_type', + 'closed', + 'closed_by_user_id', + 'closed_by_username', + 'closed_date', + 'last_modified_by_user_id', + 'last_modified_by_user_username', + 'last_modified_date', + 'opened_by_user_id', + 'opened_by_username', + 'opened_date', + 'server_last_modified_date', + 'state', + 'case_link', + ]) diff --git a/corehq/apps/case_importer/views.py b/corehq/apps/case_importer/views.py index 64ebf07acff1..66760512d2b2 100644 --- a/corehq/apps/case_importer/views.py +++ b/corehq/apps/case_importer/views.py @@ -32,7 +32,7 @@ ) from corehq.apps.case_importer.suggested_fields import ( get_suggested_case_fields, - get_system_fields, + get_non_discoverable_system_properties, ) from corehq.apps.case_importer.tracking.case_upload_tracker import CaseUpload from corehq.apps.case_importer.util import ( @@ -373,7 +373,7 @@ def excel_fields(request, domain): 'columns': columns, 'excel_fields': excel_fields, 'case_field_specs': case_field_specs, - 'system_fields': get_system_fields(), + 'system_fields': get_non_discoverable_system_properties(), 'domain': domain, 'mirroring_enabled': mirroring_enabled, 'is_bulk_import': request.POST.get('is_bulk_import', 'False') == 'True', From 2c90bb9f1e8c66b4ae2ba1d1963066d133170b87 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Tue, 10 Sep 2024 10:10:45 -0400 Subject: [PATCH 8/8] Select custom field for import if it has a "spec" A field with a "spec" is one returned by corehq.apps.case_importer.suggested_fields.get_suggested_case_fields(). Before 2772741c236d50109ba0c06b33d0fcdd142ad31e these fields were selected for import by default. --- .../apps/case_importer/static/case_importer/js/excel_fields.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/corehq/apps/case_importer/static/case_importer/js/excel_fields.js b/corehq/apps/case_importer/static/case_importer/js/excel_fields.js index 395ee3dfb6f6..dd821f3206a6 100644 --- a/corehq/apps/case_importer/static/case_importer/js/excel_fields.js +++ b/corehq/apps/case_importer/static/case_importer/js/excel_fields.js @@ -79,8 +79,7 @@ hqDefine('case_importer/js/excel_fields', [ row.isCustom(false); row.selectedCaseField(field); } else { - var spec = _(caseFieldSpecs).findWhere({field}); - row.isCustom(_.isEmpty(spec) && !systemFields.contains(field)); + row.isCustom(!systemFields.contains(field)); row.selectedCaseField(null); } };