From 743c6e2753fb8b83ef7bbbd4cb5f5b5ed707f6f2 Mon Sep 17 00:00:00 2001 From: suman Date: Mon, 26 Apr 2021 17:22:06 +0800 Subject: [PATCH 1/6] added custom command to validate existing plugins on server --- .../commands/validate_existing_plugins.py | 108 ++++++++++++++++ .../tests/test_validate_existing_plugins.py | 116 ++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 qgis-app/plugins/management/commands/validate_existing_plugins.py create mode 100644 qgis-app/plugins/tests/test_validate_existing_plugins.py diff --git a/qgis-app/plugins/management/commands/validate_existing_plugins.py b/qgis-app/plugins/management/commands/validate_existing_plugins.py new file mode 100644 index 00000000..0095da85 --- /dev/null +++ b/qgis-app/plugins/management/commands/validate_existing_plugins.py @@ -0,0 +1,108 @@ +"""A command to validate the existing zipfile Plugin Packages""" + +import os +from django.conf import settings +from django.core.files.uploadedfile import InMemoryUploadedFile +from django.core.mail import send_mail +from django.core.management.base import BaseCommand +from django.contrib.sites.models import Site +from django.utils.translation import ugettext_lazy as _ + +from plugins.models import PluginVersion +from plugins.validator import validator + + +DOMAIN = Site.objects.get_current().domain + + +def send_email_notification(plugin, version, message, url_version, recipients): + + message = ('\r\nPlease update ' + 'Plugin: %s ' + '- Version: %s\r\n' + '\r\nIt failed to pass validation with message:' + '\r\n%s\r\n' + '\r\nLink: %s') % (plugin, version, message, url_version) + send_mail( + subject='Invalid Plugin Metadata Notification', + message=_(message), + from_email=settings.DEFAULT_FROM_EMAIL, + recipient_list=recipients, + fail_silently=True + ) + + +def get_recipients_email(plugin): + receipt_email = [] + if plugin.created_by.email: + receipt_email.append(plugin.created_by.email) + if plugin.email: + receipt_email.append(plugin.email) + return receipt_email + + +def validate_zipfile_version(version): + + if not os.path.exists(version.package.url): + return { + 'plugin': f'{version.plugin.name}', + 'created_by': f'{version.plugin.created_by}', + 'version': f'{version.version}', + 'msg': [f'File does not exist. Please re-upload.'], + 'url': f'http://{DOMAIN}{version.get_absolute_url()}', + 'recipients_email': get_recipients_email(version.plugin) + } + + with open(version.package.url, 'rb') as buf: + package = InMemoryUploadedFile( + buf, + 'tempfile', + 'filename.zip', + 'application/zip', + 1000000, # ignore the filesize and assume it's 1MB + 'utf8') + try: + validator(package) + except Exception as e: + return { + 'plugin': f'{version.plugin.name}', + 'created_by': f'{version.plugin.created_by}', + 'version': f'{version.version}', + 'msg': e.messages, + 'url': f'http://{DOMAIN}{version.get_absolute_url()}', + 'recipients_email': get_recipients_email(version.plugin) + } + return None + + +class Command(BaseCommand): + + help = ('Validate existing Plugins zipfile and send a notification email ' + 'for invalid Plugin') + + def handle(self, *args, **options): + self.stdout.write('Validating existing plugins...') + versions = PluginVersion.objects.all() + num_count = 0 + for version in versions: + error_msg = validate_zipfile_version(version) + if error_msg: + send_email_notification( + plugin=error_msg['plugin'], + version=error_msg['version'], + message='\r\n'.join(error_msg['msg']), + url_version=error_msg['url'], + recipients=error_msg['recipients_email'] + ) + self.stdout.write( + _('Sent email to %s for Plugin %s - Version %s.') % ( + error_msg['recipients_email'], + error_msg['plugin'], + error_msg['version'] + ) + ) + num_count += 1 + self.stdout.write( + _('Successfully sent email notification for %s invalid plugins') + % (num_count) + ) diff --git a/qgis-app/plugins/tests/test_validate_existing_plugins.py b/qgis-app/plugins/tests/test_validate_existing_plugins.py new file mode 100644 index 00000000..6746e5b1 --- /dev/null +++ b/qgis-app/plugins/tests/test_validate_existing_plugins.py @@ -0,0 +1,116 @@ +import os + +from django.contrib.auth.models import User +from django.core import mail +from django.core.files.uploadedfile import InMemoryUploadedFile +from django.test import TestCase, override_settings + +from plugins.models import PluginVersion, Plugin +from plugins.management.commands.validate_existing_plugins import ( + validate_zipfile_version, send_email_notification) + +TESTFILE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'testfiles')) + + +@override_settings(MEDIA_ROOT='plugins/tests/testfiles/') +@override_settings(MEDIA_URL='plugins/tests/testfiles/') +class TestValidateExistingPlugin(TestCase): + def setUp(self) -> None: + self.creator = User.objects.create( + username='usertest_creator', + first_name="first_name", + last_name="last_name", + email="creator@example.com", + password="passwordtest", + is_active=True) + self.author = User.objects.create( + username='usertest_author', + first_name="author", + last_name="last_name", + email="author@example.com", + password="passwordtest", + is_active=True) + + invalid_plugin = os.path.join( + TESTFILE_DIR, "web_not_exist.zip") + self.invalid_plugin = open(invalid_plugin, 'rb') + + uploaded_zipfile = InMemoryUploadedFile( + self.invalid_plugin, + field_name='tempfile', + name='testfile.zip', + content_type='application/zip', + size=39889, + charset='utf8') + + self.plugin = Plugin.objects.create( + created_by=self.creator, + name='test_plugin', + package_name='test_plugin' + ) + + self.version = PluginVersion.objects.create( + plugin=self.plugin, + created_by=self.creator, + version='0.1', + package=uploaded_zipfile, + min_qg_version='3.10', + max_qg_version='3.18' + ) + + def tearDown(self) -> None: + self.invalid_plugin.close() + os.remove(self.version.package.url) + + def test_plugin_exist(self): + self.assertEqual(PluginVersion.objects.count(), 1) + self.assertTrue(os.path.exists(self.version.package.url)) + + def test_validate_zipfile_version(self): + expected_value = { + 'plugin': 'test_plugin', + 'created_by': 'usertest_creator', + 'version': '0.1', + 'msg': ['Please provide valid url link for Repository in metadata. ' + 'This website cannot be reached.'], + 'url': 'http://plugins.qgis.org/plugins/test_plugin/version/0.1/', + 'recipients_email': ['creator@example.com']} + self.assertEqual( + validate_zipfile_version(self.version), + expected_value, + msg=validate_zipfile_version(self.version) + ) + + @override_settings( + EMAIL_BACKEND='django.core.mail.backends.console.EmailBackend') + def test_send_email(self): + error_msg = validate_zipfile_version(self.version) + send_email_notification( + plugin=error_msg['plugin'], + version=error_msg['version'], + message='\r\n'.join(error_msg['msg']), + url_version=error_msg['url'], + recipients=error_msg['recipients_email'] + ) + + def test_send_email_must_contains(self): + error_msg = validate_zipfile_version(self.version) + send_email_notification( + plugin=error_msg['plugin'], + version=error_msg['version'], + message='\r\n'.join(error_msg['msg']), + url_version=error_msg['url'], + recipients=error_msg['recipients_email'] + ) + self.assertEqual(len(mail.outbox), 1) + self.assertEqual( + mail.outbox[0].subject, + 'Invalid Plugin Metadata Notification') + self.assertIn( + 'Please update Plugin: test_plugin - Version: 0.1', + mail.outbox[0].body) + self.assertIn( + 'Please provide valid url link for Repository in metadata. ' + 'This website cannot be reached.', + mail.outbox[0].body) From 53f67f0fde946fcef4912cb09a27ae8d1a61a2f9 Mon Sep 17 00:00:00 2001 From: suman Date: Mon, 21 Jun 2021 00:35:59 +0800 Subject: [PATCH 2/6] added PluginInvalid class model to track invalid plugins --- .../commands/validate_existing_plugins.py | 14 +++- .../plugins/migrations/0002_plugininvalid.py | 24 +++++++ qgis-app/plugins/models.py | 19 +++++ qgis-app/plugins/tests/test_models.py | 70 +++++++++++++++++++ .../tests/test_validate_existing_plugins.py | 4 +- 5 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 qgis-app/plugins/migrations/0002_plugininvalid.py create mode 100644 qgis-app/plugins/tests/test_models.py diff --git a/qgis-app/plugins/management/commands/validate_existing_plugins.py b/qgis-app/plugins/management/commands/validate_existing_plugins.py index 0095da85..5ca4e578 100644 --- a/qgis-app/plugins/management/commands/validate_existing_plugins.py +++ b/qgis-app/plugins/management/commands/validate_existing_plugins.py @@ -8,7 +8,7 @@ from django.contrib.sites.models import Site from django.utils.translation import ugettext_lazy as _ -from plugins.models import PluginVersion +from plugins.models import PluginVersion, PluginInvalid from plugins.validator import validator @@ -48,6 +48,7 @@ def validate_zipfile_version(version): 'plugin': f'{version.plugin.name}', 'created_by': f'{version.plugin.created_by}', 'version': f'{version.version}', + 'version_id': version.id, 'msg': [f'File does not exist. Please re-upload.'], 'url': f'http://{DOMAIN}{version.get_absolute_url()}', 'recipients_email': get_recipients_email(version.plugin) @@ -68,6 +69,7 @@ def validate_zipfile_version(version): 'plugin': f'{version.plugin.name}', 'created_by': f'{version.plugin.created_by}', 'version': f'{version.version}', + 'version_id': version.id, 'msg': e.messages, 'url': f'http://{DOMAIN}{version.get_absolute_url()}', 'recipients_email': get_recipients_email(version.plugin) @@ -82,7 +84,9 @@ class Command(BaseCommand): def handle(self, *args, **options): self.stdout.write('Validating existing plugins...') - versions = PluginVersion.objects.all() + # get the latest version + versions = PluginVersion.approved_objects.\ + order_by('plugin_id', '-created_on').distinct('plugin_id').all()[:3] num_count = 0 for version in versions: error_msg = validate_zipfile_version(version) @@ -102,6 +106,12 @@ def handle(self, *args, **options): ) ) num_count += 1 + plugin_version = PluginVersion.objects\ + .select_related('plugin').get(id=error_msg['version_id']) + PluginInvalid.objects.create( + plugin=plugin_version.plugin, + validated_version=plugin_version.version + ) self.stdout.write( _('Successfully sent email notification for %s invalid plugins') % (num_count) diff --git a/qgis-app/plugins/migrations/0002_plugininvalid.py b/qgis-app/plugins/migrations/0002_plugininvalid.py new file mode 100644 index 00000000..0f777898 --- /dev/null +++ b/qgis-app/plugins/migrations/0002_plugininvalid.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.18 on 2021-06-20 10:00 + +from django.db import migrations, models +import django.db.models.deletion +import plugins.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='PluginInvalid', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('validated_version', plugins.models.VersionField(db_index=True, max_length=32, verbose_name='Version')), + ('validated_at', models.DateTimeField(auto_now_add=True, verbose_name='Validated at')), + ('plugin', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='plugins.Plugin')), + ], + ), + ] diff --git a/qgis-app/plugins/models.py b/qgis-app/plugins/models.py index f0e13322..b6d089ac 100644 --- a/qgis-app/plugins/models.py +++ b/qgis-app/plugins/models.py @@ -628,6 +628,25 @@ def delete_plugin_icon(sender, instance, **kw): pass +class PluginInvalid(models.Model): + """Invalid plugins model. + + There were existing plugins on the server before the validation + mechanism updated. + We run the management command to validate them, and track the invalid one + of the latest version. + """ + + plugin = models.ForeignKey(Plugin, on_delete=models.CASCADE) + # We track the version number, not the version instance. + # So that when the version has been deleted, we keep the plugin + # in the tracking list + validated_version = VersionField( + _('Version'), max_length=32, db_index=True) + validated_at = models.DateTimeField( + _('Validated at'), auto_now_add=True, editable=False) + + models.signals.post_delete.connect( delete_version_package, sender=PluginVersion) models.signals.post_delete.connect(delete_plugin_icon, sender=Plugin) diff --git a/qgis-app/plugins/tests/test_models.py b/qgis-app/plugins/tests/test_models.py new file mode 100644 index 00000000..20dae7c1 --- /dev/null +++ b/qgis-app/plugins/tests/test_models.py @@ -0,0 +1,70 @@ +import os + +from django.contrib.auth.models import User +from django.core.files.uploadedfile import InMemoryUploadedFile +from django.test import TestCase, override_settings + +from plugins.models import PluginVersion, Plugin, PluginInvalid + +TESTFILE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'testfiles')) + + +@override_settings(MEDIA_ROOT='plugins/tests/testfiles/') +@override_settings(MEDIA_URL='plugins/tests/testfiles/') +class TestPluginInvalidModel(TestCase): + + def setUp(self) -> None: + self.creator = User.objects.create( + username='usertest_creator', + first_name="first_name", + last_name="last_name", + email="creator@example.com", + password="passwordtest", + is_active=True) + self.author = User.objects.create( + username='usertest_author', + first_name="author", + last_name="last_name", + email="author@example.com", + password="passwordtest", + is_active=True) + + invalid_plugin = os.path.join( + TESTFILE_DIR, "web_not_exist.zip") + self.invalid_plugin = open(invalid_plugin, 'rb') + + uploaded_zipfile = InMemoryUploadedFile( + self.invalid_plugin, + field_name='tempfile', + name='testfile.zip', + content_type='application/zip', + size=39889, + charset='utf8') + + self.plugin = Plugin.objects.create( + created_by=self.creator, + name='test_plugin', + package_name='test_plugin' + ) + + self.version = PluginVersion.objects.create( + plugin=self.plugin, + created_by=self.creator, + version='0.1', + package=uploaded_zipfile, + min_qg_version='3.10', + max_qg_version='3.18' + ) + + def tearDown(self) -> None: + self.invalid_plugin.close() + os.remove(self.version.package.url) + + def test_create_PluginInvalid_instance(self): + invalid_plugin = PluginInvalid.objects.create( + plugin=self.plugin, + validated_version=self.plugin.pluginversion_set.get().version + ) + self.assertEqual(invalid_plugin.validated_version, '0.1') + self.assertIsNotNone(invalid_plugin.validated_at) diff --git a/qgis-app/plugins/tests/test_validate_existing_plugins.py b/qgis-app/plugins/tests/test_validate_existing_plugins.py index 6746e5b1..1dd9c849 100644 --- a/qgis-app/plugins/tests/test_validate_existing_plugins.py +++ b/qgis-app/plugins/tests/test_validate_existing_plugins.py @@ -5,7 +5,7 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.test import TestCase, override_settings -from plugins.models import PluginVersion, Plugin +from plugins.models import PluginVersion, Plugin, PluginInvalid from plugins.management.commands.validate_existing_plugins import ( validate_zipfile_version, send_email_notification) @@ -16,6 +16,7 @@ @override_settings(MEDIA_ROOT='plugins/tests/testfiles/') @override_settings(MEDIA_URL='plugins/tests/testfiles/') class TestValidateExistingPlugin(TestCase): + def setUp(self) -> None: self.creator = User.objects.create( username='usertest_creator', @@ -72,6 +73,7 @@ def test_validate_zipfile_version(self): 'plugin': 'test_plugin', 'created_by': 'usertest_creator', 'version': '0.1', + 'version_id': self.version.id, 'msg': ['Please provide valid url link for Repository in metadata. ' 'This website cannot be reached.'], 'url': 'http://plugins.qgis.org/plugins/test_plugin/version/0.1/', From 2bc7b840f0c3ccfe332f5c4e6fdec0b673b1e220 Mon Sep 17 00:00:00 2001 From: suman Date: Sat, 26 Jun 2021 11:33:27 +0800 Subject: [PATCH 3/6] added views and template of invalid plugins --- .../commands/validate_existing_plugins.py | 5 +- .../plugins/migrations/0002_plugininvalid.py | 5 +- qgis-app/plugins/models.py | 8 +- .../templates/plugins/plugin_base.html | 1 + .../plugins/plugin_invalid_list.html | 121 ++++++++++++++++++ qgis-app/plugins/urls.py | 5 + qgis-app/plugins/views.py | 9 +- 7 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 qgis-app/plugins/templates/plugins/plugin_invalid_list.html diff --git a/qgis-app/plugins/management/commands/validate_existing_plugins.py b/qgis-app/plugins/management/commands/validate_existing_plugins.py index 5ca4e578..36d1c2d4 100644 --- a/qgis-app/plugins/management/commands/validate_existing_plugins.py +++ b/qgis-app/plugins/management/commands/validate_existing_plugins.py @@ -86,7 +86,7 @@ def handle(self, *args, **options): self.stdout.write('Validating existing plugins...') # get the latest version versions = PluginVersion.approved_objects.\ - order_by('plugin_id', '-created_on').distinct('plugin_id').all()[:3] + order_by('plugin_id', '-created_on').distinct('plugin_id').all()[:50] num_count = 0 for version in versions: error_msg = validate_zipfile_version(version) @@ -110,7 +110,8 @@ def handle(self, *args, **options): .select_related('plugin').get(id=error_msg['version_id']) PluginInvalid.objects.create( plugin=plugin_version.plugin, - validated_version=plugin_version.version + validated_version=plugin_version.version, + message=error_msg['msg'] ) self.stdout.write( _('Successfully sent email notification for %s invalid plugins') diff --git a/qgis-app/plugins/migrations/0002_plugininvalid.py b/qgis-app/plugins/migrations/0002_plugininvalid.py index 0f777898..eca9956e 100644 --- a/qgis-app/plugins/migrations/0002_plugininvalid.py +++ b/qgis-app/plugins/migrations/0002_plugininvalid.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.18 on 2021-06-20 10:00 +# Generated by Django 2.2.18 on 2021-06-25 22:31 from django.db import migrations, models import django.db.models.deletion @@ -18,7 +18,8 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('validated_version', plugins.models.VersionField(db_index=True, max_length=32, verbose_name='Version')), ('validated_at', models.DateTimeField(auto_now_add=True, verbose_name='Validated at')), - ('plugin', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='plugins.Plugin')), + ('message', models.CharField(editable=False, help_text='Invalid error message', max_length=256, verbose_name='Message')), + ('plugin', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='plugins.Plugin', unique=True)), ], ), ] diff --git a/qgis-app/plugins/models.py b/qgis-app/plugins/models.py index b6d089ac..6ebd26f9 100644 --- a/qgis-app/plugins/models.py +++ b/qgis-app/plugins/models.py @@ -637,7 +637,7 @@ class PluginInvalid(models.Model): of the latest version. """ - plugin = models.ForeignKey(Plugin, on_delete=models.CASCADE) + plugin = models.ForeignKey(Plugin, on_delete=models.CASCADE, unique=True) # We track the version number, not the version instance. # So that when the version has been deleted, we keep the plugin # in the tracking list @@ -645,6 +645,12 @@ class PluginInvalid(models.Model): _('Version'), max_length=32, db_index=True) validated_at = models.DateTimeField( _('Validated at'), auto_now_add=True, editable=False) + message = models.CharField( + _('Message'), + help_text=_('Invalid error message'), + max_length=256, + editable=False + ) models.signals.post_delete.connect( diff --git a/qgis-app/plugins/templates/plugins/plugin_base.html b/qgis-app/plugins/templates/plugins/plugin_base.html index 4bb7002b..45f9c727 100644 --- a/qgis-app/plugins/templates/plugins/plugin_base.html +++ b/qgis-app/plugins/templates/plugins/plugin_base.html @@ -31,6 +31,7 @@

{% trans "Plugins" %}

  • {% trans "Top downloads" %}
  • {% trans "Most rated" %}
  • {% trans "QGIS Server plugins" %}
    • +
  • {% trans "Invalid plugins" %}
    • diff --git a/qgis-app/plugins/templates/plugins/plugin_invalid_list.html b/qgis-app/plugins/templates/plugins/plugin_invalid_list.html new file mode 100644 index 00000000..9878f5e3 --- /dev/null +++ b/qgis-app/plugins/templates/plugins/plugin_invalid_list.html @@ -0,0 +1,121 @@ +{% extends 'plugins/plugin_base.html' %}{% load i18n bootstrap_pagination humanize static sort_anchor range_filter thumbnail %} +{% block extrajs %} + + +{% endblock %} +{% block content %} +

    {% if title %}{{title}}{% else %}{% trans "Invalid Plugins" %}{% endif %}

    + {# Filtered views menu #} + {% if object_list.count %} +
    + {% blocktrans with records_count=page_obj.paginator.count %}{{ records_count }} records found{% endblocktrans %} —  + {% trans "Click to toggle descriptions." %} +
    + + + + + + + {% if not user.is_anonymous %}{% endif %} + + + + + {% if user.is_authenticated %}{% endif %} + + + + {% for object in invalid_plugins %} + + + + + + + +{# #} +{# #} +{# #} +{# {% if user.is_authenticated %}{% if user in object.editors or user.is_staff %}{% endif %}#} +{##} + +{##} + + + {% endfor %} + +
     {% anchor name %}{% trans {% anchor author "Author" %}{% anchor validated_version "Validated Version" %}{% anchor validated_at "Validated at" %}{% anchor message "Error message" %}{% trans "Manage" %}
    + {% if object.plugin.icon and object.plugin.icon.file %} + {% thumbnail object.icon "24x24" format="PNG" as im %} + {% trans + {% endthumbnail %} + {% else %} + {% trans + {% endif %} + {{ object.plugin.name }}{{ object.plugin.author }}{{ object.validated_version }}{{ object.validated_at }}{{ object.message }}
    ({{ object.rating_votes }})
    		{% if object.stable %}{{ object.stable.version }}{% else %}—{% endif %}{% if object.experimental %}{{ object.experimental.version }}{% else %}—{% endif %}#} +{# {% else %}{% endif %}
    + +
    + + {% trans "Deprecated plugins are printed in red." %} +
    + {% else %} + {% block plugins_message %} +
    + + {% trans "This list is empty!" %} +
    + {% endblock %} + {% endif %} + +{% endblock %} diff --git a/qgis-app/plugins/urls.py b/qgis-app/plugins/urls.py index 6d0e35bc..e2ade5ab 100644 --- a/qgis-app/plugins/urls.py +++ b/qgis-app/plugins/urls.py @@ -55,6 +55,11 @@ url(r'^user/(?P\w+)/manage/$', user_permissions_manage, {}, name='user_permissions_manage'), ] +# Invalid Plugin +urlpatterns += [ + url(r'^invalid_plugins/$', InvalidPluginList.as_view(), name='invalid_plugins'), +] + # Version Management urlpatterns += [ diff --git a/qgis-app/plugins/views.py b/qgis-app/plugins/views.py index 077e7e65..e6ab2fc5 100644 --- a/qgis-app/plugins/views.py +++ b/qgis-app/plugins/views.py @@ -20,7 +20,7 @@ #from sortable_listview import SortableListView from django.views.generic.list import ListView from django.views.generic.detail import DetailView -from plugins.models import Plugin, PluginVersion, vjust +from plugins.models import Plugin, PluginVersion, vjust, PluginInvalid from plugins.forms import * from plugins.validator import PLUGIN_REQUIRED_METADATA @@ -582,6 +582,13 @@ def get_context_data(self, **kwargs): return context +class InvalidPluginList(ListView): + model = PluginInvalid + template_name = 'plugins/plugin_invalid_list.html' + context_object_name = 'invalid_plugins' + paginate_by = settings.PAGINATION_DEFAULT_PAGINATION + + @login_required @require_POST def plugin_manage(request, package_name): From bf71d7cb7acb20836555f5a67a359860db516298 Mon Sep 17 00:00:00 2001 From: suman Date: Sat, 3 Jul 2021 17:29:41 +0800 Subject: [PATCH 4/6] updated templates and views for sort and pagination --- .../commands/validate_existing_plugins.py | 4 +- .../plugins/plugin_invalid_list.html | 11 +--- qgis-app/plugins/views.py | 59 +++++++++++++++++++ 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/qgis-app/plugins/management/commands/validate_existing_plugins.py b/qgis-app/plugins/management/commands/validate_existing_plugins.py index 36d1c2d4..4ff3e2ec 100644 --- a/qgis-app/plugins/management/commands/validate_existing_plugins.py +++ b/qgis-app/plugins/management/commands/validate_existing_plugins.py @@ -111,7 +111,9 @@ def handle(self, *args, **options): PluginInvalid.objects.create( plugin=plugin_version.plugin, validated_version=plugin_version.version, - message=error_msg['msg'] + message=(error_msg['msg'] + if not isinstance(error_msg['msg'], list) + else ', '.join(error_msg['msg'])) ) self.stdout.write( _('Successfully sent email notification for %s invalid plugins') diff --git a/qgis-app/plugins/templates/plugins/plugin_invalid_list.html b/qgis-app/plugins/templates/plugins/plugin_invalid_list.html index 9878f5e3..98e8c8ca 100644 --- a/qgis-app/plugins/templates/plugins/plugin_invalid_list.html +++ b/qgis-app/plugins/templates/plugins/plugin_invalid_list.html @@ -62,12 +62,10 @@

    {% if title %}{{title}}{% else %}{% trans "Invalid Plugins" %}{% endif %}   {% anchor name %} - {% if not user.is_anonymous %}{% trans {% endif %} {% anchor author "Author" %} {% anchor validated_version "Validated Version" %} {% anchor validated_at "Validated at" %} {% anchor message "Error message" %} - {% if user.is_authenticated %}{% trans "Manage" %}{% endif %} @@ -87,15 +85,8 @@

    {% if title %}{{title}}{% else %}{% trans "Invalid Plugins" %}{% endif %}{{ object.validated_version }} {{ object.validated_at }} {{ object.message }} -{#
    ({{ object.rating_votes }})
    #} -{# {% if object.stable %}{{ object.stable.version }}{% else %}—{% endif %}#} -{# {% if object.experimental %}{{ object.experimental.version }}{% else %}—{% endif %}#} -{# {% if user.is_authenticated %}{% if user in object.editors or user.is_staff %}#} -{# {% else %}{% endif %}{% endif %}#} -{##} -{##} -  {{ object.description }} + {% endfor %} diff --git a/qgis-app/plugins/views.py b/qgis-app/plugins/views.py index e6ab2fc5..6c2d7e00 100644 --- a/qgis-app/plugins/views.py +++ b/qgis-app/plugins/views.py @@ -438,6 +438,9 @@ def plugin_update(request, package_name): # Checks for optional metadata _check_optional_metadata(form, request) + # Remove from invalid plugin list + delete_invalid_plugin(plugin) + return HttpResponseRedirect(new_object.get_absolute_url()) else: form = PluginForm(instance = plugin) @@ -588,6 +591,52 @@ class InvalidPluginList(ListView): context_object_name = 'invalid_plugins' paginate_by = settings.PAGINATION_DEFAULT_PAGINATION + def get_queryset(self): + qs = super(InvalidPluginList, self).get_queryset() + sort_by = self.request.GET.get('sort', None) + if sort_by: + if sort_by[0] == '-': + _sort_by = sort_by[1:] + _sort_desc = True + else: + _sort_by = sort_by + _sort_desc = False + + if _sort_by == 'name' or 'author': + _sort_by = f'-plugin__{_sort_by}' if _sort_desc else f'plugin__{_sort_by}' + + qs = qs.order_by(_sort_by) + else: + # default + if not qs.ordered: + qs = qs.order_by(Lower('plugin__name')) + return qs + + def get_context_data(self, **kwargs): + context = super(InvalidPluginList, self).get_context_data(**kwargs) + context['current_sort_query'] = self.get_sortstring() + context['current_querystring'] = self.get_querystring() + return context + + def get_sortstring(self): + if self.request.GET.get('sort', None): + return 'sort=%s' % self.request.GET.get('sort') + return + + def get_querystring(self): + """ + Clean existing query string (GET parameters) by removing + arguments that we don't want to preserve (sort parameter, 'page') + """ + to_remove = ['page', 'sort'] + query_string = urlparse(self.request.get_full_path()).query + query_dict = parse_qs(query_string) + for arg in to_remove: + if arg in query_dict: + del query_dict[arg] + clean_query_string = urlencode(query_dict, doseq=True) + return clean_query_string + @login_required @require_POST @@ -744,6 +793,8 @@ def version_create(request, package_name): form.cleaned_data['icon'] = form.cleaned_data.get('icon_file') _main_plugin_update(request, new_object.plugin, form) _check_optional_metadata(form, request) + # Remove from invalid plugin list + delete_invalid_plugin(plugin) return HttpResponseRedirect(new_object.plugin.get_absolute_url()) except (IntegrityError, ValidationError, DjangoUnicodeDecodeError) as e: messages.error(request, e, fail_silently=True) @@ -778,6 +829,8 @@ def version_update(request, package_name, version): _main_plugin_update(request, new_object.plugin, form) msg = _("The Plugin Version has been successfully updated.") messages.success(request, msg, fail_silently=True) + # Remove from invalid plugin list + delete_invalid_plugin(plugin) except (IntegrityError, ValidationError, DjangoUnicodeDecodeError) as e: messages.error(request, e, fail_silently=True) connection.close() @@ -1082,3 +1135,9 @@ def xml_plugins_new(request, qg_version=None, stable_only=None, package_name=Non return render(request, 'plugins/plugins.xml', {'object_list': object_list_new}, content_type='text/xml') + + +def delete_invalid_plugin(plugin): + invalid_plugin = PluginInvalid.objects.filter(plugin=plugin) + if invalid_plugin: + invalid_plugin.delete() From b9f94a8ce2f9500ad6081574dc9064e4c2f49fc5 Mon Sep 17 00:00:00 2001 From: suman Date: Sat, 17 Jul 2021 20:36:10 +0800 Subject: [PATCH 5/6] added unit test --- dockerize/docker/REQUIREMENTS.txt | 4 +- .../commands/validate_existing_plugins.py | 6 +- qgis-app/plugins/tests/model_factories.py | 88 +++++++++++++++++ qgis-app/plugins/tests/test_views.py | 94 +++++++++++++++++++ 4 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 qgis-app/plugins/tests/model_factories.py create mode 100644 qgis-app/plugins/tests/test_views.py diff --git a/dockerize/docker/REQUIREMENTS.txt b/dockerize/docker/REQUIREMENTS.txt index 01dedf14..c7634b79 100644 --- a/dockerize/docker/REQUIREMENTS.txt +++ b/dockerize/docker/REQUIREMENTS.txt @@ -41,4 +41,6 @@ djangorestframework==3.11.2 sorl-thumbnail-serializer-field==0.2.1 django-rest-auth==0.9.5 drf-yasg==1.17.1 -django-rest-multiple-models==2.1.3 \ No newline at end of file +django-rest-multiple-models==2.1.3 + +factory_boy==3.2.0 \ No newline at end of file diff --git a/qgis-app/plugins/management/commands/validate_existing_plugins.py b/qgis-app/plugins/management/commands/validate_existing_plugins.py index 4ff3e2ec..fa516389 100644 --- a/qgis-app/plugins/management/commands/validate_existing_plugins.py +++ b/qgis-app/plugins/management/commands/validate_existing_plugins.py @@ -1,4 +1,8 @@ -"""A command to validate the existing zipfile Plugin Packages""" +"""A command to validate the existing zipfile Plugin Packages. + +We are using the same validator that used in uploading plugins. +Re-run this command when modify the validator to validate the existing plugins. +""" import os from django.conf import settings diff --git a/qgis-app/plugins/tests/model_factories.py b/qgis-app/plugins/tests/model_factories.py new file mode 100644 index 00000000..14ee2ca4 --- /dev/null +++ b/qgis-app/plugins/tests/model_factories.py @@ -0,0 +1,88 @@ +"""Factories for building model instances for testing.""" + +import factory +import os + +from django.contrib.auth.models import User +from django.core.files.uploadedfile import InMemoryUploadedFile + +from plugins.models import Plugin, PluginVersion, PluginInvalid + + +# TESTFILE_DIR = os.path.abspath( +# os.path.join(os.path.dirname(__file__), 'testfiles')) +# +# +# ZIPFILE = InMemoryUploadedFile( +# os.path.join(TESTFILE_DIR, "valid_metadata_link.zip"), +# field_name='tempfile', +# name='testfile.zip', +# content_type='application/zip', +# size=39889, +# charset='utf8') + +FAKE = factory.faker.faker.Faker() + +class UserF(factory.django.DjangoModelFactory): + """User model factory.""" + + class Meta: + model = User + + username = factory.Sequence(lambda n: "username%s" % n) + first_name = FAKE.first_name() + last_name = FAKE.last_name() + email = FAKE.email() + password = '' + is_staff = False + is_active = True + is_superuser = False + + +class PluginF(factory.django.DjangoModelFactory): + """Plugin model factory.""" + + class Meta: + model = Plugin + + created_by = factory.SubFactory(UserF) + author = FAKE.name() + email = FAKE.email() + homepage = factory.Sequence(lambda n: "https://www.example-%s.com" % n) + repository = "https://github.com/qgis/QGIS-Django" + tracker = "https://github.com/qgis/QGIS-Django/issues" + + # name, desc etc. + package_name = factory.Sequence(lambda n: "package_%s" % n) + name = factory.Sequence(lambda n: "name_%s" % n) + description = factory.Sequence(lambda n: "Description of name_%s" % n) + about = factory.Sequence(lambda n: "About name_%s" % n) + + # downloads (soft trigger from versions) + downloads = factory.Sequence(lambda n: n) + + +class PluginVersionF(factory.django.DjangoModelFactory): + """PluginVersion model factory.""" + + class Meta: + model = PluginVersion + + # link to parent + plugin = factory.SubFactory(PluginF) + created_by = factory.SubFactory(UserF) + min_qg_version = "002.000.000" + max_qg_version = "002.099.003.###" + version = "1.2.3.4" + package = factory.django.FileField(filename='plugin.zip') + + +class PluginInvalidF(factory.django.DjangoModelFactory): + """PluginVersion model factory.""" + + class Meta: + model = PluginInvalid + + plugin = factory.SubFactory(PluginF) + validated_version = "0.0.0.0" + message = "File does not exist. Please re-upload." diff --git a/qgis-app/plugins/tests/test_views.py b/qgis-app/plugins/tests/test_views.py new file mode 100644 index 00000000..1608fe72 --- /dev/null +++ b/qgis-app/plugins/tests/test_views.py @@ -0,0 +1,94 @@ +import os +import tempfile + +from django.core.files.uploadedfile import InMemoryUploadedFile +from django.test import TestCase, override_settings +from django.urls import reverse + +from plugins.models import PluginInvalid +from plugins.tests.model_factories import (UserF, + PluginF, + PluginVersionF, + PluginInvalidF) + + +TESTFILE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'testfiles')) + + +@override_settings(MEDIA_ROOT=tempfile.mkdtemp()) +class TestInvalidPluginView(TestCase): + """Test InvalidPlugin View""" + + # simplemenu will be loaded on base template. We need create its model + fixtures = ['fixtures/simplemenu.json'] + + def setUp(self) -> None: + self.user = UserF.create() + self.user.set_password('password') + self.user.is_staff = True + self.user.save() + + valid_plugins = os.path.join( + TESTFILE_DIR, "valid_metadata_link.zip") + invalid_plugins = os.path.join( + TESTFILE_DIR, "invalid_metadata_link.zip") + self.valid_metadata_link = open(valid_plugins, 'rb') + self.invalid_metadata_link = open(invalid_plugins, 'rb') + + self.plugin = PluginF.create() + + # Create invalid plugins + PluginInvalid.objects.all().delete() + self.plugin_version_1 = PluginVersionF.create() + self.invalid_plugin_1 = PluginInvalidF.build( + plugin=self.plugin_version_1.plugin) + self.invalid_plugin_1.save() + self.plugin_version_2 = PluginVersionF.create() + self.invalid_plugin_2 = PluginInvalidF.build( + plugin=self.plugin_version_2.plugin) + self.invalid_plugin_2.save() + + def tearDown(self): + self.valid_metadata_link.close() + self.invalid_metadata_link.close() + + def test_PluginInvalid_list_should_return_invalid_plugin(self): + self.assertEqual(PluginInvalid.objects.count(), 2) + response = self.client.get(reverse('invalid_plugins')) + self.assertEqual(response.status_code, 200) + self.assertContains(response, self.invalid_plugin_1.plugin.name) + self.assertContains(response, self.invalid_plugin_2.plugin.name) + self.assertNotContains(response, self.plugin.name) + + def test_update_to_valid_plugin_should_remove_from_invalid_list(self): + plugin_invalid = PluginF.create() + plugin_invalid.package_name="test_modul" + plugin_invalid.save() + self.plugin_version_1.plugin = plugin_invalid + self.plugin_version_1.save() + + url = reverse( + 'version_update', + kwargs={ + "package_name": self.plugin_version_1.plugin.package_name, + "version": self.plugin_version_1.version + }) + data = { + "package": InMemoryUploadedFile( + self.valid_metadata_link, + field_name='tempfile', + name='testfile.zip', + content_type='application/zip', + size=39889, + charset='utf8' + ) + } + self.client.login(username=self.user.username, password="password") + response = self.client.post(url, data, follow=True) + # import pdb + # pdb.set_trace() + self.assertEqual(response.status_code, 200) + # TODO + # put breakpoint on update version and check what's happening + # self.assertEqual(PluginInvalid.objects.count(), 1) From e51123fb469bd0b18f78ab4b8cbbf20d7bfde070 Mon Sep 17 00:00:00 2001 From: suman Date: Fri, 23 Jul 2021 18:11:11 +0800 Subject: [PATCH 6/6] added unit test --- qgis-app/plugins/tests/test_views.py | 87 +++++++++++++++++++++++----- qgis-app/plugins/views.py | 3 - 2 files changed, 72 insertions(+), 18 deletions(-) diff --git a/qgis-app/plugins/tests/test_views.py b/qgis-app/plugins/tests/test_views.py index 1608fe72..3478b207 100644 --- a/qgis-app/plugins/tests/test_views.py +++ b/qgis-app/plugins/tests/test_views.py @@ -27,6 +27,7 @@ def setUp(self) -> None: self.user = UserF.create() self.user.set_password('password') self.user.is_staff = True + self.user.is_superuser = True self.user.save() valid_plugins = os.path.join( @@ -36,11 +37,17 @@ def setUp(self) -> None: self.valid_metadata_link = open(valid_plugins, 'rb') self.invalid_metadata_link = open(invalid_plugins, 'rb') - self.plugin = PluginF.create() + self.plugin = PluginF.create( + package_name="test_plugin" + ) # Create invalid plugins PluginInvalid.objects.all().delete() - self.plugin_version_1 = PluginVersionF.create() + self.plugin_version_1 = PluginVersionF.create( + version='0.0.0.0' + ) + self.plugin_version_1.plugin.package_name = "test_modul" + self.plugin_version_1.plugin.save() self.invalid_plugin_1 = PluginInvalidF.build( plugin=self.plugin_version_1.plugin) self.invalid_plugin_1.save() @@ -61,18 +68,37 @@ def test_PluginInvalid_list_should_return_invalid_plugin(self): self.assertContains(response, self.invalid_plugin_2.plugin.name) self.assertNotContains(response, self.plugin.name) - def test_update_to_valid_plugin_should_remove_from_invalid_list(self): - plugin_invalid = PluginF.create() - plugin_invalid.package_name="test_modul" - plugin_invalid.save() - self.plugin_version_1.plugin = plugin_invalid - self.plugin_version_1.save() + def test_update_plugin_should_remove_from_invalid_list(self): + self.assertEqual(PluginInvalid.objects.count(), 2) + url = reverse( + 'version_update', + kwargs={ + "package_name": self.invalid_plugin_1.plugin.package_name, + "version": self.invalid_plugin_1.validated_version + }) + data = { + "package": InMemoryUploadedFile( + self.valid_metadata_link, + field_name='tempfile', + name='testfile.zip', + content_type='application/zip', + size=39889, + charset='utf8' + ) + } + self.client.login(username=self.user.username, password="password") + response = self.client.post(url, data, follow=True) + self.assertEqual(response.status_code, 200) + # The updated plugin should be removed from the invalid list + self.assertEqual(PluginInvalid.objects.count(), 1) + def test_update_plugin_should_remove_from_invalid_list(self): + self.assertEqual(PluginInvalid.objects.count(), 2) url = reverse( 'version_update', kwargs={ - "package_name": self.plugin_version_1.plugin.package_name, - "version": self.plugin_version_1.version + "package_name": self.invalid_plugin_1.plugin.package_name, + "version": self.invalid_plugin_1.validated_version }) data = { "package": InMemoryUploadedFile( @@ -86,9 +112,40 @@ def test_update_to_valid_plugin_should_remove_from_invalid_list(self): } self.client.login(username=self.user.username, password="password") response = self.client.post(url, data, follow=True) - # import pdb - # pdb.set_trace() self.assertEqual(response.status_code, 200) - # TODO - # put breakpoint on update version and check what's happening - # self.assertEqual(PluginInvalid.objects.count(), 1) + # The updated plugin should be removed from the invalid list + self.assertEqual(PluginInvalid.objects.count(), 1) + self.assertFalse( + PluginInvalid.objects.filter( + plugin__package_name=self.invalid_plugin_1.plugin.package_name + ).exists() + ) + + def test_create_new_version_should_remove_plugin_from_invalid_list(self): + self.assertEqual(PluginInvalid.objects.count(), 2) + url = reverse( + 'version_create', + kwargs={ + "package_name": "test_modul" + }) + data = { + "package": InMemoryUploadedFile( + self.valid_metadata_link, + field_name='tempfile', + name='testfile.zip', + content_type='application/zip', + size=39889, + charset='utf8' + ) + } + self.client.login(username=self.user.username, password="password") + response = self.client.post(url, data, follow=True) + self.assertEqual(response.status_code, 200) + # The updated plugin should be removed from the invalid list + self.assertEqual(PluginInvalid.objects.count(), 1) + self.assertFalse( + PluginInvalid.objects.filter( + plugin__package_name="test_modul" + ).exists() + ) + diff --git a/qgis-app/plugins/views.py b/qgis-app/plugins/views.py index 6c2d7e00..b0cfbdba 100644 --- a/qgis-app/plugins/views.py +++ b/qgis-app/plugins/views.py @@ -438,9 +438,6 @@ def plugin_update(request, package_name): # Checks for optional metadata _check_optional_metadata(form, request) - # Remove from invalid plugin list - delete_invalid_plugin(plugin) - return HttpResponseRedirect(new_object.get_absolute_url()) else: form = PluginForm(instance = plugin)