Create custom command to validate existing plugins on server

dockerize/docker/REQUIREMENTS.txt

@@ -41,4 +41,6 @@ djangorestframework==3.11.2
 sorl-thumbnail-serializer-field==0.2.1
 django-rest-auth==0.9.5
 drf-yasg==1.17.1
-django-rest-multiple-models==2.1.3
+django-rest-multiple-models==2.1.3
+
+factory_boy==3.2.0

qgis-app/plugins/management/commands/validate_existing_plugins.py

@@ -0,0 +1,125 @@
+"""A command to validate the existing zipfile Plugin Packages.
+
+We are using the same validator that used in uploading plugins.
+Re-run this command when modify the validator to validate the existing plugins.
+"""
+
+import os
+from django.conf import settings
+from django.core.files.uploadedfile import InMemoryUploadedFile
+from django.core.mail import send_mail
+from django.core.management.base import BaseCommand
+from django.contrib.sites.models import Site
+from django.utils.translation import ugettext_lazy as _
+
+from plugins.models import PluginVersion, PluginInvalid
+from plugins.validator import validator
+
+
+DOMAIN = Site.objects.get_current().domain
+
+
+def send_email_notification(plugin, version, message, url_version, recipients):
+
+    message = ('\r\nPlease update '
+               'Plugin: %s '
+               '- Version: %s\r\n'
+               '\r\nIt failed to pass validation with message:'
+               '\r\n%s\r\n'
+               '\r\nLink: %s') % (plugin, version, message, url_version)
+    send_mail(
+        subject='Invalid Plugin Metadata Notification',
+        message=_(message),
+        from_email=settings.DEFAULT_FROM_EMAIL,
+        recipient_list=recipients,
+        fail_silently=True
+    )
+
+
+def get_recipients_email(plugin):
+    receipt_email = []
+    if plugin.created_by.email:
+        receipt_email.append(plugin.created_by.email)
+    if plugin.email:
+        receipt_email.append(plugin.email)
+    return receipt_email
+
+
+def validate_zipfile_version(version):
+
+    if not os.path.exists(version.package.url):
+        return {
+            'plugin': f'{version.plugin.name}',
+            'created_by': f'{version.plugin.created_by}',
+            'version': f'{version.version}',
+            'version_id': version.id,
+            'msg': [f'File does not exist. Please re-upload.'],
+            'url': f'http://{DOMAIN}{version.get_absolute_url()}',
+            'recipients_email': get_recipients_email(version.plugin)
+        }
+
+    with open(version.package.url, 'rb') as buf:
+        package = InMemoryUploadedFile(
+            buf,
+            'tempfile',
+            'filename.zip',
+            'application/zip',
+            1000000,  # ignore the filesize and assume it's 1MB
+            'utf8')
+        try:
+            validator(package)
+        except Exception as e:
+            return {
+                'plugin': f'{version.plugin.name}',
+                'created_by': f'{version.plugin.created_by}',
+                'version': f'{version.version}',
+                'version_id': version.id,
+                'msg': e.messages,
+                'url': f'http://{DOMAIN}{version.get_absolute_url()}',
+                'recipients_email': get_recipients_email(version.plugin)
+            }
+    return None
+
+
+class Command(BaseCommand):
+
+    help = ('Validate existing Plugins zipfile and send a notification email '
+            'for invalid Plugin')
+
+    def handle(self, *args, **options):
+        self.stdout.write('Validating existing plugins...')
+        # get the latest version
+        versions = PluginVersion.approved_objects.\
+            order_by('plugin_id', '-created_on').distinct('plugin_id').all()[:50]
+        num_count = 0
+        for version in versions:
+            error_msg = validate_zipfile_version(version)
+            if error_msg:
+                send_email_notification(
+                    plugin=error_msg['plugin'],
+                    version=error_msg['version'],
+                    message='\r\n'.join(error_msg['msg']),
+                    url_version=error_msg['url'],
+                    recipients=error_msg['recipients_email']
+                )
+                self.stdout.write(
+                    _('Sent email to %s for Plugin %s - Version %s.') % (
+                        error_msg['recipients_email'],
+                        error_msg['plugin'],
+                        error_msg['version']
+                    )
+                )
+                num_count += 1
+                plugin_version = PluginVersion.objects\
+                    .select_related('plugin').get(id=error_msg['version_id'])
+                PluginInvalid.objects.create(
+                    plugin=plugin_version.plugin,
+                    validated_version=plugin_version.version,
+                    message=(error_msg['msg']
+                             if not isinstance(error_msg['msg'], list)
+                             else ', '.join(error_msg['msg']))
+                )
+        self.stdout.write(
+            _('Successfully sent email notification for %s invalid plugins')
+            % (num_count)
+        )

qgis-app/plugins/migrations/0002_plugininvalid.py

@@ -0,0 +1,25 @@
+# Generated by Django 2.2.18 on 2021-06-25 22:31
+
+from django.db import migrations, models
+import django.db.models.deletion
+import plugins.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('plugins', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PluginInvalid',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('validated_version', plugins.models.VersionField(db_index=True, max_length=32, verbose_name='Version')),
+                ('validated_at', models.DateTimeField(auto_now_add=True, verbose_name='Validated at')),
+                ('message', models.CharField(editable=False, help_text='Invalid error message', max_length=256, verbose_name='Message')),
+                ('plugin', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='plugins.Plugin', unique=True)),
+            ],
+        ),
+    ]

qgis-app/plugins/models.py

@@ -628,6 +628,31 @@ def delete_plugin_icon(sender, instance, **kw):
         pass
 
 
+class PluginInvalid(models.Model):
+    """Invalid plugins model.
+
+    There were existing plugins on the server before the validation
+    mechanism updated.
+    We run the management command to validate them, and track the invalid one
+    of the latest version.
+    """
+
+    plugin = models.ForeignKey(Plugin, on_delete=models.CASCADE, unique=True)
+    # We track the version number, not the version instance.
+    # So that when the version has been deleted, we keep the plugin
+    # in the tracking list
+    validated_version = VersionField(
+        _('Version'), max_length=32, db_index=True)
+    validated_at = models.DateTimeField(
+        _('Validated at'), auto_now_add=True, editable=False)
+    message = models.CharField(
+        _('Message'),
+        help_text=_('Invalid error message'),
+        max_length=256,
+        editable=False
+    )
+
+
 models.signals.post_delete.connect(
     delete_version_package, sender=PluginVersion)
 models.signals.post_delete.connect(delete_plugin_icon, sender=Plugin)

qgis-app/plugins/templates/plugins/plugin_base.html

@@ -31,6 +31,7 @@ <h3>{% trans "Plugins" %}</h3>
         <li><a href="{% url "most_downloaded_plugins" %}">{% trans "Top downloads" %}</a></li>
         <li><a href="{% url "most_rated_plugins" %}">{% trans "Most rated" %}</a></li>
         <li><a href="{% url "server_plugins" %}">{% trans "QGIS Server plugins" %}</a></li>
+        <li><a href="{% url "invalid_plugins" %}">{% trans "Invalid plugins" %}</a></li>
 </ul>
 
 <div class="module_menu">

qgis-app/plugins/templates/plugins/plugin_invalid_list.html

@@ -0,0 +1,112 @@
+{% extends 'plugins/plugin_base.html' %}{% load i18n bootstrap_pagination humanize static sort_anchor range_filter thumbnail %}
+{% block extrajs %}
+  <script type="text/javascript" src="{% static "js/jquery.cookie.js" %}"></script>
+  <script language="javascript">
+   $(document).ready(function () {
+     if ($('tr > th > a:contains("Downloads ↓")'))
+     {
+       $('tr > th > a:contains("Downloads ↓")').html('<img src="{% static "images/down_16.png" %}" />&darr;');
+     }
+     if ( $('tr > th > a:contains("Downloads ↑")') )
+     {
+       $('tr > th > a:contains("Downloads ↑")').html('<img src="{% static "images/down_16.png" %}" />&uarr;');
+     }
+     $('tr > th > a:contains("Downloads")').html('<img src="{% static "images/down_16.png" %}" />');
+
+     if ($('tr > th > a:contains("Featured ↓")'))
+     {
+       $('tr > th > a:contains("Featured ↓")').html('<img src="{% static "images/star_16.png" %}" />&darr;');
+     }
+     if ( $('tr > th > a:contains("Featured ↑")') )
+     {
+       $('tr > th > a:contains("Featured ↑")').html('<img src="{% static "images/star_16.png" %}" />&uarr;');
+     }
+     $('tr > th > a:contains("Featured")').html('<img src="{% static "images/star_16.png" %}" />');
+   });
+
+    function toggle_desc(){
+        jQuery('.plugin-description').toggle('slow', function(){
+            jQuery.cookie('plugin-description-visible', jQuery('.plugin-description').is(':visible'));
+        });
+        if(jQuery('.plugin-description').hasClass('hidden')){
+            jQuery('.plugin-description').removeClass('hidden');
+        } else {
+            jQuery('.plugin-description').addClass('hidden');
+        }
+        return false;
+    }
+
+    // Start with descriptions visible
+    jQuery(function(){
+        if (jQuery.cookie('plugin-description-visible') == 'true'){
+            toggle_desc();
+        }
+    });
+
+
+  </script>
+{% endblock %}
+{% block content %}
+    <h2>{% if title %}{{title}}{% else %}{% trans "Invalid Plugins" %}{% endif %}</h2>
+    {# Filtered views menu #}
+    {% if object_list.count %}
+    <div id="list_commands">
+        <span class="num_items">{% blocktrans with records_count=page_obj.paginator.count %}{{ records_count }} records found{% endblocktrans %}</span>&nbsp;&mdash;&nbsp;
+        <a class="toggle_desc" href="javascript:void(0);" onclick="return toggle_desc()">{% trans "Click to toggle descriptions." %}</a>
+    </div>
+    <div class="pagination">
+    {#% include 'sortable_listview/sort_links.html' %#}
+    </div>
+    <table class="table table-striped plugins">
+        <thead>
+            <tr>
+                <th>&nbsp;</th>
+                <th>{% anchor name %}</th>
+                <th>{% anchor author "Author" %}</th>
+                <th>{% anchor validated_version "Validated Version" %}</th>
+                <th>{% anchor validated_at "Validated at" %}</th>
+                <th>{% anchor message "Error message" %}</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for object in invalid_plugins %}
+            <tr class="pmain {% if object.plugin.deprecated %} error deprecated{% endif %}" id="pmain{{object.plugin.pk}}">
+                <td><a title="{% if object.plugin.deprecated %} [DEPRECATED] {% endif %}{% trans "Click here for plugin details" %}" href="{% url "plugin_detail" object.plugin.package_name %}">
+                {% if object.plugin.icon and object.plugin.icon.file %}
+                {% thumbnail object.icon "24x24" format="PNG" as im %}
+                    <img class="plugin-icon" alt="{% trans "Plugin icon" %}" src="{{ im.url }}" width="{{ im.x }}" height="{{ im.y }}" />
+                {% endthumbnail %}
+                {% else %}
+                    <img height="32" width="32" class="plugin-icon" src="{% static "images/qgis-icon-32x32.png" %}" alt="{% trans "Plugin icon" %}" />
+                {% endif %}
+                </a></td>
+                <td><a title="{% if object.plugin.deprecated %} [DEPRECATED] {% endif %}{% trans "Click here for plugin details" %}"  href="{% url "plugin_detail" object.plugin.package_name %}">{{ object.plugin.name }}</a></td>
+                  <td><a title="{% trans "See all plugins by"%} {{ object.plugin.author }}" href="{% url "author_plugins" object.plugin.author %}">{{ object.plugin.author }}</a></td>
+                <td>{{ object.validated_version }}</td>
+                <td>{{ object.validated_at }}</td>
+                <td>{{ object.message }}</td>
+            </tr>
+
+
+            {% endfor %}
+        </tbody>
+    </table>
+    <div class="pagination">
+    {% include 'sortable_listview/pagination.html' %}
+    </div>
+    <div class="alert">
+        <button type="button" class="close" data-dismiss="alert">&times;</button>
+        {% trans "Deprecated plugins are printed in red." %}
+    </div>
+    {% else %}
+    {% block plugins_message %}
+    <div class="alert">
+        <button type="button" class="close" data-dismiss="alert">&times;</button>
+        {% trans "This list is empty!" %}
+    </div>
+    {% endblock %}
+    {% endif %}
+    <script type="text/javascript">
+        jQuery('.popover').popover();
+    </script>
+{% endblock %}