From 5868679e7ab47078da03f2861d05bb4fa808c655 Mon Sep 17 00:00:00 2001 From: Dimas Date: Wed, 19 Jul 2023 10:44:48 +0700 Subject: [PATCH] Add a background task to fetch qgis versions --- .../plugins/tasks/update_qgis_versions.py | 27 +++++ qgis-app/plugins/tests/test_task.py | 94 ++++++++++++++++++ qgis-app/plugins/tests/test_utils.py | 37 +++++++ qgis-app/plugins/utils.py | 49 +++++++++ .../whoosh_index/MAIN_6wzk7090d2stn8v8.seg | Bin 23641 -> 0 bytes qgis-app/whoosh_index/_MAIN_1.toc | Bin 3628 -> 0 bytes 6 files changed, 207 insertions(+) create mode 100644 qgis-app/plugins/tasks/update_qgis_versions.py create mode 100644 qgis-app/plugins/tests/test_task.py create mode 100644 qgis-app/plugins/tests/test_utils.py create mode 100644 qgis-app/plugins/utils.py delete mode 100644 qgis-app/whoosh_index/MAIN_6wzk7090d2stn8v8.seg delete mode 100644 qgis-app/whoosh_index/_MAIN_1.toc diff --git a/qgis-app/plugins/tasks/update_qgis_versions.py b/qgis-app/plugins/tasks/update_qgis_versions.py new file mode 100644 index 00000000..890d836c --- /dev/null +++ b/qgis-app/plugins/tasks/update_qgis_versions.py @@ -0,0 +1,27 @@ +from celery import shared_task +from base.models.site_preferences import SitePreference +from plugins.utils import get_qgis_versions + + +@shared_task +def update_qgis_versions(): + """ + This background task fetches the QGIS versions from the GitHub QGIS releases + and then updates the current QGIS version in the database. + """ + site_preference = SitePreference.objects.first() + if not site_preference: + site_preference = SitePreference.objects.create() + + qgis_versions = get_qgis_versions() + stored_qgis_versions = site_preference.qgis_versions.split(',') + for qgis_version in qgis_versions: + if qgis_version not in stored_qgis_versions: + stored_qgis_versions.append(qgis_version) + stored_qgis_versions = list(filter(None, stored_qgis_versions)) + stored_qgis_versions = [tuple(map(int, v.split('.'))) for v in stored_qgis_versions] + stored_qgis_versions.sort(reverse=True) + stored_qgis_versions = ['.'.join(map(str, v)) for v in stored_qgis_versions] + + site_preference.qgis_versions = ','.join(stored_qgis_versions) + site_preference.save() diff --git a/qgis-app/plugins/tests/test_task.py b/qgis-app/plugins/tests/test_task.py new file mode 100644 index 00000000..7cf22545 --- /dev/null +++ b/qgis-app/plugins/tests/test_task.py @@ -0,0 +1,94 @@ +import os + +from django.test import TestCase, override_settings +from django.conf import settings + +from preferences import preferences +from unittest.mock import patch, MagicMock + +from base.models.site_preferences import SitePreference +from plugins.tasks.generate_plugins_xml import generate_plugins_xml +from plugins.tasks.update_qgis_versions import update_qgis_versions + + +class TestPluginTask(TestCase): + @patch.object(SitePreference.objects, 'first') + @patch.object(SitePreference.objects, 'create') + @patch('plugins.tasks.update_qgis_versions.get_qgis_versions') + def test_update_qgis_versions(self, mock_get_qgis_versions, mock_create, mock_first): + mock_create.return_value = MagicMock() + mock_get_qgis_versions.return_value = ['3.16', '3.11', '3.12'] + site_preference = MagicMock() + site_preference.qgis_versions = '3.16,3.17' + mock_first.return_value = site_preference + + update_qgis_versions() + + self.assertEqual(site_preference.qgis_versions, '3.17,3.16,3.12,3.11') + site_preference.save.assert_called_once() + + @patch.object(SitePreference.objects, 'first') + @patch.object(SitePreference.objects, 'create') + @patch('plugins.tasks.update_qgis_versions.get_qgis_versions') + def test_update_qgis_versions_no_site_preference(self, mock_get_qgis_versions, mock_create, mock_first): + mock_get_qgis_versions.return_value = ['3.16', '3.16', '3.16'] + mock_first.return_value = None + mock_create.return_value = MagicMock() + update_qgis_versions() + mock_create.assert_called_once() + + @override_settings(DEFAULT_PLUGINS_SITE='http://test_plugins_site') + @patch('requests.get') + @patch('os.path.exists', return_value=False) + @patch('os.mkdir') + @patch('builtins.open', new_callable=MagicMock) + def test_generate_plugins_xml(self, mock_open, mock_mkdir, mock_exists, mock_get): + # Given + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = '' + mock_get.return_value = mock_response + preferences.SitePreference.qgis_versions = '3.24,3.25' + + expected_folder_path = os.path.join(settings.MEDIA_ROOT, 'cached_xmls') + + # When + generate_plugins_xml() + + # Then + mock_mkdir.assert_called_once_with(expected_folder_path) + expected_calls = [ + ((f'{settings.DEFAULT_PLUGINS_SITE}/plugins/plugins_new.xml?qgis=3.24',),), + ((f'{settings.DEFAULT_PLUGINS_SITE}/plugins/plugins_new.xml?qgis=3.25',),) + ] + mock_get.assert_has_calls(expected_calls, any_order=True) + mock_open.assert_any_call(os.path.join(expected_folder_path, 'plugins_3.24.xml'), 'w+') + mock_open.assert_any_call(os.path.join(expected_folder_path, 'plugins_3.25.xml'), 'w+') + + @override_settings(DEFAULT_PLUGINS_SITE='http://test_plugins_site') + @patch('requests.get') + @patch('os.path.exists', return_value=False) + @patch('os.mkdir') + @patch('builtins.open', new_callable=MagicMock) + def test_generate_plugins_xml_with_custom_site(self, mock_open, mock_mkdir, mock_exists, mock_get): + # Given + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = '' + mock_get.return_value = mock_response + preferences.SitePreference.qgis_versions = '3.24,3.25' + + expected_folder_path = os.path.join(settings.MEDIA_ROOT, 'cached_xmls') + + # When + generate_plugins_xml('http://custom_plugins_site') + + # Then + mock_mkdir.assert_called_once_with(expected_folder_path) + expected_calls = [ + (('http://custom_plugins_site/plugins/plugins_new.xml?qgis=3.24',),), + (('http://custom_plugins_site/plugins/plugins_new.xml?qgis=3.25',),) + ] + mock_get.assert_has_calls(expected_calls, any_order=True) + mock_open.assert_any_call(os.path.join(expected_folder_path, 'plugins_3.24.xml'), 'w+') + mock_open.assert_any_call(os.path.join(expected_folder_path, 'plugins_3.25.xml'), 'w+') diff --git a/qgis-app/plugins/tests/test_utils.py b/qgis-app/plugins/tests/test_utils.py new file mode 100644 index 00000000..c41cd8a2 --- /dev/null +++ b/qgis-app/plugins/tests/test_utils.py @@ -0,0 +1,37 @@ +from unittest.mock import patch, Mock +from django.test import TestCase +from plugins.utils import ( + get_qgis_versions, + extract_version +) + + +class TestQGISGitHubReleases(TestCase): + + @patch('requests.get') + def test_get_qgis_versions(self, mock_get): + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = [ + {'tag_name': 'final_3_22_10', 'html_url': 'https://github.com/qgis/QGIS/releases/tag/final-3_22_10'}, + {'tag_name': 'beta_3_23_0', 'html_url': 'https://github.com/qgis/QGIS/releases/tag/beta-3_23_0'} + ] + mock_get.return_value = mock_response + + versions = get_qgis_versions() + self.assertIn('3.22', versions) + + @patch('requests.get') + def test_get_github_releases_failed_request(self, mock_get): + mock_response = Mock() + mock_response.status_code = 404 + mock_get.return_value = mock_response + + with self.assertRaises(Exception) as context: + get_qgis_versions() + self.assertTrue('Request failed' in str(context.exception)) + + def test_extract_version(self): + self.assertEqual(extract_version('final-3.22.10'), '3.22') + self.assertEqual(extract_version('beta-3.23.0'), '3.23') + self.assertIsNone(extract_version('invalid-tag')) diff --git a/qgis-app/plugins/utils.py b/qgis-app/plugins/utils.py new file mode 100644 index 00000000..e180e22b --- /dev/null +++ b/qgis-app/plugins/utils.py @@ -0,0 +1,49 @@ +import requests +import re + + +def extract_version(tag): + """ + Extracts the major and minor version from a given tag. + + The tag should be in the format of x.y.z where x, y, and z are + numbers representing major, minor, and patch versions respectively. + + Args: + tag (str): The version tag to be processed. + + Returns: + str: The major and minor version as x.y, or None if no match. + """ + match = re.search(r'(\d+\.\d+\.\d+)', tag) + if match: + version = match.group(1) + version_parts = version.split('.') + return '.'.join(version_parts[:-1]) + else: + return None + + +def get_qgis_versions(): + """ + Fetches all releases from the QGIS GitHub repository and extracts their + major and minor versions. + + Returns: + list: A list of unique major and minor versions of the releases. + + Raises: + Exception: If the request to the GitHub API fails. + """ + url = 'https://api.github.com/repos/qgis/QGIS/releases' + response = requests.get(url) + if response.status_code != 200: + raise Exception('Request failed') + releases = response.json() + all_versions = [] + for release in releases: + tag_name = release['tag_name'].replace('_', '.') + version = extract_version(tag_name) + if version not in all_versions: + all_versions.append(version) + return all_versions diff --git a/qgis-app/whoosh_index/MAIN_6wzk7090d2stn8v8.seg b/qgis-app/whoosh_index/MAIN_6wzk7090d2stn8v8.seg deleted file mode 100644 index 2d585eae5ea505828175b858a378bbb5a73d8b3e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23641 zcmeHP2~-qUy6%0cUg>5>NGqt2C~h-xIYto|(3Z$VCoWL|rCGubjbMz5nnZne3`R6@ zdopo}&L}PzF*9m3nh}k0$%@fYG>&F56OCq$zPP-5`~KB+tGg&XhI#KKr|*%f@Ba0D z|K0w&_jXksB@&#*_moKFos&l=_i|_9pg`wni6o;?GO)^Rl3J~RfZ;i%Aq4ashd3Qm z@t+d?A@4bkd~yDM;Hx}*ae*`v1ZqxDBwB>@MNmKs3R6T7xM{dVBm_yBSVRzoDRL>{BlwE$|b;ufkS-p`Y5l106Gc_#5+ng3KY;$ZiRS9 zsp>!ijjj-n4z&U35UmxhM5{U<_|Ry?7w=W|haiAPV?m~9ziJ-mDWn;h1-V?CrBdHH zMzatpaK&`!FIpm#R9$**bLoXS;hu6NEJ6e}X?{YeH~IsWhLy2gsgO$Bg|9M8Kqju5 z%2$P=th>}=_6m)+K_~aIlBR&_NLIQmp z2*@qq7H~6#pkjen3&b0&zv|TjY6TH;Zy4s)f>H;!}Ac7!%&oE+|+A z0vHqi(k`gD1qv7wucci`5;z(p&}g(TOF$Nsq~YNSZa^ObccNSd`h!M0sjat=`yeag zQqwHGixcFR0v|dF@x^bXAiOZ)0-*lX)F}qbWNwZ`x{sX6@#v=gI51Gs9&n#pIdr?i-~hrYuX4m& zfMxR{kIlj@KuOz`oPCco?)`)p36zwHroApad?*>fMsoCV^AAh#)(s^^?Zqz6V1i9A zkTW@k3=G5@D3q)t$(bBiOO5#1F-n%J_;vrE4v(h zA24%6kc^Bw3O;y>T+{LDW=iXeV-qDSeh%tMasRX@l8mvba6!~yf~CB;+S}p85Mnp4 z^9a4b{7!~sV$$^5KBsax!O}@yG*4Fb@1YY&Mit&<+&h_ISws5wTLEiMddMPR%AnKf z^Zs|B(X4#?-2vI;n&}M9q4A;)L!T#k zHD0$o$vSIC9?58=9wP!5DkBO=UY1JRdp=u+XMdDrQA_?Y5l^)!1qE;K_A+4OKw|Sf z{@ksyj93FI=&Pw~9|ybid)iLpr+Ur9FAh)&g0UjGErn_L86--97Zamvz^~$SV)H#V zCiP;=zE5B6D{YUTM4)7b_L1D!{e7nsWEc{)_&k1tgi_$2$=UZ0Vwmu8kSn|T( z6cA4axQBc_JauywtUW>J7v}#C`pewp%OsW_e;tp1-J_)E&DArHF9o{=;*qc_)-7xx zNKr`6^_2xMw)YXci?`KXJBmUs|0U&|Bd$v3TkmqTPl0~5aE~% zr+!!mSbLN9*1z!W z+i4v8{5R)FUNw70?tZ|^%QT0^mx{(;B6;1IpL;NX^&4rJibf#V%;kKrj9d%Gdm|PnAglm; zF=bysZ1@-~hFL$gh{)nSSVdCsqhQt#6<@)Rf>}Rbv$a8O5yQX_;ph?{%yL2z3HmK0 znG=X&mS0Kn#V}h|HO0r6wQr*E0g8h$%W0zQ-k9ZFr#O#-!B41$^COC(A42=8B!_dys^gf|915vhp4;v5QBAwrhSkd05Q)*+gkEX6*+kyEkSzO^5?^h|+NU zUyQ+jl#TNFu$a~ML0T54QaFe5DTY~X72>mP+)nvvr2ITAW_>nOT5rs9Zc`kL!Ty+; zv;W61TUIPp*27}xtH;Js+DE~xp9NIgYbl=?v-R9V;Zq2M)`lRPY%D`wW_m~C%5WyhGcS5tO3u2L-f7;H0ZVqKc7Tt|LpK}K;6&M_HNiprg78RgER z8oR{BOf7X}I31a3Q>NG0rCheCu*NQPnKK<_Q%iG;ow-GYAd$PYna^hwW*4PRb=KGw zo?LEbja})I;giaxEzBrzfI#gED9)IgmyzvAqpTViQ&x;#jHNjpmB^3Qr67PQcWHAA zGm&w6v7^Q%O|Efi-1&S5s!?%yyu^Th5amf$p`SeWrCp_yq>H8dq~FV;WT~=wvIf}| zd9XZ5K3%>=epaDV^j8!rY89U=^~ywLrLsRR#9`x?@QzFc8q-LEli?3)=!oX!27f}xzaUvx2pJa0*DsrUsRLtdHK&}$Hz z`*jcz+L+miT4}gd0-uU%D<`jcCS_Da5A#3PCxj2a6Wdk!KXW?97mtq%iP=-~U5~lR zwcZb{pS-X-YDt%mD(`09JzxLB-x}V$rd=Mhuzy2& zV`gNRX@}#378@QI9JzN^lsit$cTCDDJN&A_TEI*muWovE>#;fd>g=n%Zx1@&yZ*-! z5vy1A-KDzv$EWwyK9Rmn8^3AfnRYeV=__B#7}e!t*ZG^H-(K79+O2-Gudf?dJ0fo0 zlk#g122Y#2=(Ro0xoZzU9(lQW*(YxddhQcuT=4ucQ7OBw?uvQOb?#16zsY@Us>1_E zR(&7z0`N1l$040!2cMM>|nfEf*g@{cY5pdsr>$I5Og|LXEsze(e-tT#2^ z{%mOT(%XyPs$YHawWPaus^0tdmU5ZyNP%==vcWx+%EYm~67wHCc(A!j7HgN|w&9$AuoK+EE=h*N}WQ zW_jk7u;88zK|2*eJ9mfd{6;Fd7-6#d-TOPoA36Tp%0DUiXH5Tc;$Lf9-6^aV*oa8& z(pEMix6wxAMyh;te*1a72fsDB{b)r*YE0)HF}+T1uYPxHb;nKB@4QpJbzk))ljKAX zs|t1{rv=KA*d_e3Zsv8%{8TOp)63`kWn>@t|J@}*o7Hf8IDg#QcC+etFS3*OcaFbv z9_?q2KXUw$XgZ{32v1M!+pv?2 z)c`ZpFE1;pm9pZj#x^UfOuVcUi>=x=D{FKsWffU9ZB|wb%ZX>aHdrlC)^dRdAl|Vd z_y8o)+~JUO)10{7)q@VLI=HIqAGhI8xMJU2I_~uq+2b0|J=`P4Gp$d&v$Vkd znX;-VE33?bz6{p8ROr*+Y-dhQ${*rgnu4OtT=YG2W=+zdFJ7*9-8d&LcSW_}3VlQz zKeZ?y`^%B1Op4D@*yrRZLAbKA+!ZS<!Vu> zKD#3#MaNm4;`MBuoBSyG3Y=esE4!MNo8eWHV7nY~_d;{)PlW%=%(!K>( zWlr=zL1$9q@*UEqxcCDT-GUf@X6{vVMR%WJZKV^YOs7h(#Bw#S6IuxF;xZ=$<7}i! SsgL!DP#BlTS1D%J#Q!fEbRWY2 diff --git a/qgis-app/whoosh_index/_MAIN_1.toc b/qgis-app/whoosh_index/_MAIN_1.toc deleted file mode 100644 index 1d9c769d7da608a28dfa9ca73c134f8ee7636c43..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3628 zcmaJ^cYGX26+X$5EvFOboG8ZCS;U&pO*D#oB<6Wp8ip-bQ=7_qIqn z!z6OR;pA+9;0?W*ZmOxKw_rjKUC$qSWV~46K zWHE|v80bbAxY*eOk=vc`8yVl19~f0|Il-*p*rjS+;w8th>xY?#-RUdR;#+=iI^`?< zZnft6R>1n2s})oTWhKW|S|@?fOx4HL1ze*pK?&Rm=2*0YujE;UO^>C|3a%xLbsX0x z?z8Ed$9&&%9X!wUn4+7st7S6gYke#;_+xZwP`=RE>bq4t3l-?^MFEu|aK2!Ar@V$8mhT3f&+Q+{^I^Kxy5g zg2>eqIbNp1d2l|6Sc3@R))Z%+HEG001Gkoj@JY1t$s8|NC0odGO4l}1*NY)#!*X<8 zDSPf==J+g7aC-r-P-R-DtN`#z8R`HSUL{9{Q^Kp|G#HZ%uaQG~y^7b$M9|@ONL$?v zJ=`g6M5(?65txI!q>ZYiKLMn43`mpZ=&XVc(0&Ep4`ZBLWPCO0ICL?h}e@ll(61c z4jSIBdcgdCSp^DK>1Btbt7@x7c5U08(%o{I=;mXsp=}`Jlt&_#8g-LIv|L6U$;?pg z2%Yq6%(g8$mp7}eUG%JzXqByL^pgmQRXPURI$BZLXOe$@hZY^74aXme0vo#F#jSY?sj|?h!n2z`ij?YxJ^};(!ia?bD zV7euSToU^L-Xwh>7D#Eh`J5uOU8kbpjLc!tqSzeC?o6Eu9g7Zux6Ep52*1Qkt7_PK z%{ENt7PW@qfkl1cRyxabma<`R+9GJj zq>j*_6|)?VwX)$h%9dakskA<6*yM0@Ok_{|W|A3Wx;bv#Ke!(Ktut=^6R+N};dX|*)q*osY(UD`0 zcc!xYLXIyIvisshc3;Brr7g1iG7w$qG;La*Tg1z$rzujTkU;had!>j_+h{|y}9*b4rp1pJnz;8qDw zf`EE6$G4|tr+iXVl)=c>KIj) zg#EM(`};Y5APxHmIetil{liJvKf>{&En)wd4Ex79e&T_`{z;B^wYu`DbHgrbv}TwS zMuq7ME}fAseVXIlsY{>X_*vo7=MtAb&+!W_E`3qD^d*k>JdjIY=J=IXm%e(gOX40C zu;~ELg182JjpNr-FYe{|4dKN%6ED8S@!Kt4dPgI&-M1270fup6#}Vb}v%*b5_&gMB~^5e`5eMqv!Dhl4N<8r%S)gZ&_nKWug| z5KRk2OZpMVAFE(M5zHKvpaP3}2#8Mc6OKP^J@3!xIM7@)D1XZz`w$A15On1Z*anZlLh=coHXE60sjG~3^-lDf5K@4&J^%paK?bM1^l-P zXW>2;$~&W8sypdd#eX>dH!Sv&PvSn5nd!Kb%Rwenh@%b7?-S8j%--Z-YaDftZyjZo zDszJPc-(U&=CK-e%OeSj$5F>f-*7IDI(=5LN^!LA@~OiU+qPV>rF5AeINK+;$I)`S z^AltYZPAMHt^IO554BP++O9caTU9HFvop~u`GL^2*n3^8?#J0nqVD