From 9a411d3bb4f68a5100378b2db9f0eb5415314f39 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Wed, 2 Aug 2023 14:18:00 +0200 Subject: [PATCH 1/3] extract module and assessment metadata from app CCZ --- commcare_connect/opportunity/app_xml.py | 79 ++++++++++++ .../tests/data/demo_app/modules-0/forms-0.xml | 54 +++++++++ .../tests/data/demo_app/modules-0/forms-1.xml | 53 ++++++++ .../tests/data/demo_app/modules-1/forms-0.xml | 53 ++++++++ .../tests/data/demo_app/modules-2/forms-0.xml | 114 ++++++++++++++++++ .../opportunity/tests/test_app_xml.py | 61 ++++++++++ requirements/dev.in | 1 + requirements/dev.txt | 24 ++++ 8 files changed, 439 insertions(+) create mode 100644 commcare_connect/opportunity/app_xml.py create mode 100644 commcare_connect/opportunity/tests/data/demo_app/modules-0/forms-0.xml create mode 100644 commcare_connect/opportunity/tests/data/demo_app/modules-0/forms-1.xml create mode 100644 commcare_connect/opportunity/tests/data/demo_app/modules-1/forms-0.xml create mode 100644 commcare_connect/opportunity/tests/data/demo_app/modules-2/forms-0.xml create mode 100644 commcare_connect/opportunity/tests/test_app_xml.py diff --git a/commcare_connect/opportunity/app_xml.py b/commcare_connect/opportunity/app_xml.py new file mode 100644 index 00000000..20622b2f --- /dev/null +++ b/commcare_connect/opportunity/app_xml.py @@ -0,0 +1,79 @@ +import itertools +import re +import tempfile +import xml.etree.ElementTree as ET +import zipfile +from dataclasses import dataclass + +import httpx +from django.conf import settings + +XMLNS = "http://commcareconnect.com/data/v1/learn" +XMLNS_PREFIX = "{%s}" % XMLNS + + +@dataclass +class Module: + id: str + name: str + description: str + time_estimate: int + + +@dataclass +class Assessment: + id: str + + +def get_connect_blocks_for_app(domain: str, app_id: str) -> list[str]: + form_xmls = get_form_xml_for_app(domain, app_id) + return list(itertools.chain.from_iterable(extract_connect_blocks(form_xml) for form_xml in form_xmls)) + + +def get_form_xml_for_app(domain: str, app_id: str) -> list[str]: + """Download the CCZ for the given app and return the XML for each form.""" + ccz_url = f"{settings.COMMCARE_HQ_URL}/a/{domain}/apps/api/download_ccz/" + params = { + "app_id": app_id, + "latest": "release", + } + response = httpx.get(ccz_url, params=params) + response.raise_for_status() + + form_xml = [] + with tempfile.NamedTemporaryFile() as file: + file.write(response.content) + file.seek(0) + + form_re = re.compile(r"modules-\d+/forms-\d+\.xml") + with zipfile.ZipFile(file, "r") as zip_ref: + for file in zip_ref.namelist(): + if form_re.match(file): + with zip_ref.open(file) as xml_file: + form_xml.append(xml_file.read().decode()) + return form_xml + + +def extract_connect_blocks(form_xml): + xml = ET.fromstring(form_xml) + yield from extract_modules(xml) + yield from extract_assessments(xml) + + +def extract_assessments(xml: ET.ElementTree) -> list[str]: + for block in xml.findall(f".//{XMLNS_PREFIX}assessment"): + yield Assessment(block.get("id")) + + +def extract_modules(xml: ET.ElementTree): + for block in xml.findall(f".//{XMLNS_PREFIX}module"): + slug = block.get("id") + name = get_element_text(block, "name") + description = get_element_text(block, "description") + time_estimate = get_element_text(block, "time_estimate") + yield Module(slug, name, description, int(time_estimate) if time_estimate is not None else None) + + +def get_element_text(parent, name) -> str | None: + element = parent.find(f"{XMLNS_PREFIX}{name}") + return element.text if element is not None else None diff --git a/commcare_connect/opportunity/tests/data/demo_app/modules-0/forms-0.xml b/commcare_connect/opportunity/tests/data/demo_app/modules-0/forms-0.xml new file mode 100644 index 00000000..4504ff9f --- /dev/null +++ b/commcare_connect/opportunity/tests/data/demo_app/modules-0/forms-0.xml @@ -0,0 +1,54 @@ + + + Module 1 + + + + + + + Module 1 + This is the first module in a series of modules +that will take you through all you need to know. + 1 + + + + + + + + + + + + + + + + + + + + Welcome to module 1 + + + + + + + + + + + + + + + + + + + + diff --git a/commcare_connect/opportunity/tests/data/demo_app/modules-0/forms-1.xml b/commcare_connect/opportunity/tests/data/demo_app/modules-0/forms-1.xml new file mode 100644 index 00000000..888479ce --- /dev/null +++ b/commcare_connect/opportunity/tests/data/demo_app/modules-0/forms-1.xml @@ -0,0 +1,53 @@ + + + Module 2 + + + + + + + Module 2 + This is module 2 of the series. + 2 + + + + + + + + + + + + + + + + + + + + Welcome to module 2 + + + + + + + + + + + + + + + + + + + + diff --git a/commcare_connect/opportunity/tests/data/demo_app/modules-1/forms-0.xml b/commcare_connect/opportunity/tests/data/demo_app/modules-1/forms-0.xml new file mode 100644 index 00000000..01c00841 --- /dev/null +++ b/commcare_connect/opportunity/tests/data/demo_app/modules-1/forms-0.xml @@ -0,0 +1,53 @@ + + + Module 3 + + + + + + + Module 3 + Module 3 in the series + 3 + + + + + + + + + + + + + + + + + + + + Welcome to module 3 + + + + + + + + + + + + + + + + + + + + diff --git a/commcare_connect/opportunity/tests/data/demo_app/modules-2/forms-0.xml b/commcare_connect/opportunity/tests/data/demo_app/modules-2/forms-0.xml new file mode 100644 index 00000000..7df56b76 --- /dev/null +++ b/commcare_connect/opportunity/tests/data/demo_app/modules-2/forms-0.xml @@ -0,0 +1,114 @@ + + + Assessment + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Did you lean anything? + + + Yes + + + No + + + How many modules are there? + + + One + + + Two + + + Three + + + Four + + + Your score is: + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/commcare_connect/opportunity/tests/test_app_xml.py b/commcare_connect/opportunity/tests/test_app_xml.py new file mode 100644 index 00000000..42d10b0e --- /dev/null +++ b/commcare_connect/opportunity/tests/test_app_xml.py @@ -0,0 +1,61 @@ +import tempfile +import zipfile +from pathlib import Path + +import pytest + +from commcare_connect.opportunity.app_xml import Assessment, Module, get_connect_blocks_for_app, get_form_xml_for_app + + +@pytest.fixture +def demo_app_ccz_content(): + """Create a temporary CCZ file with the contents of the `demo_app` folder. + Returns the contents of the CCZ file as bytes. + """ + path = Path(__file__).parent / "data" / "demo_app" + with tempfile.TemporaryFile() as file: + with zipfile.ZipFile(file, "w") as f: + for ccz_file in sorted(path.glob("**/*")): + f.write(ccz_file, ccz_file.as_posix().removeprefix(path.as_posix())) + + file.seek(0) + return file.read() + + +def test_get_form_xml_for_app(httpx_mock, demo_app_ccz_content): + httpx_mock.add_response(content=demo_app_ccz_content) + + form_xml = get_form_xml_for_app("demo_domain", "app_id") + assert len(form_xml) == 4 + assert "http://openrosa.org/formdesigner/52F02F3E-320D-4D91-9EBF-FF4F06226E98" in form_xml[0] + assert "http://openrosa.org/formdesigner/EC1AD740-D2C9-4532-AECC-2D5CF5364696" in form_xml[1] + assert "http://openrosa.org/formdesigner/11151AA2-1599-4C5E-8013-5E2197B6C68E" in form_xml[2] + assert "http://openrosa.org/formdesigner/BD70B3D5-6CB4-4A2E-AD5B-C8E3E7BC37A7" in form_xml[3] + + +def test_get_connect_blocks_for_app(httpx_mock, demo_app_ccz_content): + httpx_mock.add_response(content=demo_app_ccz_content) + + blocks = get_connect_blocks_for_app("demo_domain", "app_id") + assert blocks == [ + Module( + id="module_1", + name="Module 1", + description="This is the first module in a series of modules\n" + "that will take you through all you need to know.", + time_estimate=1, + ), + Module( + id="module_2", + name="Module 2", + description="This is module 2 of the series.", + time_estimate=2, + ), + Module( + id="module_3", + name="Module 3", + description="Module 3 in the series", + time_estimate=3, + ), + Assessment(id="demo"), + ] diff --git a/requirements/dev.in b/requirements/dev.in index d73bf437..1cddb029 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -10,6 +10,7 @@ pip-tools # Testing # ------------------------------------------------------------------------------ pytest +pytest-httpx # Code quality # ------------------------------------------------------------------------------ diff --git a/requirements/dev.txt b/requirements/dev.txt index a12f1ac5..253aa045 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -7,6 +7,7 @@ anyio==3.7.1 # via # -c requirements/base.txt + # httpcore # watchfiles asgiref==3.7.2 # via @@ -24,6 +25,11 @@ black==23.7.0 # via -r requirements/dev.in build==0.10.0 # via pip-tools +certifi==2023.7.22 + # via + # -c requirements/base.txt + # httpcore + # httpx cfgv==3.3.1 # via pre-commit click==8.1.6 @@ -67,12 +73,25 @@ flake8==6.0.0 # flake8-isort flake8-isort==6.0.0 # via -r requirements/dev.in +h11==0.14.0 + # via + # -c requirements/base.txt + # httpcore +httpcore==0.17.3 + # via + # -c requirements/base.txt + # httpx +httpx==0.24.1 + # via + # -c requirements/base.txt + # pytest-httpx identify==2.5.26 # via pre-commit idna==3.4 # via # -c requirements/base.txt # anyio + # httpx iniconfig==2.0.0 # via pytest invoke==2.2.0 @@ -160,8 +179,11 @@ pytest==7.4.0 # via # -r requirements/dev.in # pytest-django + # pytest-httpx pytest-django==4.5.2 # via -r requirements/dev.in +pytest-httpx==0.23.1 + # via -r requirements/dev.in python-dateutil==2.8.2 # via # -c requirements/base.txt @@ -179,6 +201,8 @@ sniffio==1.3.0 # via # -c requirements/base.txt # anyio + # httpcore + # httpx sqlparse==0.4.4 # via # -c requirements/base.txt From 8101d0e5ed49d60cc8a9dc04ced762b898348425 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Wed, 2 Aug 2023 14:21:46 +0200 Subject: [PATCH 2/3] add readme --- commcare_connect/opportunity/tests/data/demo_app/README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 commcare_connect/opportunity/tests/data/demo_app/README.md diff --git a/commcare_connect/opportunity/tests/data/demo_app/README.md b/commcare_connect/opportunity/tests/data/demo_app/README.md new file mode 100644 index 00000000..e558bc41 --- /dev/null +++ b/commcare_connect/opportunity/tests/data/demo_app/README.md @@ -0,0 +1,7 @@ +This content comes from the https://staging.commcarehq.org/a/skelly/apps/view/a671db914232020002c1f294c675d1d8/ app. + +To update it from the app: + +1. Download the CCZ: https://staging.commcarehq.org/a/skelly/apps/api/download_ccz/?app_id=a671db914232020002c1f294c675d1d8 +2. Extract the contents +3. Update the files in this folder From f16083277e8f9f58c374e1ed4b2120f633e24ede Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Thu, 3 Aug 2023 08:48:38 +0200 Subject: [PATCH 3/3] update requirements --- requirements/production.txt | 4 ++++ tasks.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/requirements/production.txt b/requirements/production.txt index bfffd2a4..7fc16bab 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -80,6 +80,10 @@ sqlparse==0.4.4 # via # -c requirements/base.txt # django +typing-extensions==4.7.1 + # via + # -c requirements/base.txt + # asgiref urllib3==1.26.16 # via # -c requirements/base.txt diff --git a/tasks.py b/tasks.py index 3b48dea8..adcc9349 100644 --- a/tasks.py +++ b/tasks.py @@ -33,8 +33,8 @@ def requirements(c: Context, upgrade=False): args = " -U" if upgrade else "" cmd_base = "pip-compile -q --resolver=backtracking" env = {"CUSTOM_COMPILE_COMMAND": "inv requirements"} - c.run(f"{cmd_base} --resolver=backtracking requirements/base.in{args}", env=env) - c.run(f"{cmd_base} --resolver=backtracking requirements/dev.in{args}", env=env) + c.run(f"{cmd_base} requirements/base.in{args}", env=env) + c.run(f"{cmd_base} requirements/dev.in{args}", env=env) # can't use backtracking resolver for now: https://github.com/pypa/pip/issues/8713 c.run(f"{cmd_base} requirements/production.in{args}", env=env)