Merge branch 'fix-tests-2.9'

ckan · Sep 1, 2020 · 6b7ec50 · 6b7ec50
2 parents d5cf275 + 519d353
commit 6b7ec50
Show file tree

Hide file tree

Showing 27 changed files with 113 additions and 6,665 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,25 +1,22 @@
-# the new trusty images of Travis cause build errors with psycopg2, see https://github.com/travis-ci/travis-ci/issues/8897
-dist: trusty
-group: deprecated-2017Q4
-
+os: linux
+dist: focal
 language: python
 
-install:
-    - bash bin/travis-build.bash
 services:
+    - docker
     - redis
-    - postgresql
 
+install: bash bin/travis-build.bash
 script: bash bin/travis-run.bash
-before_install:
-    - pip install codecov
-after_success:
-    - codecov
+
+stages:
+  - Flake8
+  - Tests
 
 jobs:
   include:
     - stage: Flake8
-      python: 2.7
+      python: 3.6
       env: FLAKE8=True
       install:
         - pip install flake8==3.5.0
@@ -30,21 +27,60 @@ jobs:
         - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
         # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
         - flake8 . --count --max-line-length=127 --statistics --exclude ckan --exit-zero
+
     - stage: Tests
-      python: "2.7"
+      python: "3.6"
       env: CKANVERSION=master
+      services:
+          - postgresql
+          - redis
+          - docker
+
+    - python: "2.7"
+      env: CKANVERSION=2.9
+      services:
+          - postgresql
+          - redis
+          - docker
+
+
     - python: "3.6"
-      env: CKANVERSION=master
+      env: CKANVERSION=2.9
+      services:
+          - postgresql
+          - redis
+          - docker
+
     - python: "2.7"
       env: CKANVERSION=2.8
+      addons:
+        postgresql: '11'
+        apt:
+          sources:
+            - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main'
+          packages:
+            - postgresql-11
+
     - python: "2.7"
       env: CKANVERSION=2.7
+      addons:
+        postgresql: '9.6'
+        apt:
+          sources:
+            - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main'
+          packages:
+            - postgresql-9.6
+
     - python: "2.7"
       env: CKANVERSION=2.6
-    - python: "2.7"
-      env: CKANVERSION=2.5
-    - python: "2.7"
-      env: CKANVERSION=2.4
+      addons:
+        postgresql: '9.6'
+        apt:
+          sources:
+            - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main'
+          packages:
+            - postgresql-9.6
+
 
 cache:
   directories:

diff --git a/bin/travis-build.bash b/bin/travis-build.bash
@@ -12,11 +12,6 @@ fi
 
 export PYTHON_MAJOR_VERSION=${TRAVIS_PYTHON_VERSION%.*}
 
-
-echo "Installing the packages that CKAN requires..."
-sudo apt-get update -qq
-sudo apt-get install solr-jetty
-
 echo "Installing CKAN and its Python dependencies..."
 git clone https://github.com/ckan/ckan
 cd ckan
@@ -29,48 +24,48 @@ else
     echo "CKAN version: ${CKAN_TAG#ckan-}"
 fi
 
-# install the recommended version of setuptools
+echo "Installing the recommended setuptools requirement"
 if [ -f requirement-setuptools.txt ]
 then
-    echo "Updating setuptools..."
     pip install -r requirement-setuptools.txt
 fi
 
 python setup.py develop
 
-
-# TODO: remove once 2.5.3 is relesed
-# Pin this as newer versions installed by RDFLib give setuptools troubles
-pip install "html5lib==0.9999999"
-
-if [ $CKANVERSION == '2.7' ]
-then
-    echo "Installing setuptools"
-    pip install setuptools==39.0.1
-fi
-
 if (( $CKAN_MINOR_VERSION >= 9 )) && (( $PYTHON_MAJOR_VERSION == 2 ))
 then
     pip install -r requirements-py2.txt
 else
     pip install -r requirements.txt
 fi
+
 pip install -r dev-requirements.txt
 cd -
 
 echo "Setting up Solr..."
-printf "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
-sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
-sudo service jetty restart
+docker run --name ckan-solr -p 8983:8983 -d openknowledge/ckan-solr-dev:$CKANVERSION
+
+echo "Setting up Postgres..."
+export PG_VERSION="$(pg_lsclusters | grep online | awk '{print $1}')"
+export PG_PORT="$(pg_lsclusters | grep online | awk '{print $3}')"
+echo "Using Postgres $PGVERSION on port $PG_PORT"
+if [ $PG_PORT != "5432" ]
+then
+	echo "Using non-standard Postgres port, updating configuration..."
+	sed -i -e "s/postgresql:\/\/ckan_default:pass@localhost\/ckan_test/postgresql:\/\/ckan_default:pass@localhost:$PG_PORT\/ckan_test/" ckan/test-core.ini
+	sed -i -e "s/postgresql:\/\/ckan_default:pass@localhost\/datastore_test/postgresql:\/\/ckan_default:pass@localhost:$PG_PORT\/datastore_test/" ckan/test-core.ini
+	sed -i -e "s/postgresql:\/\/datastore_default:pass@localhost\/datastore_test/postgresql:\/\/datastore_default:pass@localhost:$PG_PORT\/datastore_test/" ckan/test-core.ini
+fi
+
 
 echo "Creating the PostgreSQL user and database..."
-sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
-sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
+sudo -u postgres psql -p $PG_PORT -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
+sudo -u postgres psql -p $PG_PORT -c "CREATE USER datastore_default WITH PASSWORD 'pass';"
+sudo -u postgres psql -p $PG_PORT -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
+sudo -u postgres psql -p $PG_PORT -c 'CREATE DATABASE datastore_test WITH OWNER ckan_default;'
 
 echo "Initialising the database..."
 cd ckan
-
-
 if (( $CKAN_MINOR_VERSION >= 9 ))
 then
     ckan -c test-core.ini db init
@@ -102,6 +97,5 @@ python setup.py develop
 echo "Moving test.ini into a subdir..."
 mkdir subdir
 mv test.ini subdir
-mv test-nose.ini subdir
 
 echo "travis-build.bash is done."
diff --git a/bin/travis-run.bash b/bin/travis-run.bash
@@ -1,17 +1,4 @@
 #!/bin/bash
 set -e
 
-if [ $CKANVERSION == 'master' ]
-then
-    export CKAN_MINOR_VERSION=100
-else
-    export CKAN_MINOR_VERSION=${CKANVERSION##*.}
-fi
-
-
-if (( $CKAN_MINOR_VERSION >= 9 ))
-then
-    pytest --ckan-ini=subdir/test.ini --cov=ckanext.dcat ckanext/dcat/tests
-else
-    nosetests --ckan --nologcapture --with-pylons=subdir/test-nose.ini --with-coverage --cover-package=ckanext.dcat --cover-inclusive --cover-erase --cover-tests ckanext/dcat/tests/nose
-fi
+pytest --ckan-ini=subdir/test.ini --cov=ckanext.dcat ckanext/dcat/tests
diff --git a/ckanext/dcat/plugins/__init__.py b/ckanext/dcat/plugins/__init__.py
@@ -80,6 +80,7 @@ def update_config(self, config):
     def get_helpers(self):
         return {
             'helper_available': utils.helper_available,
+            'dcat_get_endpoint': utils.get_endpoint,
         }
 
     # IActions

diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py
@@ -459,10 +459,7 @@ def _access_rights(self, subject, predicate):
             if isinstance(obj, BNode) and self._object(obj, RDF.type) == DCT.RightsStatement:
                 result = self._object_value(obj, RDFS.label)
             elif isinstance(obj, Literal):
-                if six.PY2:
-                    result = unicode(obj)
-                else:
-                    result = str(obj)
+                result = six.text_type(obj)
         return result
 
     def _distribution_format(self, distribution, normalize_ckan_format=True):

diff --git a/ckanext/dcat/templates/home/index.html b/ckanext/dcat/templates/home/index.html
@@ -1,7 +1,7 @@
 {% ckan_extends %}
 {% block links %}
     {{ super() }}
-    {% with endpoint='dcat.read_catalog' if h.ckan_version() > '2.9' else 'dcat_catalog'  %}
+    {% with endpoint=h.dcat_get_endpoint('catalog')  %}
         <link rel="alternate" type="text/n3" href="{{ h.url_for(endpoint, _format='n3', qualified=True) }}"/>
         <link rel="alternate" type="text/ttl" href="{{ h.url_for(endpoint, _format='ttl', qualified=True) }}"/>
         <link rel="alternate" type="application/rdf+xml" href="{{ h.url_for(endpoint, _format='xml', qualified=True) }}"/>

diff --git a/ckanext/dcat/templates/package/read_base.html b/ckanext/dcat/templates/package/read_base.html
@@ -1,7 +1,7 @@
 {% ckan_extends %}
 {% block links %}
     {{ super() }}
-    {% with endpoint='dcat.read_dataset' if h.ckan_version() > '2.9' else 'dcat_dataset'  %}
+    {% with endpoint=h.dcat_get_endpoint('dataset')  %}
         <link rel="alternate" type="text/n3" href="{{ h.url_for(endpoint, _id=pkg.id, _format='n3', qualified=True) }}"/>
         <link rel="alternate" type="text/ttl" href="{{ h.url_for(endpoint, _id=pkg.id, _format='ttl', qualified=True) }}"/>
         <link rel="alternate" type="application/rdf+xml" href="{{ h.url_for(endpoint, _id=pkg.id, _format='xml', qualified=True) }}"/>

diff --git a/ckanext/dcat/templates/package/search.html b/ckanext/dcat/templates/package/search.html
@@ -1,7 +1,7 @@
 {% ckan_extends %}
 {% block links %}
     {{ super() }}
-    {% with endpoint='dcat.read_catalog' if h.ckan_version() > '2.9' else 'dcat_catalog'  %}
+    {% with endpoint=h.dcat_get_endpoint('catalog')  %}
         <link rel="alternate" type="text/n3" href="{{ h.url_for(endpoint, _format='n3', qualified=True) }}"/>
         <link rel="alternate" type="text/ttl" href="{{ h.url_for(endpoint, _format='ttl', qualified=True) }}"/>
         <link rel="alternate" type="application/rdf+xml" href="{{ h.url_for(endpoint, _format='xml', qualified=True) }}"/>

diff --git a/ckanext/dcat/tests/nose/__init__.py b/ckanext/dcat/tests/nose/__init__.py