WIP - create region or domain specific profiles

.gitignore

@@ -15,3 +15,5 @@ settings/clip/clip.prj
 settings/clip/clip.qpj
 settings/clip/clip.shp
 settings/clip/clip.shx
+SouthAfrica/south-africa.pbf
+Libraries/south-africa.pbf

Libraries/Dockerfile

@@ -0,0 +1,11 @@
+# This image is intended to be used as a storage
+# only container
+
+FROM byrnedo/alpine-curl
+MAINTAINER Tim Sutton <[email protected]>
+
+RUN mkdir /settings
+RUN curl -o /settings/country.pbf http://download.openstreetmap.fr/extracts/africa/south_africa.osm.pbf
+ADD mapping.yml /settings/mapping.yml
+ADD clip /settings/clip
+ADD qgis_style.sql /settings/qgis_style,sql

Libraries/Makefile

@@ -0,0 +1,140 @@
+PROJECT_ID := libraries
+COMPOSE_FILE := docker-compose.yml
+# Uncomment the next line if you want to display data with Leaflet.
+# COMPOSE_FILE := docker-compose-web.yml
+
+.PHONY: logs
+
+###
+#    DOCKER MANAGEMENT
+###
+
+status:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Status in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) ps
+
+
+build:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Building in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) build
+
+run:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) up -d --no-recreate
+
+rundev:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running in DEVELOPMENT mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) up
+
+stop:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Stopping in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) stop
+
+kill:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Killing in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) kill
+
+rm: kill
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Removing production instance!!! "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) rm
+
+rm-volumes:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Removing all volumes!!!! "
+	@echo "------------------------------------------------------------------"
+	@docker volume rm libraries_osm-postgis-data libraries_import_queue libraries_import_done libraries_cache
+
+logs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Logs"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) logs
+
+live_logs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Live Logs"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) logs -f
+
+
+###
+#    CLIPPING
+###
+
+
+import_clip:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Importing clip shapefile"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)_db /usr/bin/shp2pgsql -c -I -D -s 4326 /home/settings/clip/clip.shp | docker exec -i $(PROJECT_ID)_db su - postgres -c "psql gis"
+
+remove_clip:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Removing clip shapefile"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS clip;'"
+
+###
+#    STATS
+###
+
+
+timestamp:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Timestamp"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)_imposm cat /home/settings/timestamp.txt
+
+###
+#    STYLES
+###
+
+
+import_styles: remove_styles
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Importing QGIS styles"
+	@echo "------------------------------------------------------------------"
+	@docker exec -i $(PROJECT_ID)_db su - postgres -c "psql -f /home/settings/qgis_style.sql gis"
+
+remove_styles:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Removing QGIS styles"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS layer_styles;'"
+
+backup_styles:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Backup QGIS styles to BACKUP.sql"
+	@echo "------------------------------------------------------------------"
+	@echo "SET XML OPTION DOCUMENT;" > BACKUP-STYLES.sql
+	@ docker exec -t $(PROJECT_ID)_db su - postgres -c "/usr/bin/pg_dump --format plain --inserts --table public.layer_styles gis" >> BACKUP-STYLES.sql

Libraries/clip/clip.sql

@@ -0,0 +1,20 @@
+CREATE OR REPLACE FUNCTION clean_tables() RETURNS void AS
+$BODY$
+DECLARE osm_tables CURSOR FOR
+    SELECT table_name
+    FROM information_schema.tables
+    WHERE table_schema='public'
+    AND table_type='BASE TABLE'
+    AND table_name LIKE 'osm_%';
+BEGIN
+    FOR osm_table IN osm_tables LOOP
+        EXECUTE 'DELETE FROM ' || quote_ident(osm_table.table_name) || ' WHERE osm_id IN (
+            SELECT DISTINCT osm_id
+            FROM ' || quote_ident(osm_table.table_name) || '
+            LEFT JOIN clip ON ST_Intersects(geometry, geom)
+            WHERE clip.gid IS NULL)
+        ;';
+    END LOOP;
+END;
+$BODY$
+LANGUAGE plpgsql;

Libraries/docker-compose.yml

@@ -0,0 +1,104 @@
+version: '2.1'
+
+volumes:
+  # Make sure that these names are not used in another docker
+  # compose otherwise this compose that the other will share
+  # the same data volumes
+  libraries-postgis-data:
+  libraries-import-done:
+  libraries-import-queue:
+  libraries-cache:
+
+services:
+  db:
+    # About the postgresql version, it should match in the dockerfile of docker-imposm3
+    image: kartoza/postgis:9.6-2.4
+    hostname: db
+    environment:
+      - POSTGRES_USER=docker
+      - POSTGRES_PASS=docker
+      - POSTGRES_DBNAME=gis
+    volumes:
+      - 'libraries-postgis-data:/var/lib/postgresql'
+    # Uncomment to use the postgis database from outside the docker network
+    ports:
+     - "45432:5432"
+    healthcheck:
+        test: "exit 0"
+    restart: unless-stopped
+
+  imposm:
+    image: kartoza/docker-osm:imposm-latest
+    #build: docker-imposm3
+    volumes:
+      # These are sharable to other containers
+      - .:/home/settings
+      - libraries-import-done:/home/import_done
+      - libraries-import-queue:/home/import_queue
+      - libraries-cache:/home/cache
+    depends_on:
+        db:
+          condition: service_healthy
+    environment:
+      - POSTGRES_USER=docker
+      - POSTGRES_PASS=docker
+      - POSTGRES_DBNAME=gis
+      - POSTGRES_PORT=5432
+      - POSTGRES_HOST=db
+      # seconds between 2 executions of the script
+      # if 0, then no update will be done, only the first initial import from the PBF
+      - TIME=120
+      # folder for settings (with *.json and *.sql)
+      - SETTINGS=settings
+      # folder for caching
+      - CACHE=cache
+      # folder for diff which has been imported
+      - IMPORT_DONE=import_done
+      # folder for diff which hasn't been imported yet
+      - IMPORT_QUEUE=import_queue
+      # it can be 3857
+      - SRID=4326
+      # see http://imposm.org/docs/imposm3/latest/tutorial.html#optimize
+      - OPTIMIZE=false
+      # see http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+      - DBSCHEMA_PRODUCTION=public
+      # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+      - DBSCHEMA_IMPORT=import
+      # http://imposm.org/docs/imposm3/latest/tutorial.html#deploy-production-tables
+      - DBSCHEMA_BACKUP=backup
+      # Install some styles if you are using the default mapping. It can be 'yes' or 'no'
+      - QGIS_STYLE=yes
+      # Use clip in the database
+      - CLIP=no
+    restart: unless-stopped
+
+  osmupdate:
+    #build: docker-osmupdate
+    image: kartoza/docker-osm:osmupdate-latest
+    volumes:
+      # These are sharable to other containers
+      - .:/home/settings
+      - libraries-import-done:/home/import_done
+      - libraries-import-queue:/home/import_queue
+      - libraries-cache:/home/cache
+    environment:
+      # These are all currently the defaults but listed here for your
+      # convenience if you want to change them
+      # the maximum time range to assemble a cumulated changefile.
+      - MAX_DAYS=100
+      # osmupdate uses a combination of minutely, hourly and daily changefiles. This value can be minute, hour, day or sporadic.
+      - DIFF=sporadic
+      # argument to determine the maximum number of parallely processed changefiles.
+      - MAX_MERGE=7
+      # define level for gzip compression. values between 1 (low compression but fast) and 9 (high compression but slow)
+      - COMPRESSION_LEVEL=1
+      # change the URL to use a custom URL to fetch regional file updates.
+      - BASE_URL=http://planet.openstreetmap.org/replication/
+      # folder for diff which hasn't been imported yet
+      - IMPORT_QUEUE=import_queue
+      # folder for diff which has been imported
+      - IMPORT_DONE=import_done
+      # seconds between 2 executions of the script
+      # if 0, then no update will be done, only the first initial import from the PBF
+      - TIME=120
+    restart: unless-stopped

Libraries/last.state.txt

@@ -0,0 +1,3 @@
+timestamp=2018-06-21T13\:39\:25Z
+sequenceNumber=3024788
+replicationUrl=https://planet.openstreetmap.org/replication/minute/

Libraries/mapping.yml

@@ -0,0 +1,108 @@
+areas:
+  area_tags: [buildings]
+  linear_tags: []
+generalized_tables:
+tables:
+  amenities:
+    columns:
+    - name: osm_id
+      type: id
+    - name: geometry
+      type: geometry
+    - key: name
+      name: name
+      type: string
+    - name: type
+      type: mapping_value
+    mapping:
+      amenity:
+      - library
+    type: point
+  buildings:
+    columns:
+    - name: osm_id
+      type: id
+    - name: geometry
+      type: geometry
+    - key: name
+      name: name
+      type: string
+    - name: type
+      type: mapping_value
+    mapping:
+      building:
+      - __any__
+    type: polygon
+  housenumbers:
+    columns:
+    - name: osm_id
+      type: id
+    - name: geometry
+      type: geometry
+    - key: name
+      name: name
+      type: string
+    - name: type
+      type: mapping_value
+    - key: addr:street
+      name: addr:street
+      type: string
+    - key: addr:postcode
+      name: addr:postcode
+      type: string
+    - key: addr:city
+      name: addr:city
+      type: string
+    mapping:
+      addr:housenumber:
+      - __any__
+    type: point
+  housenumbers_interpolated:
+    columns:
+    - name: osm_id
+      type: id
+    - name: geometry
+      type: geometry
+    - key: name
+      name: name
+      type: string
+    - name: type
+      type: mapping_value
+    - key: addr:street
+      name: addr:street
+      type: string
+    - key: addr:postcode
+      name: addr:postcode
+      type: string
+    - key: addr:city
+      name: addr:city
+      type: string
+    - key: addr:inclusion
+      name: addr:inclusion
+      type: string
+    mapping:
+      addr:interpolation:
+      - __any__
+    type: linestring
+  landusages:
+    columns:
+    - name: osm_id
+      type: id
+    - name: geometry
+      type: geometry
+    - key: name
+      name: name
+      type: string
+    - name: type
+      type: mapping_value
+    - name: area
+      type: webmerc_area
+    - args:
+        values:
+        - library
+      name: z_order
+      type: enumerate
+    mapping:
+      amenity:
+      - library
+    type: polygon