Faster-dev

.github/actions/run-docker/action.yml

@@ -19,7 +19,7 @@ inputs:
   compose_file:
     description: 'The docker-compose file to use'
     required: false
-    default: 'docker-compose.yml:docker-compose.ci.yml'
+    default: 'docker-compose.yml'
   logs:
     description: 'Show logs'
     required: false
@@ -46,6 +46,7 @@ runs:
         # Exec the run command in the container
         # quoted 'EOF' to prevent variable expansion
         cat <<'EOF' | docker compose exec --user olympia web sh
+          ls -lan /data/olympia/docs
           ${{ inputs.run }}
         EOF
 

.github/workflows/_test.yml

@@ -41,7 +41,6 @@ jobs:
           -
             name: Needs Locale Compilation
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: |
               make compile_locales
               make test_needs_locales_compilation
@@ -53,22 +52,18 @@ jobs:
           -
             name: Internal Routes
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_internal_routes_allowed
           -
             name: Elastic Search
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_es_tests
           -
             name: Codestyle
             services: web
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make lint-codestyle
           -
             name: Manage Check
             services: web
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make check
     steps:
       - uses: actions/checkout@v4
@@ -78,5 +73,4 @@ jobs:
           version: ${{ inputs.version }}
           digest: ${{ inputs.digest }}
           services: ${{ matrix.services }}
-          compose_file: ${{ matrix.compose_file }}
           run: ${{ matrix.run }}

Dockerfile

@@ -2,12 +2,16 @@
 # Read the docs/topics/development/docker.md file for more information about this Dockerfile.
 ####################################################################################################
 
+ARG OLYMPIA_UID=9500
+
 FROM python:3.11-slim-bookworm AS olympia
 
+ARG OLYMPIA_UID
+
 # Set shell to bash with logs and errors for build
 SHELL ["/bin/bash", "-xue", "-c"]
 
-ENV OLYMPIA_UID=9500
+ENV OLYMPIA_UID=${OLYMPIA_UID}
 RUN <<EOF
 groupadd -g ${OLYMPIA_UID} olympia
 useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_UID} -s /sbin/nologin -d /data/olympia olympia
@@ -62,6 +66,9 @@ ln -s ${HOME}/package.json /deps/package.json
 ln -s ${HOME}/package-lock.json /deps/package-lock.json
 EOF
 
+# Add our custom mime types (required for for ts/json/md files)
+COPY docker/etc/mime.types /etc/mime.types
+
 USER olympia:olympia
 
 ENV PIP_USER=true
@@ -91,14 +98,21 @@ rm -rf /deps/build/*
 ${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt
 EOF
 
-# Define production dependencies as a single layer
-# let's the rest of the stages inherit prod dependencies
-# and makes copying the /deps dir to the final layer easy.
-FROM base AS pip_production
+ARG DOCKER_BUILD DOCKER_COMMIT DOCKER_VERSION
+
+ENV DOCKER_BUILD=${DOCKER_BUILD}
+ENV DOCKER_COMMIT=${DOCKER_COMMIT}
+ENV DOCKER_VERSION=${DOCKER_VERSION}
+
+# This is the terminal stage for local development
+# We skip most of the rest of the build as we don't need built assets
+# and include most copied sources from volumes at runtime.
+FROM base AS development
 
 RUN \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
+    --mount=type=bind,source=./requirements/dev.txt,target=${HOME}/requirements/dev.txt \
     # Files required to install npm dependencies
     --mount=type=bind,source=package.json,target=${HOME}/package.json \
     --mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \
@@ -107,15 +121,18 @@ RUN \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
 <<EOF
 ${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-npm ci ${NPM_ARGS} --include=prod
+${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
+npm install ${NPM_ARGS} --no-save
 EOF
 
-FROM base AS pip_development
+# Define production dependencies as a single layer
+# let's the rest of the stages inherit prod dependencies
+# and makes copying the /deps dir to the final layer easy.
+FROM base AS pip_production
 
 RUN \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
-    --mount=type=bind,source=./requirements/dev.txt,target=${HOME}/requirements/dev.txt \
     # Files required to install npm dependencies
     --mount=type=bind,source=package.json,target=${HOME}/package.json \
     --mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \
@@ -124,8 +141,7 @@ RUN \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
 <<EOF
 ${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
-npm install ${NPM_ARGS} --no-save
+npm ci ${NPM_ARGS} --include=prod
 EOF
 
 FROM base AS locales
@@ -160,16 +176,10 @@ echo "from olympia.lib.settings_base import *" > settings_local.py
 DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
 EOF
 
-FROM base AS sources
-
-ARG DOCKER_BUILD DOCKER_COMMIT DOCKER_VERSION
-
-ENV DOCKER_BUILD=${DOCKER_BUILD}
-ENV DOCKER_COMMIT=${DOCKER_COMMIT}
-ENV DOCKER_VERSION=${DOCKER_VERSION}
+FROM base AS production
 
-# Add our custom mime types (required for for ts/json/md files)
-COPY docker/etc/mime.types /etc/mime.types
+# Copy dependencies from `pip_production`
+COPY --from=pip_production --chown=olympia:olympia /deps /deps
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
 # Copy compiled locales from builder
@@ -178,17 +188,4 @@ COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
 COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
 
-# Set shell back to sh until we can prove we can use bash at runtime
-SHELL ["/bin/sh", "-c"]
-
-FROM sources AS development
-
-# Copy dependencies from `pip_development`
-COPY --from=pip_development --chown=olympia:olympia /deps /deps
-
-FROM sources AS production
-
-# Copy dependencies from `pip_production`
-COPY --from=pip_production --chown=olympia:olympia /deps /deps
-
 

Makefile-docker

@@ -157,7 +157,7 @@ dbshell: ## connect to a database shell
 	$(PYTHON_COMMAND) ./manage.py dbshell
 
 .PHONY: initialize
-initialize: initialize_db update_assets populate_data reindex_data ## init the dependencies, the database, and assets
+initialize: initialize_db populate_data reindex_data ## init the dependencies, the database, and assets
 
 PYTEST_SRC := src/olympia/
 

Makefile-os

@@ -45,6 +45,8 @@ CLEAN_PATHS := \
 	logs \
 	buildx-bake-metadata.json \
 	deps \
+	static-build \
+	site-static \
 
 .PHONY: help_redirect
 help_redirect:
@@ -155,7 +157,7 @@ clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_image
 
 .PHONY: docker_compose_up
 docker_compose_up: docker_mysqld_volume_create ## Start the docker containers
-	docker compose up $(DOCKER_SERVICES) $(DOCKER_COMPOSE_ARGS) $(ARGS)
+	docker compose up $(DOCKER_SERVICES) $(DOCKER_COMPOSE_ARGS) $(ARGS) || docker compose logs
 
 .PHONY: up
 up: setup docker_pull_or_build docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose

docker-bake.hcl

@@ -8,16 +8,29 @@ variable DOCKER_VERSION {}
 variable DOCKER_TARGET {}
 variable DOCKER_TAG {}
 
+variable HOST_UID {}
+
+# User defined function to check if the target is a local build
+function "endswith" {
+  params = [str, suffix]
+  result = (
+    strlen(suffix) <= strlen(str) &&
+    substr(str, strlen(str) - strlen(suffix), strlen(suffix)) == suffix
+  )
+}
+
+
 target "web" {
   context = "."
   dockerfile = "Dockerfile"
   target = "${DOCKER_TARGET}"
   tags = ["${DOCKER_TAG}"]
   platforms = ["linux/amd64"]
   args = {
-	DOCKER_COMMIT = "${DOCKER_COMMIT}"
-	DOCKER_VERSION = "${DOCKER_VERSION}"
-	DOCKER_BUILD = "${DOCKER_BUILD}"
+    DOCKER_COMMIT  = "${DOCKER_COMMIT}"
+    DOCKER_VERSION = "${DOCKER_VERSION}"
+    DOCKER_BUILD   = "${DOCKER_BUILD}"
+    OLYMPIA_UID    = "${endswith("${DOCKER_TAG}", ":local") ? HOST_UID : null}"
   }
   pull = true
 

docker-compose.yml

@@ -51,28 +51,25 @@ services:
       "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
     ]
     volumes:
-      - data_olympia:/data/olympia
-      # Don't mount generated files. They only exist in the container
-      # and would otherwiser be deleted by mounbting data_olympia
-      - /data/olympia/static-build
-      - /data/olympia/site-static
+      - .:/data/olympia
+      - data_static:/data/olympia/static
+      - data_site_static:/data/olympia/site-static
       - storage:/data/olympia/storage
-      - ./package.json:/deps/package.json
-      - ./package-lock.json:/deps/package-lock.json
     extra_hosts:
      - "olympia.test:127.0.0.1"
     restart: on-failure:5
-    # entrypoint.sh takes some time
-    # we can wait for services to be up and running
+    # Healtcheck determines if the container is ready to server requests
+    # A startperiod enables higher frequency healthchecks while the conatiner starts
+    # and a less frequent healtcheck runs after the start period.
     healthcheck:
       test: ["CMD-SHELL", "DJANGO_SETTINGS_MODULE=olympia celery -A olympia.amo.celery status"]
-      # The interval is 90s after the start period of 60s
+      # How often to run the healtcheck after the start period
       interval: 90s
-      # 3 failed attempts result in container failure
+      # How many failed attempts result in a container failure
       retries: 3
-      # While starting, ping faster to get the container healthy as soon as possible
+      # How often to run the healtcheck during the start period
       start_interval: 1s
-      # The start period is 60s
+      # How long the start period lasts
       start_period: 120s
 
   web:
@@ -90,7 +87,8 @@ services:
     image: nginx
     volumes:
       - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf
-      - ./static:/srv/site-static
+      - data_static:/srv/static
+      - data_site_static:/srv/site-static
       - storage:/srv/user-media
     ports:
       - "80:80"
@@ -156,7 +154,7 @@ services:
     platform: linux/amd64
     command: /go/bin/autograph -c /data/olympia/scripts/autograph_localdev_config.yaml
     volumes:
-      - data_olympia:/data/olympia
+      - .:/data/olympia
 
   addons-frontend:
     <<: *env
@@ -187,12 +185,13 @@ volumes:
     name: addons-server_data_mysqld
     external: true
   data_rabbitmq:
-  data_olympia:
+  data_site_static:
+  data_static:
     driver: local
     driver_opts:
       type: none
       o: bind
-      device: ${PWD}
+      device: ./static
   storage:
     driver: local
     driver_opts:

docker/entrypoint.sh

@@ -18,9 +18,16 @@ OLYMPIA_USER="olympia"
 function get_olympia_uid() { echo "$(id -u "$OLYMPIA_USER")"; }
 function get_olympia_gid() { echo "$(id -g "$OLYMPIA_USER")"; }
 
-if [[ -n "${HOST_UID:-}" ]]; then
+# If the host user's uid is not the same as the olympia user's uid
+# change the olympia user's uid to the host user's uid and
+# change the ownership of the /data/olympia directory to the olympia user.
+# This is necessary when using a remote image that was built with a different UID
+# than the current host user's uid.
+IMAGE_UID="$(get_olympia_uid)"
+if [[ -n "${HOST_UID:-}" && "${HOST_UID}" != "${IMAGE_UID}" ]]; then
+  echo "${OLYMPIA_USER} UID: ${IMAGE_UID} -> ${HOST_UID}"
   usermod -u ${HOST_UID} ${OLYMPIA_USER}
-  echo "${OLYMPIA_USER} UID: ${OLYMPIA_UID} -> ${HOST_UID}"
+  chown -R ${OLYMPIA_USER} /data/olympia
 fi
 
 cat <<EOF | su -s /bin/bash $OLYMPIA_USER

src/olympia/core/apps.py

@@ -60,6 +60,9 @@ def version_check(app_configs, **kwargs):
 
 @register(CustomTags.custom_setup)
 def static_check(app_configs, **kwargs):
+    if settings.DEBUG:
+        return []
+
     errors = []
     output = StringIO()