[Docker] Add support for Varnish setup with Docker

.dockerignore

@@ -3,7 +3,8 @@
 
 ### GIT files
 .git
-vendor/*/*/.git
+# Skipped as it makes build very slow and since it is needed by composer when working on dev branches
+#vendor/*/*/.git
 
 ### Symfony template
 # Cache and logs (Symfony2)

.env

@@ -25,3 +25,8 @@ APP_DOCKER_FILE=Dockerfile
 
 # Install config
 INSTALL_EZ_INSTALL_TYPE=clean
+
+# Behat / Selenium config
+EZP_TEST_REST_HOST=web
+BEHAT_SELENIUM_HOST=selenium
+BEHAT_WEB_HOST=web

.travis.yml

@@ -1,5 +1,6 @@
 sudo: required
 dist: trusty
+group: beta
 language: generic
 
 services:
@@ -16,23 +17,20 @@ env:
     - SYMFONY_DEBUG=1
   # list of behat arguments to test
   matrix:
-    # Disabled until failures on BD tests with redis is solved / figured out
+    # Disabled until failures on BDD tests with redis is solved / figured out
     #- TEST_CMD="bin/behat -vv --profile=rest --suite=fullJson --tags=~@broken" COMPOSE_FILE="doc/docker-compose/base-prod.yml:doc/docker-compose/redis.yml:doc/docker-compose/selenium.yml"
-    - TEST_CMD="bin/behat -vv --profile=rest --suite=fullJson --tags=~@broken" COMPOSE_FILE="doc/docker-compose/base-prod.yml:doc/docker-compose/selenium.yml"
-    - TEST_CMD="bin/behat -vv --profile=rest --suite=fullXml --tags=~@broken"
-    - TEST_CMD="bin/behat -vv --profile=core --tags=~@broken"
-    - TEST_CMD="bin/phpunit -v vendor/ezsystems/ezpublish-kernel/eZ/Bundle/EzPublishRestBundle/Tests/Functional"
-    - TEST_CMD="bin/behat -vv --profile=platformui --tags='@common'"
+    - TEST_CMD="bin/behat -vv --profile=rest --suite=fullJson --tags=~@broken" BEHAT_WEB_HOST="varnish" COMPOSE_FILE="doc/docker-compose/base-prod.yml:doc/docker-compose/varnish.yml:doc/docker-compose/selenium.yml"
+ #   - TEST_CMD="bin/behat -vv --profile=rest --suite=fullXml --tags=~@broken"
+ #   - TEST_CMD="bin/behat -vv --profile=core --tags=~@broken"
+ #   - TEST_CMD="bin/phpunit -v vendor/ezsystems/ezpublish-kernel/eZ/Bundle/EzPublishRestBundle/Tests/Functional"
+ #   - TEST_CMD="bin/behat -vv --profile=platformui --tags='@common'"
 
 # test only master (+ Pull requests)
 branches:
   only:
     - master
     - /^\d.\d+$/
 
-# Update Docker and Docker Compose
-before_install: ./bin/.travis/trusty/update_docker.sh
-
 before_script:
   # Internal auth token dedicated to testing with travis+composer on ezsystems repos, not for reuse!
   - echo "{\"github-oauth\":{\"github.com\":\"d0285ed5c8644f30547572ead2ed897431c1fc09\"}}" > auth.json
@@ -45,7 +43,14 @@ before_script:
 
 # Execute test command, need to use sh to get right exit code (docker/compose/issues/3379)
 # Behat will use behat.yml which is a copy of behat.yml.dist with hostnames update by doc/docker-compose/selenium.yml
-script: docker-compose exec --user www-data app sh -c "php /scripts/wait_for_db.php; php $TEST_CMD"
+script: docker-compose exec --user www-data app sh -c "php /scripts/wait_for_db.php; php -d xdebug.auto_trace=ON -d xdebug.trace_output_dir=app/logs/trace/ $TEST_CMD"
+
+after_failure:
+  # Will show us the full log of container's main processes (not counting shell process above running php and behat)
+  - docker-compose logs -t --tail="all"
+  # Will show us what is up, and how long it's been up
+  - docker ps -s
+  - docker-compose exec --user www-data app cat app/logs/trace/*
 
 # disable mail notifications
 notifications:

Dockerfile

@@ -1,4 +1,4 @@
-FROM ezsystems/php:7.0-v1
+FROM ezsystems/php:7.0-v1-dev
 
 # Build argument about keeping auth.json or not (by default on as prod images should'nt get updates via composer update)
 ARG REMOVE_AUTH=1
@@ -14,7 +14,7 @@ RUN if [ -d .git ]; then echo "ERROR: .dockerignore folders detected, exiting" &
 
 # Install and prepare install
 RUN mkdir -p web/var \
-    && composer install --optimize-autoloader --no-progress --no-interaction --prefer-dist \
+    && composer install --optimize-autoloader --no-progress --no-interaction --no-suggest --prefer-dist \
 # Clear cache again so env variables are taken into account on startup
     && rm -Rf app/logs/* app/cache/*/* \
 # Fix permissions for www-data

app/config/default_parameters.yml

@@ -28,3 +28,7 @@ parameters:
 
     cache_memcached_port: 11211
     cache_redis_port: 6379
+
+    # Settings for HttpCache
+    purge_type: "local"
+    purge_server: "http://my.varnish.server:80"

app/config/env/docker.php

@@ -94,3 +94,9 @@
     $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__ . '/../cache_pool'));
     $loader->load($pool . '.yml');
 }
+
+// HttpCache setting (for configuring Varnish purging)
+if ($purgeServer = getenv('HTTPCACHE_PURGE_SERVER')) {
+    $container->setParameter('purge_type', 'http');
+    $container->setParameter('purge_server', $purgeServer);
+}

app/config/ezplatform.yml

@@ -1,4 +1,8 @@
 ezpublish:
+    # HttpCache settings, By default 'local' (Symfony HttpCache Proxy), setting it to 'http' you can point it to Varnish
+    http_cache:
+        purge_type: %purge_type%
+
     # Repositories configuration, setup default repository to support solr if enabled
     repositories:
         default:
@@ -29,3 +33,6 @@ ezpublish:
             # so removing eng-GB from this list may lead to errors or content not being shown, unless you change
             # all eng-GB data to other locales first.
             languages: [eng-GB]
+            # HttpCache purge server(s) setting, eg Varnish, for when ezpublish.http_cache.purge_type is set to 'http'.
+            http_cache:
+                purge_servers: ["%purge_server%"]

bin/.travis/trusty/update_docker.sh

@@ -10,10 +10,10 @@
 # sudo apt-get --reinstall -y [...] install docker-engine=1.11.0-0~jessie
 # http://apt.dockerproject.org/repo/dists/debian-jessie/main/binary-amd64/Packages
 
-
-DOCKER_COMPOSE_VERSION="1.8.0"
-echo "\nUpdating Docker Compose to ${DOCKER_COMPOSE_VERSION}"
-sudo rm -f /usr/local/bin/docker-compose
-curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
-chmod +x docker-compose
-sudo mv docker-compose /usr/local/bin
+# VM docker version is now 1.12, no need for this
+#DOCKER_COMPOSE_VERSION="1.8.0"
+#echo "\nUpdating Docker Compose to ${DOCKER_COMPOSE_VERSION}"
+#sudo rm -f /usr/local/bin/docker-compose
+#curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+#chmod +x docker-compose
+#sudo mv docker-compose /usr/local/bin

doc/docker-compose/Dockerfile-varnish

@@ -0,0 +1,26 @@
+#FROM debian:jessie
+FROM debian:stretch
+# Based on https://hub.docker.com/r/kardasz/varnish/~/dockerfile/
+# stretch is used in order to install varnish-modules withouth having to compile from source
+
+# links:
+# - https://github.com/varnish/varnish-modules/blob/master/docs/vmod_xkey.rst
+# - https://wikitech.wikimedia.org/wiki/XKey
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN \
+    apt-get update -q -y && \
+#    apt-get install -q -y --force-yes --no-install-recommends apt-transport-https curl ca-certificates
+    apt-get install -q -y --force-yes --no-install-recommends ca-certificates varnish-modules varnish
+
+#RUN \
+#    curl https://repo.varnish-cache.org/GPG-key.txt | apt-key add - && \
+#    echo "deb https://repo.varnish-cache.org/debian/ jessie varnish-4.1" >> /etc/apt/sources.list.d/varnish-cache.list && \
+#    apt-get update -q -y && \
+#    apt-get install -q -y --force-yes --no-install-recommends varnish
+
+
+EXPOSE 80 6082
+
+CMD ["varnishd", "-F", "-a", ":80", "-T", ":6082", "-f", "/etc/varnish/default.vcl", "-s", "malloc,256M"]

doc/docker-compose/entrypoint/varnish/varnish4.vcl

@@ -0,0 +1,221 @@
+// Varnish 4 style - eZ 5.4+ / 2014.09+
+// Complete VCL example
+
+vcl 4.0;
+
+// Our Backend - Assuming that web server is listening on port 80
+// Replace the host to fit your setup
+backend ezplatform {
+    .host = "web";
+    .port = "80";
+}
+
+// ACL for invalidators IP
+acl invalidators {
+    "127.0.0.1";
+    "172.16.0.0"/20;
+    "app";
+}
+
+// ACL for debuggers IP
+acl debuggers {
+    "127.0.0.1";
+    "172.16.0.0"/20;
+}
+
+// Called at the beginning of a request, after the complete request has been received
+sub vcl_recv {
+
+    // Set the backend
+    set req.backend_hint = ezplatform;
+
+    // Advertise Symfony for ESI support
+    set req.http.Surrogate-Capability = "abc=ESI/1.0";
+
+    // Add a unique header containing the client address (only for master request)
+    // Please note that /_fragment URI can change in Symfony configuration
+    if (!req.url ~ "^/_fragment") {
+        if (req.http.x-forwarded-for) {
+            set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
+        } else {
+            set req.http.X-Forwarded-For = client.ip;
+        }
+    }
+
+    // Trigger cache purge if needed
+    call ez_purge;
+
+    // Don't cache requests other than GET and HEAD.
+    if (req.method != "GET" && req.method != "HEAD") {
+        return (pass);
+    }
+
+    // Normalize the Accept-Encoding headers
+    if (req.http.Accept-Encoding) {
+        if (req.http.Accept-Encoding ~ "gzip") {
+            set req.http.Accept-Encoding = "gzip";
+        } elsif (req.http.Accept-Encoding ~ "deflate") {
+            set req.http.Accept-Encoding = "deflate";
+        } else {
+            unset req.http.Accept-Encoding;
+        }
+    }
+
+    // Don't cache Authenticate & Authorization
+    // You may remove this when using REST API with basic auth.
+    if (req.http.Authenticate || req.http.Authorization) {
+        if (client.ip ~ debuggers) {
+            set req.http.X-Debug = "Not Cached according to configuration (Authorization)";
+        }
+        return (hash);
+    }
+
+    // Do a standard lookup on assets
+    // Note that file extension list below is not extensive, so consider completing it to fit your needs.
+    if (req.url ~ "\.(css|js|gif|jpe?g|bmp|png|tiff?|ico|img|tga|wmf|svg|swf|ico|mp3|mp4|m4a|ogg|mov|avi|wmv|zip|gz|pdf|ttf|eot|wof)$") {
+        return (hash);
+    }
+
+    // Retrieve client user hash and add it to the forwarded request.
+    call ez_user_hash;
+
+    // If it passes all these tests, do a lookup anyway.
+    return (hash);
+}
+
+// Called when the requested object has been retrieved from the backend
+sub vcl_backend_response {
+
+    if (bereq.http.accept ~ "application/vnd.fos.user-context-hash"
+        && beresp.status >= 500
+    ) {
+        return (abandon);
+    }
+
+    // Optimize to only parse the Response contents from Symfony
+    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
+        unset beresp.http.Surrogate-Control;
+        set beresp.do_esi = true;
+    }
+
+    // Allow stale content, in case the backend goes down or cache is not fresh any more
+    // make Varnish keep all objects for 1 hours beyond their TTL
+    set beresp.grace = 1h;
+}
+
+// Handle purge
+// You may add FOSHttpCacheBundle tagging rules
+// See http://foshttpcache.readthedocs.org/en/latest/varnish-configuration.html#id4
+sub ez_purge {
+
+    if (req.method == "BAN") {
+        if (!client.ip ~ invalidators) {
+            return (synth(405, "Method Not Allowed"));
+        }
+
+        if (req.http.X-Location-Id) {
+            ban("obj.http.X-Location-Id ~ " + req.http.X-Location-Id);
+            if (client.ip ~ debuggers) {
+                set req.http.X-Debug = "Ban done for content connected to LocationId " + req.http.X-Location-Id;
+            }
+            return (synth(200, "Banned"));
+        }
+    }
+}
+
+// Sub-routine to get client user hash, for context-aware HTTP cache.
+sub ez_user_hash {
+
+    // Prevent tampering attacks on the hash mechanism
+    if (req.restarts == 0
+        && (req.http.accept ~ "application/vnd.fos.user-context-hash"
+            || req.http.x-user-hash
+        )
+    ) {
+        return (synth(400));
+    }
+
+    if (req.restarts == 0 && (req.method == "GET" || req.method == "HEAD")) {
+        // Get User (Context) hash, for varying cache by what user has access to.
+        // https://doc.ez.no/display/EZP/Context+aware+HTTP+cache
+
+        // Anonymous user w/o session => Use hardcoded anonymous hash to avoid backend lookup for hash
+        if (req.http.Cookie !~ "eZSESSID" && !req.http.authorization) {
+            // You may update this hash with the actual one for anonymous user
+            // to get a better cache hit ratio across anonymous users.
+            // Note: You should then update it every time anonymous user rights change.
+            set req.http.X-User-Hash = "38015b703d82206ebc01d17a39c727e5";
+        }
+        // Pre-authenticate request to get shared cache, even when authenticated
+        else {
+            set req.http.x-fos-original-url    = req.url;
+            set req.http.x-fos-original-accept = req.http.accept;
+            set req.http.x-fos-original-cookie = req.http.cookie;
+            // Clean up cookie for the hash request to only keep session cookie, as hash cache will vary on cookie.
+            set req.http.cookie = ";" + req.http.cookie;
+            set req.http.cookie = regsuball(req.http.cookie, "; +", ";");
+            set req.http.cookie = regsuball(req.http.cookie, ";(eZSESSID[^=]*)=", "; \1=");
+            set req.http.cookie = regsuball(req.http.cookie, ";[^ ][^;]*", "");
+            set req.http.cookie = regsuball(req.http.cookie, "^[; ]+|[; ]+$", "");
+
+            set req.http.accept = "application/vnd.fos.user-context-hash";
+            set req.url = "/_fos_user_context_hash";
+
+            // Force the lookup, the backend must tell how to cache/vary response containing the user hash
+
+            return (hash);
+        }
+    }
+
+    // Rebuild the original request which now has the hash.
+    if (req.restarts > 0
+        && req.http.accept == "application/vnd.fos.user-context-hash"
+    ) {
+        set req.url         = req.http.x-fos-original-url;
+        set req.http.accept = req.http.x-fos-original-accept;
+        set req.http.cookie = req.http.x-fos-original-cookie;
+
+        unset req.http.x-fos-original-url;
+        unset req.http.x-fos-original-accept;
+        unset req.http.x-fos-original-cookie;
+
+        // Force the lookup, the backend must tell not to cache or vary on the
+        // user hash to properly separate cached data.
+
+        return (hash);
+    }
+}
+
+sub vcl_deliver {
+    // On receiving the hash response, copy the hash header to the original
+    // request and restart.
+    if (req.restarts == 0
+        && resp.http.content-type ~ "application/vnd.fos.user-context-hash"
+    ) {
+        set req.http.x-user-hash = resp.http.x-user-hash;
+
+        return (restart);
+    }
+
+    // If we get here, this is a real response that gets sent to the client.
+
+    // Remove the vary on context user hash, this is nothing public. Keep all
+    // other vary headers.
+    set resp.http.Vary = regsub(resp.http.Vary, "(?i),? *x-user-hash *", "");
+    set resp.http.Vary = regsub(resp.http.Vary, "^, *", "");
+    if (resp.http.Vary == "") {
+        unset resp.http.Vary;
+    }
+
+    // Sanity check to prevent ever exposing the hash to a client.
+    unset resp.http.x-user-hash;
+
+    if (client.ip ~ debuggers) {
+        if (obj.hits > 0) {
+            set resp.http.X-Cache = "HIT";
+            set resp.http.X-Cache-Hits = obj.hits;
+        } else {
+            set resp.http.X-Cache = "MISS";
+        }
+    }
+}

doc/docker-compose/install.yml

@@ -30,7 +30,7 @@ services:
     # Second chown line:  For dev and behat tests we give a bit extra rights, never do this for prod.
     command: >
       /bin/sh -c "
-      composer install --no-progress --no-interaction --prefer-dist;
+      composer install --no-progress --no-interaction --no-suggest --prefer-dist;
       mkdir -p web/var;
       php /scripts/wait_for_db.php;
       php app/console ezplatform:install ${INSTALL_EZ_INSTALL_TYPE};