diff --git a/.dockerignore b/.dockerignore index b3adbe2670..8dbb1db2d7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,8 @@ ### GIT files .git -vendor/*/*/.git +# Skipped as it makes build very slow and since it is needed by composer when working on dev branches +#vendor/*/*/.git ### Symfony template # Cache and logs (Symfony2) diff --git a/.env b/.env index f5b08cdeaf..45642a7d02 100644 --- a/.env +++ b/.env @@ -25,3 +25,8 @@ APP_DOCKER_FILE=Dockerfile # Install config INSTALL_EZ_INSTALL_TYPE=clean + +# Behat / Selenium config +EZP_TEST_REST_HOST=web +BEHAT_SELENIUM_HOST=selenium +BEHAT_WEB_HOST=web \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 8d551c8dba..3eeb535b3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ sudo: required dist: trusty +group: beta language: generic services: @@ -16,13 +17,13 @@ env: - SYMFONY_DEBUG=1 # list of behat arguments to test matrix: - # Disabled until failures on BD tests with redis is solved / figured out + # Disabled until failures on BDD tests with redis is solved / figured out #- TEST_CMD="bin/behat -vv --profile=rest --suite=fullJson --tags=~@broken" COMPOSE_FILE="doc/docker-compose/base-prod.yml:doc/docker-compose/redis.yml:doc/docker-compose/selenium.yml" - - TEST_CMD="bin/behat -vv --profile=rest --suite=fullJson --tags=~@broken" COMPOSE_FILE="doc/docker-compose/base-prod.yml:doc/docker-compose/selenium.yml" - - TEST_CMD="bin/behat -vv --profile=rest --suite=fullXml --tags=~@broken" - - TEST_CMD="bin/behat -vv --profile=core --tags=~@broken" - - TEST_CMD="bin/phpunit -v vendor/ezsystems/ezpublish-kernel/eZ/Bundle/EzPublishRestBundle/Tests/Functional" - - TEST_CMD="bin/behat -vv --profile=platformui --tags='@common'" + - TEST_CMD="bin/behat -vv --profile=rest --suite=fullJson --tags=~@broken" BEHAT_WEB_HOST="varnish" COMPOSE_FILE="doc/docker-compose/base-prod.yml:doc/docker-compose/varnish.yml:doc/docker-compose/selenium.yml" + # - TEST_CMD="bin/behat -vv --profile=rest --suite=fullXml --tags=~@broken" + # - TEST_CMD="bin/behat -vv --profile=core --tags=~@broken" + # - TEST_CMD="bin/phpunit -v vendor/ezsystems/ezpublish-kernel/eZ/Bundle/EzPublishRestBundle/Tests/Functional" + # - TEST_CMD="bin/behat -vv --profile=platformui --tags='@common'" # test only master (+ Pull requests) branches: @@ -30,9 +31,6 @@ branches: - master - /^\d.\d+$/ -# Update Docker and Docker Compose -before_install: ./bin/.travis/trusty/update_docker.sh - before_script: # Internal auth token dedicated to testing with travis+composer on ezsystems repos, not for reuse! - echo "{\"github-oauth\":{\"github.com\":\"d0285ed5c8644f30547572ead2ed897431c1fc09\"}}" > auth.json @@ -45,7 +43,14 @@ before_script: # Execute test command, need to use sh to get right exit code (docker/compose/issues/3379) # Behat will use behat.yml which is a copy of behat.yml.dist with hostnames update by doc/docker-compose/selenium.yml -script: docker-compose exec --user www-data app sh -c "php /scripts/wait_for_db.php; php $TEST_CMD" +script: docker-compose exec --user www-data app sh -c "php /scripts/wait_for_db.php; php -d xdebug.auto_trace=ON -d xdebug.trace_output_dir=app/logs/trace/ $TEST_CMD" + +after_failure: + # Will show us the full log of container's main processes (not counting shell process above running php and behat) + - docker-compose logs -t --tail="all" + # Will show us what is up, and how long it's been up + - docker ps -s + - docker-compose exec --user www-data app cat app/logs/trace/* # disable mail notifications notifications: diff --git a/Dockerfile b/Dockerfile index fe289be495..8caa011a7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ezsystems/php:7.0-v1 +FROM ezsystems/php:7.0-v1-dev # Build argument about keeping auth.json or not (by default on as prod images should'nt get updates via composer update) ARG REMOVE_AUTH=1 @@ -14,7 +14,7 @@ RUN if [ -d .git ]; then echo "ERROR: .dockerignore folders detected, exiting" & # Install and prepare install RUN mkdir -p web/var \ - && composer install --optimize-autoloader --no-progress --no-interaction --prefer-dist \ + && composer install --optimize-autoloader --no-progress --no-interaction --no-suggest --prefer-dist \ # Clear cache again so env variables are taken into account on startup && rm -Rf app/logs/* app/cache/*/* \ # Fix permissions for www-data diff --git a/app/config/default_parameters.yml b/app/config/default_parameters.yml index 6710401616..85bfeb1d90 100644 --- a/app/config/default_parameters.yml +++ b/app/config/default_parameters.yml @@ -28,3 +28,7 @@ parameters: cache_memcached_port: 11211 cache_redis_port: 6379 + + # Settings for HttpCache + purge_type: "local" + purge_server: "http://my.varnish.server:80" diff --git a/app/config/env/docker.php b/app/config/env/docker.php index 79ca6d6cb5..6eb6833f86 100644 --- a/app/config/env/docker.php +++ b/app/config/env/docker.php @@ -94,3 +94,9 @@ $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__ . '/../cache_pool')); $loader->load($pool . '.yml'); } + +// HttpCache setting (for configuring Varnish purging) +if ($purgeServer = getenv('HTTPCACHE_PURGE_SERVER')) { + $container->setParameter('purge_type', 'http'); + $container->setParameter('purge_server', $purgeServer); +} diff --git a/app/config/ezplatform.yml b/app/config/ezplatform.yml index edc4d2c479..34d3ada8b8 100644 --- a/app/config/ezplatform.yml +++ b/app/config/ezplatform.yml @@ -1,4 +1,8 @@ ezpublish: + # HttpCache settings, By default 'local' (Symfony HttpCache Proxy), setting it to 'http' you can point it to Varnish + http_cache: + purge_type: %purge_type% + # Repositories configuration, setup default repository to support solr if enabled repositories: default: @@ -29,3 +33,6 @@ ezpublish: # so removing eng-GB from this list may lead to errors or content not being shown, unless you change # all eng-GB data to other locales first. languages: [eng-GB] + # HttpCache purge server(s) setting, eg Varnish, for when ezpublish.http_cache.purge_type is set to 'http'. + http_cache: + purge_servers: ["%purge_server%"] diff --git a/bin/.travis/trusty/update_docker.sh b/bin/.travis/trusty/update_docker.sh index 9d4825827c..2a1aaa81d0 100755 --- a/bin/.travis/trusty/update_docker.sh +++ b/bin/.travis/trusty/update_docker.sh @@ -10,10 +10,10 @@ # sudo apt-get --reinstall -y [...] install docker-engine=1.11.0-0~jessie # http://apt.dockerproject.org/repo/dists/debian-jessie/main/binary-amd64/Packages - -DOCKER_COMPOSE_VERSION="1.8.0" -echo "\nUpdating Docker Compose to ${DOCKER_COMPOSE_VERSION}" -sudo rm -f /usr/local/bin/docker-compose -curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose -chmod +x docker-compose -sudo mv docker-compose /usr/local/bin +# VM docker version is now 1.12, no need for this +#DOCKER_COMPOSE_VERSION="1.8.0" +#echo "\nUpdating Docker Compose to ${DOCKER_COMPOSE_VERSION}" +#sudo rm -f /usr/local/bin/docker-compose +#curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose +#chmod +x docker-compose +#sudo mv docker-compose /usr/local/bin diff --git a/doc/docker-compose/Dockerfile-varnish b/doc/docker-compose/Dockerfile-varnish new file mode 100644 index 0000000000..496e65f36d --- /dev/null +++ b/doc/docker-compose/Dockerfile-varnish @@ -0,0 +1,26 @@ +#FROM debian:jessie +FROM debian:stretch +# Based on https://hub.docker.com/r/kardasz/varnish/~/dockerfile/ +# stretch is used in order to install varnish-modules withouth having to compile from source + +# links: +# - https://github.com/varnish/varnish-modules/blob/master/docs/vmod_xkey.rst +# - https://wikitech.wikimedia.org/wiki/XKey + +ENV DEBIAN_FRONTEND noninteractive + +RUN \ + apt-get update -q -y && \ +# apt-get install -q -y --force-yes --no-install-recommends apt-transport-https curl ca-certificates + apt-get install -q -y --force-yes --no-install-recommends ca-certificates varnish-modules varnish + +#RUN \ +# curl https://repo.varnish-cache.org/GPG-key.txt | apt-key add - && \ +# echo "deb https://repo.varnish-cache.org/debian/ jessie varnish-4.1" >> /etc/apt/sources.list.d/varnish-cache.list && \ +# apt-get update -q -y && \ +# apt-get install -q -y --force-yes --no-install-recommends varnish + + +EXPOSE 80 6082 + +CMD ["varnishd", "-F", "-a", ":80", "-T", ":6082", "-f", "/etc/varnish/default.vcl", "-s", "malloc,256M"] diff --git a/doc/docker-compose/entrypoint/varnish/varnish4.vcl b/doc/docker-compose/entrypoint/varnish/varnish4.vcl new file mode 100644 index 0000000000..7be2e95294 --- /dev/null +++ b/doc/docker-compose/entrypoint/varnish/varnish4.vcl @@ -0,0 +1,221 @@ +// Varnish 4 style - eZ 5.4+ / 2014.09+ +// Complete VCL example + +vcl 4.0; + +// Our Backend - Assuming that web server is listening on port 80 +// Replace the host to fit your setup +backend ezplatform { + .host = "web"; + .port = "80"; +} + +// ACL for invalidators IP +acl invalidators { + "127.0.0.1"; + "172.16.0.0"/20; + "app"; +} + +// ACL for debuggers IP +acl debuggers { + "127.0.0.1"; + "172.16.0.0"/20; +} + +// Called at the beginning of a request, after the complete request has been received +sub vcl_recv { + + // Set the backend + set req.backend_hint = ezplatform; + + // Advertise Symfony for ESI support + set req.http.Surrogate-Capability = "abc=ESI/1.0"; + + // Add a unique header containing the client address (only for master request) + // Please note that /_fragment URI can change in Symfony configuration + if (!req.url ~ "^/_fragment") { + if (req.http.x-forwarded-for) { + set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip; + } else { + set req.http.X-Forwarded-For = client.ip; + } + } + + // Trigger cache purge if needed + call ez_purge; + + // Don't cache requests other than GET and HEAD. + if (req.method != "GET" && req.method != "HEAD") { + return (pass); + } + + // Normalize the Accept-Encoding headers + if (req.http.Accept-Encoding) { + if (req.http.Accept-Encoding ~ "gzip") { + set req.http.Accept-Encoding = "gzip"; + } elsif (req.http.Accept-Encoding ~ "deflate") { + set req.http.Accept-Encoding = "deflate"; + } else { + unset req.http.Accept-Encoding; + } + } + + // Don't cache Authenticate & Authorization + // You may remove this when using REST API with basic auth. + if (req.http.Authenticate || req.http.Authorization) { + if (client.ip ~ debuggers) { + set req.http.X-Debug = "Not Cached according to configuration (Authorization)"; + } + return (hash); + } + + // Do a standard lookup on assets + // Note that file extension list below is not extensive, so consider completing it to fit your needs. + if (req.url ~ "\.(css|js|gif|jpe?g|bmp|png|tiff?|ico|img|tga|wmf|svg|swf|ico|mp3|mp4|m4a|ogg|mov|avi|wmv|zip|gz|pdf|ttf|eot|wof)$") { + return (hash); + } + + // Retrieve client user hash and add it to the forwarded request. + call ez_user_hash; + + // If it passes all these tests, do a lookup anyway. + return (hash); +} + +// Called when the requested object has been retrieved from the backend +sub vcl_backend_response { + + if (bereq.http.accept ~ "application/vnd.fos.user-context-hash" + && beresp.status >= 500 + ) { + return (abandon); + } + + // Optimize to only parse the Response contents from Symfony + if (beresp.http.Surrogate-Control ~ "ESI/1.0") { + unset beresp.http.Surrogate-Control; + set beresp.do_esi = true; + } + + // Allow stale content, in case the backend goes down or cache is not fresh any more + // make Varnish keep all objects for 1 hours beyond their TTL + set beresp.grace = 1h; +} + +// Handle purge +// You may add FOSHttpCacheBundle tagging rules +// See http://foshttpcache.readthedocs.org/en/latest/varnish-configuration.html#id4 +sub ez_purge { + + if (req.method == "BAN") { + if (!client.ip ~ invalidators) { + return (synth(405, "Method Not Allowed")); + } + + if (req.http.X-Location-Id) { + ban("obj.http.X-Location-Id ~ " + req.http.X-Location-Id); + if (client.ip ~ debuggers) { + set req.http.X-Debug = "Ban done for content connected to LocationId " + req.http.X-Location-Id; + } + return (synth(200, "Banned")); + } + } +} + +// Sub-routine to get client user hash, for context-aware HTTP cache. +sub ez_user_hash { + + // Prevent tampering attacks on the hash mechanism + if (req.restarts == 0 + && (req.http.accept ~ "application/vnd.fos.user-context-hash" + || req.http.x-user-hash + ) + ) { + return (synth(400)); + } + + if (req.restarts == 0 && (req.method == "GET" || req.method == "HEAD")) { + // Get User (Context) hash, for varying cache by what user has access to. + // https://doc.ez.no/display/EZP/Context+aware+HTTP+cache + + // Anonymous user w/o session => Use hardcoded anonymous hash to avoid backend lookup for hash + if (req.http.Cookie !~ "eZSESSID" && !req.http.authorization) { + // You may update this hash with the actual one for anonymous user + // to get a better cache hit ratio across anonymous users. + // Note: You should then update it every time anonymous user rights change. + set req.http.X-User-Hash = "38015b703d82206ebc01d17a39c727e5"; + } + // Pre-authenticate request to get shared cache, even when authenticated + else { + set req.http.x-fos-original-url = req.url; + set req.http.x-fos-original-accept = req.http.accept; + set req.http.x-fos-original-cookie = req.http.cookie; + // Clean up cookie for the hash request to only keep session cookie, as hash cache will vary on cookie. + set req.http.cookie = ";" + req.http.cookie; + set req.http.cookie = regsuball(req.http.cookie, "; +", ";"); + set req.http.cookie = regsuball(req.http.cookie, ";(eZSESSID[^=]*)=", "; \1="); + set req.http.cookie = regsuball(req.http.cookie, ";[^ ][^;]*", ""); + set req.http.cookie = regsuball(req.http.cookie, "^[; ]+|[; ]+$", ""); + + set req.http.accept = "application/vnd.fos.user-context-hash"; + set req.url = "/_fos_user_context_hash"; + + // Force the lookup, the backend must tell how to cache/vary response containing the user hash + + return (hash); + } + } + + // Rebuild the original request which now has the hash. + if (req.restarts > 0 + && req.http.accept == "application/vnd.fos.user-context-hash" + ) { + set req.url = req.http.x-fos-original-url; + set req.http.accept = req.http.x-fos-original-accept; + set req.http.cookie = req.http.x-fos-original-cookie; + + unset req.http.x-fos-original-url; + unset req.http.x-fos-original-accept; + unset req.http.x-fos-original-cookie; + + // Force the lookup, the backend must tell not to cache or vary on the + // user hash to properly separate cached data. + + return (hash); + } +} + +sub vcl_deliver { + // On receiving the hash response, copy the hash header to the original + // request and restart. + if (req.restarts == 0 + && resp.http.content-type ~ "application/vnd.fos.user-context-hash" + ) { + set req.http.x-user-hash = resp.http.x-user-hash; + + return (restart); + } + + // If we get here, this is a real response that gets sent to the client. + + // Remove the vary on context user hash, this is nothing public. Keep all + // other vary headers. + set resp.http.Vary = regsub(resp.http.Vary, "(?i),? *x-user-hash *", ""); + set resp.http.Vary = regsub(resp.http.Vary, "^, *", ""); + if (resp.http.Vary == "") { + unset resp.http.Vary; + } + + // Sanity check to prevent ever exposing the hash to a client. + unset resp.http.x-user-hash; + + if (client.ip ~ debuggers) { + if (obj.hits > 0) { + set resp.http.X-Cache = "HIT"; + set resp.http.X-Cache-Hits = obj.hits; + } else { + set resp.http.X-Cache = "MISS"; + } + } +} diff --git a/doc/docker-compose/install.yml b/doc/docker-compose/install.yml index 1b4d1efbaf..70cb986d95 100644 --- a/doc/docker-compose/install.yml +++ b/doc/docker-compose/install.yml @@ -30,7 +30,7 @@ services: # Second chown line: For dev and behat tests we give a bit extra rights, never do this for prod. command: > /bin/sh -c " - composer install --no-progress --no-interaction --prefer-dist; + composer install --no-progress --no-interaction --no-suggest --prefer-dist; mkdir -p web/var; php /scripts/wait_for_db.php; php app/console ezplatform:install ${INSTALL_EZ_INSTALL_TYPE}; diff --git a/doc/docker-compose/selenium.yml b/doc/docker-compose/selenium.yml index cdd9310b5e..cab7c02ed8 100644 --- a/doc/docker-compose/selenium.yml +++ b/doc/docker-compose/selenium.yml @@ -16,6 +16,6 @@ services: depends_on: - selenium environment: - - EZP_TEST_REST_HOST=web - - BEHAT_SELENIUM_HOST=selenium - - BEHAT_WEB_HOST=web + - EZP_TEST_REST_HOST + - BEHAT_SELENIUM_HOST + - BEHAT_WEB_HOST diff --git a/doc/docker-compose/varnish.yml b/doc/docker-compose/varnish.yml new file mode 100644 index 0000000000..aa6d92999f --- /dev/null +++ b/doc/docker-compose/varnish.yml @@ -0,0 +1,35 @@ +version: '2' +# Simple single server setup for prod + +services: + app: + environment: + - SYMFONY_HTTP_CACHE=0 + - SYMFONY_TRUSTED_PROXIES=varnish + - HTTPCACHE_PURGE_SERVER=http://varnish + + varnish: + build: + context: . + dockerfile: Dockerfile-varnish + image: my_varnish + mem_limit: 384M + memswap_limit: 512M + ports: + - "8081:80" + depends_on: + - web + volumes: + - ${COMPOSE_DIR}/entrypoint/varnish/varnish4.vcl:/etc/varnish/default.vcl:ro + + +## DEBUG?? +# In need of debugging all request going to Varnish, use varnishlog, example: +# docker-compose exec varnish varnishlog -c -i ReqURL,ReqMethod -I ReqHeader:X-Location-Id +# Or more relevant only BAN's with all info: +# docker-compose exec varnish varnishlog -g request -q "ReqMethod eq 'BAN'" +# +# But before doing that check that http and not local purge client is set: +# docker-compose exec app app/console --env=dev debug:container ezpublish.http_cache.purge_client +# +# And if in prod make sure you have rebuilt app container on code changes ;)