Sample VCLs

Based on the fantastic work of Mattias Geniar: https://github.com/mattiasgeniar/varnish-4.0-configuration-templates/

Configuring Varnish

ACL purge

  acl purge {
    # ACL we'll use later to allow purges
    "localhost";
    "127.0.0.1";
    "::1";
  }

Backend definition

  backend server1 { # Define one backend
    .host = "127.0.0.1";    # IP or Hostname of backend
    .port = "80";           # Port Apache or whatever is listening
    .max_connections = 300; # That's it

    .probe = {
      #.url = "/"; # short easy way (GET /)
      # We prefer to only do a HEAD /
      .request =
        "HEAD / HTTP/1.1"
        "Host: localhost"
        "Connection: close"
        "User-Agent: Varnish Health Probe";

      .interval  = 5s; # check the health of each backend every 5 seconds
      .timeout   = 1s; # timing out after 1 second.
      .window    = 5;  # If 3 out of the last 5 polls succeeded the backend is considered healthy, otherwise it will be marked as sick
      .threshold = 3;
    }

    .first_byte_timeout     = 300s;   # How long to wait before we receive a first byte from our backend?
    .connect_timeout        = 5s;     # How long to wait for a backend connection?
    .between_bytes_timeout  = 2s;     # How long to wait between bytes received from our backend?
  }

Normalizing header

  sub vcl_recv {
    # Normalize the header, remove the port (in case you're testing this on various TCP ports)
    set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
  }

Removing proxy header

  sub vcl_recv {
    # Remove the proxy header (see https://httpoxy.org/#mitigate-varnish)
    unset req.http.proxy;
  }

Normalizing query arguments

  sub vcl_recv {
    # Normalize the query arguments
    set req.url = std.querysort(req.url);
  }

Allowing purging

  sub vcl_recv {
    # Allow purging
    if (req.method == "PURGE") {
      if (!client.ip ~ purge) { # purge is the ACL defined at the begining
        # Not from an allowed IP? Then die with an error.
        return (synth(405, "This IP is not allowed to send PURGE requests."));
      }
      # If you got this stage (and didn't error out above), purge the cached result
      return (purge);
    }
  }

Dealing with selective header types

  sub vcl_recv {
    if (req.method != "GET" &&
        req.method != "HEAD" &&
        req.method != "PUT" &&
        req.method != "POST" &&
        req.method != "TRACE" &&
        req.method != "OPTIONS" &&
        req.method != "PATCH" &&
        req.method != "DELETE") {
      /* Non-RFC2616 or CONNECT which is weird. */
      return (pipe);
    }
  }

Implementing web socket support example

  sub vcl_recv {
    # Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html)
    if (req.http.Upgrade ~ "(?i)websocket") {
      return (pipe);
    }
  }
  sub vcl_pipe {
    if (req.http.upgrade) {
      set bereq.http.upgrade = req.http.upgrade;
    }

    return (pipe);
  }

Sending traffic to vdir

When using external VMODs you need this.

  sub vcl_recv {
    set req.backend_hint = vdir.backend(); # send all traffic to the vdir director
  }

URL manipulation

Making sure the POST requests are always passed

  sub vcl_recv {
    # Only cache GET or HEAD requests. This makes sure the POST requests are always passed.
    if (req.method != "GET" && req.method != "HEAD") {
      return (pass);
    }
  }

Removing Google Analytics added parameters

  sub vcl_recv {
    if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=") {
      set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
      set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
      set req.url = regsub(req.url, "\?&", "?");
      set req.url = regsub(req.url, "\?$", "");
    }
  }

Example of stripping from URL

The example below strips # from URL, because server has no use for it.

  sub vcl_recv {
    # Strip hash, server doesn't need it.
    if (req.url ~ "\#") {
      set req.url = regsub(req.url, "\#.*$", "");
    }
  }

The example below strips trailing ? from URL.

  sub vcl_recv {
    # Strip a trailing ? if it exists
    if (req.url ~ "\?$") {
      set req.url = regsub(req.url, "\?$", "");
    }
  }

ESI support

  sub vcl_recv {
    # Send Surrogate-Capability headers to announce ESI support to backend
    set req.http.Surrogate-Capability = "key=ESI/1.0";

    if (req.http.Authorization) {
      # Not cacheable by default
      return (pass);
    }
  }

Hashing cookies

  sub vcl_hash {
    # hash cookies for requests that have them
    if (req.http.Cookie) {
      hash_data(req.http.Cookie);
    }
  }

Serving queued requests with Grace

  sub vcl_hit {
    # https://www.varnish-cache.org/docs/trunk/users-guide/vcl-grace.html
    # When several clients are requesting the same page Varnish will send one request to the backend and place the others on hold while fetching one copy from the backend. In some products this is called request coalescing and Varnish does this automatically.
    # If you are serving thousands of hits per second the queue of waiting requests can get huge. There are two potential problems - one is a thundering herd problem - suddenly releasing a thousand threads to serve content might send the load sky high. Secondly - nobody likes to wait. To deal with this we can instruct Varnish to keep the objects in cache beyond their TTL and to serve the waiting requests somewhat stale content.
    # We have no fresh fish. Lets look at the stale ones.
    if (std.healthy(req.backend_hint)) {
      # Backend is healthy. Limit age to 10s.
      if (obj.ttl + 10s > 0s) {
        #set req.http.grace = "normal(limited)";
        return (deliver);
      } else {
        # No candidate for grace. Fetch a fresh object.
        return(fetch);
      }
    } else {
      # backend is sick - use full grace
        if (obj.ttl + obj.grace > 0s) {
        #set req.http.grace = "full";
        return (deliver);
      } else {
        # no graced object.
        return (fetch);
      }
    }

    # fetch & deliver once we get the result
    return (fetch); # Dead code, keep as a safeguard
  }

Passing real IP to backend

sub vcl_recv {
    if (req.restarts == 0) {
        if (req.http.X-Forwarded-For) {
           set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
       } else {
        set req.http.X-Forwarded-For = client.ip;
       }
    }
}

Handling request from backend

How Varnish handles the HTTP request coming from our backends relies on the VCL we write in our backend subroutines.

The vcl_backend_response subroutine is called after the response headers are successfully retrieved from the backend.

Pausing ESI request

  sub vcl_backend_response {
    # Pause ESI request and remove Surrogate-Control header
    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
      unset beresp.http.Surrogate-Control;
      set beresp.do_esi = true;
    }
    return (deliver);
  }

Enabling cache for static files

  sub vcl_backend_response {
    if (bereq.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") {
      unset beresp.http.set-cookie;
    }
    return (deliver);
  }

Streaming

  sub vcl_backend_response {
    # Varnish 4 fully supports Streaming, so use streaming here to avoid locking.
    if (bereq.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(\?.*)?$") {
      unset beresp.http.set-cookie;
      set beresp.do_stream = true;  # Check memory usage it'll grow in fetch_chunksize blocks (128k by default) if the backend doesn't send a Content-Length header, so only enable it for big objects
    }
    return (deliver);
  }

Redirecting

Sometimes, a 301 or 302 redirect formed via Apache’s mod_rewrite can mess with the HTTP port that is being passed along. This often happens with simple rewrite rules in a scenario where Varnish runs on :80 and Apache on :8080 on the same box. A redirect can then often redirect the end-user to a URL on :8080, where it should be :80. This may need fine-tuning on your setup.

  sub vcl_backend_response {
    # To prevent accidental replace, we only filter the 301/302 redirects for now.
    if (beresp.status == 301 || beresp.status == 302) {
      set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", "");
    }
    return (deliver);
  }

Setting cache for static files if unset

  sub vcl_backend_response {
    # Set 2min cache if unset for static files
    if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") {
      set beresp.ttl = 120s; # Important, you shouldn't rely on this, SET YOUR HEADERS in the backend
      set beresp.uncacheable = true;
      return (deliver);
    }
    return (deliver);
  }

Example of excluding from cache

  sub vcl_backend_response {
    # Don't cache 50x responses
    if (beresp.status == 500 || beresp.status == 502 || beresp.status == 503 || beresp.status == 504) {
      return (abandon);
    }
    return (deliver);
  }

Setting Grace mode

  sub vcl_backend_response {
    # Allow stale content, in case the backend goes down.
    # make Varnish keep all objects for 6 hours beyond their TTL
    set beresp.grace = 6h;
  }

Adding debug headers

  sub vcl_deliver {
    if (obj.hits > 0) { # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed
      set resp.http.X-Cache = "HIT";
    } else {
      set resp.http.X-Cache = "MISS";
    }
    return (deliver);
  }

Handling HTTP purge

  sub vcl_purge {
    # Only handle actual PURGE HTTP methods, everything else is discarded
    if (req.method != "PURGE") {
      # restart request
      set req.http.X-Purge = "Yes";
      return(restart);
    }
  }

vcl_synth

  sub vcl_synth {
    if (resp.status == 720) {
      # We use this special error status 720 to force redirects with 301 (permanent) redirects
      # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
      set resp.http.Location = resp.reason;
      set resp.status = 301;
      return (deliver);
    } elseif (resp.status == 721) {
      # And we use error status 721 to force redirects with a 302 (temporary) redirect
      # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
      set resp.http.Location = resp.reason;
      set resp.status = 302;
      return (deliver);
    }

    return (deliver);
  }