fixed templateing install and service file, updated install documentation and included new varnish vcl file

7 files changed, 187 insertions, 19 deletions

diff --git a/INSTALLING.rst b/INSTALLING.rst
index b04e226..3920775 100644
--- a/INSTALLING.rst
+++ b/INSTALLING.rst
@@ -39,8 +39,27 @@ As ``$YOURUSER``::
 
 Then visit http://ip-your-boxen/
 
+Basic authentication
+--------------------
+We include the following from /etc/varnish/auth.vcl, to keep passwords out of default vcl:
+
+`req.http.Authorization != "Basic AAAA"`
+
+where AAAA is the result of
+        `echo -n user:password | base64`
+
+Example:
+
+```
+kly@jade:~$ echo -n tech:rules | base64
+dGVjaDpydWxlcw==
+cat /etc/varnish/auth.vcl 
+req.http.Authorization != "Basic dGVjaDpydWxlcw=="
+```
+
+
 Securing InfluxDB
-.................
+-----------------
 
 Default InfluxDB is wide open, to limit this a little we need to configure a few users
 
@@ -69,7 +88,7 @@ Set the influxdb write user and password in /includes/config.pm
 
 Generate the base64 string using ``echo -n "gondulRead:funfunfun" | base64``
 
-Setting up your network...
+Setting up your network
 --------------------------
 
 Gondul tries to detect uplinks and clients on equipment automatically.
diff --git a/ansible/roles/web/files/auth.vcl b/ansible/roles/web/files/auth.vcl
new file mode 100644
index 0000000..81c21eb
--- /dev/null
+++ b/ansible/roles/web/files/auth.vcl
@@ -0,0 +1 @@
+req.http.Authorization != "Basic dGVjaDpydWxlcw==" 
diff --git a/ansible/roles/web/files/template.service b/ansible/roles/web/files/template.service
new file mode 100644
index 0000000..b33b8b8
--- /dev/null
+++ b/ansible/roles/web/files/template.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Gondul Templating
+Documentation=http://google.com
+After=network.target
+
+[Service]
+ExecStart=/opt/gondul/templating/templating.py -h ::1 -p 8081
+MountFlags=slave
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/ansible/roles/web/files/varnish.vcl b/ansible/roles/web/files/varnish.vcl
index 2cdbbfe..c0dc8b6 100644
--- a/ansible/roles/web/files/varnish.vcl
+++ b/ansible/roles/web/files/varnish.vcl
@@ -1,21 +1,97 @@
 # vim: ts=8:expandtab:sw=4:softtabstop=4
 
+# VCL for Gondul - also requires auth.vcl (see further down)
+# Also uses hitch and acmetool for ssl
 vcl 4.0;
 
+import std;
+
+# API - apache
 backend default {
     .host = "::1";
     .port = "8080";
 }
 
+# Templating engine
+backend templating {
+    .host = "::1";
+    .port = "8081";
+}
+
+# Definitely not influx
 backend influx {
     .host = "::1";
     .port = "8086";
 }
 
+# For certbot
+# WTF... isn't this apache? Apparently acmetool listens on port 402
+backend acmetool {
+   .host = "::1";
+   .port = "402";
+}
+
+# White-list localhost - PLEASE make sure this is actually smart
+acl white {
+    "::1";
+    "127.0.0.0"/8;
+    #"172.16.0.0"/12;
+    #"192.168.0.0"/16;
+    #"10.0.0.0"/8;
+}
+
+# vcl_recv is "prep-processing of requests
 sub vcl_recv {
+    # Handle certbot by passing /.well-known to acmetool
+    if (req.url ~ "^/.well-known/acme-challenge/") {
+       set req.backend_hint = acmetool;
+       return(pass);
+    }
+
+    # Redirect to https - note that this does NOT happen for 
+    # "whitelisted" stuff - e.g., templating engine.
+    #disabled as we haven't fixd hitch for ssl termination
+    #if (std.port(local.ip) == 80 && client.ip !~ white) {
+    #    set req.http.x-redir = "https://" + req.http.host + req.url;
+    #    return(synth(301));
+    #}
+
+    # Basic authentication ....
+    # We include the following from /etc/varnish/auth.vcl, to keep passwords
+    # out of default vcl:
+    # req.http.Authorization != "Basic AAAA"
+    #
+    # where AAAA is the result of:
+    # echo -n user:password | base64.
+    # Example:
+    # kly@jade:~$ echo -n tech:rules | base64 
+    # dGVjaDpydWxlcw==
+    # # cat /etc/varnish/auth.vcl 
+    # req.http.Authorization != "Basic dGVjaDpydWxlcw=="
+    if (client.ip !~ white && 
+            include "/etc/varnish/auth.vcl";) {
+        return(synth(401));
+    } else {
+        unset req.http.Authorization;
+        set req.http.X-Webauth-User = "admin";
+    }
+
+    if (req.url ~ "^/api/templates") {
+        set req.url = regsub(req.url,"^/api/templates","");
+        set req.backend_hint = templating;
+    }
+    
+    if (req.url ~ "^/query") {
+        set req.backend_hint = influx;
+    }
+
+    # More human-typable URL
     if (req.url ~ "^/where" || req.url ~ "^/location") {
         set req.url = "/api/public/location";
     }
+    
+    # Fairly standard filtering. Default VCL will do "pipe", which is
+    # pointless for us.
     if (req.method != "GET" &&
         req.method != "HEAD" &&
         req.method != "PUT" &&
@@ -27,50 +103,84 @@ sub vcl_recv {
         return (synth(418,"LOLOLOL"));
     }
 
-    if (req.url ~ "^/query") {
-        set req.backend_hint = influx;
-    }
-
+    # We can only cache GET/HEAD requests.
     if (req.method != "GET" && req.method != "HEAD") {
-        /* We only deal with GET and HEAD by default */
         return (pass);
     }
 
-    # Brukes ikke. Cookies er for nubs.
+    # We don't use cookies - so get rid of them so we don't mess up the cache
+    # by accident.
     unset req.http.Cookie;
 
-    # Tvinges gjennom for å cache med authorization-skrot.
+    # Force hash, since we want to cache with Authorization headers
     return (hash);
 }
 
-
-# Rosa magi
+# vcl_hash runs right after vcl_recv, and determines what
+# is "unique", e.g., what's part of the hash key. We simply
+# add the Authorization header, allowing caching of authenticated
+# content.
+# NOTE: We do NOT run "return" so it will fall back to the default
+# vcl builtin, which will add ip/host and URL as you'd expect.
 sub vcl_hash {
-    # Wheee. Legg til authorization-headeren i hashen.
     hash_data(req.http.authorization);
 }
 
-# Mauve magi. Hva nå enn det er.
-# Dette er WIP - Skal flyttes til backend
+# vcl_synth is run for "synthetic messages": responses generated internally
+# from Varnish, typically error messages or "return (synth...)"
+sub vcl_synth {
+    if (resp.status == 401) {
+        set resp.http.WWW-Authenticate = {"Basic realm="WHAT .... is your favorite color?""};
+    }
+
+    # Second part of redirect-logic
+    if (resp.status == 301) {
+        set resp.http.Location = req.http.x-redir;
+        return (deliver);
+    }
+}
+
+# vcl_backend_response is run when we have a reply from a backend,
+# allowing us to massage the backend response. We wish to do as little
+# as possible here to keep things transparent.
 sub vcl_backend_response {
+    # Expose the URL used for debug purposes and future
+    # cache invalidation.
     set beresp.http.x-url = bereq.url;
+
+    # If the backend response supplies the "x-ban" HTTP response
+    # header, then invalidate based on it. This is used for for
+    # invalidating e.g. switch-management if a switch is added, or the oplog.
     if (beresp.http.x-ban) {
         ban("obj.http.x-url ~ " + beresp.http.x-ban);
     }
+    
+    # Force gzip on text-based content so we don't have to
+    # rely on Apache. 
+    if (beresp.http.content-type ~ "text") {
+        set beresp.do_gzip = true;
+    }
+
+    # Do some hand-crafting for influx. Should probably be
+    # improved... e.g.: with checking error codes.
     if (bereq.url ~ "/query") {
-        # Let's blindly cache influx requests for 5+10s
         set beresp.http.Cache-Control = "max-age=5";
         unset beresp.http.Pragma;
         set beresp.uncacheable = false;
         set beresp.grace = 10s;
         set beresp.ttl = 5s;
     }
+    
+    # Wait, nvm, we catch non-200 here and make them actually cacheable for 5
+    # seconds - we don't want to nuke a backend just because it has ...issues. 
     if (beresp.status != 200) {
         set beresp.uncacheable = false;
         set beresp.ttl = 5s;
     }
-    if (bereq.url ~ "\.(html|css|js)") {
-        # Mainly for ease of development
+    # So for html/css/js there really is no sensible blackend to set
+    # smart TTL, so we hard-code it to 10s. 10s can be a bit annoying
+    # for development, but works.
+    if (bereq.url ~ "\.(html|css|js)" || bereq.url ~ "^/[^/.]*") {
         set beresp.ttl = 10s;
     }
 }
diff --git a/ansible/roles/web/handlers/main.yml b/ansible/roles/web/handlers/main.yml
index 5d36f51..4550f9d 100644
--- a/ansible/roles/web/handlers/main.yml
+++ b/ansible/roles/web/handlers/main.yml
@@ -11,6 +11,12 @@
     daemon_reload: yes
     name: varnish
 
-- name: reload systemd
+- name: restart gondul-template
   systemd:
+    state: restarted
     daemon_reload: yes
+    name: gondul-template
+
+- name: reload systemd
+  systemd:
+    daemon_reload: yes
\ No newline at end of file
diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml
index 1c89947..466dec8 100644
--- a/ansible/roles/web/tasks/main.yml
+++ b/ansible/roles/web/tasks/main.yml
@@ -31,7 +31,9 @@
           'libxml2-dev',
           'build-essential',
           'cpanminus',
-          'apt-transport-https']
+          'apt-transport-https',
+          'python-setuptools',
+          'python3-pip']
       state: present
 
 - name: Add packagecloud.io Varnish apt key.
@@ -59,6 +61,12 @@
   notify:
     - reload systemd
 
+- name: Copy varnish-acl config
+  copy:
+      dest: /etc/varnish/auth.vcl
+      src: auth.vcl
+  notify: restart varnish
+
 - name: Copy varnish config
   copy:
       dest: /etc/varnish/default.vcl
@@ -87,3 +95,12 @@
 - name: Install InfluxDB module
   cpanm:
     name: AnyEvent::InfluxDB
+
+- name: Install python dependencies
+  command: /usr/bin/pip3 install -r /opt/gondul/templating/requirements.txt
+
+- name: add systemd service file for gondul-template
+  copy:
+    dest: /etc/systemd/system/gondul-template.service
+    src: template.service
+  notify: restart gondul-template
\ No newline at end of file
diff --git a/include/config.pm.dist b/include/config.pm
index 78f03cc..78f03cc 100755
--- a/include/config.pm.dist
+++ b/include/config.pm