Script to support cloud-init and local easy deploy (#1366)

* add the install script to support cloud-init and local one-shot deployments

* update travis-ci tests

* update docs

* enable no_log again

* update docs

.travis.yml

@@ -1,15 +1,11 @@
 ---
 language: python
 python: "2.7"
-sudo: required
+dist: xenial
-dist: trusty
 
 services:
   - docker
 
-matrix:
-  fast_finish: true
-
 addons:
   apt:
     sources:
@@ -41,35 +37,52 @@ before_cache:
   - sudo tar cf $HOME/lxc/cache.tar /var/lib/lxd/images/
   - sudo chown $USER. $HOME/lxc/cache.tar
 
-env:
+matrix:
-  - LXC_NAME=docker LXC_DISTRO=ubuntu LXC_RELEASE=18.04
+  fast_finish: true
+  include:
+    - stage: Test
+      name: local deployment from docker
+      script:
+        - docker build -t travis/algo .
+        - sudo tar xf $HOME/lxc/cache.tar -C / || echo "Didn't extract cache."
+        - sudo cp -f tests/lxd-bridge /etc/default/lxd-bridge
+        - ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''
+        - echo -e "#cloud-config\nssh_authorized_keys:\n - $(cat ~/.ssh/id_rsa.pub)" | sudo lxc profile set default user.user-data -
+        - sudo service lxd restart
+        - sudo lxc launch ubuntu:18.04 algo
+        - until host algo.lxd 10.0.8.1 -t A; do sleep 3; done
+        - export LXC_IP="$(dig algo.lxd @10.0.8.1 +short)"
+        - pip install -r requirements.txt
+        - pip install ansible-lint
+        - gem install awesome_bot
+        - ansible-playbook --version
+        - tree . -L 2
+        - ansible-playbook main.yml --syntax-check
+        - ./tests/local-deploy.sh
+        - ./tests/update-users.sh
 
-before_install:
+    - stage: Test
-  - test "${LXC_NAME}" != "docker" && sudo modprobe wireguard || docker build -t travis/algo .
+      name: cloud-init deployment
+      script:
+        - sudo tar xf $HOME/lxc/cache.tar -C / || echo "Didn't extract cache."
+        - sudo cp -f tests/lxd-bridge /etc/default/lxd-bridge
+        - sudo service lxd restart
+        - bash tests/cloud-init.sh | sudo lxc profile set default user.user-data -
+        - sudo lxc profile show default
+        - sudo lxc launch ubuntu:18.04 algo
+        - until sudo lxc exec algo -- test -f /var/log/cloud-init-output.log; do echo 'Log file not found, Sleep for 3 seconds'; sleep 3; done
+        - ( sudo lxc exec algo -- tail -f /var/log/cloud-init-output.log & )
+        - |
+          until sudo lxc exec algo -- test -f /var/lib/cloud/data/result.json; do
+            echo 'Cloud init is not finished. Sleep for 30 seconds';
+            sleep 30;
+          done
+        - sudo lxc exec algo -- test -f /opt/algo/configs/localhost/.config.yml
 
-install:
+# script:
-  - sudo tar xf $HOME/lxc/cache.tar -C / || echo "Didn't extract cache."
-  - ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''
-  - chmod 0644 ~/.ssh/config
-  - echo -e "#cloud-config\nssh_authorized_keys:\n - $(cat ~/.ssh/id_rsa.pub)" | sudo lxc profile set default user.user-data -
-  - sudo cp -f tests/lxd-bridge /etc/default/lxd-bridge
-  - sudo service lxd restart
-  - sudo lxc launch ${LXC_DISTRO}:${LXC_RELEASE} ${LXC_NAME}
-  - until host ${LXC_NAME}.lxd 10.0.8.1 -t A; do sleep 3; done
-  - export LXC_IP="$(dig ${LXC_NAME}.lxd @10.0.8.1 +short)"
-  - pip install -r requirements.txt
-  - pip install ansible-lint
-  - gem install awesome_bot
-  - ansible-playbook --version
-  - tree . -L 2
-
-script:
   # - awesome_bot --allow-dupe --skip-save-results *.md docs/*.md --white-list paypal.com,do.co,microsoft.com,https://github.com/trailofbits/algo/archive/master.zip,https://github.com/trailofbits/algo/issues/new
 # - shellcheck algo
 # - ansible-lint main.yml users.yml deploy_client.yml
-  - ansible-playbook main.yml --syntax-check
-  - ./tests/local-deploy.sh
-  - ./tests/update-users.sh
 
 notifications:
   email: false

docs/deploy-from-script-or-cloud-init-to-localhost.md

@@ -0,0 +1,58 @@
+# Deploy from script or cloud-init
+
+You can use `install.sh` to prepare the environment and deploy AlgoVPN on the local Ubuntu server in one shot using cloud-init or run the script directly on the server. The script doesn't configure any parameters in your cloud, so it's on your own to configure related [firewall rules](faq.md#what-inbound-ports-are-used), a floating ip address and other resources you may need.
+
+## Cloud init deployment
+
+You can copy-paste the snippet below to the user data (cloud-init or startup script) field when creating a new server. For now it is only possible for [DigitalOcean](https://www.digitalocean.com/docs/droplets/resources/metadata/), Amazon [EC2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html) and [Lightsail](https://lightsail.aws.amazon.com/ls/docs/en/articles/lightsail-how-to-configure-server-additional-data-shell-script), [Google Cloud](https://cloud.google.com/compute/docs/startupscript) and [Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init).
+
+```
+#!/bin/bash
+curl -s https://raw.githubusercontent.com/trailofbits/algo/master/install.sh | sudo bash -x
+```
+The command will prepare the environment and install AlgoVPN with default parameters. If you want to modify the behaviour you may define additional variables.
+
+## Variables
+
+`METHOD` -  which method of the deployment to use. Possible values are local and cloud. Default: cloud. The cloud method is intended to use in cloud-init deployments only. If you are not using cloud-init to deploy the server you have to use the local method  
+`ONDEMAND_CELLULAR` - "Connect On Demand" when connected to cellular networks. Bollean. Default: false  
+`ONDEMAND_WIFI` - "Connect On Demand" when connected to Wi-Fi. Default: false  
+`ONDEMAND_WIFI_EXCLUDE` - List the names of any trusted Wi-Fi networks where macOS/iOS IPsec clients should not use "Connect On Demand". Comma-separated list.  
+`WINDOWS` - To support Windows 10 or Linux Desktop clients. Default: false  
+`STORE_CAKEY` - To retain the CA key. (required to add users in the future, but less secure). Default: false
+`LOCAL_DNS` - To install an ad blocking DNS resolver. Default: false
+`SSH_TUNNELING` -  Enable SSH tunneling for each user. Default: false  
+`ENDPOINT` - The public IP address or domain name of your server: (IMPORTANT! This is used to verify the certificate). It will be gathered automatically for DigitalOcean, AWS, GCE or Azure if the `METHOD` is cloud. Otherwise you need to define this variable according to your public IP address.  
+`USERS` - list of VPN users. Comma-separated list.
+`REPO_SLUG` - Owner and repository that used to get the installation scripts from. Default: trailofbits/algo
+`REPO_BRANCH` - Branch for `REPO_SLUG`. Default: master
+`EXTRA_VARS` - Additional extra variables.
+`ANSIBLE_EXTRA_ARGS` - Any available ansible parameters. ie: `--skip-tags apparmor`
+
+## Examples
+
+##### How to customise a cloud-init deployment by variables
+
+```
+#!/bin/bash
+export ONDEMAND_CELLULAR=true
+export WINDOWS=true
+export SSH_TUNNELING=true
+curl -s https://raw.githubusercontent.com/trailofbits/algo/master/install.sh | sudo bash -x
+```
+
+##### How to deploy locally without using cloud-init
+
+```
+export METHOD=local
+export ONDEMAND_CELLULAR=true
+curl -s https://raw.githubusercontent.com/trailofbits/algo/master/install.sh | sudo bash -x
+```
+
+##### How to deploy a server using arguments
+
+The arguments order as per [variables](#variables) above
+
+```
+curl -s https://raw.githubusercontent.com/trailofbits/algo/master/install.sh | sudo bash -x -s local true false _null true true true true myvpnserver.com
+```

install.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env sh
+
+set -ex
+
+METHOD="${1:-${METHOD:-cloud}}"
+ONDEMAND_CELLULAR="${2:-${ONDEMAND_CELLULAR:-false}}"
+ONDEMAND_WIFI="${3:-${ONDEMAND_WIFI:-false}}"
+ONDEMAND_WIFI_EXCLUDE="${4:-${ONDEMAND_WIFI_EXCLUDE:-_null}}"
+WINDOWS="${5:-${WINDOWS:-false}}"
+STORE_CAKEY="${6:-${STORE_CAKEY:-false}}"
+LOCAL_DNS="${7:-${LOCAL_DNS:-false}}"
+SSH_TUNNELING="${8:-${SSH_TUNNELING:-false}}"
+ENDPOINT="${9:-${ENDPOINT:-localhost}}"
+USERS="${10:-${USERS:-user1}}"
+REPO_SLUG="${11:-${REPO_SLUG:-trailofbits/algo}}"
+REPO_BRANCH="${12:-${REPO_BRANCH:-master}}"
+EXTRA_VARS="${13:-${EXTRA_VARS:-placeholder=null}}"
+ANSIBLE_EXTRA_ARGS="${14:-${ANSIBLE_EXTRA_ARGS}}"
+
+cd /opt/
+
+installRequirements() {
+  apt-get update
+  apt-get install \
+    software-properties-common \
+    git \
+    build-essential \
+    libssl-dev \
+    libffi-dev \
+    python-dev \
+    python-pip \
+    python-setuptools \
+    python-virtualenv \
+    bind9-host \
+    jq -y
+}
+
+getAlgo() {
+  [ ! -d "algo" ] && git clone https://github.com/${REPO_SLUG} algo
+  cd algo
+
+  git checkout ${REPO_BRANCH}
+
+  python -m virtualenv --python=`which python2` .venv
+  . .venv/bin/activate
+  python -m pip install -U pip virtualenv
+  python -m pip install -r requirements.txt
+}
+
+publicIpFromInterface() {
+  echo "Couldn't find a valid ipv4 address, using the first IP found on the interfaces as the endpoint."
+  DEFAULT_INTERFACE="$(ip -4 route list match default | grep -Eo "dev .*" | awk '{print $2}')"
+  ENDPOINT=$(ip -4 addr sh dev eth0 | grep -w inet | head -n1 | awk '{print $2}' | grep -oE '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b')
+  export ENDPOINT=$ENDPOINT
+  echo "Using ${ENDPOINT} as the endpoint"
+}
+
+publicIpFromMetadata() {
+  if curl -s http://169.254.169.254/metadata/v1/vendor-data | grep DigitalOcean >/dev/null; then
+    PROVIDER="digitalocean"
+    ENDPOINT="$(curl -s http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address)"
+  elif test "$(curl -s http://169.254.169.254/latest/meta-data/services/domain)" = "amazonaws.com"; then
+    PROVIDER="amazon"
+    ENDPOINT="$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)"
+  elif host -t A -W 10 metadata.google.internal 127.0.0.53 >/dev/null; then
+    PROVIDER="gce"
+    ENDPOINT="$(curl -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip")"
+  elif test "$(curl -s -H Metadata:true 'http://169.254.169.254/metadata/instance/compute/publisher/?api-version=2017-04-02&format=text')" = "Canonical"; then
+    PROVIDER="azure"
+    ENDPOINT="$(curl -H Metadata:true 'http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2017-04-02&format=text')"
+  fi
+
+  if echo ${ENDPOINT} | grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b"; then
+    export ENDPOINT=$ENDPOINT
+    echo "Using ${ENDPOINT} as the endpoint"
+  else
+    publicIpFromInterface
+  fi
+}
+
+deployAlgo() {
+  getAlgo
+
+  cd /opt/algo
+  . .venv/bin/activate
+
+  export HOME=/root
+  export ANSIBLE_LOCAL_TEMP=/root/.ansible/tmp
+  export ANSIBLE_REMOTE_TEMP=/root/.ansible/tmp
+
+  ansible-playbook main.yml \
+    -e provider=local \
+    -e ondemand_cellular=${ONDEMAND_CELLULAR} \
+    -e ondemand_wifi=${ONDEMAND_WIFI} \
+    -e ondemand_wifi_exclude=${ONDEMAND_WIFI_EXCLUDE} \
+    -e windows=${WINDOWS} \
+    -e store_cakey=${STORE_CAKEY} \
+    -e local_dns=${LOCAL_DNS} \
+    -e ssh_tunneling=${SSH_TUNNELING} \
+    -e endpoint=$ENDPOINT \
+    -e users=$(echo "$USERS" | jq -Rc 'split(",")') \
+    -e server=localhost \
+    -e ssh_user=root \
+    -e "${EXTRA_VARS}" \
+    --skip-tags debug ${ANSIBLE_EXTRA_ARGS} |
+      tee /var/log/algo.log
+}
+
+if test $METHOD = "cloud"; then
+  publicIpFromMetadata
+fi
+
+installRequirements
+
+deployAlgo

tests/cloud-init.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+echo "#!/bin/bash
+export METHOD=local
+export ONDEMAND_CELLULAR=true
+export ONDEMAND_WIFI=true
+export ONDEMAND_WIFI_EXCLUDE=test1,test2
+export WINDOWS=true
+export STORE_CAKEY=true
+export LOCAL_DNS=true
+export ENDPOINT=algo.lxc
+export USERS=user1,user2
+export EXTRA_VARS='install_headers=false tests=true apparmor_enabled=false'
+export ANSIBLE_EXTRA_ARGS='--skip-tags apparmor'
+export REPO_SLUG=${TRAVIS_PULL_REQUEST_SLUG:-${TRAVIS_REPO_SLUG:-trailofbits/algo}}
+export REPO_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH:-master}}
+
+curl -s https://raw.githubusercontent.com/${TRAVIS_PULL_REQUEST_SLUG:-${TRAVIS_REPO_SLUG}}/${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}/install.sh | sudo -E bash -x"