Github Actions (#1681)

.github/workflows/main.yml

@@ -0,0 +1,160 @@
+name: Main
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+
+      - name: Install dependencies
+        run: |
+          sudo apt update -y
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          sudo snap install shellcheck
+          pip install ansible-lint
+
+      - name: Checks and linters
+        run: |
+          /snap/bin/shellcheck algo install.sh
+          ansible-playbook main.yml --syntax-check
+          ansible-lint -v *.yml roles/{local,cloud-*}/*/*.yml
+
+  scripted-deploy:
+    runs-on: ubuntu-16.04
+    strategy:
+      matrix:
+        UBUNTU_VERSION: ["18.04", "19.04", "19.10"]
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+
+      - name: Install dependencies
+        run: |
+          sudo apt update -y
+          sudo add-apt-repository -yu ppa:wireguard/wireguard
+          sudo apt install -y \
+            python3-pip \
+            lxd \
+            expect-dev \
+            debootstrap \
+            tree \
+            bridge-utils \
+            dnsutils \
+            build-essential \
+            libssl-dev \
+            libffi-dev \
+            python3-dev \
+            linux-headers-$(uname -r) \
+            wireguard \
+            libxml2-utils \
+            crudini \
+            fping \
+            strongswan \
+            libstrongswan-standard-plugins \
+            resolvconf
+
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements.txt
+
+      - name: Provision
+        env:
+          DEPLOY: cloud-init
+          UBUNTU_VERSION: ${{ matrix.UBUNTU_VERSION }}
+        run: |
+          ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''
+          # sed -i "s/^reduce_mtu:\s0$/reduce_mtu: 80/" config.cfg
+          sudo -E ./tests/pre-deploy.sh
+
+      - name: Deployment
+        run: |
+          until sudo lxc exec algo -- test -f /var/log/cloud-init-output.log; do echo 'Log file not found, Sleep for 3 seconds'; sleep 3; done
+          ( sudo lxc exec algo -- tail -f /var/log/cloud-init-output.log & )
+          until sudo lxc exec algo -- test -f /var/lib/cloud/data/result.json; do
+            echo 'Cloud init is not finished. Sleep for 30 seconds';
+            sleep 30;
+          done
+          sudo lxc exec algo -- test -f /opt/algo/configs/localhost/.config.yml
+          sudo lxc exec algo -- tar zcf /root/algo-configs.tar -C /opt/algo/configs/ .
+          sudo lxc file pull algo/root/algo-configs.tar ./
+          sudo tar -C ./configs -zxf algo-configs.tar
+
+      - name: Tests
+        run: |
+          set -x
+          sudo -E bash -x ./tests/wireguard-client.sh
+          sudo env "PATH=$PATH" ./tests/ipsec-client.sh
+          sudo ./tests/ssh-tunnel.sh
+
+  local-deploy:
+    runs-on: ubuntu-16.04
+    strategy:
+      matrix:
+        UBUNTU_VERSION: ["18.04", "19.04", "19.10"]
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+
+      - name: Install dependencies
+        run: |
+          set -x
+          sudo add-apt-repository -yu ppa:wireguard/wireguard
+          sudo add-apt-repository -yu ppa:ubuntu-lxc/stable
+          sudo apt update -y
+          sudo apt install -y \
+            python3-pip \
+            lxd \
+            expect-dev \
+            debootstrap \
+            tree \
+            bridge-utils \
+            dnsutils \
+            build-essential \
+            libssl-dev \
+            libffi-dev \
+            python3-dev \
+            linux-headers-$(uname -r) \
+            wireguard \
+            libxml2-utils \
+            crudini \
+            fping \
+            strongswan \
+            libstrongswan-standard-plugins \
+            resolvconf
+
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements.txt
+
+      - name: Provision
+        env:
+          DEPLOY: docker
+          UBUNTU_VERSION: ${{ matrix.UBUNTU_VERSION }}
+        run: |
+          ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''
+          sed -i "s/^reduce_mtu:\s0$/reduce_mtu: 80/" config.cfg
+          sudo -E ./tests/pre-deploy.sh
+
+      - name: Deployment
+        env:
+          DEPLOY: docker
+          UBUNTU_VERSION: ${{ matrix.UBUNTU_VERSION }}
+        run: |
+          docker build -t local/algo .
+          ./tests/local-deploy.sh
+          ./tests/update-users.sh
+
+      - name: Tests
+        run: |
+          set -x
+          sudo bash -x ./tests/wireguard-client.sh
+          sudo env "PATH=$PATH" bash -x ./tests/ipsec-client.sh
+          sudo bash -x ./tests/ssh-tunnel.sh

.travis.yml

@@ -1,129 +0,0 @@
----
-language: python
-python: "3.7"
-dist: xenial
-
-services:
-  - docker
-
-addons:
-  apt:
-    sources: &default_sources
-      - sourceline: 'ppa:ubuntu-lxc/stable'
-      - sourceline: 'ppa:wireguard/wireguard'
-    packages: &default_packages
-      - python3-pip
-      - lxd
-      - expect-dev
-      - debootstrap
-      - tree
-      - bridge-utils
-      - dnsutils
-      - build-essential
-      - libssl-dev
-      - libffi-dev
-      - python3-dev
-      - linux-headers-$(uname -r)
-      - wireguard
-      - libxml2-utils
-      - crudini
-      - fping
-      - strongswan
-      - libstrongswan-standard-plugins
-
-cache:
-  directories:
-    - $HOME/lxc/
-  pip: true
-
-before_cache:
-  - mkdir $HOME/lxc
-  - sudo tar cf $HOME/lxc/cache.tar /var/lib/lxd/images/
-  - sudo chown $USER. $HOME/lxc/cache.tar
-
-custom_scripts:
-  provisioning: &provisioning
-    - ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''
-    - sudo ./tests/pre-deploy.sh
-    - 'sed -i "s/^reduce_mtu:\s0$/reduce_mtu: 20/" config.cfg'
-  tests: &tests
-    - sudo ./tests/wireguard-client.sh
-    - sudo env "PATH=$PATH" ./tests/ipsec-client.sh
-    - sudo ./tests/ssh-tunnel.sh
-
-stages:
-  - &tests-and-linters
-    stage: Tests
-    name: code checks and linters
-    addons:
-      apt:
-        packages:
-          - shellcheck
-    script:
-      - pip install ansible-lint
-      - shellcheck algo install.sh
-      - ansible-playbook main.yml --syntax-check
-      - ansible-lint -v *.yml roles/{local,cloud-*}/*/*.yml
-
-  - &deploy-local
-    stage: Deploy
-    name: local deployment from docker
-    addons:
-      apt:
-        sources: *default_sources
-        packages: *default_packages
-    before_install: *provisioning
-    before_script:
-      - docker build -t travis/algo .
-      - ./tests/local-deploy.sh
-      - ./tests/update-users.sh
-    script: *tests
-
-  - &deploy-cloudinit
-    stage: Deploy
-    name: cloud-init deployment
-    addons:
-      apt:
-        sources: *default_sources
-        packages: *default_packages
-    env: DEPLOY=cloud-init
-    before_install: *provisioning
-    before_script:
-      - until sudo lxc exec algo -- test -f /var/log/cloud-init-output.log; do echo 'Log file not found, Sleep for 3 seconds'; sleep 3; done
-      - ( sudo lxc exec algo -- tail -f /var/log/cloud-init-output.log & )
-      - |
-        until sudo lxc exec algo -- test -f /var/lib/cloud/data/result.json; do
-          echo 'Cloud init is not finished. Sleep for 30 seconds';
-          sleep 30;
-        done
-      - sudo lxc exec algo -- test -f /opt/algo/configs/localhost/.config.yml
-      - sudo lxc exec algo -- tar zcf /root/algo-configs.tar -C /opt/algo/configs/ .
-      - sudo lxc file pull algo/root/algo-configs.tar ./
-      - sudo tar -C ./configs -zxf algo-configs.tar
-    script: *tests
-
-matrix:
-  fast_finish: true
-  include:
-    - <<: *tests-and-linters
-    - <<: *deploy-local
-      name: 'Ubuntu 18.04: local deployment from docker'
-      env: DEPLOY=docker UBUNTU_VERSION=18.04
-    - <<: *deploy-local
-      name: 'Ubuntu 19.04: local deployment from docker'
-      env: DEPLOY=docker UBUNTU_VERSION=19.04
-    - <<: *deploy-local
-      name: 'Ubuntu 19.10: local deployment from docker'
-      env: DEPLOY=docker UBUNTU_VERSION=19.10
-    - <<: *deploy-cloudinit
-      name: 'Ubuntu 18.04: cloud-init deployment'
-      env: DEPLOY=cloud-init UBUNTU_VERSION=18.04
-    - <<: *deploy-cloudinit
-      name: 'Ubuntu 19.04: cloud-init deployment'
-      env: DEPLOY=cloud-init UBUNTU_VERSION=19.04
-    - <<: *deploy-cloudinit
-      name: 'Ubuntu 19.10: cloud-init deployment'
-      env: DEPLOY=cloud-init UBUNTU_VERSION=19.10
-
-notifications:
-  email: false

server.yml

@@ -87,8 +87,8 @@
               ipsec_enabled: {{ ipsec_enabled }}
               wireguard_enabled: {{ wireguard_enabled }}
               {% if tests|default(false)|bool %}
-              ca_password: {{ CA_password }}
+              ca_password: '{{ CA_password }}'
-              p12_password: {{ p12_export_password }}
+              p12_password: '{{ p12_export_password }}'
               {% endif %}
           become: false
           delegate_to: localhost

tests/cloud-init.sh

@@ -9,9 +9,9 @@ export DNS_ADBLOCKING=true
 export SSH_TUNNELING=true
 export ENDPOINT=10.0.8.100
 export USERS=desktop,user1,user2
-export EXTRA_VARS='install_headers=false tests=true apparmor_enabled=false local_service_ip=172.16.0.1'
+export EXTRA_VARS='install_headers=false tests=true local_service_ip=172.16.0.1'
-export ANSIBLE_EXTRA_ARGS='--skip-tags apparmor'
+export ANSIBLE_EXTRA_ARGS=''
-export REPO_SLUG=${TRAVIS_PULL_REQUEST_SLUG:-${TRAVIS_REPO_SLUG:-trailofbits/algo}}
+export REPO_SLUG=${REPOSITORY:-trailofbits/algo}
-export REPO_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH:-master}}
+export REPO_BRANCH=${BRANCH:-master}
 
-curl -s https://raw.githubusercontent.com/${TRAVIS_PULL_REQUEST_SLUG:-${TRAVIS_REPO_SLUG}}/${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}/install.sh | sudo -E bash -x"
+curl -s https://raw.githubusercontent.com/${REPOSITORY:-trailofbits/algo}/${BRANCH:-master}/install.sh | sudo -E bash -x"

tests/local-deploy.sh

@@ -2,11 +2,11 @@
 
 set -ex
 
-DEPLOY_ARGS="provider=local server=10.0.8.100 ssh_user=ubuntu endpoint=10.0.8.100 apparmor_enabled=false ondemand_cellular=true ondemand_wifi=true ondemand_wifi_exclude=test dns_adblocking=true ssh_tunneling=true store_pki=true install_headers=false tests=true local_service_ip=172.16.0.1"
+DEPLOY_ARGS="provider=local server=10.0.8.100 ssh_user=ubuntu endpoint=10.0.8.100 ondemand_cellular=true ondemand_wifi=true ondemand_wifi_exclude=test dns_adblocking=true ssh_tunneling=true store_pki=true install_headers=false tests=true local_service_ip=172.16.0.1"
 
 if [ "${DEPLOY}" == "docker" ]
 then
-  docker run -it -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "DEPLOY_ARGS=${DEPLOY_ARGS}" travis/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook main.yml -e \"${DEPLOY_ARGS}\" --skip-tags apparmor"
+  docker run -i -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "DEPLOY_ARGS=${DEPLOY_ARGS}" local/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook main.yml -e \"${DEPLOY_ARGS}\" --skip-tags debug"
 else
-  ansible-playbook main.yml -e "${DEPLOY_ARGS}" --skip-tags apparmor
+  ansible-playbook main.yml -e "${DEPLOY_ARGS}"
 fi

tests/pre-deploy.sh

@@ -8,6 +8,9 @@ tar xf $HOME/lxc/cache.tar -C / || echo "Didn't extract cache."
 cp -f tests/lxd-bridge /etc/default/lxd-bridge
 cp -f tests/algo.conf /etc/default/algo.conf
 
+export REPOSITORY=${GITHUB_REPOSITORY}
+export BRANCH=${GITHUB_REF#refs/heads/}
+
 if [[ "$DEPLOY" == "cloud-init" ]]; then
   bash tests/cloud-init.sh | lxc profile set default user.user-data -
 else

tests/ssh-tunnel.sh

@@ -2,7 +2,7 @@
 
 set -euxo pipefail
 
-PASS=$(grep ^p12_password: configs/10.0.8.100/.config.yml | awk '{print $2}')
+PASS=$(grep ^p12_password: configs/10.0.8.100/.config.yml | awk '{print $2}' | cut -f2 -d\')
 
 ssh-keygen -p -P ${PASS} -N '' -f configs/10.0.8.100/ssh-tunnel/desktop.pem
 
@@ -10,6 +10,6 @@ ssh -o StrictHostKeyChecking=no -D 127.0.0.1:1080 -f -q -C -N desktop@10.0.8.100
 
 git config --global http.proxy 'socks5://127.0.0.1:1080'
 
-git clone -vv https://github.com/trailofbits/algo /tmp/ssh-tunnel-check
+for i in {1..10}; do git clone -vv https://github.com/trailofbits/algo /tmp/ssh-tunnel-check && break || sleep 1; done
 
 echo "SSH tunneling tests passed"

tests/update-users.sh

@@ -6,7 +6,7 @@ USER_ARGS="{ 'server': '10.0.8.100', 'users': ['desktop', 'user1', 'user2'], 'lo
 
 if [ "${DEPLOY}" == "docker" ]
 then
-  docker run -it -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "USER_ARGS=${USER_ARGS}" travis/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook users.yml -e \"${USER_ARGS}\" -t update-users"
+  docker run -i -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "USER_ARGS=${USER_ARGS}" local/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook users.yml -e \"${USER_ARGS}\" -t update-users --skip-tags debug -vvvvv"
 else
   ansible-playbook users.yml -e "${USER_ARGS}" -t update-users
 fi
@@ -23,7 +23,7 @@ if sudo openssl crl -inform pem -noout -text -in configs/10.0.8.100/ipsec/.pki/c
     exit 1
 fi
 
-if sudo openssl x509 -inform pem -noout -text -in configs/10.0.8.100/ipsec/.pki/certs/user1.crt | grep CN=user1
+if sudo openssl x509 -inform pem -noout -text -in configs/10.0.8.100/ipsec/.pki/certs/user1.crt | grep -E "CN(=|\s+=\s+)user1"
   then
     echo "The new user exists"
   else