Merge branch 'trailofbits:master' into issue/14280

This commit is contained in:
Glenn Schlereth 2021-10-31 15:55:43 -03:00 committed by GitHub
commit 7d549825f4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 141 additions and 275 deletions

View file

@ -4,7 +4,7 @@ on: [push, pull_request]
jobs:
lint:
runs-on: ubuntu-18.04
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2.2.2
@ -26,12 +26,12 @@ jobs:
ansible-lint -x experimental,package-latest,unnamed-task -v *.yml roles/{local,cloud-*}/*/*.yml
scripted-deploy:
runs-on: ubuntu-16.04
runs-on: ubuntu-20.04
strategy:
matrix:
UBUNTU_VERSION: ["18.04", "20.04"]
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- uses: actions/setup-python@v2.2.2
with:
python-version: '3.7'
@ -40,29 +40,20 @@ jobs:
run: |
sudo apt update -y
sudo apt install -y \
python3-pip \
lxd \
expect-dev \
debootstrap \
tree \
bridge-utils \
dnsutils \
build-essential \
libssl-dev \
libffi-dev \
python3-dev \
linux-headers-$(uname -r) \
wireguard \
libxml2-utils \
crudini \
fping \
strongswan \
libstrongswan-standard-plugins \
resolvconf
openresolv
python3 -m pip install --upgrade pip
python3 -m pip install -r requirements.txt
sudo snap refresh lxd
sudo lxd init --auto
- name: Provision
env:
DEPLOY: cloud-init
@ -93,13 +84,13 @@ jobs:
sudo -E bash -x ./tests/wireguard-client.sh
sudo env "PATH=$PATH" ./tests/ipsec-client.sh
local-deploy:
runs-on: ubuntu-16.04
docker-deploy:
runs-on: ubuntu-20.04
strategy:
matrix:
UBUNTU_VERSION: ["18.04", "20.04"]
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- uses: actions/setup-python@v2.2.2
with:
python-version: '3.7'
@ -107,32 +98,22 @@ jobs:
- name: Install dependencies
run: |
set -x
sudo add-apt-repository -yu ppa:ubuntu-lxc/stable
sudo apt update -y
sudo apt install -y \
python3-pip \
lxd \
expect-dev \
debootstrap \
tree \
bridge-utils \
dnsutils \
build-essential \
libssl-dev \
libffi-dev \
python3-dev \
linux-headers-$(uname -r) \
wireguard \
libxml2-utils \
crudini \
fping \
strongswan \
libstrongswan-standard-plugins \
resolvconf
openresolv
python3 -m pip install --upgrade pip
python3 -m pip install -r requirements.txt
sudo snap refresh lxd
sudo lxd init --auto
- name: Provision
env:
DEPLOY: docker

1
.gitignore vendored
View file

@ -7,3 +7,4 @@ inventory_users
.DS_Store
venvs/*
!venvs/.gitinit
.vagrant

36
Vagrantfile vendored Normal file
View file

@ -0,0 +1,36 @@
Vagrant.configure("2") do |config|
config.vm.box = "bento/ubuntu-20.04"
config.vm.provider "virtualbox" do |v|
v.name = "algo-20.04"
v.memory = "512"
v.cpus = "1"
end
config.vm.synced_folder "./", "/opt/algo", create: true
config.vm.provision "ansible_local" do |ansible|
ansible.playbook = "/opt/algo/main.yml"
# https://github.com/hashicorp/vagrant/issues/12204
ansible.pip_install_cmd = "sudo apt-get install -y python3-pip python-is-python3 && sudo ln -s -f /usr/bin/pip3 /usr/bin/pip"
ansible.install_mode = "pip_args_only"
ansible.pip_args = "-r /opt/algo/requirements.txt"
ansible.inventory_path = "/opt/algo/inventory"
ansible.limit = "local"
ansible.verbose = "-vvvv"
ansible.extra_vars = {
provider: "local",
server: "localhost",
ssh_user: "",
endpoint: "127.0.0.1",
ondemand_cellular: true,
ondemand_wifi: false,
dns_adblocking: true,
ssh_tunneling: true,
store_pki: true,
tests: true,
no_log: false
}
end
end

View file

@ -88,9 +88,15 @@ dnscrypt_servers:
ipv4:
- cloudflare
# - google
# - <YourCustomServer> # E.g., if using NextDNS, this will be something like NextDNS-abc123.
# You must also fill in custom_server_stamps below. You may specify
# multiple custom servers.
ipv6:
- cloudflare-ipv6
custom_server_stamps:
# YourCustomServer: 'sdns://...'
# DNS servers which will be used if 'dns_encryption' is 'false'.
# Fallback resolvers for systemd-resolved
# The default is to use Cloudflare.
@ -198,7 +204,7 @@ cloud_providers:
image: ubuntu-20.04
openstack:
flavor_ram: ">=512"
image: Ubuntu-18.04
image: Ubuntu-20.04
cloudstack:
size: Micro
image: Linux Ubuntu 20.04 LTS 64-bit

View file

@ -1,20 +1,11 @@
### Configuration file
You need to create a configuration file in INI format with your api key in `$HOME/.cloudstack.ini`
Algo scripts will ask you for the API detail. You need to fetch the API credentials and the endpoint from the provider cocntrol panel.
```
[cloudstack]
endpoint = <endpoint>
key = <your api key>
secret = <your secret>
timeout = 30
```
Example for Exoscale (European cloud provider exposing CloudStack API), visit https://portal.exoscale.com/u/<your@account>/account/profile/api to gather the required information: CloudStack api key and secret.
Example for Exoscale (European cloud provider exposing CloudStack API), visit https://portal.exoscale.com/u/<your@account>/account/profile/api to gather the required information:
```
[exoscale]
endpoint = https://api.exoscale.com/compute
key = <your api key>
secret = <your secret>
timeout = 30
```bash
export CLOUDSTACK_KEY="<your api key>"
export CLOUDSTACK_SECRET="<your secret>"
export CLOUDSTACK_ENDPOINT="https://api.exoscale.com/compute"
```

View file

@ -22,16 +22,7 @@ installRequirements() {
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install \
software-properties-common \
git \
build-essential \
libssl-dev \
libffi-dev \
python3-dev \
python3-pip \
python3-setuptools \
python3-virtualenv \
bind9-host \
jq -y
}

View file

@ -1,110 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
ANSIBLE_METADATA = {'metadata_version': '1.1',
'status': ['preview'],
'supported_by': 'community'}
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.cloudstack import (
AnsibleCloudStack,
cs_argument_spec,
cs_required_together,
)
DOCUMENTATION = '''
---
module: cloudstack_zones
short_description: List zones on Apache CloudStack based clouds.
description:
- List zones.
version_added: '0.1'
author: Julien Bachmann (@0xmilkmix)
extends_documentation_fragment: cloudstack
'''
EXAMPLES = '''
- name: List zones
cloudstack_zones:
register: _cs_zones
'''
RETURN = '''
---
zone:
description: List of zones.
returned: success
type: list
sample:
[
{
"allocationstate": "Enabled",
"dhcpprovider": "VirtualRouter",
"id": "<id>",
"localstorageenabled": true,
"name": "ch-gva-2",
"networktype": "Basic",
"securitygroupsenabled": true,
"tags": [],
"zonetoken": "token"
},
{
"allocationstate": "Enabled",
"dhcpprovider": "VirtualRouter",
"id": "<id>",
"localstorageenabled": true,
"name": "ch-dk-2",
"networktype": "Basic",
"securitygroupsenabled": true,
"tags": [],
"zonetoken": "token"
},
{
"allocationstate": "Enabled",
"dhcpprovider": "VirtualRouter",
"id": "<id>",
"localstorageenabled": true,
"name": "at-vie-1",
"networktype": "Basic",
"securitygroupsenabled": true,
"tags": [],
"zonetoken": "token"
},
{
"allocationstate": "Enabled",
"dhcpprovider": "VirtualRouter",
"id": "<id>",
"localstorageenabled": true,
"name": "de-fra-1",
"networktype": "Basic",
"securitygroupsenabled": true,
"tags": [],
"zonetoken": "token"
}
]
'''
class AnsibleCloudStackZones(AnsibleCloudStack):
def __init__(self, module):
super(AnsibleCloudStackZones, self).__init__(module)
self.zones = None
def get_zones(self):
args = {}
if not self.zones:
zones = self.query_api('listZones', **args)
if zones:
self.zones = zones
return self.zones
def main():
module = AnsibleModule(argument_spec={})
acs_zones = AnsibleCloudStackZones(module)
result = acs_zones.get_zones()
module.exit_json(**result)
if __name__ == '__main__':
main()

View file

@ -9,7 +9,7 @@
- name: Ensure Ansible is not being run in a world writable directory
assert:
that: _playbook_dir.stat.mode|int <= 0775
that: _playbook_dir.stat.mode|int <= 775
msg: >
Ansible is being run in a world writable directory ({{ playbook_dir }}), ignoring it as an ansible.cfg source.
For more information see https://docs.ansible.com/ansible/devel/reference_appendices/config.html#cfg-in-world-writable-dir
@ -24,14 +24,14 @@
- name: Set required ansible version as a fact
set_fact:
required_ansible_version:
"{{ item | regex_replace('^ansible[\\s+]?(?P<op>[=,>,<]+)[\\s+]?(?P<ver>\\d.\\d+(.\\d+)?)$',
"{{ item | regex_replace('^ansible-core[\\s+]?(?P<op>[=,>,<]+)[\\s+]?(?P<ver>\\d.\\d+(.\\d+)?)$',
'{\"op\": \"\\g<op>\",\"ver\": \"\\g<ver>\" }') }}"
when: '"ansible" in item'
when: '"ansible-core" in item'
with_items: "{{ lookup('file', 'requirements.txt').splitlines() }}"
- name: Verify Python meets Algo VPN requirements
assert:
that: (ansible_python.version.major|string + '.' + ansible_python.version.minor|string)|float is version('3.6', '>=')
that: (ansible_python.version.major|string + '.' + ansible_python.version.minor|string) is version('3.6', '>=')
msg: >
Python version is not supported.
You must upgrade to at least Python 3.6 to use this version of Algo.

View file

@ -13,7 +13,7 @@
'algo_ssh_tunneling "{{ algo_ssh_tunneling }}"' \
'wireguard_enabled "{{ wireguard_enabled }}"' \
'dns_encryption "{{ dns_encryption }}"' \
> /dev/tty
> /dev/tty || true
tags: debug
- name: Install the requirements

View file

@ -1,3 +1,4 @@
ansible==2.9.20
jinja2==2.11.3
ansible-core==2.11.3
ansible==4.4.0
jinja2~=3.0.1
netaddr

View file

@ -1,3 +1,3 @@
---
- name: restart strongswan
service: name=strongswan state=restarted
service: name={{ strongswan_service }} state=restarted

View file

@ -1,45 +1,6 @@
---
- name: Install requirements
pip:
name:
- packaging
- requests[security]
- xmltodict
- azure-cli-core==2.16.0
- azure-cli-nspkg==3.0.2
- azure-common==1.1.11
- azure-mgmt-authorization==0.51.1
- azure-mgmt-batch==5.0.1
- azure-mgmt-cdn==3.0.0
- azure-mgmt-compute==4.4.0
- azure-mgmt-containerinstance==1.4.0
- azure-mgmt-containerregistry==2.0.0
- azure-mgmt-containerservice==4.4.0
- azure-mgmt-dns==2.1.0
- azure-mgmt-keyvault==1.1.0
- azure-mgmt-marketplaceordering==0.1.0
- azure-mgmt-monitor==0.5.2
- azure-mgmt-network==2.3.0
- azure-mgmt-nspkg==2.0.0
- azure-mgmt-redis==5.0.0
- azure-mgmt-resource==2.1.0
- azure-mgmt-rdbms==1.4.1
- azure-mgmt-servicebus==0.5.3
- azure-mgmt-sql==0.10.0
- azure-mgmt-storage==3.1.0
- azure-mgmt-trafficmanager==0.50.0
- azure-mgmt-web==0.41.0
- azure-nspkg==2.0.0
- azure-storage==0.35.1
- msrest==0.6.1
- msrestazure==0.6.4
- azure-keyvault==1.0.0a1
- azure-graphrbac==0.40.0
- azure-mgmt-cosmosdb==0.5.2
- azure-mgmt-hdinsight==0.1.0
- azure-mgmt-devtestlabs==3.0.0
- azure-mgmt-loganalytics==0.2.0
- azure-mgmt-automation==0.1.1
- azure-mgmt-iothub==0.7.0
requirements: https://raw.githubusercontent.com/ansible-collections/azure/v1.9.0/requirements-azure.txt
state: latest
virtualenv_python: python3

View file

@ -54,5 +54,6 @@
ansible_ssh_port: "{{ ssh_port }}"
cloudinit: true
environment:
CLOUDSTACK_CONFIG: "{{ algo_cs_config }}"
CLOUDSTACK_REGION: "{{ algo_cs_region }}"
CLOUDSTACK_KEY: "{{ algo_cs_key }}"
CLOUDSTACK_SECRET: "{{ algo_cs_token }}"
CLOUDSTACK_ENDPOINT: "{{ algo_cs_url }}"

View file

@ -2,36 +2,47 @@
- block:
- pause:
prompt: |
Enter path for cloudstack.ini file (https://trailofbits.github.io/algo/cloud-cloudstack.html)
[~/.cloudstack.ini]
register: _cs_config
Enter the API key (https://trailofbits.github.io/algo/cloud-cloudstack.html):
echo: false
register: _cs_key
when:
- cs_config is undefined
- lookup('env', 'CLOUDSTACK_CONFIG') | length <= 0
- cs_key is undefined
- lookup('env','CLOUDSTACK_KEY')|length <= 0
- pause:
prompt: |
Specify region to use in cloudstack.ini file
[exoscale]
register: _cs_region
Enter the API ssecret (https://trailofbits.github.io/algo/cloud-cloudstack.html):
echo: false
register: _cs_secret
when:
- cs_region is undefined
- lookup('env', 'CLOUDSTACK_REGION') | length <= 0
- cs_secret is undefined
- lookup('env','CLOUDSTACK_SECRET')|length <= 0
- pause:
prompt: |
Enter the API endpoint (https://trailofbits.github.io/algo/cloud-cloudstack.html)
[https://api.exoscale.com/compute]
register: _cs_url
when:
- cs_url is undefined
- lookup('env', 'CLOUDSTACK_ENDPOINT') | length <= 0
- set_fact:
algo_cs_config: "{{ cs_config | default(_cs_config.user_input|default(None)) | default(lookup('env', 'CLOUDSTACK_CONFIG'), true) | default('~/.cloudstack.ini', true) }}"
algo_cs_region: "{{ cs_region | default(_cs_region.user_input|default(None)) | default(lookup('env', 'CLOUDSTACK_REGION'), true) | default('exoscale', true) }}"
algo_cs_key: "{{ cs_key | default(_cs_key.user_input|default(None)) | default(lookup('env', 'CLOUDSTACK_KEY'), true) }}"
algo_cs_token: "{{ cs_secret | default(_cs_secret.user_input|default(None)) | default(lookup('env', 'CLOUDSTACK_SECRET'), true) }}"
algo_cs_url: "{{ cs_url | default(_cs_url.user_input|default(None)) | default(lookup('env', 'CLOUDSTACK_ENDPOINT'), true) | default('https://api.exoscale.com/compute', true) }}"
- name: Get zones on cloud
cloudstack_zones:
cs_zone_info:
register: _cs_zones
environment:
CLOUDSTACK_CONFIG: "{{ algo_cs_config }}"
CLOUDSTACK_REGION: "{{ algo_cs_region }}"
CLOUDSTACK_KEY: "{{ algo_cs_key }}"
CLOUDSTACK_SECRET: "{{ algo_cs_token }}"
CLOUDSTACK_ENDPOINT: "{{ algo_cs_url }}"
- name: Extract zones from output
set_fact:
cs_zones: "{{ _cs_zones['zone'] | sort(attribute='name') }}"
cs_zones: "{{ _cs_zones['zones'] | sort(attribute='name') }}"
- name: Set the default zone
set_fact:

View file

@ -6,7 +6,7 @@
import_tasks: prompts.yml
- name: Locate official AMI for region
ec2_ami_facts:
ec2_ami_info:
aws_access_key: "{{ access_key }}"
aws_secret_key: "{{ secret_key }}"
owners: "{{ cloud_providers.ec2.image.owner }}"

View file

@ -24,7 +24,7 @@
- block:
- name: Get regions
aws_region_facts:
aws_region_info:
aws_access_key: "{{ access_key }}"
aws_secret_key: "{{ secret_key }}"
region: us-east-1
@ -64,7 +64,7 @@
- block:
- name: Get existing available Elastic IPs
ec2_eip_facts:
ec2_eip_info:
aws_access_key: "{{ access_key }}"
aws_secret_key: "{{ secret_key }}"
region: "{{ algo_region }}"

View file

@ -7,14 +7,14 @@
import_tasks: venv.yml
- name: Security group created
os_security_group:
openstack.cloud.security_group:
state: "{{ state|default('present') }}"
name: "{{ algo_server_name }}-security_group"
description: AlgoVPN security group
register: os_security_group
- name: Security rules created
os_security_group_rule:
openstack.cloud.security_group_rule:
state: "{{ state|default('present') }}"
security_group: "{{ os_security_group.id }}"
protocol: "{{ item.proto }}"
@ -29,22 +29,25 @@
- { proto: udp, port_min: "{{ wireguard_port }}", port_max: "{{ wireguard_port }}", range: 0.0.0.0/0 }
- name: Gather facts about flavors
os_flavor_facts:
openstack.cloud.compute_flavor_info:
ram: "{{ cloud_providers.openstack.flavor_ram }}"
register: os_flavor
- name: Gather facts about images
os_image_facts:
openstack.cloud.image_info:
register: os_image
- name: Set image as a fact
set_fact:
image_id: "{{ item.id }}"
loop: "{{ openstack_image }}"
loop: "{{ os_image.openstack_image }}"
when:
- item.name == cloud_providers.openstack.image
- item.status == "active"
- name: Gather facts about public networks
os_networks_facts:
openstack.cloud.networks_info:
register: os_network
- name: Set the network as a fact
set_fact:
@ -53,15 +56,15 @@
- item['router:external']|default(omit)
- item['admin_state_up']|default(omit)
- item['status'] == 'ACTIVE'
with_items: "{{ openstack_networks }}"
with_items: "{{ os_network.openstack_networks }}"
- name: Set facts
set_fact:
flavor_id: "{{ (openstack_flavors | sort(attribute='ram'))[0]['id'] }}"
flavor_id: "{{ (os_flavor.openstack_flavors | sort(attribute='ram'))[0]['id'] }}"
security_group_name: "{{ os_security_group['secgroup']['name'] }}"
- name: Server created
os_server:
openstack.cloud.server:
state: "{{ state|default('present') }}"
name: "{{ algo_server_name }}"
image: "{{ image_id }}"

View file

@ -3,17 +3,19 @@
- block:
- name: Gather Scaleway organizations facts
scaleway_organization_facts:
scaleway_organization_info:
register: scaleway_org
- name: Get images
scaleway_image_facts:
scaleway_image_info:
region: "{{ algo_region }}"
register: scaleway_image
- name: Set cloud specific facts
set_fact:
organization_id: "{{ scaleway_organization_facts[0]['id'] }}"
organization_id: "{{ scaleway_org.scaleway_organization_info[0]['id'] }}"
images: >-
[{% for i in scaleway_image_facts -%}
[{% for i in scaleway_image.scaleway_image_info -%}
{% if i.name == cloud_providers.scaleway.image and
i.arch == cloud_providers.scaleway.arch -%}
'{{ i.id }}'{% if not loop.last %},{% endif %}

View file

@ -6,6 +6,9 @@
tags:
- update-users
- fail:
when: cloud_test|default(false)|bool
- include_tasks: ubuntu.yml
when: '"Ubuntu" in OS.stdout or "Linux" in OS.stdout'
tags:

View file

@ -119,7 +119,7 @@
- linux-headers-generic
- "linux-headers-{{ ansible_kernel }}"
state: present
when: install_headers
when: install_headers | bool
- name: Configure the alternative ingress ip
include_tasks: aip/main.yml

View file

@ -551,5 +551,10 @@ cache_neg_max_ttl = 600
[static]
{% if custom_server_stamps %}{% for name, stamp in custom_server_stamps.items() %}
[static.'{{ name }}']
stamp = '{{ stamp }}'
{%- endfor %}{% endif %}
# [static.'myserver']
# stamp = 'sdns:AQcAAAAAAAAAAAAQMi5kbnNjcnlwdC1jZXJ0Lg'

View file

@ -1 +0,0 @@
dhcp-host=algo,10.0.8.100

View file

@ -1,16 +0,0 @@
USE_LXD_BRIDGE="true"
LXD_BRIDGE="lxdbr0"
UPDATE_PROFILE="true"
LXD_CONFILE="/etc/default/algo.conf"
LXD_DOMAIN="lxd"
LXD_IPV4_ADDR="10.0.8.1"
LXD_IPV4_NETMASK="255.255.255.0"
LXD_IPV4_NETWORK="10.0.8.0/24"
LXD_IPV4_DHCP_RANGE="10.0.8.2,10.0.8.254"
LXD_IPV4_DHCP_MAX="250"
LXD_IPV4_NAT="true"
LXD_IPV6_ADDR=""
LXD_IPV6_MASK=""
LXD_IPV6_NETWORK=""
LXD_IPV6_NAT="false"
LXD_IPV6_PROXY="false"

View file

@ -4,10 +4,6 @@ set -euxo pipefail
sysctl net.ipv6.conf.all.disable_ipv6=0
tar xf $HOME/lxc/cache.tar -C / || echo "Didn't extract cache."
cp -f tests/lxd-bridge /etc/default/lxd-bridge
cp -f tests/algo.conf /etc/default/algo.conf
export REPOSITORY=${REPOSITORY:-${GITHUB_REPOSITORY}}
export _BRANCH=${BRANCH#refs/heads/}
export BRANCH=${_BRANCH:-${GITHUB_REF#refs/heads/}}
@ -18,12 +14,16 @@ else
echo -e "#cloud-config\nssh_authorized_keys:\n - $(cat ~/.ssh/id_rsa.pub)" | lxc profile set default user.user-data -
fi
systemctl restart lxd-bridge.service lxd-containers.service lxd.service
lxc network set lxdbr0 ipv4.address 10.0.8.1/24
lxc profile set default raw.lxc lxc.aa_profile=unconfined
lxc profile set default raw.lxc 'lxc.apparmor.profile = unconfined'
lxc profile set default security.privileged true
lxc profile show default
lxc launch ubuntu:${UBUNTU_VERSION} algo
lxc init ubuntu:${UBUNTU_VERSION} algo
lxc network attach lxdbr0 algo eth0 eth0
lxc config device set algo eth0 ipv4.address 10.0.8.100
lxc start algo
if [[ ${UBUNTU_VERSION} == "20.04" ]]; then
lxc exec algo -- apt remove snapd --purge -y || true