on-build python venvs (#1199)

.gitignore

@@ -5,3 +5,5 @@ inventory_users
 *.kate-swp
 env
 .DS_Store
+venvs/*
+!venvs/.gitinit

Dockerfile

@@ -26,7 +26,7 @@ COPY . .
 RUN chmod 0755 /algo/algo-docker.sh
 
 # Because of the bind mounting of `configs/`, we need to run as the `root` user
-# This may break in cases where user namespacing is enabled, so hopefully Docker 
+# This may break in cases where user namespacing is enabled, so hopefully Docker
 # sorts out a way to set permissions on bind-mounted volumes (`docker run -v`)
 # before userns becomes default
 # Note that not running as root will break if we don't have a matching userid

README.md

@@ -58,7 +58,7 @@ The easiest way to get an Algo server running is to let it set up a _new_ virtua
     ```bash
     $ python -m virtualenv --python=`which python2` env &&
         source env/bin/activate &&
-        python -m pip install -U pip &&
+        python -m pip install -U pip virtualenv &&
         python -m pip install -r requirements.txt
     ```
     On macOS, you may be prompted to install `cc`. You should press accept if so.

config.cfg

@@ -13,6 +13,9 @@ users:
 # If True re-init all existing certificates. Boolean
 keys_clean_all: False
 
+# Clean up cloud python environments
+clean_environment: false
+
 vpn_network: 10.19.48.0/24
 vpn_network_ipv6: 'fd9d:bc11:4020::/48'
 wireguard_enabled: true

playbooks/cloud-pre.yml

@@ -14,6 +14,16 @@
         'dns_encryption "{{ dns_encryption }}"' \
         > /dev/tty
 
+- name: Install the requirements
+  local_action:
+    module: pip
+    state: latest
+    name:
+      - pyOpenSSL
+      - jinja2==2.8
+      - segno
+  tags: always
+
 - name: Generate the SSH private key
   openssl_privatekey:
     path: "{{ SSH_keys.private }}"

requirements.txt

@@ -1,13 +1 @@
-setuptools>=11.3
+ansible==2.5.2
-SecretStorage < 3
-ansible[azure]==2.5.2
-dopy==0.3.5
-boto>=2.5
-boto3
-apache-libcloud
-six
-pyopenssl
-jinja2==2.8
-shade
-pycrypto
-segno

roles/cloud-azure/defaults/main.yml

@@ -1,4 +1,5 @@
 ---
+azure_venv: "{{ playbook_dir }}/configs/.venvs/azure"
 _azure_regions: >
   [
     {

roles/cloud-azure/tasks/main.yml

@@ -1,43 +1,48 @@
 ---
 - block:
-    - name: Include prompts
+    - name: Build python virtual environment
-      import_tasks: prompts.yml
+      import_tasks: venv.yml
 
-    - set_fact:
+    - block:
-        algo_region: >-
+      - name: Include prompts
-          {% if region is defined %}{{ region }}
+        import_tasks: prompts.yml
-          {%- elif _algo_region.user_input is defined and _algo_region.user_input != "" %}{{ azure_regions[_algo_region.user_input | int -1 ]['name'] }}
-          {%- else %}{{ azure_regions[default_region | int - 1]['name'] }}{% endif %}
 
-    - name: Create AlgoVPN Server
+      - set_fact:
-      azure_rm_deployment:
+          algo_region: >-
-        state: present
+            {% if region is defined %}{{ region }}
-        deployment_name: "AlgoVPN-{{ algo_server_name }}"
+            {%- elif _algo_region.user_input is defined and _algo_region.user_input != "" %}{{ azure_regions[_algo_region.user_input | int -1 ]['name'] }}
-        template: "{{ lookup('file', 'deployment.json') }}"
+            {%- else %}{{ azure_regions[default_region | int - 1]['name'] }}{% endif %}
-        secret: "{{ secret }}"
-        tenant: "{{ tenant }}"
-        client_id: "{{ client_id }}"
-        subscription_id: "{{ subscription_id }}"
-        resource_group_name: "AlgoVPN-{{ algo_server_name }}"
-        parameters:
-          AlgoServerName:
-            value: "{{ algo_server_name }}"
-          sshKeyData:
-            value: "{{ lookup('file', '{{ SSH_keys.public }}') }}"
-          location:
-            value: "{{ algo_region }}"
-          WireGuardPort:
-            value: "{{ wireguard_port }}"
-          vmSize:
-            value: "{{ cloud_providers.azure.size }}"
-          imageReferenceSku:
-            value: "{{ cloud_providers.azure.image }}"
-      register: azure_rm_deployment
 
-    - set_fact:
+      - name: Create AlgoVPN Server
-        cloud_instance_ip: "{{ azure_rm_deployment.deployment.outputs.publicIPAddresses.value }}"
+        azure_rm_deployment:
-        ansible_ssh_user: ubuntu
+          state: present
+          deployment_name: "AlgoVPN-{{ algo_server_name }}"
+          template: "{{ lookup('file', 'deployment.json') }}"
+          secret: "{{ secret }}"
+          tenant: "{{ tenant }}"
+          client_id: "{{ client_id }}"
+          subscription_id: "{{ subscription_id }}"
+          resource_group_name: "AlgoVPN-{{ algo_server_name }}"
+          parameters:
+            AlgoServerName:
+              value: "{{ algo_server_name }}"
+            sshKeyData:
+              value: "{{ lookup('file', '{{ SSH_keys.public }}') }}"
+            location:
+              value: "{{ algo_region }}"
+            WireGuardPort:
+              value: "{{ wireguard_port }}"
+            vmSize:
+              value: "{{ cloud_providers.azure.size }}"
+            imageReferenceSku:
+              value: "{{ cloud_providers.azure.image }}"
+        register: azure_rm_deployment
 
+      - set_fact:
+          cloud_instance_ip: "{{ azure_rm_deployment.deployment.outputs.publicIPAddresses.value }}"
+          ansible_ssh_user: ubuntu
+      environment:
+        PYTHONPATH: "{{ azure_venv }}/lib/python2.7/site-packages/"
   rescue:
     - debug: var=fail_hint
       tags: always

roles/cloud-azure/tasks/venv.yml

@@ -0,0 +1,32 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ azure_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name:
+      - packaging
+      - requests[security]
+      - azure-mgmt-compute>=2.0.0,<3
+      - azure-mgmt-network>=1.3.0,<2
+      - azure-mgmt-storage>=1.5.0,<2
+      - azure-mgmt-resource>=1.1.0,<2
+      - azure-storage>=0.35.1,<0.36
+      - azure-cli-core>=2.0.12,<3
+      - msrest==0.4.29
+      - msrestazure==0.4.31
+      - azure-mgmt-dns>=1.0.1,<2
+      - azure-mgmt-keyvault>=0.40.0,<0.41
+      - azure-mgmt-batch>=4.1.0,<5
+      - azure-mgmt-sql>=0.7.1,<0.8
+      - azure-mgmt-web>=0.32.0,<0.33
+      - azure-mgmt-containerservice>=2.0.0,<3.0.0
+      - azure-mgmt-containerregistry>=1.0.1
+      - azure-mgmt-rdbms==1.2.0
+      - azure-mgmt-containerinstance==0.4.0
+    state: latest
+    virtualenv: "{{ azure_venv }}"
+    virtualenv_python: python2.7

roles/cloud-digitalocean/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+digitalocean_venv: "{{ playbook_dir }}/configs/.venvs/digitalocean"

roles/cloud-digitalocean/tasks/main.yml

@@ -1,102 +1,108 @@
 - block:
-    - name: Include prompts
+    - name: Build python virtual environment
-      import_tasks: prompts.yml
+      import_tasks: venv.yml
-
-    - name: Set additional facts
-      set_fact:
-        algo_do_region: >-
-          {% if region is defined %}{{ region }}
-          {%- elif _algo_region.user_input is defined and _algo_region.user_input != "" %}{{ do_regions[_algo_region.user_input | int -1 ]['slug'] }}
-          {%- else %}{{ do_regions[default_region | int - 1]['slug'] }}{% endif %}
-        public_key: "{{ lookup('file', '{{ SSH_keys.public }}') }}"
 
     - block:
-        - name: "Delete the existing Algo SSH keys"
+      - name: Include prompts
-          digital_ocean:
+        import_tasks: prompts.yml
-            state: absent
-            command: ssh
-            api_token: "{{ algo_do_token }}"
-            name: "{{ SSH_keys.comment }}"
-          register: ssh_keys
-          until: ssh_keys.changed != true
-          retries: 10
-          delay: 1
 
-      rescue:
+      - name: Set additional facts
-        - name: Collect the fail error
+        set_fact:
-          digital_ocean:
+          algo_do_region: >-
-            state: absent
+            {% if region is defined %}{{ region }}
-            command: ssh
+            {%- elif _algo_region.user_input is defined and _algo_region.user_input != "" %}{{ do_regions[_algo_region.user_input | int -1 ]['slug'] }}
-            api_token: "{{ algo_do_token }}"
+            {%- else %}{{ do_regions[default_region | int - 1]['slug'] }}{% endif %}
-            name: "{{ SSH_keys.comment }}"
+          public_key: "{{ lookup('file', '{{ SSH_keys.public }}') }}"
-          register: ssh_keys
-          ignore_errors: yes
 
-        - debug: var=ssh_keys
+      - block:
+          - name: "Delete the existing Algo SSH keys"
+            digital_ocean:
+              state: absent
+              command: ssh
+              api_token: "{{ algo_do_token }}"
+              name: "{{ SSH_keys.comment }}"
+            register: ssh_keys
+            until: ssh_keys.changed != true
+            retries: 10
+            delay: 1
 
-        - fail:
+        rescue:
-            msg: "Please, ensure that your API token is not read-only."
+          - name: Collect the fail error
+            digital_ocean:
+              state: absent
+              command: ssh
+              api_token: "{{ algo_do_token }}"
+              name: "{{ SSH_keys.comment }}"
+            register: ssh_keys
+            ignore_errors: yes
 
-    - name: "Upload the SSH key"
+          - debug: var=ssh_keys
-      digital_ocean:
-        state: present
-        command: ssh
-        ssh_pub_key: "{{ public_key }}"
-        api_token: "{{ algo_do_token }}"
-        name: "{{ SSH_keys.comment }}"
-      register: do_ssh_key
 
-    - name: "Creating a droplet..."
+          - fail:
-      digital_ocean:
+              msg: "Please, ensure that your API token is not read-only."
-        state: present
-        command: droplet
-        name: "{{ algo_server_name }}"
-        region_id: "{{ algo_do_region }}"
-        size_id: "{{ cloud_providers.digitalocean.size }}"
-        image_id: "{{ cloud_providers.digitalocean.image }}"
-        ssh_key_ids: "{{ do_ssh_key.ssh_key.id }}"
-        unique_name: yes
-        api_token: "{{ algo_do_token }}"
-        ipv6: yes
-      register: do
 
-    - set_fact:
+      - name: "Upload the SSH key"
-        cloud_instance_ip: "{{ do.droplet.ip_address }}"
+        digital_ocean:
-        ansible_ssh_user: root
+          state: present
+          command: ssh
+          ssh_pub_key: "{{ public_key }}"
+          api_token: "{{ algo_do_token }}"
+          name: "{{ SSH_keys.comment }}"
+        register: do_ssh_key
 
-    - name: Tag the droplet
+      - name: "Creating a droplet..."
-      digital_ocean_tag:
+        digital_ocean:
-        name: "Environment:Algo"
+          state: present
-        resource_id: "{{ do.droplet.id }}"
+          command: droplet
-        api_token: "{{ algo_do_token }}"
+          name: "{{ algo_server_name }}"
-        state: present
+          region_id: "{{ algo_do_region }}"
+          size_id: "{{ cloud_providers.digitalocean.size }}"
+          image_id: "{{ cloud_providers.digitalocean.image }}"
+          ssh_key_ids: "{{ do_ssh_key.ssh_key.id }}"
+          unique_name: yes
+          api_token: "{{ algo_do_token }}"
+          ipv6: yes
+        register: do
 
-    - block:
+      - set_fact:
-        - name: "Delete the new Algo SSH key"
+          cloud_instance_ip: "{{ do.droplet.ip_address }}"
-          digital_ocean:
+          ansible_ssh_user: root
-            state: absent
-            command: ssh
-            api_token: "{{ algo_do_token }}"
-            name: "{{ SSH_keys.comment }}"
-          register: ssh_keys
-          until: ssh_keys.changed != true
-          retries: 10
-          delay: 1
 
-      rescue:
+      - name: Tag the droplet
-        - name: Collect the fail error
+        digital_ocean_tag:
-          digital_ocean:
+          name: "Environment:Algo"
-            state: absent
+          resource_id: "{{ do.droplet.id }}"
-            command: ssh
+          api_token: "{{ algo_do_token }}"
-            api_token: "{{ algo_do_token }}"
+          state: present
-            name: "{{ SSH_keys.comment }}"
-          register: ssh_keys
-          ignore_errors: yes
 
-        - debug: var=ssh_keys
+      - block:
+          - name: "Delete the new Algo SSH key"
+            digital_ocean:
+              state: absent
+              command: ssh
+              api_token: "{{ algo_do_token }}"
+              name: "{{ SSH_keys.comment }}"
+            register: ssh_keys
+            until: ssh_keys.changed != true
+            retries: 10
+            delay: 1
 
-        - fail:
+        rescue:
-            msg: "Please, ensure that your API token is not read-only."
+          - name: Collect the fail error
+            digital_ocean:
+              state: absent
+              command: ssh
+              api_token: "{{ algo_do_token }}"
+              name: "{{ SSH_keys.comment }}"
+            register: ssh_keys
+            ignore_errors: yes
+
+          - debug: var=ssh_keys
+
+          - fail:
+              msg: "Please, ensure that your API token is not read-only."
+      environment:
+        PYTHONPATH: "{{ digitalocean_venv }}/lib/python2.7/site-packages/"
   rescue:
     - debug: var=fail_hint
       tags: always

roles/cloud-digitalocean/tasks/venv.yml

@@ -0,0 +1,13 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ digitalocean_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name: dopy
+    version: 0.3.5
+    virtualenv: "{{ digitalocean_venv }}"
+    virtualenv_python: python2.7

roles/cloud-ec2/defaults/main.yml

@@ -4,3 +4,4 @@ encrypted: "{{ cloud_providers.ec2.encrypted }}"
 ec2_vpc_nets:
   cidr_block: 172.16.0.0/16
   subnet_cidr: 172.16.254.0/23
+ec2_venv: "{{ playbook_dir }}/configs/.venvs/aws"

roles/cloud-ec2/tasks/main.yml

@@ -1,40 +1,46 @@
 - block:
-    - name: Include prompts
+    - name: Build python virtual environment
-      import_tasks: prompts.yml
+      import_tasks: venv.yml
 
-    - set_fact:
+    - block:
-        algo_region: >-
+      - name: Include prompts
-          {% if region is defined %}{{ region }}
+        import_tasks: prompts.yml
-          {%- elif _algo_region.user_input is defined and _algo_region.user_input != "" %}{{ aws_regions[_algo_region.user_input | int -1 ]['region_name'] }}
-          {%- else %}{{ aws_regions[default_region | int - 1]['region_name'] }}{% endif %}
-        stack_name: "{{ algo_server_name | replace('.', '-') }}"
 
-    - name: Locate official AMI for region
+      - set_fact:
-      ec2_ami_facts:
+          algo_region: >-
-        aws_access_key: "{{ access_key }}"
+            {% if region is defined %}{{ region }}
-        aws_secret_key: "{{ secret_key }}"
+            {%- elif _algo_region.user_input is defined and _algo_region.user_input != "" %}{{ aws_regions[_algo_region.user_input | int -1 ]['region_name'] }}
-        owners: "{{ cloud_providers.ec2.image.owner }}"
+            {%- else %}{{ aws_regions[default_region | int - 1]['region_name'] }}{% endif %}
-        region: "{{ algo_region }}"
+          stack_name: "{{ algo_server_name | replace('.', '-') }}"
-        filters:
-          name: "ubuntu/images/hvm-ssd/{{ cloud_providers.ec2.image.name }}-amd64-server-*"
-      register: ami_search
 
-    - import_tasks: encrypt_image.yml
+      - name: Locate official AMI for region
-      when: encrypted
+        ec2_ami_facts:
+          aws_access_key: "{{ access_key }}"
+          aws_secret_key: "{{ secret_key }}"
+          owners: "{{ cloud_providers.ec2.image.owner }}"
+          region: "{{ algo_region }}"
+          filters:
+            name: "ubuntu/images/hvm-ssd/{{ cloud_providers.ec2.image.name }}-amd64-server-*"
+        register: ami_search
 
-    - name: Set the ami id as a fact
+      - import_tasks: encrypt_image.yml
-      set_fact:
+        when: encrypted
-        ami_image: >-
-          {% if ami_search_encrypted.image_id is defined %}{{ ami_search_encrypted.image_id }}
-          {%- elif search_crypt.images is defined and search_crypt.images|length >= 1 %}{{ (search_crypt.images | sort(attribute='creation_date') | last)['image_id'] }}
-          {%- else %}{{ (ami_search.images | sort(attribute='creation_date') | last)['image_id'] }}{% endif %}
 
-    - name: Deploy the stack
+      - name: Set the ami id as a fact
-      import_tasks: cloudformation.yml
+        set_fact:
+          ami_image: >-
+            {% if ami_search_encrypted.image_id is defined %}{{ ami_search_encrypted.image_id }}
+            {%- elif search_crypt.images is defined and search_crypt.images|length >= 1 %}{{ (search_crypt.images | sort(attribute='creation_date') | last)['image_id'] }}
+            {%- else %}{{ (ami_search.images | sort(attribute='creation_date') | last)['image_id'] }}{% endif %}
 
-    - set_fact:
+      - name: Deploy the stack
-        cloud_instance_ip: "{{ stack.stack_outputs.ElasticIP }}"
+        import_tasks: cloudformation.yml
-        ansible_ssh_user: ubuntu
+
+      - set_fact:
+          cloud_instance_ip: "{{ stack.stack_outputs.ElasticIP }}"
+          ansible_ssh_user: ubuntu
+      environment:
+        PYTHONPATH: "{{ ec2_venv }}/lib/python2.7/site-packages/"
   rescue:
     - debug: var=fail_hint
       tags: always

roles/cloud-ec2/tasks/venv.yml

@@ -0,0 +1,15 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ ec2_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name:
+      - boto>=2.5
+      - boto3
+    state: latest
+    virtualenv: "{{ ec2_venv }}"
+    virtualenv_python: python2.7

roles/cloud-gce/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+gce_venv: "{{ playbook_dir }}/configs/.venvs/gce"

roles/cloud-gce/tasks/main.yml

@@ -1,54 +1,60 @@
 - block:
-    - name: Include prompts
+    - name: Build python virtual environment
-      import_tasks: prompts.yml
+      import_tasks: venv.yml
-
-    - name: Network configured
-      gce_net:
-        name: "algo-net-{{ algo_server_name }}"
-        fwname: "algo-net-{{ algo_server_name }}-fw"
-        allowed: "udp:500,4500,{{ wireguard_port }};tcp:22"
-        state: "present"
-        mode: auto
-        src_range: 0.0.0.0/0
-        service_account_email: "{{ service_account_email }}"
-        credentials_file: "{{ credentials_file_path  }}"
-        project_id: "{{ project_id }}"
 
     - block:
-      - name: External IP allocated
+      - name: Include prompts
-        gce_eip:
+        import_tasks: prompts.yml
+
+      - name: Network configured
+        gce_net:
+          name: "algo-net-{{ algo_server_name }}"
+          fwname: "algo-net-{{ algo_server_name }}-fw"
+          allowed: "udp:500,4500,{{ wireguard_port }};tcp:22"
+          state: "present"
+          mode: auto
+          src_range: 0.0.0.0/0
+          service_account_email: "{{ service_account_email }}"
+          credentials_file: "{{ credentials_file_path  }}"
+          project_id: "{{ project_id }}"
+
+      - block:
+        - name: External IP allocated
+          gce_eip:
+            service_account_email: "{{ service_account_email }}"
+            credentials_file: "{{ credentials_file_path }}"
+            project_id: "{{ project_id }}"
+            name: "{{ algo_server_name }}"
+            region: "{{ algo_region.split('-')[0:2] | join('-') }}"
+            state: present
+          register: gce_eip
+
+        - name: Set External IP as a fact
+          set_fact:
+            external_ip: "{{ gce_eip.address }}"
+        when: cloud_providers.gce.external_static_ip
+
+      - name: "Creating a new instance..."
+        gce:
+          instance_names: "{{ algo_server_name }}"
+          zone: "{{ algo_region }}"
+          external_ip: "{{ external_ip | default('ephemeral') }}"
+          machine_type: "{{ cloud_providers.gce.size }}"
+          image: "{{ cloud_providers.gce.image }}"
           service_account_email: "{{ service_account_email }}"
           credentials_file: "{{ credentials_file_path }}"
           project_id: "{{ project_id }}"
-          name: "{{ algo_server_name }}"
+          metadata: '{"ssh-keys":"ubuntu:{{ ssh_public_key_lookup }}"}'
-          region: "{{ algo_region.split('-')[0:2] | join('-') }}"
+          network: "algo-net-{{ algo_server_name }}"
-          state: present
+          tags:
-        register: gce_eip
+            - "environment-algo"
+        register: google_vm
 
-      - name: Set External IP as a fact
+      - set_fact:
-        set_fact:
+          cloud_instance_ip: "{{ google_vm.instance_data[0].public_ip }}"
-          external_ip: "{{ gce_eip.address }}"
+          ansible_ssh_user: ubuntu
-      when: cloud_providers.gce.external_static_ip
+      environment:
-
+        PYTHONPATH: "{{ gce_venv }}/lib/python2.7/site-packages/"
-    - name: "Creating a new instance..."
-      gce:
-        instance_names: "{{ algo_server_name }}"
-        zone: "{{ algo_region }}"
-        external_ip: "{{ external_ip | default('ephemeral') }}"
-        machine_type: "{{ cloud_providers.gce.size }}"
-        image: "{{ cloud_providers.gce.image }}"
-        service_account_email: "{{ service_account_email }}"
-        credentials_file: "{{ credentials_file_path }}"
-        project_id: "{{ project_id }}"
-        metadata: '{"ssh-keys":"ubuntu:{{ ssh_public_key_lookup }}"}'
-        network: "algo-net-{{ algo_server_name }}"
-        tags:
-          - "environment-algo"
-      register: google_vm
-
-    - set_fact:
-        cloud_instance_ip: "{{ google_vm.instance_data[0].public_ip }}"
-        ansible_ssh_user: ubuntu
   rescue:
     - debug: var=fail_hint
       tags: always

roles/cloud-gce/tasks/venv.yml

@@ -0,0 +1,15 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ gce_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name:
+      - apache-libcloud
+      - pycrypto
+    state: latest
+    virtualenv: "{{ gce_venv }}"
+    virtualenv_python: python2.7

roles/cloud-lightsail/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+lightsail_venv: "{{ playbook_dir }}/configs/.venvs/aws"

roles/cloud-lightsail/tasks/main.yml

@@ -1,41 +1,47 @@
 - block:
-  - name: Include prompts
+  - name: Build python virtual environment
-    import_tasks: prompts.yml
+    import_tasks: venv.yml
 
-  - name: Create an instance
+  - block:
-    lightsail:
+    - name: Include prompts
-      aws_access_key: "{{ access_key }}"
+      import_tasks: prompts.yml
-      aws_secret_key: "{{ secret_key }}"
-      name: "{{ algo_server_name }}"
-      state: present
-      region: "{{ algo_region }}"
-      zone: "{{ algo_region }}a"
-      blueprint_id: "{{ cloud_providers.lightsail.image }}"
-      bundle_id: "{{ cloud_providers.lightsail.size }}"
-      wait_timeout: 300
-      open_ports:
-        - from_port: 4500
-          to_port: 4500
-          protocol: udp
-        - from_port: 500
-          to_port: 500
-          protocol: udp
-        - from_port: "{{ wireguard_port }}"
-          to_port: "{{ wireguard_port }}"
-          protocol: udp
-      user_data: |
-        #!/bin/bash
-        mkdir -p /home/ubuntu/.ssh/
-        echo "{{ lookup('file', '{{ SSH_keys.public }}') }}" >> /home/ubuntu/.ssh/authorized_keys
-        chown -R ubuntu: /home/ubuntu/.ssh/
-        chmod 0700 /home/ubuntu/.ssh/
-        chmod 0600 /home/ubuntu/.ssh/*
-        test
-    register: algo_instance
 
-  - set_fact:
+    - name: Create an instance
-      cloud_instance_ip: "{{ algo_instance['instance']['public_ip_address'] }}"
+      lightsail:
-      ansible_ssh_user: ubuntu
+        aws_access_key: "{{ access_key }}"
+        aws_secret_key: "{{ secret_key }}"
+        name: "{{ algo_server_name }}"
+        state: present
+        region: "{{ algo_region }}"
+        zone: "{{ algo_region }}a"
+        blueprint_id: "{{ cloud_providers.lightsail.image }}"
+        bundle_id: "{{ cloud_providers.lightsail.size }}"
+        wait_timeout: 300
+        open_ports:
+          - from_port: 4500
+            to_port: 4500
+            protocol: udp
+          - from_port: 500
+            to_port: 500
+            protocol: udp
+          - from_port: "{{ wireguard_port }}"
+            to_port: "{{ wireguard_port }}"
+            protocol: udp
+        user_data: |
+          #!/bin/bash
+          mkdir -p /home/ubuntu/.ssh/
+          echo "{{ lookup('file', '{{ SSH_keys.public }}') }}" >> /home/ubuntu/.ssh/authorized_keys
+          chown -R ubuntu: /home/ubuntu/.ssh/
+          chmod 0700 /home/ubuntu/.ssh/
+          chmod 0600 /home/ubuntu/.ssh/*
+          test
+      register: algo_instance
+
+    - set_fact:
+        cloud_instance_ip: "{{ algo_instance['instance']['public_ip_address'] }}"
+        ansible_ssh_user: ubuntu
+    environment:
+      PYTHONPATH: "{{ lightsail_venv }}/lib/python2.7/site-packages/"
 
   rescue:
   - debug: var=fail_hint

roles/cloud-lightsail/tasks/venv.yml

@@ -0,0 +1,15 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ lightsail_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name:
+      - boto>=2.5
+      - boto3
+    state: latest
+    virtualenv: "{{ lightsail_venv }}"
+    virtualenv_python: python2.7

roles/cloud-openstack/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+openstack_venv: "{{ playbook_dir }}/configs/.venvs/openstack"

roles/cloud-openstack/tasks/main.yml

@@ -4,77 +4,83 @@
   when: lookup('env', 'OS_AUTH_URL') == ""
 
 - block:
-  - name: Security group created
+  - name: Build python virtual environment
-    os_security_group:
+    import_tasks: venv.yml
-      state: "{{ state|default('present') }}"
-      name: "{{ algo_server_name }}-security_group"
-      description: AlgoVPN security group
-    register: os_security_group
 
-  - name: Security rules created
+  - block:
-    os_security_group_rule:
+    - name: Security group created
-      state: "{{ state|default('present') }}"
+      os_security_group:
-      security_group: "{{ os_security_group.id }}"
+        state: "{{ state|default('present') }}"
-      protocol: "{{ item.proto }}"
+        name: "{{ algo_server_name }}-security_group"
-      port_range_min: "{{ item.port_min }}"
+        description: AlgoVPN security group
-      port_range_max: "{{ item.port_max }}"
+      register: os_security_group
-      remote_ip_prefix: "{{ item.range }}"
-    with_items:
-      - { proto: tcp, port_min: 22, port_max: 22, range: 0.0.0.0/0 }
-      - { proto: icmp, port_min: -1, port_max: -1, range: 0.0.0.0/0 }
-      - { proto: udp, port_min: 4500, port_max: 4500, range: 0.0.0.0/0 }
-      - { proto: udp, port_min: 500, port_max: 500, range: 0.0.0.0/0 }
-      - { proto: udp, port_min: "{{ wireguard_port }}", port_max: "{{ wireguard_port }}", range: 0.0.0.0/0 }
 
-  - name: Keypair created
+    - name: Security rules created
-    os_keypair:
+      os_security_group_rule:
-      state: "{{ state|default('present') }}"
+        state: "{{ state|default('present') }}"
-      name: "{{ SSH_keys.comment|regex_replace('@', '_') }}"
+        security_group: "{{ os_security_group.id }}"
-      public_key_file: "{{ SSH_keys.public }}"
+        protocol: "{{ item.proto }}"
-    register: os_keypair
+        port_range_min: "{{ item.port_min }}"
+        port_range_max: "{{ item.port_max }}"
+        remote_ip_prefix: "{{ item.range }}"
+      with_items:
+        - { proto: tcp, port_min: 22, port_max: 22, range: 0.0.0.0/0 }
+        - { proto: icmp, port_min: -1, port_max: -1, range: 0.0.0.0/0 }
+        - { proto: udp, port_min: 4500, port_max: 4500, range: 0.0.0.0/0 }
+        - { proto: udp, port_min: 500, port_max: 500, range: 0.0.0.0/0 }
+        - { proto: udp, port_min: "{{ wireguard_port }}", port_max: "{{ wireguard_port }}", range: 0.0.0.0/0 }
 
-  - name: Gather facts about flavors
+    - name: Keypair created
-    os_flavor_facts:
+      os_keypair:
-      ram: "{{ cloud_providers.openstack.flavor_ram }}"
+        state: "{{ state|default('present') }}"
+        name: "{{ SSH_keys.comment|regex_replace('@', '_') }}"
+        public_key_file: "{{ SSH_keys.public }}"
+      register: os_keypair
 
-  - name: Gather facts about images
+    - name: Gather facts about flavors
-    os_image_facts:
+      os_flavor_facts:
-      image: "{{ cloud_providers.openstack.image }}"
+        ram: "{{ cloud_providers.openstack.flavor_ram }}"
 
-  - name: Gather facts about public networks
+    - name: Gather facts about images
-    os_networks_facts:
+      os_image_facts:
+        image: "{{ cloud_providers.openstack.image }}"
 
-  - name: Set the network as a fact
+    - name: Gather facts about public networks
-    set_fact:
+      os_networks_facts:
-      public_network_id: "{{ item.id }}"
-    when:
-      - item['router:external']|default(omit)
-      - item['admin_state_up']|default(omit)
-      - item['status'] == 'ACTIVE'
-    with_items: "{{ openstack_networks }}"
 
-  - name: Set facts
+    - name: Set the network as a fact
-    set_fact:
+      set_fact:
-      flavor_id: "{{ (openstack_flavors | sort(attribute='ram'))[0]['id'] }}"
+        public_network_id: "{{ item.id }}"
-      image_id: "{{ openstack_image['id'] }}"
+      when:
-      keypair_name: "{{ os_keypair.key.name }}"
+        - item['router:external']|default(omit)
-      security_group_name: "{{ os_security_group['secgroup']['name'] }}"
+        - item['admin_state_up']|default(omit)
+        - item['status'] == 'ACTIVE'
+      with_items: "{{ openstack_networks }}"
 
-  - name: Server created
+    - name: Set facts
-    os_server:
+      set_fact:
-      state: "{{ state|default('present') }}"
+        flavor_id: "{{ (openstack_flavors | sort(attribute='ram'))[0]['id'] }}"
-      name: "{{ algo_server_name }}"
+        image_id: "{{ openstack_image['id'] }}"
-      image: "{{ image_id }}"
+        keypair_name: "{{ os_keypair.key.name }}"
-      flavor: "{{ flavor_id }}"
+        security_group_name: "{{ os_security_group['secgroup']['name'] }}"
-      key_name: "{{ keypair_name }}"
-      security_groups: "{{ security_group_name }}"
-      nics:
-        - net-id: "{{ public_network_id }}"
-    register: os_server
 
-  - set_fact:
+    - name: Server created
-      cloud_instance_ip: "{{ os_server['openstack']['public_v4'] }}"
+      os_server:
-      ansible_ssh_user: ubuntu
+        state: "{{ state|default('present') }}"
+        name: "{{ algo_server_name }}"
+        image: "{{ image_id }}"
+        flavor: "{{ flavor_id }}"
+        key_name: "{{ keypair_name }}"
+        security_groups: "{{ security_group_name }}"
+        nics:
+          - net-id: "{{ public_network_id }}"
+      register: os_server
+
+    - set_fact:
+        cloud_instance_ip: "{{ os_server['openstack']['public_v4'] }}"
+        ansible_ssh_user: ubuntu
+    environment:
+      PYTHONPATH: "{{ openstack_venv }}/lib/python2.7/site-packages/"
 
   rescue:
   - debug: var=fail_hint

roles/cloud-openstack/tasks/venv.yml

@@ -0,0 +1,13 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ openstack_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name: shade
+    state: latest
+    virtualenv: "{{ openstack_venv }}"
+    virtualenv_python: python2.7

users.yml

@@ -58,6 +58,16 @@
     - config.cfg
     - "configs/{{ inventory_hostname }}/config.yml"
 
+  pre_tasks:
+    - block:
+      - name: Local pre-tasks
+        import_tasks: playbooks/cloud-pre.yml
+      rescue:
+        - debug: var=fail_hint
+          tags: always
+        - fail:
+          tags: always
+
   roles:
     - role: common
     - role: wireguard