on-build python venvs (#1199)

.gitignore

@@ -5,3 +5,5 @@ inventory_users
 *.kate-swp
 env
 .DS_Store
+venvs/*
+!venvs/.gitinit

README.md

@@ -58,7 +58,7 @@ The easiest way to get an Algo server running is to let it set up a _new_ virtua
     ```bash
     $ python -m virtualenv --python=`which python2` env &&
         source env/bin/activate &&
-        python -m pip install -U pip &&
+        python -m pip install -U pip virtualenv &&
         python -m pip install -r requirements.txt
     ```
     On macOS, you may be prompted to install `cc`. You should press accept if so.

config.cfg

@@ -13,6 +13,9 @@ users:
 # If True re-init all existing certificates. Boolean
 keys_clean_all: False
 
+# Clean up cloud python environments
+clean_environment: false
+
 vpn_network: 10.19.48.0/24
 vpn_network_ipv6: 'fd9d:bc11:4020::/48'
 wireguard_enabled: true

playbooks/cloud-pre.yml

@@ -14,6 +14,16 @@
         'dns_encryption "{{ dns_encryption }}"' \
         > /dev/tty
 
+- name: Install the requirements
+  local_action:
+    module: pip
+    state: latest
+    name:
+      - pyOpenSSL
+      - jinja2==2.8
+      - segno
+  tags: always
+
 - name: Generate the SSH private key
   openssl_privatekey:
     path: "{{ SSH_keys.private }}"

requirements.txt

@@ -1,13 +1 @@
-setuptools>=11.3
-SecretStorage < 3
-ansible[azure]==2.5.2
-dopy==0.3.5
-boto>=2.5
-boto3
-apache-libcloud
-six
-pyopenssl
-jinja2==2.8
-shade
-pycrypto
-segno
+ansible==2.5.2

roles/cloud-azure/defaults/main.yml

@@ -1,4 +1,5 @@
 ---
+azure_venv: "{{ playbook_dir }}/configs/.venvs/azure"
 _azure_regions: >
   [
     {

roles/cloud-azure/tasks/main.yml

@@ -1,4 +1,8 @@
 ---
+- block:
+    - name: Build python virtual environment
+      import_tasks: venv.yml
+
     - block:
       - name: Include prompts
         import_tasks: prompts.yml
@@ -37,7 +41,8 @@
       - set_fact:
           cloud_instance_ip: "{{ azure_rm_deployment.deployment.outputs.publicIPAddresses.value }}"
           ansible_ssh_user: ubuntu
-
+      environment:
+        PYTHONPATH: "{{ azure_venv }}/lib/python2.7/site-packages/"
   rescue:
     - debug: var=fail_hint
       tags: always

roles/cloud-azure/tasks/venv.yml

@@ -0,0 +1,32 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ azure_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name:
+      - packaging
+      - requests[security]
+      - azure-mgmt-compute>=2.0.0,<3
+      - azure-mgmt-network>=1.3.0,<2
+      - azure-mgmt-storage>=1.5.0,<2
+      - azure-mgmt-resource>=1.1.0,<2
+      - azure-storage>=0.35.1,<0.36
+      - azure-cli-core>=2.0.12,<3
+      - msrest==0.4.29
+      - msrestazure==0.4.31
+      - azure-mgmt-dns>=1.0.1,<2
+      - azure-mgmt-keyvault>=0.40.0,<0.41
+      - azure-mgmt-batch>=4.1.0,<5
+      - azure-mgmt-sql>=0.7.1,<0.8
+      - azure-mgmt-web>=0.32.0,<0.33
+      - azure-mgmt-containerservice>=2.0.0,<3.0.0
+      - azure-mgmt-containerregistry>=1.0.1
+      - azure-mgmt-rdbms==1.2.0
+      - azure-mgmt-containerinstance==0.4.0
+    state: latest
+    virtualenv: "{{ azure_venv }}"
+    virtualenv_python: python2.7

roles/cloud-digitalocean/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+digitalocean_venv: "{{ playbook_dir }}/configs/.venvs/digitalocean"

roles/cloud-digitalocean/tasks/main.yml

@@ -1,3 +1,7 @@
+- block:
+    - name: Build python virtual environment
+      import_tasks: venv.yml
+
     - block:
       - name: Include prompts
         import_tasks: prompts.yml
@@ -97,6 +101,8 @@
 
           - fail:
               msg: "Please, ensure that your API token is not read-only."
+      environment:
+        PYTHONPATH: "{{ digitalocean_venv }}/lib/python2.7/site-packages/"
   rescue:
     - debug: var=fail_hint
       tags: always

roles/cloud-digitalocean/tasks/venv.yml

@@ -0,0 +1,13 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ digitalocean_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name: dopy
+    version: 0.3.5
+    virtualenv: "{{ digitalocean_venv }}"
+    virtualenv_python: python2.7

roles/cloud-ec2/defaults/main.yml

@@ -4,3 +4,4 @@ encrypted: "{{ cloud_providers.ec2.encrypted }}"
 ec2_vpc_nets:
   cidr_block: 172.16.0.0/16
   subnet_cidr: 172.16.254.0/23
+ec2_venv: "{{ playbook_dir }}/configs/.venvs/aws"

roles/cloud-ec2/tasks/main.yml

@@ -1,3 +1,7 @@
+- block:
+    - name: Build python virtual environment
+      import_tasks: venv.yml
+
     - block:
       - name: Include prompts
         import_tasks: prompts.yml
@@ -35,6 +39,8 @@
       - set_fact:
           cloud_instance_ip: "{{ stack.stack_outputs.ElasticIP }}"
           ansible_ssh_user: ubuntu
+      environment:
+        PYTHONPATH: "{{ ec2_venv }}/lib/python2.7/site-packages/"
   rescue:
     - debug: var=fail_hint
       tags: always

roles/cloud-ec2/tasks/venv.yml

@@ -0,0 +1,15 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ ec2_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name:
+      - boto>=2.5
+      - boto3
+    state: latest
+    virtualenv: "{{ ec2_venv }}"
+    virtualenv_python: python2.7

roles/cloud-gce/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+gce_venv: "{{ playbook_dir }}/configs/.venvs/gce"

roles/cloud-gce/tasks/main.yml

@@ -1,3 +1,7 @@
+- block:
+    - name: Build python virtual environment
+      import_tasks: venv.yml
+
     - block:
       - name: Include prompts
         import_tasks: prompts.yml
@@ -49,6 +53,8 @@
       - set_fact:
           cloud_instance_ip: "{{ google_vm.instance_data[0].public_ip }}"
           ansible_ssh_user: ubuntu
+      environment:
+        PYTHONPATH: "{{ gce_venv }}/lib/python2.7/site-packages/"
   rescue:
     - debug: var=fail_hint
       tags: always

roles/cloud-gce/tasks/venv.yml

@@ -0,0 +1,15 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ gce_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name:
+      - apache-libcloud
+      - pycrypto
+    state: latest
+    virtualenv: "{{ gce_venv }}"
+    virtualenv_python: python2.7

roles/cloud-lightsail/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+lightsail_venv: "{{ playbook_dir }}/configs/.venvs/aws"

roles/cloud-lightsail/tasks/main.yml

@@ -1,3 +1,7 @@
+- block:
+  - name: Build python virtual environment
+    import_tasks: venv.yml
+
   - block:
     - name: Include prompts
       import_tasks: prompts.yml
@@ -36,6 +40,8 @@
     - set_fact:
         cloud_instance_ip: "{{ algo_instance['instance']['public_ip_address'] }}"
         ansible_ssh_user: ubuntu
+    environment:
+      PYTHONPATH: "{{ lightsail_venv }}/lib/python2.7/site-packages/"
 
   rescue:
   - debug: var=fail_hint

roles/cloud-lightsail/tasks/venv.yml

@@ -0,0 +1,15 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ lightsail_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name:
+      - boto>=2.5
+      - boto3
+    state: latest
+    virtualenv: "{{ lightsail_venv }}"
+    virtualenv_python: python2.7

roles/cloud-openstack/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+openstack_venv: "{{ playbook_dir }}/configs/.venvs/openstack"

roles/cloud-openstack/tasks/main.yml

@@ -3,6 +3,10 @@
     msg: "OpenStack credentials are not set. Download it from the OpenStack dashboard->Compute->API Access and source it in the shell (eg: source /tmp/dhc-openrc.sh)"
   when: lookup('env', 'OS_AUTH_URL') == ""
 
+- block:
+  - name: Build python virtual environment
+    import_tasks: venv.yml
+
   - block:
     - name: Security group created
       os_security_group:
@@ -75,6 +79,8 @@
     - set_fact:
         cloud_instance_ip: "{{ os_server['openstack']['public_v4'] }}"
         ansible_ssh_user: ubuntu
+    environment:
+      PYTHONPATH: "{{ openstack_venv }}/lib/python2.7/site-packages/"
 
   rescue:
   - debug: var=fail_hint

roles/cloud-openstack/tasks/venv.yml

@@ -0,0 +1,13 @@
+---
+- name: Clean up the environment
+  file:
+    dest: "{{ openstack_venv }}"
+    state: absent
+  when: clean_environment
+
+- name: Install requirements
+  pip:
+    name: shade
+    state: latest
+    virtualenv: "{{ openstack_venv }}"
+    virtualenv_python: python2.7

users.yml

@@ -58,6 +58,16 @@
     - config.cfg
     - "configs/{{ inventory_hostname }}/config.yml"
 
+  pre_tasks:
+    - block:
+      - name: Local pre-tasks
+        import_tasks: playbooks/cloud-pre.yml
+      rescue:
+        - debug: var=fail_hint
+          tags: always
+        - fail:
+          tags: always
+
   roles:
     - role: common
     - role: wireguard