From 75cfeab24a077b141f3c91341fc1546004c48d15 Mon Sep 17 00:00:00 2001 From: Jack Ivanov <17044561+jackivanov@users.noreply.github.com> Date: Wed, 17 May 2023 03:04:23 +0300 Subject: [PATCH 01/18] Ubuntu 22.04 support (#14579) * add 22.04 support * actions trigger * lighsail to 22.04 and remove 20.04 * test scripted deploy * ansible lint is advisory. moving to terraform --- .github/workflows/main.yml | 16 +++++++++---- config.cfg | 24 ++++++++++---------- docs/cloud-amazon-ec2.md | 5 +++-- docs/deploy-from-ansible.md | 35 ++++++++++++++++------------- docs/deploy-to-ubuntu.md | 5 ++++- docs/deploy-to-unsupported-cloud.md | 2 +- input.yml | 2 +- tests/pre-deploy.sh | 2 +- 8 files changed, 53 insertions(+), 38 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6676ef7..75d405f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,24 +13,26 @@ jobs: cache: 'pip' - name: Install dependencies + env: + DEBIAN_FRONTEND: noninteractive run: | sudo apt update -y python -m pip install --upgrade pip pip install -r requirements.txt sudo snap install shellcheck - pip install ansible-lint==6.3.0 + pip install ansible-lint - name: Checks and linters run: | /snap/bin/shellcheck algo install.sh ansible-playbook main.yml --syntax-check - ansible-lint -x experimental,package-latest,unnamed-task -v *.yml roles/{local,cloud-*}/*/*.yml + ansible-lint -x experimental,package-latest,unnamed-task -v *.yml roles/{local,cloud-*}/*/*.yml || true scripted-deploy: runs-on: ubuntu-20.04 strategy: matrix: - UBUNTU_VERSION: ["20.04"] + UBUNTU_VERSION: ["22.04"] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v2.3.2 @@ -39,6 +41,8 @@ jobs: cache: 'pip' - name: Install dependencies + env: + DEBIAN_FRONTEND: noninteractive run: | sudo apt update -y sudo apt install -y \ @@ -69,12 +73,14 @@ jobs: - name: Deployment run: | + set -x until sudo lxc exec algo -- test -f /var/log/cloud-init-output.log; do echo 'Log file not found, Sleep for 3 seconds'; sleep 3; done ( sudo lxc exec algo -- tail -f /var/log/cloud-init-output.log & ) until sudo lxc exec algo -- test -f /var/lib/cloud/data/result.json; do echo 'Cloud init is not finished. Sleep for 30 seconds'; sleep 30; done + sudo lxc exec algo -- cat /var/log/cloud-init-output.log sudo lxc exec algo -- test -f /opt/algo/configs/localhost/.config.yml sudo lxc exec algo -- tar zcf /root/algo-configs.tar -C /opt/algo/configs/ . sudo lxc file pull algo/root/algo-configs.tar ./ @@ -90,7 +96,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - UBUNTU_VERSION: ["20.04"] + UBUNTU_VERSION: ["22.04"] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v2.3.2 @@ -99,6 +105,8 @@ jobs: cache: 'pip' - name: Install dependencies + env: + DEBIAN_FRONTEND: noninteractive run: | set -x sudo apt update -y diff --git a/config.cfg b/config.cfg index b76aa7f..f1e102f 100644 --- a/config.cfg +++ b/config.cfg @@ -172,12 +172,12 @@ cloud_providers: type: Standard_LRS image: publisher: Canonical - offer: 0001-com-ubuntu-minimal-focal-daily - sku: minimal-20_04-daily-lts + offer: 0001-com-ubuntu-minimal-jammy-daily + sku: minimal-22_04-daily-lts version: latest digitalocean: size: s-1vcpu-1gb - image: "ubuntu-20-04-x64" + image: "ubuntu-22-04-x64" ec2: # Change the encrypted flag to "false" to disable AWS volume encryption. encrypted: true @@ -186,7 +186,7 @@ cloud_providers: use_existing_eip: false size: t2.micro image: - name: "ubuntu-focal-20.04" + name: "ubuntu-jammy-22.04" arch: x86_64 owner: "099720109477" # Change instance_market_type from "on-demand" to "spot" to launch a spot @@ -194,31 +194,31 @@ cloud_providers: instance_market_type: on-demand gce: size: e2-micro - image: ubuntu-2004-lts + image: ubuntu-2204-lts external_static_ip: false lightsail: size: nano_2_0 - image: ubuntu_20_04 + image: ubuntu_22_04 scaleway: size: DEV1-S - image: Ubuntu 20.04 Focal Fossa + image: Ubuntu 22.04 Jammy Jellyfish arch: x86_64 hetzner: server_type: cx11 - image: ubuntu-20.04 + image: ubuntu-22.04 openstack: flavor_ram: ">=512" - image: Ubuntu-20.04 + image: Ubuntu-22.04 cloudstack: size: Micro - image: Linux Ubuntu 20.04 LTS 64-bit + image: Linux Ubuntu 22.04 LTS 64-bit disk: 10 vultr: - os: Ubuntu 20.04 LTS x64 + os: Ubuntu 22.04 LTS x64 size: 1024 MB RAM,25 GB SSD,1.00 TB BW linode: type: g6-nanode-1 - image: linode/ubuntu20.04 + image: linode/ubuntu22.04 local: fail_hint: diff --git a/docs/cloud-amazon-ec2.md b/docs/cloud-amazon-ec2.md index 25d0e93..92dcf45 100644 --- a/docs/cloud-amazon-ec2.md +++ b/docs/cloud-amazon-ec2.md @@ -18,7 +18,7 @@ Addtional configurations are documented in the [EC2 section of the deploy from a In the AWS console, find the policies menu: click Services > IAM > Policies. Click Create Policy. -Here, you have the policy editor. Switch to the JSON tab and copy-paste over the existing empty policy with [the minimum required AWS policy needed for Algo deployment](https://github.com/trailofbits/algo/blob/master/docs/deploy-from-ansible.md#minimum-required-iam-permissions-for-deployment). +Here, you have the policy editor. Switch to the JSON tab and copy-paste over the existing empty policy with [the minimum required AWS policy needed for Algo deployment](https://github.com/trailofbits/algo/blob/master/docs/deploy-from-ansible.md#minimum-required-iam-permissions-for-deployment). When prompted to name the policy, name it `AlgoVPN_Provisioning`. @@ -69,7 +69,7 @@ $ ./algo 9. OpenStack (DreamCompute optimised) 10. CloudStack (Exoscale optimised) 11. Linode - 12. Install to existing Ubuntu 18.04 or 20.04 server (for more advanced users) + 12. Install to existing Ubuntu server (for more advanced users) Enter the number of your desired provider : 3 @@ -125,4 +125,5 @@ Enter the number of your desired region You will then be asked the remainder of the standard Algo setup questions. ## Cleanup + If you've installed Algo onto EC2 multiple times, your AWS account may become cluttered with unused or deleted resources e.g. instances, VPCs, subnets, etc. This may cause future installs to fail. The easiest way to clean up after you're done with a server is to go to "CloudFormation" from the console and delete the CloudFormation stack associated with that server. Please note that unless you've enabled termination protection on your instance, deleting the stack this way will delete your instance without warning, so be sure you are deleting the correct stack. diff --git a/docs/deploy-from-ansible.md b/docs/deploy-from-ansible.md index 0a4618d..00c33f0 100644 --- a/docs/deploy-from-ansible.md +++ b/docs/deploy-from-ansible.md @@ -56,19 +56,19 @@ Cloud roles: Server roles: - role: strongswan - * Installs [strongSwan](https://www.strongswan.org/) - * Enables AppArmor, limits CPU and memory access, and drops user privileges - * Builds a Certificate Authority (CA) with [easy-rsa-ipsec](https://github.com/ValdikSS/easy-rsa-ipsec) and creates one client certificate per user - * Bundles the appropriate certificates into Apple mobileconfig profiles for each user + - Installs [strongSwan](https://www.strongswan.org/) + - Enables AppArmor, limits CPU and memory access, and drops user privileges + - Builds a Certificate Authority (CA) with [easy-rsa-ipsec](https://github.com/ValdikSS/easy-rsa-ipsec) and creates one client certificate per user + - Bundles the appropriate certificates into Apple mobileconfig profiles for each user - role: dns_adblocking - * Installs DNS encryption through [dnscrypt-proxy](https://github.com/jedisct1/dnscrypt-proxy) with blacklists to be updated daily from `adblock_lists` in `config.cfg` - note this will occur even if `dns_encryption` in `config.cfg` is set to `false` - * Constrains dnscrypt-proxy with AppArmor and cgroups CPU and memory limitations + - Installs DNS encryption through [dnscrypt-proxy](https://github.com/jedisct1/dnscrypt-proxy) with blacklists to be updated daily from `adblock_lists` in `config.cfg` - note this will occur even if `dns_encryption` in `config.cfg` is set to `false` + - Constrains dnscrypt-proxy with AppArmor and cgroups CPU and memory limitations - role: ssh_tunneling - * Adds a restricted `algo` group with no shell access and limited SSH forwarding options - * Creates one limited, local account and an SSH public key for each user + - Adds a restricted `algo` group with no shell access and limited SSH forwarding options + - Creates one limited, local account and an SSH public key for each user - role: wireguard - * Installs a [Wireguard](https://www.wireguard.com/) server, with a startup script, and automatic checks for upgrades - * Creates wireguard.conf files for Linux clients as well as QR codes for Apple/Android clients + - Installs a [Wireguard](https://www.wireguard.com/) server, with a startup script, and automatic checks for upgrades + - Creates wireguard.conf files for Linux clients as well as QR codes for Apple/Android clients Note: The `strongswan` role generates Apple profiles with On-Demand Wifi and Cellular if you pass the following variables: @@ -96,7 +96,7 @@ Required variables: - do_token - region -Possible options can be gathered calling to https://api.digitalocean.com/v2/regions +Possible options can be gathered calling to ### Amazon EC2 @@ -113,20 +113,23 @@ Additional variables: - [encrypted](https://aws.amazon.com/blogs/aws/new-encrypted-ebs-boot-volumes/) - Encrypted EBS boot volume. Boolean (Default: true) - [size](https://aws.amazon.com/ec2/instance-types/) - EC2 instance type. String (Default: t2.micro) - [image](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-images.html) - AMI `describe-images` search parameters to find the OS for the hosted image. Each OS and architecture has a unique AMI-ID. The OS owner, for example [Ubuntu](https://cloud-images.ubuntu.com/locator/ec2/), updates these images often. If parameters below result in multiple results, the most recent AMI-ID is chosen + ``` # Example of equivalent cli comand - aws ec2 describe-images --owners "099720109477" --filters "Name=architecture,Values=arm64" "Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-focal-20.04*" + aws ec2 describe-images --owners "099720109477" --filters "Name=architecture,Values=arm64" "Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-jammy-22.04*" ``` + - [owners] - The operating system owner id. Default is [Canonical](https://help.ubuntu.com/community/EC2StartersGuide#Official_Ubuntu_Cloud_Guest_Amazon_Machine_Images_.28AMIs.29) (Default: 099720109477) - [arch] - The architecture (Default: x86_64, Optional: arm64) - - [name] - The wildcard string to filter available ami names. Algo appends this name with the string "-\*64-server-\*", and prepends with "ubuntu/images/hvm-ssd/" (Default: ubuntu-focal-20.04) + - [name] - The wildcard string to filter available ami names. Algo appends this name with the string "-\*64-server-\*", and prepends with "ubuntu/images/hvm-ssd/" (Default: Ubuntu latest LTS) - [instance_market_type](https://aws.amazon.com/ec2/pricing/) - Two pricing models are supported: on-demand and spot. String (Default: on-demand) - * If using spot instance types, one additional IAM permission along with the below minimum is required for deployment: + - If using spot instance types, one additional IAM permission along with the below minimum is required for deployment: + ``` "ec2:CreateLaunchTemplate" ``` -#### Minimum required IAM permissions for deployment: +#### Minimum required IAM permissions for deployment ``` { @@ -232,7 +235,7 @@ Required variables: Possible options can be gathered via cli `aws lightsail get-regions` -#### Minimum required IAM permissions for deployment: +#### Minimum required IAM permissions for deployment ``` { diff --git a/docs/deploy-to-ubuntu.md b/docs/deploy-to-ubuntu.md index 04bf094..8f11ae7 100644 --- a/docs/deploy-to-ubuntu.md +++ b/docs/deploy-to-ubuntu.md @@ -9,14 +9,17 @@ You can use Algo to configure a pre-existing server as an AlgoVPN rather than using it to create and configure a new server on a supported cloud provider. This is referred to as a **local** installation rather than a **cloud** deployment. If you're new to Algo or unfamiliar with Linux you'll find a cloud deployment to be easier. To perform a local installation, install the Algo scripts following the normal installation instructions, then choose: + ``` -Install to existing Ubuntu 18.04 or 20.04 server (for more advanced users) +Install to existing Ubuntu latest LTS server (for more advanced users) ``` + Make sure your target server is running an unmodified copy of the operating system version specified. The target can be the same system where you've installed the Algo scripts, or a remote system that you are able to access as root via SSH without needing to enter the SSH key passphrase (such as when using `ssh-agent`). ## Inbound VPN Server (also called "Road Warrior" setup) Some may find it useful to set up an Algo server on an Ubuntu box on your home LAN, with the intention of being able to securely access your LAN and any resources on it when you're traveling elsewhere (the ["road warrior" setup](https://en.wikipedia.org/wiki/Road_warrior_(computing))). A few tips if you're doing so: + - Make sure you forward any [relevant incoming ports](/docs/firewalls.md#external-firewall) to the Algo server from your router; - Change `BetweenClients_DROP` in `config.cfg` to `false`, and also consider changing `block_smb` and `block_netbios` to `false`; - If you want to use a DNS server on your LAN to resolve local domain names properly (e.g. a Pi-hole), set the `dns_encryption` flag in `config.cfg` to `false`, and change `dns_servers` to the local DNS server IP (i.e. `192.168.1.2`). diff --git a/docs/deploy-to-unsupported-cloud.md b/docs/deploy-to-unsupported-cloud.md index 6e1a5f9..5c18a5b 100644 --- a/docs/deploy-to-unsupported-cloud.md +++ b/docs/deploy-to-unsupported-cloud.md @@ -2,7 +2,7 @@ Algo officially supports the [cloud providers listed here](https://github.com/trailofbits/algo/blob/master/README.md#deploy-the-algo-server). If you want to deploy Algo on another virtual hosting provider, that provider must support: -1. the base operating system image that Algo uses (Ubuntu 18.04 or 20.04), and +1. the base operating system image that Algo uses (Ubuntu latest LTS release), and 2. a minimum of certain kernel modules required for the strongSwan IPsec server. Please see the [Required Kernel Modules](https://wiki.strongswan.org/projects/strongswan/wiki/KernelModules) documentation from strongSwan for a list of the specific required modules and a script to check for them. As a first step, we recommend running their shell script to determine initial compatibility with your new hosting provider. diff --git a/input.yml b/input.yml index 043a129..64b4805 100644 --- a/input.yml +++ b/input.yml @@ -22,7 +22,7 @@ - { name: OpenStack (DreamCompute optimised), alias: openstack } - { name: CloudStack (Exoscale optimised), alias: cloudstack } - { name: Linode, alias: linode } - - { name: Install to existing Ubuntu 18.04 or 20.04 server (for more advanced users), alias: local } + - { name: Install to existing Ubuntu latest LTS server (for more advanced users), alias: local } vars_files: - config.cfg diff --git a/tests/pre-deploy.sh b/tests/pre-deploy.sh index e5ce7c0..c26164e 100755 --- a/tests/pre-deploy.sh +++ b/tests/pre-deploy.sh @@ -32,7 +32,7 @@ until dig A +short algo.lxd @10.0.8.1 | grep -vE '^$' > /dev/null; do done case ${UBUNTU_VERSION} in - 20.04) + 20.04|22.04) lxc exec algo -- apt remove snapd --purge -y || true ;; 18.04) From 1c80cd23f59ef4bff6cdac9f94fe0cede6eac226 Mon Sep 17 00:00:00 2001 From: James Olds Date: Thu, 29 Jun 2023 13:37:09 -0400 Subject: [PATCH 02/18] update link to empirehacking slack (#14621) --- docs/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index 109093e..7ce81e8 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -17,7 +17,7 @@ ## Has Algo been audited? -No. This project is under active development. We're happy to [accept and fix issues](https://github.com/trailofbits/algo/issues) as they are identified. Use Algo at your own risk. If you find a security issue of any severity, please [contact us on Slack](https://empireslacking.herokuapp.com). +No. This project is under active development. We're happy to [accept and fix issues](https://github.com/trailofbits/algo/issues) as they are identified. Use Algo at your own risk. If you find a security issue of any severity, please [contact us on Slack](https://slack.empirehacking.nyc). ## What's the current status of WireGuard? From 7d1af5ababaa3de840eda0bcd9cefcd01af74b56 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 25 Jul 2023 13:53:34 +0200 Subject: [PATCH 03/18] Update README.md (#14591) --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 218633a..000b72a 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,8 @@ The easiest way to get an Algo server running is to run it on your local system ``` On Fedora first run `export TMPDIR=/var/tmp`, then add the option `--system-site-packages` to the first command above (after `python3 -m virtualenv`). On macOS install the C compiler if prompted. -5. **Set your configuration options.** Open the file `config.cfg` in your favorite text editor. Specify the users you wish to create in the `users` list. Create a unique user for each device you plan to connect to your VPN. If you want to add or delete users later, you **must** select `yes` at the `Do you want to retain the keys (PKI)?` prompt during the server deployment. You should also review the other options before deployment, as changing your mind about them later [may require you to deploy a brand new server](https://github.com/trailofbits/algo/blob/master/docs/faq.md#i-deployed-an-algo-server-can-you-update-it-with-new-features). +5. **Set your configuration options.** Open the file `config.cfg` in your favorite text editor. Specify the users you wish to create in the `users` list. Create a unique user for each device you plan to connect to your VPN. + > Note: [IKEv2 Only] If you want to add or delete users later, you **must** select `yes` at the `Do you want to retain the keys (PKI)?` prompt during the server deployment. You should also review the other options before deployment, as changing your mind about them later [may require you to deploy a brand new server](https://github.com/trailofbits/algo/blob/master/docs/faq.md#i-deployed-an-algo-server-can-you-update-it-with-new-features). 6. **Start the deployment.** Return to your terminal. In the Algo directory, run `./algo` and follow the instructions. There are several optional features available, none of which are required for a fully functional VPN server. These optional features are described in greater detail in [here](docs/deploy-from-ansible.md). From 1cf3d8d66d7b49a172ed2fec5d55881cc3520d53 Mon Sep 17 00:00:00 2001 From: Artem Dinaburg Date: Tue, 25 Jul 2023 07:55:28 -0400 Subject: [PATCH 04/18] Add CODEOWNERS file (#14599) --- CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..a50f8ad --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @jackivanov From abb8164054c4152489111d94e41b33f465fceca5 Mon Sep 17 00:00:00 2001 From: Brenton Bostick Date: Mon, 14 Aug 2023 11:43:02 -0400 Subject: [PATCH 05/18] Fix typos (#14635) --- docs/client-openwrt-router-wireguard.md | 6 +++--- docs/cloud-amazon-ec2.md | 2 +- docs/cloud-cloudstack.md | 2 +- docs/cloud-gce.md | 2 +- docs/deploy-from-ansible.md | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/client-openwrt-router-wireguard.md b/docs/client-openwrt-router-wireguard.md index b31781f..e3c0e70 100644 --- a/docs/client-openwrt-router-wireguard.md +++ b/docs/client-openwrt-router-wireguard.md @@ -1,4 +1,4 @@ -# Using Router wıth OpenWRT as a Client with WireGuard +# Using Router with OpenWRT as a Client with WireGuard This scenario is useful in case you want to use vpn with devices which has no vpn capability like smart tv, or make vpn connection available via router for multiple devices. This is a tested, working scenario with following environment: @@ -16,12 +16,12 @@ Make sure that you have - router and device in front of router does not have same ip . By default openwrt have 192.168.1.1 if so change it to something like 192.168.2.1 ### Install required packages(WebUI) - Open router web UI (mostly http://192.168.1.1 ) -- Login. (by default username: root, password: - System -> Software, click "Update lists" - Install following packages wireguard-tools, kmod-wireguard, luci-app-wireguard, wireguard, kmod-crypto-sha256, kmod-crypto-sha1, kmod-crypto-md5 - restart router -### Aluternative Install required packages(ssh) +### Alternative Install required packages(ssh) - Open router web UI (mostly http://192.168.1.1 ) - ssh root@192.168.1.1 - opkg update diff --git a/docs/cloud-amazon-ec2.md b/docs/cloud-amazon-ec2.md index 92dcf45..c6a0d44 100644 --- a/docs/cloud-amazon-ec2.md +++ b/docs/cloud-amazon-ec2.md @@ -12,7 +12,7 @@ The cheapest EC2 plan you can choose is the "Free Plan" a.k.a. the "AWS Free Tie As of the time of this writing (July 2018), the Free Tier limits include "750 hours of Amazon EC2 Linux t2.micro instance usage" per month, 15 GB of bandwidth (outbound) per month, and 30 GB of cloud storage. Algo will not even use 1% of the storage limit, but you may have to monitor your bandwidth usage or keep an eye out for the email from Amazon when you are about to exceed the Free Tier limits. -Addtional configurations are documented in the [EC2 section of the deploy from ansible guide](https://github.com/trailofbits/algo/blob/master/docs/deploy-from-ansible.md#amazon-ec2) +Additional configurations are documented in the [EC2 section of the deploy from ansible guide](https://github.com/trailofbits/algo/blob/master/docs/deploy-from-ansible.md#amazon-ec2) ### Create an AWS permissions policy diff --git a/docs/cloud-cloudstack.md b/docs/cloud-cloudstack.md index fe892f6..672778e 100644 --- a/docs/cloud-cloudstack.md +++ b/docs/cloud-cloudstack.md @@ -1,6 +1,6 @@ ### Configuration file -Algo scripts will ask you for the API detail. You need to fetch the API credentials and the endpoint from the provider cocntrol panel. +Algo scripts will ask you for the API detail. You need to fetch the API credentials and the endpoint from the provider control panel. Example for Exoscale (European cloud provider exposing CloudStack API), visit https://portal.exoscale.com/u//account/profile/api to gather the required information: CloudStack api key and secret. diff --git a/docs/cloud-gce.md b/docs/cloud-gce.md index c846765..f88c837 100644 --- a/docs/cloud-gce.md +++ b/docs/cloud-gce.md @@ -38,4 +38,4 @@ gcloud services enable compute.googleapis.com **Attention:** take care of the `configs/gce.json` file, which contains the credentials to manage your Google Cloud account, including create and delete servers on this project. -There are more advanced arguments available for deploynment [using ansible](deploy-from-ansible.md). +There are more advanced arguments available for deployment [using ansible](deploy-from-ansible.md). diff --git a/docs/deploy-from-ansible.md b/docs/deploy-from-ansible.md index 00c33f0..f7d6b96 100644 --- a/docs/deploy-from-ansible.md +++ b/docs/deploy-from-ansible.md @@ -115,7 +115,7 @@ Additional variables: - [image](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-images.html) - AMI `describe-images` search parameters to find the OS for the hosted image. Each OS and architecture has a unique AMI-ID. The OS owner, for example [Ubuntu](https://cloud-images.ubuntu.com/locator/ec2/), updates these images often. If parameters below result in multiple results, the most recent AMI-ID is chosen ``` - # Example of equivalent cli comand + # Example of equivalent cli command aws ec2 describe-images --owners "099720109477" --filters "Name=architecture,Values=arm64" "Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-jammy-22.04*" ``` From 17881b2d2a740ae699a41f6ee6b06d1566655bf7 Mon Sep 17 00:00:00 2001 From: dasmart Date: Wed, 27 Sep 2023 10:56:28 -0400 Subject: [PATCH 06/18] make sure cron is installed on ubuntu. #14568 (#14640) --- roles/common/tasks/ubuntu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/common/tasks/ubuntu.yml b/roles/common/tasks/ubuntu.yml index caa1d30..15d2d91 100644 --- a/roles/common/tasks/ubuntu.yml +++ b/roles/common/tasks/ubuntu.yml @@ -98,6 +98,7 @@ - cgroup-tools - openssl - gnupg2 + - cron sysctl: - item: net.ipv4.ip_forward value: 1 From 199e404ec47640c29d3045868ac91080ade70af4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 17:57:03 +0300 Subject: [PATCH 07/18] Bump docker/build-push-action from 4 to 5 (#14650) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4 to 5. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4...v5) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yaml b/.github/workflows/docker-image.yaml index d7f0dda..7a77998 100644 --- a/.github/workflows/docker-image.yaml +++ b/.github/workflows/docker-image.yaml @@ -36,7 +36,7 @@ jobs: type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - name: Build and push Docker image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . push: true From fc1d3d4d7b3dc24d78917121a8cb25d521f1e073 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 17:57:15 +0300 Subject: [PATCH 08/18] Bump actions/checkout from 3 to 4 (#14647) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-image.yaml | 2 +- .github/workflows/main.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-image.yaml b/.github/workflows/docker-image.yaml index 7a77998..1e55d8e 100644 --- a/.github/workflows/docker-image.yaml +++ b/.github/workflows/docker-image.yaml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Log in to the Container registry uses: docker/login-action@v2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 75d405f..dad810f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,7 @@ jobs: lint: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v2.3.2 with: python-version: '3.9' @@ -34,7 +34,7 @@ jobs: matrix: UBUNTU_VERSION: ["22.04"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v2.3.2 with: python-version: '3.9' @@ -98,7 +98,7 @@ jobs: matrix: UBUNTU_VERSION: ["22.04"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v2.3.2 with: python-version: '3.9' From 29f5a2f6f8d8c3aa99e3cc16cdae2d80c7250181 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 17:57:28 +0300 Subject: [PATCH 09/18] Bump docker/metadata-action from 4 to 5 (#14648) Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 4 to 5. - [Release notes](https://github.com/docker/metadata-action/releases) - [Upgrade guide](https://github.com/docker/metadata-action/blob/master/UPGRADE.md) - [Commits](https://github.com/docker/metadata-action/compare/v4...v5) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yaml b/.github/workflows/docker-image.yaml index 1e55d8e..2a05ea1 100644 --- a/.github/workflows/docker-image.yaml +++ b/.github/workflows/docker-image.yaml @@ -28,7 +28,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | From a5b30cdbfee80f648216592e85f63e8f6c04a89b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 18:02:00 +0300 Subject: [PATCH 10/18] Bump docker/login-action from 2 to 3 (#14649) Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yaml b/.github/workflows/docker-image.yaml index 2a05ea1..cbcf718 100644 --- a/.github/workflows/docker-image.yaml +++ b/.github/workflows/docker-image.yaml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v4 - name: Log in to the Container registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} From fd6efb71f255739c1e50e8222c6c5fab99df5709 Mon Sep 17 00:00:00 2001 From: Dima Scherbakov Date: Wed, 27 Sep 2023 08:15:35 -0700 Subject: [PATCH 11/18] ssh_config: ignore pre-existing SSH keys on client (#14646) sshd limits the number of authentication attempts permitted per established connection. The limit is set via the MaxAuthTries option and defaults to six attempts. Client SSH environments that define more than six SSH keys globally or in the agent would exhaust authentication attempts before they reach the algo-specified per-instance SSH private key. SSH client allows "forgetting" existing keys per connection using the IdentitiesOnly option. A client only offers an explicitly defined key when this option is set. --- roles/ssh_tunneling/templates/ssh_config.j2 | 1 + server.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/roles/ssh_tunneling/templates/ssh_config.j2 b/roles/ssh_tunneling/templates/ssh_config.j2 index 04931fc..54600b1 100644 --- a/roles/ssh_tunneling/templates/ssh_config.j2 +++ b/roles/ssh_tunneling/templates/ssh_config.j2 @@ -2,6 +2,7 @@ Host algo DynamicForward 127.0.0.1:1080 LogLevel quiet Compression yes + IdentitiesOnly yes IdentityFile {{ item }}.ssh.pem User {{ item }} Hostname {{ IP_subject_alt_name }} diff --git a/server.yml b/server.yml index 18af459..d1828ea 100644 --- a/server.yml +++ b/server.yml @@ -32,6 +32,7 @@ HostName {{ IP_subject_alt_name }} User {{ ansible_ssh_user }} Port {{ ansible_ssh_port }} + IdentitiesOnly yes IdentityFile {{ SSH_keys.private | realpath }} KeepAlive yes ServerAliveInterval 30 From 90c2b1bbebfb8763a3d0d07cf6a3defaea4fa375 Mon Sep 17 00:00:00 2001 From: Jack Ivanov <17044561+jackivanov@users.noreply.github.com> Date: Thu, 7 Dec 2023 17:54:08 -0500 Subject: [PATCH 12/18] upgrade ansible to 9.1.0 (#14673) * upgrade to 9.1.0 * python version * 3.11-alpine * missed python version --- .github/workflows/main.yml | 6 +++--- Dockerfile | 2 +- requirements.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dad810f..143ccb5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v2.3.2 with: - python-version: '3.9' + python-version: '3.11' cache: 'pip' - name: Install dependencies @@ -37,7 +37,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v2.3.2 with: - python-version: '3.9' + python-version: '3.11' cache: 'pip' - name: Install dependencies @@ -101,7 +101,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v2.3.2 with: - python-version: '3.9' + python-version: '3.11' cache: 'pip' - name: Install dependencies diff --git a/Dockerfile b/Dockerfile index 387d42e..84a9afa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3-alpine +FROM python:3.11-alpine ARG VERSION="git" ARG PACKAGES="bash libffi openssh-client openssl rsync tini gcc libffi-dev linux-headers make musl-dev openssl-dev rust cargo" diff --git a/requirements.txt b/requirements.txt index 886f8b4..6ef66f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -ansible==6.1.0 +ansible==9.1.0 jinja2~=3.0.3 netaddr From a6ad0adc942fa32855b8027d6d80d87695109188 Mon Sep 17 00:00:00 2001 From: Alex Oswald Date: Thu, 7 Dec 2023 17:56:17 -0500 Subject: [PATCH 13/18] update digitalocean docs on droplets (#14659) Make note of smaller/cheaper droplets for DigitalOcean and document it. --- config.cfg | 2 ++ docs/cloud-do.md | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/config.cfg b/config.cfg index f1e102f..e8b6559 100644 --- a/config.cfg +++ b/config.cfg @@ -176,6 +176,8 @@ cloud_providers: sku: minimal-22_04-daily-lts version: latest digitalocean: + # See docs for extended droplet options, pricing, and availability. + # Possible values: 's-1vcpu-512mb-10gb', 's-1vcpu-1gb', ... size: s-1vcpu-1gb image: "ubuntu-22-04-x64" ec2: diff --git a/docs/cloud-do.md b/docs/cloud-do.md index 59596e0..88ec8e9 100644 --- a/docs/cloud-do.md +++ b/docs/cloud-do.md @@ -18,6 +18,18 @@ You will be returned to the **Tokens/Keys** tab, and your new key will be shown Copy or note down the hash that shows below the name you entered, as this will be necessary for the steps below. This value will disappear if you leave this page, and you'll need to regenerate it if you forget it. +## Select a Droplet (optional) + +The default option is the `s-1vcpu-1gb` because it is available in all regions. However, you may want to switch to a cheaper droplet such as `s-1vcpu-512mb-10gb` even though it is not available in all regions. This can be edited in the [Configuration File](config.cfg) under `cloud_providers > digitalocean > size`. See this brief comparison between the two droplets below: + +| Droplet Type | Monthly Cost | Bandwidth | Availability | +|:--|:-:|:-:|:--| +| `s-1vcpu-512mb-10gb` | $4/month | 0.5 TB | Limited | +| `s-1vcpu-1gb` | $6/month | 1.0 TB | All regions | +| ... | ... | ... | ... | + +*Note: Exceeding bandwidth limits costs $0.01/GiB at time of writing ([docs](https://docs.digitalocean.com/products/billing/bandwidth/#droplets)). See the live list of droplets [here](https://slugs.do-api.dev/).* + ## Using DigitalOcean with Algo (interactive) These steps are for those who run Algo using Docker or using the `./algo` command. From 67aa5fe881baffb2da70a4c971a2dab57e273288 Mon Sep 17 00:00:00 2001 From: Pavel Mishkovich Date: Thu, 7 Dec 2023 22:57:57 +0000 Subject: [PATCH 14/18] Add a linode entry to troubleshooting.md (#14632) --- docs/troubleshooting.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 75c73c6..5d6d488 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -23,6 +23,7 @@ First of all, check [this](https://github.com/trailofbits/algo#features) and ens * [Wireguard: Unable to find 'configs/...' in expected paths](#wireguard-unable-to-find-configs-in-expected-paths) * [Ubuntu Error: "unable to write 'random state'" when generating CA password](#ubuntu-error-unable-to-write-random-state-when-generating-ca-password) * [Timeout when waiting for search string OpenSSH in xxx.xxx.xxx.xxx:4160](#old-networking-firewall-in-place) + * [Linode Error: "Unable to query the Linode API. Saw: 400: The requested distribution is not supported by this stackscript.; "](#linode-error-uable-to-query-the-linode-api-saw-400-the-requested-distribution-is-not-supported-by-this-stackscript) * [Connection Problems](#connection-problems) * [I'm blocked or get CAPTCHAs when I access certain websites](#im-blocked-or-get-captchas-when-i-access-certain-websites) * [I want to change the list of trusted Wifi networks on my Apple device](#i-want-to-change-the-list-of-trusted-wifi-networks-on-my-apple-device) @@ -384,6 +385,11 @@ ok: [localhost] => { If you see this error then one possible explanation is that you have a previous firewall configured in your cloud hosting provider which needs to be either updated or ideally removed. Removing this can often fix this issue. +### Linode Error: "Unable to query the Linode API. Saw: 400: The requested distribution is not supported by this stackscript.; " + +StackScript is a custom deployment script that defines a set of configurations for a Linode instance (e.g. which distribution, specs, etc.). if you used algo with default values in the past deployments, a stackscript that would've been created is 're-used' in the deployment process (in fact, go see 'create Linodes' and under 'StackScripts' tab). Thus, there's a little chance that your deployment process will generate this 'unsupported stackscript' error due to a pre-existing StackScript that doesn't support a particular configuration setting or value due to an 'old' stackscript. The quickest solution is just to change the name of your deployment from the default value of 'algo' (or any other name that you've used before, again see the dashboard) and re-run the deployment. + + ## Connection Problems Look here if you deployed an Algo server but now have a problem connecting to it with a client. From c9352a180190275cef0715d18c798c3ffb604fb0 Mon Sep 17 00:00:00 2001 From: Disconnect3d Date: Tue, 12 Dec 2023 17:05:13 +0100 Subject: [PATCH 15/18] cloud-pre.yml: use 4096 bits for ssh rsa key (#14674) The ssh-key we generated used 2048 bits while even openssh's ssh-keygen defaults to 3072 nowadays [0]. While RSA-2048 is probably ok (?) and what NIST recommends for keys until around 2030, its probably better to switch to more bits. This is also just a temporary solution as we should also switch to ed25519. Thanks to Dan M (@dmur1 or dan@hexarcana.ch) for pointing this out. [0] https://github.com/openssh/openssh-portable/blob/19d3ee2f3adf7d9a606ff015c1e153744702c4c9/ssh-keygen.c#L83 --- playbooks/cloud-pre.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/cloud-pre.yml b/playbooks/cloud-pre.yml index da0019d..d513412 100644 --- a/playbooks/cloud-pre.yml +++ b/playbooks/cloud-pre.yml @@ -32,7 +32,7 @@ - name: Generate the SSH private key openssl_privatekey: path: "{{ SSH_keys.private }}" - size: 2048 + size: 4096 mode: "0600" type: RSA From baf8a85c0b4241e748688d162b5e4e9661dd2933 Mon Sep 17 00:00:00 2001 From: Ayaan Mirza Baig Date: Tue, 12 Dec 2023 21:47:59 +0530 Subject: [PATCH 16/18] Create SECURITY.md (#14669) Added a basic skeleton for SECURITY.md Co-authored-by: William Woodruff --- SECURITY.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..8496af3 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,9 @@ +# Reporting Security Issues + +The Algo team and community take security bugs in Algo seriously. We appreciate your efforts to responsibly disclose your findings, and will make every effort to acknowledge your contributions. + +To report a security issue, please use the GitHub Security Advisory ["Report a Vulnerability"](https://github.com/trailofbits/algo/security/) tab. + +The Algo team will send a response indicating the next steps in handling your report. After the initial reply to your report, the security team will keep you informed of the progress towards a fix and full announcement, and may ask for additional information or guidance. + +Report security bugs in third-party modules to the person or team maintaining the module. From 5817300bb149996eccb881b027a2036fef9ef95c Mon Sep 17 00:00:00 2001 From: BowTiedJerboa <148892906+BowTiedJerboa@users.noreply.github.com> Date: Fri, 29 Dec 2023 21:40:39 +0100 Subject: [PATCH 17/18] Updated Python dependency from 3.8 to 3.10 (#14677) * Updated Python dependency from 3.8 to 3.10 to support version issues with Ansible * Changed install recommendations to use pyenv instead of downloading from ppa --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 000b72a..0ee6f9b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The easiest way to get an Algo server running is to run it on your local system git clone https://github.com/trailofbits/algo.git ``` -3. **Install Algo's core dependencies.** Algo requires that **Python 3.8 or later** and at least one supporting package are installed on your system. +3. **Install Algo's core dependencies.** Algo requires that **Python 3.10 or later** and at least one supporting package are installed on your system. - **macOS:** Catalina (10.15) and higher includes Python 3 as part of the optional Command Line Developer Tools package. From Terminal run: @@ -52,7 +52,7 @@ The easiest way to get an Algo server running is to run it on your local system For macOS versions prior to Catalina, see [Deploy from macOS](docs/deploy-from-macos.md) for information on installing Python 3 . - - **Linux:** Recent releases of Ubuntu, Debian, and Fedora come with Python 3 already installed. Make sure your system is up-to-date and install the supporting package(s): + - **Linux:** Recent releases of Ubuntu, Debian, and Fedora come with Python 3 already installed. If your Python version is not 3.10, then you will need to use pyenv to install Python 3.10. Make sure your system is up-to-date and install the supporting package(s): * Ubuntu and Debian: ```bash sudo apt install -y --no-install-recommends python3-virtualenv From 74051d06a26bd8fc4af1049218be29887608f438 Mon Sep 17 00:00:00 2001 From: Okan Binli Date: Thu, 4 Jan 2024 20:46:31 +0300 Subject: [PATCH 18/18] Update README.md dependencies (#14634) `file` and `lookup` are part of the ubuntu most of the time but in some cases it was missing therefore ansible fails. Co-authored-by: Jack Ivanov <17044561+jackivanov@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ee6f9b..0e62217 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ The easiest way to get an Algo server running is to run it on your local system - **Linux:** Recent releases of Ubuntu, Debian, and Fedora come with Python 3 already installed. If your Python version is not 3.10, then you will need to use pyenv to install Python 3.10. Make sure your system is up-to-date and install the supporting package(s): * Ubuntu and Debian: ```bash - sudo apt install -y --no-install-recommends python3-virtualenv + sudo apt install -y --no-install-recommends python3-virtualenv file lookup ``` On a Raspberry Pi running Ubuntu also install `libffi-dev` and `libssl-dev`.