From cc236b75981bf1369d945f33313f967e20558dd5 Mon Sep 17 00:00:00 2001 From: Vitor Hideyoshi Nakazone Batista Date: Sun, 13 Aug 2023 16:39:52 -0300 Subject: [PATCH] Better Deployment Implementation --- .github/workflows/deploy-prod.yml | 55 ++++++----- .github/workflows/deploy-staging.yml | 49 +++++----- Pipfile | 13 +++ Pipfile.lock | 37 ++++++++ setup.py | 135 +++++++++++++++++++++++++++ setup.sh | 62 ------------ 6 files changed, 244 insertions(+), 107 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 setup.py delete mode 100755 setup.sh diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index dd45caf..ae1bdac 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -3,42 +3,49 @@ name: remote ssh command on: push: branches: - - "main" + - main jobs: build: name: Build - environment: prod + environment: staging runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: "3.10" - name: Make Env File uses: SpicyPizza/create-envfile@v2.0 with: - envkey_FRONTEND_PATH: ${{ secrets.FRONTEND_PATH }} - envkey_TOKEN_SECRET: ${{ secrets.TOKEN_SECRET }} - envkey_ACCESS_TOKEN_DURATION: ${{ secrets.ACCESS_TOKEN_DURATION }} - envkey_REFRESH_TOKEN_DURATION: ${{ secrets.REFRESH_TOKEN_DURATION }} - envkey_DEFAULT_USER_FULLNAME: ${{ secrets.DEFAULT_USER_FULLNAME }} - envkey_DEFAULT_USER_EMAIL: ${{ secrets.DEFAULT_USER_EMAIL }} - envkey_DEFAULT_USER_USERNAME: ${{ secrets.DEFAULT_USER_USERNAME }} - envkey_DEFAULT_USER_PASSWORD: ${{ secrets.DEFAULT_USER_PASSWORD }} - envkey_GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} - envkey_GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} - envkey_GOOGLE_REDIRECT_URL: ${{ secrets.GOOGLE_REDIRECT_URL }} - envkey_OAUTH_GITHUB_CLIENT_ID: ${{ secrets.OAUTH_GITHUB_CLIENT_ID }} - envkey_OAUTH_GITHUB_CLIENT_SECRET: ${{ secrets.OAUTH_GITHUB_CLIENT_SECRET }} - envkey_OAUTH_GITHUB_REDIRECT_URL: ${{ secrets.OAUTH_GITHUB_REDIRECT_URL }} - envkey_POSTGRES_USER: ${{ secrets.POSTGRES_USER }} - envkey_POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} - envkey_POSTGRES_DB: ${{ secrets.POSTGRES_DB }} - envkey_REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} + envkey_BACKEND_OAUTH_URL: ${{ secrets.BACKEND_OAUTH_URL }} + envkey_BACKEND_URL: ${{ secrets.BACKEND_URL }} + envkey_FRONTEND_PATH: ${{ secrets.FRONTEND_PATH }} + envkey_GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} + envkey_GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} + envkey_GOOGLE_REDIRECT_URL: ${{ secrets.GOOGLE_REDIRECT_URL }} + envkey_OAUTH_GITHUB_CLIENT_ID: ${{ secrets.OAUTH_GITHUB_CLIENT_ID }} + envkey_OAUTH_GITHUB_CLIENT_SECRET: ${{ secrets.OAUTH_GITHUB_CLIENT_SECRET }} + envkey_OAUTH_GITHUB_REDIRECT_URL: ${{ secrets.OAUTH_GITHUB_REDIRECT_URL }} + envkey_ACCESS_TOKEN_DURATION: ${{ secrets.ACCESS_TOKEN_DURATION}} + envkey_DEFAULT_USER_EMAIL: ${{ secrets.DEFAULT_USER_EMAIL}} + envkey_DEFAULT_USER_FULLNAME: ${{ secrets.DEFAULT_USER_FULLNAME}} + envkey_DEFAULT_USER_PASSWORD: ${{ secrets.DEFAULT_USER_PASSWORD}} + envkey_DEFAULT_USER_USERNAME: ${{ secrets.DEFAULT_USER_USERNAME}} + envkey_POSTGRES_DB: ${{ secrets.POSTGRES_DB}} + envkey_POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD}} + envkey_POSTGRES_USER: ${{ secrets.POSTGRES_USER}} + envkey_REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD}} + envkey_REFRESH_TOKEN_DURATION: ${{ secrets.REFRESH_TOKEN_DURATION}} + envkey_TOKEN_SECRET: ${{ secrets.TOKEN_SECRET}} - name: Inserts Prod Enviromental Variables run: | - ./setup.sh --prod .env + python -m pip install --upgrade pip pipenv + pipenv install + pipenv run python setup.py -e prod -f .env - name: copy file via ssh uses: appleboy/scp-action@master @@ -59,4 +66,4 @@ jobs: key: ${{ secrets.SSH_KEY }} script: | cd infra-hideyoshi.com - ./deploy.sh \ No newline at end of file + ./deploy.sh --prod \ No newline at end of file diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index e337088..a6972e2 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -11,34 +11,41 @@ jobs: environment: staging runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: "3.10" - name: Make Env File uses: SpicyPizza/create-envfile@v2.0 with: - envkey_FRONTEND_PATH: ${{ secrets.FRONTEND_PATH }} - envkey_TOKEN_SECRET: ${{ secrets.TOKEN_SECRET }} - envkey_ACCESS_TOKEN_DURATION: ${{ secrets.ACCESS_TOKEN_DURATION }} - envkey_REFRESH_TOKEN_DURATION: ${{ secrets.REFRESH_TOKEN_DURATION }} - envkey_DEFAULT_USER_FULLNAME: ${{ secrets.DEFAULT_USER_FULLNAME }} - envkey_DEFAULT_USER_EMAIL: ${{ secrets.DEFAULT_USER_EMAIL }} - envkey_DEFAULT_USER_USERNAME: ${{ secrets.DEFAULT_USER_USERNAME }} - envkey_DEFAULT_USER_PASSWORD: ${{ secrets.DEFAULT_USER_PASSWORD }} - envkey_GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} - envkey_GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} - envkey_GOOGLE_REDIRECT_URL: ${{ secrets.GOOGLE_REDIRECT_URL }} - envkey_OAUTH_GITHUB_CLIENT_ID: ${{ secrets.OAUTH_GITHUB_CLIENT_ID }} - envkey_OAUTH_GITHUB_CLIENT_SECRET: ${{ secrets.OAUTH_GITHUB_CLIENT_SECRET }} - envkey_OAUTH_GITHUB_REDIRECT_URL: ${{ secrets.OAUTH_GITHUB_REDIRECT_URL }} - envkey_POSTGRES_USER: ${{ secrets.POSTGRES_USER }} - envkey_POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} - envkey_POSTGRES_DB: ${{ secrets.POSTGRES_DB }} - envkey_REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} + envkey_BACKEND_OAUTH_URL: ${{ secrets.BACKEND_OAUTH_URL }} + envkey_BACKEND_URL: ${{ secrets.BACKEND_URL }} + envkey_FRONTEND_PATH: ${{ secrets.FRONTEND_PATH }} + envkey_GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} + envkey_GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} + envkey_GOOGLE_REDIRECT_URL: ${{ secrets.GOOGLE_REDIRECT_URL }} + envkey_OAUTH_GITHUB_CLIENT_ID: ${{ secrets.OAUTH_GITHUB_CLIENT_ID }} + envkey_OAUTH_GITHUB_CLIENT_SECRET: ${{ secrets.OAUTH_GITHUB_CLIENT_SECRET }} + envkey_OAUTH_GITHUB_REDIRECT_URL: ${{ secrets.OAUTH_GITHUB_REDIRECT_URL }} + envkey_ACCESS_TOKEN_DURATION: ${{ secrets.ACCESS_TOKEN_DURATION}} + envkey_DEFAULT_USER_EMAIL: ${{ secrets.DEFAULT_USER_EMAIL}} + envkey_DEFAULT_USER_FULLNAME: ${{ secrets.DEFAULT_USER_FULLNAME}} + envkey_DEFAULT_USER_PASSWORD: ${{ secrets.DEFAULT_USER_PASSWORD}} + envkey_DEFAULT_USER_USERNAME: ${{ secrets.DEFAULT_USER_USERNAME}} + envkey_POSTGRES_DB: ${{ secrets.POSTGRES_DB}} + envkey_POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD}} + envkey_POSTGRES_USER: ${{ secrets.POSTGRES_USER}} + envkey_REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD}} + envkey_REFRESH_TOKEN_DURATION: ${{ secrets.REFRESH_TOKEN_DURATION}} + envkey_TOKEN_SECRET: ${{ secrets.TOKEN_SECRET}} - name: Inserts Prod Enviromental Variables run: | - ./setup.sh --staging .env + python -m pip install --upgrade pip pipenv + pipenv install + pipenv run python setup.py -e staging -f .env - name: copy file via ssh uses: appleboy/scp-action@master diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..70e71ab --- /dev/null +++ b/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +python-dotenv = "*" +envsubst = "*" + +[dev-packages] + +[requires] +python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..6f0e098 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,37 @@ +{ + "_meta": { + "hash": { + "sha256": "9127f5121153ee714035c045f3621c810565d451e8dd461576f24e165e55f73d" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "envsubst": { + "hashes": [ + "sha256:7188f9f2a046e45b63098ce5a7fd84126cbe9e5b73b8ff78eaf6e32122c0caaf", + "sha256:d8e402984a84dda4ea7a8a1f7afe1c41e54a1257cfb74c80cb8f991053d97b9b" + ], + "index": "pypi", + "version": "==0.1.5" + }, + "python-dotenv": { + "hashes": [ + "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba", + "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a" + ], + "index": "pypi", + "version": "==1.0.0" + } + }, + "develop": {} +} diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c023d54 --- /dev/null +++ b/setup.py @@ -0,0 +1,135 @@ +from base64 import b64decode, b64encode +from dotenv import load_dotenv +from envsubst import envsubst +from pathlib import Path, PosixPath +import argparse +import os + + +ENV_VARIABLES = [ + "FRONTEND_PATH", + "BACKEND_URL", + "BACKEND_OAUTH_URL", + "TOKEN_SECRET", + "ACCESS_TOKEN_DURATION", + "REFRESH_TOKEN_DURATION", + "DEFAULT_USER_FULLNAME", + "DEFAULT_USER_EMAIL", + "DEFAULT_USER_USERNAME", + "DEFAULT_USER_PASSWORD", + "GOOGLE_CLIENT_ID", + "GOOGLE_CLIENT_SECRET", + "GOOGLE_REDIRECT_URL", + "OAUTH_GITHUB_CLIENT_ID", + "OAUTH_GITHUB_CLIENT_SECRET", + "OAUTH_GITHUB_REDIRECT_URL", + "POSTGRES_USER", + "POSTGRES_PASSWORD", + "POSTGRES_DB", + "REDIS_PASSWORD", +] + + +FORCE_BASE64_FIELD = [ + "OAUTH_GITHUB_CLIENT_ID", + "OAUTH_GITHUB_CLIENT_SECRET" +] + + +def is_force_base64_fields(field: str) -> bool: + return field in FORCE_BASE64_FIELD + + +def is_validate_base64(value: str) -> bool: + if not isinstance(value, str): + return False + + try: + if b64encode(b64decode(value)).decode() == value: + return True + except: + pass + + return False + + +def setting_environment(environment: str): + if not environment in ("prod", "staging", "dev"): + raise ValueError("Invalid Environment Selected") + + match environment: + case "staging": + DOMAIN="staging.hideyoshi.com.br" + API_DOMAIN="api.staging.hideyoshi.com.br" + case _: + DOMAIN="hideyoshi.com.br" + API_DOMAIN="api.hideyoshi.com.br" + + os.environ["DOMAIN"] = DOMAIN + os.environ["API_DOMAIN"] = API_DOMAIN + + +def load_secret_file(file: str): + secret_file_path = Path(file) + if not secret_file_path.exists(): + raise FileNotFoundError("Secret File Doesn't Exists") + + load_dotenv(dotenv_path=secret_file_path) + + +def fetch_env_variables(): + for env in ENV_VARIABLES: + value = os.environ[env] + if not is_force_base64_fields(env) and is_validate_base64(value): + os.environ[env] = value + else: + value = value.encode("utf-8") + os.environ[env] = b64encode(value).decode() + + +def envsubst_file(file: PosixPath): + with open(file) as f: + formated_file = envsubst(f.read()) + + new_file = Path("deployment")\ + .joinpath(*[part.split('.')[0] for part in file.parts if part != "template"])\ + .with_suffix(".yaml") + + with open(new_file, 'w') as f: + f.write(formated_file) + + +def substitute_secrets_from_templates(): + for subdir in Path("template").glob("*"): + for file in subdir.glob("*.yaml"): + envsubst_file(file) + + +def main(file, environment): + setting_environment(environment) + + load_secret_file(file) + + fetch_env_variables() + + substitute_secrets_from_templates() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(prog="Setup") + parser.add_argument( + "-f", "--file", + dest="file", + default=".env", + help="Secret file [default = .secret]" + ) + parser.add_argument( + "-e", "--environment", + dest="environment", + default="prod", + help="Selected Deployment Environment [default = prod, options = [prod, staging, dev]]" + ) + + args = parser.parse_args() + + main(**vars(args)) \ No newline at end of file diff --git a/setup.sh b/setup.sh deleted file mode 100755 index dc0742d..0000000 --- a/setup.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - - -TEMPLATE='./template' -WORK_DIR='./deployment' - - -function set_working_dir { - mkdir -p $WORK_DIR; - mkdir -p $WORK_DIR/redis; - mkdir -p $WORK_DIR/postgres; - mkdir -p $WORK_DIR/frontend; - mkdir -p $WORK_DIR/backend; - - envsubst < $TEMPLATE/redis/redis-secret.template.yaml > $WORK_DIR/redis/redis-secret.yaml; - envsubst < $TEMPLATE/postgres/postgres-secret.template.yaml > $WORK_DIR/postgres/postgres-secret.yaml; - envsubst < $TEMPLATE/frontend/frontend-secret.template.yaml > $WORK_DIR/frontend/frontend-secret.yaml; - envsubst < $TEMPLATE/backend/backend-secret.template.yaml > $WORK_DIR/backend/backend-secret.yaml; - envsubst < $TEMPLATE/nginx-ingress/nginx-ingress-api.yaml > $WORK_DIR/nginx-ingress/nginx-ingress-api.yaml; - envsubst < $TEMPLATE/nginx-ingress/nginx-ingress-root.yaml > $WORK_DIR/nginx-ingress/nginx-ingress-root.yaml; - envsubst < $TEMPLATE/cert-manager/cert-manager-certificate.template.yaml > $WORK_DIR/cert-manager/cert-manager-certificate.yaml; -} - - -function main { - if [[ $1 == "--prod" || $1 == "-a" ]]; then - DOMAIN="hideyoshi.com.br" - API_DOMAIN="api.hideyoshi.com.br" - elif [[ $1 == "--staging" || $1 == "-a" ]]; then - DOMAIN="staging.hideyoshi.com.br" - API_DOMAIN="api.staging.hideyoshi.com.br" - elif [[ $1 == "--dev" || $1 == "-d" ]]; then - DOMAIN="hideyoshi.com.br" - API_DOMAIN="api.hideyoshi.com.br" - fi - - [[ -z $2 ]] && secret_file=".secret" || secret_file=$2 - - if [[ -e $secret_file ]]; then - - set -a - while read line; do - if [[ $line != "" ]]; then - variable=$(echo -n "$line" | cut -f 1 -d '=') - value=$(echo -n $(echo -n "$line" | cut -f 2- -d '=') | base64 -w 0) - declare "$variable=$value" - fi - done < $secret_file - set +a - - else - - echo "ERROR: Secret file not found."; - exit 1; - - fi - - set_working_dir -} - - -main $@