Merge pull request #20 from HideyoshiNakazone/staging

Better Deployment Implementation

.github/workflows/deploy-prod.yml

@@ -3,7 +3,7 @@ name: remote ssh command
 on:
     push:
         branches:
-            - "main"
+            - main
 
 jobs:
     build:
@@ -11,34 +11,41 @@ jobs:
         environment: prod
         runs-on: ubuntu-latest
         steps:
-            - name: checkout
+            - uses: actions/checkout@v3
-              uses: actions/checkout@v2
+            - name: Set up Python 3.8
+              uses: actions/setup-python@v3
+              with:
+                python-version: "3.10"
 
             - name: Make Env File
               uses: SpicyPizza/create-envfile@v2.0
               with:
-                envkey_FRONTEND_PATH: ${{ secrets.FRONTEND_PATH }}
+                  envkey_BACKEND_OAUTH_URL: ${{ secrets.BACKEND_OAUTH_URL }}
-                envkey_TOKEN_SECRET: ${{ secrets.TOKEN_SECRET }}
+                  envkey_BACKEND_URL: ${{ secrets.BACKEND_URL }}
-                envkey_ACCESS_TOKEN_DURATION: ${{ secrets.ACCESS_TOKEN_DURATION }}
+                  envkey_FRONTEND_PATH: ${{ secrets.FRONTEND_PATH }}
-                envkey_REFRESH_TOKEN_DURATION: ${{ secrets.REFRESH_TOKEN_DURATION }}
+                  envkey_GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
-                envkey_DEFAULT_USER_FULLNAME: ${{ secrets.DEFAULT_USER_FULLNAME }}
+                  envkey_GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
-                envkey_DEFAULT_USER_EMAIL: ${{ secrets.DEFAULT_USER_EMAIL }}
+                  envkey_GOOGLE_REDIRECT_URL: ${{ secrets.GOOGLE_REDIRECT_URL }}
-                envkey_DEFAULT_USER_USERNAME: ${{ secrets.DEFAULT_USER_USERNAME }}
+                  envkey_OAUTH_GITHUB_CLIENT_ID: ${{ secrets.OAUTH_GITHUB_CLIENT_ID }}
-                envkey_DEFAULT_USER_PASSWORD: ${{ secrets.DEFAULT_USER_PASSWORD }}
+                  envkey_OAUTH_GITHUB_CLIENT_SECRET: ${{ secrets.OAUTH_GITHUB_CLIENT_SECRET }}
-                envkey_GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
+                  envkey_OAUTH_GITHUB_REDIRECT_URL: ${{ secrets.OAUTH_GITHUB_REDIRECT_URL }}
-                envkey_GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
+                  envkey_ACCESS_TOKEN_DURATION: ${{ secrets.ACCESS_TOKEN_DURATION}}
-                envkey_GOOGLE_REDIRECT_URL: ${{ secrets.GOOGLE_REDIRECT_URL }}
+                  envkey_DEFAULT_USER_EMAIL: ${{ secrets.DEFAULT_USER_EMAIL}}
-                envkey_OAUTH_GITHUB_CLIENT_ID: ${{ secrets.OAUTH_GITHUB_CLIENT_ID }}
+                  envkey_DEFAULT_USER_FULLNAME: ${{ secrets.DEFAULT_USER_FULLNAME}}
-                envkey_OAUTH_GITHUB_CLIENT_SECRET: ${{ secrets.OAUTH_GITHUB_CLIENT_SECRET }}
+                  envkey_DEFAULT_USER_PASSWORD: ${{ secrets.DEFAULT_USER_PASSWORD}}
-                envkey_OAUTH_GITHUB_REDIRECT_URL: ${{ secrets.OAUTH_GITHUB_REDIRECT_URL }}
+                  envkey_DEFAULT_USER_USERNAME: ${{ secrets.DEFAULT_USER_USERNAME}}
-                envkey_POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+                  envkey_POSTGRES_DB: ${{ secrets.POSTGRES_DB}}
-                envkey_POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+                  envkey_POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD}}
-                envkey_POSTGRES_DB: ${{ secrets.POSTGRES_DB }}
+                  envkey_POSTGRES_USER: ${{ secrets.POSTGRES_USER}}
-                envkey_REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
+                  envkey_REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD}}
+                  envkey_REFRESH_TOKEN_DURATION: ${{ secrets.REFRESH_TOKEN_DURATION}}
+                  envkey_TOKEN_SECRET: ${{ secrets.TOKEN_SECRET}}
 
             - name: Inserts Prod Enviromental Variables
               run: |
-                ./setup.sh --prod .env
+                python -m pip install --upgrade pip pipenv
+                pipenv install
+                pipenv run python setup.py -e prod -f .env
 
             - name: copy file via ssh
               uses: appleboy/scp-action@master
@@ -59,4 +66,4 @@ jobs:
                 key: ${{ secrets.SSH_KEY }}
                 script: |
                   cd infra-hideyoshi.com
-                  ./deploy.sh
+                  ./deploy.sh --prod

.github/workflows/deploy-staging.yml

@@ -11,34 +11,41 @@ jobs:
         environment: staging
         runs-on: ubuntu-latest
         steps:
-            - name: checkout
+            - uses: actions/checkout@v3
-              uses: actions/checkout@v2
+            - name: Set up Python 3.8
+              uses: actions/setup-python@v3
+              with:
+                python-version: "3.10"
 
             - name: Make Env File
               uses: SpicyPizza/create-envfile@v2.0
               with:
-                envkey_FRONTEND_PATH: ${{ secrets.FRONTEND_PATH }}
+                  envkey_BACKEND_OAUTH_URL: ${{ secrets.BACKEND_OAUTH_URL }}
-                envkey_TOKEN_SECRET: ${{ secrets.TOKEN_SECRET }}
+                  envkey_BACKEND_URL: ${{ secrets.BACKEND_URL }}
-                envkey_ACCESS_TOKEN_DURATION: ${{ secrets.ACCESS_TOKEN_DURATION }}
+                  envkey_FRONTEND_PATH: ${{ secrets.FRONTEND_PATH }}
-                envkey_REFRESH_TOKEN_DURATION: ${{ secrets.REFRESH_TOKEN_DURATION }}
+                  envkey_GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
-                envkey_DEFAULT_USER_FULLNAME: ${{ secrets.DEFAULT_USER_FULLNAME }}
+                  envkey_GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
-                envkey_DEFAULT_USER_EMAIL: ${{ secrets.DEFAULT_USER_EMAIL }}
+                  envkey_GOOGLE_REDIRECT_URL: ${{ secrets.GOOGLE_REDIRECT_URL }}
-                envkey_DEFAULT_USER_USERNAME: ${{ secrets.DEFAULT_USER_USERNAME }}
+                  envkey_OAUTH_GITHUB_CLIENT_ID: ${{ secrets.OAUTH_GITHUB_CLIENT_ID }}
-                envkey_DEFAULT_USER_PASSWORD: ${{ secrets.DEFAULT_USER_PASSWORD }}
+                  envkey_OAUTH_GITHUB_CLIENT_SECRET: ${{ secrets.OAUTH_GITHUB_CLIENT_SECRET }}
-                envkey_GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
+                  envkey_OAUTH_GITHUB_REDIRECT_URL: ${{ secrets.OAUTH_GITHUB_REDIRECT_URL }}
-                envkey_GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
+                  envkey_ACCESS_TOKEN_DURATION: ${{ secrets.ACCESS_TOKEN_DURATION}}
-                envkey_GOOGLE_REDIRECT_URL: ${{ secrets.GOOGLE_REDIRECT_URL }}
+                  envkey_DEFAULT_USER_EMAIL: ${{ secrets.DEFAULT_USER_EMAIL}}
-                envkey_OAUTH_GITHUB_CLIENT_ID: ${{ secrets.OAUTH_GITHUB_CLIENT_ID }}
+                  envkey_DEFAULT_USER_FULLNAME: ${{ secrets.DEFAULT_USER_FULLNAME}}
-                envkey_OAUTH_GITHUB_CLIENT_SECRET: ${{ secrets.OAUTH_GITHUB_CLIENT_SECRET }}
+                  envkey_DEFAULT_USER_PASSWORD: ${{ secrets.DEFAULT_USER_PASSWORD}}
-                envkey_OAUTH_GITHUB_REDIRECT_URL: ${{ secrets.OAUTH_GITHUB_REDIRECT_URL }}
+                  envkey_DEFAULT_USER_USERNAME: ${{ secrets.DEFAULT_USER_USERNAME}}
-                envkey_POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+                  envkey_POSTGRES_DB: ${{ secrets.POSTGRES_DB}}
-                envkey_POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+                  envkey_POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD}}
-                envkey_POSTGRES_DB: ${{ secrets.POSTGRES_DB }}
+                  envkey_POSTGRES_USER: ${{ secrets.POSTGRES_USER}}
-                envkey_REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
+                  envkey_REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD}}
+                  envkey_REFRESH_TOKEN_DURATION: ${{ secrets.REFRESH_TOKEN_DURATION}}
+                  envkey_TOKEN_SECRET: ${{ secrets.TOKEN_SECRET}}
 
             - name: Inserts Prod Enviromental Variables
               run: |
-                ./setup.sh --staging .env
+                python -m pip install --upgrade pip pipenv
+                pipenv install
+                pipenv run python setup.py -e staging -f .env
 
             - name: copy file via ssh
               uses: appleboy/scp-action@master

Pipfile

@@ -0,0 +1,13 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+python-dotenv = "*"
+envsubst = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3"

Pipfile.lock

@@ -0,0 +1,37 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "9127f5121153ee714035c045f3621c810565d451e8dd461576f24e165e55f73d"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "envsubst": {
+            "hashes": [
+                "sha256:7188f9f2a046e45b63098ce5a7fd84126cbe9e5b73b8ff78eaf6e32122c0caaf",
+                "sha256:d8e402984a84dda4ea7a8a1f7afe1c41e54a1257cfb74c80cb8f991053d97b9b"
+            ],
+            "index": "pypi",
+            "version": "==0.1.5"
+        },
+        "python-dotenv": {
+            "hashes": [
+                "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba",
+                "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"
+            ],
+            "index": "pypi",
+            "version": "==1.0.0"
+        }
+    },
+    "develop": {}
+}

setup.py

@@ -0,0 +1,135 @@
+from base64 import b64decode, b64encode
+from dotenv import load_dotenv
+from envsubst import envsubst
+from pathlib import Path, PosixPath
+import argparse
+import os
+
+
+ENV_VARIABLES = [
+    "FRONTEND_PATH",
+    "BACKEND_URL",
+    "BACKEND_OAUTH_URL",
+    "TOKEN_SECRET",
+    "ACCESS_TOKEN_DURATION",
+    "REFRESH_TOKEN_DURATION",
+    "DEFAULT_USER_FULLNAME",
+    "DEFAULT_USER_EMAIL",
+    "DEFAULT_USER_USERNAME",
+    "DEFAULT_USER_PASSWORD",
+    "GOOGLE_CLIENT_ID",
+    "GOOGLE_CLIENT_SECRET",
+    "GOOGLE_REDIRECT_URL",
+    "OAUTH_GITHUB_CLIENT_ID",
+    "OAUTH_GITHUB_CLIENT_SECRET",
+    "OAUTH_GITHUB_REDIRECT_URL",
+    "POSTGRES_USER",
+    "POSTGRES_PASSWORD",
+    "POSTGRES_DB",
+    "REDIS_PASSWORD",
+]
+
+
+FORCE_BASE64_FIELD = [
+    "OAUTH_GITHUB_CLIENT_ID",
+    "OAUTH_GITHUB_CLIENT_SECRET"
+]
+
+
+def is_force_base64_fields(field: str) -> bool:
+    return field in FORCE_BASE64_FIELD
+
+
+def is_validate_base64(value: str) -> bool:
+    if not isinstance(value, str):
+        return False
+
+    try:
+        if b64encode(b64decode(value)).decode() == value:
+            return True
+    except:
+        pass
+
+    return False
+
+
+def setting_environment(environment: str):
+    if not environment in ("prod", "staging", "dev"):
+        raise ValueError("Invalid Environment Selected")
+
+    match environment:
+        case "staging":
+            DOMAIN="staging.hideyoshi.com.br"
+            API_DOMAIN="api.staging.hideyoshi.com.br"
+        case _:
+            DOMAIN="hideyoshi.com.br"
+            API_DOMAIN="api.hideyoshi.com.br"
+
+    os.environ["DOMAIN"] = DOMAIN
+    os.environ["API_DOMAIN"] = API_DOMAIN
+
+
+def load_secret_file(file: str):
+    secret_file_path = Path(file)
+    if not secret_file_path.exists():
+        raise FileNotFoundError("Secret File Doesn't Exists")
+
+    load_dotenv(dotenv_path=secret_file_path)
+
+
+def fetch_env_variables():
+    for env in ENV_VARIABLES:
+        value = os.environ[env]
+        if not is_force_base64_fields(env) and is_validate_base64(value):
+            os.environ[env] = value
+        else:
+            value = value.encode("utf-8")
+            os.environ[env] = b64encode(value).decode()
+
+
+def envsubst_file(file: PosixPath):
+    with open(file) as f:
+        formated_file = envsubst(f.read())
+
+    new_file = Path("deployment")\
+        .joinpath(*[part.split('.')[0] for part in file.parts if part != "template"])\
+        .with_suffix(".yaml")
+
+    with open(new_file, 'w') as f:
+        f.write(formated_file)
+
+
+def substitute_secrets_from_templates():
+    for subdir in Path("template").glob("*"):
+        for file in subdir.glob("*.yaml"):
+            envsubst_file(file)
+
+
+def main(file, environment):
+    setting_environment(environment)
+
+    load_secret_file(file)
+
+    fetch_env_variables()
+
+    substitute_secrets_from_templates()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog="Setup")
+    parser.add_argument(
+        "-f", "--file",
+        dest="file",
+        default=".env",
+        help="Secret file [default = .secret]"
+    )
+    parser.add_argument(
+        "-e", "--environment",
+        dest="environment",
+        default="prod",
+        help="Selected Deployment Environment [default = prod, options = [prod, staging, dev]]"
+    )
+
+    args = parser.parse_args()
+
+    main(**vars(args))

setup.sh

@@ -1,62 +0,0 @@
-#!/bin/bash
-
-
-TEMPLATE='./template'
-WORK_DIR='./deployment'
-
-
-function set_working_dir {
-    mkdir -p $WORK_DIR;
-    mkdir -p $WORK_DIR/redis;
-    mkdir -p $WORK_DIR/postgres;
-    mkdir -p $WORK_DIR/frontend;
-    mkdir -p $WORK_DIR/backend;
-
-    envsubst < $TEMPLATE/redis/redis-secret.template.yaml > $WORK_DIR/redis/redis-secret.yaml;
-    envsubst < $TEMPLATE/postgres/postgres-secret.template.yaml > $WORK_DIR/postgres/postgres-secret.yaml;
-    envsubst < $TEMPLATE/frontend/frontend-secret.template.yaml > $WORK_DIR/frontend/frontend-secret.yaml;
-    envsubst < $TEMPLATE/backend/backend-secret.template.yaml > $WORK_DIR/backend/backend-secret.yaml;
-    envsubst < $TEMPLATE/nginx-ingress/nginx-ingress-api.yaml > $WORK_DIR/nginx-ingress/nginx-ingress-api.yaml;
-    envsubst < $TEMPLATE/nginx-ingress/nginx-ingress-root.yaml > $WORK_DIR/nginx-ingress/nginx-ingress-root.yaml;
-    envsubst < $TEMPLATE/cert-manager/cert-manager-certificate.template.yaml > $WORK_DIR/cert-manager/cert-manager-certificate.yaml;
-}
-
-
-function main {
-    if [[ $1 == "--prod" || $1 == "-a" ]]; then
-        DOMAIN="hideyoshi.com.br"
-        API_DOMAIN="api.hideyoshi.com.br"
-    elif [[ $1 == "--staging" || $1 == "-a" ]]; then
-        DOMAIN="staging.hideyoshi.com.br"
-        API_DOMAIN="api.staging.hideyoshi.com.br"
-    elif [[ $1 == "--dev" || $1 == "-d" ]]; then
-        DOMAIN="hideyoshi.com.br"
-        API_DOMAIN="api.hideyoshi.com.br"
-    fi
-    
-    [[ -z $2 ]] && secret_file=".secret" || secret_file=$2
-
-    if [[ -e $secret_file ]]; then
-
-        set -a
-        while read line; do
-            if [[ $line != "" ]]; then
-                variable=$(echo -n "$line" | cut -f 1 -d '=')
-                value=$(echo -n $(echo -n "$line" | cut -f 2- -d '=') | base64 -w 0)
-                declare "$variable=$value"
-            fi
-        done < $secret_file
-        set +a
-
-    else 
-
-        echo "ERROR: Secret file not found.";
-        exit 1;
-
-    fi
-
-    set_working_dir
-}
-
-
-main $@