From e5e14d7ce696fbe69a4e66d857cbd875dbf6379c Mon Sep 17 00:00:00 2001 From: Vitor Hideyoshi Date: Wed, 23 Aug 2023 10:17:28 -0300 Subject: [PATCH 1/6] Adds Storage Service --- .gitignore | 2 + deploy.sh | 6 ++ deployment/backend/backend.yaml | 6 ++ deployment/storage/storage-config.yaml | 8 ++ deployment/storage/storage.yaml | 102 ++++++++++++++++++ setup.py | 5 + template/storage/storage-secret.template.yaml | 12 +++ 7 files changed, 141 insertions(+) create mode 100644 deployment/storage/storage-config.yaml create mode 100644 deployment/storage/storage.yaml create mode 100644 template/storage/storage-secret.template.yaml diff --git a/.gitignore b/.gitignore index 8e8cad2..0b4e79b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ .vscode/ +**/storage-secret.yaml + **/backend-secret.yaml **/frontend-secret.yaml diff --git a/deploy.sh b/deploy.sh index 30be59f..8f75173 100755 --- a/deploy.sh +++ b/deploy.sh @@ -44,6 +44,12 @@ function application_deploy() { deployment.apps/frontend-deployment \ -n portfolio; + kubectl apply -f ./deployment/storage; + kubectl wait --for=condition=available \ + --timeout=600s \ + deployment.apps/storage-deployment \ + -n portfolio; + kubectl apply -f ./deployment/backend; kubectl wait --for=condition=available \ --timeout=600s \ diff --git a/deployment/backend/backend.yaml b/deployment/backend/backend.yaml index edfedf9..91620f8 100644 --- a/deployment/backend/backend.yaml +++ b/deployment/backend/backend.yaml @@ -155,6 +155,12 @@ spec: name: redis-secret key: redis-password + - name: STORAGE_SERVICE_PATH + valueFrom: + configMapKeyRef: + name: storage-config + key: storage_url + --- apiVersion: v1 kind: Service diff --git a/deployment/storage/storage-config.yaml b/deployment/storage/storage-config.yaml new file mode 100644 index 0000000..229d64e --- /dev/null +++ b/deployment/storage/storage-config.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: portfolio + name: storage-config +data: + storage_url: storage-service + storage_port: "8000" \ No newline at end of file diff --git a/deployment/storage/storage.yaml b/deployment/storage/storage.yaml new file mode 100644 index 0000000..10ff8ed --- /dev/null +++ b/deployment/storage/storage.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: portfolio + name: storage-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: storage + template: + metadata: + labels: + app: storage + spec: + containers: + - name: storage + image: yoshiunfriendly/storage-hideyoshi.com:latest + imagePullPolicy: "Always" + ports: + - containerPort: 8000 + env: + - name: ALLOWED_ORIGINS + valueFrom: + secretKeyRef: + name: frontend-secret + key: backend_url + + - name: EXPIRES_IN + valueFrom: + secretKeyRef: + name: backend-secret + key: access_token_duration + + - name: SERVER_PORT + valueFrom: + configMapKeyRef: + name: storage-config + key: storage_port + + - name: REDIS_HOST + valueFrom: + configMapKeyRef: + name: redis-config + key: redis-url + + - name: REDIS_PORT + valueFrom: + configMapKeyRef: + name: redis-config + key: redis-port + + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + + - name: STORAGE_TYPE + valueFrom: + secretKeyRef: + name: storage-secret + key: storage_type + + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: storage-secret + key: aws_access_key_id + + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: storage-secret + key: aws_access_access_key + + - name: AWS_REGION_NAME + valueFrom: + secretKeyRef: + name: storage-secret + key: aws_region_name + + - name: AWS_BUCKET_NAME + valueFrom: + secretKeyRef: + name: storage-secret + key: aws_bucket_name + +--- +apiVersion: v1 +kind: Service +metadata: + namespace: portfolio + name: storage-service +spec: + selector: + app: storage + ports: + - port: 8000 + protocol: TCP + targetPort: 8000 + type: ClusterIP diff --git a/setup.py b/setup.py index c023d54..64f6dc5 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,11 @@ ENV_VARIABLES = [ "POSTGRES_PASSWORD", "POSTGRES_DB", "REDIS_PASSWORD", + "STORAGE_TYPE", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_REGION_NAME", + "AWS_BUCKET_NAME", ] diff --git a/template/storage/storage-secret.template.yaml b/template/storage/storage-secret.template.yaml new file mode 100644 index 0000000..4da3faf --- /dev/null +++ b/template/storage/storage-secret.template.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + namespace: portfolio + name: storage-secret +type: Opaque +data: + storage_type: $STORAGE_TYPE + aws_access_key_id: $AWS_ACCESS_KEY_ID + aws_access_access_key: $AWS_SECRET_ACCESS_KEY + aws_region_name: $AWS_REGION_NAME + aws_bucket_name: $AWS_BUCKET_NAME \ No newline at end of file From 30fcf41c77d3422d3518defd0b8e850bc7113b06 Mon Sep 17 00:00:00 2001 From: Vitor Hideyoshi Date: Wed, 23 Aug 2023 10:25:38 -0300 Subject: [PATCH 2/6] Fixes Github Actions --- .github/workflows/deploy-prod.yml | 5 +++++ .github/workflows/deploy-staging.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index fc71348..848c6e3 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -40,6 +40,11 @@ jobs: envkey_REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD}} envkey_REFRESH_TOKEN_DURATION: ${{ secrets.REFRESH_TOKEN_DURATION}} envkey_TOKEN_SECRET: ${{ secrets.TOKEN_SECRET}} + envkey_STORAGE_TYPE: ${{ secrets.STORAGE_TYPE }} + envkey_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + envkey_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + envkey_AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }} + envkey_AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} - name: Inserts Prod Enviromental Variables run: | diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 970347b..756a6a5 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -40,6 +40,11 @@ jobs: envkey_REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD}} envkey_REFRESH_TOKEN_DURATION: ${{ secrets.REFRESH_TOKEN_DURATION}} envkey_TOKEN_SECRET: ${{ secrets.TOKEN_SECRET}} + envkey_STORAGE_TYPE: ${{ secrets.STORAGE_TYPE }} + envkey_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + envkey_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + envkey_AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }} + envkey_AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} - name: Inserts Prod Enviromental Variables run: | From dcd0da0cd8f2b28b068f053fc8561005ce32c9bc Mon Sep 17 00:00:00 2001 From: Vitor Hideyoshi Date: Wed, 23 Aug 2023 10:55:27 -0300 Subject: [PATCH 3/6] Fixes MicroService Pathing --- deployment/backend/backend.yaml | 4 ++-- setup.py | 8 +++++++- template/nginx-ingress/nginx-ingress-api.yaml | 14 ++++++++++++++ template/nginx-ingress/nginx-ingress-root.yaml | 11 +++++++++++ template/storage/storage-secret.template.yaml | 1 + 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/deployment/backend/backend.yaml b/deployment/backend/backend.yaml index 91620f8..46d71d1 100644 --- a/deployment/backend/backend.yaml +++ b/deployment/backend/backend.yaml @@ -157,8 +157,8 @@ spec: - name: STORAGE_SERVICE_PATH valueFrom: - configMapKeyRef: - name: storage-config + secretKeyRef: + name: storage-secret key: storage_url --- diff --git a/setup.py b/setup.py index 64f6dc5..289aa8d 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ ENV_VARIABLES = [ "POSTGRES_PASSWORD", "POSTGRES_DB", "REDIS_PASSWORD", + "STORAGE_URL", "STORAGE_TYPE", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", @@ -37,7 +38,9 @@ ENV_VARIABLES = [ FORCE_BASE64_FIELD = [ "OAUTH_GITHUB_CLIENT_ID", - "OAUTH_GITHUB_CLIENT_SECRET" + "OAUTH_GITHUB_CLIENT_SECRET", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", ] @@ -66,12 +69,15 @@ def setting_environment(environment: str): case "staging": DOMAIN="staging.hideyoshi.com.br" API_DOMAIN="api.staging.hideyoshi.com.br" + STORAGE_API_DOMAIN="storage.staging.hideyoshi.com.br" case _: DOMAIN="hideyoshi.com.br" API_DOMAIN="api.hideyoshi.com.br" + STORAGE_API_DOMAIN="storage.hideyoshi.com.br" os.environ["DOMAIN"] = DOMAIN os.environ["API_DOMAIN"] = API_DOMAIN + os.environ["STORAGE_API_DOMAIN"] = STORAGE_API_DOMAIN def load_secret_file(file: str): diff --git a/template/nginx-ingress/nginx-ingress-api.yaml b/template/nginx-ingress/nginx-ingress-api.yaml index 1c3d214..7c94e11 100644 --- a/template/nginx-ingress/nginx-ingress-api.yaml +++ b/template/nginx-ingress/nginx-ingress-api.yaml @@ -16,6 +16,13 @@ spec: - host: ${DOMAIN} http: paths: + - path: /api/storage + pathType: Prefix + backend: + service: + name: storage-service + port: + number: 8000 - path: /api(/|$)(.*) pathType: Prefix backend: @@ -25,6 +32,13 @@ spec: number: 8070 - http: paths: + - path: /api/storage + pathType: Prefix + backend: + service: + name: storage-service + port: + number: 8000 - path: /api(/|$)(.*) pathType: Prefix backend: diff --git a/template/nginx-ingress/nginx-ingress-root.yaml b/template/nginx-ingress/nginx-ingress-root.yaml index e752f56..243089e 100644 --- a/template/nginx-ingress/nginx-ingress-root.yaml +++ b/template/nginx-ingress/nginx-ingress-root.yaml @@ -12,6 +12,7 @@ spec: - hosts: - ${DOMAIN} - ${API_DOMAIN} + - ${STORAGE_API_DOMAIN} secretName: letsencrypt-cluster-certificate-tls rules: - host: ${DOMAIN} @@ -34,6 +35,16 @@ spec: name: backend-service port: number: 8070 + - host: ${STORAGE_API_DOMAIN} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: storage-service + port: + number: 8000 - http: paths: - path: / diff --git a/template/storage/storage-secret.template.yaml b/template/storage/storage-secret.template.yaml index 4da3faf..0d3c07d 100644 --- a/template/storage/storage-secret.template.yaml +++ b/template/storage/storage-secret.template.yaml @@ -5,6 +5,7 @@ metadata: name: storage-secret type: Opaque data: + storage_url: $STORAGE_URL storage_type: $STORAGE_TYPE aws_access_key_id: $AWS_ACCESS_KEY_ID aws_access_access_key: $AWS_SECRET_ACCESS_KEY From 271af3fdaa8b733dd9cef75d852e412c66d1f089 Mon Sep 17 00:00:00 2001 From: Vitor Hideyoshi Date: Wed, 23 Aug 2023 10:58:37 -0300 Subject: [PATCH 4/6] Fixes Github Actions --- .github/workflows/deploy-staging.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 756a6a5..c0d2b3c 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -40,6 +40,7 @@ jobs: envkey_REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD}} envkey_REFRESH_TOKEN_DURATION: ${{ secrets.REFRESH_TOKEN_DURATION}} envkey_TOKEN_SECRET: ${{ secrets.TOKEN_SECRET}} + envkey_STORAGE_URL: ${{ secrets.STORAGE_URL }} envkey_STORAGE_TYPE: ${{ secrets.STORAGE_TYPE }} envkey_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} envkey_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From e4b637e07e878357d6f451d745339e6ca019b5b7 Mon Sep 17 00:00:00 2001 From: Vitor Hideyoshi Date: Wed, 23 Aug 2023 11:01:45 -0300 Subject: [PATCH 5/6] Adds MicroService API do SSL Certificate --- template/cert-manager/cert-manager-certificate.template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/template/cert-manager/cert-manager-certificate.template.yaml b/template/cert-manager/cert-manager-certificate.template.yaml index 09d5a1b..a5b05e7 100644 --- a/template/cert-manager/cert-manager-certificate.template.yaml +++ b/template/cert-manager/cert-manager-certificate.template.yaml @@ -7,6 +7,7 @@ spec: dnsNames: - ${DOMAIN} - ${API_DOMAIN} + - ${STORAGE_API_DOMAIN} secretName: letsencrypt-cluster-certificate-tls issuerRef: name: cluster-certificate-issuer From 338e33401a282ae6c83d7355663d564d4932efd6 Mon Sep 17 00:00:00 2001 From: Vitor Hideyoshi Date: Wed, 23 Aug 2023 18:48:11 -0300 Subject: [PATCH 6/6] Fixes Storage Service Integration --- deploy.sh | 7 +++++++ deployment/backend/backend.yaml | 17 +++++++++++++---- setup.py | 19 +++++++------------ .../cert-manager-certificate.template.yaml | 1 - template/nginx-ingress/nginx-ingress-api.yaml | 14 -------------- .../nginx-ingress/nginx-ingress-root.yaml | 11 ----------- template/storage/storage-secret.template.yaml | 1 - 7 files changed, 27 insertions(+), 43 deletions(-) diff --git a/deploy.sh b/deploy.sh index 8f75173..786940a 100755 --- a/deploy.sh +++ b/deploy.sh @@ -23,6 +23,13 @@ function application_deploy() { kubectl apply -f ./deployment/portfolio-namespace.yaml; + + kubectl apply -f ./deployment/postgres/postgres-secret.yaml; + kubectl apply -f ./deployment/redis/redis-secret.yaml; + kubectl apply -f ./deployment/storage/storage-secret.yaml; + kubectl apply -f ./deployment/backend/backend-secret.yaml; + kubectl apply -f ./deployment/frontend/frontend-secret.yaml; + kubectl apply -f \ ./deployment/cert-manager/cert-manager-certificate.yaml; diff --git a/deployment/backend/backend.yaml b/deployment/backend/backend.yaml index 46d71d1..7f4662a 100644 --- a/deployment/backend/backend.yaml +++ b/deployment/backend/backend.yaml @@ -16,7 +16,7 @@ spec: containers: - name: backend image: yoshiunfriendly/backend-hideyoshi.com - imagePullPolicy: "Always" + imagePullPolicy: Always ports: - containerPort: 8070 env: @@ -155,12 +155,21 @@ spec: name: redis-secret key: redis-password - - name: STORAGE_SERVICE_PATH + - name: STORAGE_SERVICE_URL valueFrom: - secretKeyRef: - name: storage-secret + configMapKeyRef: + name: storage-config key: storage_url + - name: STORAGE_SERVICE_PORT + valueFrom: + configMapKeyRef: + name: storage-config + key: storage_port + + - name: STORAGE_SERVICE_PATH + value: "http://$(STORAGE_SERVICE_URL):$(STORAGE_SERVICE_PORT)" + --- apiVersion: v1 kind: Service diff --git a/setup.py b/setup.py index 289aa8d..02676e5 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,6 @@ from pathlib import Path, PosixPath import argparse import os - ENV_VARIABLES = [ "FRONTEND_PATH", "BACKEND_URL", @@ -35,7 +34,6 @@ ENV_VARIABLES = [ "AWS_BUCKET_NAME", ] - FORCE_BASE64_FIELD = [ "OAUTH_GITHUB_CLIENT_ID", "OAUTH_GITHUB_CLIENT_SECRET", @@ -67,17 +65,14 @@ def setting_environment(environment: str): match environment: case "staging": - DOMAIN="staging.hideyoshi.com.br" - API_DOMAIN="api.staging.hideyoshi.com.br" - STORAGE_API_DOMAIN="storage.staging.hideyoshi.com.br" + DOMAIN = "staging.hideyoshi.com.br" + API_DOMAIN = "api.staging.hideyoshi.com.br" case _: - DOMAIN="hideyoshi.com.br" - API_DOMAIN="api.hideyoshi.com.br" - STORAGE_API_DOMAIN="storage.hideyoshi.com.br" + DOMAIN = "hideyoshi.com.br" + API_DOMAIN = "api.hideyoshi.com.br" os.environ["DOMAIN"] = DOMAIN os.environ["API_DOMAIN"] = API_DOMAIN - os.environ["STORAGE_API_DOMAIN"] = STORAGE_API_DOMAIN def load_secret_file(file: str): @@ -102,8 +97,8 @@ def envsubst_file(file: PosixPath): with open(file) as f: formated_file = envsubst(f.read()) - new_file = Path("deployment")\ - .joinpath(*[part.split('.')[0] for part in file.parts if part != "template"])\ + new_file = Path("deployment") \ + .joinpath(*[part.split('.')[0] for part in file.parts if part != "template"]) \ .with_suffix(".yaml") with open(new_file, 'w') as f: @@ -143,4 +138,4 @@ if __name__ == "__main__": args = parser.parse_args() - main(**vars(args)) \ No newline at end of file + main(**vars(args)) diff --git a/template/cert-manager/cert-manager-certificate.template.yaml b/template/cert-manager/cert-manager-certificate.template.yaml index a5b05e7..09d5a1b 100644 --- a/template/cert-manager/cert-manager-certificate.template.yaml +++ b/template/cert-manager/cert-manager-certificate.template.yaml @@ -7,7 +7,6 @@ spec: dnsNames: - ${DOMAIN} - ${API_DOMAIN} - - ${STORAGE_API_DOMAIN} secretName: letsencrypt-cluster-certificate-tls issuerRef: name: cluster-certificate-issuer diff --git a/template/nginx-ingress/nginx-ingress-api.yaml b/template/nginx-ingress/nginx-ingress-api.yaml index 7c94e11..1c3d214 100644 --- a/template/nginx-ingress/nginx-ingress-api.yaml +++ b/template/nginx-ingress/nginx-ingress-api.yaml @@ -16,13 +16,6 @@ spec: - host: ${DOMAIN} http: paths: - - path: /api/storage - pathType: Prefix - backend: - service: - name: storage-service - port: - number: 8000 - path: /api(/|$)(.*) pathType: Prefix backend: @@ -32,13 +25,6 @@ spec: number: 8070 - http: paths: - - path: /api/storage - pathType: Prefix - backend: - service: - name: storage-service - port: - number: 8000 - path: /api(/|$)(.*) pathType: Prefix backend: diff --git a/template/nginx-ingress/nginx-ingress-root.yaml b/template/nginx-ingress/nginx-ingress-root.yaml index 243089e..e752f56 100644 --- a/template/nginx-ingress/nginx-ingress-root.yaml +++ b/template/nginx-ingress/nginx-ingress-root.yaml @@ -12,7 +12,6 @@ spec: - hosts: - ${DOMAIN} - ${API_DOMAIN} - - ${STORAGE_API_DOMAIN} secretName: letsencrypt-cluster-certificate-tls rules: - host: ${DOMAIN} @@ -35,16 +34,6 @@ spec: name: backend-service port: number: 8070 - - host: ${STORAGE_API_DOMAIN} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: storage-service - port: - number: 8000 - http: paths: - path: / diff --git a/template/storage/storage-secret.template.yaml b/template/storage/storage-secret.template.yaml index 0d3c07d..4da3faf 100644 --- a/template/storage/storage-secret.template.yaml +++ b/template/storage/storage-secret.template.yaml @@ -5,7 +5,6 @@ metadata: name: storage-secret type: Opaque data: - storage_url: $STORAGE_URL storage_type: $STORAGE_TYPE aws_access_key_id: $AWS_ACCESS_KEY_ID aws_access_access_key: $AWS_SECRET_ACCESS_KEY