feat: implements a better kubernetes deployment setup
This commit is contained in:
4
.gitignore
vendored
4
.gitignore
vendored
@@ -4,3 +4,7 @@
|
|||||||
.idea
|
.idea
|
||||||
|
|
||||||
**/__pycache__/
|
**/__pycache__/
|
||||||
|
|
||||||
|
.k8s/*.yml
|
||||||
|
!.k8s/*.template.yml
|
||||||
|
!.k8s/*.enc.yml
|
||||||
5
.k8s/.sops.yaml
Normal file
5
.k8s/.sops.yaml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
creation_rules:
|
||||||
|
- path_regex: ^secrets(\.enc)?\.yml$
|
||||||
|
encrypted_regex: '^(data|stringData)$'
|
||||||
|
pgp: >-
|
||||||
|
8C8D94A7639C87559B0F2F64B7E1F62F69798EB6
|
||||||
12
.k8s/config.template.yml
Normal file
12
.k8s/config.template.yml
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: storage-config
|
||||||
|
namespace: ${KUBE_NAMESPACE}
|
||||||
|
data:
|
||||||
|
SERVER_PORT: "8000"
|
||||||
|
|
||||||
|
REDIS_HOST: "storage-redis-service"
|
||||||
|
REDIS_PORT: "6379"
|
||||||
|
|
||||||
|
EXPIRES_IN: "1800000"
|
||||||
63
.k8s/deployment.template.yml
Normal file
63
.k8s/deployment.template.yml
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: storage-deployment
|
||||||
|
namespace: ${KUBE_NAMESPACE}
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: storage
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: storage
|
||||||
|
spec:
|
||||||
|
nodeSelector:
|
||||||
|
${WORKER_NODE_LABEL}
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: ghcr-secret
|
||||||
|
containers:
|
||||||
|
- name: storage
|
||||||
|
image: ${IMAGE_BASE}:${IMAGE_TAG}
|
||||||
|
imagePullPolicy: Always
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: "256Mi"
|
||||||
|
cpu: "250m"
|
||||||
|
limits:
|
||||||
|
memory: "256Mi"
|
||||||
|
cpu: "1000m"
|
||||||
|
ports:
|
||||||
|
- containerPort: 8000
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 8000
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 8000
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: storage-config
|
||||||
|
- secretRef:
|
||||||
|
name: storage-secret
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
namespace: ${KUBE_NAMESPACE}
|
||||||
|
name: storage-service
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: backend
|
||||||
|
ports:
|
||||||
|
- port: 8000
|
||||||
|
protocol: TCP
|
||||||
|
targetPort: 8000
|
||||||
|
type: ClusterIP
|
||||||
47
.k8s/redis.template.yml
Normal file
47
.k8s/redis.template.yml
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
namespace: ${KUBE_NAMESPACE}
|
||||||
|
name: storage-redis-deployment
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: storage-redis
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: storage-redis
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: redis
|
||||||
|
image: valkey/valkey:8.0.6-alpine
|
||||||
|
imagePullPolicy: "IfNotPresent"
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: "256Mi"
|
||||||
|
cpu: "75m"
|
||||||
|
limits:
|
||||||
|
memory: "256Mi"
|
||||||
|
cpu: "256m"
|
||||||
|
ports:
|
||||||
|
- containerPort: 6379
|
||||||
|
env:
|
||||||
|
- name: VALKEY_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: storage-secret
|
||||||
|
key: REDIS_PASSWORD
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
namespace: ${KUBE_NAMESPACE}
|
||||||
|
name: storage-redis-service
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: storage-redis
|
||||||
|
ports:
|
||||||
|
- port: 6379
|
||||||
|
type: ClusterIP
|
||||||
38
.k8s/secrets.enc.yml
Normal file
38
.k8s/secrets.enc.yml
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: storage-secret
|
||||||
|
namespace: ${KUBE_NAMESPACE}
|
||||||
|
data:
|
||||||
|
REDIS_PASSWORD: ENC[AES256_GCM,data:QAjxHjGqOIun8jTN,iv:tfOsGEP0+kGfqvL90Ek6ZNOcueUFnYpYGE6f7r2EvoM=,tag:lAxPPRGE2dO//3hLFItOnA==,type:str]
|
||||||
|
AWS_ACCESS_KEY_ID: ENC[AES256_GCM,data:AtT8rcJxkxgWQ7SEPj5UEVvR+1b/CJt1Rc2zZQ==,iv:862ChLY0MkOwj+HG/sI+NqWFeK1mz6L3m8mWSKDqH4E=,tag:jpRNIXo7jMTYX/BAQG/7aQ==,type:str]
|
||||||
|
AWS_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:vudblTMIDdRZiAmJeYqYtCXd4Jdv++QnvdIUH8PX3IJceW/vES2UybfcKwtKGJgMt6ZPnskOamM=,iv:JHbcu/Tv81d9NWPqPXcFZBSTmrml8xBEU4+G/pVgjGM=,tag:HnAf3tGcmv3CBUOA9px/rw==,type:str]
|
||||||
|
AWS_REGION_NAME: ENC[AES256_GCM,data:GfwdCPgAe7vAtrZPYRp0Ng==,iv:RkZ69hQkuobcOL9DSfhcmLIKsK0rnevHTWF+0Mbw7Dg=,tag:0XAm2oXhETKeUbuN2UPgyA==,type:str]
|
||||||
|
AWS_BUCKET_NAME: ENC[AES256_GCM,data:atzKtMWzlco3rZf5vsFmse1zd+f3xWHUYldeF8O2egmQOmkf,iv:lYw2tM65fszSG8+MhN6edZgvGlwxw0mhOIDvp93LWnE=,tag:iNR7JJzksrFkXrg4/87E/w==,type:str]
|
||||||
|
VIRUS_CHECKER_API_KEY: ENC[AES256_GCM,data:4Ick9JIfUIIZ/LEp09kMiJM9T4FaLmXKiksoeW0BiTGJlYcps5fZFgDEYBUzEcQTGMK8SD5yPZhEbytXXAvyNFanASsVa8rXX+sO98Xr6XFJqVZEjCJbiw==,iv:5/yl9sUEF7cFqf1RKhoV0oGWTLBxTeJDxShUns8/Qsg=,tag:LvEbdmPIeWQBw6G7wnWQkg==,type:str]
|
||||||
|
sops:
|
||||||
|
lastmodified: "2025-11-08T22:29:57Z"
|
||||||
|
mac: ENC[AES256_GCM,data:s15EYKavUjMl9CNqHY6D6XHJZMSwcWhdfW6IFeGhYtjHi6SXGnx/Vpj+L96011xuDca20gU8YS/m69ML2qPV3D5wTMogXYNYGW70LCnXm4YQK+QAIJmhRYPqjlZoQpI85c698+abWuh25zER4tukSZ/ZmmNxnNGZQEyOOnaw8oU=,iv:4nMkWcFmYzBFRoiqoFLO3pR0NTb3QXxqduBqQxfQ0Sc=,tag:uwcqo8+ErSMCOuZM/eUoQA==,type:str]
|
||||||
|
pgp:
|
||||||
|
- created_at: "2025-11-08T22:29:57Z"
|
||||||
|
enc: |-
|
||||||
|
-----BEGIN PGP MESSAGE-----
|
||||||
|
|
||||||
|
hQIMAwzdivR1H/BQAQ//WSCkM5pDyRMbSSMEP8lfREjwHszaXZLF0MkNfjlDsAEC
|
||||||
|
h1yVM2+v5snJj5clipD+8FRiTl69Jho7GGjCEDfoahNyhtgr7t/UH7XVDJKJeI8W
|
||||||
|
55JrWU/T8PBvXo+Ld0c3lKqqdNQ18nWYoEMJeHyUT0MBWYmYlJbvmxKlrgLyQPs5
|
||||||
|
EUBG77aJUXIn3j0O39GEUTTmHLNITNyincZQsx4Hndgi9T7IriJfGwqQP1+9WMYw
|
||||||
|
2Hzld8Qb3VeByrVn+Ybykem6vB8IR5aULpfTU9bKvOrQh1iSgVew9bSXZcqO34Kl
|
||||||
|
OA+DpfJ8LhspPd/00zfto+2h/pwo7tAxaaJtE3NjZfAysFuSF6lX+pkPqj7FOoxh
|
||||||
|
P/ps7wzsk3F3nI/z62ITh91m/O+WhSBVsOou4xAGIatDBapH2YYy9qF/31IP9u3W
|
||||||
|
OOiXNoY1AOOu1/+c9/2ROjQ9/ENhJxiDU/Vl921+PoA+E4EGsFtgXdYiulG5TqZu
|
||||||
|
rkF9eOYw/ds/ptviA4YafFOCmk2oKxDRwhb3oNChqEFQ5hGZ/FqwcwMU+NEI8Blu
|
||||||
|
nmA/UHR/MBdxAWH/H6HMf/DJ1CNLNSgMDaEkvAA/QLPMswuoecpPLstwpOeGC/nL
|
||||||
|
qULiv7+wqlXhggT6QcmekKaw6tjOkMOA/sjqpZMpVnXMA0RpV1lPbK52Z8jINufS
|
||||||
|
XgEyDLyvaqEPLzhG7rNx1IXNBDymXKPmySvopbzjVWXao75verYNK3o/cV+OFihL
|
||||||
|
sgPoqDMMRHhLA6oepxqw9fK5f5jLOVJEc4TPbpjANZIOTIaxvir3E6avoaO3Qx8=
|
||||||
|
=7oeD
|
||||||
|
-----END PGP MESSAGE-----
|
||||||
|
fp: 8C8D94A7639C87559B0F2F64B7E1F62F69798EB6
|
||||||
|
encrypted_regex: ^(data|stringData)$
|
||||||
|
version: 3.11.0
|
||||||
@@ -1,4 +1,3 @@
|
|||||||
from storage_service.config.config_allowed_origins import get_allowed_origins
|
|
||||||
from storage_service.controller import health_router, storage_router
|
from storage_service.controller import health_router, storage_router
|
||||||
from storage_service.utils.exception_handler import (
|
from storage_service.utils.exception_handler import (
|
||||||
http_exception_handler,
|
http_exception_handler,
|
||||||
@@ -7,7 +6,6 @@ from storage_service.utils.exception_handler import (
|
|||||||
|
|
||||||
from fastapi import FastAPI, HTTPException
|
from fastapi import FastAPI, HTTPException
|
||||||
from fastapi.exceptions import RequestValidationError
|
from fastapi.exceptions import RequestValidationError
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
|
||||||
|
|
||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
|
|
||||||
@@ -15,14 +13,5 @@ app = FastAPI()
|
|||||||
app.add_exception_handler(HTTPException, http_exception_handler)
|
app.add_exception_handler(HTTPException, http_exception_handler)
|
||||||
app.add_exception_handler(RequestValidationError, validation_exception_handler)
|
app.add_exception_handler(RequestValidationError, validation_exception_handler)
|
||||||
|
|
||||||
|
|
||||||
app.add_middleware(
|
|
||||||
CORSMiddleware,
|
|
||||||
allow_origins=get_allowed_origins(),
|
|
||||||
allow_credentials=True,
|
|
||||||
allow_methods=["*"],
|
|
||||||
allow_headers=["*"],
|
|
||||||
)
|
|
||||||
|
|
||||||
app.include_router(storage_router)
|
app.include_router(storage_router)
|
||||||
app.include_router(health_router)
|
app.include_router(health_router)
|
||||||
|
|||||||
@@ -1,14 +0,0 @@
|
|||||||
from dotenv import load_dotenv
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
|
|
||||||
def get_allowed_origins():
|
|
||||||
load_dotenv()
|
|
||||||
|
|
||||||
origins = os.environ.get("ALLOWED_ORIGINS", None)
|
|
||||||
|
|
||||||
if origins is None:
|
|
||||||
return []
|
|
||||||
|
|
||||||
return origins.split(",")
|
|
||||||
@@ -1,5 +1,3 @@
|
|||||||
from storage_service.utils.enums.storage_type import StorageType
|
|
||||||
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
import os
|
import os
|
||||||
@@ -14,5 +12,5 @@ def get_config_s3():
|
|||||||
"aws_secret_access_key": os.environ.get("AWS_SECRET_ACCESS_KEY", None),
|
"aws_secret_access_key": os.environ.get("AWS_SECRET_ACCESS_KEY", None),
|
||||||
"region_name": os.environ.get("AWS_REGION_NAME", None),
|
"region_name": os.environ.get("AWS_REGION_NAME", None),
|
||||||
"bucket_name": os.environ.get("AWS_BUCKET_NAME", None),
|
"bucket_name": os.environ.get("AWS_BUCKET_NAME", None),
|
||||||
"expires_in": os.environ.get("EXPIRES_IN", 3600),
|
"expires_in": os.environ.get("EXPIRES_IN", "3600"),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from storage_service.utils.exceptions.file_not_found_exception import (
|
|||||||
from storage_service.utils.file.file_hash_generator import generate_file_hash
|
from storage_service.utils.file.file_hash_generator import generate_file_hash
|
||||||
from storage_service.worker.storage_file_worker import storage_file_worker
|
from storage_service.worker.storage_file_worker import storage_file_worker
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
from fastapi import APIRouter, Depends
|
||||||
from fastapi_utils.cbv import cbv
|
from fastapi_utils.cbv import cbv
|
||||||
from rq import Queue
|
from rq import Queue
|
||||||
|
|
||||||
|
|||||||
@@ -37,12 +37,15 @@ def build_client_s3(config: dict) -> botocore.client.BaseClient:
|
|||||||
def dependency_storage_service() -> StorageService:
|
def dependency_storage_service() -> StorageService:
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
|
|
||||||
if StorageType(os.environ["STORAGE_TYPE"]) == StorageType.S3_STORAGE:
|
storage_type = StorageType(os.environ.get("STORAGE_TYPE", "s3"))
|
||||||
s3_config = get_config_s3()
|
|
||||||
|
|
||||||
return AmazonS3Service(
|
match storage_type:
|
||||||
build_client_s3(s3_config),
|
case StorageType.S3_STORAGE:
|
||||||
s3_config["bucket_name"],
|
s3_config = get_config_s3()
|
||||||
)
|
|
||||||
|
|
||||||
raise RuntimeError("Invalid Storage Type")
|
return AmazonS3Service(
|
||||||
|
build_client_s3(s3_config),
|
||||||
|
s3_config["bucket_name"],
|
||||||
|
)
|
||||||
|
case _:
|
||||||
|
raise RuntimeError("Invalid Storage Type")
|
||||||
|
|||||||
@@ -23,12 +23,9 @@ from functools import cache
|
|||||||
def dependency_virus_checker_service() -> VirusCheckerService:
|
def dependency_virus_checker_service() -> VirusCheckerService:
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
|
|
||||||
try:
|
checker_type = VirusCheckerType(os.environ.get("VIRUS_CHECKER_TYPE", "total_virus"))
|
||||||
type = VirusCheckerType(os.environ["VIRUS_CHECKER_TYPE"])
|
|
||||||
except ValueError:
|
|
||||||
raise RuntimeError("Invalid Virus Checker Type")
|
|
||||||
|
|
||||||
match type:
|
match checker_type:
|
||||||
case VirusCheckerType.TOTAL_VIRUS:
|
case VirusCheckerType.TOTAL_VIRUS:
|
||||||
virus_checker = Virustotal(get_virus_checker_api_key())
|
virus_checker = Virustotal(get_virus_checker_api_key())
|
||||||
return VirusTotalService(virus_checker)
|
return VirusTotalService(virus_checker)
|
||||||
|
|||||||
Reference in New Issue
Block a user