feat: implements a better kubernetes deployment setup
This commit is contained in:
4
.gitignore
vendored
4
.gitignore
vendored
@@ -4,3 +4,7 @@
|
||||
.idea
|
||||
|
||||
**/__pycache__/
|
||||
|
||||
.k8s/*.yml
|
||||
!.k8s/*.template.yml
|
||||
!.k8s/*.enc.yml
|
||||
5
.k8s/.sops.yaml
Normal file
5
.k8s/.sops.yaml
Normal file
@@ -0,0 +1,5 @@
|
||||
creation_rules:
|
||||
- path_regex: ^secrets(\.enc)?\.yml$
|
||||
encrypted_regex: '^(data|stringData)$'
|
||||
pgp: >-
|
||||
8C8D94A7639C87559B0F2F64B7E1F62F69798EB6
|
||||
12
.k8s/config.template.yml
Normal file
12
.k8s/config.template.yml
Normal file
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: storage-config
|
||||
namespace: ${KUBE_NAMESPACE}
|
||||
data:
|
||||
SERVER_PORT: "8000"
|
||||
|
||||
REDIS_HOST: "storage-redis-service"
|
||||
REDIS_PORT: "6379"
|
||||
|
||||
EXPIRES_IN: "1800000"
|
||||
63
.k8s/deployment.template.yml
Normal file
63
.k8s/deployment.template.yml
Normal file
@@ -0,0 +1,63 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: storage-deployment
|
||||
namespace: ${KUBE_NAMESPACE}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: storage
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: storage
|
||||
spec:
|
||||
nodeSelector:
|
||||
${WORKER_NODE_LABEL}
|
||||
imagePullSecrets:
|
||||
- name: ghcr-secret
|
||||
containers:
|
||||
- name: storage
|
||||
image: ${IMAGE_BASE}:${IMAGE_TAG}
|
||||
imagePullPolicy: Always
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "256Mi"
|
||||
cpu: "1000m"
|
||||
ports:
|
||||
- containerPort: 8000
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 60
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 60
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: storage-config
|
||||
- secretRef:
|
||||
name: storage-secret
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
namespace: ${KUBE_NAMESPACE}
|
||||
name: storage-service
|
||||
spec:
|
||||
selector:
|
||||
app: backend
|
||||
ports:
|
||||
- port: 8000
|
||||
protocol: TCP
|
||||
targetPort: 8000
|
||||
type: ClusterIP
|
||||
47
.k8s/redis.template.yml
Normal file
47
.k8s/redis.template.yml
Normal file
@@ -0,0 +1,47 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
namespace: ${KUBE_NAMESPACE}
|
||||
name: storage-redis-deployment
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: storage-redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: storage-redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: valkey/valkey:8.0.6-alpine
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "75m"
|
||||
limits:
|
||||
memory: "256Mi"
|
||||
cpu: "256m"
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
env:
|
||||
- name: VALKEY_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: storage-secret
|
||||
key: REDIS_PASSWORD
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
namespace: ${KUBE_NAMESPACE}
|
||||
name: storage-redis-service
|
||||
spec:
|
||||
selector:
|
||||
app: storage-redis
|
||||
ports:
|
||||
- port: 6379
|
||||
type: ClusterIP
|
||||
38
.k8s/secrets.enc.yml
Normal file
38
.k8s/secrets.enc.yml
Normal file
@@ -0,0 +1,38 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: storage-secret
|
||||
namespace: ${KUBE_NAMESPACE}
|
||||
data:
|
||||
REDIS_PASSWORD: ENC[AES256_GCM,data:QAjxHjGqOIun8jTN,iv:tfOsGEP0+kGfqvL90Ek6ZNOcueUFnYpYGE6f7r2EvoM=,tag:lAxPPRGE2dO//3hLFItOnA==,type:str]
|
||||
AWS_ACCESS_KEY_ID: ENC[AES256_GCM,data:AtT8rcJxkxgWQ7SEPj5UEVvR+1b/CJt1Rc2zZQ==,iv:862ChLY0MkOwj+HG/sI+NqWFeK1mz6L3m8mWSKDqH4E=,tag:jpRNIXo7jMTYX/BAQG/7aQ==,type:str]
|
||||
AWS_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:vudblTMIDdRZiAmJeYqYtCXd4Jdv++QnvdIUH8PX3IJceW/vES2UybfcKwtKGJgMt6ZPnskOamM=,iv:JHbcu/Tv81d9NWPqPXcFZBSTmrml8xBEU4+G/pVgjGM=,tag:HnAf3tGcmv3CBUOA9px/rw==,type:str]
|
||||
AWS_REGION_NAME: ENC[AES256_GCM,data:GfwdCPgAe7vAtrZPYRp0Ng==,iv:RkZ69hQkuobcOL9DSfhcmLIKsK0rnevHTWF+0Mbw7Dg=,tag:0XAm2oXhETKeUbuN2UPgyA==,type:str]
|
||||
AWS_BUCKET_NAME: ENC[AES256_GCM,data:atzKtMWzlco3rZf5vsFmse1zd+f3xWHUYldeF8O2egmQOmkf,iv:lYw2tM65fszSG8+MhN6edZgvGlwxw0mhOIDvp93LWnE=,tag:iNR7JJzksrFkXrg4/87E/w==,type:str]
|
||||
VIRUS_CHECKER_API_KEY: ENC[AES256_GCM,data:4Ick9JIfUIIZ/LEp09kMiJM9T4FaLmXKiksoeW0BiTGJlYcps5fZFgDEYBUzEcQTGMK8SD5yPZhEbytXXAvyNFanASsVa8rXX+sO98Xr6XFJqVZEjCJbiw==,iv:5/yl9sUEF7cFqf1RKhoV0oGWTLBxTeJDxShUns8/Qsg=,tag:LvEbdmPIeWQBw6G7wnWQkg==,type:str]
|
||||
sops:
|
||||
lastmodified: "2025-11-08T22:29:57Z"
|
||||
mac: ENC[AES256_GCM,data:s15EYKavUjMl9CNqHY6D6XHJZMSwcWhdfW6IFeGhYtjHi6SXGnx/Vpj+L96011xuDca20gU8YS/m69ML2qPV3D5wTMogXYNYGW70LCnXm4YQK+QAIJmhRYPqjlZoQpI85c698+abWuh25zER4tukSZ/ZmmNxnNGZQEyOOnaw8oU=,iv:4nMkWcFmYzBFRoiqoFLO3pR0NTb3QXxqduBqQxfQ0Sc=,tag:uwcqo8+ErSMCOuZM/eUoQA==,type:str]
|
||||
pgp:
|
||||
- created_at: "2025-11-08T22:29:57Z"
|
||||
enc: |-
|
||||
-----BEGIN PGP MESSAGE-----
|
||||
|
||||
hQIMAwzdivR1H/BQAQ//WSCkM5pDyRMbSSMEP8lfREjwHszaXZLF0MkNfjlDsAEC
|
||||
h1yVM2+v5snJj5clipD+8FRiTl69Jho7GGjCEDfoahNyhtgr7t/UH7XVDJKJeI8W
|
||||
55JrWU/T8PBvXo+Ld0c3lKqqdNQ18nWYoEMJeHyUT0MBWYmYlJbvmxKlrgLyQPs5
|
||||
EUBG77aJUXIn3j0O39GEUTTmHLNITNyincZQsx4Hndgi9T7IriJfGwqQP1+9WMYw
|
||||
2Hzld8Qb3VeByrVn+Ybykem6vB8IR5aULpfTU9bKvOrQh1iSgVew9bSXZcqO34Kl
|
||||
OA+DpfJ8LhspPd/00zfto+2h/pwo7tAxaaJtE3NjZfAysFuSF6lX+pkPqj7FOoxh
|
||||
P/ps7wzsk3F3nI/z62ITh91m/O+WhSBVsOou4xAGIatDBapH2YYy9qF/31IP9u3W
|
||||
OOiXNoY1AOOu1/+c9/2ROjQ9/ENhJxiDU/Vl921+PoA+E4EGsFtgXdYiulG5TqZu
|
||||
rkF9eOYw/ds/ptviA4YafFOCmk2oKxDRwhb3oNChqEFQ5hGZ/FqwcwMU+NEI8Blu
|
||||
nmA/UHR/MBdxAWH/H6HMf/DJ1CNLNSgMDaEkvAA/QLPMswuoecpPLstwpOeGC/nL
|
||||
qULiv7+wqlXhggT6QcmekKaw6tjOkMOA/sjqpZMpVnXMA0RpV1lPbK52Z8jINufS
|
||||
XgEyDLyvaqEPLzhG7rNx1IXNBDymXKPmySvopbzjVWXao75verYNK3o/cV+OFihL
|
||||
sgPoqDMMRHhLA6oepxqw9fK5f5jLOVJEc4TPbpjANZIOTIaxvir3E6avoaO3Qx8=
|
||||
=7oeD
|
||||
-----END PGP MESSAGE-----
|
||||
fp: 8C8D94A7639C87559B0F2F64B7E1F62F69798EB6
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.11.0
|
||||
@@ -1,4 +1,3 @@
|
||||
from storage_service.config.config_allowed_origins import get_allowed_origins
|
||||
from storage_service.controller import health_router, storage_router
|
||||
from storage_service.utils.exception_handler import (
|
||||
http_exception_handler,
|
||||
@@ -7,7 +6,6 @@ from storage_service.utils.exception_handler import (
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@@ -15,14 +13,5 @@ app = FastAPI()
|
||||
app.add_exception_handler(HTTPException, http_exception_handler)
|
||||
app.add_exception_handler(RequestValidationError, validation_exception_handler)
|
||||
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=get_allowed_origins(),
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(storage_router)
|
||||
app.include_router(health_router)
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
from dotenv import load_dotenv
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def get_allowed_origins():
|
||||
load_dotenv()
|
||||
|
||||
origins = os.environ.get("ALLOWED_ORIGINS", None)
|
||||
|
||||
if origins is None:
|
||||
return []
|
||||
|
||||
return origins.split(",")
|
||||
@@ -1,5 +1,3 @@
|
||||
from storage_service.utils.enums.storage_type import StorageType
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
import os
|
||||
@@ -14,5 +12,5 @@ def get_config_s3():
|
||||
"aws_secret_access_key": os.environ.get("AWS_SECRET_ACCESS_KEY", None),
|
||||
"region_name": os.environ.get("AWS_REGION_NAME", None),
|
||||
"bucket_name": os.environ.get("AWS_BUCKET_NAME", None),
|
||||
"expires_in": os.environ.get("EXPIRES_IN", 3600),
|
||||
"expires_in": os.environ.get("EXPIRES_IN", "3600"),
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ from storage_service.utils.exceptions.file_not_found_exception import (
|
||||
from storage_service.utils.file.file_hash_generator import generate_file_hash
|
||||
from storage_service.worker.storage_file_worker import storage_file_worker
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi_utils.cbv import cbv
|
||||
from rq import Queue
|
||||
|
||||
|
||||
@@ -37,12 +37,15 @@ def build_client_s3(config: dict) -> botocore.client.BaseClient:
|
||||
def dependency_storage_service() -> StorageService:
|
||||
load_dotenv()
|
||||
|
||||
if StorageType(os.environ["STORAGE_TYPE"]) == StorageType.S3_STORAGE:
|
||||
s3_config = get_config_s3()
|
||||
storage_type = StorageType(os.environ.get("STORAGE_TYPE", "s3"))
|
||||
|
||||
return AmazonS3Service(
|
||||
build_client_s3(s3_config),
|
||||
s3_config["bucket_name"],
|
||||
)
|
||||
match storage_type:
|
||||
case StorageType.S3_STORAGE:
|
||||
s3_config = get_config_s3()
|
||||
|
||||
raise RuntimeError("Invalid Storage Type")
|
||||
return AmazonS3Service(
|
||||
build_client_s3(s3_config),
|
||||
s3_config["bucket_name"],
|
||||
)
|
||||
case _:
|
||||
raise RuntimeError("Invalid Storage Type")
|
||||
|
||||
@@ -23,12 +23,9 @@ from functools import cache
|
||||
def dependency_virus_checker_service() -> VirusCheckerService:
|
||||
load_dotenv()
|
||||
|
||||
try:
|
||||
type = VirusCheckerType(os.environ["VIRUS_CHECKER_TYPE"])
|
||||
except ValueError:
|
||||
raise RuntimeError("Invalid Virus Checker Type")
|
||||
checker_type = VirusCheckerType(os.environ.get("VIRUS_CHECKER_TYPE", "total_virus"))
|
||||
|
||||
match type:
|
||||
match checker_type:
|
||||
case VirusCheckerType.TOTAL_VIRUS:
|
||||
virus_checker = Virustotal(get_virus_checker_api_key())
|
||||
return VirusTotalService(virus_checker)
|
||||
|
||||
Reference in New Issue
Block a user