Refactors Services and Initial Test Implementation
This commit is contained in:
27
.github/workflows/run-tests.yml
vendored
Normal file
27
.github/workflows/run-tests.yml
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push
|
||||
|
||||
jobs:
|
||||
|
||||
run-tests:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install poetry
|
||||
poetry install
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
poetry run python -m unittest
|
||||
66
poetry.lock
generated
66
poetry.lock
generated
@@ -248,6 +248,70 @@ files = [
|
||||
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "coverage"
|
||||
version = "7.5.1"
|
||||
description = "Code coverage measurement for Python"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "coverage-7.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0884920835a033b78d1c73b6d3bbcda8161a900f38a488829a83982925f6c2e"},
|
||||
{file = "coverage-7.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:39afcd3d4339329c5f58de48a52f6e4e50f6578dd6099961cf22228feb25f38f"},
|
||||
{file = "coverage-7.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a7b0ceee8147444347da6a66be737c9d78f3353b0681715b668b72e79203e4a"},
|
||||
{file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a9ca3f2fae0088c3c71d743d85404cec8df9be818a005ea065495bedc33da35"},
|
||||
{file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd215c0c7d7aab005221608a3c2b46f58c0285a819565887ee0b718c052aa4e"},
|
||||
{file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4bf0655ab60d754491004a5efd7f9cccefcc1081a74c9ef2da4735d6ee4a6223"},
|
||||
{file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61c4bf1ba021817de12b813338c9be9f0ad5b1e781b9b340a6d29fc13e7c1b5e"},
|
||||
{file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:db66fc317a046556a96b453a58eced5024af4582a8dbdc0c23ca4dbc0d5b3146"},
|
||||
{file = "coverage-7.5.1-cp310-cp310-win32.whl", hash = "sha256:b016ea6b959d3b9556cb401c55a37547135a587db0115635a443b2ce8f1c7228"},
|
||||
{file = "coverage-7.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:df4e745a81c110e7446b1cc8131bf986157770fa405fe90e15e850aaf7619bc8"},
|
||||
{file = "coverage-7.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:796a79f63eca8814ca3317a1ea443645c9ff0d18b188de470ed7ccd45ae79428"},
|
||||
{file = "coverage-7.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4fc84a37bfd98db31beae3c2748811a3fa72bf2007ff7902f68746d9757f3746"},
|
||||
{file = "coverage-7.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6175d1a0559986c6ee3f7fccfc4a90ecd12ba0a383dcc2da30c2b9918d67d8a3"},
|
||||
{file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fc81d5878cd6274ce971e0a3a18a8803c3fe25457165314271cf78e3aae3aa2"},
|
||||
{file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:556cf1a7cbc8028cb60e1ff0be806be2eded2daf8129b8811c63e2b9a6c43bca"},
|
||||
{file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9981706d300c18d8b220995ad22627647be11a4276721c10911e0e9fa44c83e8"},
|
||||
{file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d7fed867ee50edf1a0b4a11e8e5d0895150e572af1cd6d315d557758bfa9c057"},
|
||||
{file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ef48e2707fb320c8f139424a596f5b69955a85b178f15af261bab871873bb987"},
|
||||
{file = "coverage-7.5.1-cp311-cp311-win32.whl", hash = "sha256:9314d5678dcc665330df5b69c1e726a0e49b27df0461c08ca12674bcc19ef136"},
|
||||
{file = "coverage-7.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fa567e99765fe98f4e7d7394ce623e794d7cabb170f2ca2ac5a4174437e90dd"},
|
||||
{file = "coverage-7.5.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b6cf3764c030e5338e7f61f95bd21147963cf6aa16e09d2f74f1fa52013c1206"},
|
||||
{file = "coverage-7.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ec92012fefebee89a6b9c79bc39051a6cb3891d562b9270ab10ecfdadbc0c34"},
|
||||
{file = "coverage-7.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16db7f26000a07efcf6aea00316f6ac57e7d9a96501e990a36f40c965ec7a95d"},
|
||||
{file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:beccf7b8a10b09c4ae543582c1319c6df47d78fd732f854ac68d518ee1fb97fa"},
|
||||
{file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8748731ad392d736cc9ccac03c9845b13bb07d020a33423fa5b3a36521ac6e4e"},
|
||||
{file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7352b9161b33fd0b643ccd1f21f3a3908daaddf414f1c6cb9d3a2fd618bf2572"},
|
||||
{file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7a588d39e0925f6a2bff87154752481273cdb1736270642aeb3635cb9b4cad07"},
|
||||
{file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:68f962d9b72ce69ea8621f57551b2fa9c70509af757ee3b8105d4f51b92b41a7"},
|
||||
{file = "coverage-7.5.1-cp312-cp312-win32.whl", hash = "sha256:f152cbf5b88aaeb836127d920dd0f5e7edff5a66f10c079157306c4343d86c19"},
|
||||
{file = "coverage-7.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:5a5740d1fb60ddf268a3811bcd353de34eb56dc24e8f52a7f05ee513b2d4f596"},
|
||||
{file = "coverage-7.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e2213def81a50519d7cc56ed643c9e93e0247f5bbe0d1247d15fa520814a7cd7"},
|
||||
{file = "coverage-7.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5037f8fcc2a95b1f0e80585bd9d1ec31068a9bcb157d9750a172836e98bc7a90"},
|
||||
{file = "coverage-7.5.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3721c2c9e4c4953a41a26c14f4cef64330392a6d2d675c8b1db3b645e31f0e"},
|
||||
{file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca498687ca46a62ae590253fba634a1fe9836bc56f626852fb2720f334c9e4e5"},
|
||||
{file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cdcbc320b14c3e5877ee79e649677cb7d89ef588852e9583e6b24c2e5072661"},
|
||||
{file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:57e0204b5b745594e5bc14b9b50006da722827f0b8c776949f1135677e88d0b8"},
|
||||
{file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fe7502616b67b234482c3ce276ff26f39ffe88adca2acf0261df4b8454668b4"},
|
||||
{file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9e78295f4144f9dacfed4f92935fbe1780021247c2fabf73a819b17f0ccfff8d"},
|
||||
{file = "coverage-7.5.1-cp38-cp38-win32.whl", hash = "sha256:1434e088b41594baa71188a17533083eabf5609e8e72f16ce8c186001e6b8c41"},
|
||||
{file = "coverage-7.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:0646599e9b139988b63704d704af8e8df7fa4cbc4a1f33df69d97f36cb0a38de"},
|
||||
{file = "coverage-7.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4cc37def103a2725bc672f84bd939a6fe4522310503207aae4d56351644682f1"},
|
||||
{file = "coverage-7.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc0b4d8bfeabd25ea75e94632f5b6e047eef8adaed0c2161ada1e922e7f7cece"},
|
||||
{file = "coverage-7.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d0a0f5e06881ecedfe6f3dd2f56dcb057b6dbeb3327fd32d4b12854df36bf26"},
|
||||
{file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9735317685ba6ec7e3754798c8871c2f49aa5e687cc794a0b1d284b2389d1bd5"},
|
||||
{file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d21918e9ef11edf36764b93101e2ae8cc82aa5efdc7c5a4e9c6c35a48496d601"},
|
||||
{file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c3e757949f268364b96ca894b4c342b41dc6f8f8b66c37878aacef5930db61be"},
|
||||
{file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:79afb6197e2f7f60c4824dd4b2d4c2ec5801ceb6ba9ce5d2c3080e5660d51a4f"},
|
||||
{file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d1d0d98d95dd18fe29dc66808e1accf59f037d5716f86a501fc0256455219668"},
|
||||
{file = "coverage-7.5.1-cp39-cp39-win32.whl", hash = "sha256:1cc0fe9b0b3a8364093c53b0b4c0c2dd4bb23acbec4c9240b5f284095ccf7981"},
|
||||
{file = "coverage-7.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:dde0070c40ea8bb3641e811c1cfbf18e265d024deff6de52c5950677a8fb1e0f"},
|
||||
{file = "coverage-7.5.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:6537e7c10cc47c595828b8a8be04c72144725c383c4702703ff4e42e44577312"},
|
||||
{file = "coverage-7.5.1.tar.gz", hash = "sha256:54de9ef3a9da981f7af93eafde4ede199e0846cd819eb27c88e2b712aae9708c"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
toml = ["tomli"]
|
||||
|
||||
[[package]]
|
||||
name = "dnspython"
|
||||
version = "2.6.1"
|
||||
@@ -1602,4 +1666,4 @@ files = [
|
||||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = "^3.12"
|
||||
content-hash = "fc6576b524c2ac90df27c5266de1cde8d5003fcc11aab1435b11d413347334f4"
|
||||
content-hash = "ce4c7be96b74b18514b8e28438bc5b68c7cdad495d7fcfdeae899252ed414979"
|
||||
|
||||
@@ -25,6 +25,7 @@ typing-inspect = "^0.9.0"
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
isort = "^5.12.0"
|
||||
black = "^23.7.0"
|
||||
coverage = "^7.5.1"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
|
||||
@@ -10,10 +10,11 @@ from storage_service.model.storage.process_file_request import (
|
||||
)
|
||||
from storage_service.model.storage.signed_url_response import SignedUrlResponse
|
||||
from storage_service.service.storage.storage_service import StorageService
|
||||
from storage_service.utils.file_name_hash import file_name_hash
|
||||
from storage_service.utils.exceptions.file_not_found_exception import FileNotFoundException
|
||||
from storage_service.utils.file.file_hash_generator import generate_file_hash
|
||||
from storage_service.worker.storage_file_worker import storage_file_worker
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi_utils.cbv import cbv
|
||||
from rq import Queue
|
||||
|
||||
@@ -29,7 +30,7 @@ class StorageController:
|
||||
|
||||
@s3_router.post("/file", status_code=200)
|
||||
def new_file_url(self, new_file_request: NewFileURLRequest) -> SignedUrlResponse:
|
||||
hashed_file_name = file_name_hash(
|
||||
hashed_file_name = generate_file_hash(
|
||||
new_file_request.file_key, new_file_request.file_postfix
|
||||
)
|
||||
|
||||
@@ -39,13 +40,16 @@ class StorageController:
|
||||
|
||||
@s3_router.get("/file", status_code=200)
|
||||
def file_url(self, file_key: str, file_postfix: str) -> SignedUrlResponse:
|
||||
try:
|
||||
return self.storage_service.get_temp_read_link(
|
||||
file_name_hash(file_key, file_postfix)
|
||||
generate_file_hash(file_key, file_postfix)
|
||||
)
|
||||
except Exception as _:
|
||||
raise FileNotFoundException("File not found")
|
||||
|
||||
@s3_router.delete("/file", status_code=204)
|
||||
def delete_file(self, file_key: str, file_postfix: str):
|
||||
return self.storage_service.delete_file(file_name_hash(file_key, file_postfix))
|
||||
return self.storage_service.delete_file(generate_file_hash(file_key, file_postfix))
|
||||
|
||||
@s3_router.post("/file/process", status_code=200)
|
||||
def process_file(self, process_file_request: ProcessFileRequest):
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
from .storage_service import StorageService
|
||||
from .amazon_s3_service import AmazonS3Service
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from storage_service.depends.depend_virus_checker_service import (
|
||||
dependency_virus_checker_service,
|
||||
)
|
||||
@@ -9,13 +11,15 @@ from storage_service.service.virus_checker.virus_checker_service import (
|
||||
VirusCheckerService,
|
||||
)
|
||||
from storage_service.utils.enums.file_type import FileType
|
||||
from storage_service.utils.file_handler import FILE_HANDLER
|
||||
|
||||
from botocore.client import BaseClient
|
||||
|
||||
import io
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AmazonS3Service(StorageService):
|
||||
virus_checker_service: VirusCheckerService
|
||||
|
||||
@@ -28,7 +32,7 @@ class AmazonS3Service(StorageService):
|
||||
self,
|
||||
s3_client: BaseClient,
|
||||
bucket_name: str,
|
||||
virus_checker_service=dependency_virus_checker_service(),
|
||||
virus_checker_service=None,
|
||||
**kwargs,
|
||||
):
|
||||
self.virus_checker_service = virus_checker_service
|
||||
@@ -41,6 +45,9 @@ class AmazonS3Service(StorageService):
|
||||
raise RuntimeError("Invalid S3 Config: Missing bucket_name")
|
||||
self.bucket_name = bucket_name
|
||||
|
||||
if virus_checker_service is None:
|
||||
self.virus_checker_service = dependency_virus_checker_service()
|
||||
|
||||
if "expires_in" in kwargs:
|
||||
self.expires_in = kwargs["expires_in"]
|
||||
|
||||
@@ -59,15 +66,30 @@ class AmazonS3Service(StorageService):
|
||||
def delete_file(self, file_name: str) -> None:
|
||||
self._delete_file(file_name)
|
||||
|
||||
def process_file(self, file_name: str, file_type: FileType = FileType.PNG) -> None:
|
||||
def process_file(self, file_name: str, file_type: FileType = FileType.PNG) -> dict:
|
||||
try:
|
||||
file_bytes = self._get_file_obj(file_name)
|
||||
except Exception as _:
|
||||
raise FileNotFoundError("File not found")
|
||||
|
||||
if not self.virus_checker_service.check_virus(file_bytes):
|
||||
self._delete_file(file_name)
|
||||
raise ValueError("Virus Detected")
|
||||
|
||||
handler = FILE_HANDLER[file_type]["handler"]
|
||||
try:
|
||||
old_size = file_bytes.getbuffer().nbytes
|
||||
|
||||
self._upload_file(file_name, handler(file_bytes))
|
||||
file_bytes = file_type.get_validator()(file_bytes)
|
||||
|
||||
new_size = file_bytes.getbuffer().nbytes
|
||||
except Exception as _:
|
||||
raise RuntimeError("Error Processing")
|
||||
|
||||
self._upload_file(file_name, file_bytes)
|
||||
|
||||
return {
|
||||
"previous_size": old_size,
|
||||
"current_size": new_size,
|
||||
}
|
||||
|
||||
def _get_presigned_write_url(self, file_name, file_type: FileType) -> str:
|
||||
return self.s3_client.generate_presigned_url(
|
||||
@@ -75,7 +97,7 @@ class AmazonS3Service(StorageService):
|
||||
Params={
|
||||
"Bucket": self.bucket_name,
|
||||
"Key": file_name,
|
||||
"ContentType": FILE_HANDLER[file_type]["content_type"],
|
||||
"ContentType": file_type.get_content_type(),
|
||||
},
|
||||
ExpiresIn=self.expires_in,
|
||||
)
|
||||
@@ -91,7 +113,8 @@ class AmazonS3Service(StorageService):
|
||||
Params={"Bucket": self.bucket_name, "Key": file_name},
|
||||
ExpiresIn=self.expires_in,
|
||||
)
|
||||
return None
|
||||
|
||||
raise FileNotFoundError("File not found")
|
||||
|
||||
def _get_file_obj(self, file_name: str) -> io.BytesIO:
|
||||
return io.BytesIO(
|
||||
|
||||
@@ -20,5 +20,5 @@ class StorageService(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def process_file(self, file_name: str, file_type: FileType) -> None:
|
||||
def process_file(self, file_name: str, file_type: FileType) -> dict:
|
||||
pass
|
||||
|
||||
@@ -34,7 +34,7 @@ class VirusTotalService(VirusCheckerService):
|
||||
@staticmethod
|
||||
def _is_valid_file(file_stats: dict) -> bool:
|
||||
match file_stats:
|
||||
case {"malicious": 0, "suspicious": 0, "undetected": 0, "harmless": 0}:
|
||||
case {"malicious": 0, "suspicious": 0, "harmless": 0}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
@@ -1,6 +1,26 @@
|
||||
from enum import Enum
|
||||
from io import BytesIO
|
||||
from typing import Callable
|
||||
|
||||
from storage_service.utils.file.validators import image_validator
|
||||
|
||||
|
||||
class FileType(Enum):
|
||||
PNG = "png"
|
||||
JPEG = "jpeg"
|
||||
|
||||
def get_content_type(self) -> str:
|
||||
match self:
|
||||
case FileType.PNG:
|
||||
return "image/png"
|
||||
case FileType.JPEG:
|
||||
return "image/jpeg"
|
||||
case _:
|
||||
raise ValueError("File Type Not Implemented")
|
||||
|
||||
def get_validator(self) -> Callable[[BytesIO], BytesIO]:
|
||||
match self:
|
||||
case FileType.PNG | FileType.JPEG:
|
||||
return image_validator
|
||||
case _:
|
||||
raise ValueError("File Type Not Implemented")
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
|
||||
class FileNotFoundException(HTTPException):
|
||||
def __init__(self, message: str):
|
||||
super().__init__(
|
||||
status.HTTP_400_BAD_REQUEST, detail=message
|
||||
)
|
||||
0
storage_service/utils/file/__init__.py
Normal file
0
storage_service/utils/file/__init__.py
Normal file
@@ -2,7 +2,7 @@ import base64
|
||||
from hashlib import md5
|
||||
|
||||
|
||||
def file_name_hash(file_key: str, file_postfix: str) -> str:
|
||||
def generate_file_hash(file_key: str, file_postfix: str) -> str:
|
||||
hashed_file_key = md5(file_key.encode("utf-8")).digest()
|
||||
hashed_file_key = base64.b64encode(hashed_file_key).decode()
|
||||
|
||||
1
storage_service/utils/file/validators/__init__.py
Normal file
1
storage_service/utils/file/validators/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
from .image_handler import image_validator
|
||||
@@ -3,10 +3,10 @@ from PIL import Image
|
||||
import io
|
||||
|
||||
|
||||
def image_handler(file_bytes: io.BytesIO) -> io.BytesIO:
|
||||
def image_validator(file_bytes: io.BytesIO) -> io.BytesIO:
|
||||
img = Image.open(file_bytes)
|
||||
|
||||
img.thumbnail((320, 320))
|
||||
img.thumbnail((180, 180))
|
||||
|
||||
data = list(img.getdata())
|
||||
image_without_exif = Image.new(img.mode, img.size)
|
||||
@@ -1,9 +0,0 @@
|
||||
from storage_service.utils.enums.file_type import FileType
|
||||
from storage_service.utils.file_handler.handlers.image_handler import (
|
||||
image_handler,
|
||||
)
|
||||
|
||||
FILE_HANDLER = {
|
||||
FileType.PNG: {"content_type": "image/png", "handler": image_handler},
|
||||
FileType.JPEG: {"content_type": "image/jpeg", "handler": image_handler},
|
||||
}
|
||||
@@ -1,9 +1,30 @@
|
||||
import logging
|
||||
|
||||
from storage_service.depends.depend_s3_service import (
|
||||
dependency_storage_service,
|
||||
)
|
||||
from storage_service.utils.enums.file_type import FileType
|
||||
from storage_service.utils.file_name_hash import file_name_hash
|
||||
from storage_service.utils.file.file_hash_generator import generate_file_hash
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def storage_file_worker(username: str, file_postfix: str) -> None:
|
||||
dependency_storage_service().process_file(file_name_hash(username, file_postfix))
|
||||
storage_service = dependency_storage_service()
|
||||
|
||||
file_name = generate_file_hash(username, file_postfix)
|
||||
try:
|
||||
stats = storage_service.process_file(file_name)
|
||||
|
||||
print(
|
||||
f"File processed: {file_name} - "
|
||||
f"Previous Size: {stats["previous_size"]/1_000}kb - "
|
||||
f"New Size: {stats["current_size"]/1_000}kb"
|
||||
)
|
||||
except Exception as e:
|
||||
print(
|
||||
f"Error processing file: {e}."
|
||||
f" Deleting file: {file_name}."
|
||||
)
|
||||
|
||||
storage_service.delete_file(file_name)
|
||||
|
||||
0
tests/__init__.py
Normal file
0
tests/__init__.py
Normal file
0
tests/storage_service/__init__.py
Normal file
0
tests/storage_service/__init__.py
Normal file
140
tests/storage_service/test_amazon_s3_service.py
Normal file
140
tests/storage_service/test_amazon_s3_service.py
Normal file
@@ -0,0 +1,140 @@
|
||||
from unittest import TestCase
|
||||
from unittest.mock import Mock
|
||||
|
||||
from storage_service.service.storage import AmazonS3Service
|
||||
from storage_service.utils.enums.file_type import FileType
|
||||
|
||||
|
||||
class TestAmazonS3Service(TestCase):
|
||||
def setUp(self):
|
||||
self.s3_client_mock = Mock()
|
||||
self.virus_checker_service_mock = Mock()
|
||||
|
||||
def test_get_temp_upload_link(self):
|
||||
self.s3_client_mock.generate_presigned_url.return_value = "https://test.com"
|
||||
|
||||
storage_service = AmazonS3Service(
|
||||
s3_client=self.s3_client_mock,
|
||||
bucket_name="test_bucket",
|
||||
virus_checker_service=self.virus_checker_service_mock
|
||||
)
|
||||
|
||||
response = storage_service.get_temp_upload_link("test_file", FileType.JPEG)
|
||||
|
||||
self.assertEqual(response.signed_url, "https://test.com")
|
||||
self.assertEqual(response.expires_in, 3600)
|
||||
|
||||
self.s3_client_mock.generate_presigned_url.assert_called_once_with(
|
||||
"put_object",
|
||||
Params={
|
||||
"Bucket": "test_bucket",
|
||||
"Key": "test_file",
|
||||
"ContentType": "image/jpeg",
|
||||
},
|
||||
ExpiresIn=3600,
|
||||
)
|
||||
|
||||
def test_get_temp_read_link(self):
|
||||
self.s3_client_mock.generate_presigned_url.return_value = "https://test.com"
|
||||
self.s3_client_mock.list_objects.return_value = {
|
||||
"Contents": [
|
||||
{
|
||||
"Key": "test_file"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
storage_service = AmazonS3Service(
|
||||
s3_client=self.s3_client_mock,
|
||||
bucket_name="test_bucket",
|
||||
virus_checker_service=self.virus_checker_service_mock
|
||||
)
|
||||
|
||||
response = storage_service.get_temp_read_link("test_file")
|
||||
|
||||
self.assertEqual(response.signed_url, "https://test.com")
|
||||
self.assertEqual(response.expires_in, 3600)
|
||||
|
||||
self.s3_client_mock.generate_presigned_url.assert_called_once_with(
|
||||
"get_object",
|
||||
Params={
|
||||
"Bucket": "test_bucket",
|
||||
"Key": "test_file"
|
||||
},
|
||||
ExpiresIn=3600,
|
||||
)
|
||||
|
||||
def test_delete_file(self):
|
||||
storage_service = AmazonS3Service(
|
||||
s3_client=self.s3_client_mock,
|
||||
bucket_name="test_bucket",
|
||||
virus_checker_service=self.virus_checker_service_mock
|
||||
)
|
||||
|
||||
storage_service.delete_file("test_file")
|
||||
|
||||
self.s3_client_mock.delete_object.assert_called_once_with(
|
||||
Bucket="test_bucket",
|
||||
Key="test_file"
|
||||
)
|
||||
|
||||
def test_process_file_if_file_invalid(self):
|
||||
mock_body = Mock()
|
||||
mock_body.read.return_value = b"test_file"
|
||||
self.s3_client_mock.get_object.return_value = {
|
||||
"Body": mock_body
|
||||
}
|
||||
self.virus_checker_service_mock.check_virus.return_value = True
|
||||
|
||||
storage_service = AmazonS3Service(
|
||||
s3_client=self.s3_client_mock,
|
||||
bucket_name="test_bucket",
|
||||
virus_checker_service=self.virus_checker_service_mock
|
||||
)
|
||||
|
||||
with self.assertRaises(RuntimeError):
|
||||
storage_service.process_file("test_file", FileType.JPEG)
|
||||
|
||||
def test_process_file_if_file_is_virus(self):
|
||||
mock_body = Mock()
|
||||
mock_body.read.return_value = b"test_file"
|
||||
self.s3_client_mock.get_object.return_value = {
|
||||
"Body": mock_body
|
||||
}
|
||||
|
||||
mock_file_type = Mock()
|
||||
mock_file_type.get_validator.return_value = lambda x: x
|
||||
mock_file_type.get_content_type.return_value = "image/fake"
|
||||
|
||||
self.virus_checker_service_mock.check_virus.return_value = False
|
||||
|
||||
storage_service = AmazonS3Service(
|
||||
s3_client=self.s3_client_mock,
|
||||
bucket_name="test_bucket",
|
||||
virus_checker_service=self.virus_checker_service_mock
|
||||
)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
storage_service.process_file("test_file", mock_file_type)
|
||||
|
||||
def test_process_file(self):
|
||||
mock_body = Mock()
|
||||
mock_body.read.return_value = b"test_file"
|
||||
self.s3_client_mock.get_object.return_value = {
|
||||
"Body": mock_body
|
||||
}
|
||||
self.virus_checker_service_mock.check_virus.return_value = True
|
||||
|
||||
mock_file_type = Mock()
|
||||
mock_file_type.get_validator.return_value = lambda x: x
|
||||
mock_file_type.get_content_type.return_value = "image/fake"
|
||||
|
||||
storage_service = AmazonS3Service(
|
||||
s3_client=self.s3_client_mock,
|
||||
bucket_name="test_bucket",
|
||||
virus_checker_service=self.virus_checker_service_mock
|
||||
)
|
||||
|
||||
storage_service.process_file("test_file", mock_file_type)
|
||||
|
||||
self.s3_client_mock.upload_fileobj.assert_called()
|
||||
0
tests/virus_checker_service/__init__.py
Normal file
0
tests/virus_checker_service/__init__.py
Normal file
Reference in New Issue
Block a user