From 7eb4d5b64dec08ad51655e7eee892801553e582e Mon Sep 17 00:00:00 2001 From: Vitor Hideyoshi Nakazone Batista Date: Tue, 21 May 2024 03:35:24 -0300 Subject: [PATCH] Refactors Services and Initial Test Implementation --- .github/workflows/run-tests.yml | 27 ++++ poetry.lock | 66 ++++++++- pyproject.toml | 1 + .../controller/storage_controller.py | 18 ++- storage_service/service/storage/__init__.py | 2 + .../service/storage/amazon_s3_service.py | 41 +++-- .../service/storage/storage_service.py | 2 +- .../virus_checker/virus_total_service.py | 2 +- storage_service/utils/enums/file_type.py | 20 +++ .../handlers => exceptions}/__init__.py | 0 .../exceptions/file_not_found_exception.py | 8 + storage_service/utils/file/__init__.py | 0 .../file_hash_generator.py} | 2 +- .../utils/file/validators/__init__.py | 1 + .../validators}/image_handler.py | 4 +- .../utils/file_handler/__init__.py | 9 -- storage_service/worker/storage_file_worker.py | 27 +++- tests/__init__.py | 0 tests/storage_service/__init__.py | 0 .../storage_service/test_amazon_s3_service.py | 140 ++++++++++++++++++ tests/virus_checker_service/__init__.py | 0 21 files changed, 336 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/run-tests.yml rename storage_service/utils/{file_handler/handlers => exceptions}/__init__.py (100%) create mode 100644 storage_service/utils/exceptions/file_not_found_exception.py create mode 100644 storage_service/utils/file/__init__.py rename storage_service/utils/{file_name_hash.py => file/file_hash_generator.py} (76%) create mode 100644 storage_service/utils/file/validators/__init__.py rename storage_service/utils/{file_handler/handlers => file/validators}/image_handler.py (78%) delete mode 100644 storage_service/utils/file_handler/__init__.py create mode 100644 tests/__init__.py create mode 100644 tests/storage_service/__init__.py create mode 100644 tests/storage_service/test_amazon_s3_service.py create mode 100644 tests/virus_checker_service/__init__.py diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000..37169e1 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,27 @@ +name: ci + +on: + push + +jobs: + + run-tests: + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install + + - name: Run tests + run: | + poetry run python -m unittest \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 206a074..fa07d3d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -248,6 +248,70 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "coverage" +version = "7.5.1" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "coverage-7.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0884920835a033b78d1c73b6d3bbcda8161a900f38a488829a83982925f6c2e"}, + {file = "coverage-7.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:39afcd3d4339329c5f58de48a52f6e4e50f6578dd6099961cf22228feb25f38f"}, + {file = "coverage-7.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a7b0ceee8147444347da6a66be737c9d78f3353b0681715b668b72e79203e4a"}, + {file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a9ca3f2fae0088c3c71d743d85404cec8df9be818a005ea065495bedc33da35"}, + {file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd215c0c7d7aab005221608a3c2b46f58c0285a819565887ee0b718c052aa4e"}, + {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4bf0655ab60d754491004a5efd7f9cccefcc1081a74c9ef2da4735d6ee4a6223"}, + {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61c4bf1ba021817de12b813338c9be9f0ad5b1e781b9b340a6d29fc13e7c1b5e"}, + {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:db66fc317a046556a96b453a58eced5024af4582a8dbdc0c23ca4dbc0d5b3146"}, + {file = "coverage-7.5.1-cp310-cp310-win32.whl", hash = "sha256:b016ea6b959d3b9556cb401c55a37547135a587db0115635a443b2ce8f1c7228"}, + {file = "coverage-7.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:df4e745a81c110e7446b1cc8131bf986157770fa405fe90e15e850aaf7619bc8"}, + {file = "coverage-7.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:796a79f63eca8814ca3317a1ea443645c9ff0d18b188de470ed7ccd45ae79428"}, + {file = "coverage-7.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4fc84a37bfd98db31beae3c2748811a3fa72bf2007ff7902f68746d9757f3746"}, + {file = "coverage-7.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6175d1a0559986c6ee3f7fccfc4a90ecd12ba0a383dcc2da30c2b9918d67d8a3"}, + {file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fc81d5878cd6274ce971e0a3a18a8803c3fe25457165314271cf78e3aae3aa2"}, + {file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:556cf1a7cbc8028cb60e1ff0be806be2eded2daf8129b8811c63e2b9a6c43bca"}, + {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9981706d300c18d8b220995ad22627647be11a4276721c10911e0e9fa44c83e8"}, + {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d7fed867ee50edf1a0b4a11e8e5d0895150e572af1cd6d315d557758bfa9c057"}, + {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ef48e2707fb320c8f139424a596f5b69955a85b178f15af261bab871873bb987"}, + {file = "coverage-7.5.1-cp311-cp311-win32.whl", hash = "sha256:9314d5678dcc665330df5b69c1e726a0e49b27df0461c08ca12674bcc19ef136"}, + {file = "coverage-7.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fa567e99765fe98f4e7d7394ce623e794d7cabb170f2ca2ac5a4174437e90dd"}, + {file = "coverage-7.5.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b6cf3764c030e5338e7f61f95bd21147963cf6aa16e09d2f74f1fa52013c1206"}, + {file = "coverage-7.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ec92012fefebee89a6b9c79bc39051a6cb3891d562b9270ab10ecfdadbc0c34"}, + {file = "coverage-7.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16db7f26000a07efcf6aea00316f6ac57e7d9a96501e990a36f40c965ec7a95d"}, + {file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:beccf7b8a10b09c4ae543582c1319c6df47d78fd732f854ac68d518ee1fb97fa"}, + {file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8748731ad392d736cc9ccac03c9845b13bb07d020a33423fa5b3a36521ac6e4e"}, + {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7352b9161b33fd0b643ccd1f21f3a3908daaddf414f1c6cb9d3a2fd618bf2572"}, + {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7a588d39e0925f6a2bff87154752481273cdb1736270642aeb3635cb9b4cad07"}, + {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:68f962d9b72ce69ea8621f57551b2fa9c70509af757ee3b8105d4f51b92b41a7"}, + {file = "coverage-7.5.1-cp312-cp312-win32.whl", hash = "sha256:f152cbf5b88aaeb836127d920dd0f5e7edff5a66f10c079157306c4343d86c19"}, + {file = "coverage-7.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:5a5740d1fb60ddf268a3811bcd353de34eb56dc24e8f52a7f05ee513b2d4f596"}, + {file = "coverage-7.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e2213def81a50519d7cc56ed643c9e93e0247f5bbe0d1247d15fa520814a7cd7"}, + {file = "coverage-7.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5037f8fcc2a95b1f0e80585bd9d1ec31068a9bcb157d9750a172836e98bc7a90"}, + {file = "coverage-7.5.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3721c2c9e4c4953a41a26c14f4cef64330392a6d2d675c8b1db3b645e31f0e"}, + {file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca498687ca46a62ae590253fba634a1fe9836bc56f626852fb2720f334c9e4e5"}, + {file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cdcbc320b14c3e5877ee79e649677cb7d89ef588852e9583e6b24c2e5072661"}, + {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:57e0204b5b745594e5bc14b9b50006da722827f0b8c776949f1135677e88d0b8"}, + {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fe7502616b67b234482c3ce276ff26f39ffe88adca2acf0261df4b8454668b4"}, + {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9e78295f4144f9dacfed4f92935fbe1780021247c2fabf73a819b17f0ccfff8d"}, + {file = "coverage-7.5.1-cp38-cp38-win32.whl", hash = "sha256:1434e088b41594baa71188a17533083eabf5609e8e72f16ce8c186001e6b8c41"}, + {file = "coverage-7.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:0646599e9b139988b63704d704af8e8df7fa4cbc4a1f33df69d97f36cb0a38de"}, + {file = "coverage-7.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4cc37def103a2725bc672f84bd939a6fe4522310503207aae4d56351644682f1"}, + {file = "coverage-7.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc0b4d8bfeabd25ea75e94632f5b6e047eef8adaed0c2161ada1e922e7f7cece"}, + {file = "coverage-7.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d0a0f5e06881ecedfe6f3dd2f56dcb057b6dbeb3327fd32d4b12854df36bf26"}, + {file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9735317685ba6ec7e3754798c8871c2f49aa5e687cc794a0b1d284b2389d1bd5"}, + {file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d21918e9ef11edf36764b93101e2ae8cc82aa5efdc7c5a4e9c6c35a48496d601"}, + {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c3e757949f268364b96ca894b4c342b41dc6f8f8b66c37878aacef5930db61be"}, + {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:79afb6197e2f7f60c4824dd4b2d4c2ec5801ceb6ba9ce5d2c3080e5660d51a4f"}, + {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d1d0d98d95dd18fe29dc66808e1accf59f037d5716f86a501fc0256455219668"}, + {file = "coverage-7.5.1-cp39-cp39-win32.whl", hash = "sha256:1cc0fe9b0b3a8364093c53b0b4c0c2dd4bb23acbec4c9240b5f284095ccf7981"}, + {file = "coverage-7.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:dde0070c40ea8bb3641e811c1cfbf18e265d024deff6de52c5950677a8fb1e0f"}, + {file = "coverage-7.5.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:6537e7c10cc47c595828b8a8be04c72144725c383c4702703ff4e42e44577312"}, + {file = "coverage-7.5.1.tar.gz", hash = "sha256:54de9ef3a9da981f7af93eafde4ede199e0846cd819eb27c88e2b712aae9708c"}, +] + +[package.extras] +toml = ["tomli"] + [[package]] name = "dnspython" version = "2.6.1" @@ -1602,4 +1666,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "fc6576b524c2ac90df27c5266de1cde8d5003fcc11aab1435b11d413347334f4" +content-hash = "ce4c7be96b74b18514b8e28438bc5b68c7cdad495d7fcfdeae899252ed414979" diff --git a/pyproject.toml b/pyproject.toml index 5e9fcbd..b86bc38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ typing-inspect = "^0.9.0" [tool.poetry.group.dev.dependencies] isort = "^5.12.0" black = "^23.7.0" +coverage = "^7.5.1" [build-system] requires = ["poetry-core"] diff --git a/storage_service/controller/storage_controller.py b/storage_service/controller/storage_controller.py index ff759db..86ecbe5 100644 --- a/storage_service/controller/storage_controller.py +++ b/storage_service/controller/storage_controller.py @@ -10,10 +10,11 @@ from storage_service.model.storage.process_file_request import ( ) from storage_service.model.storage.signed_url_response import SignedUrlResponse from storage_service.service.storage.storage_service import StorageService -from storage_service.utils.file_name_hash import file_name_hash +from storage_service.utils.exceptions.file_not_found_exception import FileNotFoundException +from storage_service.utils.file.file_hash_generator import generate_file_hash from storage_service.worker.storage_file_worker import storage_file_worker -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException from fastapi_utils.cbv import cbv from rq import Queue @@ -29,7 +30,7 @@ class StorageController: @s3_router.post("/file", status_code=200) def new_file_url(self, new_file_request: NewFileURLRequest) -> SignedUrlResponse: - hashed_file_name = file_name_hash( + hashed_file_name = generate_file_hash( new_file_request.file_key, new_file_request.file_postfix ) @@ -39,13 +40,16 @@ class StorageController: @s3_router.get("/file", status_code=200) def file_url(self, file_key: str, file_postfix: str) -> SignedUrlResponse: - return self.storage_service.get_temp_read_link( - file_name_hash(file_key, file_postfix) - ) + try: + return self.storage_service.get_temp_read_link( + generate_file_hash(file_key, file_postfix) + ) + except Exception as _: + raise FileNotFoundException("File not found") @s3_router.delete("/file", status_code=204) def delete_file(self, file_key: str, file_postfix: str): - return self.storage_service.delete_file(file_name_hash(file_key, file_postfix)) + return self.storage_service.delete_file(generate_file_hash(file_key, file_postfix)) @s3_router.post("/file/process", status_code=200) def process_file(self, process_file_request: ProcessFileRequest): diff --git a/storage_service/service/storage/__init__.py b/storage_service/service/storage/__init__.py index e69de29..9433f92 100644 --- a/storage_service/service/storage/__init__.py +++ b/storage_service/service/storage/__init__.py @@ -0,0 +1,2 @@ +from .storage_service import StorageService +from .amazon_s3_service import AmazonS3Service diff --git a/storage_service/service/storage/amazon_s3_service.py b/storage_service/service/storage/amazon_s3_service.py index edc6841..41a6a32 100644 --- a/storage_service/service/storage/amazon_s3_service.py +++ b/storage_service/service/storage/amazon_s3_service.py @@ -1,5 +1,7 @@ from __future__ import annotations +import logging + from storage_service.depends.depend_virus_checker_service import ( dependency_virus_checker_service, ) @@ -9,13 +11,15 @@ from storage_service.service.virus_checker.virus_checker_service import ( VirusCheckerService, ) from storage_service.utils.enums.file_type import FileType -from storage_service.utils.file_handler import FILE_HANDLER from botocore.client import BaseClient import io +logger = logging.getLogger(__name__) + + class AmazonS3Service(StorageService): virus_checker_service: VirusCheckerService @@ -28,7 +32,7 @@ class AmazonS3Service(StorageService): self, s3_client: BaseClient, bucket_name: str, - virus_checker_service=dependency_virus_checker_service(), + virus_checker_service=None, **kwargs, ): self.virus_checker_service = virus_checker_service @@ -41,6 +45,9 @@ class AmazonS3Service(StorageService): raise RuntimeError("Invalid S3 Config: Missing bucket_name") self.bucket_name = bucket_name + if virus_checker_service is None: + self.virus_checker_service = dependency_virus_checker_service() + if "expires_in" in kwargs: self.expires_in = kwargs["expires_in"] @@ -59,15 +66,30 @@ class AmazonS3Service(StorageService): def delete_file(self, file_name: str) -> None: self._delete_file(file_name) - def process_file(self, file_name: str, file_type: FileType = FileType.PNG) -> None: - file_bytes = self._get_file_obj(file_name) + def process_file(self, file_name: str, file_type: FileType = FileType.PNG) -> dict: + try: + file_bytes = self._get_file_obj(file_name) + except Exception as _: + raise FileNotFoundError("File not found") if not self.virus_checker_service.check_virus(file_bytes): - self._delete_file(file_name) + raise ValueError("Virus Detected") - handler = FILE_HANDLER[file_type]["handler"] + try: + old_size = file_bytes.getbuffer().nbytes - self._upload_file(file_name, handler(file_bytes)) + file_bytes = file_type.get_validator()(file_bytes) + + new_size = file_bytes.getbuffer().nbytes + except Exception as _: + raise RuntimeError("Error Processing") + + self._upload_file(file_name, file_bytes) + + return { + "previous_size": old_size, + "current_size": new_size, + } def _get_presigned_write_url(self, file_name, file_type: FileType) -> str: return self.s3_client.generate_presigned_url( @@ -75,7 +97,7 @@ class AmazonS3Service(StorageService): Params={ "Bucket": self.bucket_name, "Key": file_name, - "ContentType": FILE_HANDLER[file_type]["content_type"], + "ContentType": file_type.get_content_type(), }, ExpiresIn=self.expires_in, ) @@ -91,7 +113,8 @@ class AmazonS3Service(StorageService): Params={"Bucket": self.bucket_name, "Key": file_name}, ExpiresIn=self.expires_in, ) - return None + + raise FileNotFoundError("File not found") def _get_file_obj(self, file_name: str) -> io.BytesIO: return io.BytesIO( diff --git a/storage_service/service/storage/storage_service.py b/storage_service/service/storage/storage_service.py index 40c84b1..cdab80b 100644 --- a/storage_service/service/storage/storage_service.py +++ b/storage_service/service/storage/storage_service.py @@ -20,5 +20,5 @@ class StorageService(ABC): pass @abstractmethod - def process_file(self, file_name: str, file_type: FileType) -> None: + def process_file(self, file_name: str, file_type: FileType) -> dict: pass diff --git a/storage_service/service/virus_checker/virus_total_service.py b/storage_service/service/virus_checker/virus_total_service.py index e96e551..9a7c0cc 100644 --- a/storage_service/service/virus_checker/virus_total_service.py +++ b/storage_service/service/virus_checker/virus_total_service.py @@ -34,7 +34,7 @@ class VirusTotalService(VirusCheckerService): @staticmethod def _is_valid_file(file_stats: dict) -> bool: match file_stats: - case {"malicious": 0, "suspicious": 0, "undetected": 0, "harmless": 0}: + case {"malicious": 0, "suspicious": 0, "harmless": 0}: return True case _: return False diff --git a/storage_service/utils/enums/file_type.py b/storage_service/utils/enums/file_type.py index 1b8ddc1..149e4c3 100644 --- a/storage_service/utils/enums/file_type.py +++ b/storage_service/utils/enums/file_type.py @@ -1,6 +1,26 @@ from enum import Enum +from io import BytesIO +from typing import Callable + +from storage_service.utils.file.validators import image_validator class FileType(Enum): PNG = "png" JPEG = "jpeg" + + def get_content_type(self) -> str: + match self: + case FileType.PNG: + return "image/png" + case FileType.JPEG: + return "image/jpeg" + case _: + raise ValueError("File Type Not Implemented") + + def get_validator(self) -> Callable[[BytesIO], BytesIO]: + match self: + case FileType.PNG | FileType.JPEG: + return image_validator + case _: + raise ValueError("File Type Not Implemented") diff --git a/storage_service/utils/file_handler/handlers/__init__.py b/storage_service/utils/exceptions/__init__.py similarity index 100% rename from storage_service/utils/file_handler/handlers/__init__.py rename to storage_service/utils/exceptions/__init__.py diff --git a/storage_service/utils/exceptions/file_not_found_exception.py b/storage_service/utils/exceptions/file_not_found_exception.py new file mode 100644 index 0000000..2e231e1 --- /dev/null +++ b/storage_service/utils/exceptions/file_not_found_exception.py @@ -0,0 +1,8 @@ +from fastapi import HTTPException, status + + +class FileNotFoundException(HTTPException): + def __init__(self, message: str): + super().__init__( + status.HTTP_400_BAD_REQUEST, detail=message + ) diff --git a/storage_service/utils/file/__init__.py b/storage_service/utils/file/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/storage_service/utils/file_name_hash.py b/storage_service/utils/file/file_hash_generator.py similarity index 76% rename from storage_service/utils/file_name_hash.py rename to storage_service/utils/file/file_hash_generator.py index 8c993e2..23bc6e5 100644 --- a/storage_service/utils/file_name_hash.py +++ b/storage_service/utils/file/file_hash_generator.py @@ -2,7 +2,7 @@ import base64 from hashlib import md5 -def file_name_hash(file_key: str, file_postfix: str) -> str: +def generate_file_hash(file_key: str, file_postfix: str) -> str: hashed_file_key = md5(file_key.encode("utf-8")).digest() hashed_file_key = base64.b64encode(hashed_file_key).decode() diff --git a/storage_service/utils/file/validators/__init__.py b/storage_service/utils/file/validators/__init__.py new file mode 100644 index 0000000..b5bed77 --- /dev/null +++ b/storage_service/utils/file/validators/__init__.py @@ -0,0 +1 @@ +from .image_handler import image_validator diff --git a/storage_service/utils/file_handler/handlers/image_handler.py b/storage_service/utils/file/validators/image_handler.py similarity index 78% rename from storage_service/utils/file_handler/handlers/image_handler.py rename to storage_service/utils/file/validators/image_handler.py index 764bf64..fb24884 100644 --- a/storage_service/utils/file_handler/handlers/image_handler.py +++ b/storage_service/utils/file/validators/image_handler.py @@ -3,10 +3,10 @@ from PIL import Image import io -def image_handler(file_bytes: io.BytesIO) -> io.BytesIO: +def image_validator(file_bytes: io.BytesIO) -> io.BytesIO: img = Image.open(file_bytes) - img.thumbnail((320, 320)) + img.thumbnail((180, 180)) data = list(img.getdata()) image_without_exif = Image.new(img.mode, img.size) diff --git a/storage_service/utils/file_handler/__init__.py b/storage_service/utils/file_handler/__init__.py deleted file mode 100644 index 43930da..0000000 --- a/storage_service/utils/file_handler/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from storage_service.utils.enums.file_type import FileType -from storage_service.utils.file_handler.handlers.image_handler import ( - image_handler, -) - -FILE_HANDLER = { - FileType.PNG: {"content_type": "image/png", "handler": image_handler}, - FileType.JPEG: {"content_type": "image/jpeg", "handler": image_handler}, -} diff --git a/storage_service/worker/storage_file_worker.py b/storage_service/worker/storage_file_worker.py index fd74d46..afc6752 100644 --- a/storage_service/worker/storage_file_worker.py +++ b/storage_service/worker/storage_file_worker.py @@ -1,9 +1,30 @@ +import logging + from storage_service.depends.depend_s3_service import ( dependency_storage_service, ) -from storage_service.utils.enums.file_type import FileType -from storage_service.utils.file_name_hash import file_name_hash +from storage_service.utils.file.file_hash_generator import generate_file_hash + + +logger = logging.getLogger(__name__) def storage_file_worker(username: str, file_postfix: str) -> None: - dependency_storage_service().process_file(file_name_hash(username, file_postfix)) + storage_service = dependency_storage_service() + + file_name = generate_file_hash(username, file_postfix) + try: + stats = storage_service.process_file(file_name) + + print( + f"File processed: {file_name} - " + f"Previous Size: {stats["previous_size"]/1_000}kb - " + f"New Size: {stats["current_size"]/1_000}kb" + ) + except Exception as e: + print( + f"Error processing file: {e}." + f" Deleting file: {file_name}." + ) + + storage_service.delete_file(file_name) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/storage_service/__init__.py b/tests/storage_service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/storage_service/test_amazon_s3_service.py b/tests/storage_service/test_amazon_s3_service.py new file mode 100644 index 0000000..e85f267 --- /dev/null +++ b/tests/storage_service/test_amazon_s3_service.py @@ -0,0 +1,140 @@ +from unittest import TestCase +from unittest.mock import Mock + +from storage_service.service.storage import AmazonS3Service +from storage_service.utils.enums.file_type import FileType + + +class TestAmazonS3Service(TestCase): + def setUp(self): + self.s3_client_mock = Mock() + self.virus_checker_service_mock = Mock() + + def test_get_temp_upload_link(self): + self.s3_client_mock.generate_presigned_url.return_value = "https://test.com" + + storage_service = AmazonS3Service( + s3_client=self.s3_client_mock, + bucket_name="test_bucket", + virus_checker_service=self.virus_checker_service_mock + ) + + response = storage_service.get_temp_upload_link("test_file", FileType.JPEG) + + self.assertEqual(response.signed_url, "https://test.com") + self.assertEqual(response.expires_in, 3600) + + self.s3_client_mock.generate_presigned_url.assert_called_once_with( + "put_object", + Params={ + "Bucket": "test_bucket", + "Key": "test_file", + "ContentType": "image/jpeg", + }, + ExpiresIn=3600, + ) + + def test_get_temp_read_link(self): + self.s3_client_mock.generate_presigned_url.return_value = "https://test.com" + self.s3_client_mock.list_objects.return_value = { + "Contents": [ + { + "Key": "test_file" + } + ] + } + + storage_service = AmazonS3Service( + s3_client=self.s3_client_mock, + bucket_name="test_bucket", + virus_checker_service=self.virus_checker_service_mock + ) + + response = storage_service.get_temp_read_link("test_file") + + self.assertEqual(response.signed_url, "https://test.com") + self.assertEqual(response.expires_in, 3600) + + self.s3_client_mock.generate_presigned_url.assert_called_once_with( + "get_object", + Params={ + "Bucket": "test_bucket", + "Key": "test_file" + }, + ExpiresIn=3600, + ) + + def test_delete_file(self): + storage_service = AmazonS3Service( + s3_client=self.s3_client_mock, + bucket_name="test_bucket", + virus_checker_service=self.virus_checker_service_mock + ) + + storage_service.delete_file("test_file") + + self.s3_client_mock.delete_object.assert_called_once_with( + Bucket="test_bucket", + Key="test_file" + ) + + def test_process_file_if_file_invalid(self): + mock_body = Mock() + mock_body.read.return_value = b"test_file" + self.s3_client_mock.get_object.return_value = { + "Body": mock_body + } + self.virus_checker_service_mock.check_virus.return_value = True + + storage_service = AmazonS3Service( + s3_client=self.s3_client_mock, + bucket_name="test_bucket", + virus_checker_service=self.virus_checker_service_mock + ) + + with self.assertRaises(RuntimeError): + storage_service.process_file("test_file", FileType.JPEG) + + def test_process_file_if_file_is_virus(self): + mock_body = Mock() + mock_body.read.return_value = b"test_file" + self.s3_client_mock.get_object.return_value = { + "Body": mock_body + } + + mock_file_type = Mock() + mock_file_type.get_validator.return_value = lambda x: x + mock_file_type.get_content_type.return_value = "image/fake" + + self.virus_checker_service_mock.check_virus.return_value = False + + storage_service = AmazonS3Service( + s3_client=self.s3_client_mock, + bucket_name="test_bucket", + virus_checker_service=self.virus_checker_service_mock + ) + + with self.assertRaises(ValueError): + storage_service.process_file("test_file", mock_file_type) + + def test_process_file(self): + mock_body = Mock() + mock_body.read.return_value = b"test_file" + self.s3_client_mock.get_object.return_value = { + "Body": mock_body + } + self.virus_checker_service_mock.check_virus.return_value = True + + mock_file_type = Mock() + mock_file_type.get_validator.return_value = lambda x: x + mock_file_type.get_content_type.return_value = "image/fake" + + storage_service = AmazonS3Service( + s3_client=self.s3_client_mock, + bucket_name="test_bucket", + virus_checker_service=self.virus_checker_service_mock + ) + + storage_service.process_file("test_file", mock_file_type) + + self.s3_client_mock.upload_fileobj.assert_called() diff --git a/tests/virus_checker_service/__init__.py b/tests/virus_checker_service/__init__.py new file mode 100644 index 0000000..e69de29