From 4d0ad4b104d1dd43e2e52a3b2ca2b4a82a758c15 Mon Sep 17 00:00:00 2001 From: Vitor Hideyoshi Date: Wed, 23 Aug 2023 09:35:17 -0300 Subject: [PATCH] Updates Dependencies, Creates Dockerfile and Formats Code --- .github/workflows/docker-publish.yml | 31 +++++++++++++ Dockerfile | 28 +++++++++++ poetry.lock | 46 +++++++++---------- .../config/config_allowed_origins.py | 6 +-- storage_service/controller/__init__.py | 1 - .../controller/storage_controller.py | 20 ++++---- storage_service/service/amazon_s3_service.py | 1 - storage_service/worker/storage_file_worker.py | 4 +- 8 files changed, 98 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 Dockerfile diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..91f9315 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,31 @@ +name: ci + +on: + push: + branches: + - 'main' + +jobs: + + docker: + + runs-on: ubuntu-latest + steps: + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + push: true + tags: yoshiunfriendly/storage-hideyoshi.com:latest \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..150a16c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.10 +LABEL authors="hideyoshi" + +# Configure Poetry +ENV POETRY_VERSION=1.5.1 +ENV POETRY_HOME=/opt/poetry +ENV POETRY_VENV=/opt/poetry-venv +ENV POETRY_CACHE_DIR=/opt/.cache + +# Install poetry separated from system interpreter +RUN python3 -m venv $POETRY_VENV \ + && $POETRY_VENV/bin/pip install -U pip setuptools \ + && $POETRY_VENV/bin/pip install poetry==${POETRY_VERSION} + +# Add `poetry` to PATH +ENV PATH="${PATH}:${POETRY_VENV}/bin" + +WORKDIR /app + +# Install dependencies +COPY . . + +RUN poetry install + +EXPOSE 5000-9000 + +# Run your app +CMD [ "poetry", "run", "python", "-m", "storage_service" ] \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 0258c01..2d7b19d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -23,13 +23,13 @@ trio = ["trio (<0.22)"] [[package]] name = "async-timeout" -version = "4.0.2" +version = "4.0.3" description = "Timeout context manager for asyncio programs" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, ] [[package]] @@ -80,17 +80,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.28.21" +version = "1.28.32" description = "The AWS SDK for Python" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.28.21-py3-none-any.whl", hash = "sha256:28e1ea098e43e764d990b4466e377c322b5d57829e2eb1395eca52d4209a6c11"}, - {file = "boto3-1.28.21.tar.gz", hash = "sha256:0ad6932b2469f4fa4e63f4baf8508ccc1b1bc215b9c835df73505aa85210fc27"}, + {file = "boto3-1.28.32-py3-none-any.whl", hash = "sha256:ed787f250ce2562c7744395bdf32b5a7bc9184126ef50a75e97bcb66043dccf3"}, + {file = "boto3-1.28.32.tar.gz", hash = "sha256:b505faa126db84e226f6f8d242a798fae30a725f0cac8a76c6aca9ace4e8eb28"}, ] [package.dependencies] -botocore = ">=1.31.21,<1.32.0" +botocore = ">=1.31.32,<1.32.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -99,13 +99,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.31.21" +version = "1.31.32" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.31.21-py3-none-any.whl", hash = "sha256:c20a5c46eaf49b18b76fdfaec5583320e18abd551b1bc3cd7b1e718372675e21"}, - {file = "botocore-1.31.21.tar.gz", hash = "sha256:9a13736b16aea3f16829b00edfb2c656fee72ecbfe5eb396cc2f8632e31fd524"}, + {file = "botocore-1.31.32-py3-none-any.whl", hash = "sha256:8992ac186988c4b4cc168e8e479e9472da1442b193c1bf7c9dcd1877ec62d23c"}, + {file = "botocore-1.31.32.tar.gz", hash = "sha256:7a07d8dc8cc47bf23af39409ada81f388eb78233e1bb2cde0c415756da753664"}, ] [package.dependencies] @@ -213,13 +213,13 @@ files = [ [[package]] name = "click" -version = "8.1.6" +version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, - {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [package.dependencies] @@ -238,13 +238,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.1.2" +version = "1.1.3" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, - {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, + {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, + {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, ] [package.extras] @@ -252,13 +252,13 @@ test = ["pytest (>=6)"] [[package]] name = "fastapi" -version = "0.101.0" +version = "0.101.1" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.7" files = [ - {file = "fastapi-0.101.0-py3-none-any.whl", hash = "sha256:494eb3494d89e8079c20859d7ca695f66eaccc40f46fe8c75ab6186d15f05ffd"}, - {file = "fastapi-0.101.0.tar.gz", hash = "sha256:ca2ae65fe42f6a34b5cf6c994337149154b1b400c39809d7b2dccdceb5ae77af"}, + {file = "fastapi-0.101.1-py3-none-any.whl", hash = "sha256:aef5f8676eb1b8389952e1fe734abe20f04b71f6936afcc53b320ba79b686a4b"}, + {file = "fastapi-0.101.1.tar.gz", hash = "sha256:7b32000d14ca9992f7461117b81e4ef9ff0c07936af641b4fe40e67d5f9d63cb"}, ] [package.dependencies] @@ -675,13 +675,13 @@ redis = ">=4.0.0" [[package]] name = "s3transfer" -version = "0.6.1" +version = "0.6.2" description = "An Amazon S3 Transfer Manager" optional = false python-versions = ">= 3.7" files = [ - {file = "s3transfer-0.6.1-py3-none-any.whl", hash = "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346"}, - {file = "s3transfer-0.6.1.tar.gz", hash = "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9"}, + {file = "s3transfer-0.6.2-py3-none-any.whl", hash = "sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084"}, + {file = "s3transfer-0.6.2.tar.gz", hash = "sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861"}, ] [package.dependencies] diff --git a/storage_service/config/config_allowed_origins.py b/storage_service/config/config_allowed_origins.py index 36c5e94..4fbbd5e 100644 --- a/storage_service/config/config_allowed_origins.py +++ b/storage_service/config/config_allowed_origins.py @@ -1,7 +1,7 @@ -import os - from dotenv import load_dotenv +import os + def get_allowed_origins(): load_dotenv() @@ -11,4 +11,4 @@ def get_allowed_origins(): if origins is None: return [] - return origins.split(",") \ No newline at end of file + return origins.split(",") diff --git a/storage_service/controller/__init__.py b/storage_service/controller/__init__.py index aaaf5d4..164de38 100644 --- a/storage_service/controller/__init__.py +++ b/storage_service/controller/__init__.py @@ -4,7 +4,6 @@ from storage_service.controller.storage_controller import s3_router from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware - app = FastAPI() diff --git a/storage_service/controller/storage_controller.py b/storage_service/controller/storage_controller.py index ca09890..2c4793c 100644 --- a/storage_service/controller/storage_controller.py +++ b/storage_service/controller/storage_controller.py @@ -20,14 +20,16 @@ s3_router = InferringRouter() @cbv(s3_router) class StorageController: queue: Queue = Depends(dependency_queue, use_cache=True) - storage_service: StorageService = Depends(dependency_storage_service, use_cache=True) + storage_service: StorageService = Depends( + dependency_storage_service, use_cache=True + ) @s3_router.post("/new_file_url/", status_code=200) def new_file_url( - self, - username: Annotated[str, Body(embed=True)], - file_postfix: Annotated[str, Body(embed=True)], - file_type: Annotated[FileType, Body(embed=True)], + self, + username: Annotated[str, Body(embed=True)], + file_postfix: Annotated[str, Body(embed=True)], + file_type: Annotated[FileType, Body(embed=True)], ) -> dict[str, str]: return self.storage_service.get_temp_upload_link( file_name_hash(username, file_postfix), file_type @@ -40,7 +42,9 @@ class StorageController: ) @s3_router.post("/process_file/", status_code=200) - def process_file(self, - username: Annotated[str, Body(embed=True)], - file_postfix: Annotated[str, Body(embed=True)]): + def process_file( + self, + username: Annotated[str, Body(embed=True)], + file_postfix: Annotated[str, Body(embed=True)], + ): self.queue.enqueue(storage_file_worker, username, file_postfix) diff --git a/storage_service/service/amazon_s3_service.py b/storage_service/service/amazon_s3_service.py index a400c7a..a23bbb2 100644 --- a/storage_service/service/amazon_s3_service.py +++ b/storage_service/service/amazon_s3_service.py @@ -5,7 +5,6 @@ from storage_service.utils.enums.file_type import FileType from storage_service.utils.file_handler import FILE_HANDLER import boto3 -from PIL import Image import io from typing import Any diff --git a/storage_service/worker/storage_file_worker.py b/storage_service/worker/storage_file_worker.py index 6e03ff2..fd74d46 100644 --- a/storage_service/worker/storage_file_worker.py +++ b/storage_service/worker/storage_file_worker.py @@ -6,6 +6,4 @@ from storage_service.utils.file_name_hash import file_name_hash def storage_file_worker(username: str, file_postfix: str) -> None: - dependency_storage_service().process_file( - file_name_hash(username, file_postfix) - ) + dependency_storage_service().process_file(file_name_hash(username, file_postfix))