| Loading...
Azure Blob Storage
, AWS
, or MongoDB
. The aim is to deploy a data object server (Blob
) and a database service in production.API
with the FastApi
framework to facilitate downloading, adding, updating or deleting files, and a cluster Minio
.In the case of my site www.roo7690.me for example, the blog pages are markdown files that the server retrieves from the server blob storage and transforms into html before delivering it.
Minio
instance, to which we'll connect.aodocker run -d --name i_minio -p 7090:7090 -p 7091:7091 \ -e "MINIO_ROOT_USER=user" \ -e "MINIO_ROOT_PASSWORD=password" \ -v vol_data:/data -v vol_config:/root/.minio \ server --address ":7090" --console-address ":7091"
FastApi
framework we need to use the Uvicorn
package.aos/api/__main__.pimport uvicorn,os from argparse import ArgumentParser,ArgumentTypeError def getServerOptions(): parser=ArgumentParser() parser.add_argument('-p',type=int,help='server port') parser.add_argument('-H',type=str,help='server hostname') parser.add_argument('-M',type=str,help='Server launch mode') args=parser.parse_args() if args.M not in ["dev","prod"]: raise ArgumentTypeError("server launch mode not specified (-M prod or dev)") return { "server_address":{"hostname":args.H or 'localhost',"port":args.p or 7090}, "env":{ "mode":args.M } } if __name__=='__main__': opt=getServerOptions() for var, val in opt["env"].items(): os.environ[var]=val uvicorn.run("api.index:app", # application to launch host=opt["server_address"]["hostname"], # hostname to use port=opt["server_address"]["port"], # port on which to run the server log_level='info', reload=True if opt["env"]["mode"]=='dev' else False, proxy_headers=False if opt["env"]["mode"]=='dev' else True) # In production, the server will be proxied behind a nginx server
api
with just the necessary features. Let's start by connecting to the minio server.aos/api/manager/__init__.pfrom minio import Minio from dotenv import load_dotenv import os load_dotenv('.env.local') def key(): key={ "endpoint":'localhost:7090', "access_key":"minioadmin", "secret_key":"minioadmin" } if os.getenv('mode')=='prod': key={ "endpoint":os.getenv('SERVER_NAME'), "access_key":os.getenv('MINIO_ROOT_USER'), "secret_key":os.getenv('MINIO_ROOT_PASSWORD') } return key Manager=Minio(**key(), cert_check=False, secure=False) # the certification file and ssl security will be managed by nginx.
token
on all the application's routes, except on, the routes:aos/api/router/__init__.pfrom fastapi import Request, Response from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint from key import Token # module in my workspaces import os,json from ..manager import Manager # manager of the minio server token=Token(os.getenv('APP_NAME')) # Development of middleware with token verification logic. class SecuriseMiddleware(BaseHTTPMiddleware): async def dispatch(self,req:Request,call_next:RequestResponseEndpoint): if req.url.path!='/' and req.url.path.startswith('/dashboard')==False \ and req.url.path.startswith('/download/public')==False : res=Response() key=req.headers.get('Access-Key') if not key: res.status_code=403 res.body='{"message":"Access-Key missing"}' return res if not token.verifyToken(key): res.status_code=403 res.body='{"message":"Access-Key invalid"}' return res client_id=token.getClient(key) if Manager.bucket_exists(client_id)==False: res.status_code=403 res.body='{"message":"Access denied"}' return res req.state.client_id=client_id return await call_next(req)
upload
, delete
and download
.\ features.
The aim is to set up the following endpoints:endpoint{ "endpoints": [ { "url": "/upload", "method": "POST" }, { "url": "/download/:filepath", "method": "GET" }, { "url": "/delete", "method": "POST" } ] }
/download/:filepath
route, a uri rewriting and redirection middleware, to transform all /download/path/to/file
routes into /download/{path_to_file}
routes.aos/api/router/download.pfrom fastapi import Response,Request from fastapi.responses import RedirectResponse from starlette.middleware.base import BaseHTTPMiddleware,RequestResponseEndpoint from . import splitter class DownloadMiddleware(BaseHTTPMiddleware): async def dispatch(self,req:Request,call_next:RequestResponseEndpoint): if req.url.path.startswith("/download"): nbrSlash=req.url.path.count("/") if nbrSlash>2: url=req.url.path[len("/download/"):] url=url.replace("/",splitter) return RedirectResponse("/download/"+url,status_code=307) pass res=await call_next(req) return res
aos/api/router/download.pfrom fastapi import APIRouter,Response,Request from api.manager import Manager from fastapi import Response,Request from fastapi.responses import RedirectResponse from starlette.middleware.base import BaseHTTPMiddleware,RequestResponseEndpoint from . import splitter class DownloadMiddleware(BaseHTTPMiddleware): ... return res router= APIRouter(prefix='/download') @router.get('/{file}') async def download(file:str,res:Response,req:Request): obj=None response=None file=file.replace(splitter,"/") bucket_name:str if file.startswith("public/"): file=file[len("public/"):] bucket_name="public" else: bucket_name=req.state.client_id try: response=Manager.get_object(bucket_name,file) obj=response.read() except Exception as e: res.status_code=404 return {"erreur":"fichier introuvable"} finally: if response is not None: response.close() response.release_conn() return Response(content=obj,media_type="application/octet-stream",headers={"Content-Disposition":f"attachment; filename={file}"})
aos/api/router/action.pfrom fastapi import APIRouter,Response,Request, \ File, UploadFile, Form from ..manager import Manager import random, string from pydantic import BaseModel from minio.deleteobjects import DeleteObject router= APIRouter() def random_string(): letters= string.ascii_letters return ''.join(random.choice(letters) for i in range(7)) @router.post('/upload') async def upload(req:Request,res:Response, \ file:UploadFile=File(...),to:str=Form(...)): filename=to+'/'+random_string()+'.'+file.filename filename=filename.replace("//","/") try: Manager.put_object(req.state.client_id, filename,file.file,file.size, content_type=file.content_type) except Exception as e: res.status_code=500 return {"erreur":"unsaved file"} return {"filename":filename} class Object(BaseModel): name:str|None=None names:list[str]|None=None @router.post("/delete") async def delete(req:Request,res:Response,object:Object): if not ((object.name!=None) ^ (object.names!=None)): res.status_code=400 return {"erreur":"missing or too many filenames with names property"} try: errors=[] if object.name!=None: Manager.remove_object(req.state.client_id,object.name) else: objects=[] for name in object.names: objects.append(DeleteObject(name)) errors=Manager.remove_objects(req.state.client_id,objects) except Exception as e: res.status_code=500 return {"erreur":"an error occurred while deleting the file"} finally: for error in errors: print(f"an error occurred when deleting {error.name}",error) return {"message":f"file(s) {object.name or object.names} delete"}
FastApi
, lol.aos/api/index.pfrom fastapi import FastAPI from .router import SecuriseMiddleware, router as _home from .router.dashboard import router as dashboard_router from .router.download import router as download_router, \ DownloadMiddleware from .router.action import router as action_router app= FastAPI() app.add_middleware(SecuriseMiddleware) app.add_middleware(DownloadMiddleware) app.include_router(_home) app.include_router(dashboard_router) app.include_router(download_router) app.include_router(action_router)
aos/docker-compose.ymversion: '1.0.0' networks: object-storage: driver: bridge x-minio-client: &minio-client # property anchoring image: quay.io/minio/minio:latest env_file: - .env.local # environment variables such as username, password, and console-address path rewriting strategy networks: - object-storage command: server --address ":7090" --console-address ":7091" <http://minio-client>{1...4}/data # command to be run at container launch # `http://minio-client{1...4}/data` specifies all the agents that will form the cluster healthcheck: # command to be performed within a given interval, to determine container health test: ["CMD", "curl", "-f", "<http://localhost:7090/minio/health/live>"] interval: 30s timeout: 20s retries: 3 services: # Setting up the cluster ##[ minio-client1: <<: *minio-client container_name: minio1.object-storage volumes: - minio1_data:/data minio-client2: <<: *minio-client container_name: minio2.object-storage volumes: - minio2_data:/data minio-client3: <<: *minio-client container_name: minio3.object-storage volumes: - minio3_data:/data minio-client4: <<: *minio-client container_name: minio4.object-storage volumes: - minio4_data:/data ##] nginx: container_name: nginx.object-storage image: nginx ports: - '8081:8081' # port on which to connect to the stack volumes: - ./nginx.conf:/etc/nginx/conf.d/default.conf # nginx server configuration (TODO: facilitate traffic to minio storage with nginx configuration) networks: - object-storage api: container_name: api.object-storage image: api.object-storage:1.0.0 build: dockerfile: api.dockerfile # dockerfile for building the api image (TODO) networks: - object-storage volumes: minio1_data: minio2_data: minio3_data: minio4_data:
aos/api.dockerfilFROM python:3.12 WORKDIR /api COPY ./requirements.txt /api/requirements.txt # file containing the necessary packages. COPY ./.venv/packages /tmp/packages # archive and copy the packages into my workspaces that I used RUN pip install --no-cache-dir --upgrade -r /api/requirements.txt COPY ./.env.local /api/.env.local COPY ./api /api/app COPY ./res /api/res CMD [ "python","-m","app","-H","0.0.0.0","-M","prod" ]
```nginaos/nginx.conf server { listen 8081; server_name localhost; return 301 $scheme://blob.roo7690.me$request_uri; } upstream admin_store { least_conn; server minio-client1:7091; server minio-client2:7091; server minio-client3:7091; server minio-client4:7091; } upstream server_store{ least_conn; server minio-client1:7090; server minio-client2:7090; server minio-client3:7090; server minio-client4:7090; } server { listen 8081; server_name blob.roo7690.me blob.roo7690.server; client_max_body_size 0; location /admin/ { # direct /admin to minio's console-address. To do this, you also need to set the environment variable # MINIO_BROWSER_REDIRECT_URL=http://this/admin in the .env.local file rewrite ^/admin/(.*) /$1 break; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-NginX-Proxy true; real_ip_header X-Real-IP; proxy_connect_timeout 300; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; chunked_transfer_encoding off; proxy_pass http://admin_store/; } location / { proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection ''; proxy_redirect off; proxy_buffering off; proxy_pass <http://api:7090/>; } } server { listen 8081; server_name nginx; client_max_body_size 0; location / { proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header XX-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_connect_timeout 300; proxy_set_header Connection ""; chunked_transfer_encoding off; proxy_pass http://server_store; } }
blob.roo7690.me
.