What's Minio?

In the case of my site www.roo7690.me for example, the blog pages are markdown files that the server retrieves from the server blob storage and transforms into html before delivering it.

Developing the API

aos
docker run -d --name i_minio -p 7090:7090 -p 7091:7091 \
-e "MINIO_ROOT_USER=user" \
-e "MINIO_ROOT_PASSWORD=password" \
-v vol_data:/data -v vol_config:/root/.minio \
server --address ":7090" --console-address ":7091"

aos/api/__main__.py
import uvicorn,os
from argparse import ArgumentParser,ArgumentTypeError


def getServerOptions():
  parser=ArgumentParser()
  parser.add_argument('-p',type=int,help='server port')
  parser.add_argument('-H',type=str,help='server hostname')
  parser.add_argument('-M',type=str,help='Server launch mode')
  args=parser.parse_args()

  if args.M not in ["dev","prod"]:
    raise ArgumentTypeError("server launch mode not specified (-M prod or dev)")

  return {
    "server_address":{"hostname":args.H or 'localhost',"port":args.p or 7090},
    "env":{
      "mode":args.M
    }
  }

if __name__=='__main__':
  opt=getServerOptions()

  for var, val in opt["env"].items():
    os.environ[var]=val

  uvicorn.run("api.index:app", # application to launch
    host=opt["server_address"]["hostname"], # hostname to use
    port=opt["server_address"]["port"], # port on which to run the server
    log_level='info',
    reload=True if opt["env"]["mode"]=='dev' else False,
    proxy_headers=False if opt["env"]["mode"]=='dev' else True) # In production, the server will be proxied behind a nginx server

aos/api/manager/__init__.py
from minio import Minio
from dotenv import load_dotenv
import os

load_dotenv('.env.local')

def key():
  key={
    "endpoint":'localhost:7090',
    "access_key":"minioadmin",
    "secret_key":"minioadmin"
  }
  if os.getenv('mode')=='prod':
    key={
      "endpoint":os.getenv('SERVER_NAME'),
      "access_key":os.getenv('MINIO_ROOT_USER'),
      "secret_key":os.getenv('MINIO_ROOT_PASSWORD')
    }
  return key

Manager=Minio(**key(),
  cert_check=False,
  secure=False) # the certification file and ssl security will be managed by nginx.

• / # home page
• /dashboard/(.*) # for our administration area
• /download/public/(.*) # for downloading public data.

aos/api/router/__init__.py
from fastapi import Request, Response
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
from key import Token # module in my workspaces
import os,json
from ..manager import Manager # manager of the minio server

token=Token(os.getenv('APP_NAME'))

# Development of middleware with token verification logic.
class SecuriseMiddleware(BaseHTTPMiddleware):
  async def dispatch(self,req:Request,call_next:RequestResponseEndpoint):
    if req.url.path!='/' and req.url.path.startswith('/dashboard')==False \
      and req.url.path.startswith('/download/public')==False :
      res=Response()
      key=req.headers.get('Access-Key')
      if not key:
        res.status_code=403
        res.body='{"message":"Access-Key missing"}'
        return res
      if not token.verifyToken(key):
        res.status_code=403
        res.body='{"message":"Access-Key invalid"}'
        return res
      client_id=token.getClient(key)
      if Manager.bucket_exists(client_id)==False:
        res.status_code=403
        res.body='{"message":"Access denied"}'
        return res
      req.state.client_id=client_id
    return await call_next(req)

endpoints
{
  "endpoints": [
    {
      "url": "/upload",
      "method": "POST"
    },
    {
      "url": "/download/:filepath",
      "method": "GET"
    },
    {
      "url": "/delete",
      "method": "POST"
    }
  ]
}

aos/api/router/download.py
from fastapi import Response,Request
from fastapi.responses import RedirectResponse
from starlette.middleware.base import BaseHTTPMiddleware,RequestResponseEndpoint
from . import splitter

class DownloadMiddleware(BaseHTTPMiddleware):
  async def dispatch(self,req:Request,call_next:RequestResponseEndpoint):
    if req.url.path.startswith("/download"):
      nbrSlash=req.url.path.count("/")
      if nbrSlash>2:
        url=req.url.path[len("/download/"):]
        url=url.replace("/",splitter)
        return RedirectResponse("/download/"+url,status_code=307)
      pass
    res=await call_next(req)
    return res

aos/api/router/download.py
from fastapi import APIRouter,Response,Request 
from api.manager import Manager 
from fastapi import Response,Request 
from fastapi.responses import RedirectResponse
from starlette.middleware.base import BaseHTTPMiddleware,RequestResponseEndpoint
from . import splitter

class DownloadMiddleware(BaseHTTPMiddleware):
  ...
  return res

router= APIRouter(prefix='/download')
@router.get('/{file}')
async def download(file:str,res:Response,req:Request):
  obj=None
  response=None
  file=file.replace(splitter,"/")
  bucket_name:str
  if file.startswith("public/"):
    file=file[len("public/"):]
    bucket_name="public"
  else:
    bucket_name=req.state.client_id

  try:
    response=Manager.get_object(bucket_name,file)
    obj=response.read()
  except Exception as e:
    res.status_code=404
    return {"erreur":"fichier introuvable"}
  finally:
    if response is not None:
      response.close()
      response.release_conn()
  return Response(content=obj,media_type="application/octet-stream",headers={"Content-Disposition":f"attachment; filename={file}"})

aos/api/router/action.py
from fastapi import APIRouter,Response,Request, \
  File, UploadFile, Form
from ..manager import Manager
import random, string
from pydantic import BaseModel
from minio.deleteobjects import DeleteObject

router= APIRouter()

def random_string():
  letters= string.ascii_letters
  return ''.join(random.choice(letters) for i in range(7))

@router.post('/upload')
async def upload(req:Request,res:Response, \
  file:UploadFile=File(...),to:str=Form(...)):
  filename=to+'/'+random_string()+'.'+file.filename
  filename=filename.replace("//","/")
  try:
    Manager.put_object(req.state.client_id,
      filename,file.file,file.size,
      content_type=file.content_type)
  except Exception as e:
    res.status_code=500
    return {"erreur":"unsaved file"}
  return {"filename":filename}

class Object(BaseModel):
  name:str|None=None
  names:list[str]|None=None

@router.post("/delete")
async def delete(req:Request,res:Response,object:Object):
  if not ((object.name!=None) ^ (object.names!=None)):
    res.status_code=400
    return {"erreur":"missing or too many filenames with names property"}
  try:
    errors=[]
    if object.name!=None:
      Manager.remove_object(req.state.client_id,object.name)
    else:
      objects=[]
      for name in object.names:
        objects.append(DeleteObject(name))
      errors=Manager.remove_objects(req.state.client_id,objects)
  except Exception as e:
    res.status_code=500
    return {"erreur":"an error occurred while deleting the file"}
  finally:
    for error in errors:
      print(f"an error occurred when deleting {error.name}",error)

  return {"message":f"file(s) {object.name or object.names} delete"}

aos/api/index.py
from fastapi import FastAPI
from .router import SecuriseMiddleware, router as _home
from .router.dashboard import router as dashboard_router
from .router.download import router as download_router, \
  DownloadMiddleware
from .router.action import router as action_router

app= FastAPI()
app.add_middleware(SecuriseMiddleware)
app.add_middleware(DownloadMiddleware)

app.include_router(_home)
app.include_router(dashboard_router)
app.include_router(download_router)
app.include_router(action_router)

aos/docker-compose.yml
version: '1.0.0'

networks:
  object-storage:
    driver: bridge

x-minio-client: &minio-client # property anchoring
  image: quay.io/minio/minio:latest
  env_file:
    - .env.local
    # environment variables such as username, password, and console-address path rewriting strategy
  networks:
    - object-storage
  command: server --address ":7090" --console-address ":7091" <http://minio-client>{1...4}/data  # command to be run at container launch
  # `http://minio-client{1...4}/data` specifies all the agents that will form the cluster

  healthcheck: # command to be performed within a given interval, to determine container health
    test: ["CMD", "curl", "-f", "<http://localhost:7090/minio/health/live>"]
    interval: 30s
    timeout: 20s
    retries: 3

services:
  # Setting up the cluster
  ##[
  minio-client1:
    <<: *minio-client
    container_name: minio1.object-storage
    volumes:
      - minio1_data:/data

  minio-client2:
    <<: *minio-client
    container_name: minio2.object-storage
    volumes:
      - minio2_data:/data

  minio-client3:
    <<: *minio-client
    container_name: minio3.object-storage
    volumes:
      - minio3_data:/data

  minio-client4:
    <<: *minio-client
    container_name: minio4.object-storage
    volumes:
      - minio4_data:/data
  ##]

  nginx:
    container_name: nginx.object-storage
    image: nginx
    ports:
      - '8081:8081'  # port on which to connect to the stack
    volumes:
      - ./nginx.conf:/etc/nginx/conf.d/default.conf  # nginx server configuration (TODO: facilitate traffic to minio storage with nginx configuration)
    networks:
      - object-storage

  api:
    container_name: api.object-storage
    image: api.object-storage:1.0.0
    build:
      dockerfile: api.dockerfile  # dockerfile for building the api image (TODO)
    networks:
      - object-storage

volumes:
  minio1_data:
  minio2_data:
  minio3_data:
  minio4_data:

aos/api.dockerfile
FROM python:3.12

WORKDIR /api

COPY ./requirements.txt /api/requirements.txt  # file containing the necessary packages.
COPY ./.venv/packages /tmp/packages # archive and copy the packages into my workspaces that I used
RUN pip install --no-cache-dir --upgrade -r /api/requirements.txt

COPY ./.env.local /api/.env.local
COPY ./api /api/app
COPY ./res /api/res

CMD [ "python","-m","app","-H","0.0.0.0","-M","prod" ]

```nginx
aos/nginx.conf
server {
  listen 8081;
  server_name localhost;

  return 301 $scheme://blob.roo7690.me$request_uri;
}

upstream admin_store {
  least_conn;
  server minio-client1:7091;
  server minio-client2:7091;
  server minio-client3:7091;
  server minio-client4:7091;
}
upstream server_store{
  least_conn;
  server minio-client1:7090;
  server minio-client2:7090;
  server minio-client3:7090;
  server minio-client4:7090;
}

server {
  listen 8081;
  server_name blob.roo7690.me blob.roo7690.server;
  client_max_body_size 0;

  location /admin/ {
    # direct /admin to minio's console-address. To do this, you also need to set the environment variable
    # MINIO_BROWSER_REDIRECT_URL=http://this/admin in the .env.local file
    rewrite ^/admin/(.*) /$1 break;
    proxy_set_header Host $http_host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    proxy_set_header X-NginX-Proxy true;
    real_ip_header X-Real-IP;
    proxy_connect_timeout 300;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection "upgrade";
    chunked_transfer_encoding off;
    proxy_pass http://admin_store/;
  }

  location / {
    proxy_set_header Host $http_host;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection '';
    proxy_redirect off;
    proxy_buffering off;
    proxy_pass <http://api:7090/>;
  }
}

server {
  listen 8081;
  server_name nginx;
  client_max_body_size 0;

    location / {
    proxy_set_header Host $http_host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header XX-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    proxy_connect_timeout 300;
    proxy_set_header Connection "";
    chunked_transfer_encoding off;
    proxy_pass http://server_store;
  }
}