Stop Downloading 8GB Models on Every Pod Restart - Use OCI Object Storage as a Model Cache

# Download model locally
pip install huggingface_hub
python3 -c "
from huggingface_hub import snapshot_download
snapshot_download('microsoft/Phi-3-mini-4k-instruct', local_dir='./phi3-mini')
"

# Create OCI bucket
oci os bucket create \
  --compartment-id $COMPARTMENT_ID \
  --name ai-model-cache \
  --storage-tier Standard

# Upload model files
oci os object bulk-upload \
  --bucket-name ai-model-cache \
  --src-dir ./phi3-mini \
  --prefix "models/phi3-mini/" \
  --content-type application/octet-stream

apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-inference
spec:
  replicas: 1
  selector:
    matchLabels:
      app: vllm
  template:
    metadata:
      labels:
        app: vllm
    spec:
      initContainers:
        - name: model-loader
          image: ghcr.io/oracle/oci-cli:latest
          command:
            - /bin/bash
            - -c
            - |
              # Skip if model already cached on PVC
              if [ -f /models/phi3-mini/.complete ]; then
                echo "Model already cached, skipping download"
                exit 0
              fi

              echo "Downloading model from OCI Object Storage..."
              oci os object bulk-download \
                --bucket-name ai-model-cache \
                --prefix "models/phi3-mini/" \
                --download-dir /models/phi3-mini \
                --auth instance_principal

              # Mark as complete so we don't re-download
              touch /models/phi3-mini/.complete
              echo "Model download complete"
          volumeMounts:
            - name: model-cache
              mountPath: /models

      containers:
        - name: vllm
          image: iad.ocir.io/mytenancy/vllm:v1
          args:
            - "--model"
            - "/models/phi3-mini"
            - "--max-model-len"
            - "4096"
          ports:
            - containerPort: 8000
          resources:
            limits:
              nvidia.com/gpu: 1
              memory: 32Gi
          volumeMounts:
            - name: model-cache
              mountPath: /models

      volumes:
        - name: model-cache
          persistentVolumeClaim:
            claimName: model-cache-pvc

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: model-cache-pvc
spec:
  accessModes: ["ReadWriteOnce"]
  storageClassName: oci-bv
  resources:
    requests:
      storage: 50Gi

# Terraform — allow GPU nodes to read model bucket
resource "oci_identity_policy" "model_access" {
  compartment_id = var.tenancy_id
  name           = "gpu-nodes-model-access"
  description    = "Allow GPU node pool to read model cache bucket"

  statements = [
    "Allow dynamic-group gpu-node-pool to read objects in compartment ${var.compartment_name} where target.bucket.name='ai-model-cache'"
  ]
}

oci os object bulk-upload --bucket-name ai-model-cache \
  --src-dir ./llama3-8b --prefix "models/llama3-8b/"

oci os object bulk-upload --bucket-name ai-model-cache \
  --src-dir ./mistral-7b --prefix "models/mistral-7b/"

env:
  - name: MODEL_NAME
    value: "llama3-8b"
command:
  - /bin/bash
  - -c
  - |
    if [ -f /models/$MODEL_NAME/.complete ]; then exit 0; fi
    oci os object bulk-download \
      --bucket-name ai-model-cache \
      --prefix "models/$MODEL_NAME/" \
      --download-dir /models/$MODEL_NAME \
      --auth instance_principal
    touch /models/$MODEL_NAME/.complete