holetron-lab/fleet-memory
1
0
Fork 0
Code Releases Packages Activity Actions

fix(docker): add retry logic for ML model downloads (#248)

Browse source 
- Add 3 retries with exponential backoff (10s -> 20s -> 40s)
- Set HF_HUB_DOWNLOAD_TIMEOUT=600 for longer timeout
- Fixes transient network failures during HuggingFace downloads
- Applied to both api-only and standalone stages

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Anton Evseev 2026-01-30 18:42:16 +10:00 committed by GitHub
parent 381c96c093
commit ecc590cb79
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 38 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
docker/standalone/Dockerfile
View file

@ -169,16 +169,33 @@ ENV PATH="/app/api/.venv/bin:${PATH}"
# Pre-download ML models to avoid runtime download (conditional)
# Only runs if both PRELOAD_ML_MODELS=true AND INCLUDE_LOCAL_MODELS=true
# Includes retry logic with exponential backoff for transient network failures
ARG PRELOAD_ML_MODELS
ARG INCLUDE_LOCAL_MODELS
ENV HF_HUB_DOWNLOAD_TIMEOUT=600
RUN if [ "$PRELOAD_ML_MODELS" = "true" ] && [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
/app/api/.venv/bin/python -c "\
MAX_RETRIES=3; \
RETRY_DELAY=10; \
for i in $(seq 1 $MAX_RETRIES); do \
echo "Attempt $i/$MAX_RETRIES: Downloading ML models..."; \
/app/api/.venv/bin/python -c "\
import os; os.environ['HF_HUB_DOWNLOAD_TIMEOUT'] = '600'; \
from sentence_transformers import SentenceTransformer, CrossEncoder; \
print('Downloading embedding model...'); \
SentenceTransformer('BAAI/bge-small-en-v1.5'); \
print('Downloading cross-encoder model...'); \
CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'); \
print('Models cached successfully')"; \
print('Models cached successfully')" && break; \
if [ $i -lt $MAX_RETRIES ]; then \
echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
sleep $RETRY_DELAY; \
RETRY_DELAY=$((RETRY_DELAY * 2)); \
fi; \
done; \
if [ $i -eq $MAX_RETRIES ] && ! /app/api/.venv/bin/python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('BAAI/bge-small-en-v1.5')" 2>/dev/null; then \
echo "ERROR: Failed to download models after $MAX_RETRIES attempts"; \
exit 1; \
fi; \
elif [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then echo "Skipping ML model preload (local-models not included)"; \
else echo "Skipping ML model preload"; fi
@ -277,16 +294,33 @@ ENV PATH="/app/api/.venv/bin:${PATH}"
# Pre-download ML models to avoid runtime download (conditional)
# Only runs if both PRELOAD_ML_MODELS=true AND INCLUDE_LOCAL_MODELS=true
# Includes retry logic with exponential backoff for transient network failures
ARG PRELOAD_ML_MODELS
ARG INCLUDE_LOCAL_MODELS
ENV HF_HUB_DOWNLOAD_TIMEOUT=600
RUN if [ "$PRELOAD_ML_MODELS" = "true" ] && [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
/app/api/.venv/bin/python -c "\
MAX_RETRIES=3; \
RETRY_DELAY=10; \
for i in $(seq 1 $MAX_RETRIES); do \
echo "Attempt $i/$MAX_RETRIES: Downloading ML models..."; \
/app/api/.venv/bin/python -c "\
import os; os.environ['HF_HUB_DOWNLOAD_TIMEOUT'] = '600'; \
from sentence_transformers import SentenceTransformer, CrossEncoder; \
print('Downloading embedding model...'); \
SentenceTransformer('BAAI/bge-small-en-v1.5'); \
print('Downloading cross-encoder model...'); \
CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'); \
print('Models cached successfully')"; \
print('Models cached successfully')" && break; \
if [ $i -lt $MAX_RETRIES ]; then \
echo "Attempt $i failed, retrying in ${RETRY_DELAY}s..."; \
sleep $RETRY_DELAY; \
RETRY_DELAY=$((RETRY_DELAY * 2)); \
fi; \
done; \
if [ $i -eq $MAX_RETRIES ] && ! /app/api/.venv/bin/python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('BAAI/bge-small-en-v1.5')" 2>/dev/null; then \
echo "ERROR: Failed to download models after $MAX_RETRIES attempts"; \
exit 1; \
fi; \
elif [ "$INCLUDE_LOCAL_MODELS" != "true" ]; then echo "Skipping ML model preload (local-models not included)"; \
else echo "Skipping ML model preload"; fi