autoupgrade: a detached job is not "serving a request"

Proven on zeiss with a real LoRA training run: the skip fired correctly but
read "SKIP: serving 1 request(s) [engines=0 jobs=1]". There was no request --
that is the entire case this check exists for, since /v1/loras/train with
wait=false returns at once and the work continues for hours. The log now says
"work in progress", which is what it means.
Co-Authored-By: 's avatarClaude Opus 5 <noreply@anthropic.com>
parent dadb0196
...@@ -167,7 +167,9 @@ inflight(){ ...@@ -167,7 +167,9 @@ inflight(){
echo "$best $detail" echo "$best $detail"
} }
# Serving anything, or we cannot tell? Then this run does nothing. # Any work in progress -- a request OR a detached job like training -- or we
# cannot tell? Then this run does nothing. Calling it "serving" was wrong:
# the case this exists for has no request in flight at all.
guard_idle(){ guard_idle(){
local out count detail local out count detail
out="$(inflight)"; count="${out%% *}"; detail="${out#* }" out="$(inflight)"; count="${out%% *}"; detail="${out#* }"
...@@ -182,7 +184,7 @@ guard_idle(){ ...@@ -182,7 +184,7 @@ guard_idle(){
return 0 return 0
fi fi
if [ "$count" -gt 0 ]; then if [ "$count" -gt 0 ]; then
say "SKIP: serving $count request(s) [$detail] — leaving the image alone" say "SKIP: $count piece(s) of work in progress [$detail] — leaving the image alone"
return 1 return 1
fi fi
say "idle ($detail)" say "idle ($detail)"
...@@ -255,16 +257,16 @@ OUT="$(inflight)"; BUSY="${OUT%% *}"; WHAT="${OUT#* }" ...@@ -255,16 +257,16 @@ OUT="$(inflight)"; BUSY="${OUT%% *}"; WHAT="${OUT#* }"
if [ "$BUSY" = unknown ] || { [ "$BUSY" -gt 0 ] 2>/dev/null; }; then if [ "$BUSY" = unknown ] || { [ "$BUSY" -gt 0 ] 2>/dev/null; }; then
if [ "$N" -lt "$MAX_DEFERRALS" ]; then if [ "$N" -lt "$MAX_DEFERRALS" ]; then
echo $((N + 1)) > "$DEFERRALS" echo $((N + 1)) > "$DEFERRALS"
say "serving [$WHAT] — deferring the restart ($((N + 1))/$MAX_DEFERRALS)" say "work in progress [$WHAT] — deferring the restart ($((N + 1))/$MAX_DEFERRALS)"
exit 0 exit 0
fi fi
if [ "$MAX_DEFERRALS" -le 0 ]; then if [ "$MAX_DEFERRALS" -le 0 ]; then
echo $((N + 1)) > "$DEFERRALS" echo $((N + 1)) > "$DEFERRALS"
say "serving [$WHAT] — the new code is in the image and waits for an idle" say "work in progress [$WHAT] — the new code is in the image and waits for an idle"
say " moment ($((N + 1)) run(s) so far). Set MAX_DEFERRALS>0 to restart anyway." say " moment ($((N + 1)) run(s) so far). Set MAX_DEFERRALS>0 to restart anyway."
exit 0 exit 0
fi fi
say "serving [$WHAT] but $N deferrals already — restarting anyway" say "work in progress [$WHAT] but $N deferrals already — restarting anyway"
fi fi
if restart_and_verify; then if restart_and_verify; then
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment