Skip to content

Commit a7bb653

Browse files
MrudhulrajMrudhulraj
authored
Merge branch 'apache:main' into fix/dataset-fetch-query-fix
2 parents 594a697 + 3f87773 commit a7bb653

8 files changed

Lines changed: 990 additions & 11 deletions

File tree

‎bin/local-dev/main.sh‎

Lines changed: 95 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,22 @@
2929
# double-click for logs, ↑/↓
3030
# history, Ctrl-C twice to quit).
3131
# Requires Python + textual.
32-
# bin/local-dev.sh status same as no-arg invocation.
33-
# bin/local-dev.sh up [--fresh|--build|--no-build] [--skip=svc1,svc2]
32+
# bin/local-dev.sh status [--json] same as no-arg invocation. With
33+
# --json, print one machine-readable
34+
# JSON object (no table) and exit 0
35+
# iff every service is running — the
36+
# contract for agents/scripts.
37+
# bin/local-dev.sh up [--fresh|--build|--no-build] [--skip=svc1,svc2] [--json]
3438
# Default: skip build if no source/lock
3539
# changes since last build. --build forces
3640
# incremental sbt dist + yarn/bun install.
3741
# --fresh runs `sbt clean dist`. --no-build
38-
# skips the build step entirely.
39-
# bin/local-dev.sh down [--skip=svc1,svc2] stop every non-skipped service.
42+
# skips the build step entirely. --json
43+
# sends progress to stderr and the final
44+
# status JSON to stdout.
45+
# bin/local-dev.sh down [--skip=svc1,svc2] [--json]
46+
# stop every non-skipped service
47+
# (--json: summary JSON on stdout).
4048
# bin/local-dev.sh start <service> start one service (no rebuild).
4149
# bin/local-dev.sh stop <service> stop one service.
4250
# bin/local-dev.sh <service> rebuild only that service incrementally
@@ -60,9 +68,10 @@
6068
# agent-service :3001 Bun --watch (cd agent-service && bun run dev)
6169
# frontend :4200 ng serve via cd frontend && yarn start
6270
#
63-
# Docker infra (postgres / minio / lakefs / lakekeeper / litellm) is NOT managed
64-
# here -- bring those up yourself before `up`. The script will warn if expected
65-
# ports are unreachable.
71+
# Docker infra (postgres / minio / lakefs / lakekeeper / litellm) IS managed
72+
# here: `up` brings it up via `docker compose` (project texera-local-dev) and
73+
# `down` tears down any docker targets. The script warns if expected ports are
74+
# unreachable.
6675
#
6776
# Logs and pid book-keeping live under: ${TEXERA_LOCAL_DEV_DIR:-/tmp/texera-local-dev}
6877

@@ -716,7 +725,24 @@ tui_state_color() {
716725
tui_spinner() {
717726
local pid="$1" msg="$2"
718727
if [[ ! -t 1 ]]; then
719-
printf " ${BLUE}${SYM_PROG}${RESET} ${DIM}%s (no-TTY, no spinner)${RESET}\n" "$msg"
728+
# No cursor control on a pipe, so we can't spin in place. Print one
729+
# line up front, then a heartbeat every TUI_HEARTBEAT_SECS while the
730+
# job runs — otherwise a long silent step (e.g. `sbt dist`, whose
731+
# output is redirected to a log) looks hung to a non-interactive
732+
# caller polling the stream.
733+
printf " ${BLUE}${SYM_PROG}${RESET} ${DIM}%s (no-TTY)${RESET}\n" "$msg"
734+
# Poll every 1s (so we return within ~1s of the job finishing — no
735+
# trailing dead time) but only print a heartbeat every
736+
# TUI_HEARTBEAT_SECS so the log stays readable.
737+
local hb_start=$SECONDS hb_every="${TUI_HEARTBEAT_SECS:-15}" hb_last=0 hb_now=0
738+
while kill -0 "$pid" 2>/dev/null; do
739+
sleep 1
740+
hb_now=$((SECONDS - hb_start))
741+
if (( hb_now - hb_last >= hb_every )); then
742+
printf " ${BLUE}${SYM_PROG}${RESET} ${DIM}… still running (%ds)${RESET}\n" "$hb_now"
743+
hb_last=$hb_now
744+
fi
745+
done
720746
return
721747
fi
722748
# Use an array (vs a single multibyte string + byte indexing) because
@@ -1777,7 +1803,49 @@ refresh_node_deps() {
17771803
}
17781804
17791805
# --------- subcommands ---------
1806+
# Machine-readable counterpart to cmd_status: one JSON object on stdout, no
1807+
# colours, no decorative table. The stable contract for agents/scripts that
1808+
# would otherwise scrape the dashboard. Exit code mirrors health: 0 iff every
1809+
# service is running, else 1.
1810+
emit_status_json() {
1811+
local branch="" sha=""
1812+
branch=$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "?")
1813+
sha=$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || echo "?")
1814+
1815+
local n_running=0 n_total=0 first=true svc="" type="" port="" state="" pid="" rows=""
1816+
for svc in "${SERVICES[@]}"; do
1817+
n_total=$((n_total+1))
1818+
type=$(amap_get SVC_TYPE "$svc")
1819+
port=$(amap_get SVC_PORT "$svc")
1820+
pid="null"
1821+
if [[ "$type" == "docker" ]]; then
1822+
state=$(docker_svc_state "$svc")
1823+
case "$state" in running|exited) n_running=$((n_running+1)) ;; esac
1824+
else
1825+
local p=""
1826+
p=$(svc_running_pid "$svc")
1827+
if [[ -n "$p" ]]; then
1828+
state="running"; pid="$p"; n_running=$((n_running+1))
1829+
else
1830+
state="stopped"
1831+
fi
1832+
fi
1833+
$first || rows+=","
1834+
first=false
1835+
rows+=$(printf '{"service":"%s","port":%s,"type":"%s","pid":%s,"state":"%s"}' \
1836+
"$svc" "$port" "$type" "$pid" "$state")
1837+
done
1838+
printf '{"branch":"%s","sha":"%s","running":%d,"total":%d,"services":[%s]}\n' \
1839+
"$branch" "$sha" "$n_running" "$n_total" "$rows"
1840+
(( n_running == n_total ))
1841+
}
1842+
17801843
cmd_status() {
1844+
case "${1:-}" in
1845+
--json) emit_status_json; return $? ;;
1846+
"") ;;
1847+
*) tui_err "unknown flag: $1" >&2; exit 2 ;;
1848+
esac
17811849
local branch="" sha=""
17821850
branch=$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "?")
17831851
sha=$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || echo "?")
@@ -1874,17 +1942,25 @@ cmd_up() {
18741942
SKIP_LIST=""
18751943
FRESH=false
18761944
BUILD=auto # auto (skip if no source change) | force | no
1945+
JSON_OUT=false
18771946
while [[ $# -gt 0 ]]; do
18781947
case "$1" in
18791948
--skip=*) SKIP_LIST="${1#--skip=}" ;;
18801949
--fresh) FRESH=true; BUILD=force ;;
18811950
--build) BUILD=force ;;
18821951
--no-build) BUILD=no ;;
1952+
--json) JSON_OUT=true ;;
18831953
*) tui_err "unknown flag: $1" >&2; exit 2 ;;
18841954
esac
18851955
shift
18861956
done
18871957
1958+
# --json: the final summary on stdout must be pure JSON, so push all the
1959+
# human progress (banner, sections, in-place health panel) to stderr and
1960+
# keep the real stdout on fd 3 for emit_status_json. stderr is unbuffered,
1961+
# so a non-interactive caller still sees progress live on the side stream.
1962+
if $JSON_OUT; then exec 3>&1 1>&2; fi
1963+
18881964
local n_skip=0
18891965
[[ -n "$SKIP_LIST" ]] && n_skip=$(echo "$SKIP_LIST" | tr ',' '\n' | wc -l | tr -d ' ')
18901966
local skip_label="none"
@@ -1915,6 +1991,7 @@ cmd_up() {
19151991
tui_ok "no source/lock changes since last build"
19161992
tui_ok "all ${#SERVICES[@]} services already running"
19171993
printf "\n ${BOLD}${GREEN}${SYM_OK} nothing to do${RESET} ${DIM}(use \`u --build\` to force a rebuild, or \`<svc>\` to bounce just one)${RESET}\n\n"
1994+
$JSON_OUT && { emit_status_json >&3 || true; }
19181995
return 0
19191996
fi
19201997
fi
@@ -2010,7 +2087,7 @@ cmd_up() {
20102087
fi
20112088
printf "\n"
20122089
2013-
cmd_status
2090+
if $JSON_OUT; then emit_status_json >&3 || true; else cmd_status; fi
20142091
[[ $ec -eq 0 ]]
20152092
}
20162093
@@ -2251,13 +2328,17 @@ cmd_auto() {
22512328
22522329
cmd_down() {
22532330
SKIP_LIST=""
2331+
JSON_OUT=false
22542332
while [[ $# -gt 0 ]]; do
22552333
case "$1" in
22562334
--skip=*) SKIP_LIST="${1#--skip=}" ;;
2335+
--json) JSON_OUT=true ;;
22572336
*) tui_err "unknown flag: $1" >&2; exit 2 ;;
22582337
esac
22592338
shift
22602339
done
2340+
# See cmd_up: human progress to stderr, JSON summary on real stdout (fd 3).
2341+
if $JSON_OUT; then exec 3>&1 1>&2; fi
22612342
tui_banner "Texera Local Dev — stopping stack" "skip=${SKIP_LIST:-none}"
22622343
tui_section "Stopping (reverse order)"
22632344
local svc=""
@@ -2284,6 +2365,8 @@ cmd_down() {
22842365
done
22852366
$has_docker_targets && infra_down
22862367
printf "\n"
2368+
$JSON_OUT && { emit_status_json >&3 || true; }
2369+
return 0
22872370
}
22882371
22892372
cmd_update_one() {
@@ -2555,7 +2638,8 @@ cmd_interactive() {
25552638
_precompute_src_dirs
25562639
25572640
case "${1:-}" in
2558-
""|status) cmd_status ;; # default: one-shot dashboard (safe in scripts/CI)
2641+
"") cmd_status ;; # default: one-shot dashboard (safe in scripts/CI)
2642+
status) shift; cmd_status "$@" ;; # `status [--json]`
25592643
-i|--interactive) cmd_interactive ;; # opt in to the live TUI
25602644
up) shift; cmd_up "$@" ;;
25612645
auto) shift; cmd_auto "$@" ;;
@@ -2565,6 +2649,6 @@ case "${1:-}" in
25652649
logs) shift; cmd_logs "${1:-}" ;;
25662650
w|watch) shift; cmd_watch "${1:-2}" ;;
25672651
version) printf "%s\n" "$TEXERA_VERSION" ;;
2568-
-h|--help) sed -n '18,67p' "$0" ;;
2652+
-h|--help) sed -n '18,75p' "$0" ;;
25692653
*) cmd_update_one "$1" ;;
25702654
esac

‎bin/local-dev/tests/test_local_dev_sh.sh‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,5 +230,89 @@ for fn in cmd_up cmd_auto; do
230230
fi
231231
done
232232

233+
# 12) `status --json` emits a single machine-readable JSON object — the stable
234+
# contract for agents/scripts that would otherwise grep the dashboard.
235+
# Must parse, expose running/total/services, list every service exactly
236+
# once, and stay internally consistent (len(services)==total, running<=total).
237+
if command -v python3 >/dev/null 2>&1; then
238+
json_out=$("$SCRIPT" status --json 2>/dev/null)
239+
if printf '%s' "$json_out" | python3 -c '
240+
import sys, json
241+
d = json.load(sys.stdin)
242+
assert isinstance(d["services"], list), "services not a list"
243+
assert isinstance(d["running"], int) and isinstance(d["total"], int)
244+
assert d["total"] == len(d["services"]), "total != len(services)"
245+
assert 0 <= d["running"] <= d["total"], "running out of range"
246+
names = {s["service"] for s in d["services"]}
247+
need = {"texera-web", "frontend", "postgres"}
248+
assert need <= names, f"missing services: {need - names}"
249+
for s in d["services"]:
250+
assert isinstance(s["port"], int), "port not int"
251+
assert s["type"] in {"jvm", "docker", "yarn", "bun"}, "bad service type"
252+
assert s["pid"] is None or isinstance(s["pid"], int), "pid not int|null"
253+
' 2>/tmp/.local-dev-json.err; then
254+
_pass "status --json emits valid, consistent JSON with all services"
255+
else
256+
_fail "status --json invalid/inconsistent" \
257+
"$(tail -1 /tmp/.local-dev-json.err 2>/dev/null); out=$(printf '%s' "$json_out" | head -c 160)"
258+
fi
259+
rm -f /tmp/.local-dev-json.err
260+
261+
# 13) Exit code mirrors health: 0 iff running == total, else 1. Lets an
262+
# agent gate on `if status --json; then` without parsing the body.
263+
running=$(printf '%s' "$json_out" | python3 -c 'import sys,json;print(json.load(sys.stdin)["running"])' 2>/dev/null)
264+
total=$(printf '%s' "$json_out" | python3 -c 'import sys,json;print(json.load(sys.stdin)["total"])' 2>/dev/null)
265+
"$SCRIPT" status --json >/dev/null 2>&1; rc_json=$?
266+
if { [[ "$running" == "$total" ]] && (( rc_json == 0 )); } \
267+
|| { [[ "$running" != "$total" ]] && (( rc_json == 1 )); }; then
268+
_pass "status --json exit code reflects health (running=$running total=$total rc=$rc_json)"
269+
else
270+
_fail "status --json exit code wrong" "running=$running total=$total rc=$rc_json"
271+
fi
272+
else
273+
_pass "skip: python3 not on PATH (status --json shape check)"
274+
fi
275+
276+
# 14) Negative: an unknown flag to `status` must refuse with rc 2 and a clear
277+
# message — bad input is not silently ignored.
278+
out=$("$SCRIPT" status --definitely-bogus 2>&1)
279+
rc=$?
280+
if (( rc == 2 )) && [[ "$out" == *"unknown flag"* ]]; then
281+
_pass "status rejects unknown flag (rc=2, clear error)"
282+
else
283+
_fail "status didn't reject unknown flag" "rc=$rc out=$(echo "$out" | head -1)"
284+
fi
285+
286+
# 15) `--help` documents --json so the contract is discoverable.
287+
help_out=$("$SCRIPT" --help 2>&1)
288+
if [[ "$help_out" == *"--json"* ]]; then
289+
_pass "--help documents --json"
290+
else
291+
_fail "--help doesn't mention --json"
292+
fi
293+
294+
# 16) Regression: in non-TTY mode tui_spinner can't spin in place, so a long
295+
# silent step (sbt dist → log) must emit a heartbeat or it looks hung to a
296+
# non-interactive caller. Guard the sentinel inside the function body.
297+
spinner_body=$(awk '/^tui_spinner\(\)/{f=1} f{print} f&&/^}/{exit}' "$REPO_ROOT/bin/local-dev/main.sh")
298+
if [[ "$spinner_body" == *"! -t 1"* && "$spinner_body" == *"still running"* && "$spinner_body" == *"kill -0"* ]]; then
299+
_pass "tui_spinner emits a non-TTY heartbeat (no silent long-running steps)"
300+
else
301+
_fail "tui_spinner missing non-TTY heartbeat loop"
302+
fi
303+
304+
# 17) `up` and `down` accept --json (route the human stream to stderr, emit the
305+
# JSON summary on stdout). Structural guard — invoking them for real would
306+
# build/stop the stack, out of scope here.
307+
for fn in cmd_up cmd_down; do
308+
body=$(awk -v fn="$fn" '$0 ~ "^" fn "\\(\\)" {f=1} f{print} f&&/^}/{exit}' \
309+
"$REPO_ROOT/bin/local-dev/main.sh")
310+
if [[ "$body" == *"--json"* && "$body" == *"emit_status_json"* ]]; then
311+
_pass "$fn accepts --json and emits JSON summary"
312+
else
313+
_fail "$fn doesn't wire up --json"
314+
fi
315+
done
316+
233317
printf "\n%d passed, %d failed\n" "$PASS" "$FAIL"
234318
(( FAIL == 0 ))

‎frontend/src/app/app.module.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ import { AgentChatComponent } from "./workspace/component/agent/agent-panel/agen
108108
import { AgentRegistrationComponent } from "./workspace/component/agent/agent-panel/agent-registration/agent-registration.component";
109109
import { HuggingFaceImageUploadComponent } from "./workspace/component/hugging-face-image-upload/hugging-face-image-upload.component";
110110
import { HuggingFaceComponent } from "./workspace/component/hugging-face/hugging-face.component";
111+
import { HuggingFaceAudioUploadComponent } from "./workspace/component/hugging-face-audio-upload/hugging-face-audio-upload.component";
111112
import { DatasetFileSelectorComponent } from "./workspace/component/dataset-file-selector/dataset-file-selector.component";
112113
import { DatasetVersionSelectorComponent } from "./workspace/component/dataset-version-selector/dataset-version-selector.component";
113114
import { DatasetSelectionModalComponent } from "./workspace/component/dataset-selection-modal/dataset-selection-modal.component";
@@ -333,6 +334,7 @@ registerLocaleData(en);
333334
AgentRegistrationComponent,
334335
AgentInteractionComponent,
335336
HuggingFaceComponent,
337+
HuggingFaceAudioUploadComponent,
336338
HuggingFaceImageUploadComponent,
337339
DatasetFileSelectorComponent,
338340
DatasetVersionSelectorComponent,

‎frontend/src/app/common/formly/formly-config.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import { UiUdfParametersComponent } from "../../workspace/component/ui-udf-param
3131
import { DatasetVersionSelectorComponent } from "../../workspace/component/dataset-version-selector/dataset-version-selector.component";
3232
import { HuggingFaceImageUploadComponent } from "../../workspace/component/hugging-face-image-upload/hugging-face-image-upload.component";
3333
import { HuggingFaceComponent } from "../../workspace/component/hugging-face/hugging-face.component";
34+
import { HuggingFaceAudioUploadComponent } from "../../workspace/component/hugging-face-audio-upload/hugging-face-audio-upload.component";
3435

3536
/**
3637
* Configuration for using Json Schema with Formly.
@@ -83,6 +84,7 @@ export const TEXERA_FORMLY_CONFIG = {
8384
{ name: "inputautocomplete", component: DatasetFileSelectorComponent, wrappers: ["form-field"] },
8485
{ name: "datasetversionselector", component: DatasetVersionSelectorComponent, wrappers: ["form-field"] },
8586
{ name: "huggingface", component: HuggingFaceComponent, wrappers: ["form-field"] },
87+
{ name: "huggingface-audio-upload", component: HuggingFaceAudioUploadComponent, wrappers: ["form-field"] },
8688
{ name: "huggingface-image-upload", component: HuggingFaceImageUploadComponent, wrappers: ["form-field"] },
8789
{ name: "repeat-section-dnd", component: FormlyRepeatDndComponent },
8890
{ name: "ui-udf-parameters", component: UiUdfParametersComponent, wrappers: ["form-field"] },

‎frontend/src/app/workspace/component/hugging-face-audio-upload/hugging-face-audio-upload.component.html‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing,
13+
software distributed under the License is distributed on an
14+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
KIND, either express or implied. See the License for the
16+
specific language governing permissions and limitations
17+
under the License.
18+
-->
19+
20+
<div class="hf-audio-upload">
21+
<div class="hf-audio-guidance">
22+
Audio files are uploaded to temporary backend storage and referenced from the operator, so larger clips can be used
23+
without bloating the workflow JSON.
24+
</div>
25+
26+
<input
27+
#fileInput
28+
type="file"
29+
accept="audio/*"
30+
class="hf-audio-upload-input"
31+
[disabled]="isUploading"
32+
(change)="onFileSelected($event)" />
33+
34+
<div
35+
*ngIf="previewSrc"
36+
class="hf-audio-preview">
37+
<audio
38+
controls
39+
[src]="previewSrc"></audio>
40+
<div class="hf-audio-meta">
41+
<span>{{ fileName || "Selected audio" }}</span>
42+
<span
43+
*ngIf="isUploading"
44+
class="hf-audio-status"
45+
>Uploading...</span
46+
>
47+
<button
48+
nz-button
49+
nzSize="small"
50+
type="button"
51+
[disabled]="isUploading"
52+
(click)="clearAudio(fileInput)">
53+
Clear
54+
</button>
55+
</div>
56+
</div>
57+
58+
<div
59+
*ngIf="errorMessage"
60+
class="hf-audio-error">
61+
{{ errorMessage }}
62+
</div>
63+
</div>

0 commit comments

Comments
 (0)