Route internal metastore queries through the overridable select seams #6442
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Studio Tests | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| workflow_dispatch: | |
| env: | |
| FORCE_COLOR: "1" | |
| BRANCH: ${{ github.head_ref || github.ref_name }} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| studio: | |
| if: '!github.event.pull_request.head.repo.fork' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| pyv: ['3.13'] | |
| group: [1, 2, 3, 4, 5, 6] | |
| services: | |
| postgres: | |
| image: postgres:16.3 | |
| ports: | |
| - 5432:5432 | |
| env: | |
| POSTGRES_USER: test | |
| POSTGRES_DB: database | |
| POSTGRES_HOST_AUTH_METHOD: trust | |
| options: >- | |
| --add-host=host.docker.internal:host-gateway | |
| clickhouse: | |
| image: clickhouse/clickhouse-server:25.8 | |
| ports: | |
| - 8123:8123 | |
| - 9010:9000 | |
| env: | |
| CLICKHOUSE_DB: studio_local_db | |
| CLICKHOUSE_USER: studio_local | |
| CLICKHOUSE_PASSWORD: ch123456789! | |
| CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 | |
| options: >- | |
| --add-host=host.docker.internal:host-gateway | |
| redis: | |
| image: redis:7.2.5 | |
| ports: | |
| - 6379:6379 | |
| options: >- | |
| --add-host=host.docker.internal:host-gateway | |
| steps: | |
| - name: Studio branch name | |
| env: | |
| BRANCH: ${{ env.BRANCH }} | |
| STUDIO_READ_ACCESS_TOKEN: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }} | |
| run: | | |
| echo "DataChain branch: $BRANCH" | |
| if git ls-remote --heads https://"$STUDIO_READ_ACCESS_TOKEN"@github.com/datachain-ai/studio.git "$BRANCH" | grep -F "$BRANCH" 2>&1>/dev/null | |
| then | |
| STUDIO_BRANCH="$BRANCH" | |
| else | |
| STUDIO_BRANCH=main | |
| fi | |
| echo "STUDIO_BRANCH=$STUDIO_BRANCH" >> $GITHUB_ENV | |
| echo "Studio branch: $STUDIO_BRANCH" | |
| - name: Check out Studio | |
| uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 1 | |
| repository: datachain-ai/studio | |
| ref: ${{ env.STUDIO_BRANCH }} | |
| token: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }} | |
| - name: Check out DataChain | |
| uses: actions/checkout@v7 | |
| with: | |
| path: './datachain' | |
| fetch-depth: 1 | |
| - name: Set up Python ${{ matrix.pyv }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.pyv }} | |
| - name: Setup uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| cache-suffix: studio | |
| cache-dependency-glob: | | |
| datachain_saas/pyproject.toml | |
| datachain/pyproject.toml | |
| - name: Use CPU-only PyTorch on Linux | |
| # torch 2.11+ on PyPI pulls CUDA deps missing libnppicc (nvidia-npp) | |
| run: echo "UV_TORCH_BACKEND=cpu" >> "$GITHUB_ENV" | |
| - name: Update DataChain requirement in Studio | |
| if: ${{ env.BRANCH != 'main' }} | |
| working-directory: backend | |
| run: make update_datachain_deps "${{ env.BRANCH }}" | |
| - name: Install FFmpeg | |
| run: | | |
| sudo apt update | |
| sudo apt install -y ffmpeg | |
| - name: Install dependencies | |
| run: uv pip install --system ./datachain_saas[tests] ./datachain[tests] | |
| - name: Print installed system packages | |
| run: uv pip list --system | |
| - name: Initialize datachain venv | |
| env: | |
| PYTHON_VERSION: ${{ matrix.pyv }} | |
| DATACHAIN_VENV_DIR: /tmp/local/datachain_venv/python${{ matrix.pyv }} | |
| run: | | |
| virtualenv -p "$(which python"${PYTHON_VERSION}")" "${DATACHAIN_VENV_DIR}" | |
| pip_cache_dir="${DATACHAIN_VENV_DIR}/.cache/pip" | |
| pip_wheel_dir="${pip_cache_dir}/wheels" | |
| pip_bin="${DATACHAIN_VENV_DIR}/bin/pip" | |
| mkdir -p "$pip_cache_dir" | |
| mkdir -p "$pip_wheel_dir" | |
| uv_cache_dir="${DATACHAIN_VENV_DIR}/.cache/uv" | |
| mkdir -p "$uv_cache_dir" | |
| $pip_bin install -U pip wheel setuptools \ | |
| --cache-dir="$pip_cache_dir" | |
| $pip_bin wheel ./datachain_saas \ | |
| --wheel-dir="$pip_wheel_dir" \ | |
| --cache-dir="$pip_cache_dir" | |
| uv venv --python "$PYTHON_VERSION" "${DATACHAIN_VENV_DIR}/default" | |
| uv pip install -r ./compute/requirements-worker-venv.txt \ | |
| --find-links="$pip_wheel_dir" \ | |
| --cache-dir="${DATACHAIN_VENV_DIR}/.cache/uv" \ | |
| -p "${DATACHAIN_VENV_DIR}/default/bin/python" | |
| - name: Print installed worker venv packages | |
| env: | |
| DATACHAIN_VENV_DIR: /tmp/local/datachain_venv/python${{ matrix.pyv }} | |
| run: uv pip list -p "${DATACHAIN_VENV_DIR}/default/bin/python" | |
| - name: Run tests | |
| # Generate `.test_durations` file with `pytest --store-durations --durations-path ../.github/.test_durations ...` | |
| run: > | |
| PYTHONPATH="$(pwd)/..:${PYTHONPATH}" | |
| pytest | |
| --config-file=pyproject.toml -rs | |
| --splits=6 --group=${{ matrix.group }} --durations-path=../../.github/.test_durations | |
| --benchmark-skip | |
| tests ../datachain/tests | |
| working-directory: datachain_saas |