mack/.github/workflows/ci.yml at 7013891da4c361d8a56532e10eabc120ae0d90b5 · MrPowers/mack · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
name: Unit tests

on:
  push:
    branches:
      - main
  pull_request:
    branches:
    - main

jobs:

  test:
    strategy:
      fail-fast: false
      matrix:
        # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        include:
          - delta-lake-version: "3.0"
            pyspark-version: "3.5"
          - delta-lake-version: "2.4"
            pyspark-version: "3.4"
          - delta-lake-version: "2.3"
            pyspark-version: "3.3"
          - delta-lake-version: "2.2"
            pyspark-version: "3.3"
          - delta-lake-version: "2.1"
            pyspark-version: "3.3"
          - delta-lake-version: "2.0"
            pyspark-version: "3.2"
          - delta-lake-version: "2.4"
            pyspark-version: "3.4"

    runs-on: ubuntu-latest
    name: Unit Test (Python 3.10, Delta ${{ matrix.delta-lake-version }}, PySpark ${{ matrix.pyspark-version }})
    steps:
      - uses: actions/checkout@v1
        with:
          fetch-depth: 1

      - name: Set up Python 3.10
        uses: actions/setup-python@v2
        with:
          python-version: "3.10"

      - name: Install Poetry
        uses: snok/install-poetry@v1

      - name: Cache Poetry virtualenv
        uses: actions/cache@v1
        id: cache
        with:
          path: ~/.virtualenvs
          key: poetry-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version}}-${{ matrix.delta-lake-version}}-${{ matrix.pyspark-version }}
          restore-keys: |
            poetry-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version}}-${{ matrix.delta-lake-version}}-${{ matrix.pyspark-version }}

      - name: Install specific delta-spark version
        if: steps.cache.outputs.cache-hit != 'true'
        run: poetry update delta-spark==${{ matrix.delta-lake-version }} pyspark==${{ matrix.pyspark-version }}

      - name: Run mack tests with pytest
        run: poetry run pytest