mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-06-13 23:36:45 +03:00
Chris Lu
caadd6ca79
* ci(s3tables): drop docker pre-pull from Lakekeeper job The lakekeeper repro is pure Go against the local weed binary; the job kept failing on Docker Hub timeouts pulling python:3 and localstack images the test never runs. Also drop the stale python-in-docker comments left from the old harness. * ci(s3tables): serve python:3 from GHA cache in the STS job Retried pulls still die when both mirror.gcr.io and registry-1.docker.io are unreachable from the runner. Cache the saved image tarball under a weekly key: an exact hit skips the registry entirely, a miss pulls fresh and refreshes the cache, and a stale tarball from a previous week is the fallback when Docker Hub is down. * ci(spark): pre-pull the spark tag the test actually runs The workflow warmed apache/spark:3.5.8 with retries while the testcontainers setup runs apache/spark:3.5.1, so the real image was pulled at test time with no retry at all.
944 lines
27 KiB
YAML
944 lines
27 KiB
YAML
name: "S3 Tables Integration Tests"
|
|
|
|
on:
|
|
pull_request:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
|
|
jobs:
|
|
s3-tables-tests:
|
|
name: S3 Tables Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Run go mod tidy
|
|
run: go mod tidy
|
|
|
|
- name: Install SeaweedFS
|
|
run: |
|
|
go install -buildvcs=false ./weed
|
|
|
|
- name: Run S3 Tables Integration Tests
|
|
timeout-minutes: 25
|
|
working-directory: test/s3tables/table-buckets
|
|
run: |
|
|
set -x
|
|
set -o pipefail
|
|
echo "=== System Information ==="
|
|
uname -a
|
|
free -h
|
|
df -h
|
|
echo "=== Starting S3 Tables Tests ==="
|
|
|
|
# Run S3 Tables integration tests
|
|
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
|
echo "S3 Tables integration tests failed"
|
|
exit 1
|
|
}
|
|
|
|
- name: Show test output on failure
|
|
if: failure()
|
|
working-directory: test/s3tables/table-buckets
|
|
run: |
|
|
echo "=== Test Output ==="
|
|
if [ -f test-output.log ]; then
|
|
tail -200 test-output.log
|
|
fi
|
|
|
|
echo "=== Process information ==="
|
|
ps aux | grep -E "(weed|test)" || true
|
|
|
|
- name: Upload test logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: s3-tables-test-logs
|
|
path: test/s3tables/table-buckets/test-output.log
|
|
retention-days: 3
|
|
|
|
iceberg-catalog-tests:
|
|
name: Iceberg Catalog Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Configure Docker Hub mirror
|
|
run: |
|
|
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
|
sudo systemctl restart docker
|
|
|
|
- name: Run go mod tidy
|
|
run: go mod tidy
|
|
|
|
- name: Run Iceberg Catalog Integration Tests
|
|
timeout-minutes: 25
|
|
working-directory: test/s3tables/catalog
|
|
run: |
|
|
set -x
|
|
set -o pipefail
|
|
echo "=== System Information ==="
|
|
uname -a
|
|
free -h
|
|
df -h
|
|
echo "=== Starting Iceberg Catalog Tests ==="
|
|
|
|
# Run Iceberg catalog integration tests using Makefile (handles build)
|
|
make test 2>&1 | tee test-output.log || {
|
|
echo "Iceberg catalog integration tests failed"
|
|
exit 1
|
|
}
|
|
|
|
- name: Show test output on failure
|
|
if: failure()
|
|
working-directory: test/s3tables/catalog
|
|
run: |
|
|
echo "=== Test Output ==="
|
|
if [ -f test-output.log ]; then
|
|
tail -200 test-output.log
|
|
fi
|
|
|
|
echo "=== Process information ==="
|
|
ps aux | grep -E "(weed|test|docker)" || true
|
|
|
|
- name: Upload test logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: iceberg-catalog-test-logs
|
|
path: test/s3tables/catalog/test-output.log
|
|
retention-days: 3
|
|
|
|
trino-iceberg-catalog-tests:
|
|
name: Trino Iceberg Catalog Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Configure Docker Hub mirror
|
|
run: |
|
|
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
|
sudo systemctl restart docker
|
|
|
|
- name: Pre-pull Trino image
|
|
run: |
|
|
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
|
pull trinodb/trino:479
|
|
|
|
- name: Run go mod tidy
|
|
run: go mod tidy
|
|
|
|
- name: Install SeaweedFS
|
|
run: |
|
|
go install -buildvcs=false ./weed
|
|
|
|
- name: Run Trino Iceberg Catalog Integration Tests
|
|
timeout-minutes: 25
|
|
working-directory: test/s3tables/catalog_trino
|
|
run: |
|
|
set -x
|
|
set -o pipefail
|
|
echo "=== System Information ==="
|
|
uname -a
|
|
free -h
|
|
df -h
|
|
echo "=== Starting Trino Iceberg Catalog Tests ==="
|
|
|
|
# Run Trino + Iceberg catalog integration tests
|
|
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
|
echo "Trino Iceberg catalog integration tests failed"
|
|
exit 1
|
|
}
|
|
|
|
- name: Show test output on failure
|
|
if: failure()
|
|
working-directory: test/s3tables/catalog_trino
|
|
run: |
|
|
echo "=== Test Output ==="
|
|
if [ -f test-output.log ]; then
|
|
tail -200 test-output.log
|
|
fi
|
|
|
|
echo "=== Process information ==="
|
|
ps aux | grep -E "(weed|test|docker)" || true
|
|
|
|
- name: Upload test logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: trino-iceberg-catalog-test-logs
|
|
path: test/s3tables/catalog_trino/test-output.log
|
|
retention-days: 3
|
|
|
|
dremio-iceberg-catalog-tests:
|
|
name: Dremio Iceberg Catalog Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Configure Docker Hub mirror
|
|
run: |
|
|
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
|
sudo systemctl restart docker
|
|
|
|
- name: Pre-pull images
|
|
run: |
|
|
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
|
pull dremio/dremio-oss:25.2.0
|
|
pull python:3.11-slim
|
|
|
|
- name: Run go mod tidy
|
|
run: go mod tidy
|
|
|
|
- name: Install SeaweedFS
|
|
run: |
|
|
go install -buildvcs=false ./weed
|
|
|
|
- name: Run Dremio Iceberg Catalog Integration Tests
|
|
timeout-minutes: 25
|
|
working-directory: test/s3tables/catalog_dremio
|
|
run: |
|
|
set -x
|
|
set -o pipefail
|
|
echo "=== System Information ==="
|
|
uname -a
|
|
free -h
|
|
df -h
|
|
docker info
|
|
echo "=== Starting Dremio Iceberg Catalog Tests ==="
|
|
|
|
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
|
echo "Dremio Iceberg catalog integration tests failed"
|
|
exit 1
|
|
}
|
|
|
|
- name: Show test output on failure
|
|
if: failure()
|
|
working-directory: test/s3tables/catalog_dremio
|
|
run: |
|
|
echo "=== Test Output ==="
|
|
if [ -f test-output.log ]; then
|
|
tail -200 test-output.log
|
|
fi
|
|
|
|
echo "=== Process information ==="
|
|
ps aux | grep -E "(weed|test|docker|dremio)" || true
|
|
echo "=== Dremio containers ==="
|
|
docker ps -a --filter "name=seaweed-dremio" || true
|
|
|
|
- name: Upload test logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: dremio-iceberg-catalog-test-logs
|
|
path: test/s3tables/catalog_dremio/test-output.log
|
|
retention-days: 3
|
|
|
|
doris-iceberg-catalog-tests:
|
|
name: Doris Iceberg Catalog Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 35
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Configure Docker Hub mirror
|
|
run: |
|
|
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
|
sudo systemctl restart docker
|
|
|
|
- name: Pre-pull images
|
|
run: |
|
|
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
|
pull apache/doris:doris-all-in-one-2.1.0
|
|
pull python:3.11-slim
|
|
|
|
- name: Run go mod tidy
|
|
run: go mod tidy
|
|
|
|
- name: Install SeaweedFS
|
|
run: |
|
|
go install -buildvcs=false ./weed
|
|
|
|
- name: Run Doris Iceberg Catalog Integration Tests
|
|
timeout-minutes: 30
|
|
working-directory: test/s3tables/catalog_doris
|
|
run: |
|
|
set -x
|
|
set -o pipefail
|
|
echo "=== System Information ==="
|
|
uname -a
|
|
free -h
|
|
df -h
|
|
docker info
|
|
echo "=== Starting Doris Iceberg Catalog Tests ==="
|
|
|
|
go test -v -timeout 25m . 2>&1 | tee test-output.log || {
|
|
echo "Doris Iceberg catalog integration tests failed"
|
|
exit 1
|
|
}
|
|
|
|
- name: Show test output on failure
|
|
if: failure()
|
|
working-directory: test/s3tables/catalog_doris
|
|
run: |
|
|
echo "=== Test Output ==="
|
|
if [ -f test-output.log ]; then
|
|
tail -200 test-output.log
|
|
fi
|
|
|
|
echo "=== Process information ==="
|
|
ps aux | grep -E "(weed|test|docker|doris)" || true
|
|
echo "=== Doris containers ==="
|
|
docker ps -a --filter "name=seaweed-doris" || true
|
|
|
|
- name: Upload test logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: doris-iceberg-catalog-test-logs
|
|
path: test/s3tables/catalog_doris/test-output.log
|
|
retention-days: 3
|
|
|
|
polaris-integration-tests:
|
|
name: Polaris Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Run go mod tidy
|
|
run: go mod tidy
|
|
|
|
- name: Install SeaweedFS
|
|
run: |
|
|
go install -buildvcs=false ./weed
|
|
|
|
- name: Configure Docker Hub mirror
|
|
run: |
|
|
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
|
sudo systemctl restart docker
|
|
|
|
- name: Pre-pull Polaris image
|
|
run: |
|
|
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
|
pull apache/polaris:latest
|
|
|
|
- name: Run Polaris Integration Tests
|
|
timeout-minutes: 25
|
|
run: |
|
|
set -x
|
|
set -o pipefail
|
|
echo "=== System Information ==="
|
|
uname -a
|
|
free -h
|
|
df -h
|
|
echo "=== Starting Polaris Tests ==="
|
|
|
|
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
|
|
echo "Polaris integration tests failed"
|
|
exit 1
|
|
}
|
|
|
|
- name: Show test output on failure
|
|
if: failure()
|
|
working-directory: test/s3tables/polaris
|
|
run: |
|
|
echo "=== Test Output ==="
|
|
if [ -f test-output.log ]; then
|
|
tail -200 test-output.log
|
|
fi
|
|
|
|
echo "=== Process information ==="
|
|
ps aux | grep -E "(weed|test|docker)" || true
|
|
|
|
- name: Upload test logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: polaris-test-logs
|
|
path: test/s3tables/polaris/test-output.log
|
|
retention-days: 3
|
|
|
|
spark-iceberg-catalog-tests:
|
|
name: Spark Iceberg Catalog Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Configure Docker Hub mirror
|
|
run: |
|
|
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
|
sudo systemctl restart docker
|
|
|
|
- name: Pre-pull Spark image
|
|
run: |
|
|
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
|
pull apache/spark:3.5.1
|
|
|
|
- name: Run go mod tidy
|
|
run: go mod tidy
|
|
|
|
- name: Install SeaweedFS
|
|
run: |
|
|
go install -buildvcs=false ./weed
|
|
|
|
- name: Run Spark Iceberg Catalog Integration Tests
|
|
timeout-minutes: 25
|
|
working-directory: test/s3tables/catalog_spark
|
|
run: |
|
|
set -x
|
|
set -o pipefail
|
|
echo "=== System Information ==="
|
|
uname -a
|
|
free -h
|
|
df -h
|
|
echo "=== Starting Spark Iceberg Catalog Tests ==="
|
|
|
|
# Run Spark + Iceberg catalog integration tests
|
|
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
|
echo "Spark Iceberg catalog integration tests failed"
|
|
exit 1
|
|
}
|
|
|
|
- name: Show test output on failure
|
|
if: failure()
|
|
working-directory: test/s3tables/catalog_spark
|
|
run: |
|
|
echo "=== Test Output ==="
|
|
if [ -f test-output.log ]; then
|
|
tail -200 test-output.log
|
|
fi
|
|
|
|
echo "=== Process information ==="
|
|
ps aux | grep -E "(weed|test|docker)" || true
|
|
|
|
- name: Upload test logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: spark-iceberg-catalog-test-logs
|
|
path: test/s3tables/catalog_spark/test-output.log
|
|
retention-days: 3
|
|
|
|
risingwave-catalog-tests:
|
|
name: RisingWave Catalog Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Configure Docker Hub mirror
|
|
run: |
|
|
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
|
sudo systemctl restart docker
|
|
|
|
- name: Pre-pull RisingWave image
|
|
run: |
|
|
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
|
pull risingwavelabs/risingwave:v2.5.0
|
|
pull postgres:16-alpine
|
|
|
|
- name: Run go mod tidy
|
|
run: go mod tidy
|
|
|
|
- name: Install SeaweedFS
|
|
run: |
|
|
go install -buildvcs=false ./weed
|
|
|
|
- name: Run RisingWave Catalog Integration Tests
|
|
timeout-minutes: 25
|
|
working-directory: test/s3tables/catalog_risingwave
|
|
run: |
|
|
set -x
|
|
set -o pipefail
|
|
echo "=== System Information ==="
|
|
uname -a
|
|
free -h
|
|
df -h
|
|
echo "=== Starting RisingWave Catalog Tests ==="
|
|
|
|
# Run RisingWave catalog integration tests
|
|
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
|
echo "RisingWave catalog integration tests failed"
|
|
exit 1
|
|
}
|
|
|
|
- name: Show test output on failure
|
|
if: failure()
|
|
working-directory: test/s3tables/catalog_risingwave
|
|
run: |
|
|
echo "=== Test Output ==="
|
|
if [ -f test-output.log ]; then
|
|
tail -200 test-output.log
|
|
fi
|
|
|
|
echo "=== Process information ==="
|
|
ps aux | grep -E "(weed|test|docker)" || true
|
|
|
|
- name: Upload test logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: risingwave-catalog-test-logs
|
|
path: test/s3tables/catalog_risingwave/test-output.log
|
|
retention-days: 3
|
|
|
|
sts-integration-tests:
|
|
name: STS Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Configure Docker Hub mirror
|
|
run: |
|
|
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
|
sudo systemctl restart docker
|
|
|
|
- name: Week stamp for image cache key
|
|
id: week
|
|
run: echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Restore python:3 image cache
|
|
id: python-image
|
|
uses: actions/cache@v5
|
|
with:
|
|
path: /tmp/python3-image.tar
|
|
key: python3-image-${{ steps.week.outputs.week }}
|
|
restore-keys: |
|
|
python3-image-
|
|
|
|
- name: Load or pull python:3
|
|
run: |
|
|
if [ "${{ steps.python-image.outputs.cache-hit }}" = "true" ]; then
|
|
docker load -i /tmp/python3-image.tar
|
|
exit 0
|
|
fi
|
|
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
|
if pull python:3; then
|
|
docker save -o /tmp/python3-image.tar python:3
|
|
elif [ -f /tmp/python3-image.tar ]; then
|
|
# Docker Hub unreachable; fall back to last week's cached image
|
|
docker load -i /tmp/python3-image.tar
|
|
else
|
|
exit 1
|
|
fi
|
|
|
|
- name: Run go mod tidy
|
|
run: go mod tidy
|
|
|
|
- name: Install SeaweedFS
|
|
run: |
|
|
go install -buildvcs=false ./weed
|
|
|
|
- name: Run STS Integration Tests
|
|
timeout-minutes: 25
|
|
working-directory: test/s3tables/sts_integration
|
|
run: |
|
|
set -x
|
|
set -o pipefail
|
|
echo "=== System Information ==="
|
|
uname -a
|
|
free -h
|
|
df -h
|
|
echo "=== Starting STS Integration Tests ==="
|
|
|
|
# Run STS integration tests
|
|
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
|
echo "STS integration tests failed"
|
|
exit 1
|
|
}
|
|
|
|
- name: Show test output on failure
|
|
if: failure()
|
|
working-directory: test/s3tables/sts_integration
|
|
run: |
|
|
echo "=== Test Output ==="
|
|
if [ -f test-output.log ]; then
|
|
tail -200 test-output.log
|
|
fi
|
|
|
|
echo "=== Process information ==="
|
|
ps aux | grep -E "(weed|test|docker)" || true
|
|
|
|
- name: Upload test logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: sts-integration-test-logs
|
|
path: test/s3tables/sts_integration/test-output.log
|
|
retention-days: 3
|
|
|
|
lakekeeper-integration-tests:
|
|
name: Lakekeeper Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Run go mod tidy
|
|
run: go mod tidy
|
|
|
|
- name: Install SeaweedFS
|
|
run: |
|
|
go install -buildvcs=false ./weed
|
|
|
|
- name: Run Lakekeeper Integration Tests
|
|
timeout-minutes: 25
|
|
working-directory: test/s3tables/lakekeeper
|
|
run: |
|
|
set -x
|
|
set -o pipefail
|
|
echo "=== System Information ==="
|
|
uname -a
|
|
free -h
|
|
df -h
|
|
echo "=== Starting Lakekeeper Integration Tests ==="
|
|
|
|
# Run Lakekeeper integration tests
|
|
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
|
echo "Lakekeeper integration tests failed"
|
|
exit 1
|
|
}
|
|
|
|
- name: Show test output on failure
|
|
if: failure()
|
|
working-directory: test/s3tables/lakekeeper
|
|
run: |
|
|
echo "=== Test Output ==="
|
|
if [ -f test-output.log ]; then
|
|
tail -200 test-output.log
|
|
fi
|
|
|
|
echo "=== Process information ==="
|
|
ps aux | grep -E "(weed|test|docker)" || true
|
|
|
|
- name: Upload test logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: lakekeeper-integration-test-logs
|
|
path: test/s3tables/lakekeeper/test-output.log
|
|
retention-days: 3
|
|
|
|
unity-catalog-integration-tests:
|
|
name: Unity Catalog Integration Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Configure Docker Hub mirror
|
|
run: |
|
|
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
|
sudo systemctl restart docker
|
|
|
|
- name: Pre-pull images
|
|
run: |
|
|
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
|
pull unitycatalog/unitycatalog:v0.4.0
|
|
pull python:3.11-slim
|
|
|
|
- name: Run go mod tidy
|
|
run: go mod tidy
|
|
|
|
- name: Install SeaweedFS
|
|
run: |
|
|
go install -buildvcs=false ./weed
|
|
|
|
- name: Run Unity Catalog Integration Tests
|
|
timeout-minutes: 25
|
|
working-directory: test/s3tables/unity_catalog
|
|
run: |
|
|
set -x
|
|
set -o pipefail
|
|
echo "=== System Information ==="
|
|
uname -a
|
|
free -h
|
|
df -h
|
|
docker info
|
|
echo "=== Starting Unity Catalog Tests ==="
|
|
|
|
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
|
echo "Unity Catalog integration tests failed"
|
|
exit 1
|
|
}
|
|
|
|
- name: Show test output on failure
|
|
if: failure()
|
|
working-directory: test/s3tables/unity_catalog
|
|
run: |
|
|
echo "=== Test Output ==="
|
|
if [ -f test-output.log ]; then
|
|
tail -200 test-output.log
|
|
fi
|
|
|
|
echo "=== Process information ==="
|
|
ps aux | grep -E "(weed|test|docker|unitycatalog)" || true
|
|
echo "=== Unity Catalog containers ==="
|
|
docker ps -a --filter "name=seaweed-unity-catalog" || true
|
|
|
|
- name: Upload test logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: unity-catalog-integration-test-logs
|
|
path: test/s3tables/unity_catalog/test-output.log
|
|
retention-days: 3
|
|
|
|
s3-tables-build-verification:
|
|
name: S3 Tables Build Verification
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 15
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Verify S3 Tables Package Builds
|
|
run: |
|
|
set -x
|
|
echo "=== Building S3 Tables package ==="
|
|
go build ./weed/s3api/s3tables || {
|
|
echo "S3 Tables package build failed"
|
|
exit 1
|
|
}
|
|
echo "S3 Tables package built successfully"
|
|
|
|
- name: Verify S3 API Integration Builds
|
|
run: |
|
|
set -x
|
|
echo "=== Building S3 API with S3 Tables integration ==="
|
|
go build ./weed/s3api || {
|
|
echo "S3 API build with S3 Tables failed"
|
|
exit 1
|
|
}
|
|
echo "S3 API with S3 Tables integration built successfully"
|
|
|
|
- name: Run Go Tests for S3 Tables Package
|
|
run: |
|
|
set -x
|
|
echo "=== Running Go unit tests for S3 Tables ==="
|
|
go test -v -race -timeout 5m ./weed/s3api/s3tables/... || {
|
|
echo "S3 Tables unit tests failed"
|
|
exit 1
|
|
}
|
|
echo "S3 Tables unit tests passed"
|
|
|
|
- name: Verify Iceberg Package Builds
|
|
run: |
|
|
set -x
|
|
echo "=== Building Iceberg package ==="
|
|
go build ./weed/s3api/iceberg || {
|
|
echo "Iceberg package build failed"
|
|
exit 1
|
|
}
|
|
echo "Iceberg package built successfully"
|
|
|
|
- name: Run Go Tests for Iceberg Package
|
|
run: |
|
|
set -x
|
|
echo "=== Running Go unit tests for Iceberg ==="
|
|
go test -v -race -timeout 5m ./weed/s3api/iceberg/... || {
|
|
echo "Iceberg unit tests failed"
|
|
exit 1
|
|
}
|
|
echo "Iceberg unit tests passed"
|
|
|
|
s3-tables-fmt-check:
|
|
name: S3 Tables Format Check
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Check Go Format
|
|
run: |
|
|
set -x
|
|
echo "=== Checking S3 Tables Go format ==="
|
|
unformatted=$(gofmt -l ./weed/s3api/s3tables)
|
|
if [ -n "$unformatted" ]; then
|
|
echo "Go format check failed - files need formatting"
|
|
echo "$unformatted"
|
|
exit 1
|
|
fi
|
|
echo "All S3 Tables files are properly formatted"
|
|
|
|
- name: Check S3 Tables Test Format
|
|
run: |
|
|
set -x
|
|
echo "=== Checking S3 Tables test format ==="
|
|
unformatted=$(gofmt -l ./test/s3tables)
|
|
if [ -n "$unformatted" ]; then
|
|
echo "Go format check failed for tests"
|
|
echo "$unformatted"
|
|
exit 1
|
|
fi
|
|
echo "All S3 Tables test files are properly formatted"
|
|
|
|
- name: Check Iceberg Format
|
|
run: |
|
|
set -x
|
|
echo "=== Checking Iceberg Go format ==="
|
|
unformatted=$(gofmt -l ./weed/s3api/iceberg)
|
|
if [ -n "$unformatted" ]; then
|
|
echo "Go format check failed for Iceberg - files need formatting"
|
|
echo "$unformatted"
|
|
exit 1
|
|
fi
|
|
echo "All Iceberg files are properly formatted"
|
|
|
|
s3-tables-vet:
|
|
name: S3 Tables Go Vet Check
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Run Go Vet
|
|
run: |
|
|
set -x
|
|
echo "=== Running go vet on S3 Tables package ==="
|
|
go vet ./weed/s3api/s3tables/... || {
|
|
echo "go vet check failed"
|
|
exit 1
|
|
}
|
|
echo "go vet checks passed"
|
|
|
|
- name: Run Go Vet on Tests
|
|
run: |
|
|
set -x
|
|
echo "=== Running go vet on S3 Tables tests ==="
|
|
go vet ./test/s3tables/... || {
|
|
echo "go vet check failed for tests"
|
|
exit 1
|
|
}
|
|
echo "go vet checks passed for tests"
|
|
|
|
- name: Run Go Vet on Iceberg
|
|
run: |
|
|
set -x
|
|
echo "=== Running go vet on Iceberg package ==="
|
|
go vet ./weed/s3api/iceberg/... || {
|
|
echo "go vet check failed for Iceberg"
|
|
exit 1
|
|
}
|
|
echo "go vet checks passed for Iceberg"
|