From a10607f90a7fb916524badbc91a5beadf5d51f0c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 30 May 2026 23:43:17 -0700 Subject: [PATCH] Add Terraform support for VM-based SeaweedFS deployment (#9754) * terraform: add cloud-agnostic core renderer module Renders per-node weed argv, systemd units, config files, disk-mount and secret-fetch scripts, and cloud-init from an address map. Creates zero cloud resources. Flags verified against the weed binary: volume uses -mserver for the master list, gRPC is -port.grpc (auto http+10000), minFreeSpacePercent is a string, filer store via -defaultStoreDir. * terraform: add mTLS and JWT security module Generates the CA, per-component certs with distinct CNs, and JWT signing keys via the tls/random providers. Emits a core_security object plus PEMs for secret-store delivery. * terraform: add AWS deployment module and examples Reserves stable ENIs first, renders config via the core, then creates instances, prevent_destroy EBS data disks mounted at /data, and the cluster security group. With enable_security, generates certs/JWT, stores them in SSM SecureString, grants an instance role, and fetches them at boot so secrets stay out of user_data. Keyed for_each on every stateful tier. * terraform: add local cluster test harnesses run_local_cluster.sh and run_local_secure.sh render a cluster with the core and run real weed processes, asserting master quorum, volume registration, filer/s3 round-trips, mutual-TLS formation, and JWT enforcement. Use an isolated high port range with a guard so they never touch a cluster already running on the machine. The weed binary defaults to $(go env GOPATH)/bin/weed. * terraform: add CI workflow and README fmt/validate/tofu-test plus smoke jobs that build weed and run both harnesses. * terraform: guard against empty filesystem UUID in mount script An empty UUID made grep -q match any fstab line, skipping the fstab entry and breaking the mount. Fail fast when blkid returns no UUID. * terraform: sanitize cluster name in WEED_CLUSTER env keys Hyphens or spaces in cluster_name produced invalid systemd/bash env var names; map non-alphanumerics to underscores. * terraform: omit empty jwt.signing block from security.toml With enable_security and no JWT key, the template emitted [jwt.signing] key="". Gate the block on a non-empty key and cover it with a test. * terraform: mark core security input as sensitive The security object carries JWT signing keys; keep them out of plan output and known values. * terraform: enforce jwt_length minimum of 32 * terraform: note region/AZ coupling in HA example * terraform: guard WORKDIR before recursive delete in test harnesses * terraform: fix README fence language and test count * terraform: handle embedded s3 with no filer nodes Indexing sort(keys(var.filers))[0] errored at plan time when embedded S3 was enabled but no filers were defined; fall back to an empty config source. * terraform: scope kms:Decrypt to a configurable key arn Replace the hardcoded Resource="*" with a kms_key_arn variable (default "*") so production can restrict decrypt to a specific CMK. * terraform: encrypt EBS data volumes at rest Set encrypted = true on the volume/filer data disks and the all-in-one example disk. * terraform: protect filer instances from API termination Filers hold the leveldb2 metadata store, so they are stateful and get the same disable_api_termination as masters and volumes. * terraform: stop instance before detaching in all-in-one example * terraform: drop stale references to the removed plan doc * terraform: correct stale mount-step comment in aws module * terraform: mark Terraform support as experimental in README --- .github/workflows/terraform_ci.yml | 80 ++++ terraform/.gitignore | 15 + terraform/README.md | 128 ++++++ terraform/examples/aws-all-in-one/main.tf | 133 +++++++ terraform/examples/aws-ha-distributed/main.tf | 117 ++++++ terraform/modules/aws/main.tf | 205 ++++++++++ terraform/modules/aws/outputs.tf | 34 ++ terraform/modules/aws/security.tf | 136 +++++++ terraform/modules/aws/security_group.tf | 62 +++ .../aws/templates/fetch-secrets.sh.tftpl | 21 + terraform/modules/aws/variables.tf | 171 ++++++++ terraform/modules/aws/versions.tf | 12 + terraform/modules/core/main.tf | 373 ++++++++++++++++++ terraform/modules/core/outputs.tf | 32 ++ .../core/templates/cloud-init.yaml.tftpl | 44 +++ .../core/templates/mount-disks.sh.tftpl | 73 ++++ .../core/templates/security.toml.tftpl | 53 +++ .../modules/core/templates/weed.service.tftpl | 39 ++ .../modules/core/tests/render.tftest.hcl | 234 +++++++++++ terraform/modules/core/variables.tf | 318 +++++++++++++++ terraform/modules/core/versions.tf | 6 + terraform/modules/security/main.tf | 68 ++++ terraform/modules/security/outputs.tf | 65 +++ terraform/modules/security/variables.tf | 89 +++++ terraform/modules/security/versions.tf | 13 + terraform/test/local-secure/main.tf | 74 ++++ .../test/local-secure/run_local_secure.sh | 128 ++++++ terraform/test/local/main.tf | 100 +++++ terraform/test/local/run_local_cluster.sh | 170 ++++++++ 29 files changed, 2993 insertions(+) create mode 100644 .github/workflows/terraform_ci.yml create mode 100644 terraform/.gitignore create mode 100644 terraform/README.md create mode 100644 terraform/examples/aws-all-in-one/main.tf create mode 100644 terraform/examples/aws-ha-distributed/main.tf create mode 100644 terraform/modules/aws/main.tf create mode 100644 terraform/modules/aws/outputs.tf create mode 100644 terraform/modules/aws/security.tf create mode 100644 terraform/modules/aws/security_group.tf create mode 100644 terraform/modules/aws/templates/fetch-secrets.sh.tftpl create mode 100644 terraform/modules/aws/variables.tf create mode 100644 terraform/modules/aws/versions.tf create mode 100644 terraform/modules/core/main.tf create mode 100644 terraform/modules/core/outputs.tf create mode 100644 terraform/modules/core/templates/cloud-init.yaml.tftpl create mode 100644 terraform/modules/core/templates/mount-disks.sh.tftpl create mode 100644 terraform/modules/core/templates/security.toml.tftpl create mode 100644 terraform/modules/core/templates/weed.service.tftpl create mode 100644 terraform/modules/core/tests/render.tftest.hcl create mode 100644 terraform/modules/core/variables.tf create mode 100644 terraform/modules/core/versions.tf create mode 100644 terraform/modules/security/main.tf create mode 100644 terraform/modules/security/outputs.tf create mode 100644 terraform/modules/security/variables.tf create mode 100644 terraform/modules/security/versions.tf create mode 100644 terraform/test/local-secure/main.tf create mode 100755 terraform/test/local-secure/run_local_secure.sh create mode 100644 terraform/test/local/main.tf create mode 100755 terraform/test/local/run_local_cluster.sh diff --git a/.github/workflows/terraform_ci.yml b/.github/workflows/terraform_ci.yml new file mode 100644 index 000000000..b587f3ffe --- /dev/null +++ b/.github/workflows/terraform_ci.yml @@ -0,0 +1,80 @@ +name: "terraform: validate and test modules" + +on: + push: + branches: [ master ] + paths: ['terraform/**', '.github/workflows/terraform_ci.yml'] + pull_request: + branches: [ master ] + paths: ['terraform/**', '.github/workflows/terraform_ci.yml'] + +permissions: + contents: read + +jobs: + validate: + name: fmt, validate, plan-level tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Set up OpenTofu + uses: opentofu/setup-opentofu@v1 + with: + tofu_version: 1.12.1 + + - name: fmt check + working-directory: terraform + run: tofu fmt -recursive -check -diff + + - name: validate core + working-directory: terraform/modules/core + run: | + tofu init -backend=false -input=false + tofu validate + + - name: validate security + working-directory: terraform/modules/security + run: | + tofu init -backend=false -input=false + tofu validate + + - name: plan-level tests (core) + working-directory: terraform/modules/core + run: tofu test + + - name: validate examples + run: | + set -e + for ex in terraform/examples/*/; do + echo "== validate $ex ==" + tofu -chdir="$ex" init -backend=false -input=false + tofu -chdir="$ex" validate + done + + smoke: + name: local cluster smoke test (real weed) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Build weed + run: go build -o "$RUNNER_TEMP/weed" ./weed + + - name: Set up OpenTofu + uses: opentofu/setup-opentofu@v1 + with: + tofu_version: 1.12.1 + + - name: Run local cluster harness + working-directory: terraform/test/local + run: WEED="$RUNNER_TEMP/weed" ./run_local_cluster.sh + + - name: Run local mTLS cluster harness + working-directory: terraform/test/local-secure + run: WEED="$RUNNER_TEMP/weed" ./run_local_secure.sh diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 000000000..5e59a3acb --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,15 @@ +# Terraform / OpenTofu working files +.terraform/ +.terraform.lock.hcl +*.tfstate +*.tfstate.* +crash.log +crash.*.log +*.tfvars +!*.tfvars.example +override.tf +override.tf.json +*_override.tf +*_override.tf.json +.terraformrc +terraform.rc diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 000000000..053278f4a --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,128 @@ +# SeaweedFS on Terraform + +> **Experimental.** This Terraform support is an early scaffold under active +> development. Interfaces (variables, outputs, module layout) may change without +> notice, and not every tier is implemented yet. Not recommended for production +> without your own review and testing. + +Self-contained Terraform/OpenTofu modules to deploy SeaweedFS on cloud VMs +running the `weed` binary directly under systemd. No Helm and no Kubernetes +required. + +What works today is verified end-to-end against a real `weed` cluster (see +"Test it locally" below). + +## Layout + +```text +terraform/ + modules/ + core/ cloud-agnostic renderer (ZERO cloud resources): turns an + address map + config into per-node weed argv, systemd units, + config files, disk-mount + secret-fetch scripts, and cloud-init. + Both the cloud wrappers and the local harnesses consume `nodes`. + security/ cloud-agnostic CA + per-component mTLS certs (distinct CNs) + + JWT signing keys (tls/random providers). Emits a `core_security` + object ready for the core, plus PEMs for secret-store delivery. + aws/ thin AWS wrapper: reserves stable ENIs (fixed private IPs) + first, feeds them to core, creates instances + protected EBS + data disks + SG; with enable_security it generates certs/JWT, + stores them in SSM SecureString, grants an instance role, and + renders a boot fetch-secrets.sh. Keyed for_each throughout. + examples/ + aws-ha-distributed/ 3 masters + 3 volumes (1/AZ) + 2 filers + 1 S3, + secure-by-default (mTLS via SSM-delivered certs) + aws-all-in-one/ single `weed server` instance via core directly + test/ + local/ render a cluster with core and run it as real weed processes + on 127.0.0.1 (no cloud, no docker), then assert it works + local-secure/ generate certs/JWT with the security module, run a real + mTLS cluster, and assert it forms + enforces JWT auth +``` + +## Design in one paragraph + +The chart is the structural reference, not a dependency. A cloud-agnostic +**core** renders everything portable; thin per-cloud wrappers provision infra. +Addressing is an **input** to the core (wrapper reserves static IPs first), so +the wrapper -> core dependency is one-way with no apply-time cycle. Stateful +tiers (master/volume/filer) are keyed `for_each` maps, never `count`, so a +middle node can be replaced without reindexing its peers or reattaching the +wrong disk. Flag names are verified against the real `weed` binary +(notably: volume uses `-mserver` for the master list; gRPC is `-port.grpc`, +auto = http+10000; `minFreeSpacePercent` is a string). + +## Test it locally + +Requires `tofu` (or `terraform`), `jq`, `curl`, and a `weed` binary. Renders a +3-master + volume + filer + S3 cluster from the core module and runs it as real +processes, asserting quorum, volume registration, and filer/S3 round-trips: + +```bash +cd terraform/test/local +WEED=/path/to/weed ./run_local_cluster.sh +# => 7 passed, 0 failed +``` + +The harness uses a high port range (29333/28080/28888/28333) so it does not +collide with a SeaweedFS cluster already running on the machine, and aborts if a +required port is taken. `KEEP=1 ./run_local_cluster.sh` leaves the cluster up. + +### mTLS end-to-end + +```bash +cd terraform/test/local-secure +WEED=/path/to/weed ./run_local_secure.sh +# => generates a CA + component certs + JWT, renders security.toml, runs a real +# mTLS cluster, asserts master/volume/filer form over mutual TLS and that the +# filer enforces JWT signing (unsigned writes get 401). 5 passed. +``` + +## Plan-level tests (no cloud) + +```bash +cd terraform/modules/core && tofu test +# => 11 passed: peers list, -mserver vs -master, metrics gating, +# security.toml conditions, all-in-one inheritance, ... +``` + +## Validate the cloud wrappers + +```bash +cd terraform/examples/aws-ha-distributed && tofu init && tofu validate +``` + +`apply` needs AWS credentials, a VPC, subnets, and an AMI with `weed` installed +(bake with Packer, or install at boot). + +## Status + +Implemented and verified: +- Tiers: master / volume / filer / s3 / all-in-one rendered by the core. +- Disk mount: cloud-init runs a `mount-disks.sh` that auto-discovers (or takes + explicit candidate devices), `blkid`-guards `mkfs`, mounts, and persists to + `/etc/fstab` by UUID. The AWS wrapper wires the protected EBS disk to `/data`. +- mTLS + JWT: the `security` module generates the CA + per-component certs + (distinct CNs) + JWT keys; the AWS wrapper stores them in SSM SecureString, + grants an instance role, and the core renders a boot fetch script so secrets + stay OUT of user_data. Proven end-to-end by `test/local-secure`. + +Not yet implemented: +- sftp / admin / worker tiers (core does not render them yet). +- GCP/Azure wrappers, the data-plane provider, and the K8s-native module. +- CA in Vault PKI (currently TF-generated, so the CA key lives in state). KMS + decrypt in the IAM policy is scoped by + `ViaService` but uses `Resource="*"`; tighten to the SSM CMK in production. + +## Gotchas + +- This `weed` build starts an Iceberg REST catalog on `:8181` by default; set + `s3.iceberg_port = 0` to disable, or a port to relocate it. +- Disk auto-discovery assumes a single attached data disk per node. For multiple + data disks, pass explicit `devices` candidates in `disk_mounts`. + +## Security note + +`tofu output`/state can contain secrets. Generate secrets outside Terraform +(cloud secret manager / Vault) and fetch them at boot; never commit secret +material. diff --git a/terraform/examples/aws-all-in-one/main.tf b/terraform/examples/aws-all-in-one/main.tf new file mode 100644 index 000000000..201d3311d --- /dev/null +++ b/terraform/examples/aws-all-in-one/main.tf @@ -0,0 +1,133 @@ +# SeaweedFS all-in-one on a single AWS instance. +# +# Demonstrates the layered design directly: call the cloud-agnostic core to +# render cloud-init for a `weed server` node, then attach it to one instance +# with a protected data disk. No wrapper module needed for the simple case. +# +# tofu init && tofu validate +# tofu apply # requires AWS credentials + a weed AMI + +terraform { + required_version = ">= 1.3.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.40" + } + } +} + +provider "aws" { + region = var.region +} + +variable "region" { + type = string + default = "us-east-1" +} +variable "vpc_id" { + type = string +} +variable "subnet_id" { + type = string +} +variable "availability_zone" { + type = string + default = "us-east-1a" +} +variable "ami_id" { + type = string +} +variable "private_ip" { + type = string + default = "10.0.1.50" +} + +module "core" { + source = "../../modules/core" + weed_binary = "/usr/bin/weed" + + # disable the distributed tiers; run a single all-in-one process + master = { enabled = false } + volume = { enabled = false } + filer = { enabled = false } + + all_in_one = { + enabled = true + nodes = { a0 = { address = var.private_ip, data_dir = "/data" } } + s3 = { enabled = true } + } + + s3_identities = [{ + name = "anonymous" + access_key = "" + secret_key = "" + actions = ["Read", "List"] + }] +} + +resource "aws_network_interface" "aio" { + subnet_id = var.subnet_id + private_ips = [var.private_ip] + security_groups = [aws_security_group.aio.id] + tags = { Name = "seaweedfs-all-in-one" } +} + +resource "aws_security_group" "aio" { + name_prefix = "seaweedfs-aio-" + vpc_id = var.vpc_id + lifecycle { + create_before_destroy = true + } +} + +resource "aws_vpc_security_group_ingress_rule" "s3" { + security_group_id = aws_security_group.aio.id + cidr_ipv4 = "10.0.0.0/8" + ip_protocol = "tcp" + from_port = 8333 + to_port = 8333 +} + +resource "aws_vpc_security_group_egress_rule" "all" { + security_group_id = aws_security_group.aio.id + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "-1" +} + +resource "aws_instance" "aio" { + ami = var.ami_id + instance_type = "m5.large" + user_data = module.core.cloud_init_by_node["all-in-one-a0"] + + network_interface { + network_interface_id = aws_network_interface.aio.id + device_index = 0 + } + metadata_options { + http_tokens = "required" + } + tags = { Name = "seaweedfs-all-in-one", Role = "all-in-one" } +} + +resource "aws_ebs_volume" "data" { + availability_zone = var.availability_zone + size = 200 + type = "gp3" + encrypted = true + tags = { Name = "seaweedfs-all-in-one-data" } + lifecycle { + prevent_destroy = true + } +} + +resource "aws_volume_attachment" "data" { + device_name = "/dev/xvdf" + volume_id = aws_ebs_volume.data.id + instance_id = aws_instance.aio.id + stop_instance_before_detaching = true +} + +output "instance_id" { + value = aws_instance.aio.id +} diff --git a/terraform/examples/aws-ha-distributed/main.tf b/terraform/examples/aws-ha-distributed/main.tf new file mode 100644 index 000000000..4997b70c8 --- /dev/null +++ b/terraform/examples/aws-ha-distributed/main.tf @@ -0,0 +1,117 @@ +# SeaweedFS HA on AWS: 3-master quorum + 3 volume servers (one per AZ) + 2 +# filers (leveldb2-replicated HA) + 1 standalone S3 gateway. +# +# tofu init && tofu validate +# tofu apply # requires AWS credentials, a VPC, subnets, and a weed AMI +# +# This is a scaffold: it provisions instances, protected EBS data disks, and the +# security group. Mounting the EBS disk at /data and secret-store cert delivery +# are documented follow-ups (see terraform/README.md). + +terraform { + required_version = ">= 1.3.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.40" + } + } +} + +provider "aws" { + region = var.region +} + +variable "region" { + type = string + default = "us-east-1" +} + +variable "vpc_id" { + type = string +} + +variable "ami_id" { + description = "AMI with the weed binary at /usr/bin/weed." + type = string +} + +# subnet per AZ +variable "subnet_a" { type = string } +variable "subnet_b" { type = string } +variable "subnet_c" { type = string } + +# AZ defaults assume region us-east-1. When you change `region`, override az_a/az_b/az_c +# (and the matching subnets) with AZs that belong to that region, or the volume/filer +# EBS volumes will fail to create in a mismatched AZ. +variable "az_a" { + type = string + default = "us-east-1a" +} +variable "az_b" { + type = string + default = "us-east-1b" +} +variable "az_c" { + type = string + default = "us-east-1c" +} + +variable "client_ingress_cidrs" { + type = list(string) + default = ["10.0.0.0/8"] +} + +variable "ssh_ingress_cidrs" { + type = list(string) + default = [] +} + +module "seaweedfs" { + source = "../../modules/aws" + + name = "seaweedfs" + vpc_id = var.vpc_id + ami_id = var.ami_id + + # secure-by-default flagship: mTLS (certs + JWT generated by the security + # submodule, delivered via SSM and fetched at boot) plus monitoring. + enable_security = true + monitoring_enabled = true + + masters = { + m0 = { subnet_id = var.subnet_a, private_ip = "10.0.1.10" } + m1 = { subnet_id = var.subnet_b, private_ip = "10.0.2.10" } + m2 = { subnet_id = var.subnet_c, private_ip = "10.0.3.10" } + } + + volumes = { + v0 = { subnet_id = var.subnet_a, availability_zone = var.az_a, private_ip = "10.0.1.20", rack = var.az_a, data_center = var.region, data_volume_size_gb = 500 } + v1 = { subnet_id = var.subnet_b, availability_zone = var.az_b, private_ip = "10.0.2.20", rack = var.az_b, data_center = var.region, data_volume_size_gb = 500 } + v2 = { subnet_id = var.subnet_c, availability_zone = var.az_c, private_ip = "10.0.3.20", rack = var.az_c, data_center = var.region, data_volume_size_gb = 500 } + } + + filers = { + f0 = { subnet_id = var.subnet_a, availability_zone = var.az_a, private_ip = "10.0.1.30" } + f1 = { subnet_id = var.subnet_b, availability_zone = var.az_b, private_ip = "10.0.2.30" } + } + + s3_nodes = { + s0 = { subnet_id = var.subnet_a, private_ip = "10.0.1.40" } + } + + client_ingress_cidrs = var.client_ingress_cidrs + ssh_ingress_cidrs = var.ssh_ingress_cidrs +} + +output "master_peers" { + value = module.seaweedfs.master_peers +} + +output "instance_ids" { + value = module.seaweedfs.instance_ids +} + +output "security_group_id" { + value = module.seaweedfs.security_group_id +} diff --git a/terraform/modules/aws/main.tf b/terraform/modules/aws/main.tf new file mode 100644 index 000000000..3f52ecb73 --- /dev/null +++ b/terraform/modules/aws/main.tf @@ -0,0 +1,205 @@ +# ---- step 1: reserve stable addressing (ENIs with fixed private IPs) ------- +# These exist BEFORE the core renders config, so the address map is known and +# the core <- wrapper dependency is one-way (no apply-time cycle). + +resource "aws_network_interface" "master" { + for_each = var.masters + subnet_id = each.value.subnet_id + private_ips = [each.value.private_ip] + security_groups = [aws_security_group.cluster.id] + tags = merge(var.tags, { Name = "${var.name}-master-${each.key}", Role = "master" }) +} + +resource "aws_network_interface" "volume" { + for_each = var.volumes + subnet_id = each.value.subnet_id + private_ips = [each.value.private_ip] + security_groups = [aws_security_group.cluster.id] + tags = merge(var.tags, { Name = "${var.name}-volume-${each.key}", Role = "volume" }) +} + +resource "aws_network_interface" "filer" { + for_each = var.filers + subnet_id = each.value.subnet_id + private_ips = [each.value.private_ip] + security_groups = [aws_security_group.cluster.id] + tags = merge(var.tags, { Name = "${var.name}-filer-${each.key}", Role = "filer" }) +} + +resource "aws_network_interface" "s3" { + for_each = var.s3_nodes + subnet_id = each.value.subnet_id + private_ips = [each.value.private_ip] + security_groups = [aws_security_group.cluster.id] + tags = merge(var.tags, { Name = "${var.name}-s3-${each.key}", Role = "s3" }) +} + +# ---- step 2: render config with the cloud-agnostic core -------------------- +module "core" { + source = "../core" + + weed_binary = var.weed_binary + monitoring_enabled = var.monitoring_enabled + enable_security = var.enable_security + security = local.core_security + + # Keep secrets out of cloud-init user_data when security is on; deliver them + # from SSM via the boot fetch script instead. + render_secret_files = !var.enable_security + boot_fetch_script = local.fetch_script + + master = { + nodes = { for k, v in var.masters : k => { address = v.private_ip } } + } + + volume = { + enabled = length(var.volumes) > 0 + nodes = { for k, v in var.volumes : k => { + address = v.private_ip + rack = v.rack + data_center = v.data_center + data_dirs = [{ path = "/data", max_volumes = 0 }] + # auto-discover the single attached data disk and mount it at /data + disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }] + } } + } + + filer = { + enabled = length(var.filers) > 0 + nodes = { for k, v in var.filers : k => { + address = v.private_ip + data_dir = "/data/filerldb2" + disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }] + } } + s3 = var.embedded_s3 + } + + s3 = { + enabled = length(var.s3_nodes) > 0 + nodes = { for k, v in var.s3_nodes : k => { address = v.private_ip } } + } + + s3_identities = var.s3_identities +} + +# ---- step 3: instances (keyed for_each; user_data = rendered cloud-init) ---- +resource "aws_instance" "master" { + for_each = var.masters + ami = var.ami_id + instance_type = each.value.instance_type + key_name = var.key_name + user_data = module.core.cloud_init_by_node["master-${each.key}"] + iam_instance_profile = var.enable_security ? aws_iam_instance_profile.node[0].name : null + disable_api_termination = var.termination_protection + + network_interface { + network_interface_id = aws_network_interface.master[each.key].id + device_index = 0 + } + metadata_options { + http_tokens = "required" # IMDSv2 + } + tags = merge(var.tags, { Name = "${var.name}-master-${each.key}", Role = "master" }) +} + +resource "aws_instance" "volume" { + for_each = var.volumes + ami = var.ami_id + instance_type = each.value.instance_type + key_name = var.key_name + user_data = module.core.cloud_init_by_node["volume-${each.key}"] + iam_instance_profile = var.enable_security ? aws_iam_instance_profile.node[0].name : null + disable_api_termination = var.termination_protection + + network_interface { + network_interface_id = aws_network_interface.volume[each.key].id + device_index = 0 + } + metadata_options { + http_tokens = "required" + } + tags = merge(var.tags, { Name = "${var.name}-volume-${each.key}", Role = "volume" }) +} + +resource "aws_instance" "filer" { + for_each = var.filers + ami = var.ami_id + instance_type = each.value.instance_type + key_name = var.key_name + user_data = module.core.cloud_init_by_node["filer-${each.key}"] + iam_instance_profile = var.enable_security ? aws_iam_instance_profile.node[0].name : null + disable_api_termination = var.termination_protection + + network_interface { + network_interface_id = aws_network_interface.filer[each.key].id + device_index = 0 + } + metadata_options { + http_tokens = "required" + } + tags = merge(var.tags, { Name = "${var.name}-filer-${each.key}", Role = "filer" }) +} + +resource "aws_instance" "s3" { + for_each = var.s3_nodes + ami = var.ami_id + instance_type = each.value.instance_type + key_name = var.key_name + user_data = module.core.cloud_init_by_node["s3-${each.key}"] + iam_instance_profile = var.enable_security ? aws_iam_instance_profile.node[0].name : null + + network_interface { + network_interface_id = aws_network_interface.s3[each.key].id + device_index = 0 + } + metadata_options { + http_tokens = "required" + } + tags = merge(var.tags, { Name = "${var.name}-s3-${each.key}", Role = "s3" }) +} + +# ---- step 4: protected data disks (decoupled; survive instance replacement) - +# The core's mount-disks.sh (wired via disk_mounts above) mkfs+mounts these at +# /data before the weed unit starts. +resource "aws_ebs_volume" "volume_data" { + for_each = var.volumes + availability_zone = each.value.availability_zone + size = each.value.data_volume_size_gb + type = each.value.data_volume_type + iops = each.value.data_volume_iops + encrypted = true + tags = merge(var.tags, { Name = "${var.name}-volume-${each.key}-data", Role = "volume" }) + + lifecycle { + prevent_destroy = true # break-glass: remove to allow destroy + } +} + +resource "aws_volume_attachment" "volume_data" { + for_each = var.volumes + device_name = "/dev/xvdf" + volume_id = aws_ebs_volume.volume_data[each.key].id + instance_id = aws_instance.volume[each.key].id + stop_instance_before_detaching = true +} + +resource "aws_ebs_volume" "filer_data" { + for_each = var.filers + availability_zone = each.value.availability_zone + size = each.value.data_volume_size_gb + type = "gp3" + encrypted = true + tags = merge(var.tags, { Name = "${var.name}-filer-${each.key}-data", Role = "filer" }) + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_volume_attachment" "filer_data" { + for_each = var.filers + device_name = "/dev/xvdf" + volume_id = aws_ebs_volume.filer_data[each.key].id + instance_id = aws_instance.filer[each.key].id + stop_instance_before_detaching = true +} diff --git a/terraform/modules/aws/outputs.tf b/terraform/modules/aws/outputs.tf new file mode 100644 index 000000000..918562bcf --- /dev/null +++ b/terraform/modules/aws/outputs.tf @@ -0,0 +1,34 @@ +output "security_group_id" { + description = "Cluster security group id." + value = aws_security_group.cluster.id +} + +output "master_private_ips" { + description = "Master private IPs keyed by node id." + value = { for k, v in var.masters : k => v.private_ip } +} + +output "master_peers" { + description = "Master peer list passed to the weed processes." + value = module.core.master_peers +} + +output "filer_private_ips" { + description = "Filer private IPs keyed by node id." + value = { for k, v in var.filers : k => v.private_ip } +} + +output "s3_private_ips" { + description = "Standalone S3 gateway private IPs keyed by node id." + value = { for k, v in var.s3_nodes : k => v.private_ip } +} + +output "instance_ids" { + description = "All instance ids keyed by role-node." + value = merge( + { for k, v in aws_instance.master : "master-${k}" => v.id }, + { for k, v in aws_instance.volume : "volume-${k}" => v.id }, + { for k, v in aws_instance.filer : "filer-${k}" => v.id }, + { for k, v in aws_instance.s3 : "s3-${k}" => v.id }, + ) +} diff --git a/terraform/modules/aws/security.tf b/terraform/modules/aws/security.tf new file mode 100644 index 000000000..7fc644e1b --- /dev/null +++ b/terraform/modules/aws/security.tf @@ -0,0 +1,136 @@ +# ============================================================================= +# mTLS + JWT generation and secret delivery (active when enable_security=true). +# +# Generates the CA + per-component certs + JWT keys with the security submodule, +# stores them (plus the rendered security.toml / S3 config) in SSM Parameter +# Store as SecureString, and renders a boot fetch-secrets.sh that pulls them via +# the instance role. This keeps secrets OUT of cloud-init user_data / IMDS. +# Secrets still live in Terraform state (TF-generated); prefer Vault PKI for the +# CA in production. +# ============================================================================= + +locals { + all_private_ips = concat( + [for k, v in var.masters : v.private_ip], + [for k, v in var.volumes : v.private_ip], + [for k, v in var.filers : v.private_ip], + [for k, v in var.s3_nodes : v.private_ip], + ) + + ssm_prefix = "/seaweedfs/${var.name}" + deliver_s3_config = length(var.s3_nodes) > 0 || var.embedded_s3.enabled + + # security.toml is identical across nodes; take it from the first master. + first_master = var.enable_security ? sort(keys(var.masters))[0] : "" + security_toml_path = "/etc/seaweedfs/security.toml" + s3_config_path = "/etc/seaweedfs/s3_config.json" + + # Boot fetch entries: cert files + security.toml + (optional) S3 config. + fetch_entries = var.enable_security ? concat( + [for m in module.security[0].secret_manifest : { param = "${local.ssm_prefix}/certs/${m.key}", path = m.path, mode = m.mode }], + [{ param = "${local.ssm_prefix}/security.toml", path = local.security_toml_path, mode = "0600" }], + local.deliver_s3_config ? [{ param = "${local.ssm_prefix}/s3_config.json", path = local.s3_config_path, mode = "0600" }] : [], + ) : [] + + fetch_script = var.enable_security ? templatefile("${path.module}/templates/fetch-secrets.sh.tftpl", { + run_as_user = "seaweedfs" + entries = local.fetch_entries + }) : "" + + core_security = var.enable_security ? module.security[0].core_security : var.security +} + +module "security" { + count = var.enable_security ? 1 : 0 + source = "../security" + internal_domain = var.internal_domain + cert_dir = var.cert_dir + ip_sans = local.all_private_ips +} + +# ---- SSM SecureString parameters ------------------------------------------- +resource "aws_ssm_parameter" "cert" { + for_each = var.enable_security ? { for m in module.security[0].secret_manifest : m.key => m } : {} + name = "${local.ssm_prefix}/certs/${each.key}" + type = "SecureString" + value = module.security[0].secret_contents[each.key] + tags = var.tags +} + +resource "aws_ssm_parameter" "security_toml" { + count = var.enable_security ? 1 : 0 + name = "${local.ssm_prefix}/security.toml" + type = "SecureString" + value = module.core.secret_files_by_node["master-${local.first_master}"][local.security_toml_path] + tags = var.tags +} + +resource "aws_ssm_parameter" "s3_config" { + count = var.enable_security && local.deliver_s3_config ? 1 : 0 + name = "${local.ssm_prefix}/s3_config.json" + type = "SecureString" + # identical content across s3/filer nodes; render once via the security module's + # JSON is not available here, so read it back from any node that carries it. + value = local.s3_config_source + tags = var.tags +} + +locals { + # Pull the rendered S3 identity JSON from whichever node carries it. + s3_config_source = !local.deliver_s3_config ? "" : ( + length(var.s3_nodes) > 0 ? + module.core.secret_files_by_node["s3-${sort(keys(var.s3_nodes))[0]}"][local.s3_config_path] : + (length(var.filers) > 0 ? module.core.secret_files_by_node["filer-${sort(keys(var.filers))[0]}"][local.s3_config_path] : "") + ) +} + +# ---- IAM: instance role allowed to read the SSM params ---------------------- +data "aws_iam_policy_document" "assume" { + count = var.enable_security ? 1 : 0 + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "ssm_read" { + count = var.enable_security ? 1 : 0 + statement { + sid = "ReadSeaweedfsParams" + actions = ["ssm:GetParameter", "ssm:GetParameters"] + resources = ["arn:aws:ssm:*:*:parameter${local.ssm_prefix}/*"] + } + statement { + sid = "DecryptSecureStrings" + actions = ["kms:Decrypt"] + resources = [var.kms_key_arn] # defaults to "*"; set a CMK ARN in production + condition { + test = "StringEquals" + variable = "kms:ViaService" + values = ["ssm.*.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "node" { + count = var.enable_security ? 1 : 0 + name_prefix = "${var.name}-node-" + assume_role_policy = data.aws_iam_policy_document.assume[0].json + tags = var.tags +} + +resource "aws_iam_role_policy" "ssm_read" { + count = var.enable_security ? 1 : 0 + name = "ssm-read" + role = aws_iam_role.node[0].id + policy = data.aws_iam_policy_document.ssm_read[0].json +} + +resource "aws_iam_instance_profile" "node" { + count = var.enable_security ? 1 : 0 + name_prefix = "${var.name}-node-" + role = aws_iam_role.node[0].name +} diff --git a/terraform/modules/aws/security_group.tf b/terraform/modules/aws/security_group.tf new file mode 100644 index 000000000..93fa32af7 --- /dev/null +++ b/terraform/modules/aws/security_group.tf @@ -0,0 +1,62 @@ +# Cluster security group + the SeaweedFS port matrix. +# Intra-cluster: all TCP allowed within the SG (master 9333/19333, volume +# 8080/18080, filer 8888/18888, admin 33646, ...). Client-facing: only S3 (8333) +# and filer (8888). Metrics ports (9327) are never opened to clients. + +resource "aws_security_group" "cluster" { + name_prefix = "${var.name}-cluster-" + description = "SeaweedFS cluster" + vpc_id = var.vpc_id + tags = merge(var.tags, { Name = "${var.name}-cluster" }) + + lifecycle { + create_before_destroy = true + } +} + +# All intra-cluster TCP via SG self-reference. +resource "aws_vpc_security_group_ingress_rule" "intra_cluster" { + security_group_id = aws_security_group.cluster.id + referenced_security_group_id = aws_security_group.cluster.id + ip_protocol = "tcp" + from_port = 0 + to_port = 65535 + description = "intra-cluster (http + gRPC for all roles)" +} + +resource "aws_vpc_security_group_ingress_rule" "s3" { + for_each = toset(var.client_ingress_cidrs) + security_group_id = aws_security_group.cluster.id + cidr_ipv4 = each.value + ip_protocol = "tcp" + from_port = 8333 + to_port = 8333 + description = "S3 gateway" +} + +resource "aws_vpc_security_group_ingress_rule" "filer" { + for_each = toset(var.client_ingress_cidrs) + security_group_id = aws_security_group.cluster.id + cidr_ipv4 = each.value + ip_protocol = "tcp" + from_port = 8888 + to_port = 8888 + description = "filer HTTP" +} + +resource "aws_vpc_security_group_ingress_rule" "ssh" { + for_each = toset(var.ssh_ingress_cidrs) + security_group_id = aws_security_group.cluster.id + cidr_ipv4 = each.value + ip_protocol = "tcp" + from_port = 22 + to_port = 22 + description = "SSH" +} + +resource "aws_vpc_security_group_egress_rule" "all" { + security_group_id = aws_security_group.cluster.id + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "-1" + description = "all egress" +} diff --git a/terraform/modules/aws/templates/fetch-secrets.sh.tftpl b/terraform/modules/aws/templates/fetch-secrets.sh.tftpl new file mode 100644 index 000000000..86346dccb --- /dev/null +++ b/terraform/modules/aws/templates/fetch-secrets.sh.tftpl @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Rendered by terraform-aws-seaweedfs. Pulls certs + secret config from SSM +# Parameter Store (SecureString) at boot using the instance role. Region is +# auto-resolved by the AWS CLI from IMDS, so no region is baked in. Requires +# the AWS CLI and a working instance profile (see security.tf). +set -euo pipefail + +run_as_user="${run_as_user}" + +fetch() { + local param="$1" dest="$2" mode="$3" + mkdir -p "$(dirname "$dest")" + aws ssm get-parameter --with-decryption --name "$param" \ + --query 'Parameter.Value' --output text > "$dest" + chmod "$mode" "$dest" + chown "$run_as_user:$run_as_user" "$dest" 2>/dev/null || true +} + +%{ for e in entries ~} +fetch "${e.param}" "${e.path}" "${e.mode}" +%{ endfor ~} diff --git a/terraform/modules/aws/variables.tf b/terraform/modules/aws/variables.tf new file mode 100644 index 000000000..a07f653c4 --- /dev/null +++ b/terraform/modules/aws/variables.tf @@ -0,0 +1,171 @@ +# ============================================================================= +# terraform-aws-seaweedfs - thin AWS wrapper around the cloud-agnostic core. +# +# Reserves stable per-node addressing (ENIs with fixed private IPs) FIRST, feeds +# that address map to the core to render cloud-init, then creates instances and +# protected EBS data disks. All tiers are keyed for_each (never count), so a +# middle node can be replaced without reindexing its peers/disks. +# ============================================================================= + +variable "name" { + description = "Name prefix for all resources." + type = string + default = "seaweedfs" +} + +variable "vpc_id" { + description = "VPC to deploy into." + type = string +} + +variable "ami_id" { + description = "AMI with the weed binary installed (e.g. baked with Packer)." + type = string +} + +variable "weed_binary" { + description = "Path to weed on the AMI." + type = string + default = "/usr/bin/weed" +} + +variable "key_name" { + description = "EC2 key pair for SSH (optional)." + type = string + default = null +} + +variable "monitoring_enabled" { + description = "Bind metrics ports and pass them through to the core." + type = bool + default = false +} + +variable "enable_security" { + description = "Enable mTLS (security.toml). Cert material must be supplied via the security variable / secret store." + type = bool + default = false +} + +variable "security" { + description = "Passed through to the core security variable when enable_security=false. When enable_security=true this is ignored and certs/JWT are generated by the security submodule." + type = any + default = {} +} + +variable "internal_domain" { + description = "Internal domain for generated component cert CNs / peer-auth wildcard (enable_security=true)." + type = string + default = "seaweedfs.internal" +} + +variable "cert_dir" { + description = "On-host directory where certs are fetched/placed." + type = string + default = "/usr/local/share/ca-certificates" +} + +variable "kms_key_arn" { + description = "KMS key ARN the instance role may use to decrypt SSM SecureStrings. Defaults to \"*\" (the SSM default key); set to a specific CMK ARN in production." + type = string + default = "*" +} + +variable "termination_protection" { + description = "Set disable_api_termination on stateful instances (masters, volumes)." + type = bool + default = true +} + +# ---- stateful tiers: keyed node maps --------------------------------------- +variable "masters" { + description = "Master nodes keyed by stable id (m0/m1/m2). Quorum size must be odd." + type = map(object({ + subnet_id = string + private_ip = string + instance_type = optional(string, "t3.medium") + })) + validation { + condition = length(var.masters) % 2 == 1 + error_message = "Master quorum must be an odd number of nodes (1, 3, 5)." + } +} + +variable "volumes" { + description = "Volume nodes keyed by stable id. Each owns one protected EBS data disk." + type = map(object({ + subnet_id = string + availability_zone = string + private_ip = string + instance_type = optional(string, "m5.large") + rack = optional(string) + data_center = optional(string) + data_volume_size_gb = optional(number, 100) + data_volume_type = optional(string, "gp3") + data_volume_iops = optional(number, null) + })) + default = {} +} + +variable "filers" { + description = "Filer nodes keyed by stable id. Each owns a protected EBS disk for its leveldb2 store." + type = map(object({ + subnet_id = string + availability_zone = string + private_ip = string + instance_type = optional(string, "t3.medium") + data_volume_size_gb = optional(number, 50) + })) + default = {} +} + +variable "s3_nodes" { + description = "Standalone S3 gateway nodes keyed by stable id (stateless)." + type = map(object({ + subnet_id = string + private_ip = string + instance_type = optional(string, "t3.medium") + })) + default = {} +} + +variable "embedded_s3" { + description = "Embed the S3 gateway in the filer instead of standalone nodes." + type = object({ + enabled = optional(bool, false) + port = optional(number, 8333) + domain_name = optional(string, "") + }) + default = {} +} + +variable "s3_identities" { + description = "Non-admin S3 identities for the gateway config (admin key goes via EnvironmentFile)." + type = list(object({ + name = string + access_key = string + secret_key = string + actions = list(string) + })) + default = [] + sensitive = true +} + +# ---- networking ------------------------------------------------------------ +variable "client_ingress_cidrs" { + description = "CIDRs allowed to reach the client-facing S3 (8333) and filer (8888) ports." + type = list(string) + default = [] +} + +variable "ssh_ingress_cidrs" { + description = "CIDRs allowed SSH (22)." + type = list(string) + default = [] +} + +variable "tags" { + description = "Extra tags applied to all resources." + type = map(string) + default = {} +} diff --git a/terraform/modules/aws/versions.tf b/terraform/modules/aws/versions.tf new file mode 100644 index 000000000..6d5d186bf --- /dev/null +++ b/terraform/modules/aws/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 1.3.0" + required_providers { + aws = { + source = "hashicorp/aws" + # >= 5.40 for the modern aws_vpc_security_group_*_rule resources. + # Provider >= 6.0 is recommended for the EBS-attachment no-replace behavior; + # bump the floor here once you standardize on it. + version = ">= 5.40" + } + } +} diff --git a/terraform/modules/core/main.tf b/terraform/modules/core/main.tf new file mode 100644 index 000000000..5c6516ca8 --- /dev/null +++ b/terraform/modules/core/main.tf @@ -0,0 +1,373 @@ +# ============================================================================= +# SeaweedFS Terraform core - pure renderer (zero cloud resources). +# +# Computes, per node: the verified `weed` argv, the full systemd ExecStart, +# config files, environment, and rendered systemd unit + cloud-init. +# ============================================================================= + +locals { + # ---- shared security.toml (rendered when mTLS on OR any JWT key set) ---- + jwt_any = anytrue([ + var.security.jwt_signing_key != "", + var.security.jwt_signing_read_key != "", + var.security.jwt_filer_signing_key != "", + var.security.jwt_filer_signing_read_key != "", + ]) + render_security = var.enable_security || local.jwt_any + security_toml = local.render_security ? templatefile("${path.module}/templates/security.toml.tftpl", { + enable_security = var.enable_security + cert_dir = var.security.cert_dir + allowed_wildcard_domain = var.security.allowed_wildcard_domain + allowed_common_names = var.security.allowed_common_names + jwt_signing_key = var.security.jwt_signing_key + jwt_signing_read_key = var.security.jwt_signing_read_key + jwt_filer_signing_key = var.security.jwt_filer_signing_key + jwt_filer_signing_read_key = var.security.jwt_filer_signing_read_key + }) : "" + security_files = local.render_security ? { "/etc/seaweedfs/security.toml" = local.security_toml } : {} + + # ---- S3 identity JSON (non-admin identities only) ---- + # Empty access_key => anonymous identity (no credentials block), per SeaweedFS + # convention where the identity literally named "anonymous" grants anon access. + s3_config_json = jsonencode({ + identities = [for id in var.s3_identities : { + name = id.name + credentials = id.access_key != "" ? [{ accessKey = id.access_key, secretKey = id.secret_key }] : [] + actions = id.actions + }] + }) + + # ---- global flags (before the subcommand) ---- + global_flags = compact([ + format("-v=%d", var.log_level), + var.log_to_stderr ? "-logtostderr=true" : "", + ]) + + # ---- master peers (sorted by key for determinism) ---- + master_keys = sort(keys(var.master.nodes)) + master_peer_list = [for k in local.master_keys : + format("%s:%d", var.master.nodes[k].address, coalesce(var.master.nodes[k].port, var.master.port)) + ] + master_peers = join(",", local.master_peer_list) + + # ---- first filer (default s3 target + cluster env) ---- + filer_keys = sort(keys(var.filer.nodes)) + first_filer = length(local.filer_keys) > 0 ? format("%s:%d", var.filer.nodes[local.filer_keys[0]].address, coalesce(var.filer.nodes[local.filer_keys[0]].port, var.filer.port)) : "" + + # ---- cluster discovery env (WEED_CLUSTER_*) ---- + cluster_env = merge( + { WEED_CLUSTER_DEFAULT = var.cluster_name }, + local.master_peers != "" ? { "WEED_CLUSTER_${replace(upper(var.cluster_name), "/[^A-Z0-9_]/", "_")}_MASTER" = local.master_peers } : {}, + local.first_filer != "" ? { "WEED_CLUSTER_${replace(upper(var.cluster_name), "/[^A-Z0-9_]/", "_")}_FILER" = local.first_filer } : {}, + ) + + metrics_flags_master = var.monitoring_enabled ? compact([ + format("-metricsPort=%d", var.master.metrics_port), + var.master.metrics_ip != "" ? format("-metricsIp=%s", var.master.metrics_ip) : "", + ]) : [] + metrics_flags_volume = var.monitoring_enabled ? compact([ + format("-metricsPort=%d", var.volume.metrics_port), + var.volume.metrics_ip != "" ? format("-metricsIp=%s", var.volume.metrics_ip) : "", + ]) : [] + metrics_flags_filer = var.monitoring_enabled ? compact([ + format("-metricsPort=%d", var.filer.metrics_port), + var.filer.metrics_ip != "" ? format("-metricsIp=%s", var.filer.metrics_ip) : "", + ]) : [] + metrics_flags_s3 = var.monitoring_enabled ? [format("-metricsPort=%d", var.s3.metrics_port)] : [] + + # =========================================================================== + # MASTER nodes + # =========================================================================== + master_nodes = var.master.enabled ? { + for k in local.master_keys : "master-${k}" => { + role = "master" + name = "master-${k}" + address = var.master.nodes[k].address + disk_mounts = var.master.nodes[k].disk_mounts + data_dir = coalesce(var.master.nodes[k].data_dir, var.master.data_dir) + data_dirs = [coalesce(var.master.nodes[k].data_dir, var.master.data_dir)] + ports = { + http = coalesce(var.master.nodes[k].port, var.master.port) + metrics = var.master.metrics_port + } + env = local.cluster_env + config_files = local.security_files + argv = concat( + local.global_flags, + ["master"], + [ + format("-port=%d", coalesce(var.master.nodes[k].port, var.master.port)), + format("-mdir=%s", coalesce(var.master.nodes[k].data_dir, var.master.data_dir)), + format("-ip=%s", var.master.nodes[k].address), + format("-ip.bind=%s", var.master.ip_bind), + format("-peers=%s", local.master_peers), + format("-defaultReplication=%s", var.master.default_replication), + format("-volumeSizeLimitMB=%d", var.master.volume_size_limit_mb), + format("-electionTimeout=%s", var.master.election_timeout), + format("-heartbeatInterval=%s", var.master.heartbeat_interval), + ], + var.master.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.master.nodes[k].grpc_port)] : ( + var.master.grpc_port != null ? [format("-port.grpc=%d", var.master.grpc_port)] : []), + var.master.volume_preallocate ? ["-volumePreallocate"] : [], + var.master.garbage_threshold != null ? [format("-garbageThreshold=%s", tostring(var.master.garbage_threshold))] : [], + var.master.raft_hashicorp ? ["-raftHashicorp"] : [], + var.master.resume_state ? ["-resumeState"] : [], + var.master.disable_http ? ["-disableHttp"] : [], + var.master.white_list != "" ? [format("-whiteList=%s", var.master.white_list)] : [], + local.metrics_flags_master, + var.master.extra_args, + ) + } + } : {} + + # =========================================================================== + # VOLUME nodes + # =========================================================================== + volume_keys = sort(keys(var.volume.nodes)) + volume_nodes = var.volume.enabled ? { + for k in local.volume_keys : "volume-${k}" => { + role = "volume" + name = "volume-${k}" + address = var.volume.nodes[k].address + disk_mounts = var.volume.nodes[k].disk_mounts + data_dirs = [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : d.path] + data_dir = (var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs)[0].path + ports = { + http = coalesce(var.volume.nodes[k].port, var.volume.port) + metrics = var.volume.metrics_port + } + env = local.cluster_env + config_files = local.security_files + argv = concat( + local.global_flags, + ["volume"], + [ + format("-port=%d", coalesce(var.volume.nodes[k].port, var.volume.port)), + format("-dir=%s", join(",", [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : d.path])), + format("-max=%s", join(",", [for d in(var.volume.nodes[k].data_dirs != null ? var.volume.nodes[k].data_dirs : var.volume.data_dirs) : tostring(d.max_volumes)])), + format("-ip=%s", var.volume.nodes[k].address), + format("-ip.bind=%s", var.volume.ip_bind), + format("-mserver=%s", local.master_peers), + format("-readMode=%s", var.volume.read_mode), + format("-compactionMBps=%d", var.volume.compaction_mbps), + format("-minFreeSpacePercent=%s", var.volume.min_free_space_percent), + format("-index=%s", var.volume.index), + ], + var.volume.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.volume.nodes[k].grpc_port)] : [], + (var.volume.nodes[k].idx_dir != null && var.volume.nodes[k].idx_dir != "") ? [format("-dir.idx=%s", var.volume.nodes[k].idx_dir)] : (var.volume.idx_dir != "" ? [format("-dir.idx=%s", var.volume.idx_dir)] : []), + var.volume.nodes[k].data_center != null ? [format("-dataCenter=%s", var.volume.nodes[k].data_center)] : [], + var.volume.nodes[k].rack != null ? [format("-rack=%s", var.volume.nodes[k].rack)] : [], + var.volume.nodes[k].public_url != null ? [format("-publicUrl=%s", var.volume.nodes[k].public_url)] : [], + var.volume.file_size_limit_mb != null ? [format("-fileSizeLimitMB=%d", var.volume.file_size_limit_mb)] : [], + var.volume.images_fix_orientation ? ["-images.fix.orientation"] : [], + var.volume.white_list != "" ? [format("-whiteList=%s", var.volume.white_list)] : [], + local.metrics_flags_volume, + var.volume.extra_args, + ) + } + } : {} + + # =========================================================================== + # FILER nodes (+ optional embedded S3) + # =========================================================================== + filer_s3_files = var.filer.s3.enabled ? { (var.filer.s3.config_path) = local.s3_config_json } : {} + filer_nodes = var.filer.enabled ? { + for k in local.filer_keys : "filer-${k}" => { + role = "filer" + name = "filer-${k}" + address = var.filer.nodes[k].address + disk_mounts = var.filer.nodes[k].disk_mounts + data_dir = coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir) + data_dirs = [coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir)] + ports = { + http = coalesce(var.filer.nodes[k].port, var.filer.port) + metrics = var.filer.metrics_port + } + env = merge(local.cluster_env, var.filer.extra_env) + config_files = merge(local.security_files, local.filer_s3_files) + argv = concat( + local.global_flags, + ["filer"], + [ + format("-port=%d", coalesce(var.filer.nodes[k].port, var.filer.port)), + format("-ip=%s", var.filer.nodes[k].address), + format("-ip.bind=%s", var.filer.ip_bind), + format("-master=%s", local.master_peers), + format("-defaultReplicaPlacement=%s", var.filer.default_replica_placement), + format("-dirListLimit=%d", var.filer.dir_list_limit), + format("-defaultStoreDir=%s", coalesce(var.filer.nodes[k].data_dir, var.filer.data_dir)), + ], + var.filer.nodes[k].grpc_port != null ? [format("-port.grpc=%d", var.filer.nodes[k].grpc_port)] : [], + var.filer.max_mb != null ? [format("-maxMB=%d", var.filer.max_mb)] : [], + var.filer.disable_dir_listing ? ["-disableDirListing"] : [], + var.filer.disable_http ? ["-disableHttp"] : [], + var.filer.encrypt_volume_data ? ["-encryptVolumeData"] : [], + var.filer.rack != "" ? [format("-rack=%s", var.filer.rack)] : [], + var.filer.data_center != "" ? [format("-dataCenter=%s", var.filer.data_center)] : [], + var.filer.filer_group != "" ? [format("-filerGroup=%s", var.filer.filer_group)] : [], + var.filer.s3.enabled ? [ + "-s3", + format("-s3.port=%d", var.filer.s3.port), + format("-s3.config=%s", var.filer.s3.config_path), + ] : [], + (var.filer.s3.enabled && var.filer.s3.https_port > 0) ? [format("-s3.port.https=%d", var.filer.s3.https_port)] : [], + (var.filer.s3.enabled && var.filer.s3.domain_name != "") ? [format("-s3.domainName=%s", var.filer.s3.domain_name)] : [], + local.metrics_flags_filer, + var.filer.extra_args, + ) + } + } : {} + + # =========================================================================== + # S3 standalone nodes + # =========================================================================== + s3_keys = sort(keys(var.s3.nodes)) + s3_target = var.s3.filer_address != "" ? var.s3.filer_address : local.first_filer + s3_std_files = merge(local.security_files, { (var.s3.config_path) = local.s3_config_json }) + s3_nodes = var.s3.enabled ? { + for k in local.s3_keys : "s3-${k}" => { + role = "s3" + name = "s3-${k}" + address = var.s3.nodes[k].address + disk_mounts = [] + data_dir = "" + data_dirs = [] + ports = { + http = coalesce(var.s3.nodes[k].port, var.s3.port) + metrics = var.s3.metrics_port + } + env = local.cluster_env + config_files = local.s3_std_files + argv = concat( + local.global_flags, + ["s3"], + [ + format("-port=%d", coalesce(var.s3.nodes[k].port, var.s3.port)), + format("-ip.bind=%s", var.s3.ip_bind), + format("-filer=%s", local.s3_target), + format("-config=%s", var.s3.config_path), + ], + var.s3.https_port > 0 ? [format("-port.https=%d", var.s3.https_port)] : [], + var.s3.iceberg_port != null ? [format("-port.iceberg=%d", var.s3.iceberg_port)] : [], + var.s3.domain_name != "" ? [format("-domainName=%s", var.s3.domain_name)] : [], + var.s3.audit_log_config_path != "" ? [format("-auditLogConfig=%s", var.s3.audit_log_config_path)] : [], + var.s3.cert_file != "" ? [format("-cert.file=%s", var.s3.cert_file)] : [], + var.s3.key_file != "" ? [format("-key.file=%s", var.s3.key_file)] : [], + var.s3.cacert_file != "" ? [format("-cacert.file=%s", var.s3.cacert_file)] : [], + var.s3.verify_client_cert ? ["-tlsVerifyClientCert"] : [], + local.metrics_flags_s3, + var.s3.extra_args, + ) + } + } : {} + + # =========================================================================== + # ALL-IN-ONE nodes (weed server) + # =========================================================================== + aio_keys = sort(keys(var.all_in_one.nodes)) + aio_s3_files = var.all_in_one.s3.enabled ? { (var.all_in_one.s3.config_path) = local.s3_config_json } : {} + aio_nodes = var.all_in_one.enabled ? { + for k in local.aio_keys : "all-in-one-${k}" => { + role = "server" + name = "all-in-one-${k}" + address = var.all_in_one.nodes[k].address + disk_mounts = var.all_in_one.nodes[k].disk_mounts + data_dir = coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir) + data_dirs = [coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir)] + ports = { + http = var.all_in_one.master_port + metrics = var.all_in_one.metrics_port + } + env = local.cluster_env + config_files = merge(local.security_files, local.aio_s3_files) + argv = concat( + local.global_flags, + ["server"], + [ + format("-dir=%s", coalesce(var.all_in_one.nodes[k].data_dir, var.all_in_one.data_dir)), + format("-ip=%s", var.all_in_one.nodes[k].address), + format("-ip.bind=%s", var.all_in_one.ip_bind), + "-master", + format("-master.port=%d", var.all_in_one.master_port), + format("-master.peers=%s:%d", var.all_in_one.nodes[k].address, var.all_in_one.master_port), + format("-master.defaultReplication=%s", var.all_in_one.default_replication), + format("-master.volumeSizeLimitMB=%d", var.all_in_one.volume_size_limit_mb), + "-volume", + format("-volume.port=%d", var.all_in_one.volume_port), + "-filer", + format("-filer.port=%d", var.all_in_one.filer_port), + format("-idleTimeout=%d", var.all_in_one.idle_timeout), + ], + var.all_in_one.disable_http ? [format("-disableHttp=%t", var.all_in_one.disable_http)] : [], + var.all_in_one.s3.enabled ? [ + "-s3", + format("-s3.port=%d", var.all_in_one.s3.port), + format("-s3.config=%s", var.all_in_one.s3.config_path), + ] : [], + (var.all_in_one.s3.enabled && var.all_in_one.s3.domain_name != "") ? [format("-s3.domainName=%s", var.all_in_one.s3.domain_name)] : [], + var.monitoring_enabled ? [format("-metricsPort=%d", var.all_in_one.metrics_port)] : [], + var.all_in_one.extra_args, + ) + } + } : {} + + # =========================================================================== + # Merge + render systemd unit and cloud-init per node + # =========================================================================== + nodes_base = merge(local.master_nodes, local.volume_nodes, local.filer_nodes, local.s3_nodes, local.aio_nodes) + + # Disk-mount script per node (empty unless the node declares disk_mounts). + mount_scripts = { + for name, n in local.nodes_base : name => length(n.disk_mounts) > 0 ? templatefile("${path.module}/templates/mount-disks.sh.tftpl", { + run_as_user = var.hardening.run_as_user + mounts = n.disk_mounts + }) : "" + } + + nodes = { + for name, n in local.nodes_base : name => merge(n, { + exec_start = "${var.weed_binary} ${join(" ", n.argv)}" + # secret_files: the secret-bearing config a wrapper should deliver from a + # secret store when render_secret_files=false (else they go in cloud-init). + secret_files = n.config_files + mount_script = local.mount_scripts[name] + file_modes = { for p in keys(n.config_files) : p => "0600" } + systemd_unit = templatefile("${path.module}/templates/weed.service.tftpl", { + role = n.role + name = n.name + run_as_user = var.hardening.run_as_user + exec_start = "${var.weed_binary} ${join(" ", n.argv)}" + no_new_privileges = var.hardening.no_new_privileges + protect_system = var.hardening.protect_system + cap_drop_all = var.hardening.cap_drop_all + read_write_paths = n.data_dirs + requires_mounts_for = n.data_dirs + env_file = var.env_file + environment = n.env + }) + cloud_init = templatefile("${path.module}/templates/cloud-init.yaml.tftpl", { + role = n.role + name = n.name + run_as_user = var.hardening.run_as_user + config_files = var.render_secret_files ? n.config_files : {} + file_modes = { for p in keys(n.config_files) : p => "0600" } + pre_runcmd = [for p in n.data_dirs : "install -d -o ${var.hardening.run_as_user} -g ${var.hardening.run_as_user} ${p}"] + mount_script = local.mount_scripts[name] + fetch_script = var.boot_fetch_script + systemd_unit = templatefile("${path.module}/templates/weed.service.tftpl", { + role = n.role + name = n.name + run_as_user = var.hardening.run_as_user + exec_start = "${var.weed_binary} ${join(" ", n.argv)}" + no_new_privileges = var.hardening.no_new_privileges + protect_system = var.hardening.protect_system + cap_drop_all = var.hardening.cap_drop_all + read_write_paths = n.data_dirs + requires_mounts_for = n.data_dirs + env_file = var.env_file + environment = n.env + }) + }) + }) + } +} diff --git a/terraform/modules/core/outputs.tf b/terraform/modules/core/outputs.tf new file mode 100644 index 000000000..478c64d3d --- /dev/null +++ b/terraform/modules/core/outputs.tf @@ -0,0 +1,32 @@ +output "nodes" { + description = "Per-node rendered artifacts keyed by node name. Each value: role, name, address, ports, data_dir(s), argv (list, after the weed binary), exec_start, env, config_files, systemd_unit, cloud_init." + value = local.nodes + sensitive = true # config_files/env may carry JWT keys or S3 identities +} + +output "master_peers" { + description = "Comma-separated master peer list (address:port), as passed to -peers / -mserver / -master." + value = local.master_peers +} + +output "first_filer" { + description = "address:port of the lexically-first filer node (default S3 target)." + value = local.first_filer +} + +output "node_names" { + description = "List of rendered node names (non-sensitive)." + value = sort(keys(local.nodes)) +} + +output "cloud_init_by_node" { + description = "Map of node name -> rendered cloud-init user_data (for the per-cloud wrappers)." + value = { for name, n in local.nodes : name => n.cloud_init } + sensitive = true +} + +output "secret_files_by_node" { + description = "Map of node name -> {path => content} of secret-bearing files (security.toml, S3 identity JSON). Wrappers push these to a secret store and fetch them at boot when render_secret_files=false." + value = { for name, n in local.nodes : name => n.secret_files } + sensitive = true +} diff --git a/terraform/modules/core/templates/cloud-init.yaml.tftpl b/terraform/modules/core/templates/cloud-init.yaml.tftpl new file mode 100644 index 000000000..ca6df56a5 --- /dev/null +++ b/terraform/modules/core/templates/cloud-init.yaml.tftpl @@ -0,0 +1,44 @@ +#cloud-config +# Rendered by the SeaweedFS Terraform core module for node ${name} (role ${role}). +# When render_secret_files=false, secret-bearing files are NOT here; they are +# fetched at boot by fetch-secrets.sh from the cloud secret store. +users: + - name: ${run_as_user} + system: true + shell: /usr/sbin/nologin +write_files: +%{ for path, content in config_files ~} + - path: ${path} + owner: "${run_as_user}:${run_as_user}" + permissions: '${lookup(file_modes, path, "0644")}' + content: | + ${indent(6, content)} +%{ endfor ~} +%{ if mount_script != "" ~} + - path: /opt/seaweedfs/mount-disks.sh + permissions: '0755' + content: | + ${indent(6, mount_script)} +%{ endif ~} +%{ if fetch_script != "" ~} + - path: /opt/seaweedfs/fetch-secrets.sh + permissions: '0755' + content: | + ${indent(6, fetch_script)} +%{ endif ~} + - path: /etc/systemd/system/weed-${role}.service + permissions: '0644' + content: | + ${indent(6, systemd_unit)} +runcmd: +%{ if mount_script != "" ~} + - /opt/seaweedfs/mount-disks.sh +%{ endif ~} +%{ if fetch_script != "" ~} + - /opt/seaweedfs/fetch-secrets.sh +%{ endif ~} +%{ for cmd in pre_runcmd ~} + - ${cmd} +%{ endfor ~} + - systemctl daemon-reload + - systemctl enable --now weed-${role}.service diff --git a/terraform/modules/core/templates/mount-disks.sh.tftpl b/terraform/modules/core/templates/mount-disks.sh.tftpl new file mode 100644 index 000000000..0bfcde008 --- /dev/null +++ b/terraform/modules/core/templates/mount-disks.sh.tftpl @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Rendered by the SeaweedFS Terraform core module. Formats (only if blank) and +# mounts each data disk, persisting by UUID to /etc/fstab. Safe to re-run. +set -euo pipefail + +run_as_user="${run_as_user}" + +# Base block device backing "/" so auto-discovery never touches the root disk. +root_src="$(findmnt -no SOURCE / 2>/dev/null || true)" +root_base="$(lsblk -no pkname "$root_src" 2>/dev/null || true)" +[ -n "$root_base" ] && root_base="/dev/$root_base" + +is_mounted() { findmnt -rno TARGET "$1" >/dev/null 2>&1; } + +# Pick the first unmounted, non-root whole disk (single-data-disk node). +discover_dev() { + local d base + for d in /dev/nvme*n1 /dev/xvd[b-z] /dev/sd[b-z] /dev/vd[b-z]; do + [ -b "$d" ] || continue + [ "$d" = "$root_src" ] && continue + [ "$d" = "$root_base" ] && continue + base="/dev/$(lsblk -no pkname "$d" 2>/dev/null || true)" + [ "$base" = "$root_base" ] && continue + is_mounted "$d" && continue + echo "$d"; return 0 + done + return 1 +} + +# Wait up to ~120s for one of the explicit candidate devices to appear. +wait_dev() { + local i d + for i in $(seq 1 60); do + for d in "$@"; do [ -b "$d" ] && { echo "$d"; return 0; }; done + sleep 2 + done + return 1 +} + +mount_one() { + local mountpoint="$1" fstype="$2"; shift 2 + local dev="" + if [ "$#" -gt 0 ]; then + dev="$(wait_dev "$@" || true)" + fi + if [ -z "$dev" ]; then + dev="$(discover_dev || true)" + fi + if [ -z "$dev" ]; then + echo "mount-disks: no device found for $mountpoint" >&2 + return 1 + fi + if ! blkid "$dev" >/dev/null 2>&1; then + echo "mount-disks: formatting $dev as $fstype" + "mkfs.$fstype" -q "$dev" + fi + mkdir -p "$mountpoint" + local uuid; uuid="$(blkid -s UUID -o value "$dev")" + if [ -z "$uuid" ]; then + echo "mount-disks: failed to read a filesystem UUID for $dev" >&2 + return 1 + fi + if ! grep -q "$uuid" /etc/fstab; then + printf 'UUID=%s %s %s defaults,nofail 0 2\n' "$uuid" "$mountpoint" "$fstype" >> /etc/fstab + fi + is_mounted "$mountpoint" || mount "$mountpoint" + chown -R "$run_as_user:$run_as_user" "$mountpoint" + echo "mount-disks: $dev mounted at $mountpoint" +} + +%{ for m in mounts ~} +mount_one "${m.mountpoint}" "${m.fstype}"${length(m.devices) > 0 ? " ${join(" ", m.devices)}" : ""} +%{ endfor ~} diff --git a/terraform/modules/core/templates/security.toml.tftpl b/terraform/modules/core/templates/security.toml.tftpl new file mode 100644 index 000000000..54af7d8ba --- /dev/null +++ b/terraform/modules/core/templates/security.toml.tftpl @@ -0,0 +1,53 @@ +# SeaweedFS security.toml rendered by the Terraform core module. +# Rendered when enable_security OR any non-default JWT signing option is set. +# Presence of this file does NOT imply mTLS: the [grpc] TLS block is emitted +# only under enable_security; a JWT-only config carries [jwt.*] and no TLS. + +%{ if enable_security ~} +[grpc] +ca = "${cert_dir}/ca/tls.crt" +# Peer-identity authorization: without an allow-list, mTLS collapses to +# "trusted the CA" with no authZ. Set allowed_wildcard_domain / commonNames. +%{ if allowed_wildcard_domain != "" ~} +allowed_wildcard_domain = "${allowed_wildcard_domain}" +%{ endif ~} + +[grpc.master] +cert = "${cert_dir}/master/tls.crt" +key = "${cert_dir}/master/tls.key" +%{ if allowed_common_names != "" ~} +allowed_commonNames = "${allowed_common_names}" +%{ endif ~} + +[grpc.volume] +cert = "${cert_dir}/volume/tls.crt" +key = "${cert_dir}/volume/tls.key" + +[grpc.filer] +cert = "${cert_dir}/filer/tls.crt" +key = "${cert_dir}/filer/tls.key" + +[grpc.client] +cert = "${cert_dir}/client/tls.crt" +key = "${cert_dir}/client/tls.key" +%{ endif ~} +%{ if jwt_signing_key != "" ~} + +[jwt.signing] +key = "${jwt_signing_key}" +%{ endif ~} +%{ if jwt_signing_read_key != "" ~} + +[jwt.signing.read] +key = "${jwt_signing_read_key}" +%{ endif ~} +%{ if jwt_filer_signing_key != "" ~} + +[jwt.filer_signing] +key = "${jwt_filer_signing_key}" +%{ endif ~} +%{ if jwt_filer_signing_read_key != "" ~} + +[jwt.filer_signing.read] +key = "${jwt_filer_signing_read_key}" +%{ endif ~} diff --git a/terraform/modules/core/templates/weed.service.tftpl b/terraform/modules/core/templates/weed.service.tftpl new file mode 100644 index 000000000..2e1e07fd6 --- /dev/null +++ b/terraform/modules/core/templates/weed.service.tftpl @@ -0,0 +1,39 @@ +[Unit] +Description=SeaweedFS ${role} (${name}) +Documentation=https://github.com/seaweedfs/seaweedfs/wiki +After=network-online.target +Wants=network-online.target +%{ for path in requires_mounts_for ~} +RequiresMountsFor=${path} +%{ endfor ~} + +[Service] +Type=simple +User=${run_as_user} +Group=${run_as_user} +ExecStart=${exec_start} +Restart=always +RestartSec=5 +LimitNOFILE=1048576 +TimeoutStopSec=60 +# Hardening (OpenShift SCC analogs; relax via the hardening variable) +NoNewPrivileges=${no_new_privileges} +%{ if protect_system ~} +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +%{ for path in read_write_paths ~} +ReadWritePaths=${path} +%{ endfor ~} +%{ endif ~} +%{ if cap_drop_all ~} +CapabilityBoundingSet= +AmbientCapabilities= +%{ endif ~} +EnvironmentFile=-${env_file} +%{ for k, v in environment ~} +Environment="${k}=${v}" +%{ endfor ~} + +[Install] +WantedBy=multi-user.target diff --git a/terraform/modules/core/tests/render.tftest.hcl b/terraform/modules/core/tests/render.tftest.hcl new file mode 100644 index 000000000..4711d7c51 --- /dev/null +++ b/terraform/modules/core/tests/render.tftest.hcl @@ -0,0 +1,234 @@ +# Plan-level tests for the core renderer. No cloud credentials required. +# cd terraform/modules/core && tofu test + +variables { + weed_binary = "/usr/bin/weed" +} + +# --------------------------------------------------------------------------- +run "master_peers_and_count" { + command = plan + variables { + master = { + nodes = { + m0 = { address = "10.0.0.10" } + m1 = { address = "10.0.0.11" } + m2 = { address = "10.0.0.12" } + } + } + volume = { enabled = false } + filer = { enabled = false } + } + assert { + condition = output.master_peers == "10.0.0.10:9333,10.0.0.11:9333,10.0.0.12:9333" + error_message = "master -peers list is wrong" + } + assert { + condition = length(output.node_names) == 3 + error_message = "expected exactly 3 master nodes" + } +} + +# --------------------------------------------------------------------------- +run "volume_uses_mserver_not_master" { + command = plan + variables { + master = { nodes = { m0 = { address = "10.0.0.10" } } } + volume = { nodes = { v0 = { address = "10.0.0.20", rack = "rack-a", data_center = "dc1" } } } + filer = { enabled = false } + } + assert { + condition = contains(output.nodes["volume-v0"].argv, "-mserver=10.0.0.10:9333") + error_message = "volume must pass the master list via -mserver (verified flag), not -master" + } + assert { + condition = !contains(output.nodes["volume-v0"].argv, "-master=10.0.0.10:9333") + error_message = "volume should not use -master for the master list" + } + assert { + condition = contains(output.nodes["volume-v0"].argv, "-rack=rack-a") && contains(output.nodes["volume-v0"].argv, "-dataCenter=dc1") + error_message = "per-node rack/dataCenter must render" + } +} + +# --------------------------------------------------------------------------- +run "filer_uses_master_and_store_dir" { + command = plan + variables { + master = { nodes = { m0 = { address = "10.0.0.10" } } } + volume = { enabled = false } + filer = { nodes = { f0 = { address = "10.0.0.30", data_dir = "/srv/filer" } } } + } + assert { + condition = contains(output.nodes["filer-f0"].argv, "-master=10.0.0.10:9333") + error_message = "filer must use -master for the master list" + } + assert { + condition = contains(output.nodes["filer-f0"].argv, "-defaultStoreDir=/srv/filer") + error_message = "filer leveldb2 store dir must render via -defaultStoreDir" + } +} + +# --------------------------------------------------------------------------- +run "metrics_gated_off_by_default" { + command = plan + variables { + master = { nodes = { m0 = { address = "10.0.0.10" } } } + volume = { enabled = false } + filer = { enabled = false } + } + assert { + condition = length([for a in output.nodes["master-m0"].argv : a if startswith(a, "-metricsPort")]) == 0 + error_message = "metrics port must NOT be bound when monitoring is disabled" + } +} + +run "metrics_on_when_enabled" { + command = plan + variables { + monitoring_enabled = true + master = { nodes = { m0 = { address = "10.0.0.10" } }, metrics_port = 9327 } + volume = { enabled = false } + filer = { enabled = false } + } + assert { + condition = contains(output.nodes["master-m0"].argv, "-metricsPort=9327") + error_message = "metrics port must bind when monitoring is enabled" + } +} + +# --------------------------------------------------------------------------- +run "security_toml_off_by_default" { + command = plan + variables { + master = { nodes = { m0 = { address = "10.0.0.10" } } } + volume = { enabled = false } + filer = { enabled = false } + } + assert { + condition = length(keys(output.nodes["master-m0"].config_files)) == 0 + error_message = "security.toml must not render when security is off and no JWT keys set" + } +} + +run "security_toml_on_with_mtls" { + command = plan + variables { + enable_security = true + security = { + allowed_wildcard_domain = ".seaweedfs.internal" + jwt_signing_key = "test-signing-key-not-a-real-secret-0123456789" + } + master = { nodes = { m0 = { address = "10.0.0.10" } } } + volume = { enabled = false } + filer = { enabled = false } + } + assert { + condition = contains(keys(output.nodes["master-m0"].config_files), "/etc/seaweedfs/security.toml") + error_message = "security.toml must render when enable_security is true" + } +} + +# --------------------------------------------------------------------------- +run "all_in_one_renders_server" { + command = plan + variables { + master = { enabled = false } + volume = { enabled = false } + filer = { enabled = false } + all_in_one = { + enabled = true + nodes = { a0 = { address = "10.0.0.40" } } + s3 = { enabled = true } + } + } + assert { + condition = contains(output.nodes["all-in-one-a0"].argv, "server") + error_message = "all-in-one must invoke the weed server subcommand" + } + assert { + condition = contains(output.nodes["all-in-one-a0"].argv, "-s3") && contains(output.nodes["all-in-one-a0"].argv, "-s3.port=8333") + error_message = "all-in-one S3 must render when enabled" + } +} + +# --------------------------------------------------------------------------- +run "security_toml_mtls_only_no_jwt" { + command = plan + variables { + enable_security = true + master = { nodes = { m0 = { address = "10.0.0.10" } } } + volume = { enabled = false } + filer = { enabled = false } + } + assert { + condition = contains(keys(output.nodes["master-m0"].config_files), "/etc/seaweedfs/security.toml") + error_message = "security.toml must render for mTLS even without JWT keys" + } + assert { + condition = !strcontains(output.nodes["master-m0"].config_files["/etc/seaweedfs/security.toml"], "[jwt.signing]") + error_message = "the jwt.signing block must be omitted when no signing key is set" + } +} + +run "mount_fetch_and_secret_delivery" { + command = plan + variables { + enable_security = true + render_secret_files = false + boot_fetch_script = "#!/usr/bin/env bash\necho fetch\n" + security = { + jwt_signing_key = "test-signing-key-not-a-real-secret-0123456789" + } + master = { nodes = { m0 = { address = "10.0.0.10", disk_mounts = [{ mountpoint = "/data", fstype = "xfs" }] } } } + volume = { enabled = false } + filer = { enabled = false } + } + assert { + condition = strcontains(output.nodes["master-m0"].cloud_init, "/opt/seaweedfs/mount-disks.sh") + error_message = "cloud-init must write+run the mount script when disk_mounts is set" + } + assert { + condition = strcontains(output.nodes["master-m0"].cloud_init, "/opt/seaweedfs/fetch-secrets.sh") + error_message = "cloud-init must write+run the fetch script when boot_fetch_script is set" + } + assert { + condition = !strcontains(output.nodes["master-m0"].cloud_init, "[jwt.signing]") + error_message = "security.toml must NOT be inlined in cloud-init when render_secret_files=false" + } + assert { + condition = contains(keys(output.nodes["master-m0"].secret_files), "/etc/seaweedfs/security.toml") + error_message = "secret_files must expose security.toml for secret-store delivery" + } +} + +run "mount_script_absent_without_disks" { + command = plan + variables { + master = { nodes = { m0 = { address = "10.0.0.10" } } } + volume = { enabled = false } + filer = { enabled = false } + } + assert { + condition = !strcontains(output.nodes["master-m0"].cloud_init, "mount-disks.sh") + error_message = "no mount script should render when the node declares no disk_mounts" + } +} + +run "s3_standalone_targets_filer" { + command = plan + variables { + master = { nodes = { m0 = { address = "10.0.0.10" } } } + volume = { enabled = false } + filer = { nodes = { f0 = { address = "10.0.0.30" } } } + s3 = { enabled = true, nodes = { s0 = { address = "10.0.0.50" } } } + } + assert { + condition = contains(output.nodes["s3-s0"].argv, "-filer=10.0.0.30:8888") + error_message = "standalone S3 must point -filer at the first filer node" + } + assert { + condition = contains(output.nodes["s3-s0"].argv, "-config=/etc/seaweedfs/s3_config.json") + error_message = "standalone S3 must reference the rendered config path" + } +} diff --git a/terraform/modules/core/variables.tf b/terraform/modules/core/variables.tf new file mode 100644 index 000000000..82bb508b9 --- /dev/null +++ b/terraform/modules/core/variables.tf @@ -0,0 +1,318 @@ +# ============================================================================= +# SeaweedFS Terraform core module - input variables +# +# This module is a PURE RENDERER. It creates zero cloud resources. It takes a +# per-node address map plus configuration and emits, per node, the rendered +# `weed` argv, systemd unit, cloud-init, and config files. Both the per-cloud +# infra wrappers (consume cloud_init/systemd_unit) and the local test harness +# (consume argv/config_files) read its `nodes` output. +# +# Flag names below are verified against the real `weed` binary, not the chart. +# Notably: volume uses -mserver for the master list; gRPC is -port.grpc +# (auto = http+10000); minFreeSpacePercent is a string. +# ============================================================================= + +variable "weed_binary" { + description = "Path to the weed executable on the target host." + type = string + default = "/usr/bin/weed" +} + +variable "cluster_name" { + description = "Logical cluster name (WEED_CLUSTER_DEFAULT)." + type = string + default = "sw" +} + +variable "log_level" { + description = "Global glog verbosity (-v)." + type = number + default = 1 +} + +variable "log_to_stderr" { + description = "Log to stderr (journald) instead of -logdir. Recommended on systemd." + type = bool + default = true +} + +variable "monitoring_enabled" { + description = "Gate the -metricsPort/-metrics.address flags. When false the metrics port is not bound, matching the chart." + type = bool + default = false +} + +variable "enable_security" { + description = "Emit the security.toml [grpc] mTLS block and reference per-component certs. Independent of JWT." + type = bool + default = false +} + +# ---------------------------------------------------------------------------- +# Security / secrets (the core only references paths + renders security.toml; +# cert/secret material is generated by the wrapper, never defaulted here). +# ---------------------------------------------------------------------------- +variable "security" { + description = "mTLS + JWT signing configuration. Secret values must be supplied (never defaulted in module code)." + type = object({ + cert_dir = optional(string, "/usr/local/share/ca-certificates") + allowed_wildcard_domain = optional(string, "") + allowed_common_names = optional(string, "") + jwt_signing_key = optional(string, "") + jwt_signing_read_key = optional(string, "") + jwt_filer_signing_key = optional(string, "") + jwt_filer_signing_read_key = optional(string, "") + }) + default = {} + sensitive = true +} + +variable "hardening" { + description = "systemd hardening directives (OpenShift SCC analogs). Relax per field if needed." + type = object({ + run_as_user = optional(string, "seaweedfs") + no_new_privileges = optional(bool, true) + protect_system = optional(bool, true) + cap_drop_all = optional(bool, true) + }) + default = {} +} + +variable "env_file" { + description = "Path to the systemd EnvironmentFile holding secret env (DB creds, S3 admin key) fetched at boot. Optional (-prefixed in the unit, so a missing file is tolerated)." + type = string + default = "/etc/seaweedfs/weed.env" +} + +variable "render_secret_files" { + description = <<-EOT + When true (default), secret-bearing config (security.toml, S3 identity JSON) + is written directly into cloud-init user_data. Convenient for the local test + harness and simple deployments. Set to false to keep secrets OUT of user_data + (and out of the cloud metadata service): the files are then exposed only via + the secret_files_by_node output, and the wrapper is expected to deliver them + from a secret store via boot_fetch_script. + EOT + type = bool + default = true +} + +variable "boot_fetch_script" { + description = "Optional shell script written to /opt/seaweedfs/fetch-secrets.sh (root, 0755) and run before the weed unit starts. Used by wrappers to pull certs/secrets from a cloud secret store at boot." + type = string + default = "" +} + +# ---------------------------------------------------------------------------- +# Master (Raft quorum). Keyed `nodes` map -> for_each, never count. +# ---------------------------------------------------------------------------- +variable "master" { + description = "Master tier. nodes is a map keyed by stable node id (m0/m1/m2)." + type = object({ + enabled = optional(bool, true) + nodes = optional(map(object({ + address = string + port = optional(number) + grpc_port = optional(number) + data_dir = optional(string) + disk_mounts = optional(list(object({ + mountpoint = string + fstype = optional(string, "ext4") + devices = optional(list(string), []) + })), []) + })), {}) + port = optional(number, 9333) + grpc_port = optional(number, null) # null => weed auto = port+10000 + metrics_port = optional(number, 9327) + metrics_ip = optional(string, "") + ip_bind = optional(string, "0.0.0.0") + data_dir = optional(string, "/var/lib/seaweedfs/master") + default_replication = optional(string, "000") + volume_size_limit_mb = optional(number, 1000) + volume_preallocate = optional(bool, false) + garbage_threshold = optional(number, null) + disable_http = optional(bool, false) + raft_hashicorp = optional(bool, false) + resume_state = optional(bool, true) # deliberate deviation: binary default false + election_timeout = optional(string, "10s") + heartbeat_interval = optional(string, "300ms") + white_list = optional(string, "") + extra_args = optional(list(string), []) + }) + default = {} +} + +# ---------------------------------------------------------------------------- +# Volume (stateful disk owners). Keyed `nodes` map. +# ---------------------------------------------------------------------------- +variable "volume" { + description = "Volume tier. Each node owns one or more data dirs (-> -dir / -max)." + type = object({ + enabled = optional(bool, true) + nodes = optional(map(object({ + address = string + port = optional(number) + grpc_port = optional(number) + public_url = optional(string) + rack = optional(string) + data_center = optional(string) + data_dirs = optional(list(object({ + path = string + max_volumes = optional(number, 0) # 0 => auto on non-windows + }))) + idx_dir = optional(string) + disk_mounts = optional(list(object({ + mountpoint = string + fstype = optional(string, "ext4") + devices = optional(list(string), []) + })), []) + })), {}) + port = optional(number, 8080) + metrics_port = optional(number, 9327) + metrics_ip = optional(string, "") + ip_bind = optional(string, "0.0.0.0") + data_dirs = optional(list(object({ + path = string + max_volumes = optional(number, 0) + })), [{ path = "/data", max_volumes = 0 }]) + idx_dir = optional(string, "") + read_mode = optional(string, "proxy") + compaction_mbps = optional(number, 50) + min_free_space_percent = optional(string, "1") + file_size_limit_mb = optional(number, null) + index = optional(string, "memory") + images_fix_orientation = optional(bool, false) + white_list = optional(string, "") + extra_args = optional(list(string), []) + }) + default = {} +} + +# ---------------------------------------------------------------------------- +# Filer (+ optional embedded S3). Keyed `nodes` map. +# ---------------------------------------------------------------------------- +variable "filer" { + description = "Filer tier. Default leveldb2-replicated HA (each filer has its own store)." + type = object({ + enabled = optional(bool, true) + nodes = optional(map(object({ + address = string + port = optional(number) + grpc_port = optional(number) + data_dir = optional(string) + disk_mounts = optional(list(object({ + mountpoint = string + fstype = optional(string, "ext4") + devices = optional(list(string), []) + })), []) + })), {}) + port = optional(number, 8888) + metrics_port = optional(number, 9327) + metrics_ip = optional(string, "") + ip_bind = optional(string, "0.0.0.0") + data_dir = optional(string, "/var/lib/seaweedfs/filer") # -defaultStoreDir for leveldb2 + default_replica_placement = optional(string, "000") + dir_list_limit = optional(number, 100000) + max_mb = optional(number, null) + disable_dir_listing = optional(bool, false) + disable_http = optional(bool, false) + encrypt_volume_data = optional(bool, false) + rack = optional(string, "") + data_center = optional(string, "") + filer_group = optional(string, "") + extra_env = optional(map(string), {}) + extra_args = optional(list(string), []) + s3 = optional(object({ + enabled = optional(bool, false) + port = optional(number, 8333) + https_port = optional(number, 0) + domain_name = optional(string, "") + config_path = optional(string, "/etc/seaweedfs/s3_config.json") + }), {}) + }) + default = {} +} + +# ---------------------------------------------------------------------------- +# S3 gateway (standalone). Keyed `nodes` map (stateless, but kept keyed for +# parity; wrappers may instead drive an ASG). +# ---------------------------------------------------------------------------- +variable "s3" { + description = "Standalone S3 gateway tier." + type = object({ + enabled = optional(bool, false) + nodes = optional(map(object({ + address = string + port = optional(number) + grpc_port = optional(number) + })), {}) + port = optional(number, 8333) + https_port = optional(number, 0) + iceberg_port = optional(number, null) + metrics_port = optional(number, 9327) + ip_bind = optional(string, "0.0.0.0") + domain_name = optional(string, "") + filer_address = optional(string, "") # override; else first filer node + config_path = optional(string, "/etc/seaweedfs/s3_config.json") + audit_log_config_path = optional(string, "") + cert_file = optional(string, "") + key_file = optional(string, "") + cacert_file = optional(string, "") + verify_client_cert = optional(bool, false) + extra_args = optional(list(string), []) + }) + default = {} +} + +# ---------------------------------------------------------------------------- +# S3 identities -> rendered into the s3 config JSON (non-admin only; the admin +# credential is delivered via EnvironmentFile, never the JSON). +# ---------------------------------------------------------------------------- +variable "s3_identities" { + description = "Non-admin S3 identities rendered into the config JSON. Admin key goes in the EnvironmentFile, not here." + type = list(object({ + name = string + access_key = string + secret_key = string + actions = list(string) + })) + default = [] + sensitive = true +} + +# ---------------------------------------------------------------------------- +# All-in-one (single `weed server` process). +# ---------------------------------------------------------------------------- +variable "all_in_one" { + description = "All-in-one tier (weed server). Mutually exclusive with the distributed tiers." + type = object({ + enabled = optional(bool, false) + nodes = optional(map(object({ + address = string + data_dir = optional(string) + disk_mounts = optional(list(object({ + mountpoint = string + fstype = optional(string, "ext4") + devices = optional(list(string), []) + })), []) + })), {}) + data_dir = optional(string, "/data") + ip_bind = optional(string, "0.0.0.0") + master_port = optional(number, 9333) + volume_port = optional(number, 8080) + filer_port = optional(number, 8888) + default_replication = optional(string, "000") + volume_size_limit_mb = optional(number, 1000) + idle_timeout = optional(number, 30) + metrics_port = optional(number, 9324) + disable_http = optional(bool, false) + s3 = optional(object({ + enabled = optional(bool, false) + port = optional(number, 8333) + config_path = optional(string, "/etc/seaweedfs/s3/s3_config.json") + domain_name = optional(string, "") + }), {}) + extra_args = optional(list(string), []) + }) + default = {} +} diff --git a/terraform/modules/core/versions.tf b/terraform/modules/core/versions.tf new file mode 100644 index 000000000..c264e1783 --- /dev/null +++ b/terraform/modules/core/versions.tf @@ -0,0 +1,6 @@ +terraform { + # 1.3+ is required for optional() object attributes with defaults. + # The published wrappers pin a single honest floor; + # the core itself only needs language features available since 1.3. + required_version = ">= 1.3.0" +} diff --git a/terraform/modules/security/main.tf b/terraform/modules/security/main.tf new file mode 100644 index 000000000..294257623 --- /dev/null +++ b/terraform/modules/security/main.tf @@ -0,0 +1,68 @@ +locals { + components = ["master", "volume", "filer", "client"] + ip_sans = distinct(concat(["127.0.0.1"], var.ip_sans)) +} + +# ---- CA --------------------------------------------------------------------- +resource "tls_private_key" "ca" { + algorithm = var.key_algorithm + ecdsa_curve = var.ecdsa_curve + rsa_bits = var.rsa_bits +} + +resource "tls_self_signed_cert" "ca" { + private_key_pem = tls_private_key.ca.private_key_pem + is_ca_certificate = true + + subject { + common_name = "SeaweedFS CA" + organization = "SeaweedFS" + } + + validity_period_hours = var.ca_validity_hours + allowed_uses = ["cert_signing", "crl_signing", "digital_signature"] +} + +# ---- per-component leaf certs (distinct CN per component) -------------------- +resource "tls_private_key" "component" { + for_each = toset(local.components) + algorithm = var.key_algorithm + ecdsa_curve = var.ecdsa_curve + rsa_bits = var.rsa_bits +} + +resource "tls_cert_request" "component" { + for_each = toset(local.components) + private_key_pem = tls_private_key.component[each.key].private_key_pem + + subject { + # Distinct CN per component so peer-identity authZ (allowed_wildcard_domain + # / allowed_commonNames) is meaningful. Do NOT use one CN for all. + common_name = "${each.key}.${var.internal_domain}" + organization = "SeaweedFS" + } + + dns_names = distinct(concat( + ["${each.key}.${var.internal_domain}", "localhost"], + var.extra_dns_sans, + )) + ip_addresses = local.ip_sans +} + +resource "tls_locally_signed_cert" "component" { + for_each = toset(local.components) + cert_request_pem = tls_cert_request.component[each.key].cert_request_pem + ca_private_key_pem = tls_private_key.ca.private_key_pem + ca_cert_pem = tls_self_signed_cert.ca.cert_pem + + validity_period_hours = var.cert_validity_hours + early_renewal_hours = var.cert_early_renewal_hours + allowed_uses = ["digital_signature", "key_agreement", "server_auth", "client_auth"] +} + +# ---- JWT signing keys ------------------------------------------------------- +resource "random_password" "jwt" { + for_each = var.generate_jwt ? toset(["signing", "signing_read", "filer_signing", "filer_signing_read"]) : toset([]) + length = var.jwt_length + special = false # TOML-safe +} diff --git a/terraform/modules/security/outputs.tf b/terraform/modules/security/outputs.tf new file mode 100644 index 000000000..5c48b00dc --- /dev/null +++ b/terraform/modules/security/outputs.tf @@ -0,0 +1,65 @@ +output "ca_cert_pem" { + description = "CA certificate PEM." + value = tls_self_signed_cert.ca.cert_pem +} + +output "certs" { + description = "Per-component { cert_pem, private_key_pem }." + value = { + for c in local.components : c => { + cert_pem = tls_locally_signed_cert.component[c].cert_pem + private_key_pem = tls_private_key.component[c].private_key_pem + } + } + sensitive = true +} + +output "jwt" { + description = "Generated JWT keys (empty strings when generate_jwt = false)." + value = { + signing = var.generate_jwt ? random_password.jwt["signing"].result : "" + signing_read = var.generate_jwt ? random_password.jwt["signing_read"].result : "" + filer_signing = var.generate_jwt ? random_password.jwt["filer_signing"].result : "" + filer_signing_read = var.generate_jwt ? random_password.jwt["filer_signing_read"].result : "" + } + sensitive = true +} + +# Ready to pass straight to the core module's `security` variable. +output "core_security" { + description = "Object shaped for the core module's `security` input (cert_dir, allowed_wildcard_domain, JWT keys)." + value = { + cert_dir = var.cert_dir + allowed_wildcard_domain = ".${var.internal_domain}" + allowed_common_names = "" + jwt_signing_key = var.generate_jwt ? random_password.jwt["signing"].result : "" + jwt_signing_read_key = var.generate_jwt ? random_password.jwt["signing_read"].result : "" + jwt_filer_signing_key = var.generate_jwt ? random_password.jwt["filer_signing"].result : "" + jwt_filer_signing_read_key = var.generate_jwt ? random_password.jwt["filer_signing_read"].result : "" + } + sensitive = true +} + +# Non-sensitive manifest (key, on-host path, mode) so a wrapper can for_each +# over SSM/secret-store resources without tripping the sensitive-for_each rule. +output "secret_manifest" { + description = "List of { key, path, mode } for the CA + component cert/key files (no content)." + value = concat( + [{ key = "ca/tls.crt", path = "${var.cert_dir}/ca/tls.crt", mode = "0644" }], + flatten([for c in local.components : [ + { key = "${c}/tls.crt", path = "${var.cert_dir}/${c}/tls.crt", mode = "0644" }, + { key = "${c}/tls.key", path = "${var.cert_dir}/${c}/tls.key", mode = "0600" }, + ]]), + ) +} + +# Sensitive map keyed by the manifest key -> file content. +output "secret_contents" { + description = "Map of manifest key -> PEM content." + value = merge( + { "ca/tls.crt" = tls_self_signed_cert.ca.cert_pem }, + { for c in local.components : "${c}/tls.crt" => tls_locally_signed_cert.component[c].cert_pem }, + { for c in local.components : "${c}/tls.key" => tls_private_key.component[c].private_key_pem }, + ) + sensitive = true +} diff --git a/terraform/modules/security/variables.tf b/terraform/modules/security/variables.tf new file mode 100644 index 000000000..5e31add4b --- /dev/null +++ b/terraform/modules/security/variables.tf @@ -0,0 +1,89 @@ +# ============================================================================= +# SeaweedFS security material generator (cloud-agnostic). +# +# Generates the CA, per-component mTLS certs with DISTINCT CommonNames, and JWT +# signing keys. Outputs a `core_security` object ready to feed the core module's +# `security` variable, plus the raw PEMs for a wrapper to deliver via a secret +# store. NOTE: TF-generated secrets live in state; prefer Vault PKI for the CA +# in production. +# ============================================================================= + +variable "internal_domain" { + description = "Internal domain for component CNs (e.g. master.). Drives the allowed_wildcard_domain peer-auth check." + type = string + default = "seaweedfs.internal" +} + +variable "ip_sans" { + description = "IP addresses to include as SANs on every component cert (all node private IPs + 127.0.0.1 is added automatically)." + type = list(string) + default = [] +} + +variable "extra_dns_sans" { + description = "Additional DNS SANs to include on every component cert." + type = list(string) + default = [] +} + +variable "key_algorithm" { + description = "CA/leaf key algorithm: ECDSA (default, modern) or RSA." + type = string + default = "ECDSA" + validation { + condition = contains(["ECDSA", "RSA"], var.key_algorithm) + error_message = "key_algorithm must be ECDSA or RSA." + } +} + +variable "ecdsa_curve" { + description = "ECDSA curve when key_algorithm = ECDSA." + type = string + default = "P256" +} + +variable "rsa_bits" { + description = "RSA key size when key_algorithm = RSA." + type = number + default = 3072 +} + +variable "ca_validity_hours" { + description = "CA certificate validity (default 10 years)." + type = number + default = 87600 +} + +variable "cert_validity_hours" { + description = "Component certificate validity. Short by default; certs hot-reload so reissue is cheap." + type = number + default = 72 +} + +variable "cert_early_renewal_hours" { + description = "Renew component certs this many hours before expiry." + type = number + default = 24 +} + +variable "cert_dir" { + description = "On-host directory where certs are placed; surfaced in core_security.cert_dir." + type = string + default = "/usr/local/share/ca-certificates" +} + +variable "generate_jwt" { + description = "Generate JWT signing keys (signing, signing.read, filer_signing, filer_signing.read)." + type = bool + default = true +} + +variable "jwt_length" { + description = "Length of generated JWT keys (>= 32 hardened)." + type = number + default = 40 + validation { + condition = var.jwt_length >= 32 + error_message = "jwt_length must be >= 32 (hardened minimum)." + } +} diff --git a/terraform/modules/security/versions.tf b/terraform/modules/security/versions.tf new file mode 100644 index 000000000..a68ec872a --- /dev/null +++ b/terraform/modules/security/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.3.0" + required_providers { + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.5" + } + } +} diff --git a/terraform/test/local-secure/main.tf b/terraform/test/local-secure/main.tf new file mode 100644 index 000000000..c3eb33b4b --- /dev/null +++ b/terraform/test/local-secure/main.tf @@ -0,0 +1,74 @@ +# ============================================================================= +# Local mTLS test harness: generate the CA + component certs + JWT keys with the +# security submodule, render security.toml with the core, then run a real +# master+volume+filer cluster with mTLS enabled on 127.0.0.1 and assert it works. +# Proves the generated security material is valid and accepted by weed. +# ============================================================================= + +terraform { + required_version = ">= 1.3.0" +} + +variable "weed_binary" { + type = string + default = "/usr/bin/weed" +} + +variable "workdir" { + type = string + default = "/tmp/seaweedfs-tftest-secure" +} + +module "security" { + source = "../../modules/security" + internal_domain = "seaweedfs.internal" + ip_sans = ["127.0.0.1"] + cert_dir = "${var.workdir}/certs" +} + +module "core" { + source = "../../modules/core" + weed_binary = var.weed_binary + enable_security = true + security = module.security.core_security + # We read security.toml back from the output and place it ourselves (under + # $HOME/.seaweedfs), so keep it in the rendered file set. + render_secret_files = true + + # High, distinct ports so this never collides with a running SeaweedFS or the + # non-secure harness. + master = { + nodes = { m0 = { address = "127.0.0.1", port = 29336, data_dir = "${var.workdir}/master-m0" } } + election_timeout = "3s" + heartbeat_interval = "200ms" + } + volume = { + nodes = { v0 = { address = "127.0.0.1", port = 28081, data_dirs = [{ path = "${var.workdir}/volume-v0", max_volumes = 20 }] } } + } + filer = { + nodes = { f0 = { address = "127.0.0.1", port = 28889, data_dir = "${var.workdir}/filer-f0" } } + } +} + +output "cluster" { + sensitive = true + value = { + for name, n in module.core.nodes : name => { + role = n.role + http_port = n.ports.http + data_dirs = n.data_dirs + argv = n.argv + env = n.env + } + } +} + +output "security_toml" { + sensitive = true + value = module.core.secret_files_by_node["master-m0"]["/etc/seaweedfs/security.toml"] +} + +output "certs" { + sensitive = true + value = [for m in module.security.secret_manifest : { path = m.path, mode = m.mode, content = module.security.secret_contents[m.key] }] +} diff --git a/terraform/test/local-secure/run_local_secure.sh b/terraform/test/local-secure/run_local_secure.sh new file mode 100755 index 000000000..5e5588d83 --- /dev/null +++ b/terraform/test/local-secure/run_local_secure.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# Generate mTLS material + security.toml with Terraform, then run a real +# master+volume+filer cluster with mTLS enabled and assert it works. +# ./run_local_secure.sh # render + run + assert + teardown +# KEEP=1 ./run_local_secure.sh +set -u + +HERE="$(cd "$(dirname "$0")" && pwd)" +export PATH="/opt/homebrew/bin:$PATH" +TOFU="${TOFU:-tofu}" +WEED="${WEED:-$(go env GOPATH 2>/dev/null || echo "$HOME/go")/bin/weed}" +WORKDIR="${WORKDIR:-/tmp/seaweedfs-tftest-secure}" +LOGDIR="$WORKDIR/logs" +RUNDIR="$WORKDIR/run" + +PASS=0 +FAIL=0 +ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } +bad() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } +info() { echo "==> $1"; } + +cleanup() { + info "tearing down" + if [ -d "$RUNDIR" ]; then + for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill "$(cat "$pf")" 2>/dev/null; done + sleep 1 + for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill -9 "$(cat "$pf")" 2>/dev/null; done + fi +} + +info "cleaning $WORKDIR" +case "$WORKDIR" in "" | "/" | "$HOME") echo "refusing to delete '$WORKDIR'" >&2; exit 2 ;; esac +rm -rf "$WORKDIR" +mkdir -p "$LOGDIR" "$RUNDIR" "$WORKDIR/.seaweedfs" +[ -x "$WEED" ] || { echo "weed not found at $WEED" >&2; exit 2; } + +info "generating certs + rendering config with OpenTofu" +cd "$HERE" +"$TOFU" init -backend=false -input=false -no-color >/dev/null 2>&1 || { echo "tofu init failed"; exit 2; } +if ! "$TOFU" apply -auto-approve -input=false -no-color \ + -var "weed_binary=$WEED" -var "workdir=$WORKDIR" >"$LOGDIR/tofu.log" 2>&1; then + echo "tofu apply failed"; tail -30 "$LOGDIR/tofu.log"; exit 2 +fi + +# write certs to their on-host paths +"$TOFU" output -json certs | jq -c '.[]' | while IFS= read -r c; do + p="$(echo "$c" | jq -r '.path')"; m="$(echo "$c" | jq -r '.mode')" + mkdir -p "$(dirname "$p")" + echo "$c" | jq -r '.content' > "$p" + chmod "$m" "$p" +done +# place security.toml where weed searches ($HOME/.seaweedfs) +"$TOFU" output -raw security_toml > "$WORKDIR/.seaweedfs/security.toml" +info "security.toml + $(ls "$WORKDIR"/certs | wc -l | tr -d ' ') cert dirs written" + +OUT="$("$TOFU" output -json cluster)" + +# derive ports from the rendered config (high range; never hardcoded) +port_of() { echo "$OUT" | jq -r --arg r "$1" '.[] | select(.role==$r) | .http_port' | head -1; } +MPORT="$(port_of master)"; VPORT="$(port_of volume)"; FPORT="$(port_of filer)" + +busy="" +for p in $(echo "$OUT" | jq -r '.[].http_port'); do + lsof -nP -iTCP:"$p" -sTCP:LISTEN >/dev/null 2>&1 && busy="$busy $p" +done +[ -n "$busy" ] && { echo "Required port(s) already in use:$busy -- is another SeaweedFS running?" >&2; exit 3; } + +[ "${KEEP:-0}" = "1" ] || trap cleanup EXIT INT TERM + +# launch a node with HOME pointed at $WORKDIR so weed loads security.toml +launch() { + n="$1" + for d in $(echo "$OUT" | jq -r --arg n "$n" '.[$n].data_dirs[]?'); do mkdir -p "$d"; done + ENVS=("HOME=$WORKDIR") + while IFS= read -r e; do [ -n "$e" ] && ENVS+=("$e"); done \ + < <(echo "$OUT" | jq -r --arg n "$n" '.[$n].env | to_entries[] | "\(.key)=\(.value)"') + ARGV=() + while IFS= read -r a; do ARGV+=("$a"); done \ + < <(echo "$OUT" | jq -r --arg n "$n" '.[$n].argv[]') + info "launching $n (mTLS)" + env "${ENVS[@]}" "$WEED" "${ARGV[@]}" >"$LOGDIR/$n.log" 2>&1 & + echo "$!" > "$RUNDIR/$n.pid" +} + +wait_http() { + url="$1"; t="${2:-30}"; i=0 + while [ "$i" -lt "$t" ]; do + curl -fsS -o /dev/null --max-time 2 "$url" 2>/dev/null && return 0 + i=$((i + 1)); sleep 1 + done + return 1 +} + +launch master-m0 +QOK=0; i=0 +while [ "$i" -lt 40 ]; do + st="$(curl -fsS --max-time 2 "http://127.0.0.1:$MPORT/cluster/status" 2>/dev/null)" || { i=$((i+1)); sleep 1; continue; } + [ "$(echo "$st" | jq -r '.IsLeader // false')" = "true" ] && { QOK=1; break; } + i=$((i + 1)); sleep 1 +done +[ "$QOK" -eq 1 ] && ok "master elected leader under mTLS" || bad "master leader (mTLS)" + +launch volume-v0 +# registration (master gRPC over mTLS) confirms the volume joined the cluster +AOK=0; i=0 +while [ "$i" -lt 40 ]; do + fid="$(curl -fsS --max-time 2 "http://127.0.0.1:$MPORT/dir/assign" 2>/dev/null | jq -r '.fid // ""')" + [ -n "$fid" ] && { AOK=1; break; } + i=$((i + 1)); sleep 1 +done +[ "$AOK" -eq 1 ] && ok "volume registered via mTLS gRPC (fid $fid)" || bad "volume registration over mTLS" +# /healthz flips to 200 after the first heartbeat completes (slower under mTLS) +wait_http "http://127.0.0.1:$VPORT/healthz" 60 && ok "volume /healthz up (mTLS)" || bad "volume /healthz" + +launch filer-f0 +wait_http "http://127.0.0.1:$FPORT/" 30 && ok "filer / up (mTLS)" || bad "filer /" +# [jwt.filer_signing] is active, so the filer requires a signed JWT for writes. +# An unsigned write MUST be rejected with 401 -- this proves the JWT signing key +# rendered into security.toml is enforced (positive security assertion). +echo "smoke" > "$WORKDIR/hello.txt" +code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -X POST \ + -F "file=@$WORKDIR/hello.txt" "http://127.0.0.1:$FPORT/smoke/hello.txt" 2>/dev/null)" +[ "$code" = "401" ] && ok "filer rejects unsigned write (HTTP 401) => JWT signing enforced" \ + || bad "filer JWT enforcement (expected 401, got HTTP $code)" + +echo +info "RESULTS: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] diff --git a/terraform/test/local/main.tf b/terraform/test/local/main.tf new file mode 100644 index 000000000..4de8a6dc7 --- /dev/null +++ b/terraform/test/local/main.tf @@ -0,0 +1,100 @@ +# ============================================================================= +# Local test harness: render a small SeaweedFS cluster with the core module +# and run it as real `weed` processes on 127.0.0.1 (no cloud, no docker). +# +# 3 masters (quorum) + 1 volume + 1 filer + 1 standalone S3, each on a distinct +# port. run_local_cluster.sh consumes the `cluster` output, launches the weed +# processes from the rendered argv, and asserts the cluster actually works. +# ============================================================================= + +terraform { + required_version = ">= 1.3.0" +} + +variable "weed_binary" { + description = "Path to the weed executable used for the local cluster." + type = string + default = "/usr/bin/weed" +} + +variable "workdir" { + description = "Scratch directory for per-node data dirs and config files." + type = string + default = "/tmp/seaweedfs-tftest" +} + +module "core" { + source = "../../modules/core" + + weed_binary = var.weed_binary + monitoring_enabled = false + enable_security = false + + # High port range so the harness does not collide with a SeaweedFS cluster + # that may already be running on this dev machine (default 9333/8080/8888/8333). + master = { + nodes = { + m0 = { address = "127.0.0.1", port = 29333, data_dir = "${var.workdir}/master-m0" } + m1 = { address = "127.0.0.1", port = 29334, data_dir = "${var.workdir}/master-m1" } + m2 = { address = "127.0.0.1", port = 29335, data_dir = "${var.workdir}/master-m2" } + } + # Tighten election so a 3-node local quorum converges quickly. + election_timeout = "3s" + heartbeat_interval = "200ms" + } + + volume = { + nodes = { + v0 = { + address = "127.0.0.1" + port = 28080 + rack = "rack-a" + data_dirs = [{ path = "${var.workdir}/volume-v0", max_volumes = 20 }] + } + } + } + + filer = { + nodes = { + f0 = { address = "127.0.0.1", port = 28888, data_dir = "${var.workdir}/filer-f0" } + } + } + + s3 = { + enabled = true + nodes = { s0 = { address = "127.0.0.1", port = 28333 } } + # this weed build starts an Iceberg REST catalog on 8181 by default; disable + # it so the test does not collide with a cluster already using that port. + iceberg_port = 0 + config_path = "${var.workdir}/s3_config.json" + } + + # Anonymous identity so the smoke test can PUT/GET without sigv4 signing. + s3_identities = [{ + name = "anonymous" + access_key = "" + secret_key = "" + actions = ["Admin", "Read", "Write", "List", "Tagging"] + }] +} + +output "cluster" { + description = "Node specs consumed by run_local_cluster.sh (read via `tofu output -json`, which emits values even when sensitive)." + sensitive = true + value = { + for name, n in module.core.nodes : name => { + role = n.role + address = n.address + http_port = n.ports.http + data_dirs = n.data_dirs + argv = n.argv + env = n.env + config_files = n.config_files + exec_start = n.exec_start + } + } +} + +output "master_peers" { + value = module.core.master_peers +} diff --git a/terraform/test/local/run_local_cluster.sh b/terraform/test/local/run_local_cluster.sh new file mode 100755 index 000000000..c25593ae5 --- /dev/null +++ b/terraform/test/local/run_local_cluster.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# Render a SeaweedFS cluster with the Terraform core module and run it as real +# `weed` processes locally, then assert it actually works. No cloud, no docker. +# +# ./run_local_cluster.sh # render + run + assert + teardown +# KEEP=1 ./run_local_cluster.sh # leave the cluster running after asserts +# WEED=/path/to/weed ./run_local_cluster.sh +# +# Ports come from the rendered config (high range, to avoid colliding with a +# SeaweedFS cluster already running on this machine). Exits non-zero on any +# failed assertion or if a required port is already in use. +set -u + +HERE="$(cd "$(dirname "$0")" && pwd)" +export PATH="/opt/homebrew/bin:$PATH" +TOFU="${TOFU:-tofu}" +WEED="${WEED:-$(go env GOPATH 2>/dev/null || echo "$HOME/go")/bin/weed}" +WORKDIR="${WORKDIR:-/tmp/seaweedfs-tftest}" +LOGDIR="$WORKDIR/logs" +RUNDIR="$WORKDIR/run" + +PASS=0 +FAIL=0 +ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } +bad() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } +info() { echo "==> $1"; } + +cleanup() { + info "tearing down" + if [ -d "$RUNDIR" ]; then + for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill "$(cat "$pf")" 2>/dev/null; done + sleep 1 + for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill -9 "$(cat "$pf")" 2>/dev/null; done + fi +} + +info "cleaning $WORKDIR" +case "$WORKDIR" in "" | "/" | "$HOME") echo "refusing to delete '$WORKDIR'" >&2; exit 2 ;; esac +rm -rf "$WORKDIR" +mkdir -p "$LOGDIR" "$RUNDIR" +[ -x "$WEED" ] || { echo "weed binary not found/executable at $WEED" >&2; exit 2; } + +info "rendering cluster config with OpenTofu" +cd "$HERE" +"$TOFU" init -backend=false -input=false -no-color >/dev/null 2>&1 || { echo "tofu init failed"; exit 2; } +if ! "$TOFU" apply -auto-approve -input=false -no-color \ + -var "weed_binary=$WEED" -var "workdir=$WORKDIR" >"$LOGDIR/tofu.log" 2>&1; then + echo "tofu apply failed; see $LOGDIR/tofu.log"; tail -30 "$LOGDIR/tofu.log"; exit 2 +fi +OUT="$("$TOFU" output -json cluster)" +[ -n "$OUT" ] || { echo "empty cluster output"; exit 2; } + +# ---- derive ports by role (never hardcoded) --------------------------------- +port_of() { echo "$OUT" | jq -r --arg r "$1" '.[] | select(.role==$r) | .http_port' | head -1; } +mports() { echo "$OUT" | jq -r '.[] | select(.role=="master") | .http_port' | sort; } +MPORT1="$(mports | head -1)" +VPORT="$(port_of volume)"; FPORT="$(port_of filer)"; SPORT="$(port_of s3)" +NMASTERS="$(mports | wc -l | tr -d ' ')" + +# ---- refuse to run if a required port is already taken (foreign cluster) ----- +busy="" +for p in $(echo "$OUT" | jq -r '.[].http_port'); do + lsof -nP -iTCP:"$p" -sTCP:LISTEN >/dev/null 2>&1 && busy="$busy $p" +done +[ -n "$busy" ] && { echo "Required port(s) already in use:$busy -- is another SeaweedFS running?" >&2; exit 3; } + +[ "${KEEP:-0}" = "1" ] || trap cleanup EXIT INT TERM + +node_names() { echo "$OUT" | jq -r 'keys[]'; } +node_field() { echo "$OUT" | jq -r --arg n "$1" --arg f "$2" '.[$n][$f]'; } + +launch_node() { + n="$1" + for p in $(echo "$OUT" | jq -r --arg n "$n" '.[$n].config_files | keys[]?'); do + mkdir -p "$(dirname "$p")" + echo "$OUT" | jq -r --arg n "$n" --arg p "$p" '.[$n].config_files[$p]' > "$p" + done + for d in $(echo "$OUT" | jq -r --arg n "$n" '.[$n].data_dirs[]?'); do mkdir -p "$d"; done + ENVS=() + while IFS= read -r e; do [ -n "$e" ] && ENVS+=("$e"); done \ + < <(echo "$OUT" | jq -r --arg n "$n" '.[$n].env | to_entries[] | "\(.key)=\(.value)"') + ARGV=() + while IFS= read -r a; do ARGV+=("$a"); done \ + < <(echo "$OUT" | jq -r --arg n "$n" '.[$n].argv[]') + info "launching $n" + if [ "${#ENVS[@]}" -gt 0 ]; then + env "${ENVS[@]}" "$WEED" "${ARGV[@]}" >"$LOGDIR/$n.log" 2>&1 & + else + "$WEED" "${ARGV[@]}" >"$LOGDIR/$n.log" 2>&1 & + fi + echo "$!" > "$RUNDIR/$n.pid" +} + +wait_http() { + url="$1"; t="${2:-30}"; i=0 + while [ "$i" -lt "$t" ]; do + curl -fsS -o /dev/null --max-time 2 "$url" 2>/dev/null && return 0 + i=$((i + 1)); sleep 1 + done + return 1 +} + +# ---- launch masters and wait for quorum ------------------------------------- +for n in $(node_names); do + case "$(node_field "$n" role)" in master) launch_node "$n";; esac +done + +info "waiting for master quorum across ports: $(mports | tr '\n' ' ')" +QUORUM_OK=0; i=0 +while [ "$i" -lt 40 ]; do + LEADERS=0; AGREED_LEADER=""; mismatch=0; reachable=0 + for port in $(mports); do + st="$(curl -fsS --max-time 2 "http://127.0.0.1:$port/cluster/status" 2>/dev/null)" || continue + reachable=$((reachable + 1)) + [ "$(echo "$st" | jq -r '.IsLeader // false')" = "true" ] && LEADERS=$((LEADERS + 1)) + ldr="$(echo "$st" | jq -r '.Leader // ""')" + if [ -n "$ldr" ]; then + if [ -z "$AGREED_LEADER" ]; then AGREED_LEADER="$ldr"; elif [ "$AGREED_LEADER" != "$ldr" ]; then mismatch=1; fi + fi + done + if [ "$reachable" -eq "$NMASTERS" ] && [ "$LEADERS" -eq 1 ] && [ "$mismatch" -eq 0 ] && [ -n "$AGREED_LEADER" ]; then + QUORUM_OK=1; break + fi + i=$((i + 1)); sleep 1 +done +if [ "$QUORUM_OK" -eq 1 ]; then + ok "$NMASTERS-master quorum: exactly one leader ($AGREED_LEADER), all agree" +else + bad "master quorum (reachable=$reachable leaders=$LEADERS leader='$AGREED_LEADER')" +fi + +# ---- volume ----------------------------------------------------------------- +for n in $(node_names); do + case "$(node_field "$n" role)" in volume) launch_node "$n";; esac +done +wait_http "http://127.0.0.1:$VPORT/healthz" 40 && ok "volume /healthz up" || bad "volume /healthz" +ASSIGN_OK=0; i=0 +while [ "$i" -lt 20 ]; do + fid="$(curl -fsS --max-time 2 "http://127.0.0.1:$MPORT1/dir/assign" 2>/dev/null | jq -r '.fid // ""')" + [ -n "$fid" ] && { ASSIGN_OK=1; break; } + i=$((i + 1)); sleep 1 +done +[ "$ASSIGN_OK" -eq 1 ] && ok "master assigned fid ($fid) => volume registered" || bad "master /dir/assign (no volume?)" + +# ---- filer ------------------------------------------------------------------ +for n in $(node_names); do + case "$(node_field "$n" role)" in filer) launch_node "$n";; esac +done +wait_http "http://127.0.0.1:$FPORT/" 30 && ok "filer / up" || bad "filer /" +PAYLOAD="seaweedfs-terraform-smoke-$$" +echo "$PAYLOAD" > "$WORKDIR/hello.txt" +curl -fsS -F "file=@$WORKDIR/hello.txt" "http://127.0.0.1:$FPORT/smoke/hello.txt" >/dev/null 2>&1 +got="$(curl -fsS --max-time 5 "http://127.0.0.1:$FPORT/smoke/hello.txt" 2>/dev/null)" +[ "$got" = "$PAYLOAD" ] && ok "filer PUT/GET round-trip" || bad "filer round-trip (got '$got')" + +# ---- s3 --------------------------------------------------------------------- +for n in $(node_names); do + case "$(node_field "$n" role)" in s3) launch_node "$n";; esac +done +wait_http "http://127.0.0.1:$SPORT/status" 30 && ok "s3 /status up" || bad "s3 /status" +curl -fsS -X PUT "http://127.0.0.1:$SPORT/smoke-bucket" >/dev/null 2>&1 +sleep 1 +curl -fsS -X PUT --data-binary "$PAYLOAD" "http://127.0.0.1:$SPORT/smoke-bucket/obj.txt" >/dev/null 2>&1 +s3got="$(curl -fsS --max-time 5 "http://127.0.0.1:$SPORT/smoke-bucket/obj.txt" 2>/dev/null)" +[ "$s3got" = "$PAYLOAD" ] && ok "s3 PUT/GET round-trip" || bad "s3 round-trip (got '$s3got')" + +echo +info "RESULTS: $PASS passed, $FAIL failed" +[ "${KEEP:-0}" = "1" ] && info "KEEP=1: cluster left running. Logs in $LOGDIR." +[ "$FAIL" -eq 0 ]