diff --git a/.gitignore b/.gitignore index f4152160..2e67a9be 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ requirements.yaml .extra_vars.yaml sonic-vs.img *.bak +server_pid.txt +.kubeconfig_insecure .ansible files/certs/*.pem files/certs/**/*.pem diff --git a/Makefile b/Makefile index 89501e64..1ec6069e 100644 --- a/Makefile +++ b/Makefile @@ -26,11 +26,18 @@ MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-la MACHINE_OS=debian-12.0 MAX_RETRIES := 30 + +HOSTNAME_IP := $(shell hostname -I | awk '{print $$1}') + + + + # Machine flavors ifeq ($(MINI_LAB_FLAVOR),cumulus) MACHINE_OS=ubuntu-24.4 LAB_TOPOLOGY=mini-lab.cumulus.yaml VRF=vrf20 +VM_ARGS= else ifeq ($(MINI_LAB_FLAVOR),sonic) LAB_TOPOLOGY=mini-lab.sonic.yaml VRF=Vrf20 @@ -43,6 +50,12 @@ GARDENER_ENABLED=true K8S_VERSION=1.30.8 LAB_TOPOLOGY=mini-lab.sonic.yaml VRF=Vrf20 +VM_ARGS= +else ifeq ($(MINI_LAB_FLAVOR),capms) +LAB_MACHINES=machine01,machine02,machine03 +LAB_TOPOLOGY=mini-lab.capms.yaml +VRF=Vrf20 +VM_ARGS=-e QEMU_MACHINE_CPU_CORES=2 -e QEMU_MACHINE_DISK_SIZE=20G else $(error Unknown flavor $(MINI_LAB_FLAVOR)) endif @@ -140,6 +153,69 @@ external_network: env: @./env.sh +configure-bgp: + @docker exec -it $$(docker ps -qf "name=inet") bash -c "\ + vtysh -c 'configure terminal' \ + -c 'router bgp 4200000021' \ + -c 'network 172.17.0.0/16' \ + -c 'end' \ + -c 'write memory' \ + -c 'show run'" + + + +deploy-fc: configure-bgp _privatenet insecure-kubeconfig deploy-firewall-controller-manager build-firewall-controller create-firewall-image + +deploy-firewall-controller-manager: + @echo "Deploying firewall-controller-manager" + $(MAKE) -C ../firewall-controller-manager deploy + +build-firewall-controller: + @echo "Building firewall-controller docker image" + $(MAKE) -C ../firewall-controller docker + +firewall-metal-images: + @echo "Building firewall image in ../metal-images" + $(MAKE) -C ../metal-images firewall + +create-firewall-image: firewall-metal-images + @echo "Starting HTTP server in ../metal-images on port 8000" + @cd ../metal-images && python3 -m http.server 8000 & + @echo $$! > server_pid.txt + @sleep 5 # Wait for the server to start + @echo "Using URL: http://$(HOSTNAME_IP):8000/images/firewall/3.0-ubuntu/img.tar.lz4" + @metalctl image create \ + --id firewall-ubuntu-4.0 \ + --url http://$(HOSTNAME_IP):8000/images/firewall/3.0-ubuntu/img.tar.lz4 \ + --features "firewall" + +start-server: + @echo "Starting HTTP server on port 8000" + @cd ../metal-images && python3 -m http.server 8000 & echo $$! > server_pid.txt + @echo "HTTP server started with PID: $$(cat server_pid.txt)" + +shut-down-server: + @if [ -f server_pid.txt ]; then \ + PID=$$(cat server_pid.txt); \ + if [ -n "$$PID" ] && ps -p $$PID > /dev/null 2>&1; then \ + echo "Shutting down HTTP server with PID: $$PID"; \ + kill $$PID && rm server_pid.txt; \ + else \ + echo "No running process found for PID: $$PID. Cleaning up."; \ + rm -f server_pid.txt; \ + fi; \ + else \ + echo "Error: server_pid.txt not found."; \ + fi + + + +insecure-kubeconfig: + @sed -e 's/certificate-authority-data: .*/insecure-skip-tls-verify: true/' \ + -e 's/server: https:\/\/0.0.0.0:6443/server: https:\/\/172.17.0.1:6443/' \ + .kubeconfig > .kubeconfig_insecure + @echo "Exporting insecure kubeconfig into .kubeconfig_insecure" + .PHONY: cleanup cleanup: cleanup-control-plane cleanup-partition diff --git a/README.md b/README.md index a24e20f9..04417d3c 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy Here is some code that should help you to set up most of the requirements: - ```bash +```bash # If UFW enabled. # Disable the firewall or allow traffic through Docker network IP range. sudo ufw status @@ -58,11 +58,11 @@ sudo chmod +x /usr/local/bin/kind The following ports are used statically on your host machine: | Port | Bind Address | Description | -|:----:|:------------ |:---------------------------------- | -| 6443 | 0.0.0.0 | kube-apiserver of the kind cluster | -| 4443 | 0.0.0.0 | HTTPS ingress | -| 4150 | 0.0.0.0 | nsqd | -| 8080 | 0.0.0.0 | HTTP ingress | +| :--: | :----------- | :--------------------------------- | +| 6443 | 0.0.0.0 | kube-apiserver of the kind cluster | +| 4443 | 0.0.0.0 | HTTPS ingress | +| 4150 | 0.0.0.0 | nsqd | +| 8080 | 0.0.0.0 | HTTP ingress | ## Known Limitations @@ -117,7 +117,7 @@ make firewall make machine ``` -__Alternatively__, you may want to issue the `metalctl` commands on your own: +**Alternatively**, you may want to issue the `metalctl` commands on your own: ```bash docker compose run --rm metalctl network allocate \ @@ -202,6 +202,125 @@ export MINI_LAB_FLAVOR=sonic make ``` +# Connect Firewall Controller to Kind Cluster + +We need a few different repositories, if you see a 💿 icon you need to change the to the repository that is written after the icon. Using tmux or any terminal multiplexer is recommended. 🌞 + +To establish a connection between the Firewall Controller (FC) and the Kind cluster, you need to configure routing between the FC and the Kind cluster. Follow the steps below to set this up: + +## 1. Configure BGP + +### 💿 MINI-LAB + +Run the following command to configure BGP: + +```bash +make configure-bgp +``` + +```bash +make _privatenet +``` + +## 2. Deploy FC to Connect to the Kind Cluster + +### 💿 FIREWALL-CONTROLLER-MANAGER + +Before executing replace the args of the config/examples/deployment.yaml with: + +```yaml +args: + - -metal-api-url=http://metal-api.metal-control-plane.svc.cluster.local:8080/metal + - -cert-dir=/certs + - -log-level=info + - -seed-api-url=http://172.17.0.1 + - -enable-leader-election + - -namespace=firewall + - -shoot-kubeconfig-secret-name=generic-token-kubeconfig ## Need to create this urself bc usually gardener creates this + - -shoot-token-secret-name=firewall-controller-shoot-access-firewall + - -ssh-key-secret-name=ssh-secret +``` + +Now run: + +```bash +make deploy +``` + +## 3. Update .seed-kubeconfig + +You also need to update the .seed-kubeconfig file on the machine. Follow these steps: + + Connect to the machine. + Replace the existing /etc/firewall-controller/.seed-kubeconfig with the output of the following command: + +### 💿 MINI-LAB + +```bash +make generate-insecure-kubeconfig +``` + +## Running Your Local Firewall Controller in Mini-Lab + +To run your own local Firewall Controller (FC) in the mini-lab, follow these steps: + +### 1. Clone the Metal-Images Repository + +Pull the metal-images repository. 2. Build Your Local Firewall Controller + +### 💿 FIREWALL-CONTROLLER + +```bash +make docker +``` + +### 3. Modify the Firewall/Dockerfile + +### 💿 METAL-IMAGES + +Replace: + +```bash +FROM ghcr.io/metal-stack/firewall-controller:${FIREWALL_CONTROLLER_VERSION} AS firewall-controller-artifacts +``` + +With: + +```bash +FROM my-local-firewall-controller:latest AS firewall-controller-artifacts +``` + +### 4. Build the Firewall Image + +```bash +make firewall +``` + +Sometimes the test.sh line in the **builds.after** fail Just remove that whole line in all the docker-make files. + +### 5. Serve the Built Images + +Start a local HTTP server to serve the images. From within the metal-images repository: + +```bash +cd images +python3 -m http.server 8000 --bind 0.0.0.0 +``` + +### 6. Create Firewall Image in Mini-Lab + +### 💿 MINI-LAB + +```bash +make create-firewall-image +``` + +**Make sure that spec.template.spec.image is firewall-ubuntu-4.0.❗** + +Now create the firewalldeployment.yaml inside the config/examples directory inside the fcm repository. + +Now your local Firewall Controller should be running in the mini-lab environment and also has connection to your cluster! Great job! 💪 + ## Network topology An Nginx is running inside of the www container to allow automatic testing of outgoing connections. diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index 0599f7d8..997edde4 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -1,5 +1,5 @@ name: mini-lab -prefix: "" +prefix: '' mgmt: network: bridge