Liberator

.AddDevice("Liberator", DeviceTypes.Server)
    .WithSerialNumber("lib")
    .WithDescription("Liberator")

.AddDevice("Power-1", DeviceTypes.Server)
    .WithSerialNumber("power-1")
    .WithDescription("Power Station")

var runFullNetworkFlow =
    GetOrderedVmsForHost(config, host.SerialNumber)
        .Any(v => v.OperatingSystem == OS.OPNSense);

@startuml Liberator-Network

title Liberator network topology - hAP ax2 + Proxmox + OPNsense + VLANs

' Role discriminator = host.Description (DeviceTranslator.cs:297 pattern):
'   "Liberator"     -> full flow: Mikrotik upstream + OPNsense on host + VLANs
'   "Power Station" -> bare Proxmox, future cluster compute, no router/VLAN
'   "Proxmox" / other -> bare Proxmox, no router/VLAN
' Defensive double-gate: isLiberator && hasOpnsenseVm in config for host.SerialNumber.
'
' VLAN map: 10 = server tier, 20 = LAN clients, 250 = OOB/MGMT, 254 = Guest.
' WAN on Mikrotik: dual-WAN auto-detect. ether1 untagged (plain DHCP) AND ether1-vlan10 tagged
'   (XGS-PON modem delivers WAN on VLAN10). dhcp-client on BOTH; only one leases (same wire);
'   both are members of the WAN interface-list. Built by flash-mikrotik .rsc (MikrotikRscBuilder).
' DHCP source of truth: OPNsense per VLAN. Static reservations from config.Computers[].MAC.
' DNS resolver: NixDns VM running Unbound. OPNsense forwards all clients to NixDns IP.

skinparam rectangle {
  BackgroundColor<<infra>> #E8F0FE
  BackgroundColor<<svc>>   #E6F4EA
  BackgroundColor<<fw>>    #FCE8E6
  BackgroundColor<<dns>>   #FEF7E0
  BackgroundColor<<net>>   #F3E8FD
}

cloud "ISP / GPON CPE\nuntagged DHCP OR XGS-PON tagged VLAN10" as NET

node "Mikrotik hAP ax2 (L2 only post phase 2)\nmgmt IP VLAN250 .1\nNO NAT, NO DHCP server" as MT <<net>> {
  rectangle "ether1 (2.5G) WAN\ndual-WAN: ether1 untagged DHCP\n+ ether1-vlan10 tagged DHCP (XGS-PON)\nboth in WAN interface-list" as MT_E1
  rectangle "ether2 + ether3 trunk\n(LACP 802.3ad from flash-mikrotik;\nMikrotikDesiredStateBuilder pushes\nbridge-ports + VLANs only)\ntrunk: 10,20,250,254" as MT_BOND
  rectangle "ether4 access\nVLAN20 untagged" as MT_E4
  rectangle "ether5 access (spare LAN port)\nVLAN20 untagged" as MT_E5
  rectangle "wlan 2.4 + 5 GHz\nSSID Liberator\nWPA2/3 -> VLAN20" as MT_W_LAN
  rectangle "wlan 2.4 GHz\nSSID Liberator-Guest\nopen, no portal -> VLAN254" as MT_W_GUEST
  rectangle "wlan 5 GHz hidden\nSSID admin-oob-lib\nWPA3 -> VLAN250" as MT_W_ADMIN
}

NET <--> MT_E1 : public IP

node "Proxmox host (Liberator)\nbond0 = enp1s0 + enp2s0 LACP\nvmbr0 VLAN-aware trunk\nPhase 1 (fresh install, answer.toml):\n  10.0.0.200/24 gw 10.0.0.1 dns 10.0.0.1 (Mikrotik LAN)\nPhase 2 (every netsetup update cycle):\n  vmbr0v250 = 10.0.250.2/24 gw 10.0.250.1 dns NixDns\n  (ConfigureProxmoxBridges Phase 3e)" as PVE <<infra>> {

  rectangle "OPNsense VM (Option Y, 5 vNIC)\nserial 1001@lib\n  net0 untagged   = WAN\n  net1 tag=10     = .1 server tier gw\n  net2 tag=20     = .1 LAN gw + DHCP\n  net3 tag=250    = .3 MGMT\n  net4 tag=254    = .1 Guest gw\nFW deny-default inter-VLAN\nNAT outbound\nDNS forwarder -> NixDns\nREST API creds deterministic (HMAC),\nopnsense-api.env (re)written every update run" as OPN <<fw>>

  rectangle "NixDns VM (coredns + Traefik)\nserial 1002@lib\nVLAN10\nIP = OU default /24 .2\nresolver + reverse + DDNS\nSINGLE inbound ingress (Traefik):\n  80/443 HTTP host-route\n  25/587/993 TCP passthrough" as DNS <<dns>>
  rectangle "Win Server DC\nserial dc@lib\nvNIC tag=10\nVLAN10" as DC <<svc>>
  rectangle "Nextcloud VM\nserial 1020@lib\nvNIC tag=10 = .20\nVLAN10" as NC <<svc>>
  rectangle "Mail VM\nserial 1090@lib\nvNIC tag=10 = .90\nVLAN10" as MAIL <<svc>>
  rectangle "PMG LXC\nctid 1091@lib\nvNIC tag=10 = .91\nVLAN10" as PMG <<svc>>
  rectangle "Win11 client VM\nvNIC tag=20\nDHCP from OPNsense VLAN20" as W11 <<svc>>
}

MT_BOND <==> PVE : LACP 802.3ad trunk\n10 / 20 / 250 / 254

OPN -[hidden]- DNS
OPN -[hidden]- DC

DNS --> OPN  : VLAN10
DC --> OPN   : VLAN10
NC --> OPN   : VLAN10
MAIL --> OPN : VLAN10
PMG --> OPN  : VLAN10
W11 --> OPN  : VLAN20

MT_E4 ..> OPN : VLAN20 access (LAN clients reach OPNsense via Mikrotik bridge)
MT_W_LAN ..> W11 : VLAN20 (WiFi clients)
MT_W_ADMIN ..> PVE : VLAN250 OOB (admin laptop -> Proxmox host independent of OPNsense)

note bottom of OPN
  **Option Y: 5 tagged vNICs**
  Hypervisor enforces VLAN, defence in depth.
  qm create:
    -net0 virtio,bridge=vmbr0
    -net1 virtio,bridge=vmbr0,tag=10
    -net2 virtio,bridge=vmbr0,tag=20
    -net3 virtio,bridge=vmbr0,tag=250
    -net4 virtio,bridge=vmbr0,tag=254
  Other VMs: single tagged vNIC each.
end note

note right of PVE
  Out-of-band mgmt path:
    laptop -> admin-oob-lib WiFi (hidden, VLAN250)
    -> Mikrotik bridges VLAN250 over LACP trunk
    -> Proxmox host .2 reachable
    even if OPNsense down.
end note

note left of MT
  **Phase 1 bootstrap** (today's flash-mikrotik + Liberator-gated answer.toml):
    flat NAT, Mikrotik DHCP, single vmbr0 on host.
    Proxmox IP = 10.0.0.200/24 gw 10.0.0.1 dns 10.0.0.1.
    Mikrotik IP = factory 10.0.0.1/24.
  **Phase 2 cutover** (additive then strip, every netsetup update cycle):
    ConfigureMikrotikVlans  -> bridge VLAN-aware, VLAN 10/20/250/254
                               Mikrotik mgmt 10.0.250.1 (LiberatorNetwork.MikrotikMgmtIp)
    ConfigureProxmoxBridges -> rewrite /etc/network/interfaces:
                               bond0 + vmbr0 + vmbr0v250 = 10.0.250.2/24 gw .1 dns NixDns
                               (idempotent: only ifreload -a on diff)
    RewireVmNicsToVlans     -> qm set <vmid> -netN ...,tag=<vlan> drift fix
    Enable OPNsense WAN VLAN if -> internet flows via OPNsense
    Strip Mikrotik flat NAT + DHCP -> Mikrotik = pure L2.
  All IPs + bridge names come from LiberatorNetwork.cs constants - no literals.
  Re-pushed every cycle, drift-correcting.
end note

note top of NET
  **Role gate**: host.Description == "Liberator"
  AND OPNsense VM in config for host.SerialNumber.
  Otherwise (Power Station / Proxmox / other):
  skip entire Mikrotik + OPNsense + VLAN flow.
  VMs still provisioned on flat vmbr0 untagged.
end note

note as INBOUND
  **Inbound published services (DNAT + firewall-allow)**
  Goal locked 2026-05-22, IMPLEMENTED. On-prem authoritative, NO VPS.
  DNS VM is the **SINGLE ingress** - edge DNAT does NOT split per service.
  Config-driven, ManagedPrefix "netsetup-" (converge deletes only its own).

  Edge DNAT chain (per ServedPort, single target):
    Mikrotik WAN: dst-nat {port} in-interface-list=WAN -> OPNsense WAN IP
        nat   "netsetup-dnat-{port}"  (chain dstnat, action dst-nat,
                  in-interface-list=WAN, in-interface empty)
        filter"netsetup-allow-{port}" (chain forward, action accept,
                  in-interface-list=WAN, in-interface empty)
      Matches WAN LIST (ether1 + ether1-vlan10), NOT literal ether1, so
      port-forward fires on whichever WAN iface leases (plain DHCP vs
      XGS-PON VLAN10). Converge REFERENCES the WAN list but does NOT
      create it - flash-mikrotik .rsc builds the list + members first
      (flash-then-update order). Const MikrotikDesiredStateBuilder.WanInterfaceList.
      ONLY when a NETSetupMikrotik is in config. Else operator
      forwards the modem port -> OPNsense WAN IP by hand.
    OPNsense WAN: d_nat {port} (DestinationNet=wanip) -> NixDns LAN IP
        rdr    "netsetup-dnat-{port}" (descr=identity, DestinationNet wanip,
                  target NixDns LAN IP, local-port {port}, disabled "0"=active)
        filter "netsetup-fwd-{port}"  (pass WAN -> NixDns LAN IP:{port})

  NixDns Traefik sorts (host.dns.reverseProxy):
    80/443    -> HTTP host-route by Host header:
                   nextcloud.<domain> -> Nextcloud .20
                   mail.<domain>      -> Mail .90 (roundcube)
    25        -> TCP passthrough L4 (HostSNI catch-all, tls.passthrough,
                   NO termination, STARTTLS untouched) -> PMG .91
                   -> spam/virus filter + pmgsh domain+transport
                   -> Mail .90 (postfix smtpd 25)
    587, 993  -> TCP passthrough L4 -> Mail .90 (submission / IMAPS)
    tcpForwards { listenPort; backendHost; backendPort; } per mail port.

  PMG is the per-domain inbound relay (NOT a hand-edited main.cf).
    netsetup pmgsh push every cycle:
      relay domain + transport <domain> -> smtp:[mailVmIp]:25 use_mx 0
      default smarthost mailVmIp:25
    Idempotent, self-heals. Replaces OLD broken relayhost mail.<domain>:26.
  Mail VM postfix mynetworks includes PMG /32 so PMG forwards are trusted
    (host.mail.trustedNetworks, resolved by NixTemplateFactory.ResolvePmgIp).

  Internal DNS auto-emit (NixDns Unbound, split-horizon, 2026-05-26 + 2026-05-27):
    mail.<domain>     IN A   Mail .90    (NixDnsTemplate.ExtraARecords)
    proxmox.<domain>  IN A   10.0.250.2  (NixDnsTemplate.ExtraARecords, Liberator only,
                                          LiberatorVmHostnames.ProxmoxDnsKey, 2026-05-27)
    <domain>.         IN MX  10 mail.<domain>.   (NixDnsTemplate.MxRecords)
  Public DNS MX + A at registrar is STILL MANUAL (no DomainProvider MX-write yet).

  Public IP follows dyndns: MX laufentaler.org -> laufentaler.ddns.net (No-IP)
  -> current WAN IP, refreshed by ddns-updater on NixDns.
  Outbound mail: Mail VM -> Brevo SMTP 587 (SASL). No direct MX (dynamic IP).
end note

@enduml

auto bond0
iface bond0 inet manual
    bond-slaves enp1s0 enp2s0
    bond-mode 802.3ad
    bond-miimon 100
    bond-xmit-hash-policy layer3+4

auto vmbr0
iface vmbr0 inet manual
    bridge-ports bond0
    bridge-vlan-aware yes
    bridge-vids 2-4094

auto vmbr0v250
iface vmbr0v250 inet static
    address 10.0.250.2/24
    gateway 10.0.250.1
    dns-nameservers <NixDns LAN IP>
    bridge_ports bond0.250
    bridge_stp off
    bridge_fd 0

qm create <vmid> \
  --bios seabios \
  --memory 2048 --cores 2 \
  --agent enabled=1 \
  --scsihw virtio-scsi-single \
  --net0 virtio,bridge=vmbr0 \
  --net1 virtio,bridge=vmbr0,tag=10 \
  --net2 virtio,bridge=vmbr0,tag=20 \
  --net3 virtio,bridge=vmbr0,tag=250 \
  --net4 virtio,bridge=vmbr0,tag=254
qm importdisk <vmid> /var/lib/vz/template/iso/opnsense.img <storage>
qm set <vmid> --scsi0 <storage>:vm-<vmid>-disk-0 --boot order=scsi0
qm start <vmid>

@startuml Liberator-Network-Push

title netsetup update push - Liberator network converge (every cycle, idempotent)

actor Admin
participant "netsetup update\n(on Proxmox host)" as NS
participant "UpdateMethods\nRunProxmoxUpdate" as UM
participant "EnsureCustomerVms" as ECV
participant "ProvisionOpnsenseVm" as POPN
participant "OpnsenseInstall\n(SSH first-boot)" as INST
participant "ConfigureOpnsenseVlans\n(REST API push)" as COV
participant "ConfigureMikrotikVlans\n(REST API push)" as CMV
participant "ConfigureProxmoxBridges\n(/etc/network/interfaces +\nsafe ifreload)" as CPB
participant "RewireVmNicsToVlans\n(qm set tag=...)" as RWN
participant "OPNsense VM" as OPN
participant "Mikrotik hAP ax2" as MT
participant "Other VMs (Nix / Win / PMG)" as VMS

note over NS
  Pre-state on fresh Liberator install:
    Proxmox host IP = 10.0.0.200/24 gw 10.0.0.1 dns 10.0.0.1
    (set by Liberator-gated answer.toml emit:
     ProxmoxAutoInstallExtensions.ToProxmoxAutoInstall)
    Mikrotik = factory 10.0.0.1/24 (post-flash-mikrotik)
  This cycle moves the host to 10.0.250.2 mgmt VLAN
  during Phase 3e ConfigureProxmoxBridges.
end note

Admin -> NS: netsetup update
NS -> UM: RunProxmoxUpdate(host, config)
UM -> UM: post-install gate (log file)\napt update + full-upgrade\nEnsureProxmoxResolvConfSecondaryDns

== Role gate ==
UM -> UM: isLiberator = host.Description == "Liberator"
UM -> UM: hasOpnsenseVm = GetOrderedVmsForHost(config, host.SerialNumber)\n  .Any(v => v.OS == OS.OPNSense)
UM -> UM: runFullNetworkFlow = isLiberator && hasOpnsenseVm

alt runFullNetworkFlow

  UM -> ECV: Run(host, config, services)

  == Phase 3d.3: OPNsense provision (Option Y, 5 vNIC) ==
  ECV -> POPN: provision (default raw image, first only)
  POPN -> OPN: qm create <vmid>\n  --net0 virtio,bridge=vmbr0\n  --net1 virtio,bridge=vmbr0,tag=10\n  --net2 virtio,bridge=vmbr0,tag=20\n  --net3 virtio,bridge=vmbr0,tag=250\n  --net4 virtio,bridge=vmbr0,tag=254\nqm importdisk + qm set --scsi0 + qm start

  alt /conf/.netsetup-installed missing
    ECV -> INST: OpnsenseInstall.Install(host, machine, config)
    INST -> INST: build config.xml:\n  vtnet0..4 assigned\n  DHCP per VLAN (10/20/254)\n  static maps from config.Computers[].MAC\n  DNS forwarder -> NixDns .53\n  FW deny-default inter-VLAN\n  allow VLAN20 -> VLAN10 (AD/DNS/SMB/print/HTTPS)\n  admin user + api_key/api_secret
    INST -> OPN: ssh root@<wanIp> (master key)\nscp config.xml -> /conf/\nconfigctl service reload all
    INST -> INST: persist api_key/secret to\n/etc/netsetup/opnsense-api.env
  end

  == Phase 3d.x: other VMs (single tagged vNIC each) ==
  ECV -> VMS: NixDns/DC/NC/Mail/PMG -> tag=10\nWin11 client -> tag=20\n(serial host matching unchanged)

  == Phase 3e: VLAN converge (every cycle, drift-correcting) ==

  UM -> COV: push OPNsense REST API
  COV -> OPN: probe /api/core/system/status (basic auth)
  COV -> OPN: PUT /api/dhcpv4/static_map\nPUT /api/dnsforwarder/settings\nPUT /api/firewall/alias/...\nPUT /api/firewall/filter/...\nPOST /api/firewall/apply

  UM -> CMV: push Mikrotik REST API (via osisa.Mikrotik converge)\n  CMV -> MT: bridge1 vlan-aware\nbridge-ports ether2/ether3 trunk PVID=1 tagged 10,20,250,254\nbridge-ports ether4/ether5 access PVID=20 untagged\nbridge-vlans for 10,20,250,254\nip 10.0.250.1/24 on bridge1 (mgmt)\nwifi SSIDs: Liberator(20)\n             Liberator-Guest(254)\n             admin-oob-lib(250 hidden)\nadmin User + SystemIdentity\ndst-nat WAN ports -> OPNsense WAN IP\n(NOT pushed: LACP bond - factory flash-mikrotik\nstep handles bond + WAN DHCP + NAT/firewall;\nMikrotikDesiredStateBuilder manages only what\ncarries the netsetup- prefix)

  UM -> CPB: rewrite /etc/network/interfaces:\n  bond0 LACP slaves = first 2 physical NICs\n    (PhysicalNicProvider seam, default ip -j link, sorted)\n    bond-mode 802.3ad, miimon 100, xmit-hash-policy layer3+4\n  vmbr0 bridge-vlan-aware (vids 2-4094)\n  vmbr0v250 host mgmt = 10.0.250.2/24\n    gw 10.0.250.1 (Mikrotik mgmt)\n    dns NixDns LAN IP (fallback Mikrotik mgmt)\nAll IPs + bridge names from LiberatorNetwork.cs consts\nDiff vs current; only ifreload -a on change\nifupdown2 reload safe-mode\n(rollback if mgmt lost)\nInner gate: LiberatorGate.IsFullLiberatorFlow re-check

  UM -> RWN: for each existing VM, drift-check tag:\n  qm set <vmid> -netN ...,tag=<vlan>\nonly when tag missing or wrong\n(idempotent)

  UM -> UM: EnsureGatewayTakeover (Phase 3e tail)\n  if OPNsense LAN reachable on HTTPS:443\n  then awk-rewrite gateway under iface vmbr0\n  + ifreload -a + flip in-memory config.GatewayIP\n  (orthogonal to vmbr0v250 mgmt above)\n  see docs/firewall.adoc

  == Phase 3f: btrfs scrub ==
  UM -> UM: btrfs scrub each mounted fs

else !runFullNetworkFlow (Power Station / vanilla Proxmox / other)
  UM -> ECV: Run(host, config, services)
  ECV -> VMS: VMs provisioned on flat vmbr0 untagged\n(serial host matching unchanged)
  note right
    Skipped:
      ProvisionOpnsenseVm
      OpnsenseInstall
      ConfigureOpnsenseVlans
      ConfigureMikrotikVlans
      ConfigureProxmoxBridges
      RewireVmNicsToVlans
    EnsureGatewayTakeover still runs but
    no-ops with NoOpnsense outcome.
    Bare Proxmox path. Power Station
    cluster wiring = future TODO.
  end note
  UM -> UM: btrfs scrub each mounted fs
end

UM --> NS: update complete
NS --> Admin: exit 0

note over UM
  Cutover sequence (additive then strip) handled by the
  ordering above: OPNsense API stages all interfaces first
  (VLAN children created), Mikrotik adds VLAN bridge alongside
  flat, Proxmox swings VM NICs to tags, OPNsense WAN VLAN if
  enabled, finally Mikrotik strips flat NAT + DHCP.
  Every step idempotent so re-running netsetup update
  is always safe and converges drift.
end note

@enduml