HAProxy Cluster

Here’s a detailed step-by-step guide on how to create and secure a cluster of 2 HAProxy VMs with Debian 12, each with 4 network interfaces. This guide assumes that you have already set up the VMs and installed Debian 12 on them.

Step 1: Install HAProxy, Keepalived, inotify-tools, sendmail, and iptables-persistent

sudo apt-get update
sudo apt-get install haproxy keepalived rsync inotify-tools iptables-persistent sendmail -y

Step 2: Configure Network Interfaces Edit the /etc/network/interfaces file and add the following for each interface:

# The loopback network interface
auto lo
iface lo inet loopback

# MGMT - eth0
auto eth0
iface eth0 inet static
    address 192.168.0.31
    netmask 255.255.255.0
    gateway 192.168.0.1
    dns-nameservers 192.168.62.41 192.168.62.42
    metric 1

# VLAN61 - eth1
auto eth1
iface eth1 inet static
    address 192.168.61.11
    netmask 255.255.255.0
    metric 2
    post-up route add -net 192.168.61.0 netmask 255.255.255.0 gw 192.168.61.1
    post-up route add -net <public_ip> netmask 255.255.255.255 gw 192.168.61.1

# VLAN62 - eth2
auto eth2
iface eth2 inet static
    address 192.168.62.11
    netmask 255.255.255.0

# VLAN63 - eth3
auto eth3
iface eth3 inet static
    address 192.168.63.11
    netmask 255.255.255.0

Replace the IP addresses with the ones for your VLANs.

Step 3: Configure HAProxy Edit the /etc/haproxy/haproxy.cfg file and replace the content with the following:

global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
    stats socket ipv4@127.0.0.1:9999  level operator  expose-fd listeners
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

defaults
    log     global
    mode    http
    option  tcplog
    option  dontlognull
    timeout connect 5000
    timeout client  50000
    timeout server  50000

# Stats page
listen stats
    bind 127.0.0.1:8404
    mode http
    stats enable
    stats uri /
    stats realm HAProxy-N1\ Statistics
    stats auth user:password
    stats admin if TRUE

# Frontend defined
frontend apache-frontend
        bind *:80
        bind *:443 ssl crt /etc/haproxy/ssl/example.com.pem
        redirect scheme https code 301 if !{ ssl_fc }
        mode http

# ACL STATS
    acl STATS_URL hdr_dom(host) -i stats.example.com
    use_backend STATS if STATS_URL

# Frontend LDAP-3268
frontend 3268-frontend
        bind *:3268
        mode tcp
        use_backend LDAP-3268 if { dst_port 3268 }

# Frontend LDAP-3269
frontend 3269-frontend
        bind *:3269
        mode tcp
        use_backend LDAP-3269 if { dst_port 3269 }

# Frontend mail
frontend mail-frontend
        bind *:25587
        mode tcp
        use_backend MAIL

backend LDAP-3268
    mode tcp
    balance roundrobin
    server dc02 192.168.62.22:3268 check
    server dc03 192.168.62.23:3268 check

backend LDAP-3269
    mode tcp
    balance roundrobin
    server dc03 192.168.62.23:3269 check
    server dc02 192.168.62.22:3269 check

backend MAIL
    mode tcp
    balance roundrobin
    server mail <public_ip>:25587 check

backend STATS
    server HA-N1 127.0.0.1:8404 check

Replace the IP addresses and ports with your actual servers. Also, replace /etc/ssl/private/example.com.pem with the path to your actual SSL certificate.

Step 4: Configure Keepalived Edit the /etc/keepalived/keepalived.conf file and add the following:

global_defs {
   notification_email {
      user@example.com
   }
   notification_email_from noreply@example.com
   smtp_server smtp.example.com
   smtp_connect_timeout 30
   router_id HAProxy_N2
   vrrp_skip_check_adv_addr
}

vrrp_script chk_haproxy {
    script "/bin/bash -c 'pid_count=$(ps aux | grep -c [h]aproxy.pid); [ $pid_count -ge 2 ]'"
    interval 2
    weight 2
}

vrrp_instance VI_1 {
    state MASTER
    interface eth1
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass P@ssword
    }

    unicast_src_ip 192.168.61.11
    unicast_peer {
        192.168.61.12
    }

    virtual_ipaddress {
        192.168.61.10
    }

    track_script {
        chk_haproxy
    }

    notify_master "/etc/keepalived/master-backup.sh MASTER"
    notify_backup "/etc/keepalived/master-backup.sh BACKUP"
    notify_fault "/etc/keepalived/master-backup.sh FAULT"

Replace eth1 with the interface connected to VLAN61, and 192.168.61.10 with your VIP.

Step 5: Create the /etc/keepalived/master-backup.sh script

#!/bin/bash -x

STATE=$1
NOW=$(date +"%D %T")
KEEPALIVED="/etc/keepalived"

case $STATE in
        "MASTER") touch $KEEPALIVED/MASTER
                  echo "$NOW Becoming MASTER" >> $KEEPALIVED/COUNTER
                  /usr/sbin/sendmail -Am -f noreply@example.com -t < /etc/keepalived/notif_master
                  exit 0
                  ;;
        "BACKUP") rm $KEEPALIVED/MASTER
                  echo "$NOW Becoming BACKUP" >> $KEEPALIVED/COUNTER
                  /usr/sbin/sendmail -Am -f noreply@example.com -t < /etc/keepalived/notif_backup
                  exit 0
                  ;;
        "FAULT")  rm $KEEPALIVED/MASTER
                  echo "$NOW Becoming FAULT" >> $KEEPALIVED/COUNTER
                  /usr/sbin/sendmail -Am -f noreply@example.com -t < /etc/keepalived/notif_fault
                  exit 0
                  ;;
        *)        echo "unknown state"
                  echo "$NOW Becoming UNKOWN" >> $KEEPALIVED/COUNTER
                  exit 1
                  ;;
esac

Step 6: Configure sendmail

Edit /etc/mail/sendmail.mc and add at the end

LOCAL_CONFIG
Djexample.com
MAILER_DEFINITIONS
FEATURE(masquerade_envelope)dnl
FEATURE(masquerade_entire_domain)dnl
dnl MASQUERADE_AS(`example.com')dnl
dnl MASQUERADE_DOMAIN(`example.com')dnl
dnl MASQUERADE_DOMAIN(`haproxyn1.example.com')dnl
define(`SMART_HOST',`[smtp.example.com]')dnl
define(`RELAY_MAILER_ARGS', `TCP $h 25587')dnl
define(`ESMTP_MAILER_ARGS', `TCP $h 25587')dnl
define(`confAUTH_OPTIONS', `A p')dnl
define(`confTRUSTED_USERS',`root daemon asterisk example.user')
TRUST_AUTH_MECH(`EXTERNAL DIGEST-MD5 CRAM-MD5 LOGIN PLAIN')dnl
define(`confAUTH_MECHANISMS', `EXTERNAL GSSAPI DIGEST-MD5 CRAM-MD5 LOGIN PLAIN')dnl
FEATURE(`authinfo',`hash -o /etc/mail/authinfo/smtp-auth.db')dnl
MAILER(`local')dnl
MAILER(`smtp')dnl

then compile the conf file

m4 /etc/mail/sendmail.mc > /etc/mail/sendmail.cf

and restart sendmail

systemctl restart sendmail

Step 7: Configure Rsync with inotify-tools

It’s a good practice to create a dedicated user for rsync operations to enhance security. Here are the steps to create a user for rsync and set up SSH key-based authentication:

a) Create a dedicated user on both nodes:

Follow the prompts to set a password and fill in any additional information.

sudo adduser rsyncuser

b) Grant sudo privileges (if needed):

If you want the rsyncuser to have the ability to execute commands with elevated privileges (e.g., for updating files in protected directories), you can add it to the sudo group and you MUST add it to the root group in order to be able to replace the haproxy.cfg file which is owned by root:root:

sudo usermod -aG sudo rsyncuser
sudo usermod -aG root rsyncuser

c) Set up SSH key-based authentication:

On the first node, generate an SSH key pair for the rsyncuser:

sudo -u rsyncuser ssh-keygen -t rsa

Press Enter through the prompts to use the default settings.

d) Copy the public key to the second node:

Use ssh-copy-id to copy the public key to the second node. Replace second_node_ip with the actual IP address or hostname of the second node:

sudo -u rsyncuser ssh-copy-id rsyncuser@second_node_ip

You’ll be prompted to enter the password for rsyncuser on the second node.

e) Test SSH connectivity:

Ensure that you can SSH from the first node to the second node without entering a password:

sudo -u rsyncuser ssh rsyncuser@second_node_ip

If successful, you should be able to connect without a password prompt.

f) Adjust permissions (if necessary):

Make sure the rsyncuser has appropriate read and write permissions for the directories and files you intend to synchronize.

chown -R rsync: /etc/haproxy/

Create a script, let’s call it sync-haproxy.sh, with the following content:

#!/bin/bash

# Define your source and destination information
SOURCE_PATH="/etc/haproxy/haproxy.cfg"
DESTINATION_IP="192.168.0.32"  # Replace with the IP address of the second node
DESTINATION_PATH="/etc/haproxy/haproxy.cfg"

# Watch for changes using inotifywait
inotifywait -e modify,create,delete -m "$SOURCE_PATH" |
while read -r directory events filename; do
    # When a change occurs, sync the file using rsync
    rsync -avz --delete -e "ssh -i /home/rsync/.ssh/id_rsa" \
  --chmod=Dg+rwx,Du+rw,Dgo+rx,Fg+rw,Fo+r "$SOURCE_PATH" "rsync@192.168.0.32:$DESTINATION_PATH"
done

Replace 192.168.1.32 with the IP address of the other HAProxy VM.

Make the script executable and run it in the background:

chmod +x sync-haproxy.sh
nohup ./sync-haproxy.sh &

Step 8: Configure iptables-persistent First, flush all current rules:

iptables -F

Then, add the necessary rules:

iptables -A INPUT -i lo -j ACCEPT
iptables -A INPUT ! -i lo -d 127.0.0.0/8 -j DROP
iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
iptables -A OUTPUT -j ACCEPT
iptables -A INPUT -p tcp --dport 80 -j ACCEPT
iptables -A INPUT -p tcp --dport 443 -j ACCEPT
iptables -A INPUT -p tcp --dport 3268 -j ACCEPT
iptables -A INPUT -p tcp --dport 3269 -j ACCEPT
iptables -A INPUT -p tcp --dport 25587 -j ACCEPT
iptables -A INPUT -p tcp -m state --state NEW --dport 22 -j ACCEPT
iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
iptables -A INPUT -m limit --limit 5/min -j LOG --log-prefix "iptables denied: " --log-level 7

Finally, set DROP as the default policy and save the rules:

iptables -P INPUT DROP
iptables -P FORWARD DROP
iptables-save > /etc/iptables/rules.v4

Replace the IP addresses with the ones for your backend servers and ethX with the corresponding network interfaces.

Clusters

HAProxy Cluster