diff --git a/util/linuxfw/fake_netfilter.go b/util/linuxfw/fake_netfilter.go index d760edfcf..eac5d904c 100644 --- a/util/linuxfw/fake_netfilter.go +++ b/util/linuxfw/fake_netfilter.go @@ -71,6 +71,8 @@ func (f *FakeNetfilterRunner) AddHooks() error { retur func (f *FakeNetfilterRunner) DelHooks(logf logger.Logf) error { return nil } func (f *FakeNetfilterRunner) AddSNATRule() error { return nil } func (f *FakeNetfilterRunner) DelSNATRule() error { return nil } +func (f *FakeNetfilterRunner) AddConnmarkSaveRule() error { return nil } +func (f *FakeNetfilterRunner) DelConnmarkSaveRule() error { return nil } func (f *FakeNetfilterRunner) AddStatefulRule(tunname string) error { return nil } func (f *FakeNetfilterRunner) DelStatefulRule(tunname string) error { return nil } func (f *FakeNetfilterRunner) AddLoopbackRule(addr netip.Addr) error { return nil } diff --git a/util/linuxfw/iptables_runner.go b/util/linuxfw/iptables_runner.go index ed55960b3..b8eb39f21 100644 --- a/util/linuxfw/iptables_runner.go +++ b/util/linuxfw/iptables_runner.go @@ -527,6 +527,104 @@ func (i *iptablesRunner) DelStatefulRule(tunname string) error { return nil } +// AddConnmarkSaveRule adds conntrack marking rules to save and restore marks. +// These rules run in mangle/PREROUTING (to restore marks from conntrack) and +// mangle/OUTPUT (to save marks to conntrack) before rp_filter checks, enabling +// proper routing table lookups for exit nodes and subnet routers. +func (i *iptablesRunner) AddConnmarkSaveRule() error { + // Check if rules already exist (idempotency) + for _, ipt := range []iptablesInterface{i.ipt4, i.ipt6} { + rules, err := ipt.List("mangle", "PREROUTING") + if err != nil { + continue + } + // Look for existing connmark restore rule + for _, rule := range rules { + if strings.Contains(rule, "CONNMARK") && + strings.Contains(rule, "restore-mark") && + strings.Contains(rule, "ctmask 0xff0000") { + // Rules already exist, skip adding + return nil + } + } + } + + // mangle/PREROUTING: Restore mark from conntrack for ESTABLISHED/RELATED connections + // This runs BEFORE routing decision and rp_filter check + for _, ipt := range []iptablesInterface{i.ipt4, i.ipt6} { + args := []string{ + "-m", "conntrack", + "--ctstate", "ESTABLISHED,RELATED", + "-j", "CONNMARK", + "--restore-mark", + "--nfmask", fwmarkMask, + "--ctmask", fwmarkMask, + } + if err := ipt.Insert("mangle", "PREROUTING", 1, args...); err != nil { + return fmt.Errorf("adding %v in mangle/PREROUTING: %w", args, err) + } + } + + // mangle/OUTPUT: Save mark to conntrack for NEW connections with non-zero marks + for _, ipt := range []iptablesInterface{i.ipt4, i.ipt6} { + args := []string{ + "-m", "conntrack", + "--ctstate", "NEW", + "-m", "mark", + "!", "--mark", "0x0/" + fwmarkMask, + "-j", "CONNMARK", + "--save-mark", + "--nfmask", fwmarkMask, + "--ctmask", fwmarkMask, + } + if err := ipt.Insert("mangle", "OUTPUT", 1, args...); err != nil { + return fmt.Errorf("adding %v in mangle/OUTPUT: %w", args, err) + } + } + + return nil +} + +// DelConnmarkSaveRule removes conntrack marking rules added by AddConnmarkSaveRule. +func (i *iptablesRunner) DelConnmarkSaveRule() error { + for _, ipt := range []iptablesInterface{i.ipt4, i.ipt6} { + // Delete PREROUTING rule + args := []string{ + "-m", "conntrack", + "--ctstate", "ESTABLISHED,RELATED", + "-j", "CONNMARK", + "--restore-mark", + "--nfmask", fwmarkMask, + "--ctmask", fwmarkMask, + } + if err := ipt.Delete("mangle", "PREROUTING", args...); err != nil { + if !isNotExistError(err) { + return fmt.Errorf("deleting connmark rule in mangle/PREROUTING: %w", err) + } + // Rule doesn't exist - this is fine for idempotency + } + + // Delete OUTPUT rule + args = []string{ + "-m", "conntrack", + "--ctstate", "NEW", + "-m", "mark", + "!", "--mark", "0x0/" + fwmarkMask, + "-j", "CONNMARK", + "--save-mark", + "--nfmask", fwmarkMask, + "--ctmask", fwmarkMask, + } + if err := ipt.Delete("mangle", "OUTPUT", args...); err != nil { + if !isNotExistError(err) { + return fmt.Errorf("deleting connmark rule in mangle/OUTPUT: %w", err) + } + // Rule doesn't exist - this is fine for idempotency + } + } + return nil +} + // buildMagicsockPortRule generates the string slice containing the arguments // to describe a rule accepting traffic on a particular port to iptables. It is // separated out here to avoid repetition in AddMagicsockPortRule and diff --git a/util/linuxfw/nftables_runner.go b/util/linuxfw/nftables_runner.go index 2c44a6218..7496e7034 100644 --- a/util/linuxfw/nftables_runner.go +++ b/util/linuxfw/nftables_runner.go @@ -521,6 +521,15 @@ type NetfilterRunner interface { // using conntrack. DelStatefulRule(tunname string) error + // AddConnmarkSaveRule adds conntrack marking rules to save marks from packets. + // These rules run in mangle/PREROUTING and mangle/OUTPUT to mark connections + // and restore marks on reply packets before rp_filter checks, enabling proper + // routing table lookups for exit nodes and subnet routers. + AddConnmarkSaveRule() error + + // DelConnmarkSaveRule removes conntrack marking rules added by AddConnmarkSaveRule. + DelConnmarkSaveRule() error + // HasIPV6 reports true if the system supports IPv6. HasIPV6() bool @@ -1950,6 +1959,242 @@ func (n *nftablesRunner) DelStatefulRule(tunname string) error { return nil } +// makeConnmarkRestoreExprs creates nftables expressions to restore mark from conntrack. +// Implements: ct state established,related ct mark & 0xff0000 != 0 meta mark set ct mark & 0xff0000 +func makeConnmarkRestoreExprs() []expr.Any { + return []expr.Any{ + // Load conntrack state into register 1 + &expr.Ct{ + Register: 1, + Key: expr.CtKeySTATE, + }, + // Check if state is ESTABLISHED or RELATED + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Mask: nativeUint32( + expr.CtStateBitESTABLISHED | + expr.CtStateBitRELATED), + Xor: nativeUint32(0), + }, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{0, 0, 0, 0}, + }, + // Load conntrack mark into register 1 + &expr.Ct{ + Register: 1, + Key: expr.CtKeyMARK, + }, + // Mask to Tailscale mark bits (0xff0000) + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Mask: getTailscaleFwmarkMask(), + Xor: []byte{0x00, 0x00, 0x00, 0x00}, + }, + // Set packet mark from register 1 + &expr.Meta{ + Key: expr.MetaKeyMARK, + SourceRegister: true, + Register: 1, + }, + } +} + +// makeConnmarkSaveExprs creates nftables expressions to save mark to conntrack. +// Implements: ct state new meta mark & 0xff0000 != 0 ct mark set meta mark & 0xff0000 +func makeConnmarkSaveExprs() []expr.Any { + return []expr.Any{ + // Load conntrack state into register 1 + &expr.Ct{ + Register: 1, + Key: expr.CtKeySTATE, + }, + // Check if state is NEW + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Mask: nativeUint32(expr.CtStateBitNEW), + Xor: nativeUint32(0), + }, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{0, 0, 0, 0}, + }, + // Load packet mark into register 1 + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + }, + // Mask to Tailscale mark bits (0xff0000) + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Mask: getTailscaleFwmarkMask(), + Xor: []byte{0x00, 0x00, 0x00, 0x00}, + }, + // Check if mark is non-zero + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{0, 0, 0, 0}, + }, + // Load packet mark again for saving + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + }, + // Mask again + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Mask: getTailscaleFwmarkMask(), + Xor: []byte{0x00, 0x00, 0x00, 0x00}, + }, + // Set conntrack mark from register 1 + &expr.Ct{ + Key: expr.CtKeyMARK, + SourceRegister: true, + Register: 1, + }, + } +} + +// AddConnmarkSaveRule adds conntrack marking rules to save and restore marks. +// These rules run in mangle/PREROUTING (to restore marks from conntrack) and +// mangle/OUTPUT (to save marks to conntrack) before rp_filter checks, enabling +// proper routing table lookups for exit nodes and subnet routers. +func (n *nftablesRunner) AddConnmarkSaveRule() error { + conn := n.conn + + // Check if rules already exist (idempotency) + for _, table := range n.getTables() { + mangleTable := &nftables.Table{ + Family: table.Proto, + Name: "mangle", + } + + // Check PREROUTING chain for restore rule + preroutingChain, err := getChainFromTable(conn, mangleTable, "PREROUTING") + if err == nil { + rules, _ := conn.GetRules(preroutingChain.Table, preroutingChain) + for _, rule := range rules { + if string(rule.UserData) == "ts-connmark-restore" { + // Rules already exist, skip adding + return nil + } + } + } + } + + // Add rules for both IPv4 and IPv6 + for _, table := range n.getTables() { + // Get or create mangle table + mangleTable := &nftables.Table{ + Family: table.Proto, + Name: "mangle", + } + conn.AddTable(mangleTable) + + // Get or create PREROUTING chain + preroutingChain, err := getChainFromTable(conn, mangleTable, "PREROUTING") + if err != nil { + // Chain doesn't exist, create it + preroutingChain = conn.AddChain(&nftables.Chain{ + Name: "PREROUTING", + Table: mangleTable, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityMangle, + }) + } + + // Add PREROUTING rule to restore mark from conntrack + conn.InsertRule(&nftables.Rule{ + Table: mangleTable, + Chain: preroutingChain, + Exprs: makeConnmarkRestoreExprs(), + UserData: []byte("ts-connmark-restore"), + }) + + // Get or create OUTPUT chain + outputChain, err := getChainFromTable(conn, mangleTable, "OUTPUT") + if err != nil { + // Chain doesn't exist, create it + outputChain = conn.AddChain(&nftables.Chain{ + Name: "OUTPUT", + Table: mangleTable, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookOutput, + Priority: nftables.ChainPriorityMangle, + }) + } + + // Add OUTPUT rule to save mark to conntrack + conn.InsertRule(&nftables.Rule{ + Table: mangleTable, + Chain: outputChain, + Exprs: makeConnmarkSaveExprs(), + UserData: []byte("ts-connmark-save"), + }) + } + + if err := conn.Flush(); err != nil { + return fmt.Errorf("flush add connmark rules: %w", err) + } + + return nil +} + +// DelConnmarkSaveRule removes conntrack marking rules added by AddConnmarkSaveRule. +func (n *nftablesRunner) DelConnmarkSaveRule() error { + conn := n.conn + + for _, table := range n.getTables() { + mangleTable := &nftables.Table{ + Family: table.Proto, + Name: "mangle", + } + + // Remove PREROUTING rule - look for restore-mark rule by UserData + preroutingChain, err := getChainFromTable(conn, mangleTable, "PREROUTING") + if err == nil { + rules, _ := conn.GetRules(preroutingChain.Table, preroutingChain) + for _, rule := range rules { + if string(rule.UserData) == "ts-connmark-restore" { + conn.DelRule(rule) + break + } + } + } + + // Remove OUTPUT rule - look for save-mark rule by UserData + outputChain, err := getChainFromTable(conn, mangleTable, "OUTPUT") + if err == nil { + rules, _ := conn.GetRules(outputChain.Table, outputChain) + for _, rule := range rules { + if string(rule.UserData) == "ts-connmark-save" { + conn.DelRule(rule) + break + } + } + } + } + + // Ignore errors during deletion - rules might not exist + conn.Flush() + + return nil +} + // cleanupChain removes a jump rule from hookChainName to tsChainName, and then // the entire chain tsChainName. Errors are logged, but attempts to remove both // the jump rule and chain continue even if one errors. diff --git a/util/linuxfw/nftables_runner_test.go b/util/linuxfw/nftables_runner_test.go index dc4d3194a..8299a9cbd 100644 --- a/util/linuxfw/nftables_runner_test.go +++ b/util/linuxfw/nftables_runner_test.go @@ -1070,3 +1070,246 @@ func checkSNATRule_nft(t *testing.T, runner *nftablesRunner, fam nftables.TableF wantsRule := snatRule(chain.Table, chain, src, dst, meta) checkRule(t, wantsRule, runner.conn) } + +// TestNFTAddAndDelConnmarkRules tests adding and removing connmark rules +// in a real network namespace. This verifies the rules are correctly created +// and cleaned up. +func TestNFTAddAndDelConnmarkRules(t *testing.T) { + conn := newSysConn(t) + runner := newFakeNftablesRunnerWithConn(t, conn, true) + + // Helper to get mangle chains + getMangleChains := func(fam nftables.TableFamily) (prerouting, output *nftables.Chain, err error) { + chains, err := conn.ListChainsOfTableFamily(fam) + if err != nil { + return nil, nil, err + } + for _, ch := range chains { + if ch.Table.Name != "mangle" { + continue + } + if ch.Name == "PREROUTING" { + prerouting = ch + } else if ch.Name == "OUTPUT" { + output = ch + } + } + return prerouting, output, nil + } + + // Check initial state - mangle chains might not exist yet + prerouting4Before, output4Before, _ := getMangleChains(nftables.TableFamilyIPv4) + prerouting6Before, output6Before, _ := getMangleChains(nftables.TableFamilyIPv6) + + var prerouting4RulesBefore, output4RulesBefore, prerouting6RulesBefore, output6RulesBefore int + if prerouting4Before != nil { + rules, _ := conn.GetRules(prerouting4Before.Table, prerouting4Before) + prerouting4RulesBefore = len(rules) + } + if output4Before != nil { + rules, _ := conn.GetRules(output4Before.Table, output4Before) + output4RulesBefore = len(rules) + } + if prerouting6Before != nil { + rules, _ := conn.GetRules(prerouting6Before.Table, prerouting6Before) + prerouting6RulesBefore = len(rules) + } + if output6Before != nil { + rules, _ := conn.GetRules(output6Before.Table, output6Before) + output6RulesBefore = len(rules) + } + + // Add connmark rules + if err := runner.AddConnmarkSaveRule(); err != nil { + t.Fatalf("AddConnmarkSaveRule() failed: %v", err) + } + + // Verify rules were added + prerouting4After, output4After, err := getMangleChains(nftables.TableFamilyIPv4) + if err != nil { + t.Fatalf("Failed to get IPv4 mangle chains: %v", err) + } + if prerouting4After == nil || output4After == nil { + t.Fatal("IPv4 mangle chains not created") + } + + prerouting4Rules, err := conn.GetRules(prerouting4After.Table, prerouting4After) + if err != nil { + t.Fatalf("GetRules(PREROUTING) failed: %v", err) + } + output4Rules, err := conn.GetRules(output4After.Table, output4After) + if err != nil { + t.Fatalf("GetRules(OUTPUT) failed: %v", err) + } + + // Should have added 1 rule to each chain + if len(prerouting4Rules) != prerouting4RulesBefore+1 { + t.Fatalf("PREROUTING rules: got %d, want %d", len(prerouting4Rules), prerouting4RulesBefore+1) + } + if len(output4Rules) != output4RulesBefore+1 { + t.Fatalf("OUTPUT rules: got %d, want %d", len(output4Rules), output4RulesBefore+1) + } + + // Verify IPv6 rules + prerouting6After, output6After, err := getMangleChains(nftables.TableFamilyIPv6) + if err != nil { + t.Fatalf("Failed to get IPv6 mangle chains: %v", err) + } + if prerouting6After == nil || output6After == nil { + t.Fatal("IPv6 mangle chains not created") + } + + prerouting6Rules, err := conn.GetRules(prerouting6After.Table, prerouting6After) + if err != nil { + t.Fatalf("GetRules(IPv6 PREROUTING) failed: %v", err) + } + output6Rules, err := conn.GetRules(output6After.Table, output6After) + if err != nil { + t.Fatalf("GetRules(IPv6 OUTPUT) failed: %v", err) + } + + if len(prerouting6Rules) != prerouting6RulesBefore+1 { + t.Fatalf("IPv6 PREROUTING rules: got %d, want %d", len(prerouting6Rules), prerouting6RulesBefore+1) + } + if len(output6Rules) != output6RulesBefore+1 { + t.Fatalf("IPv6 OUTPUT rules: got %d, want %d", len(output6Rules), output6RulesBefore+1) + } + + // Verify the rules contain conntrack expressions + foundCtInPrerouting := false + foundCtInOutput := false + for _, e := range prerouting4Rules[0].Exprs { + if _, ok := e.(*expr.Ct); ok { + foundCtInPrerouting = true + break + } + } + for _, e := range output4Rules[0].Exprs { + if _, ok := e.(*expr.Ct); ok { + foundCtInOutput = true + break + } + } + if !foundCtInPrerouting { + t.Error("PREROUTING rule doesn't contain conntrack expression") + } + if !foundCtInOutput { + t.Error("OUTPUT rule doesn't contain conntrack expression") + } + + // Delete connmark rules + if err := runner.DelConnmarkSaveRule(); err != nil { + t.Fatalf("DelConnmarkSaveRule() failed: %v", err) + } + + // Verify rules were deleted + prerouting4After, output4After, _ = getMangleChains(nftables.TableFamilyIPv4) + if prerouting4After != nil { + rules, _ := conn.GetRules(prerouting4After.Table, prerouting4After) + if len(rules) != prerouting4RulesBefore { + t.Fatalf("IPv4 PREROUTING rules after delete: got %d, want %d", len(rules), prerouting4RulesBefore) + } + } + if output4After != nil { + rules, _ := conn.GetRules(output4After.Table, output4After) + if len(rules) != output4RulesBefore { + t.Fatalf("IPv4 OUTPUT rules after delete: got %d, want %d", len(rules), output4RulesBefore) + } + } + + prerouting6After, output6After, _ = getMangleChains(nftables.TableFamilyIPv6) + if prerouting6After != nil { + rules, _ := conn.GetRules(prerouting6After.Table, prerouting6After) + if len(rules) != prerouting6RulesBefore { + t.Fatalf("IPv6 PREROUTING rules after delete: got %d, want %d", len(rules), prerouting6RulesBefore) + } + } + if output6After != nil { + rules, _ := conn.GetRules(output6After.Table, output6After) + if len(rules) != output6RulesBefore { + t.Fatalf("IPv6 OUTPUT rules after delete: got %d, want %d", len(rules), output6RulesBefore) + } + } +} + +// TestMakeConnmarkRestoreExprs tests the nftables expressions for restoring +// marks from conntrack. This is a regression test that ensures the byte encoding +// doesn't change unexpectedly. +func TestMakeConnmarkRestoreExprs(t *testing.T) { + // Expected netlink bytes for the restore rule + // Generated by running makeConnmarkRestoreExprs() and capturing the output + want := [][]byte{ + // batch begin + []byte("\x00\x00\x00\x0a"), + // nft add table ip mangle + []byte("\x02\x00\x00\x00\x0b\x00\x01\x00\x6d\x61\x6e\x67\x6c\x65\x00\x00\x08\x00\x02\x00\x00\x00\x00\x00"), + // nft add chain ip mangle PREROUTING { type filter hook prerouting priority mangle; } + []byte("\x02\x00\x00\x00\x0b\x00\x01\x00\x6d\x61\x6e\x67\x6c\x65\x00\x00\x0f\x00\x03\x00\x50\x52\x45\x52\x4f\x55\x54\x49\x4e\x47\x00\x00\x14\x00\x04\x80\x08\x00\x01\x00\x00\x00\x00\x00\x08\x00\x02\x00\xff\xff\xff\x6a\x0b\x00\x07\x00\x66\x69\x6c\x74\x65\x72\x00\x00"), + // nft add rule ip mangle PREROUTING ct state established,related ct mark & 0xff0000 != 0 meta mark set ct mark & 0xff0000 + []byte("\x02\x00\x00\x00\x0b\x00\x01\x00\x6d\x61\x6e\x67\x6c\x65\x00\x00\x0f\x00\x02\x00\x50\x52\x45\x52\x4f\x55\x54\x49\x4e\x47\x00\x00\x1c\x01\x04\x80\x20\x00\x01\x80\x07\x00\x01\x00\x63\x74\x00\x00\x14\x00\x02\x80\x08\x00\x02\x00\x00\x00\x00\x00\x08\x00\x01\x00\x00\x00\x00\x01\x44\x00\x01\x80\x0c\x00\x01\x00\x62\x69\x74\x77\x69\x73\x65\x00\x34\x00\x02\x80\x08\x00\x01\x00\x00\x00\x00\x01\x08\x00\x02\x00\x00\x00\x00\x01\x08\x00\x03\x00\x00\x00\x00\x04\x0c\x00\x04\x80\x08\x00\x01\x00\x06\x00\x00\x00\x0c\x00\x05\x80\x08\x00\x01\x00\x00\x00\x00\x00\x2c\x00\x01\x80\x08\x00\x01\x00\x63\x6d\x70\x00\x20\x00\x02\x80\x08\x00\x01\x00\x00\x00\x00\x01\x08\x00\x02\x00\x00\x00\x00\x01\x0c\x00\x03\x80\x08\x00\x01\x00\x00\x00\x00\x00\x20\x00\x01\x80\x07\x00\x01\x00\x63\x74\x00\x00\x14\x00\x02\x80\x08\x00\x02\x00\x00\x00\x00\x03\x08\x00\x01\x00\x00\x00\x00\x01\x44\x00\x01\x80\x0c\x00\x01\x00\x62\x69\x74\x77\x69\x73\x65\x00\x34\x00\x02\x80\x08\x00\x01\x00\x00\x00\x00\x01\x08\x00\x02\x00\x00\x00\x00\x01\x08\x00\x03\x00\x00\x00\x00\x04\x0c\x00\x04\x80\x08\x00\x01\x00\x00\xff\x00\x00\x0c\x00\x05\x80\x08\x00\x01\x00\x00\x00\x00\x00\x24\x00\x01\x80\x09\x00\x01\x00\x6d\x65\x74\x61\x00\x00\x00\x00\x14\x00\x02\x80\x08\x00\x02\x00\x00\x00\x00\x03\x08\x00\x03\x00\x00\x00\x00\x01"), + // batch end + []byte("\x00\x00\x00\x0a"), + } + + testConn := newTestConn(t, want, nil) + table := testConn.AddTable(&nftables.Table{ + Family: nftables.TableFamilyIPv4, + Name: "mangle", + }) + chain := testConn.AddChain(&nftables.Chain{ + Name: "PREROUTING", + Table: table, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityMangle, + }) + testConn.InsertRule(&nftables.Rule{ + Table: table, + Chain: chain, + Exprs: makeConnmarkRestoreExprs(), + }) + if err := testConn.Flush(); err != nil { + t.Fatalf("Flush() failed: %v", err) + } +} + +// TestMakeConnmarkSaveExprs tests the nftables expressions for saving marks +// to conntrack. This is a regression test that ensures the byte encoding +// doesn't change unexpectedly. +func TestMakeConnmarkSaveExprs(t *testing.T) { + // Expected netlink bytes for the save rule + // Generated by running makeConnmarkSaveExprs() and capturing the output + want := [][]byte{ + // batch begin + []byte("\x00\x00\x00\x0a"), + // nft add table ip mangle + []byte("\x02\x00\x00\x00\x0b\x00\x01\x00\x6d\x61\x6e\x67\x6c\x65\x00\x00\x08\x00\x02\x00\x00\x00\x00\x00"), + // nft add chain ip mangle OUTPUT { type route hook output priority mangle; } + []byte("\x02\x00\x00\x00\x0b\x00\x01\x00\x6d\x61\x6e\x67\x6c\x65\x00\x00\x0b\x00\x03\x00\x4f\x55\x54\x50\x55\x54\x00\x00\x14\x00\x04\x80\x08\x00\x01\x00\x00\x00\x00\x03\x08\x00\x02\x00\xff\xff\xff\x6a\x0a\x00\x07\x00\x72\x6f\x75\x74\x65\x00\x00\x00"), + // nft add rule ip mangle OUTPUT ct state new meta mark & 0xff0000 != 0 ct mark set meta mark & 0xff0000 + []byte("\x02\x00\x00\x00\x0b\x00\x01\x00\x6d\x61\x6e\x67\x6c\x65\x00\x00\x0b\x00\x02\x00\x4f\x55\x54\x50\x55\x54\x00\x00\xb0\x01\x04\x80\x20\x00\x01\x80\x07\x00\x01\x00\x63\x74\x00\x00\x14\x00\x02\x80\x08\x00\x02\x00\x00\x00\x00\x00\x08\x00\x01\x00\x00\x00\x00\x01\x44\x00\x01\x80\x0c\x00\x01\x00\x62\x69\x74\x77\x69\x73\x65\x00\x34\x00\x02\x80\x08\x00\x01\x00\x00\x00\x00\x01\x08\x00\x02\x00\x00\x00\x00\x01\x08\x00\x03\x00\x00\x00\x00\x04\x0c\x00\x04\x80\x08\x00\x01\x00\x08\x00\x00\x00\x0c\x00\x05\x80\x08\x00\x01\x00\x00\x00\x00\x00\x2c\x00\x01\x80\x08\x00\x01\x00\x63\x6d\x70\x00\x20\x00\x02\x80\x08\x00\x01\x00\x00\x00\x00\x01\x08\x00\x02\x00\x00\x00\x00\x01\x0c\x00\x03\x80\x08\x00\x01\x00\x00\x00\x00\x00\x24\x00\x01\x80\x09\x00\x01\x00\x6d\x65\x74\x61\x00\x00\x00\x00\x14\x00\x02\x80\x08\x00\x02\x00\x00\x00\x00\x03\x08\x00\x01\x00\x00\x00\x00\x01\x44\x00\x01\x80\x0c\x00\x01\x00\x62\x69\x74\x77\x69\x73\x65\x00\x34\x00\x02\x80\x08\x00\x01\x00\x00\x00\x00\x01\x08\x00\x02\x00\x00\x00\x00\x01\x08\x00\x03\x00\x00\x00\x00\x04\x0c\x00\x04\x80\x08\x00\x01\x00\x00\xff\x00\x00\x0c\x00\x05\x80\x08\x00\x01\x00\x00\x00\x00\x00\x2c\x00\x01\x80\x08\x00\x01\x00\x63\x6d\x70\x00\x20\x00\x02\x80\x08\x00\x01\x00\x00\x00\x00\x01\x08\x00\x02\x00\x00\x00\x00\x01\x0c\x00\x03\x80\x08\x00\x01\x00\x00\x00\x00\x00\x24\x00\x01\x80\x09\x00\x01\x00\x6d\x65\x74\x61\x00\x00\x00\x00\x14\x00\x02\x80\x08\x00\x02\x00\x00\x00\x00\x03\x08\x00\x01\x00\x00\x00\x00\x01\x44\x00\x01\x80\x0c\x00\x01\x00\x62\x69\x74\x77\x69\x73\x65\x00\x34\x00\x02\x80\x08\x00\x01\x00\x00\x00\x00\x01\x08\x00\x02\x00\x00\x00\x00\x01\x08\x00\x03\x00\x00\x00\x00\x04\x0c\x00\x04\x80\x08\x00\x01\x00\x00\xff\x00\x00\x0c\x00\x05\x80\x08\x00\x01\x00\x00\x00\x00\x00\x20\x00\x01\x80\x07\x00\x01\x00\x63\x74\x00\x00\x14\x00\x02\x80\x08\x00\x02\x00\x00\x00\x00\x03\x08\x00\x04\x00\x00\x00\x00\x01"), + // batch end + []byte("\x00\x00\x00\x0a"), + } + + testConn := newTestConn(t, want, nil) + table := testConn.AddTable(&nftables.Table{ + Family: nftables.TableFamilyIPv4, + Name: "mangle", + }) + chain := testConn.AddChain(&nftables.Chain{ + Name: "OUTPUT", + Table: table, + Type: nftables.ChainTypeRoute, + Hooknum: nftables.ChainHookOutput, + Priority: nftables.ChainPriorityMangle, + }) + testConn.InsertRule(&nftables.Rule{ + Table: table, + Chain: chain, + Exprs: makeConnmarkSaveExprs(), + }) + if err := testConn.Flush(); err != nil { + t.Fatalf("Flush() failed: %v", err) + } +} diff --git a/wgengine/router/osrouter/router_linux.go b/wgengine/router/osrouter/router_linux.go index 8ca38f9ec..3c261c912 100644 --- a/wgengine/router/osrouter/router_linux.go +++ b/wgengine/router/osrouter/router_linux.go @@ -86,6 +86,7 @@ type linuxRouter struct { localRoutes map[netip.Prefix]bool snatSubnetRoutes bool statefulFiltering bool + connmarkEnabled bool // whether connmark rules are currently enabled netfilterMode preftype.NetfilterMode netfilterKind string magicsockPortV4 uint16 @@ -370,6 +371,12 @@ func (r *linuxRouter) Close() error { r.unregNetMon() } r.eventClient.Close() + + // Clean up connmark rules + if err := r.nfr.DelConnmarkSaveRule(); err != nil { + r.logf("warning: failed to delete connmark rules: %v", err) + } + if err := r.downInterface(); err != nil { return err } @@ -479,6 +486,35 @@ func (r *linuxRouter) Set(cfg *router.Config) error { r.statefulFiltering = cfg.StatefulFiltering r.updateStatefulFilteringWithDockerWarning(cfg) + // Connmark rules for rp_filter compatibility. + // Always enabled when netfilter is ON to handle all rp_filter=1 scenarios + // (normal operation, exit nodes, subnet routers, and clients using exit nodes). + netfilterOn := cfg.NetfilterMode == netfilterOn + switch { + case netfilterOn == r.connmarkEnabled: + // state already correct, nothing to do. + case netfilterOn: + r.logf("enabling connmark-based rp_filter workaround") + if err := r.nfr.AddConnmarkSaveRule(); err != nil { + r.logf("warning: failed to add connmark rules (rp_filter workaround may not work): %v", err) + errs = append(errs, fmt.Errorf("enabling connmark rules: %w", err)) + } else { + // Only update state on success to keep it in sync with actual rules + r.connmarkEnabled = true + } + default: + r.logf("disabling connmark-based rp_filter workaround") + if err := r.nfr.DelConnmarkSaveRule(); err != nil { + // Deletion errors are only logged, not returned, because: + // 1. Rules may not exist (e.g., first run or after manual deletion) + // 2. Failure to delete is less critical than failure to add + // 3. We still want to update state to attempt re-add on next enable + r.logf("warning: failed to delete connmark rules: %v", err) + } + // Always clear state when disabling, even if delete failed + r.connmarkEnabled = false + } + // Issue 11405: enable IP forwarding on gokrazy. advertisingRoutes := len(cfg.SubnetRoutes) > 0 if getDistroFunc() == distro.Gokrazy && advertisingRoutes { diff --git a/wgengine/router/osrouter/router_linux_test.go b/wgengine/router/osrouter/router_linux_test.go index bce0ea092..bae997e33 100644 --- a/wgengine/router/osrouter/router_linux_test.go +++ b/wgengine/router/osrouter/router_linux_test.go @@ -124,6 +124,8 @@ func TestRouterStates(t *testing.T) { v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v4/nat/POSTROUTING -j ts-postrouting v4/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE v6/filter/FORWARD -j ts-forward @@ -132,6 +134,8 @@ func TestRouterStates(t *testing.T) { v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT v6/filter/ts-forward -o tailscale0 -m conntrack ! --ctstate ESTABLISHED,RELATED -j DROP v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v6/nat/POSTROUTING -j ts-postrouting v6/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE `, @@ -160,6 +164,8 @@ func TestRouterStates(t *testing.T) { v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v4/nat/POSTROUTING -j ts-postrouting v4/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE v6/filter/FORWARD -j ts-forward @@ -167,6 +173,8 @@ func TestRouterStates(t *testing.T) { v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v6/nat/POSTROUTING -j ts-postrouting v6/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE `, @@ -192,12 +200,16 @@ func TestRouterStates(t *testing.T) { v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v4/nat/POSTROUTING -j ts-postrouting v6/filter/FORWARD -j ts-forward v6/filter/INPUT -j ts-input v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v6/nat/POSTROUTING -j ts-postrouting `, }, @@ -225,12 +237,16 @@ func TestRouterStates(t *testing.T) { v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v4/nat/POSTROUTING -j ts-postrouting v6/filter/FORWARD -j ts-forward v6/filter/INPUT -j ts-input v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v6/nat/POSTROUTING -j ts-postrouting `, }, @@ -255,12 +271,16 @@ func TestRouterStates(t *testing.T) { v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v4/nat/POSTROUTING -j ts-postrouting v6/filter/FORWARD -j ts-forward v6/filter/INPUT -j ts-input v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v6/nat/POSTROUTING -j ts-postrouting `, }, @@ -310,12 +330,16 @@ func TestRouterStates(t *testing.T) { v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v4/nat/POSTROUTING -j ts-postrouting v6/filter/FORWARD -j ts-forward v6/filter/INPUT -j ts-input v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v6/nat/POSTROUTING -j ts-postrouting `, }, @@ -342,12 +366,16 @@ func TestRouterStates(t *testing.T) { v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v4/nat/POSTROUTING -j ts-postrouting v6/filter/FORWARD -j ts-forward v6/filter/INPUT -j ts-input v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 v6/nat/POSTROUTING -j ts-postrouting `, }, @@ -367,6 +395,120 @@ func TestRouterStates(t *testing.T) { ip route add throw 10.0.0.0/8 table 52 ip route add throw 192.168.0.0/24 table 52` + basic, }, + { + name: "subnet routes with connmark for rp_filter", + in: &Config{ + LocalAddrs: mustCIDRs("100.101.102.104/10"), + Routes: mustCIDRs("100.100.100.100/32"), + SubnetRoutes: mustCIDRs("10.0.0.0/16"), + SNATSubnetRoutes: true, + NetfilterMode: netfilterOn, + }, + want: ` +up +ip addr add 100.101.102.104/10 dev tailscale0 +ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic + + `v4/filter/FORWARD -j ts-forward +v4/filter/INPUT -j ts-input +v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 +v4/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT +v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP +v4/filter/ts-forward -o tailscale0 -j ACCEPT +v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT +v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN +v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/nat/POSTROUTING -j ts-postrouting +v4/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE +v6/filter/FORWARD -j ts-forward +v6/filter/INPUT -j ts-input +v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 +v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT +v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/nat/POSTROUTING -j ts-postrouting +v6/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE +`, + }, + { + name: "subnet routes (connmark always enabled)", + in: &Config{ + LocalAddrs: mustCIDRs("100.101.102.104/10"), + Routes: mustCIDRs("100.100.100.100/32"), + SubnetRoutes: mustCIDRs("10.0.0.0/16"), + SNATSubnetRoutes: true, + NetfilterMode: netfilterOn, + }, + want: ` +up +ip addr add 100.101.102.104/10 dev tailscale0 +ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic + + `v4/filter/FORWARD -j ts-forward +v4/filter/INPUT -j ts-input +v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 +v4/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT +v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP +v4/filter/ts-forward -o tailscale0 -j ACCEPT +v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT +v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN +v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/nat/POSTROUTING -j ts-postrouting +v4/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE +v6/filter/FORWARD -j ts-forward +v6/filter/INPUT -j ts-input +v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 +v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT +v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/nat/POSTROUTING -j ts-postrouting +v6/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE +`, + }, + { + name: "connmark with stateful filtering", + in: &Config{ + LocalAddrs: mustCIDRs("100.101.102.104/10"), + Routes: mustCIDRs("100.100.100.100/32"), + SubnetRoutes: mustCIDRs("10.0.0.0/16"), + SNATSubnetRoutes: true, + StatefulFiltering: true, + NetfilterMode: netfilterOn, + }, + want: ` +up +ip addr add 100.101.102.104/10 dev tailscale0 +ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic + + `v4/filter/FORWARD -j ts-forward +v4/filter/INPUT -j ts-input +v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 +v4/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT +v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP +v4/filter/ts-forward -o tailscale0 -m conntrack ! --ctstate ESTABLISHED,RELATED -j DROP +v4/filter/ts-forward -o tailscale0 -j ACCEPT +v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT +v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN +v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 +v4/nat/POSTROUTING -j ts-postrouting +v4/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE +v6/filter/FORWARD -j ts-forward +v6/filter/INPUT -j ts-input +v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000/0xff0000 +v6/filter/ts-forward -m mark --mark 0x40000/0xff0000 -j ACCEPT +v6/filter/ts-forward -o tailscale0 -m conntrack ! --ctstate ESTABLISHED,RELATED -j DROP +v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/mangle/OUTPUT -m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/mangle/PREROUTING -m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000 +v6/nat/POSTROUTING -j ts-postrouting +v6/nat/ts-postrouting -m mark --mark 0x40000/0xff0000 -j MASQUERADE +`, + }, } bus := eventbus.New() @@ -426,20 +568,24 @@ func newIPTablesRunner(t *testing.T) linuxfw.NetfilterRunner { return &fakeIPTablesRunner{ t: t, ipt4: map[string][]string{ - "filter/INPUT": nil, - "filter/OUTPUT": nil, - "filter/FORWARD": nil, - "nat/PREROUTING": nil, - "nat/OUTPUT": nil, - "nat/POSTROUTING": nil, + "filter/INPUT": nil, + "filter/OUTPUT": nil, + "filter/FORWARD": nil, + "nat/PREROUTING": nil, + "nat/OUTPUT": nil, + "nat/POSTROUTING": nil, + "mangle/PREROUTING": nil, + "mangle/OUTPUT": nil, }, ipt6: map[string][]string{ - "filter/INPUT": nil, - "filter/OUTPUT": nil, - "filter/FORWARD": nil, - "nat/PREROUTING": nil, - "nat/OUTPUT": nil, - "nat/POSTROUTING": nil, + "filter/INPUT": nil, + "filter/OUTPUT": nil, + "filter/FORWARD": nil, + "nat/PREROUTING": nil, + "nat/OUTPUT": nil, + "nat/POSTROUTING": nil, + "mangle/PREROUTING": nil, + "mangle/OUTPUT": nil, }, } } @@ -775,6 +921,38 @@ func (n *fakeIPTablesRunner) DelMagicsockPortRule(port uint16, network string) e return nil } +func (n *fakeIPTablesRunner) AddConnmarkSaveRule() error { + // PREROUTING rule: restore mark from conntrack + prerouteRule := "-m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000" + for _, ipt := range []map[string][]string{n.ipt4, n.ipt6} { + if err := insertRule(n, ipt, "mangle/PREROUTING", prerouteRule); err != nil { + return err + } + } + + // OUTPUT rule: save mark to conntrack for NEW connections + outputRule := "-m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000" + for _, ipt := range []map[string][]string{n.ipt4, n.ipt6} { + if err := insertRule(n, ipt, "mangle/OUTPUT", outputRule); err != nil { + return err + } + } + return nil +} + +func (n *fakeIPTablesRunner) DelConnmarkSaveRule() error { + prerouteRule := "-m conntrack --ctstate ESTABLISHED,RELATED -j CONNMARK --restore-mark --nfmask 0xff0000 --ctmask 0xff0000" + for _, ipt := range []map[string][]string{n.ipt4, n.ipt6} { + deleteRule(n, ipt, "mangle/PREROUTING", prerouteRule) // ignore errors + } + + outputRule := "-m conntrack --ctstate NEW -m mark ! --mark 0x0/0xff0000 -j CONNMARK --save-mark --nfmask 0xff0000 --ctmask 0xff0000" + for _, ipt := range []map[string][]string{n.ipt4, n.ipt6} { + deleteRule(n, ipt, "mangle/OUTPUT", outputRule) // ignore errors + } + return nil +} + func (n *fakeIPTablesRunner) HasIPV6() bool { return true } func (n *fakeIPTablesRunner) HasIPV6NAT() bool { return true } func (n *fakeIPTablesRunner) HasIPV6Filter() bool { return true }