[test] add tests for multi-BR network resilience during BR removal (#9990)

This commit adds new test cases for a topology with two BRs when the
primary BR is abruptly powered off. It validates that the second BR
takes over and all nodes successfully re-register their SRP services
with the new BR using new OMR addresses.

The first test (`501`) covers the case where neither BR is acting as
mesh leader. The second test (`502`) covers the case where the
powered-off BR is also acting a mesh leader.

4 files changed, 428 insertions, 0 deletions

diff --git a/tests/toranj/cli/cli.py b/tests/toranj/cli/cli.py
index 89be4701e..8f278f437 100644
--- a/tests/toranj/cli/cli.py
+++ b/tests/toranj/cli/cli.py
@@ -657,6 +657,12 @@ class Node(object):
     def srp_server_disable(self):
         self._cli_no_output('srp server disable')
 
+    def srp_server_auto_enable(self):
+        self._cli_no_output('srp server auto enable')
+
+    def srp_server_auto_disable(self):
+        self._cli_no_output('srp server auto disable')
+
     def srp_server_set_lease(self, min_lease, max_lease, min_key_lease, max_key_lease):
         self._cli_no_output('srp server lease', min_lease, max_lease, min_key_lease, max_key_lease)
 
diff --git a/tests/toranj/cli/test-501-multi-br-failure-recovery.py b/tests/toranj/cli/test-501-multi-br-failure-recovery.py
new file mode 100755
index 000000000..ecf41b999
--- /dev/null
+++ b/tests/toranj/cli/test-501-multi-br-failure-recovery.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+#
+#  Copyright (c) 2024, The OpenThread Authors.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#  3. Neither the name of the copyright holder nor the
+#     names of its contributors may be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+#  POSSIBILITY OF SUCH DAMAGE.
+
+from cli import verify
+from cli import verify_within
+import cli
+import time
+
+# -----------------------------------------------------------------------------------------------------------------------
+# Test description:
+#
+# Network with two BRs, none of them acting as leader. Removing BR1 ensuring BR2 taking over.
+#
+#      ________________
+#     /                \
+#   br1 --- leader --- br2
+#   /         / \        \
+#  c1       c2  c3       c4
+#
+
+test_name = __file__[:-3] if __file__.endswith('.py') else __file__
+print('-' * 120)
+print('Starting \'{}\''.format(test_name))
+
+# -----------------------------------------------------------------------------------------------------------------------
+# Creating `cli.Nodes` instances
+
+speedup = 60
+cli.Node.set_time_speedup_factor(speedup)
+
+leader = cli.Node()
+br1 = cli.Node()
+br2 = cli.Node()
+c1 = cli.Node()
+c2 = cli.Node()
+c3 = cli.Node()
+c4 = cli.Node()
+
+IF_INDEX = 1
+
+# -----------------------------------------------------------------------------------------------------------------------
+# Form topology
+
+leader.allowlist_node(br1)
+leader.allowlist_node(br2)
+leader.allowlist_node(c2)
+leader.allowlist_node(c3)
+
+br1.allowlist_node(leader)
+br1.allowlist_node(br2)
+br1.allowlist_node(c1)
+
+br2.allowlist_node(leader)
+br2.allowlist_node(br1)
+br2.allowlist_node(c4)
+
+c1.allowlist_node(br1)
+
+c2.allowlist_node(leader)
+c3.allowlist_node(leader)
+
+c4.allowlist_node(br2)
+
+leader.form("multi-br")
+br1.join(leader)
+br2.join(leader)
+c1.join(leader, cli.JOIN_TYPE_END_DEVICE)
+c2.join(leader, cli.JOIN_TYPE_END_DEVICE)
+c3.join(leader, cli.JOIN_TYPE_END_DEVICE)
+c4.join(leader, cli.JOIN_TYPE_END_DEVICE)
+
+verify(leader.get_state() == 'leader')
+verify(br1.get_state() == 'router')
+verify(br2.get_state() == 'router')
+verify(c1.get_state() == 'child')
+verify(c2.get_state() == 'child')
+verify(c3.get_state() == 'child')
+verify(c4.get_state() == 'child')
+
+nodes_non_br = [leader, c1, c2, c3, c4]
+
+# -----------------------------------------------------------------------------------------------------------------------
+# Test implementation
+
+# Start the first BR
+
+br1.srp_server_set_addr_mode('unicast')
+br1.srp_server_auto_enable()
+
+br1.br_init(IF_INDEX, 1)
+br1.br_enable()
+
+time.sleep(1)
+verify(br1.br_get_state() == 'running')
+
+br1_local_omr = br1.br_get_local_omrprefix()
+br1_favored_omr = br1.br_get_favored_omrprefix().split()[0]
+verify(br1_local_omr == br1_favored_omr)
+
+br1_local_onlink = br1.br_get_local_onlinkprefix()
+br1_favored_onlink = br1.br_get_favored_onlinkprefix().split()[0]
+verify(br1_local_onlink == br1_favored_onlink)
+
+# Start the second BR
+
+br2.br_init(IF_INDEX, 1)
+br2.br_enable()
+
+time.sleep(1)
+verify(br2.br_get_state() == 'running')
+
+br2_local_omr = br2.br_get_local_omrprefix()
+br2_favored_omr = br2.br_get_favored_omrprefix().split()[0]
+verify(br2_favored_omr == br1_favored_omr)
+
+br2_favored_onlink = br2.br_get_favored_onlinkprefix().split()[0]
+verify(br2_favored_onlink == br1_favored_onlink)
+
+verify(br1.srp_server_get_state() == 'running')
+verify(br2.srp_server_get_state() == 'disabled')
+
+# Register SRP services on all nodes
+
+for node in nodes_non_br:
+    verify(node.srp_client_get_auto_start_mode() == 'Enabled')
+    node.srp_client_set_host_name('host' + str(node.index))
+    node.srp_client_enable_auto_host_address()
+    node.srp_client_add_service('srv' + str(node.index), '_test._udp', 777, 0, 0)
+
+time.sleep(1)
+
+hosts = br1.srp_server_get_hosts()
+verify(len(hosts) == len(nodes_non_br))
+
+services = br1.srp_server_get_services()
+verify(len(services) == len(nodes_non_br))
+
+# Ensure that all registered addresses are derived from BR1 OMR.
+
+for host in hosts:
+    verify(host['addresses'][0].startswith(br1_local_omr[:-4]))
+
+# Start SRP server on BR2
+
+br2.srp_server_set_addr_mode('unicast')
+br2.srp_server_auto_enable()
+
+time.sleep(1)
+
+verify(br2.srp_server_get_state() == 'running')
+
+# De-activate BR1
+
+br1.br_disable()
+br1.thread_stop()
+br1.interface_down()
+del br1
+
+c1.allowlist_node(br2)
+br2.allowlist_node(c1)
+
+# Wait long enough for BR2 to take over
+
+time.sleep(5)
+
+# Validate that everything is registered with BR2
+
+hosts = br2.srp_server_get_hosts()
+verify(len(hosts) == len(nodes_non_br))
+
+services = br2.srp_server_get_services()
+verify(len(services) == len(nodes_non_br))
+
+# Ensure that all registered addresses are now derived from BR2
+# OMR prefix.
+
+for host in hosts:
+    verify(host['addresses'][0].startswith(br2_local_omr[:-4]))
+
+# -----------------------------------------------------------------------------------------------------------------------
+# Test finished
+
+cli.Node.finalize_all_nodes()
+
+print('\'{}\' passed.'.format(test_name))
diff --git a/tests/toranj/cli/test-502-multi-br-leader-failure-recovery.py b/tests/toranj/cli/test-502-multi-br-leader-failure-recovery.py
new file mode 100755
index 000000000..71d2ebc1b
--- /dev/null
+++ b/tests/toranj/cli/test-502-multi-br-leader-failure-recovery.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+#
+#  Copyright (c) 2024, The OpenThread Authors.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#  3. Neither the name of the copyright holder nor the
+#     names of its contributors may be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+#  POSSIBILITY OF SUCH DAMAGE.
+
+from cli import verify
+from cli import verify_within
+import cli
+import time
+
+# -----------------------------------------------------------------------------------------------------------------------
+# Test description:
+#
+# Network with two BRs, BR1 acting as leader. Removing BR1 ensuring BR2 taking over.
+#
+#      ________________
+#     /                \
+#   br1 --- router --- br2
+#   /        / \        \
+#  c1       c2  c3      c4
+#
+
+test_name = __file__[:-3] if __file__.endswith('.py') else __file__
+print('-' * 120)
+print('Starting \'{}\''.format(test_name))
+
+# -----------------------------------------------------------------------------------------------------------------------
+# Creating `cli.Nodes` instances
+
+speedup = 60
+cli.Node.set_time_speedup_factor(speedup)
+
+br1 = cli.Node()
+br2 = cli.Node()
+router = cli.Node()
+c1 = cli.Node()
+c2 = cli.Node()
+c3 = cli.Node()
+c4 = cli.Node()
+
+IF_INDEX = 1
+
+# -----------------------------------------------------------------------------------------------------------------------
+# Form topology
+
+br1.allowlist_node(router)
+br1.allowlist_node(br2)
+br1.allowlist_node(c1)
+
+br2.allowlist_node(router)
+br2.allowlist_node(br1)
+br2.allowlist_node(c4)
+
+router.allowlist_node(br1)
+router.allowlist_node(br2)
+router.allowlist_node(c2)
+router.allowlist_node(c3)
+
+c1.allowlist_node(br1)
+
+c2.allowlist_node(router)
+c3.allowlist_node(router)
+
+c4.allowlist_node(br2)
+
+br1.form("multi-br")
+br2.join(br1)
+router.join(br1)
+c1.join(br1, cli.JOIN_TYPE_END_DEVICE)
+c2.join(br1, cli.JOIN_TYPE_END_DEVICE)
+c3.join(br1, cli.JOIN_TYPE_END_DEVICE)
+c4.join(br1, cli.JOIN_TYPE_END_DEVICE)
+
+verify(br1.get_state() == 'leader')
+verify(br2.get_state() == 'router')
+verify(router.get_state() == 'router')
+verify(c1.get_state() == 'child')
+verify(c2.get_state() == 'child')
+verify(c3.get_state() == 'child')
+verify(c4.get_state() == 'child')
+
+nodes_non_br = [router, c1, c2, c3, c4]
+
+# -----------------------------------------------------------------------------------------------------------------------
+# Test implementation
+
+# Start the first BR
+
+br1.srp_server_set_addr_mode('unicast')
+br1.srp_server_auto_enable()
+
+br1.br_init(IF_INDEX, 1)
+br1.br_enable()
+
+time.sleep(1)
+verify(br1.br_get_state() == 'running')
+
+br1_local_omr = br1.br_get_local_omrprefix()
+br1_favored_omr = br1.br_get_favored_omrprefix().split()[0]
+verify(br1_local_omr == br1_favored_omr)
+
+br1_local_onlink = br1.br_get_local_onlinkprefix()
+br1_favored_onlink = br1.br_get_favored_onlinkprefix().split()[0]
+verify(br1_local_onlink == br1_favored_onlink)
+
+# Start the second BR
+
+br2.br_init(IF_INDEX, 1)
+br2.br_enable()
+
+time.sleep(1)
+verify(br2.br_get_state() == 'running')
+
+br2_local_omr = br2.br_get_local_omrprefix()
+br2_favored_omr = br2.br_get_favored_omrprefix().split()[0]
+verify(br2_favored_omr == br1_favored_omr)
+
+br2_favored_onlink = br2.br_get_favored_onlinkprefix().split()[0]
+verify(br2_favored_onlink == br1_favored_onlink)
+
+verify(br1.srp_server_get_state() == 'running')
+verify(br2.srp_server_get_state() == 'disabled')
+
+# Register SRP services on all nodes
+
+for node in nodes_non_br:
+    verify(node.srp_client_get_auto_start_mode() == 'Enabled')
+    node.srp_client_set_host_name('host' + str(node.index))
+    node.srp_client_enable_auto_host_address()
+    node.srp_client_add_service('srv' + str(node.index), '_test._udp', 777, 0, 0)
+
+time.sleep(1)
+
+hosts = br1.srp_server_get_hosts()
+verify(len(hosts) == len(nodes_non_br))
+
+services = br1.srp_server_get_services()
+verify(len(services) == len(nodes_non_br))
+
+# Ensure that all registered addresses are derived from BR1 OMR.
+
+for host in hosts:
+    verify(host['addresses'][0].startswith(br1_local_omr[:-4]))
+
+# Start SRP server on BR2
+
+br2.srp_server_set_addr_mode('unicast')
+br2.srp_server_auto_enable()
+
+time.sleep(1)
+
+verify(br2.srp_server_get_state() == 'running')
+
+# De-activate BR1
+
+br1.br_disable()
+br1.thread_stop()
+br1.interface_down()
+del br1
+
+c1.allowlist_node(br2)
+br2.allowlist_node(c1)
+
+# Wait long enough for BR2 to take over
+
+time.sleep(5)
+
+# Validate that everything is registered with BR2
+
+hosts = br2.srp_server_get_hosts()
+verify(len(hosts) == len(nodes_non_br))
+
+services = br2.srp_server_get_services()
+verify(len(services) == len(nodes_non_br))
+
+# Ensure that all registered addresses are now derived from BR2
+# OMR prefix.
+
+for host in hosts:
+    verify(host['addresses'][0].startswith(br2_local_omr[:-4]))
+
+# -----------------------------------------------------------------------------------------------------------------------
+# Test finished
+
+cli.Node.finalize_all_nodes()
+
+print('\'{}\' passed.'.format(test_name))
diff --git a/tests/toranj/start.sh b/tests/toranj/start.sh
index ebbd3d68a..04cef966c 100755
--- a/tests/toranj/start.sh
+++ b/tests/toranj/start.sh
@@ -196,6 +196,8 @@ if [ "$TORANJ_CLI" = 1 ]; then
     run cli/test-400-srp-client-server.py
     run cli/test-401-srp-server-address-cache-snoop.py
     run cli/test-500-two-brs-two-networks.py
+    run cli/test-501-multi-br-failure-recovery.py
+    run cli/test-502-multi-br-leader-failure-recovery.py
     run cli/test-601-channel-manager-channel-change.py
     # Skip the "channel-select" test on a TREL only radio link, since it
     # requires energy scan which is not supported in this case.