[ 37.275301][ T545] 8021q: 802.1Q VLAN Support v1.8 [ 38.048868][ T574] br1: port 1(veth1) entered blocking state [ 38.049132][ T574] br1: port 1(veth1) entered disabled state [ 38.049383][ T574] veth1: entered allmulticast mode [ 38.051248][ T574] veth1: entered promiscuous mode [ 38.078348][ T36] br1: port 1(veth1) entered blocking state [ 38.078579][ T36] br1: port 1(veth1) entered forwarding state [ 52.305867][ T708] [ 52.305955][ T708] ======================================================== [ 52.306087][ T708] WARNING: possible irq lock inversion dependency detected [ 52.306213][ T708] 7.1.0-rc5-virtme #1 Not tainted [ 52.306298][ T708] -------------------------------------------------------- [ 52.306424][ T708] sysctl/708 just changed the state of lock: [ 52.306544][ T708] ffffffffb6d7e8b8 (acaddr_hash_lock){+.+.}-{3:3}, at: __ipv6_dev_ac_dec+0x236/0x5d0 [ 52.306698][ T708] but this lock was taken by another, SOFTIRQ-safe lock in the past: [ 52.306823][ T708] (&ndev->lock){++--}-{3:3} [ 52.306825][ T708] [ 52.306825][ T708] [ 52.306825][ T708] and interrupts could create inverse lock ordering between them. [ 52.306825][ T708] [ 52.307134][ T708] [ 52.307134][ T708] other info that might help us debug this: [ 52.307267][ T708] Possible interrupt unsafe locking scenario: [ 52.307267][ T708] [ 52.307397][ T708] CPU0 CPU1 [ 52.307486][ T708] ---- ---- [ 52.307573][ T708] lock(acaddr_hash_lock); [ 52.307656][ T708] local_irq_disable(); [ 52.307763][ T708] lock(&ndev->lock); [ 52.307871][ T708] lock(acaddr_hash_lock); [ 52.307972][ T708] [ 52.308033][ T708] lock(&ndev->lock); [ 52.308096][ T708] [ 52.308096][ T708] *** DEADLOCK *** [ 52.308096][ T708] [ 52.308216][ T708] 2 locks held by sysctl/708: [ 52.308303][ T708] #0: ff11000009aac3e0 (sb_writers#3){.+.+}-{0:0}, at: ksys_write+0x116/0x250 [ 52.308451][ T708] #1: ffffffffb6c8c9c0 (rtnl_mutex){+.+.}-{4:4}, at: addrconf_fixup_forwarding+0x4d/0x4d0 [ 52.308620][ T708] [ 52.308620][ T708] the shortest dependencies between 2nd lock and 1st lock: [ 52.308759][ T708] -> (&ndev->lock){++--}-{3:3} { [ 52.308845][ T708] HARDIRQ-ON-W at: [ 52.308913][ T708] __lock_acquire+0x378/0xc10 [ 52.309020][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.309123][ T708] _raw_write_lock_bh+0x38/0x50 [ 52.309232][ T708] addrconf_permanent_addr+0x108/0x9c0 [ 52.309354][ T708] addrconf_notify+0x151/0xf30 [ 52.309456][ T708] notifier_call_chain+0xb0/0x320 [ 52.309570][ T708] __dev_notify_flags+0xde/0x280 [ 52.309672][ T708] netif_change_flags+0xfe/0x190 [ 52.309816][ T708] do_setlink.isra.0+0x1cc3/0x2750 [ 52.309918][ T708] rtnl_newlink+0x8d1/0xef0 [ 52.310019][ T708] rtnetlink_rcv_msg+0x6fd/0xbd0 [ 52.310122][ T708] netlink_rcv_skb+0x14e/0x3a0 [ 52.310277][ T708] netlink_unicast+0x486/0x750 [ 52.310380][ T708] netlink_sendmsg+0x735/0xc60 [ 52.310484][ T708] ____sys_sendmsg+0x419/0x850 [ 52.310634][ T708] ___sys_sendmsg+0x14e/0x1d0 [ 52.310737][ T708] __sys_sendmsg+0x145/0x1f0 [ 52.310841][ T708] do_syscall_64+0x117/0x590 [ 52.310986][ T708] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 52.311109][ T708] HARDIRQ-ON-R at: [ 52.311172][ T708] __lock_acquire+0x378/0xc10 [ 52.311318][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.311424][ T708] _raw_read_lock_bh+0x44/0x80 [ 52.311532][ T708] inet6_fill_ifla6_attrs+0x1aee/0x21e0 [ 52.311695][ T708] inet6_fill_link_af+0x5c/0xe0 [ 52.311797][ T708] rtnl_fill_ifinfo.isra.0+0x1a45/0x2c20 [ 52.311918][ T708] rtnl_getlink+0x9c9/0xeb0 [ 52.312064][ T708] rtnetlink_rcv_msg+0x6fd/0xbd0 [ 52.312176][ T708] netlink_rcv_skb+0x14e/0x3a0 [ 52.312277][ T708] netlink_unicast+0x486/0x750 [ 52.312379][ T708] netlink_sendmsg+0x735/0xc60 [ 52.312484][ T708] ____sys_sendmsg+0x419/0x850 [ 52.312589][ T708] ___sys_sendmsg+0x14e/0x1d0 [ 52.312691][ T708] __sys_sendmsg+0x145/0x1f0 [ 52.312835][ T708] do_syscall_64+0x117/0x590 [ 52.312937][ T708] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 52.313057][ T708] IN-SOFTIRQ-W at: [ 52.313161][ T708] __lock_acquire+0x378/0xc10 [ 52.313265][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.313376][ T708] _raw_write_lock+0x33/0x40 [ 52.313532][ T708] addrconf_rs_timer+0xb0/0x770 [ 52.313638][ T708] call_timer_fn+0x163/0x4f0 [ 52.313741][ T708] __run_timers+0x68f/0xab0 [ 52.313844][ T708] run_timer_softirq+0xf0/0x160 [ 52.313992][ T708] handle_softirqs+0x1d8/0x940 [ 52.314095][ T708] __irq_exit_rcu+0x103/0x1c0 [ 52.314197][ T708] irq_exit_rcu+0xe/0x30 [ 52.314349][ T708] sysvec_apic_timer_interrupt+0x9d/0xe0 [ 52.314472][ T708] asm_sysvec_apic_timer_interrupt+0x1a/0x20 [ 52.314599][ T708] pv_native_safe_halt+0xf/0x10 [ 52.314744][ T708] default_idle+0x9/0x10 [ 52.314847][ T708] default_idle_call+0x6e/0xb0 [ 52.314949][ T708] cpuidle_idle_call.constprop.0+0x237/0x410 [ 52.315113][ T708] do_idle+0xf5/0x160 [ 52.315212][ T708] cpu_startup_entry+0x53/0x70 [ 52.315315][ T708] start_secondary+0x204/0x2b0 [ 52.315476][ T708] common_startup_64+0x13e/0x148 [ 52.315583][ T708] IN-SOFTIRQ-R at: [ 52.315651][ T708] __lock_acquire+0x378/0xc10 [ 52.315795][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.315900][ T708] _raw_read_lock_bh+0x44/0x80 [ 52.316002][ T708] ipv6_get_lladdr+0xa8/0x3f0 [ 52.316104][ T708] addrconf_rs_timer+0x251/0x770 [ 52.316248][ T708] call_timer_fn+0x163/0x4f0 [ 52.316349][ T708] __run_timers+0x68f/0xab0 [ 52.316457][ T708] run_timer_softirq+0xf0/0x160 [ 52.316610][ T708] handle_softirqs+0x1d8/0x940 [ 52.316709][ T708] __irq_exit_rcu+0x103/0x1c0 [ 52.316808][ T708] irq_exit_rcu+0xe/0x30 [ 52.316952][ T708] sysvec_apic_timer_interrupt+0x9d/0xe0 [ 52.317073][ T708] asm_sysvec_apic_timer_interrupt+0x1a/0x20 [ 52.317196][ T708] pv_native_safe_halt+0xf/0x10 [ 52.317342][ T708] default_idle+0x9/0x10 [ 52.317444][ T708] default_idle_call+0x6e/0xb0 [ 52.317550][ T708] cpuidle_idle_call.constprop.0+0x237/0x410 [ 52.317714][ T708] do_idle+0xf5/0x160 [ 52.317820][ T708] cpu_startup_entry+0x53/0x70 [ 52.317922][ T708] start_secondary+0x204/0x2b0 [ 52.318066][ T708] common_startup_64+0x13e/0x148 [ 52.318167][ T708] INITIAL USE at: [ 52.318233][ T708] __lock_acquire+0x378/0xc10 [ 52.318336][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.318482][ T708] _raw_write_lock_bh+0x38/0x50 [ 52.318595][ T708] addrconf_permanent_addr+0x108/0x9c0 [ 52.318719][ T708] addrconf_notify+0x151/0xf30 [ 52.318863][ T708] notifier_call_chain+0xb0/0x320 [ 52.318963][ T708] __dev_notify_flags+0xde/0x280 [ 52.319064][ T708] netif_change_flags+0xfe/0x190 [ 52.319207][ T708] do_setlink.isra.0+0x1cc3/0x2750 [ 52.319310][ T708] rtnl_newlink+0x8d1/0xef0 [ 52.319419][ T708] rtnetlink_rcv_msg+0x6fd/0xbd0 [ 52.319571][ T708] netlink_rcv_skb+0x14e/0x3a0 [ 52.319676][ T708] netlink_unicast+0x486/0x750 [ 52.319778][ T708] netlink_sendmsg+0x735/0xc60 [ 52.319922][ T708] ____sys_sendmsg+0x419/0x850 [ 52.320024][ T708] ___sys_sendmsg+0x14e/0x1d0 [ 52.320125][ T708] __sys_sendmsg+0x145/0x1f0 [ 52.320281][ T708] do_syscall_64+0x117/0x590 [ 52.320382][ T708] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 52.320503][ T708] INITIAL READ USE at: [ 52.320588][ T708] __lock_acquire+0x378/0xc10 [ 52.320731][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.320853][ T708] _raw_read_lock_bh+0x44/0x80 [ 52.320955][ T708] inet6_fill_ifla6_attrs+0x1aee/0x21e0 [ 52.321118][ T708] inet6_fill_link_af+0x5c/0xe0 [ 52.321221][ T708] rtnl_fill_ifinfo.isra.0+0x1a45/0x2c20 [ 52.321383][ T708] rtnl_getlink+0x9c9/0xeb0 [ 52.321487][ T708] rtnetlink_rcv_msg+0x6fd/0xbd0 [ 52.321591][ T708] netlink_rcv_skb+0x14e/0x3a0 [ 52.321694][ T708] netlink_unicast+0x486/0x750 [ 52.321838][ T708] netlink_sendmsg+0x735/0xc60 [ 52.321943][ T708] ____sys_sendmsg+0x419/0x850 [ 52.322045][ T708] ___sys_sendmsg+0x14e/0x1d0 [ 52.322199][ T708] __sys_sendmsg+0x145/0x1f0 [ 52.322299][ T708] do_syscall_64+0x117/0x590 [ 52.322405][ T708] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 52.322571][ T708] } [ 52.322613][ T708] ... key at: [] __key.37+0x0/0x40 [ 52.322740][ T708] ... acquired at: [ 52.322801][ T708] __lock_acquire+0x508/0xc10 [ 52.322929][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.323009][ T708] _raw_spin_lock+0x33/0x40 [ 52.323091][ T708] ipv6_add_acaddr_hash.isra.0+0x16d/0x350 [ 52.323190][ T708] __ipv6_dev_ac_inc+0x57e/0x930 [ 52.323312][ T708] addrconf_join_anycast+0x1bc/0x260 [ 52.323396][ T708] __ipv6_ifa_notify+0x75b/0xad0 [ 52.323476][ T708] addrconf_dad_completed+0x14b/0xe10 [ 52.323562][ T708] addrconf_dad_work+0x3c1/0x930 [ 52.323684][ T708] process_one_work+0xdf8/0x1410 [ 52.323766][ T708] worker_thread+0x4f1/0xd60 [ 52.323845][ T708] kthread+0x367/0x460 [ 52.323907][ T708] ret_from_fork+0x474/0x6b0 [ 52.324032][ T708] ret_from_fork_asm+0x11/0x20 [ 52.324116][ T708] [ 52.324159][ T708] -> (acaddr_hash_lock){+.+.}-{3:3} { [ 52.324251][ T708] HARDIRQ-ON-W at: [ 52.324314][ T708] __lock_acquire+0x378/0xc10 [ 52.324468][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.324574][ T708] _raw_spin_lock+0x33/0x40 [ 52.324680][ T708] ipv6_add_acaddr_hash.isra.0+0x16d/0x350 [ 52.324844][ T708] __ipv6_dev_ac_inc+0x57e/0x930 [ 52.324946][ T708] addrconf_join_anycast+0x1bc/0x260 [ 52.325056][ T708] __ipv6_ifa_notify+0x75b/0xad0 [ 52.325201][ T708] addrconf_dad_completed+0x14b/0xe10 [ 52.325304][ T708] addrconf_dad_work+0x3c1/0x930 [ 52.325406][ T708] process_one_work+0xdf8/0x1410 [ 52.325552][ T708] worker_thread+0x4f1/0xd60 [ 52.325653][ T708] kthread+0x367/0x460 [ 52.325735][ T708] ret_from_fork+0x474/0x6b0 [ 52.325881][ T708] ret_from_fork_asm+0x11/0x20 [ 52.325992][ T708] SOFTIRQ-ON-W at: [ 52.326054][ T708] __lock_acquire+0x378/0xc10 [ 52.326156][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.326300][ T708] _raw_spin_lock+0x33/0x40 [ 52.326400][ T708] __ipv6_dev_ac_dec+0x236/0x5d0 [ 52.326500][ T708] addrconf_leave_anycast+0x1bc/0x260 [ 52.326654][ T708] dev_forward_change+0x3b3/0x850 [ 52.326759][ T708] addrconf_fixup_forwarding+0x2af/0x4d0 [ 52.326891][ T708] addrconf_sysctl_forward+0x209/0x2b0 [ 52.327036][ T708] proc_sys_call_handler+0x31b/0x480 [ 52.327139][ T708] new_sync_write+0x333/0x750 [ 52.327243][ T708] vfs_write+0x6a4/0xc10 [ 52.327387][ T708] ksys_write+0x116/0x250 [ 52.327494][ T708] do_syscall_64+0x117/0x590 [ 52.327601][ T708] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 52.327766][ T708] INITIAL USE at: [ 52.327831][ T708] __lock_acquire+0x378/0xc10 [ 52.327933][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.328042][ T708] _raw_spin_lock+0x33/0x40 [ 52.328188][ T708] ipv6_add_acaddr_hash.isra.0+0x16d/0x350 [ 52.328310][ T708] __ipv6_dev_ac_inc+0x57e/0x930 [ 52.328411][ T708] addrconf_join_anycast+0x1bc/0x260 [ 52.328563][ T708] __ipv6_ifa_notify+0x75b/0xad0 [ 52.328663][ T708] addrconf_dad_completed+0x14b/0xe10 [ 52.328764][ T708] addrconf_dad_work+0x3c1/0x930 [ 52.328908][ T708] process_one_work+0xdf8/0x1410 [ 52.329012][ T708] worker_thread+0x4f1/0xd60 [ 52.329113][ T708] kthread+0x367/0x460 [ 52.329195][ T708] ret_from_fork+0x474/0x6b0 [ 52.329336][ T708] ret_from_fork_asm+0x11/0x20 [ 52.329437][ T708] } [ 52.329477][ T708] ... key at: [] acaddr_hash_lock+0x18/0x45a0 [ 52.329642][ T708] ... acquired at: [ 52.329702][ T708] mark_lock+0x1d7/0xa00 [ 52.329785][ T708] mark_usage+0x105/0x170 [ 52.329866][ T708] __lock_acquire+0x378/0xc10 [ 52.329987][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.330067][ T708] _raw_spin_lock+0x33/0x40 [ 52.330148][ T708] __ipv6_dev_ac_dec+0x236/0x5d0 [ 52.330230][ T708] addrconf_leave_anycast+0x1bc/0x260 [ 52.330362][ T708] dev_forward_change+0x3b3/0x850 [ 52.330446][ T708] addrconf_fixup_forwarding+0x2af/0x4d0 [ 52.330551][ T708] addrconf_sysctl_forward+0x209/0x2b0 [ 52.330631][ T708] proc_sys_call_handler+0x31b/0x480 [ 52.330755][ T708] new_sync_write+0x333/0x750 [ 52.330834][ T708] vfs_write+0x6a4/0xc10 [ 52.330923][ T708] ksys_write+0x116/0x250 [ 52.331002][ T708] do_syscall_64+0x117/0x590 [ 52.331126][ T708] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 52.331229][ T708] [ 52.331270][ T708] [ 52.331270][ T708] stack backtrace: [ 52.331378][ T708] CPU: 0 UID: 0 PID: 708 Comm: sysctl Not tainted 7.1.0-rc5-virtme #1 PREEMPT(full) [ 52.331381][ T708] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 52.331383][ T708] Call Trace: [ 52.331385][ T708] [ 52.331386][ T708] dump_stack_lvl+0x6f/0xa0 [ 52.331390][ T708] print_irq_inversion_bug.part.0.cold+0xe6/0x143 [ 52.331394][ T708] mark_lock_irq+0x989/0x9c0 [ 52.331396][ T708] mark_lock+0x1d7/0xa00 [ 52.331398][ T708] mark_usage+0x105/0x170 [ 52.331400][ T708] __lock_acquire+0x378/0xc10 [ 52.331402][ T708] lock_acquire.part.0+0xbc/0x260 [ 52.331404][ T708] ? __ipv6_dev_ac_dec+0x236/0x5d0 [ 52.331405][ T708] ? rcu_is_watching+0x15/0xd0 [ 52.331408][ T708] ? do_raw_read_unlock+0x70/0x70 [ 52.331410][ T708] ? lock_acquire+0x134/0x160 [ 52.331412][ T708] _raw_spin_lock+0x33/0x40 [ 52.331414][ T708] ? __ipv6_dev_ac_dec+0x236/0x5d0 [ 52.331415][ T708] __ipv6_dev_ac_dec+0x236/0x5d0 [ 52.331417][ T708] addrconf_leave_anycast+0x1bc/0x260 [ 52.331419][ T708] ? find_held_lock+0x2b/0x80 [ 52.331422][ T708] ? __ipv6_isatap_ifid+0x210/0x210 [ 52.331424][ T708] ? mark_held_locks+0x40/0x70 [ 52.331426][ T708] ? lockdep_hardirqs_on+0x8c/0x130 [ 52.331428][ T708] dev_forward_change+0x3b3/0x850 [ 52.331430][ T708] ? addrconf_fixup_forwarding+0x4d/0x4d0 [ 52.331432][ T708] ? addrconf_sysctl_proxy_ndp+0x2f0/0x2f0 [ 52.331435][ T708] addrconf_fixup_forwarding+0x2af/0x4d0 [ 52.331438][ T708] addrconf_sysctl_forward+0x209/0x2b0 [ 52.331440][ T708] ? addrconf_fixup_forwarding+0x4d0/0x4d0 [ 52.331442][ T708] ? addrconf_fixup_forwarding+0x4d0/0x4d0 [ 52.331444][ T708] ? __kvmalloc_node_noprof+0x305/0x8a0 [ 52.331447][ T708] ? proc_sys_call_handler+0x25d/0x480 [ 52.331450][ T708] proc_sys_call_handler+0x31b/0x480 [ 52.331452][ T708] ? proc_sys_lookup+0x3d0/0x3d0 [ 52.331454][ T708] ? rcu_read_unlock+0x1b/0x70 [ 52.331457][ T708] ? do_raw_spin_unlock+0x59/0x250 [ 52.331458][ T708] ? rcu_lockdep_current_cpu_online+0x39/0x1b0 [ 52.331460][ T708] ? rcu_read_lock_any_held+0x3c/0x90 [ 52.331462][ T708] ? proc_sys_call_handler+0x480/0x480 [ 52.331464][ T708] new_sync_write+0x333/0x750 [ 52.331466][ T708] ? __lock_acquire+0x508/0xc10 [ 52.331467][ T708] ? new_sync_read+0x740/0x740 [ 52.331469][ T708] ? lock_acquire.part.0+0xbc/0x260 [ 52.331470][ T708] ? ksys_write+0x116/0x250 [ 52.331472][ T708] vfs_write+0x6a4/0xc10 [ 52.331475][ T708] ksys_write+0x116/0x250 [ 52.331476][ T708] ? __ia32_sys_read+0xc0/0xc0 [ 52.331478][ T708] ? rcu_is_watching+0x15/0xd0 [ 52.331479][ T708] ? rcu_is_watching+0x15/0xd0 [ 52.331481][ T708] do_syscall_64+0x117/0x590 [ 52.331482][ T708] ? trace_hardirqs_off+0xd/0x30 [ 52.331485][ T708] ? exc_page_fault+0xee/0x100 [ 52.331487][ T708] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 52.331489][ T708] RIP: 0033:0x7f0cf4df708e [ 52.331492][ T708] Code: 4d 89 d8 e8 94 bd 00 00 4c 8b 5d f8 41 8b 93 08 03 00 00 59 5e 48 83 f8 fc 74 11 c9 c3 0f 1f 80 00 00 00 00 48 8b 45 10 0f 05 c3 83 e2 39 83 fa 08 75 e7 e8 03 ff ff ff 0f 1f 00 f3 0f 1e fa [ 52.331494][ T708] RSP: 002b:00007ffe1f4d64e0 EFLAGS: 00000202 ORIG_RAX: 0000000000000001 [ 52.331497][ T708] RAX: ffffffffffffffda RBX: 0000000000000002 RCX: 00007f0cf4df708e [ 52.331498][ T708] RDX: 0000000000000002 RSI: 000055e5295134b0 RDI: 0000000000000005 [ 52.331499][ T708] RBP: 00007ffe1f4d64f0 R08: 0000000000000000 R09: 0000000000000000 [ 52.331500][ T708] R10: 0000000000000000 R11: 0000000000000202 R12: 000055e5295155e0 [ 52.331501][ T708] R13: 000055e529513470 R14: 0000000000000002 R15: 0000000000000000 [ 52.331503][ T708] [ 52.480253][ T718] br1: port 1(veth1) entered disabled state [ 52.498646][ T719] veth1: left allmulticast mode [ 52.498767][ T719] veth1: left promiscuous mode [ 52.498920][ T719] br1: port 1(veth1) entered disabled state