[ 9.759795][ T186] ip (186) used greatest stack depth: 24048 bytes left [ 19.910054][ C0] clocksource: Watchdog remote CPU 2 read timed out [ 154.406798][ T6337] ping6 (6337) used greatest stack depth: 23656 bytes left [ 374.773875][T17156] [ 374.773966][T17156] ======================================================== [ 374.774093][T17156] WARNING: possible irq lock inversion dependency detected [ 374.774210][T17156] 7.1.0-rc5-virtme #1 Not tainted [ 374.774288][T17156] -------------------------------------------------------- [ 374.774401][T17156] sysctl/17156 just changed the state of lock: [ 374.774500][T17156] ffffffff93b7e8b8 (acaddr_hash_lock){+.+.}-{3:3}, at: __ipv6_dev_ac_dec+0x236/0x5d0 [ 374.774647][T17156] but this lock was taken by another, SOFTIRQ-READ-safe lock in the past: [ 374.774762][T17156] (&ndev->lock){++.-}-{3:3} [ 374.774765][T17156] [ 374.774765][T17156] [ 374.774765][T17156] and interrupts could create inverse lock ordering between them. [ 374.774765][T17156] [ 374.775055][T17156] [ 374.775055][T17156] other info that might help us debug this: [ 374.775176][T17156] Possible interrupt unsafe locking scenario: [ 374.775176][T17156] [ 374.775291][T17156] CPU0 CPU1 [ 374.775367][T17156] ---- ---- [ 374.775445][T17156] lock(acaddr_hash_lock); [ 374.775524][T17156] local_irq_disable(); [ 374.775620][T17156] lock(&ndev->lock); [ 374.775717][T17156] lock(acaddr_hash_lock); [ 374.775824][T17156] [ 374.775883][T17156] lock(&ndev->lock); [ 374.775944][T17156] [ 374.775944][T17156] *** DEADLOCK *** [ 374.775944][T17156] [ 374.776063][T17156] 2 locks held by sysctl/17156: [ 374.776141][T17156] #0: ff11000009d143e0 (sb_writers#3){.+.+}-{0:0}, at: ksys_write+0x116/0x250 [ 374.776282][T17156] #1: ffffffff93a8c9c0 (rtnl_mutex){+.+.}-{4:4}, at: addrconf_fixup_forwarding+0x4d/0x4d0 [ 374.776436][T17156] [ 374.776436][T17156] the shortest dependencies between 2nd lock and 1st lock: [ 374.776570][T17156] -> (&ndev->lock){++.-}-{3:3} { [ 374.776650][T17156] HARDIRQ-ON-W at: [ 374.776711][T17156] __lock_acquire+0x378/0xc10 [ 374.776816][T17156] lock_acquire.part.0+0xbc/0x260 [ 374.776911][T17156] _raw_write_lock_bh+0x38/0x50 [ 374.777012][T17156] addrconf_permanent_addr+0x108/0x9c0 [ 374.777127][T17156] addrconf_notify+0x151/0xf30 [ 374.777225][T17156] notifier_call_chain+0xb0/0x320 [ 374.777321][T17156] __dev_notify_flags+0xde/0x280 [ 374.777417][T17156] netif_change_flags+0xfe/0x190 [ 374.777511][T17156] do_setlink.isra.0+0x1cc3/0x2750 [ 374.777612][T17156] rtnl_newlink+0x8d1/0xef0 [ 374.777710][T17156] rtnetlink_rcv_msg+0x6fd/0xbd0 [ 374.777813][T17156] netlink_rcv_skb+0x14e/0x3a0 [ 374.777916][T17156] netlink_unicast+0x486/0x750 [ 374.778019][T17156] netlink_sendmsg+0x735/0xc60 [ 374.778159][T17156] ____sys_sendmsg+0x419/0x850 [ 374.778256][T17156] ___sys_sendmsg+0x14e/0x1d0 [ 374.778353][T17156] __sys_sendmsg+0x145/0x1f0 [ 374.778490][T17156] do_syscall_64+0x117/0x590 [ 374.778585][T17156] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 374.778699][T17156] HARDIRQ-ON-R at: [ 374.778804][T17156] __lock_acquire+0x378/0xc10 [ 374.778900][T17156] lock_acquire.part.0+0xbc/0x260 [ 374.779000][T17156] _raw_read_lock_bh+0x44/0x80 [ 374.779096][T17156] inet6_fill_ifla6_attrs+0x1aee/0x21e0 [ 374.779251][T17156] inet6_fill_link_af+0x5c/0xe0 [ 374.779347][T17156] rtnl_fill_ifinfo.isra.0+0x1a45/0x2c20 [ 374.779503][T17156] rtnl_getlink+0x9c9/0xeb0 [ 374.779600][T17156] rtnetlink_rcv_msg+0x6fd/0xbd0 [ 374.779698][T17156] netlink_rcv_skb+0x14e/0x3a0 [ 374.779800][T17156] netlink_unicast+0x486/0x750 [ 374.779941][T17156] netlink_sendmsg+0x735/0xc60 [ 374.780040][T17156] ____sys_sendmsg+0x419/0x850 [ 374.780136][T17156] ___sys_sendmsg+0x14e/0x1d0 [ 374.780281][T17156] __sys_sendmsg+0x145/0x1f0 [ 374.780379][T17156] do_syscall_64+0x117/0x590 [ 374.780476][T17156] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 374.780632][T17156] IN-SOFTIRQ-R at: [ 374.780690][T17156] __lock_acquire+0x378/0xc10 [ 374.780791][T17156] lock_acquire.part.0+0xbc/0x260 [ 374.780926][T17156] _raw_read_lock_bh+0x44/0x80 [ 374.781026][T17156] ipv6_get_lladdr+0xa8/0x3f0 [ 374.781126][T17156] ndisc_send_ns+0xf3/0x120 [ 374.781222][T17156] ndisc_solicit+0x314/0x480 [ 374.781361][T17156] neigh_probe+0xb3/0x100 [ 374.781458][T17156] __neigh_event_send+0x802/0x1070 [ 374.781555][T17156] neigh_resolve_output+0xbe/0x140 [ 374.781693][T17156] ip6_finish_output2+0x423/0x1300 [ 374.781794][T17156] ip6_finish_output+0x646/0xda0 [ 374.781891][T17156] ip6_output+0x23f/0x7f0 [ 374.782035][T17156] ip6_forward+0x10f4/0x2bd0 [ 374.782133][T17156] ipv6_rcv+0x3c1/0x5a0 [ 374.782228][T17156] __netif_receive_skb_one_core+0xfc/0x180 [ 374.782381][T17156] process_backlog+0x2bc/0x1490 [ 374.782478][T17156] __napi_poll+0xa7/0x3b0 [ 374.782572][T17156] net_rx_action+0x513/0xf50 [ 374.782707][T17156] handle_softirqs+0x1d8/0x940 [ 374.782814][T17156] do_softirq+0xa9/0xe0 [ 374.782912][T17156] __local_bh_enable_ip+0x113/0x140 [ 374.783055][T17156] __neigh_event_send+0x30a/0x1070 [ 374.783153][T17156] neigh_resolve_output+0xbe/0x140 [ 374.783249][T17156] ip6_finish_output2+0x423/0x1300 [ 374.783344][T17156] ip6_finish_output+0x646/0xda0 [ 374.783483][T17156] ip6_output+0x23f/0x7f0 [ 374.783579][T17156] ip6_send_skb+0xee/0x350 [ 374.783674][T17156] rawv6_push_pending_frames+0x4cb/0xa90 [ 374.783836][T17156] rawv6_sendmsg+0x1434/0x1a50 [ 374.783935][T17156] __sys_sendto+0x2c9/0x400 [ 374.784036][T17156] __x64_sys_sendto+0xe4/0x1f0 [ 374.784173][T17156] do_syscall_64+0x117/0x590 [ 374.784269][T17156] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 374.784383][T17156] INITIAL USE at: [ 374.784481][T17156] __lock_acquire+0x378/0xc10 [ 374.784576][T17156] lock_acquire.part.0+0xbc/0x260 [ 374.784670][T17156] _raw_write_lock_bh+0x38/0x50 [ 374.784811][T17156] addrconf_permanent_addr+0x108/0x9c0 [ 374.784930][T17156] addrconf_notify+0x151/0xf30 [ 374.785033][T17156] notifier_call_chain+0xb0/0x320 [ 374.785170][T17156] __dev_notify_flags+0xde/0x280 [ 374.785266][T17156] netif_change_flags+0xfe/0x190 [ 374.785361][T17156] do_setlink.isra.0+0x1cc3/0x2750 [ 374.785457][T17156] rtnl_newlink+0x8d1/0xef0 [ 374.785595][T17156] rtnetlink_rcv_msg+0x6fd/0xbd0 [ 374.785690][T17156] netlink_rcv_skb+0x14e/0x3a0 [ 374.785795][T17156] netlink_unicast+0x486/0x750 [ 374.785934][T17156] netlink_sendmsg+0x735/0xc60 [ 374.786031][T17156] ____sys_sendmsg+0x419/0x850 [ 374.786131][T17156] ___sys_sendmsg+0x14e/0x1d0 [ 374.786268][T17156] __sys_sendmsg+0x145/0x1f0 [ 374.786365][T17156] do_syscall_64+0x117/0x590 [ 374.786459][T17156] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 374.786615][T17156] INITIAL READ USE at: [ 374.786693][T17156] __lock_acquire+0x378/0xc10 [ 374.786794][T17156] lock_acquire.part.0+0xbc/0x260 [ 374.786950][T17156] _raw_read_lock_bh+0x44/0x80 [ 374.787049][T17156] inet6_fill_ifla6_attrs+0x1aee/0x21e0 [ 374.787162][T17156] inet6_fill_link_af+0x5c/0xe0 [ 374.787299][T17156] rtnl_fill_ifinfo.isra.0+0x1a45/0x2c20 [ 374.787412][T17156] rtnl_getlink+0x9c9/0xeb0 [ 374.787508][T17156] rtnetlink_rcv_msg+0x6fd/0xbd0 [ 374.787646][T17156] netlink_rcv_skb+0x14e/0x3a0 [ 374.787742][T17156] netlink_unicast+0x486/0x750 [ 374.787842][T17156] netlink_sendmsg+0x735/0xc60 [ 374.787981][T17156] ____sys_sendmsg+0x419/0x850 [ 374.788077][T17156] ___sys_sendmsg+0x14e/0x1d0 [ 374.788176][T17156] __sys_sendmsg+0x145/0x1f0 [ 374.788270][T17156] do_syscall_64+0x117/0x590 [ 374.788408][T17156] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 374.788522][T17156] } [ 374.788561][T17156] ... key at: [] __key.37+0x0/0x40 [ 374.788720][T17156] ... acquired at: [ 374.788782][T17156] __lock_acquire+0x508/0xc10 [ 374.788860][T17156] lock_acquire.part.0+0xbc/0x260 [ 374.788935][T17156] _raw_spin_lock+0x33/0x40 [ 374.789055][T17156] ipv6_add_acaddr_hash.isra.0+0x16d/0x350 [ 374.789149][T17156] __ipv6_dev_ac_inc+0x57e/0x930 [ 374.789225][T17156] addrconf_join_anycast+0x1bc/0x260 [ 374.789301][T17156] __ipv6_ifa_notify+0x75b/0xad0 [ 374.789417][T17156] addrconf_dad_completed+0x14b/0xe10 [ 374.789493][T17156] addrconf_dad_work+0x3c1/0x930 [ 374.789568][T17156] process_one_work+0xdf8/0x1410 [ 374.789645][T17156] worker_thread+0x4f1/0xd60 [ 374.789760][T17156] kthread+0x367/0x460 [ 374.789822][T17156] ret_from_fork+0x474/0x6b0 [ 374.789901][T17156] ret_from_fork_asm+0x11/0x20 [ 374.789983][T17156] [ 374.790023][T17156] -> (acaddr_hash_lock){+.+.}-{3:3} { [ 374.790141][T17156] HARDIRQ-ON-W at: [ 374.790199][T17156] __lock_acquire+0x378/0xc10 [ 374.790293][T17156] lock_acquire.part.0+0xbc/0x260 [ 374.790429][T17156] _raw_spin_lock+0x33/0x40 [ 374.790527][T17156] ipv6_add_acaddr_hash.isra.0+0x16d/0x350 [ 374.790642][T17156] __ipv6_dev_ac_inc+0x57e/0x930 [ 374.790782][T17156] addrconf_join_anycast+0x1bc/0x260 [ 374.790879][T17156] __ipv6_ifa_notify+0x75b/0xad0 [ 374.790983][T17156] addrconf_dad_completed+0x14b/0xe10 [ 374.791083][T17156] addrconf_dad_work+0x3c1/0x930 [ 374.791221][T17156] process_one_work+0xdf8/0x1410 [ 374.791317][T17156] worker_thread+0x4f1/0xd60 [ 374.791412][T17156] kthread+0x367/0x460 [ 374.791529][T17156] ret_from_fork+0x474/0x6b0 [ 374.791625][T17156] ret_from_fork_asm+0x11/0x20 [ 374.791723][T17156] SOFTIRQ-ON-W at: [ 374.791785][T17156] __lock_acquire+0x378/0xc10 [ 374.791921][T17156] lock_acquire.part.0+0xbc/0x260 [ 374.792020][T17156] _raw_spin_lock+0x33/0x40 [ 374.792116][T17156] __ipv6_dev_ac_dec+0x236/0x5d0 [ 374.792253][T17156] addrconf_leave_anycast+0x1bc/0x260 [ 374.792352][T17156] dev_forward_change+0x3b3/0x850 [ 374.792447][T17156] addrconf_fixup_forwarding+0x2af/0x4d0 [ 374.792601][T17156] addrconf_sysctl_forward+0x209/0x2b0 [ 374.792697][T17156] proc_sys_call_handler+0x31b/0x480 [ 374.792801][T17156] new_sync_write+0x333/0x750 [ 374.792942][T17156] vfs_write+0x6a4/0xc10 [ 374.793042][T17156] ksys_write+0x116/0x250 [ 374.793138][T17156] do_syscall_64+0x117/0x590 [ 374.793274][T17156] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 374.793389][T17156] INITIAL USE at: [ 374.793447][T17156] __lock_acquire+0x378/0xc10 [ 374.793544][T17156] lock_acquire.part.0+0xbc/0x260 [ 374.793679][T17156] _raw_spin_lock+0x33/0x40 [ 374.793781][T17156] ipv6_add_acaddr_hash.isra.0+0x16d/0x350 [ 374.793895][T17156] __ipv6_dev_ac_inc+0x57e/0x930 [ 374.793993][T17156] addrconf_join_anycast+0x1bc/0x260 [ 374.794089][T17156] __ipv6_ifa_notify+0x75b/0xad0 [ 374.794184][T17156] addrconf_dad_completed+0x14b/0xe10 [ 374.794323][T17156] addrconf_dad_work+0x3c1/0x930 [ 374.794419][T17156] process_one_work+0xdf8/0x1410 [ 374.794515][T17156] worker_thread+0x4f1/0xd60 [ 374.794651][T17156] kthread+0x367/0x460 [ 374.794729][T17156] ret_from_fork+0x474/0x6b0 [ 374.794831][T17156] ret_from_fork_asm+0x11/0x20 [ 374.794926][T17156] } [ 374.794968][T17156] ... key at: [] acaddr_hash_lock+0x18/0x45a0 [ 374.795084][T17156] ... acquired at: [ 374.795139][T17156] mark_lock+0x1d7/0xa00 [ 374.795218][T17156] mark_usage+0x105/0x170 [ 374.795334][T17156] __lock_acquire+0x378/0xc10 [ 374.795411][T17156] lock_acquire.part.0+0xbc/0x260 [ 374.795487][T17156] _raw_spin_lock+0x33/0x40 [ 374.795564][T17156] __ipv6_dev_ac_dec+0x236/0x5d0 [ 374.795681][T17156] addrconf_leave_anycast+0x1bc/0x260 [ 374.795756][T17156] dev_forward_change+0x3b3/0x850 [ 374.795838][T17156] addrconf_fixup_forwarding+0x2af/0x4d0 [ 374.795977][T17156] addrconf_sysctl_forward+0x209/0x2b0 [ 374.796053][T17156] proc_sys_call_handler+0x31b/0x480 [ 374.796129][T17156] new_sync_write+0x333/0x750 [ 374.796204][T17156] vfs_write+0x6a4/0xc10 [ 374.796319][T17156] ksys_write+0x116/0x250 [ 374.796396][T17156] do_syscall_64+0x117/0x590 [ 374.796471][T17156] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 374.796568][T17156] [ 374.796606][T17156] [ 374.796606][T17156] stack backtrace: [ 374.796741][T17156] CPU: 2 UID: 0 PID: 17156 Comm: sysctl Not tainted 7.1.0-rc5-virtme #1 PREEMPT(full) [ 374.796744][T17156] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 374.796746][T17156] Call Trace: [ 374.796747][T17156] [ 374.796749][T17156] dump_stack_lvl+0x6f/0xa0 [ 374.796752][T17156] print_irq_inversion_bug.part.0.cold+0xe6/0x143 [ 374.796755][T17156] mark_lock_irq+0x989/0x9c0 [ 374.796758][T17156] mark_lock+0x1d7/0xa00 [ 374.796760][T17156] mark_usage+0x105/0x170 [ 374.796761][T17156] __lock_acquire+0x378/0xc10 [ 374.796763][T17156] lock_acquire.part.0+0xbc/0x260 [ 374.796765][T17156] ? __ipv6_dev_ac_dec+0x236/0x5d0 [ 374.796766][T17156] ? rcu_is_watching+0x15/0xd0 [ 374.796769][T17156] ? do_raw_read_unlock+0x70/0x70 [ 374.796771][T17156] ? lock_acquire+0x134/0x160 [ 374.796773][T17156] _raw_spin_lock+0x33/0x40 [ 374.796774][T17156] ? __ipv6_dev_ac_dec+0x236/0x5d0 [ 374.796775][T17156] __ipv6_dev_ac_dec+0x236/0x5d0 [ 374.796783][T17156] addrconf_leave_anycast+0x1bc/0x260 [ 374.796784][T17156] ? find_held_lock+0x2b/0x80 [ 374.796787][T17156] ? __ipv6_isatap_ifid+0x210/0x210 [ 374.796789][T17156] ? mark_held_locks+0x40/0x70 [ 374.796790][T17156] ? lockdep_hardirqs_on+0x8c/0x130 [ 374.796793][T17156] dev_forward_change+0x3b3/0x850 [ 374.796795][T17156] ? addrconf_fixup_forwarding+0x4d/0x4d0 [ 374.796797][T17156] ? addrconf_sysctl_proxy_ndp+0x2f0/0x2f0 [ 374.796799][T17156] addrconf_fixup_forwarding+0x2af/0x4d0 [ 374.796802][T17156] addrconf_sysctl_forward+0x209/0x2b0 [ 374.796804][T17156] ? addrconf_fixup_forwarding+0x4d0/0x4d0 [ 374.796806][T17156] ? addrconf_fixup_forwarding+0x4d0/0x4d0 [ 374.796808][T17156] ? __kvmalloc_node_noprof+0x305/0x8a0 [ 374.796811][T17156] ? proc_sys_call_handler+0x25d/0x480 [ 374.796813][T17156] proc_sys_call_handler+0x31b/0x480 [ 374.796815][T17156] ? proc_sys_lookup+0x3d0/0x3d0 [ 374.796817][T17156] ? rcu_read_unlock+0x1b/0x70 [ 374.796820][T17156] ? do_raw_spin_unlock+0x59/0x250 [ 374.796821][T17156] ? rcu_lockdep_current_cpu_online+0x39/0x1b0 [ 374.796823][T17156] ? rcu_read_lock_any_held+0x3c/0x90 [ 374.796825][T17156] ? proc_sys_call_handler+0x480/0x480 [ 374.796827][T17156] new_sync_write+0x333/0x750 [ 374.796828][T17156] ? __lock_acquire+0x508/0xc10 [ 374.796830][T17156] ? new_sync_read+0x740/0x740 [ 374.796831][T17156] ? lock_acquire.part.0+0xbc/0x260 [ 374.796833][T17156] ? ksys_write+0x116/0x250 [ 374.796835][T17156] vfs_write+0x6a4/0xc10 [ 374.796837][T17156] ksys_write+0x116/0x250 [ 374.796838][T17156] ? __ia32_sys_read+0xc0/0xc0 [ 374.796839][T17156] ? rcu_is_watching+0x15/0xd0 [ 374.796841][T17156] ? rcu_is_watching+0x15/0xd0 [ 374.796843][T17156] do_syscall_64+0x117/0x590 [ 374.796844][T17156] ? trace_hardirqs_off+0xd/0x30 [ 374.796847][T17156] ? exc_page_fault+0xee/0x100 [ 374.796849][T17156] entry_SYSCALL_64_after_hwframe+0x4b/0x53 [ 374.796850][T17156] RIP: 0033:0x7fb7094c708e [ 374.796853][T17156] Code: 4d 89 d8 e8 94 bd 00 00 4c 8b 5d f8 41 8b 93 08 03 00 00 59 5e 48 83 f8 fc 74 11 c9 c3 0f 1f 80 00 00 00 00 48 8b 45 10 0f 05 c3 83 e2 39 83 fa 08 75 e7 e8 03 ff ff ff 0f 1f 00 f3 0f 1e fa [ 374.796855][T17156] RSP: 002b:00007ffd040d0f40 EFLAGS: 00000202 ORIG_RAX: 0000000000000001 [ 374.796858][T17156] RAX: ffffffffffffffda RBX: 0000000000000002 RCX: 00007fb7094c708e [ 374.796859][T17156] RDX: 0000000000000002 RSI: 00005578371304b0 RDI: 0000000000000005 [ 374.796860][T17156] RBP: 00007ffd040d0f50 R08: 0000000000000000 R09: 0000000000000000 [ 374.796861][T17156] R10: 0000000000000000 R11: 0000000000000202 R12: 00005578371325e0 [ 374.796862][T17156] R13: 0000557837130470 R14: 0000000000000002 R15: 0000000000000000 [ 374.796864][T17156]