`ERROR` in dmesg for slow read attack

[RomanBelozerov]

For TempestaFW config:

        cache 0;
        keepalive_timeout 60;
        listen 443 proto=h2;
        tls_match_any_server_name;
        srv_group default {
            server ${server_ip}:8000;
        }
        vhost tempesta-tech.com {
           tls_certificate ${tempesta_workdir}/tempesta.crt;
           tls_certificate_key ${tempesta_workdir}/tempesta.key;
           proxy_pass default;
        }

1. create http2 connection with initial_window_size = 1 and open a many streams (I created 100 streams)
2. send simple GET request to a response with a large body (I used 100 MB response body)

first traceback

[ 4707.780010] [tempesta fw] Warning: tfw_h2_sk_prepare_xmit: failed to add overhead to current TCP socket control data.
[ 4737.236662] TCP: out of memory -- consider tuning tcp_mem
[ 4737.292952] ------------[ cut here ]------------
[ 4737.292960] WARNING: CPU: 8 PID: 15432 at net/core/stream.c:197 sk_stream_kill_queues+0x114/0x130
[ 4737.292961] Modules linked in: tempesta_fw(OE) tempesta_db(OE) tempesta_tls(OE) tempesta_lib(OE) nfnetlink_queue nfnetlink_log tls sha256_ssse3 sha512_ssse3 nvme_tcp nvme_fabrics nf_conntrack_netlink xfrm_user xfrm_algo xt_addrtype br_netfilter nft_masq vhost_vsock vmw_vsock_virtio_transport_common vhost vhost_iotlb vsock xt_CHECKSUM xt_MASQUERADE xt_conntrack ipt_REJECT nf_reject_ipv4 xt_tcpudp nft_compat nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 rfcomm nft_counter nf_tables libcrc32c nfnetlink bridge stp llc cmac algif_hash algif_skcipher af_alg bnep overlay binfmt_misc snd_soc_skl_hda_dsp snd_soc_hdac_hdmi snd_hda_codec_hdmi snd_hda_codec_realtek snd_hda_codec_generic nvidia_uvm(POE) nvidia_drm(POE) nvidia_modeset(POE) nvidia(POE) intel_rapl_msr intel_rapl_common x86_pkg_temp_thermal intel_powerclamp snd_soc_dmic snd_sof_pci snd_sof_intel_byt snd_sof_intel_ipc snd_sof_intel_hda_common snd_soc_hdac_hda snd_sof_xtensa_dsp snd_sof_intel_hda snd_sof snd_hda_ext_core
[ 4737.292990]  snd_soc_acpi_intel_match snd_soc_acpi ledtrig_audio snd_soc_core snd_compress ac97_bus snd_pcm_dmaengine snd_hda_intel snd_intel_dspcfg snd_hda_codec iwlmvm snd_hda_core nls_iso8859_1 snd_hwdep coretemp snd_pcm mac80211 kvm_intel mei_hdcp snd_seq_midi snd_seq_midi_event i915 libarc4 kvm snd_rawmidi crct10dif_pclmul ghash_clmulni_intel snd_seq btusb i2c_algo_bit btrtl snd_seq_device drm_kms_helper iwlwifi btbcm snd_timer aesni_intel btintel cec bluetooth snd crypto_simd cryptd glue_helper input_leds fb_sys_fops joydev ecdh_generic syscopyarea mei_me msi_wmi cfg80211 soundcore ecc sysfillrect intel_cstate sparse_keymap wmi_bmof sysimgblt ee1004 serio_raw mei mac_hid soc_button_array acpi_pad acpi_tad sch_fq_codel msr parport_pc ppdev lp parport drm efi_pstore ip_tables x_tables autofs4 hid_generic usbhid crc32_pclmul psmouse intel_lpss_pci r8169 ahci nvme i2c_i801 intel_lpss i2c_smbus idma64 realtek libahci nvme_core virt_dma i2c_hid hid wmi video [last unloaded: tempesta_lib]
[ 4737.293030] CPU: 8 PID: 15432 Comm: python3 Kdump: loaded Tainted: P        W  OE     5.10.35.tfw-04d37a1 #1
[ 4737.293030] Hardware name: Micro-Star International Co., Ltd. GF63 Thin 11UC/MS-16R6, BIOS E16R6IMS.10D 06/23/2022
[ 4737.293032] RIP: 0010:sk_stream_kill_queues+0x114/0x130
[ 4737.293033] Code: 83 48 01 00 00 85 c0 75 21 85 f6 75 23 5b 41 5c 5d c3 48 89 df e8 7c f5 fe ff 8b 83 48 01 00 00 8b b3 00 01 00 00 85 c0 74 df <0f> 0b 85 f6 74 dd 0f 0b 5b 41 5c 5d c3 0f 0b eb a8 66 66 2e 0f 1f
[ 4737.293034] RSP: 0018:ffffabaa00334e48 EFLAGS: 00010282
[ 4737.293035] RAX: 00000000fffffff7 RBX: ffff9a24d8a58000 RCX: 0000000000000007
[ 4737.293036] RDX: 0000000000000020 RSI: 0000000000000009 RDI: ffff9a24d8a580b0
[ 4737.293037] RBP: ffffabaa00334e58 R08: 0000000000000000 R09: 0000000000000000
[ 4737.293037] R10: ffff9a257fbd5d00 R11: ffffabaa00334d28 R12: ffff9a24d8a580b0
[ 4737.293037] R13: ffff9a24d8a580d0 R14: 0000000000000000 R15: ffff9a24d8a58000
[ 4737.293038] FS:  00007f11e33b51c0(0000) GS:ffff9a256fa00000(0000) knlGS:0000000000000000
[ 4737.293039] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 4737.293040] CR2: 0000345c01e54000 CR3: 000000027f008001 CR4: 0000000000770ee0
[ 4737.293040] PKRU: 55555554
[ 4737.293041] Call Trace:
[ 4737.293043]  <IRQ>
[ 4737.293047]  inet_csk_destroy_sock+0x64/0x150
[ 4737.293059]  ss_do_close+0x2e4/0x450 [tempesta_fw]
[ 4737.293065]  __sk_close_locked+0x16/0xb0 [tempesta_fw]
[ 4737.293071]  ss_tx_action+0x30c/0x670 [tempesta_fw]
[ 4737.293073]  ? process_backlog+0xdb/0x160
[ 4737.293075]  net_tx_action+0x9c/0x250
[ 4737.293077]  __do_softirq+0xd9/0x291
[ 4737.293078]  asm_call_irq_on_stack+0xf/0x20
[ 4737.293079]  </IRQ>
[ 4737.293081]  do_softirq_own_stack+0x3d/0x50
[ 4737.293083]  do_softirq.part.0+0x46/0x50
[ 4737.293084]  __local_bh_enable_ip+0x50/0x60
[ 4737.293085]  ip_finish_output2+0x1ab/0x590
[ 4737.293087]  ? __cgroup_bpf_run_filter_skb+0x3c3/0x3d0
[ 4737.293088]  __ip_finish_output+0xd8/0x220
[ 4737.293089]  ip_finish_output+0x2d/0xb0
[ 4737.293090]  ip_output+0x7a/0x100
[ 4737.293091]  ? __ip_finish_output+0x220/0x220
[ 4737.293092]  ip_local_out+0x3d/0x50
[ 4737.293093]  __ip_queue_xmit+0x1b3/0x480
[ 4737.293094]  ip_queue_xmit+0x15/0x20
[ 4737.293096]  __tcp_transmit_skb+0xa20/0xbe0
[ 4737.293098]  tcp_write_xmit+0x2cc/0x1210
[ 4737.293099]  __tcp_push_pending_frames+0x37/0x100
[ 4737.293100]  tcp_send_fin+0x4e/0x190
[ 4737.293101]  tcp_close+0x33c/0x4e0
[ 4737.293103]  tls_sk_proto_close+0x104/0x2d0 [tls]
[ 4737.293104]  ? __cgroup_bpf_run_filter_sk+0xa0/0x100
[ 4737.293106]  inet_release+0x4b/0x90
[ 4737.293108]  __sock_release+0x42/0xb0
[ 4737.293109]  sock_close+0x15/0x20
[ 4737.293111]  __fput+0xa9/0x260
[ 4737.293111]  ____fput+0xe/0x10
[ 4737.293114]  task_work_run+0x70/0xb0
[ 4737.293115]  exit_to_user_mode_prepare+0x14b/0x170
[ 4737.293117]  syscall_exit_to_user_mode+0x2d/0x150
[ 4737.293118]  do_syscall_64+0x45/0x90
[ 4737.293119]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
[ 4737.293121] RIP: 0033:0x7f11e34caf8b
[ 4737.293122] Code: 03 00 00 00 0f 05 48 3d 00 f0 ff ff 77 41 c3 48 83 ec 18 89 7c 24 0c e8 73 ba f7 ff 8b 7c 24 0c 41 89 c0 b8 03 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 35 44 89 c7 89 44 24 0c e8 c1 ba f7 ff 8b 44
[ 4737.293123] RSP: 002b:00007ffd177109b0 EFLAGS: 00000293 ORIG_RAX: 0000000000000003
[ 4737.293123] RAX: 0000000000000000 RBX: 00007f11de915a80 RCX: 00007f11e34caf8b
[ 4737.293124] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000004
[ 4737.293124] RBP: 0000000000000004 R08: 0000000000000000 R09: 0000000000000000
[ 4737.293125] R10: 0000557acbf1f0c0 R11: 0000000000000293 R12: 0000557acbf3b0d0
[ 4737.293125] R13: 00007f11e0869200 R14: 0000000000000000 R15: 0000557acbcff300
[ 4737.293127] ---[ end trace f96ad0dfe52204d5 ]---

and second

[ 4737.410348] ------------[ cut here ]------------
[ 4737.410357] WARNING: CPU: 10 PID: 15619 at net/ipv4/af_inet.c:156 inet_sock_destruct+0x1bb/0x1c0
[ 4737.410358] Modules linked in: tempesta_fw(OE) tempesta_db(OE) tempesta_tls(OE) tempesta_lib(OE) nfnetlink_queue nfnetlink_log tls sha256_ssse3 sha512_ssse3 nvme_tcp nvme_fabrics nf_conntrack_netlink xfrm_user xfrm_algo xt_addrtype br_netfilter nft_masq vhost_vsock vmw_vsock_virtio_transport_common vhost vhost_iotlb vsock xt_CHECKSUM xt_MASQUERADE xt_conntrack ipt_REJECT nf_reject_ipv4 xt_tcpudp nft_compat nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 rfcomm nft_counter nf_tables libcrc32c nfnetlink bridge stp llc cmac algif_hash algif_skcipher af_alg bnep overlay binfmt_misc snd_soc_skl_hda_dsp snd_soc_hdac_hdmi snd_hda_codec_hdmi snd_hda_codec_realtek snd_hda_codec_generic nvidia_uvm(POE) nvidia_drm(POE) nvidia_modeset(POE) nvidia(POE) intel_rapl_msr intel_rapl_common x86_pkg_temp_thermal intel_powerclamp snd_soc_dmic snd_sof_pci snd_sof_intel_byt snd_sof_intel_ipc snd_sof_intel_hda_common snd_soc_hdac_hda snd_sof_xtensa_dsp snd_sof_intel_hda snd_sof snd_hda_ext_core
[ 4737.410389]  snd_soc_acpi_intel_match snd_soc_acpi ledtrig_audio snd_soc_core snd_compress ac97_bus snd_pcm_dmaengine snd_hda_intel snd_intel_dspcfg snd_hda_codec iwlmvm snd_hda_core nls_iso8859_1 snd_hwdep coretemp snd_pcm mac80211 kvm_intel mei_hdcp snd_seq_midi snd_seq_midi_event i915 libarc4 kvm snd_rawmidi crct10dif_pclmul ghash_clmulni_intel snd_seq btusb i2c_algo_bit btrtl snd_seq_device drm_kms_helper iwlwifi btbcm snd_timer aesni_intel btintel cec bluetooth snd crypto_simd cryptd glue_helper input_leds fb_sys_fops joydev ecdh_generic syscopyarea mei_me msi_wmi cfg80211 soundcore ecc sysfillrect intel_cstate sparse_keymap wmi_bmof sysimgblt ee1004 serio_raw mei mac_hid soc_button_array acpi_pad acpi_tad sch_fq_codel msr parport_pc ppdev lp parport drm efi_pstore ip_tables x_tables autofs4 hid_generic usbhid crc32_pclmul psmouse intel_lpss_pci r8169 ahci nvme i2c_i801 intel_lpss i2c_smbus idma64 realtek libahci nvme_core virt_dma i2c_hid hid wmi video [last unloaded: tempesta_lib]
[ 4737.410435] CPU: 10 PID: 15619 Comm: sysctl Kdump: loaded Tainted: P        W  OE     5.10.35.tfw-04d37a1 #1
[ 4737.410436] Hardware name: Micro-Star International Co., Ltd. GF63 Thin 11UC/MS-16R6, BIOS E16R6IMS.10D 06/23/2022
[ 4737.410438] RIP: 0010:inet_sock_destruct+0x1bb/0x1c0
[ 4737.410439] Code: e8 da e3 ef ff e9 6f ff ff ff 0f 0b eb b4 0f 0b 41 8b 84 24 4c 01 00 00 85 c0 74 8e 0f 0b 41 8b 94 24 48 01 00 00 85 d2 74 8c <0f> 0b eb 88 90 0f 1f 44 00 00 55 48 89 e5 41 54 41 89 f4 53 48 89
[ 4737.410440] RSP: 0018:ffffabaa0459bae8 EFLAGS: 00010282
[ 4737.410441] RAX: 0000000000000000 RBX: ffff9a24d8a580b0 RCX: 0000000000400025
[ 4737.410442] RDX: 00000000fffffff7 RSI: 0000000000000009 RDI: ffff9a24d8a580b0
[ 4737.410442] RBP: ffffabaa0459baf8 R08: ffff9a24d8a58080 R09: ffffffff94c6d800
[ 4737.410443] R10: ffff9a23ff508300 R11: 0000000000000001 R12: ffff9a24d8a58000
[ 4737.410443] R13: ffff9a2376008d60 R14: ffff9a23151d9890 R15: 0000000000000000
[ 4737.410444] FS:  00007f8280bf0740(0000) GS:ffff9a256fa80000(0000) knlGS:0000000000000000
[ 4737.410445] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 4737.410446] CR2: 00005607f7b8a6d8 CR3: 000000034e73a006 CR4: 0000000000770ee0
[ 4737.410446] PKRU: 55555554
[ 4737.410447] Call Trace:
[ 4737.410452]  __sk_destruct+0x2b/0x1d0
[ 4737.410453]  sk_destruct+0x4a/0x50
[ 4737.410454]  __sk_free+0x30/0xc0
[ 4737.410455]  sk_free+0x26/0x40
[ 4737.410470]  tfw_cli_conn_release+0x70/0x90 [tempesta_fw]
[ 4737.410477]  tfw_tls_conn_dtor+0x134/0x1b0 [tempesta_fw]
[ 4737.410482]  tfw_http_conn_msg_free+0x8f/0xa0 [tempesta_fw]
[ 4737.410488]  tfw_http_conn_release+0x201/0x270 [tempesta_fw]
[ 4737.410492]  tfw_connection_release+0x2b/0x50 [tempesta_fw]
[ 4737.410498]  tfw_sock_srv_disconnect+0x11a/0x190 [tempesta_fw]
[ 4737.410504]  ? tfw_sock_srv_disconnect+0x190/0x190 [tempesta_fw]
[ 4737.410509]  tfw_sock_srv_disconnect_srv+0x52/0x80 [tempesta_fw]
[ 4737.410512]  ? _cond_resched+0x19/0x30
[ 4737.410518]  tfw_sg_for_each_srv+0x91/0xf0 [tempesta_fw]
[ 4737.410524]  tfw_sock_srv_stop+0x152/0x1b0 [tempesta_fw]
[ 4737.410529]  tfw_mods_stop+0x38/0xb0 [tempesta_fw]
[ 4737.410535]  tfw_ctlfn_state_io+0x129/0x340 [tempesta_fw]
[ 4737.410540]  ? tfw_cleanup+0x30/0x30 [tempesta_fw]
[ 4737.410543]  proc_sys_call_handler+0x13f/0x240
[ 4737.410544]  proc_sys_write+0x13/0x20
[ 4737.410546]  new_sync_write+0x117/0x1b0
[ 4737.410548]  vfs_write+0x185/0x250
[ 4737.410549]  ksys_write+0x67/0xe0
[ 4737.410550]  __x64_sys_write+0x1a/0x20
[ 4737.410552]  do_syscall_64+0x38/0x90
[ 4737.410553]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
[ 4737.410554] RIP: 0033:0x7f8280d07887
[ 4737.410555] Code: 10 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24
[ 4737.410556] RSP: 002b:00007fff9b6edb68 EFLAGS: 00000246 ORIG_RAX: 0000000000000001
[ 4737.410556] RAX: ffffffffffffffda RBX: 00005607f7b86560 RCX: 00007f8280d07887
[ 4737.410557] RDX: 0000000000000005 RSI: 00005607f7b865a0 RDI: 0000000000000004
[ 4737.410558] RBP: 00005607f7b886d0 R08: 0000000000000010 R09: 00005607f7b886d0
[ 4737.410558] R10: 0000000000000077 R11: 0000000000000246 R12: 0000000000000005
[ 4737.410559] R13: 0000000000000005 R14: 00007f8280e09b80 R15: 00007f8280e09a00
[ 4737.410560] ---[ end trace f96ad0dfe52204d7 ]---

Testing

Tests example

[EvgeniiMekhanik]

#2168 fixed in this PR

[EvgeniiMekhanik]

This warning can be easy reproduced on the new master branch if tfw_tls_encrypt fails with -ENOMEM in ss_add_overhead.