From 22ce9d668cca740c8b1af0fcb3f889a492315439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Andr=C3=A9=20Santoni?= Date: Mon, 10 Dec 2018 09:48:19 +0700 Subject: [PATCH 1/3] Refined nginx config to scale --- cloudinit/mainnet.yml | 31 +++++++++++++++++++++++++------ cloudinit/private.yml | 29 ++++++++++++++++++++++++----- cloudinit/rinkeby.yml | 31 +++++++++++++++++++++++++------ cloudinit/ropsten.yml | 31 +++++++++++++++++++++++++------ 4 files changed, 99 insertions(+), 23 deletions(-) diff --git a/cloudinit/mainnet.yml b/cloudinit/mainnet.yml index d84e088..0ae70b7 100644 --- a/cloudinit/mainnet.yml +++ b/cloudinit/mainnet.yml @@ -22,7 +22,7 @@ mounts: - [ "@@DEVICE@@1", "/datadrive", "ext4", "defaults", "0", "2" ] write_files: - owner: root:root - - path: /lib/systemd/system/geth.service + path: /lib/systemd/system/geth.service content: | [Unit] Description=Geth @@ -38,12 +38,31 @@ write_files: [Install] WantedBy=multi-user.target - owner: root:root - path: /etc/nginx/sites-available/default + path: /etc/nginx/nginx.conf content: | - server { - listen 8545; - location /@@API_KEY@@ { - proxy_pass http://127.0.0.1:8445; + user www-data; + worker_processes auto; + pid /run/nginx.pid; + worker_rlimit_nofile 100000; + + events { + worker_connections 4096; + multi_accept on; + } + + http { + tcp_nopush on; + tcp_nodelay on; + reset_timedout_connection on; + keepalive_timeout 30; + client_body_timeout 10; + send_timeout 2; + + server { + listen 8545; + location /@@API_KEY@@ { + proxy_pass http://127.0.0.1:8445; + } } } packages: diff --git a/cloudinit/private.yml b/cloudinit/private.yml index 386aab3..6c2cb8e 100644 --- a/cloudinit/private.yml +++ b/cloudinit/private.yml @@ -99,12 +99,31 @@ write_files: [Install] WantedBy=multi-user.target - owner: root:root - path: /etc/nginx/sites-available/default + path: /etc/nginx/nginx.conf content: | - server { - listen 8545; - location /@@API_KEY@@ { - proxy_pass http://127.0.0.1:8445; + user www-data; + worker_processes auto; + pid /run/nginx.pid; + worker_rlimit_nofile 100000; + + events { + worker_connections 4096; + multi_accept on; + } + + http { + tcp_nopush on; + tcp_nodelay on; + reset_timedout_connection on; + keepalive_timeout 30; + client_body_timeout 10; + send_timeout 2; + + server { + listen 8545; + location /@@API_KEY@@ { + proxy_pass http://127.0.0.1:8445; + } } } packages: diff --git a/cloudinit/rinkeby.yml b/cloudinit/rinkeby.yml index 4eb91ab..d8b561c 100644 --- a/cloudinit/rinkeby.yml +++ b/cloudinit/rinkeby.yml @@ -22,7 +22,7 @@ mounts: - [ "@@DEVICE@@1", "/datadrive", "ext4", "defaults", "0", "2" ] write_files: - owner: root:root - - path: /lib/systemd/system/geth.service + path: /lib/systemd/system/geth.service content: | [Unit] Description=Geth @@ -38,12 +38,31 @@ write_files: [Install] WantedBy=multi-user.target - owner: root:root - path: /etc/nginx/sites-available/default + path: /etc/nginx/nginx.conf content: | - server { - listen 8545; - location /@@API_KEY@@ { - proxy_pass http://127.0.0.1:8445; + user www-data; + worker_processes auto; + pid /run/nginx.pid; + worker_rlimit_nofile 100000; + + events { + worker_connections 4096; + multi_accept on; + } + + http { + tcp_nopush on; + tcp_nodelay on; + reset_timedout_connection on; + keepalive_timeout 30; + client_body_timeout 10; + send_timeout 2; + + server { + listen 8545; + location /@@API_KEY@@ { + proxy_pass http://127.0.0.1:8445; + } } } packages: diff --git a/cloudinit/ropsten.yml b/cloudinit/ropsten.yml index 8ecfe83..df6f696 100644 --- a/cloudinit/ropsten.yml +++ b/cloudinit/ropsten.yml @@ -22,7 +22,7 @@ mounts: - [ "@@DEVICE@@1", "/datadrive", "ext4", "defaults", "0", "2" ] write_files: - owner: root:root - - path: /lib/systemd/system/geth.service + path: /lib/systemd/system/geth.service content: | [Unit] Description=Geth @@ -38,12 +38,31 @@ write_files: [Install] WantedBy=multi-user.target - owner: root:root - path: /etc/nginx/sites-available/default + path: /etc/nginx/nginx.conf content: | - server { - listen 8545; - location /@@API_KEY@@ { - proxy_pass http://127.0.0.1:8445; + user www-data; + worker_processes auto; + pid /run/nginx.pid; + worker_rlimit_nofile 100000; + + events { + worker_connections 4096; + multi_accept on; + } + + http { + tcp_nopush on; + tcp_nodelay on; + reset_timedout_connection on; + keepalive_timeout 30; + client_body_timeout 10; + send_timeout 2; + + server { + listen 8545; + location /@@API_KEY@@ { + proxy_pass http://127.0.0.1:8445; + } } } packages: From 09f386ff40c0ff351ee6fc62828deab078a31f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Andr=C3=A9=20Santoni?= Date: Mon, 10 Dec 2018 10:22:09 +0700 Subject: [PATCH 2/3] Increase the limit at the systemd level --- cloudinit/mainnet.yml | 8 +++++++- cloudinit/private.yml | 8 +++++++- cloudinit/rinkeby.yml | 8 +++++++- cloudinit/ropsten.yml | 8 +++++++- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/cloudinit/mainnet.yml b/cloudinit/mainnet.yml index 0ae70b7..a5319d4 100644 --- a/cloudinit/mainnet.yml +++ b/cloudinit/mainnet.yml @@ -37,13 +37,18 @@ write_files: ExecStart=/usr/bin/geth --datadir /datadrive --syncmode fast --networkid 1 --rpc --rpcapi net,eth,web3,personal --rpcaddr 127.0.0.1 --rpcport 8445 --rpccorsdomain * --rpcvhosts * [Install] WantedBy=multi-user.target + - owner: root:root + path: /etc/systemd/system/nginx.service.d/limit.conf + content: | + [Service] + LimitNOFILE=65536 - owner: root:root path: /etc/nginx/nginx.conf content: | user www-data; worker_processes auto; pid /run/nginx.pid; - worker_rlimit_nofile 100000; + worker_rlimit_nofile 65536; events { worker_connections 4096; @@ -73,6 +78,7 @@ runcmd: - tar xf geth-linux-amd64-1.8.13-225171a4.tar.gz - cp geth-linux-amd64-1.8.13-225171a4/geth /usr/bin/geth - chown -R blockform:blockform /datadrive + - systemctl daemon-reload - systemctl enable geth - systemctl start geth - systemctl restart nginx diff --git a/cloudinit/private.yml b/cloudinit/private.yml index 6c2cb8e..02fd648 100644 --- a/cloudinit/private.yml +++ b/cloudinit/private.yml @@ -98,13 +98,18 @@ write_files: ExecStart=/usr/bin/faucet [Install] WantedBy=multi-user.target + - owner: root:root + path: /etc/systemd/system/nginx.service.d/limit.conf + content: | + [Service] + LimitNOFILE=65536 - owner: root:root path: /etc/nginx/nginx.conf content: | user www-data; worker_processes auto; pid /run/nginx.pid; - worker_rlimit_nofile 100000; + worker_rlimit_nofile 65536; events { worker_connections 4096; @@ -140,6 +145,7 @@ runcmd: - cat /genesis.json | sed "s/@@SEALER@@/`cat /sealer`/g" > /genesis.json - cat /lib/systemd/system/geth.service | sed "s/@@SEALER@@/`cat /sealer`/g" > /lib/systemd/system/geth.service - geth --nodiscover --datadir /datadrive init /genesis.json + - systemctl daemon-reload - systemctl enable geth - systemctl start geth - systemctl enable faucet diff --git a/cloudinit/rinkeby.yml b/cloudinit/rinkeby.yml index d8b561c..da6e3b2 100644 --- a/cloudinit/rinkeby.yml +++ b/cloudinit/rinkeby.yml @@ -37,13 +37,18 @@ write_files: ExecStart=/usr/bin/geth --datadir /datadrive --syncmode fast --networkid 4 --rpc --rpcapi net,eth,web3,personal --rpcaddr 127.0.0.1 --rpcport 8445 --rpccorsdomain * --rpcvhosts * [Install] WantedBy=multi-user.target + - owner: root:root + path: /etc/systemd/system/nginx.service.d/limit.conf + content: | + [Service] + LimitNOFILE=65536 - owner: root:root path: /etc/nginx/nginx.conf content: | user www-data; worker_processes auto; pid /run/nginx.pid; - worker_rlimit_nofile 100000; + worker_rlimit_nofile 65536; events { worker_connections 4096; @@ -75,6 +80,7 @@ runcmd: - wget https://www.rinkeby.io/rinkeby.json - chown -R blockform:blockform /datadrive - geth --datadir=/datadrive init rinkeby.json + - systemctl daemon-reload - systemctl enable geth - systemctl start geth - systemctl restart nginx diff --git a/cloudinit/ropsten.yml b/cloudinit/ropsten.yml index df6f696..4642d6b 100644 --- a/cloudinit/ropsten.yml +++ b/cloudinit/ropsten.yml @@ -37,13 +37,18 @@ write_files: ExecStart=/usr/bin/geth --datadir /datadrive --syncmode fast --networkid 3 --rpc --rpcapi net,eth,web3,personal --rpcaddr 127.0.0.1 --rpcport 8445 --rpccorsdomain * --rpcvhosts * [Install] WantedBy=multi-user.target + - owner: root:root + path: /etc/systemd/system/nginx.service.d/limit.conf + content: | + [Service] + LimitNOFILE=65536 - owner: root:root path: /etc/nginx/nginx.conf content: | user www-data; worker_processes auto; pid /run/nginx.pid; - worker_rlimit_nofile 100000; + worker_rlimit_nofile 65536; events { worker_connections 4096; @@ -73,6 +78,7 @@ runcmd: - tar xf geth-linux-amd64-1.8.13-225171a4.tar.gz - cp geth-linux-amd64-1.8.13-225171a4/geth /usr/bin/geth - chown -R blockform:blockform /datadrive + - systemctl daemon-reload - systemctl enable geth - systemctl start geth - systemctl restart nginx From 852094f432d022703cddecdfc32a23379bc00010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Andr=C3=A9=20Santoni?= Date: Mon, 10 Dec 2018 10:48:16 +0700 Subject: [PATCH 3/3] Increase limits --- cloudinit/mainnet.yml | 25 +++++++++++++++++++++++++ cloudinit/private.yml | 25 +++++++++++++++++++++++++ cloudinit/rinkeby.yml | 25 +++++++++++++++++++++++++ cloudinit/ropsten.yml | 25 +++++++++++++++++++++++++ 4 files changed, 100 insertions(+) diff --git a/cloudinit/mainnet.yml b/cloudinit/mainnet.yml index a5319d4..ceba7e3 100644 --- a/cloudinit/mainnet.yml +++ b/cloudinit/mainnet.yml @@ -37,6 +37,31 @@ write_files: ExecStart=/usr/bin/geth --datadir /datadrive --syncmode fast --networkid 1 --rpc --rpcapi net,eth,web3,personal --rpcaddr 127.0.0.1 --rpcport 8445 --rpccorsdomain * --rpcvhosts * [Install] WantedBy=multi-user.target + - owner: root:root + path: /etc/security/limits.conf + content: | + * soft nofile 1048576 + * hard nofile 1048576 + root soft nofile 1048576 + root hard nofile 1048576 + - owner: root:root + path: /etc/pam.d/common-session + content: | + session [default=1] pam_permit.so + session requisite pam_deny.so + session required pam_permit.so + session optional pam_umask.so + session required pam_unix.so + session optional pam_systemd.so + session required pam_limits.so + - owner: root:root + path: /etc/sysctl.conf + content: | + fs.file-max = 2097152 + fs.nr_open = 1048576 + net.ipv4.netfilter.ip_conntrack_max = 1048576 + net.nf_conntrack_max = 1048576 + net.core.somaxconn = 1048576 - owner: root:root path: /etc/systemd/system/nginx.service.d/limit.conf content: | diff --git a/cloudinit/private.yml b/cloudinit/private.yml index 02fd648..10ff2f7 100644 --- a/cloudinit/private.yml +++ b/cloudinit/private.yml @@ -98,6 +98,31 @@ write_files: ExecStart=/usr/bin/faucet [Install] WantedBy=multi-user.target + - owner: root:root + path: /etc/security/limits.conf + content: | + * soft nofile 1048576 + * hard nofile 1048576 + root soft nofile 1048576 + root hard nofile 1048576 + - owner: root:root + path: /etc/pam.d/common-session + content: | + session [default=1] pam_permit.so + session requisite pam_deny.so + session required pam_permit.so + session optional pam_umask.so + session required pam_unix.so + session optional pam_systemd.so + session required pam_limits.so + - owner: root:root + path: /etc/sysctl.conf + content: | + fs.file-max = 2097152 + fs.nr_open = 1048576 + net.ipv4.netfilter.ip_conntrack_max = 1048576 + net.nf_conntrack_max = 1048576 + net.core.somaxconn = 1048576 - owner: root:root path: /etc/systemd/system/nginx.service.d/limit.conf content: | diff --git a/cloudinit/rinkeby.yml b/cloudinit/rinkeby.yml index da6e3b2..98a90e2 100644 --- a/cloudinit/rinkeby.yml +++ b/cloudinit/rinkeby.yml @@ -37,6 +37,31 @@ write_files: ExecStart=/usr/bin/geth --datadir /datadrive --syncmode fast --networkid 4 --rpc --rpcapi net,eth,web3,personal --rpcaddr 127.0.0.1 --rpcport 8445 --rpccorsdomain * --rpcvhosts * [Install] WantedBy=multi-user.target + - owner: root:root + path: /etc/security/limits.conf + content: | + * soft nofile 1048576 + * hard nofile 1048576 + root soft nofile 1048576 + root hard nofile 1048576 + - owner: root:root + path: /etc/pam.d/common-session + content: | + session [default=1] pam_permit.so + session requisite pam_deny.so + session required pam_permit.so + session optional pam_umask.so + session required pam_unix.so + session optional pam_systemd.so + session required pam_limits.so + - owner: root:root + path: /etc/sysctl.conf + content: | + fs.file-max = 2097152 + fs.nr_open = 1048576 + net.ipv4.netfilter.ip_conntrack_max = 1048576 + net.nf_conntrack_max = 1048576 + net.core.somaxconn = 1048576 - owner: root:root path: /etc/systemd/system/nginx.service.d/limit.conf content: | diff --git a/cloudinit/ropsten.yml b/cloudinit/ropsten.yml index 4642d6b..c1dd670 100644 --- a/cloudinit/ropsten.yml +++ b/cloudinit/ropsten.yml @@ -37,6 +37,31 @@ write_files: ExecStart=/usr/bin/geth --datadir /datadrive --syncmode fast --networkid 3 --rpc --rpcapi net,eth,web3,personal --rpcaddr 127.0.0.1 --rpcport 8445 --rpccorsdomain * --rpcvhosts * [Install] WantedBy=multi-user.target + - owner: root:root + path: /etc/security/limits.conf + content: | + * soft nofile 1048576 + * hard nofile 1048576 + root soft nofile 1048576 + root hard nofile 1048576 + - owner: root:root + path: /etc/pam.d/common-session + content: | + session [default=1] pam_permit.so + session requisite pam_deny.so + session required pam_permit.so + session optional pam_umask.so + session required pam_unix.so + session optional pam_systemd.so + session required pam_limits.so + - owner: root:root + path: /etc/sysctl.conf + content: | + fs.file-max = 2097152 + fs.nr_open = 1048576 + net.ipv4.netfilter.ip_conntrack_max = 1048576 + net.nf_conntrack_max = 1048576 + net.core.somaxconn = 1048576 - owner: root:root path: /etc/systemd/system/nginx.service.d/limit.conf content: |