From ccb248a0b5591e13af71c66b14e9c5c5236be662 Mon Sep 17 00:00:00 2001 From: Joshua May Date: Tue, 17 Sep 2019 09:46:07 -0700 Subject: [PATCH 1/9] Bump .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4123d6a..f12fe6b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # Custom +.vscode/ venv/ .env From aa98fa4e7f360bc8fa5032694622126363d45a34 Mon Sep 17 00:00:00 2001 From: Joshua May Date: Tue, 17 Sep 2019 11:13:42 -0700 Subject: [PATCH 2/9] Switched to headless Chromium from PhantomJS --- sirius/coding/image_encoding.py | 44 ++++++++++++++++++++++++++---- sirius/coding/test_image_coding.py | 16 ++++++++--- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/sirius/coding/image_encoding.py b/sirius/coding/image_encoding.py index 62625bb..39ee96a 100644 --- a/sirius/coding/image_encoding.py +++ b/sirius/coding/image_encoding.py @@ -110,10 +110,36 @@ def html_to_png(html): driver = None try: caps = {'acceptSslCerts': True} - driver = webdriver.PhantomJS( - 'phantomjs', desired_capabilities=caps, - service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any']) - driver.set_window_size(384, 5) + options = webdriver.ChromeOptions() + options.accept_untrusted_certs = True + options.assume_untrusted_cert_issuer = True + + options.set_headless() + + options.add_argument("--no-sandbox") + options.add_argument("--disable-impl-side-painting") + options.add_argument("--disable-setuid-sandbox") + options.add_argument("--disable-seccomp-filter-sandbox") + options.add_argument("--disable-breakpad") + options.add_argument("--disable-client-side-phishing-detection") + options.add_argument("--disable-cast") + options.add_argument("--disable-cast-streaming-hw-encoding") + options.add_argument("--disable-cloud-import") + options.add_argument("--disable-gpu") + options.add_argument("--disable-popup-blocking") + options.add_argument("--ignore-certificate-errors") + options.add_argument("--disable-session-crashed-bubble") + options.add_argument("--disable-ipv6") + options.add_argument("--allow-http-screen-capture") + options.add_argument('--force-device-scale-factor=1') + + # TODO: how can the we find `chromedriver` on Heroku? + driver = webdriver.Chrome( + chrome_options=options, + desired_capabilities=caps, + service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'] + ) + driver.set_window_size(384, 8) # note that the .html suffix is required to make phantomjs # pick up the mime-type and render correctly. @@ -124,7 +150,15 @@ def html_to_png(html): f.write(html) f.flush() driver.get('file://' + f.name) - data = io.BytesIO(driver.get_screenshot_as_png()) + + total_height = driver.execute_script("return document.body.parentNode.scrollHeight") + driver.set_window_size(384, total_height) + screenshot = driver.get_screenshot_as_png() + + # body = driver.find_element_by_tag_name('body') + # screenshot = body.screenshot_as_png + + data = io.BytesIO(screenshot) return data finally: diff --git a/sirius/coding/test_image_coding.py b/sirius/coding/test_image_coding.py index 43d1fc5..87cd148 100644 --- a/sirius/coding/test_image_coding.py +++ b/sirius/coding/test_image_coding.py @@ -6,21 +6,29 @@ class ImageCase(unittest.TestCase): - def _test_normal_text(self): + def test_normal_text(self): data = image_encoding.html_to_png('') image = Image.open(data) self.assertEquals(image.size[0], 384) - def _test_normal_height(self): + def test_normal_height(self): data = image_encoding.html_to_png( '') image = Image.open(data) + self.assertEquals(image.size[0], 384) self.assertEquals(image.size[1], 100) - def _test_rle(self): + def test_probably_beyond_viewport_height(self): + data = image_encoding.html_to_png( + '') + image = Image.open(data) + self.assertEquals(image.size[0], 384) + self.assertEquals(image.size[1], 10000) + + def test_rle(self): data = image_encoding.html_to_png('') image = Image.open(data) - n_bytes, _ = image_encoding.rle_image(data) + n_bytes, _ = image_encoding.rle_from_bw(image) self.assertEquals(n_bytes, 3072) From 9d74a80efabd9c997aad083db3e685218bb1eafb Mon Sep 17 00:00:00 2001 From: Joshua May Date: Tue, 17 Sep 2019 12:28:54 -0700 Subject: [PATCH 3/9] Docker knows how to run Chrome instead of Phantom --- Dockerfile | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 11c3390..a8e1423 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,14 @@ FROM python:3.7.4-slim-buster WORKDIR /sirius RUN apt-get update -y && \ - mkdir -p /usr/share/man/man1 && \ - mkdir -p /usr/share/man/man7 && \ + apt-get install -y --no-install-recommends \ + curl \ + gnupg + +RUN curl -sS -o - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add \ + && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list + +RUN apt-get update -y && \ apt-get install -y --no-install-recommends \ python3 \ python3-dev \ @@ -14,20 +20,24 @@ RUN apt-get update -y && \ libjpeg-dev \ libpq-dev \ zlib1g-dev \ - bzip2 \ fontconfig \ gcc \ - phantomjs \ - wget \ + google-chrome-stable \ postgresql-11 \ - git - -RUN apt-get autoremove -y - -RUN wget https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 -RUN tar jxf phantomjs-2.1.1-linux-x86_64.tar.bz2 && \ - rm phantomjs-2.1.1-linux-x86_64.tar.bz2 && \ - mv phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/local/bin/phantomjs + unzip \ + git && \ + apt-get autoremove -y + +# Find the latest version for `chromedriver` that matches our installed `google-chrome` & install it to be available on $PATH +RUN COMMON_VERSION=$(google-chrome --version | sed -nre "s/.* ([0-9]+\.[0-9]+\.[0-9]+)/\1/p" | cut -d"." -f1-3) && \ + URL="https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${COMMON_VERSION}" && \ + CHROMEDRIVER_VERSION=$(curl -Ss "${URL}") && \ + CHROMEDRIVER_URL="https://chromedriver.storage.googleapis.com/${CHROMEDRIVER_VERSION}/chromedriver_linux64.zip" && \ + curl "${CHROMEDRIVER_URL}" -o "${HOME}/chromedriver_linux64.zip" && \ + unzip "${HOME}/chromedriver_linux64.zip" -d "${HOME}/" && \ + rm "${HOME}/chromedriver_linux64.zip" && \ + mv -f "${HOME}/chromedriver" /usr/local/bin/chromedriver && \ + chmod 0755 /usr/local/bin/chromedriver RUN pip install --upgrade pip From a2c123e68b7cf4a4aa7f8482bc8487c0e5b075a4 Mon Sep 17 00:00:00 2001 From: Joshua May Date: Tue, 17 Sep 2019 13:00:24 -0700 Subject: [PATCH 4/9] Matching fonts to match PhantomJS's DejaVu Sans usage --- Dockerfile | 1 + sirius/coding/default_template.html | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a8e1423..b45a7c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ RUN apt-get update -y && \ libpq-dev \ zlib1g-dev \ fontconfig \ + fonts-dejavu \ gcc \ google-chrome-stable \ postgresql-11 \ diff --git a/sirius/coding/default_template.html b/sirius/coding/default_template.html index ef38bf3..fcb3b80 100644 --- a/sirius/coding/default_template.html +++ b/sirius/coding/default_template.html @@ -1,9 +1,10 @@ - +