From a870b4483a259dc600d95c2a42a38d0aa8a976b6 Mon Sep 17 00:00:00 2001 From: Dominic Cerquetti Date: Sun, 26 Jun 2022 20:46:01 -0400 Subject: [PATCH 1/4] fix import for Mapping in later python versions > 3.6ish - change from 'collections' to 'collections.abc' - later python versions changed where this was imported from --- sideboard/config.py | 2 +- sideboard/lib/_utils.py | 3 ++- sideboard/lib/_websockets.py | 2 +- sideboard/lib/sa/_crud.py | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sideboard/config.py b/sideboard/config.py index c78f985..d6a27b2 100755 --- a/sideboard/config.py +++ b/sideboard/config.py @@ -3,7 +3,7 @@ import re from os import unlink -from collections import Sized, Iterable, Mapping +from collections.abc import Sized, Iterable, Mapping from copy import deepcopy from tempfile import NamedTemporaryFile diff --git a/sideboard/lib/_utils.py b/sideboard/lib/_utils.py index 93b3706..3f774bb 100644 --- a/sideboard/lib/_utils.py +++ b/sideboard/lib/_utils.py @@ -5,7 +5,8 @@ from datetime import datetime, date from contextlib import contextmanager from threading import RLock, Condition, current_thread -from collections import Sized, Iterable, Mapping, defaultdict +from collections.abc import Sized, Iterable, Mapping +from collections import defaultdict def is_listy(x): diff --git a/sideboard/lib/_websockets.py b/sideboard/lib/_websockets.py index e47b10e..5b999d3 100644 --- a/sideboard/lib/_websockets.py +++ b/sideboard/lib/_websockets.py @@ -6,7 +6,7 @@ from itertools import count from threading import RLock, Event from datetime import datetime, timedelta -from collections import Mapping, MutableMapping +from collections.abc import Mapping, MutableMapping import six from ws4py.client.threadedclient import WebSocketClient diff --git a/sideboard/lib/sa/_crud.py b/sideboard/lib/sa/_crud.py index b7bf6dc..2711f68 100644 --- a/sideboard/lib/sa/_crud.py +++ b/sideboard/lib/sa/_crud.py @@ -157,7 +157,8 @@ import inspect import collections from copy import deepcopy -from collections import Mapping, defaultdict +from collections.abc import Mapping +from collections import defaultdict from datetime import datetime, date, time from itertools import chain from functools import wraps From c75383e8850c815acf4f808dc66ce545fe15c610 Mon Sep 17 00:00:00 2001 From: Dominic Cerquetti Date: Sun, 26 Jun 2022 20:54:41 -0400 Subject: [PATCH 2/4] ignore any postgres data that shows up in this dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a334441..171bca0 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ distribute-*.tar.gz distribute_setup.py .eggs/ .coverage +db-data/ \ No newline at end of file From 151ed360db69e083f5d2e074cf146762cfde0d4d Mon Sep 17 00:00:00 2001 From: Dominic Cerquetti Date: Sun, 26 Jun 2022 20:58:06 -0400 Subject: [PATCH 3/4] WIP fix Docker support - upgrade to python 3.10 - update many packages for python 3.10 compatibility - rip apart setup.py for uber and ubersystem. I for SURE broke stuff here. - change pavement.py to explicitly install plugin dependencies - remove NPM install stuff from Dockerfile - implement multi-stage dockerfile support for dev vs prod (to mount your own code in the container for easier dev support) - this doesn't quite get a running app, but, it gets as far as cherrypy starting - DO NOT USE THIS without a TON more testing - does allow debugging in PyCharm :) --- .dockerignore | 4 ++ Dockerfile | 94 ++++++++++++----------------------------------- pavement.py | 28 ++++++++------ requirements.txt | 4 +- run_dev_server.sh | 10 +++++ run_server.sh | 6 +++ setup.py | 50 ++++++++++++------------- 7 files changed, 86 insertions(+), 110 deletions(-) create mode 100644 .dockerignore create mode 100644 run_dev_server.sh create mode 100644 run_server.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..dfd507f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +/.gitignore +/.travis.yml +/package-support +db-data/ diff --git a/Dockerfile b/Dockerfile index 95e4d5e..9241dd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,79 +1,33 @@ -FROM python:3.6.14 +FROM python:3.10-slim-bullseye AS builder MAINTAINER RAMS Project "code@magfest.org" -LABEL version.sideboard ="1.0" -WORKDIR /app +LABEL version.sideboard ="1.1" +EXPOSE 8282 +WORKDIR /app/sideboard -# This is actually the least bad way to compose two Dockerfile tech stacks right now. -# The following is copied and pasted from the Node Dockerfile at -# https://github.com/nodejs/docker-node/blob/main/12/buster/Dockerfile -# Update this comment and change the entire copypasta section to upgrade Node version +# libcap-dev and gcc is required for python-prctl (for viewing names of python threads in system tools like htop) +# TODO: need to install npm and nodejs for building static js / gradle stuff -######################################### -# START NODEJS DOCKERFILE COPYPASTA # -# https://github.com/nodejs/docker-node # -######################################### -RUN groupadd --gid 1000 node \ - && useradd --uid 1000 --gid node --shell /bin/bash --create-home node +RUN apt-get update \ + && apt-get install -y \ + libcap-dev gcc \ + git \ + && rm -rf /var/lib/apt/lists/* \ + && pip3 install virtualenv \ + && virtualenv --always-copy /app/env/ \ + && /app/env/bin/pip3 install paver -ENV NODE_VERSION 12.22.3 +# use for production configs. no volume mounting of code/etc +FROM builder as production +COPY . /app/sideboard/ -RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ - && case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - ppc64el) ARCH='ppc64le';; \ - s390x) ARCH='s390x';; \ - arm64) ARCH='arm64';; \ - armhf) ARCH='armv7l';; \ - i386) ARCH='x86';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac \ - # gpg keys listed at https://github.com/nodejs/node#release-keys - && set -ex \ - && for key in \ - 4ED778F539E3634C779C87C6D7062848A1AB005C \ - 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ - 74F12602B6F1C4E913FAA37AD3A89613643B6201 \ - 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ - 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ - C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ - C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ - DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ - A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ - 108F52B48DB57BB0CC439B2997B01419BD92F80A \ - B9E2F5981AA6E0CD28160D9FF13993A75599653C \ - ; do \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" || \ - gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ - && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ - && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ - && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ - && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ - # smoke tests - && node --version \ - && npm --version +RUN cd /app/sideboard/ && \ + /app/env/bin/paver install_deps --env_path=/app/env/ -RUN curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz" \ - && curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ - && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ - && grep " node-v$NODE_VERSION-linux-x64.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ - && tar -xJf "node-v$NODE_VERSION-linux-x64.tar.xz" -C /usr/local --strip-components=1 \ - && rm "node-v$NODE_VERSION-linux-x64.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ - && ln -s /usr/local/bin/node /usr/local/bin/nodejs -################################### -# END NODEJS DOCKERFILE COPYPASTA # -################################### +CMD /bin/bash /app/sideboard/run_server.sh -# required for python-prctl -RUN apt-get update && apt-get install -y libcap-dev && rm -rf /var/lib/apt/lists/* +FROM builder as dev -ADD . /app/ -RUN pip3 install virtualenv \ - && virtualenv --always-copy /app/env \ - && /app/env/bin/pip3 install paver -RUN /app/env/bin/paver install_deps +# use for dev builds. developers: you should mount your local sideboard/ directory (with plugins/etc) in /app/sideboard/ +# then, the startup command will run the paver dependency installations on container startup (instead of image build) -CMD /app/env/bin/python3 /app/sideboard/run_server.py -EXPOSE 8282 +CMD /bin/bash /app/sideboard/run_dev_server.sh \ No newline at end of file diff --git a/pavement.py b/pavement.py index b3006b6..a1cd100 100644 --- a/pavement.py +++ b/pavement.py @@ -75,16 +75,17 @@ def make_venv(): develop_sideboard() -def install_pip_requirements_in_dir(dir_of_requirements_txt): - path_to_pip = __here__ / path('env/bin/pip') +def install_pip_requirements_in_dir(dir_of_requirements_txt, env_path=None): + if env_path is None: + pip = __here__ / path('env/bin/pip') + else: + pip = path(env_path) / path('bin/pip') - print("---- installing dependencies in {} ----" - .format(dir_of_requirements_txt)) + print("---- **PAVER** installing dependencies in {} ----".format(dir_of_requirements_txt))\ - sh('{pip} install -e {dir_of_requirements_txt}' - .format( - pip=path_to_pip, - dir_of_requirements_txt=dir_of_requirements_txt)) + # "-e" means "editable mode" which installs .egg-info in the local dir + sh(f'{pip} install -e {dir_of_requirements_txt}') + sh(f'{pip} install -r {dir_of_requirements_txt}/requirements.txt') def run_setup_py(path): @@ -207,10 +208,13 @@ def create_plugin(options): @task -def install_deps(): - install_pip_requirements_in_dir(__here__) - for pdir in collect_plugin_dirs(): - install_pip_requirements_in_dir(pdir) +@cmdopts([ + ('env_path=', 'e', "override path to virtualenv"), # -e /some/path or --env_path=/some/path + ]) +def install_deps(options): + install_pip_requirements_in_dir(__here__, options.env_path) + for plugin_dir in collect_plugin_dirs(): + install_pip_requirements_in_dir(plugin_dir, options.env_path) @task diff --git a/requirements.txt b/requirements.txt index b777457..12ceeb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,9 @@ SQLAlchemy>=1.1.0 six>=1.5.2 Jinja2>=2.7 rpctools>=0.3.1 -logging_unterpolation>=0.2.0 +# logging_unterpolation>=0.2.1 # this is right and fixes a bug we need, BUT it hasn't been published yet +# instead, we can do this: +git+https://github.com/robdennis/logging_unterpolation@62bbb658493f5f0cf04ad3b82a521b3b330a11f7#egg=logging_unterpolation requests>=2.2.1 paver>=1.2.2 wheel>=0.24.0 diff --git a/run_dev_server.sh b/run_dev_server.sh new file mode 100644 index 0000000..908ee36 --- /dev/null +++ b/run_dev_server.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# run a "dev" server (for local development) + +set -e +cd /app/sideboard/ + +# TODO: need to add 'develop' in here +/app/env/bin/paver install_deps --env_path=/app/env/ + +bash ./run_server.sh \ No newline at end of file diff --git a/run_server.sh b/run_server.sh new file mode 100644 index 0000000..9eacbe1 --- /dev/null +++ b/run_server.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +echo "-----------STARTING UBER SERVER--------------" + +cd /app/sideboard/ +/app/env/bin/python3 sideboard/run_server.py \ No newline at end of file diff --git a/setup.py b/setup.py index 2365747..3df9814 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -import os import platform import sys import os.path @@ -11,45 +10,40 @@ with open(os.path.join(__here__, pkg_name, '_version.py')) as version: exec(version.read()) # __version__ is now defined -req_data = open(os.path.join(__here__, 'requirements.txt')).readlines() -raw_requires = [r.strip() for r in req_data if r.strip() != ''] -# Ugly hack to reconcile pip requirements.txt and setup.py install_requires os_name = os.name sys_platform = sys.platform platform_release = platform.release() implementation_name = sys.implementation.name platform_machine = platform.machine() platform_python_implementation = platform.python_implementation() -requires = [] -for s in reversed(raw_requires): - if ';' in s: - req, env_marker = s.split(';') - if eval(env_marker): - requires.append(s) - else: - requires.append(s) +# -------------- +# Ugly, old hack to reconcile pip requirements.txt and setup.py install_requires +# right now try to get away without this. +# ------------- +# req_data = open(os.path.join(__here__, 'requirements.txt')).readlines() +# raw_requires = [r.strip() for r in req_data if r.strip() != ''] +# requires = [] +# for s in reversed(raw_requires): +# if ';' in s: +# req, env_marker = s.split(';') +# if eval(env_marker): +# requires.append(s) +# else: +# requires.append(s) # testing dependencies -req_data = open(os.path.join(__here__, 'test_requirements.txt')).read() -tests_require = [r.strip() for r in req_data.split() if r.strip() != ''] -tests_require = list(reversed(tests_require)) - -# temporary workaround for a Python 2 CherryPy bug, for which we opened a pull request: -# https://bitbucket.org/cherrypy/cherrypy/pull-request/85/1285-python-2-now-accepts-both-bytestrings/ -if sys.version_info[0] == 2: - requires = ['CherryPy==3.2.2' if 'cherrypy' in r.lower() else r for r in requires] +# req_data = open(os.path.join(__here__, 'test_requirements.txt')).read() +# tests_require = [r.strip() for r in req_data.split() if r.strip() != ''] +# tests_require = list(reversed(tests_require)) if __name__ == '__main__': - setup_requires = {'setup_requires': ['distribute']} if sys.version_info[0] == 2 else {} setup( name=pkg_name, version=__version__, description='Sideboard plugin container.', license='BSD', scripts=[], - install_requires=requires, - tests_require=tests_require, packages=find_packages(), include_package_data=True, package_data={pkg_name: []}, @@ -59,8 +53,10 @@ 'sep = sideboard.sep:run_plugin_entry_point' ] }, - extras_require={ - 'perftrace': ['python-prctl>=1.6.1', 'psutil>=4.3.0'] - }, - **setup_requires + + # extras_require={ + # 'perftrace': ['python-prctl>=1.6.1', 'psutil>=4.3.0'] + # }, + # install_requires=requires, + # tests_require=tests_require, ) From fae6b6b3eabe87fa49a12150afa6b64cf78b41ee Mon Sep 17 00:00:00 2001 From: Dominic Cerquetti Date: Sun, 26 Jun 2022 21:00:13 -0400 Subject: [PATCH 4/4] add docker-compose support for local testing - assumes you have plugins/uber and other plugins populated - this is a good base to build on to work up to kubernetes support --- docker-compose-nginx.yaml | 38 ++++++++++++++++++++++++++++++++++++ docker-compose.dev.yaml | 21 ++++++++++++++++++++ docker-compose.yaml | 41 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 docker-compose-nginx.yaml create mode 100644 docker-compose.dev.yaml create mode 100644 docker-compose.yaml diff --git a/docker-compose-nginx.yaml b/docker-compose-nginx.yaml new file mode 100644 index 0000000..23e5f4c --- /dev/null +++ b/docker-compose-nginx.yaml @@ -0,0 +1,38 @@ +version: '3.9' + +# NOTE: to use this file, you need to set HTTPS_SERVER_HOSTNAME in an "environment" section. +# in uber this is done one level up in docker-compose-config.yaml +# also need to add files to ./nginx/templates + +services: + frontend: + image: nginx:stable-alpine + restart: "always" + depends_on: + - backend + ports: + - "80:80" # warning: if backend service is also listening on 80 on the host, this will conflict and fail to start + - "443:443" + volumes: + - ./nginx/templates:/etc/nginx/templates/:ro + + # NOTE: it is expected that deploys on real servers will override these values in an external downstream config file. + # the two root paths here are the ones you need to use though. they match what's in nginx.conf + # + # DEV NOTE: This gets weird under windows sometimes, be careful if you're hitting errors, could be a windows-specific thing. + # If you have issues, try this from inside WSL2 instead of native windows. + # example1: + # - /etc/ssl/certs/_.uber.org/final-bundle.crt:/__server_bundle.crt:ro + # - /etc/ssl/certs/_.uber.org/_.uber.org.key:/__server.key:ro + # + # example2: (works on the prod server where the config dir is two down from uber_server/ root dir) + # - ../../config/certs/_.uber.org/final-bundle.crt:/__server_bundle.crt:ro + # - ../../config/certs/_.uber.org/_.uber.org.key:/__server.key:ro + # + # NOTE2: anything in templates directory will automatically have envsubst ran against it. + # ENV vars defined in docker-compose can be used in the nginx conf files like this: ${SOME_ENV_VAR} + + # defaults: use self-signed certs so at least the server actually starts + # these files need to exist and should be checked in + - ./self-signed.crt:/__server_bundle.crt:ro + - ./self-signed.key:/__server.key:ro \ No newline at end of file diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml new file mode 100644 index 0000000..7fcada6 --- /dev/null +++ b/docker-compose.dev.yaml @@ -0,0 +1,21 @@ +version: '3.9' + +services: + backend: + # for live-editing code + volumes: + - ./:/app/sideboard/ + + restart: "no" + + ports: + - "8282:8282" + + # option 1: build container every time (decent for development) + build: + context: ./ + dockerfile: Dockerfile + target: dev # if using multi-stage dockerfiles, useful for selection of a particular stage + + # option 2: use a prebuilt image (better for production) + # image: ghcr.io/magfest/sideboard-whatever:dev diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..d2de9be --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,41 @@ +version: '3.9' + +# default settings. it's expected that other YAML files will override these in production +x-postgres-default-vars: &postgres-default-vars + POSTGRES_USER: uber_db_user + POSTGRES_PASSWORD: uber_db_password + POSTGRES_DB: uber_db + +x-http-default-vars: &http-default-vars + HTTPS_SERVER_HOSTNAME: changeme.ubersystem.com + +services: + db: + restart: "always" + image: 'postgres:latest' + + environment: + <<: *postgres-default-vars + ports: + - "5432:5432" + + # to prevent heartache from accidentally deleting the DB, our default is to have the + # volume be served from a local directory. feel free to remove or change this. + volumes: + - ./db-data/:/var/lib/postgresql/data/ + + # our backend uber app. expose via HTTP internally only (no SSL, and not publicly exposed) + backend: + # image: ghcr.io/magfest/ubersystem_prime:v1.0.0.0 # for production, populate this with container builds + restart: "always" + depends_on: + - db + environment: + <<: *postgres-default-vars + <<: *http-default-vars + # example of how to set other env vars + SOME_OTHER_ENV_DB: "some_value" + + # if you want to expose the backend ports directly (not recommended. go through nginx) + # ports: + # - "80:80" # warning: if nginx is also trying to listen on 80, one of the two services will fail to start. \ No newline at end of file