From 63976456f8bccbdd89e5c410d937963f8b6f06b5 Mon Sep 17 00:00:00 2001 From: Lee Penkman Date: Sat, 9 Aug 2025 16:16:54 +1200 Subject: [PATCH] Add SQLite support and R2 storage utilities --- .gitignore | 2 ++ README.md | 10 ++++-- app.yaml | 81 ------------------------------------------ appengine_config.py | 4 --- database.py | 46 ++++++++++++++++++++++++ fixtures.py | 10 +++--- index.yaml | 44 ----------------------- init_db.py | 6 ++++ main.py | 19 ++++++---- requirements.txt | 77 ++++----------------------------------- storage.py | 15 ++++++++ tests/system_test.py | 35 ++++++------------ tests/test_database.py | 13 +++++++ tests/test_storage.py | 18 ++++++++++ 14 files changed, 142 insertions(+), 238 deletions(-) delete mode 100644 app.yaml delete mode 100644 appengine_config.py create mode 100644 database.py delete mode 100644 index.yaml create mode 100644 init_db.py create mode 100644 storage.py create mode 100644 tests/test_database.py create mode 100644 tests/test_storage.py diff --git a/.gitignore b/.gitignore index 60bb349..c45c2a1 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,5 @@ nosetests.xml .DS_Store sellerinfo.py static/js/templates.js +.venv/ +wordsmashing.db diff --git a/README.md b/README.md index aa64f89..c0becdf 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,14 @@ wordsmashing ============ -WordSmashing.com an online word search puzzle game and app +WordSmashing.com an online word search puzzle game and app. -Word Smashing uses Twitter Bootstrap, JQuery and Python running on the Google app engine. +The application now runs using a local SQLite database and can upload files to a Cloudflare R2 bucket using the S3 API. -dependencies are managed through pip ```pip install -r requirements.txt``` +Dependencies are managed through pip: + +``` +pip install -r requirements.txt +``` diff --git a/app.yaml b/app.yaml deleted file mode 100644 index f98d24b..0000000 --- a/app.yaml +++ /dev/null @@ -1,81 +0,0 @@ -runtime: python27 -api_version: 1 -threadsafe: yes - -default_expiration: "300d 5h" - -handlers: -- url: /manifest.webapp - static_files: manifest/manifest.json - upload: manifest/manifest.json - http_headers: - Content-Type: application/x-web-app-manifest+json - -- url: /manifest-noads.webapp - static_files: manifest/manifest-noads.json - upload: manifest/manifest-noads.json - http_headers: - Content-Type: application/x-web-app-manifest+json - -- url: /static/* - static_dir: static - http_headers: - Vary: Accept-Encoding -- url: /transient/* - static_dir: transient - http_headers: - Vary: Accept-Encoding - expiration: "0d 1h" - -- url: /favicon.ico - static_files: static/favicon.ico - upload: static/favicon.ico - -- url: /robots.txt - static_files: transient/robots.txt - upload: transient/robots.txt - -- url: /apple-touch-icon.png - static_files: static/img/apple-touch-icon.png - upload: static/img/apple-touch-icon.png - -- url: /BingSiteAuth.xml - static_files: static/BingSiteAuth.xml - upload: static/BingSiteAuth.xml -- url: /channel.html - static_files: static/channel.html - upload: static/channel.html - -- url: /gameon/static/* - static_dir: gameon/static - http_headers: - Vary: Accept-Encoding -- url: .* - script: main.app - -libraries: -- name: webapp2 - version: "2.5.2" -- name: jinja2 - version: "2.6" - -instance_class: F1 -automatic_scaling: - max_pending_latency: 15s -# automatic_scaling: -# min_idle_instances: 2 -# max_pending_latency: 4.5s - -inbound_services: -- warmup - -builtins: -- appstats: on - -skip_files: -- ^(.*/)?#.*#$ -- ^(.*/)?.*~$ -- ^(.*/)?.*\.py[co]$ -- ^(.*/)?.*/RCS/.*$ -- ^(.*/)?\..*$ -- ^node_modules.*$ diff --git a/appengine_config.py b/appengine_config.py deleted file mode 100644 index 71ffe0c..0000000 --- a/appengine_config.py +++ /dev/null @@ -1,4 +0,0 @@ -def webapp_add_wsgi_middleware(app): - from google.appengine.ext.appstats import recording - app = recording.appstats_wsgi_middleware(app) - return app \ No newline at end of file diff --git a/database.py b/database.py new file mode 100644 index 0000000..de11d87 --- /dev/null +++ b/database.py @@ -0,0 +1,46 @@ +import os +import sqlite3 +from contextlib import contextmanager + +DB_PATH = os.environ.get('DATABASE_URL', os.path.join(os.path.dirname(__file__), 'wordsmashing.db')) + + +def init_db(): + """Create required tables if they do not exist.""" + with sqlite3.connect(DB_PATH) as conn: + conn.execute( + 'CREATE TABLE IF NOT EXISTS page_views (path TEXT PRIMARY KEY, count INTEGER NOT NULL)' + ) + conn.commit() + + +@contextmanager +def get_db(): + conn = sqlite3.connect(DB_PATH) + try: + yield conn + finally: + conn.close() + + +def record_page_view(path): + with get_db() as conn: + cur = conn.cursor() + cur.execute( + 'INSERT INTO page_views(path, count) VALUES(?, 1) ' + 'ON CONFLICT(path) DO UPDATE SET count=count+1', + (path,), + ) + conn.commit() + + +def get_page_views(): + with get_db() as conn: + cur = conn.cursor() + cur.execute('SELECT path, count FROM page_views') + return dict(cur.fetchall()) + + +def get_page_view(path): + return get_page_views().get(path, 0) + diff --git a/fixtures.py b/fixtures.py index e190635..b3aacd1 100644 --- a/fixtures.py +++ b/fixtures.py @@ -206,24 +206,24 @@ def set_hardness(self, h): # 2 more ] -for i in xrange(len(EASY_LEVELS)): +for i in range(len(EASY_LEVELS)): EASY_LEVELS[i].set_hardness(i) -for i in xrange(len(MEDIUM_LEVELS)): +for i in range(len(MEDIUM_LEVELS)): MEDIUM_LEVELS[i].set_hardness(i + len(EASY_LEVELS)) MEDIUM_LEVELS[i].difficulty = MEDIUM -for i in xrange(len(HARD_LEVELS)): +for i in range(len(HARD_LEVELS)): HARD_LEVELS[i].set_hardness(i + len(EASY_LEVELS) + len(MEDIUM_LEVELS)) HARD_LEVELS[i].difficulty = HARD -for i in xrange(len(EXPERT_LEVELS)): +for i in range(len(EXPERT_LEVELS)): EXPERT_LEVELS[i].set_hardness(i + len(EASY_LEVELS) + len(MEDIUM_LEVELS) + len(HARD_LEVELS)) EXPERT_LEVELS[i].difficulty = EXPERT LEVELS = EASY_LEVELS + MEDIUM_LEVELS + HARD_LEVELS + EXPERT_LEVELS -for i in xrange(len(LEVELS)): +for i in range(len(LEVELS)): LEVELS[i].id = i + 1 diff --git a/index.yaml b/index.yaml deleted file mode 100644 index bb97432..0000000 --- a/index.yaml +++ /dev/null @@ -1,44 +0,0 @@ -indexes: - -# AUTOGENERATED - -# This index.yaml is automatically updated whenever the dev_appserver -# detects that a new type of query is run. If you want to manage the -# index.yaml file manually, remove the above marker line (the line -# saying "# AUTOGENERATED"). If you want to manage some indexes -# manually, move them above the marker line. The index.yaml file is -# automatically uploaded to Cloud Datastore when you next deploy -# your application using 'gcloud app deploy' or create the indexes -# using 'gcloud datastore indexes create'. - -- kind: HighScore - properties: - - name: difficulty - - name: timedMode - - name: user - - name: score - direction: desc - -- kind: HighScore - properties: - - name: difficulty - - name: user - - name: score - direction: desc - -- kind: HighScore - properties: - - name: user - - name: difficulty - -- kind: HighScore - properties: - - name: user - - name: difficulty - - name: score - -- kind: HighScore - properties: - - name: user - - name: difficulty - direction: desc diff --git a/init_db.py b/init_db.py new file mode 100644 index 0000000..49a60df --- /dev/null +++ b/init_db.py @@ -0,0 +1,6 @@ +"""Initialize the SQLite database.""" +import database + +if __name__ == '__main__': + database.init_db() + print('Database initialized at', database.DB_PATH) diff --git a/main.py b/main.py index db5c149..12686a7 100644 --- a/main.py +++ b/main.py @@ -4,15 +4,13 @@ import json import urllib -from google.appengine.ext import ndb import logging import webapp2 import jinja2 import fixtures -from gameon import gameon -from gameon.gameon_utils import GameOnUtils from ws import ws +import database FACEBOOK_APP_ID = "138831849632195" @@ -25,6 +23,8 @@ loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), extensions=['jinja2.ext.autoescape']) +database.init_db() + class BaseHandler(webapp2.RequestHandler): @@ -37,7 +37,6 @@ def render(self, view_name, extraParams={}): 'fixtures': fixtures, 'ws': ws, 'json': json, - 'GameOnUtils': GameOnUtils, # 'facebook_app_id': FACEBOOK_APP_ID, # 'glogin_url': users.create_login_url(self.request.uri), # 'glogout_url': users.create_logout_url(self.request.uri), @@ -50,6 +49,7 @@ def render(self, view_name, extraParams={}): template_values.update(extraParams) template = JINJA_ENVIRONMENT.get_template(view_name) + database.record_page_view(self.request.path) self.response.write(template.render(template_values)) @@ -189,12 +189,18 @@ def get(self): self.response.write(template.render(template_values)) +class StatsHandler(BaseHandler): + def get(self): + self.response.headers['Content-Type'] = 'application/json' + self.response.write(json.dumps(database.get_page_views())) + + class SlashMurdererApp(webapp2.RequestHandler): def get(self, url): self.redirect(url) -app = ndb.toplevel(webapp2.WSGIApplication([ +app = webapp2.WSGIApplication([ ('/', MainHandler), ('(.*)/$', SlashMurdererApp), @@ -223,5 +229,6 @@ def get(self, url): ('/buy', BuyHandler), ('/sitemap', SitemapHandler), + ('/stats', StatsHandler), - ] + gameon.routes, debug=ws.debug, config=config)) + ], debug=ws.debug, config=config) diff --git a/requirements.txt b/requirements.txt index d89a77e..20015d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,71 +1,6 @@ -Flask==0.10.1 -Jinja2==2.7.1 -MarkupSafe==0.18 -PIL==1.1.7 -PyRSS2Gen==1.1 -Twisted==13.0.0 -WebOb==1.2.3 -WebTest==2.0.9 -Werkzeug==0.9.4 -altgraph==0.10.2 -bdist-mpkg==0.4.4 -beautifulsoup4==4.2.1 -bonjour-py==0.3 -cssselect==0.9 -itsdangerous==0.23 -lxml==3.2.3 -macholib==1.5.1 -mock==1.0.1 -modulegraph==0.10.4 -nose==1.3.0 -numpy==1.7.1 -py2app==0.7.3 -pyOpenSSL==0.13 -pyobjc-core==2.3.2a0 -pyobjc-framework-AddressBook==2.3.2a0 -pyobjc-framework-AppleScriptKit==2.3.2a0 -pyobjc-framework-AppleScriptObjC==2.3.2a0 -pyobjc-framework-Automator==2.3.2a0 -pyobjc-framework-CFNetwork==2.3.2a0 -pyobjc-framework-CalendarStore==2.3.2a0 -pyobjc-framework-Cocoa==2.3.2a0 -pyobjc-framework-Collaboration==2.3.2a0 -pyobjc-framework-CoreData==2.3.2a0 -pyobjc-framework-CoreLocation==2.3.2a0 -pyobjc-framework-CoreText==2.3.2a0 -pyobjc-framework-DictionaryServices==2.3.2a0 -pyobjc-framework-ExceptionHandling==2.3.2a0 -pyobjc-framework-FSEvents==2.3.2a0 -pyobjc-framework-InputMethodKit==2.3.2a0 -pyobjc-framework-InstallerPlugins==2.3.2a0 -pyobjc-framework-InstantMessage==2.3.2a0 -pyobjc-framework-InterfaceBuilderKit==2.3.2a0 -pyobjc-framework-LatentSemanticMapping==2.3.2a0 -pyobjc-framework-LaunchServices==2.3.2a0 -pyobjc-framework-Message==2.3.2a0 -pyobjc-framework-OpenDirectory==2.3.2a0 -pyobjc-framework-PreferencePanes==2.3.2a0 -pyobjc-framework-PubSub==2.3.2a0 -pyobjc-framework-QTKit==2.3.2a0 -pyobjc-framework-Quartz==2.3.2a0 -pyobjc-framework-ScreenSaver==2.3.2a0 -pyobjc-framework-ScriptingBridge==2.3.2a0 -pyobjc-framework-SearchKit==2.3.2a0 -pyobjc-framework-ServerNotification==2.3.2a0 -pyobjc-framework-ServiceManagement==2.3.2a0 -pyobjc-framework-SyncServices==2.3.2a0 -pyobjc-framework-SystemConfiguration==2.3.2a0 -pyobjc-framework-WebKit==2.3.2a0 -pyobjc-framework-XgridFoundation==2.3.2a0 -pyquery==1.2.6 -python-dateutil==2.1 -selenium==2.36.0 -six==1.3.0 -splinter==0.5.4 -unittest2==0.5.1 -vboxapi==1.0 -virtualenv==1.10.1 -waitress==0.8.7 -wsgiref==0.1.2 -xattr==0.6.4 -zope.interface==4.0.5 +webapp2==2.5.2 +Jinja2>=3.1 +WebTest>=3.0.0 +boto3>=1.28 +moto>=4.1 +pytest>=7.0 diff --git a/storage.py b/storage.py new file mode 100644 index 0000000..cb53df8 --- /dev/null +++ b/storage.py @@ -0,0 +1,15 @@ +import os +import boto3 + + +def upload_file_to_r2(file_path, bucket, key, endpoint_url=None): + """Upload a file to an R2 bucket using S3 API.""" + session = boto3.session.Session() + client = session.client( + 's3', + endpoint_url=endpoint_url or os.getenv('R2_ENDPOINT_URL'), + aws_access_key_id=os.getenv('R2_ACCESS_KEY_ID'), + aws_secret_access_key=os.getenv('R2_SECRET_ACCESS_KEY'), + region_name=os.getenv('R2_REGION', 'us-east-1'), + ) + client.upload_file(file_path, bucket, key) diff --git a/tests/system_test.py b/tests/system_test.py index 73f910a..05ff15d 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -14,20 +14,11 @@ import unittest import os -from google.appengine.ext import testbed - - -# import boilerplate -# from boilerplate import models -# from boilerplate import routes -# from boilerplate import routes as boilerplate_routes -# from boilerplate import config as boilerplate_config -# from boilerplate.lib import utils -# from boilerplate.lib import captcha -# from boilerplate.lib import i18n import yaml import main import webtest +import database +import json # setting HTTP_HOST in extra_environ parameter for TestApp is not enough for taskqueue stub os.environ['HTTP_HOST'] = 'localhost' @@ -46,18 +37,8 @@ def setUp(self): # boilerplate_routes.add_routes(self.app) # self.testapp = webtest.TestApp(self.app, extra_environ={'REMOTE_ADDR' : '127.0.0.1'}) - # activate GAE stubs - self.testbed = testbed.Testbed() - self.testbed.activate() - self.testbed.init_datastore_v3_stub() - self.testbed.init_memcache_stub() - self.testbed.init_urlfetch_stub() - self.testbed.init_taskqueue_stub() - self.testbed.init_mail_stub() - self.mail_stub = self.testbed.get_stub(testbed.MAIL_SERVICE_NAME) - self.taskqueue_stub = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) - self.testbed.init_user_stub() - + os.environ['DATABASE_URL'] = ':memory:' + database.init_db() self.headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) Version/6.0 Safari/536.25', 'Accept-Language': 'en_US'} self.app = webtest.TestApp(main.app) @@ -69,7 +50,7 @@ def setUp(self): # self.app.config['contact_recipient'] = "support-testapp@example.com" def tearDown(self): - self.testbed.deactivate() + pass class WebsiteUnitTest(AppTest): @@ -89,3 +70,9 @@ def test_homepage(self): response = self.app.get('/') self.assertEqual(response.status_int, 200) self.assertTrue(response.html()) + + def test_stats_endpoint(self): + self.app.get('/') + response = self.app.get('/stats') + data = json.loads(response.body) + self.assertEqual(data.get('/'), 1) diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..8752aff --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,13 @@ +import os +import database + + +def setup_function(_): + os.environ['DATABASE_URL'] = ':memory:' + database.init_db() + + +def test_record_page_view(): + database.record_page_view('/test') + database.record_page_view('/test') + assert database.get_page_view('/test') == 2 diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..b9ad5e3 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,18 @@ +import os +from moto import mock_s3 +import boto3 +from storage import upload_file_to_r2 + + +@mock_s3 +def test_upload_file(tmp_path): + os.environ['R2_ACCESS_KEY_ID'] = 'test' + os.environ['R2_SECRET_ACCESS_KEY'] = 'test' + bucket = 'mybucket' + s3 = boto3.client('s3', region_name='us-east-1') + s3.create_bucket(Bucket=bucket) + file_path = tmp_path / 'hello.txt' + file_path.write_text('hi') + upload_file_to_r2(str(file_path), bucket, 'hello.txt', endpoint_url=None) + body = s3.get_object(Bucket=bucket, Key='hello.txt')['Body'].read().decode() + assert body == 'hi'