diff --git a/README.md b/README.md index 123cc20..450b46d 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,9 @@ For the **quick setup instructions** scroll to the bottom of the page * Plugin support to allow content to be built programmatically ## Requirements +* git CLI * Python +* make * Jinja2 * Beautifulsoup * LXML @@ -43,13 +45,13 @@ Web pages can either be a made up from a single piece of content, or can be generated from lots of little bits we'll call "components". The source material used to create (whole) web pages is in the "content/" -folder +folder. The source material used to build pages from components is stored in the "components/" folder. These "generated pages" are constructed from the -"components" and saved in the folder "_meta/" +"components" and saved in the folder "\_meta/". -"pages" in "content/" and "generated pages" in "_meta"/ are then merged with +"pages" in "content/" and "generated pages" in "\_meta"/ are then merged with the appropriate theme template (a "skin" if you like) to generate the resultant HTML page in the "output/" folder. You need to specify the default theme name in a file called 'config.json'. @@ -132,13 +134,13 @@ Components are constructed as follows: 1. Any executable program in the "components/" folder that starts with 3 digits followed by either an underscore or a hyphen is executed (they are - executed in numerical order). Each one writes its data to the "_meta/" + executed in numerical order). Each one writes its data to the "\_meta/" folder; either writing an unthemed HTML page or a data file. 1. There are two ready-made component generators (000_collect_data.py and 999_mksitemap.py) which are used to build the navigation pages (sitemap, category listings and tag listings). 000_collect_data.py generates a set - of JSON-encoded data files in "_meta/", and 999_mksitemap.py uses these + of JSON-encoded data files in "\_meta/", and 999_mksitemap.py uses these data files to construct unthemed HTML files 1. Any other component generator can be written that may modify the @@ -279,3 +281,23 @@ Then create your website: * Run `make` * The generated content will be created in `output/`. Load it in a browser +## Uploading to your web server +There is a script called `upload.py` in the `bits_box` folder. This script +will FTP the files in the `output/` folder to your web server. + +Copy `upload.py` to the top level of your project. + +Set up the configuration for the FTP server in the file `config.json` in the +top level of your project. The file should look like this: + +```json +{ + "theme": "womble", + + "ftp_hostname": "ftp.example.com", + "ftp_remote_path": "/public_html/test_12345", + "ftp_username": "my_username" +} +``` +**Note** that the password should not be stored in the config file. `upload.py` +will ask to type the password. \ No newline at end of file diff --git a/bits_box/001_make_homepage.py b/bits_box/001_make_homepage.py index 0e6ff28..107d922 100755 --- a/bits_box/001_make_homepage.py +++ b/bits_box/001_make_homepage.py @@ -1,13 +1,15 @@ #!/usr/bin/env python -# An attempt at writing a content generator -# loads all filenames and prints out some of the content. -# Don't forget to "chmod +x " +""" +This is an example page generator. -# TODO: Pagination? +It loads all the files from the `content` directory and generates +a summary of each one to produce the homepage. +If you don't want this script, delete it or make it non-executable. +(chmod -x 001_make_homepage.py) +""" -import json import os import re @@ -32,7 +34,12 @@ def main(): all_content_files = awcm.get_all_filenames(CONTENT_DIR) - output = ['Home'] + output = [ + 'Home', + '

Autogenerated homepage

', + '

This page is autogenerated by components/001_make_homepage.py

', + '

The content below is a summary of all other pages.

' + ] # Ensure they're in the right order all_content_files.sort(reverse=True) diff --git a/bits_box/003_joke.sh b/bits_box/003_joke.sh index 5d363c2..632d56d 100644 --- a/bits_box/003_joke.sh +++ b/bits_box/003_joke.sh @@ -1,6 +1,11 @@ #!/bin/bash +# Bare minimum component example +# * a is required +# * the file must be written to the ../_meta folder +# * the script must be executable (chmod +x) to be picked up. -echo "Q: What does a skeleton order at a restaurant?" -echo "A: Spare ribs!" -echo " " +echo "<title>A joke" > ../_meta/a_joke.html +echo "Q: What does a skeleton order at a restaurant?
" >> ../_meta/a_joke.html +echo "A: Spare ribs!" >> ../_meta/a_joke.html +echo " " >> ../_meta/a_joke.html diff --git a/bits_box/998_prevent_directory_listings.py b/bits_box/998_prevent_directory_listings.py new file mode 100755 index 0000000..0444453 --- /dev/null +++ b/bits_box/998_prevent_directory_listings.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +""" +This script adds an empty index.html file to any directory that +does not have an index.html file, to prevent Directory Listings. + +As a side effect, it also creates the directory if it doesn't exist. + +To use this script +* chmod +x components/998_prevent_directory_listings.py +* make + +Minor bug: + * if an index.html exists in a folder in content/ but not in _meta/, this + script will create an index.html, but it will be replaced by the output + of AWCM, with the file from content/ + This should not be an issue. +""" + +import json +import os + +from awcm import awcm + +CONTENT_DIR = "../content" +META_DIR = "../_meta" +THEMES_DIR = "../themes" +OUTPUT_PATH = "../output" + +INDEX_HTML_CONTENT = """
+""" + + +def create_index_html_files(): + folder_without_index_file = ['./'] + + # Look in both 'content/' nd '_meta/' + for source_folder in [CONTENT_DIR, META_DIR]: + all_content_files = awcm.get_all_filenames(source_folder) + + for page in all_content_files: + if '/' in page: + folder, file = page.rsplit("/", 1) + if folder not in folder_without_index_file: + if file == 'index.html': + pass # print(f"No need to add an index.html to {folder} ") + else: + # print(f": Adding index.html to {folder} ") + folder_without_index_file.append(folder) + + # Ensure there are index.html files for al the theme folders too + folder_without_index_file.append(os.path.join(OUTPUT_PATH, 'themes')) + for theme_name in os.listdir(THEMES_DIR): + folder_without_index_file.append(os.path.join(OUTPUT_PATH, 'themes', theme_name)) + + # iterate all the folders discovered above and create an index.html file in each one. + for folder in folder_without_index_file: + new_dir_name = os.path.relpath(os.path.join(OUTPUT_PATH, folder)) + new_file_name = os.path.join(new_dir_name, 'index.html') + if os.path.isfile(new_file_name): + print(f" [INFO] index.html already exists in: {new_dir_name}") + else: + print(f" [INFO] Creating index.html in: {new_dir_name}") + os.makedirs(new_dir_name, exist_ok=True) + with open(new_file_name, "w") as f: + f.write(INDEX_HTML_CONTENT) + + +create_index_html_files() diff --git a/bits_box/upload.py b/bits_box/upload.py index f5ed99a..b71dfdc 100755 --- a/bits_box/upload.py +++ b/bits_box/upload.py @@ -1,9 +1,26 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Utility to upload files to a remote FTP server + +History +-------- First working version: 19 Dec 2012 Updated for AWCM (Python2): 12 Dec 2017 +Bugfixes: 16 Dec 2023 + + +Usage +------ +* Add some fields to config.json, as below + + "ftp_hostname": "ftp.example.com", + "ftp_remote_path": "/public_html/test_12345", + "ftp_username": "my_username" + +* python3 upload.py + + """ import os @@ -11,6 +28,8 @@ import getpass import json +DRY_RUN = False # set to True to avoid making changes + def read_from_config_file(): """ Read FTP parameters from the config.json file""" @@ -22,11 +41,10 @@ def read_from_config_file(): if 'ftp_username' in list(config.keys()): ftp_username = config['ftp_username'] else: - ftp_username = raw_input("FTP username: ") - if 'ftp_password' in list(config.keys()): - ftp_password = config['ftp_password'] - else: - ftp_password = getpass.getpass("FTP password: ") + ftp_username = input("FTP username: ") + + # Always prompt for the password. never save it in config + ftp_password = getpass.getpass("FTP password: ") # Ensure there's a leading slash on the remote path. if ftp_path[0] != '/': @@ -67,20 +85,28 @@ def make_folders_on_target(remote_dir, ftp, folders): under remote_dir """ - debug = False + debug = True ftp.cwd(remote_dir) - for f in folders: + if debug: + print("Ensure these folders exist:") + print(json.dumps(folders, indent=4)) + + for folder in folders: + if debug: - print("Folder %s" % f) + print(f"Testing for existence of Folder {folder} in {remote_dir}") try: - ftp.cwd(f) - ftp.cwd('..') + previous_pwd = ftp.pwd() + ftp.cwd(folder) + ftp.cwd(previous_pwd) + if ftp.pwd() != previous_pwd: + raise Exception("FAIL: Failed to change back to previous folder") except error_perm: if debug: - print("Currently in %s Cannot cd into %s" % (ftp.pwd(), f)) + print("Currently in %s Cannot cd into %s" % (ftp.pwd(), folder)) - parent_folder = os.path.normpath(os.path.join(f, '..')) + parent_folder = os.path.normpath(os.path.join(folder, '..')) if debug: print("Trying to make parent folders: %s" % parent_folder) @@ -90,8 +116,12 @@ def make_folders_on_target(remote_dir, ftp, folders): make_folders_on_target('/' + remote_dir, ftp, [parent_folder]) if debug: - print("Finished mk-parent-dir. Try mkd on %s" % f) - ftp.mkd(f) + print("Finished mk-parent-dir. Try mkd on %s" % folder) + print("We're currently in %s" % ftp.pwd()) + if DRY_RUN: + print("DRY_RUN: would make folder %s" % folder) + else: + ftp.mkd(folder) def ftp_files_to_target(local_dir, remote_dir, ftp, file_list):