Improving the components scripts

README.md

@@ -21,7 +21,9 @@ For the **quick setup instructions** scroll to the bottom of the page
 * Plugin support to allow content to be built programmatically
 
 ## Requirements
+* git CLI
 * Python
+* make
 * Jinja2
 * Beautifulsoup
 * LXML
@@ -43,13 +45,13 @@ Web pages can either be a made up from a single piece of content, or can be
 generated from lots of little bits we'll call "components". 
 
 The source material used to create (whole) web pages is in the "content/"
-folder 
+folder.
 
 The source material used to build pages from components is stored in the
 "components/" folder. These "generated pages" are constructed from the
-"components" and saved in the folder "_meta/" 
+"components" and saved in the folder "\_meta/".
 
-"pages" in "content/" and "generated pages" in "_meta"/ are then merged with
+"pages" in "content/" and "generated pages" in "\_meta"/ are then merged with
 the appropriate theme template (a "skin" if you like) to generate the resultant
 HTML page in the "output/" folder. You need to specify the default theme name
 in a file called 'config.json'. 
@@ -132,13 +134,13 @@ Components are constructed as follows:
 
 1. Any executable program in the "components/" folder that starts with 3
    digits followed by either an underscore or a hyphen is executed (they are 
-   executed in numerical order). Each one writes its data to the "_meta/"
+   executed in numerical order). Each one writes its data to the "\_meta/"
    folder; either writing an unthemed HTML page or a data file.
 
 1. There are two ready-made component generators (000_collect_data.py and 
    999_mksitemap.py) which are used to build the navigation pages (sitemap,
    category listings and tag listings). 000_collect_data.py generates a set
-   of JSON-encoded data files in "_meta/", and 999_mksitemap.py uses these
+   of JSON-encoded data files in "\_meta/", and 999_mksitemap.py uses these
    data files to construct unthemed HTML files
 
 1. Any other component generator can be written that may modify the
@@ -279,3 +281,23 @@ Then create your website:
 * Run `make`
 * The generated content will be created in `output/`. Load it in a browser
 
+## Uploading to your web server
+There is a script called `upload.py` in the `bits_box` folder. This script
+will FTP the files in the `output/` folder to your web server.
+
+Copy `upload.py` to the top level of your project. 
+
+Set up the configuration for the FTP server in the file `config.json` in the
+top level of your project. The file should look like this:
+
+```json
+{
+    "theme": "womble",
+
+    "ftp_hostname": "ftp.example.com",
+    "ftp_remote_path": "/public_html/test_12345",
+    "ftp_username": "my_username"
+}
+```
+**Note** that the password should not be stored in the config file. `upload.py` 
+will ask to type the password.

bits_box/001_make_homepage.py

@@ -1,13 +1,15 @@
 #!/usr/bin/env python
 
-# An attempt at writing a content generator
-# loads all filenames and prints out some of the content.
-# Don't forget to "chmod +x <filename>"
+"""
+This is an example page generator.
 
-# TODO: Pagination?
+It loads all the files from the `content` directory and generates
+a summary of each one to produce the homepage.
 
+If you don't want this script, delete it or make it non-executable.
+(chmod -x 001_make_homepage.py)
+"""
 
-import json
 import os
 import re
 
@@ -32,7 +34,12 @@
 def main():
     all_content_files = awcm.get_all_filenames(CONTENT_DIR)
 
-    output = ['<title>Home</title>']
+    output = [
+        '<title>Home</title>',
+        '<h2>Autogenerated homepage</h2>',
+        '<p>This page is autogenerated by <code>components/001_make_homepage.py</code></p>',
+        '<p>The content below is a summary of all other pages.</p>'
+    ]
 
     # Ensure they're in the right order
     all_content_files.sort(reverse=True)

bits_box/003_joke.sh

@@ -1,6 +1,11 @@
 #!/bin/bash
 
+# Bare minimum component example
+#  * a <title> is required
+#  * the file must be written to the ../_meta folder
+#  * the script must be executable (chmod +x) to be picked up.
 
-echo "Q: What does a skeleton order at a restaurant?"
-echo "A: Spare ribs!"
-echo " "
+echo "<title>A joke</title>" > ../_meta/a_joke.html
+echo "Q: What does a skeleton order at a restaurant?<br> " >> ../_meta/a_joke.html
+echo "A: Spare ribs!" >> ../_meta/a_joke.html
+echo " " >> ../_meta/a_joke.html

bits_box/998_prevent_directory_listings.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+"""
+This script adds an empty index.html file to any directory that
+does not have an index.html file, to prevent Directory Listings.
+
+As a side effect, it also creates the directory if it doesn't exist.
+
+To use this script
+* chmod +x components/998_prevent_directory_listings.py
+* make
+
+Minor bug:
+  * if an index.html exists in a folder in content/ but not in _meta/, this
+    script will create an index.html, but it will be replaced by the output
+    of AWCM, with the file from content/
+    This should not be an issue.
+"""
+
+import json
+import os
+
+from awcm import awcm
+
+CONTENT_DIR = "../content"
+META_DIR = "../_meta"
+THEMES_DIR = "../themes"
+OUTPUT_PATH = "../output"
+
+INDEX_HTML_CONTENT = """<br>
+"""
+
+
+def create_index_html_files():
+    folder_without_index_file = ['./']
+
+    # Look in both 'content/' nd '_meta/'
+    for source_folder in [CONTENT_DIR, META_DIR]:
+        all_content_files = awcm.get_all_filenames(source_folder)
+
+        for page in all_content_files:
+            if '/' in page:
+                folder, file = page.rsplit("/", 1)
+                if folder not in folder_without_index_file:
+                    if file == 'index.html':
+                        pass  # print(f"No need to add an index.html to {folder} ")
+                    else:
+                        # print(f": Adding index.html to {folder} ")
+                        folder_without_index_file.append(folder)
+
+    # Ensure there are index.html files for al the theme folders too
+    folder_without_index_file.append(os.path.join(OUTPUT_PATH, 'themes'))
+    for theme_name in os.listdir(THEMES_DIR):
+        folder_without_index_file.append(os.path.join(OUTPUT_PATH, 'themes', theme_name))
+
+    # iterate all the folders discovered above and create an index.html file in each one.
+    for folder in folder_without_index_file:
+        new_dir_name = os.path.relpath(os.path.join(OUTPUT_PATH, folder))
+        new_file_name = os.path.join(new_dir_name, 'index.html')
+        if os.path.isfile(new_file_name):
+            print(f"  [INFO] index.html already exists in: {new_dir_name}")
+        else:
+            print(f"  [INFO] Creating index.html in: {new_dir_name}")
+            os.makedirs(new_dir_name, exist_ok=True)
+            with open(new_file_name, "w") as f:
+                f.write(INDEX_HTML_CONTENT)
+
+
+create_index_html_files()

bits_box/upload.py

@@ -1,16 +1,35 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
 Utility to upload files to a remote FTP server
 
+
+History
+--------
 First working version: 19 Dec 2012
 Updated for AWCM (Python2): 12 Dec 2017
+Bugfixes: 16 Dec 2023
+
+
+Usage
+------
+* Add some fields to config.json, as below
+
+    "ftp_hostname": "ftp.example.com",
+    "ftp_remote_path": "/public_html/test_12345",
+    "ftp_username": "my_username"
+
+* python3 upload.py
+
+
 """
 
 import os
 from ftplib import FTP, error_perm
 import getpass
 import json
 
+DRY_RUN = False  # set to True to avoid making changes
+
 
 def read_from_config_file():
     """ Read FTP parameters from the config.json file"""
@@ -22,11 +41,10 @@ def read_from_config_file():
     if 'ftp_username' in list(config.keys()):
         ftp_username = config['ftp_username']
     else:
-        ftp_username = raw_input("FTP username: ")
-    if 'ftp_password' in list(config.keys()):
-        ftp_password = config['ftp_password']
-    else:
-        ftp_password = getpass.getpass("FTP password: ")
+        ftp_username = input("FTP username: ")
+
+    # Always prompt for the password. never save it in config
+    ftp_password = getpass.getpass("FTP password: ")
 
     # Ensure there's a leading slash on the remote path.
     if ftp_path[0] != '/':
@@ -67,20 +85,28 @@ def make_folders_on_target(remote_dir, ftp, folders):
     under remote_dir
 
     """
-    debug = False
+    debug = True
     ftp.cwd(remote_dir)
 
-    for f in folders:
+    if debug:
+        print("Ensure these folders exist:")
+        print(json.dumps(folders, indent=4))
+
+    for folder in folders:
+
         if debug:
-            print("Folder %s" % f)
+            print(f"Testing for existence of Folder {folder} in {remote_dir}")
         try:
-            ftp.cwd(f)
-            ftp.cwd('..')
+            previous_pwd = ftp.pwd()
+            ftp.cwd(folder)
+            ftp.cwd(previous_pwd)
+            if ftp.pwd() != previous_pwd:
+                raise Exception("FAIL: Failed to change back to previous folder")
         except error_perm:
             if debug:
-                print("Currently in %s Cannot cd into %s" % (ftp.pwd(), f))
+                print("Currently in %s Cannot cd into %s" % (ftp.pwd(), folder))
 
-            parent_folder = os.path.normpath(os.path.join(f, '..'))
+            parent_folder = os.path.normpath(os.path.join(folder, '..'))
 
             if debug:
                 print("Trying to make parent folders: %s" % parent_folder)
@@ -90,8 +116,12 @@ def make_folders_on_target(remote_dir, ftp, folders):
                 make_folders_on_target('/' + remote_dir, ftp, [parent_folder])
 
             if debug:
-                print("Finished mk-parent-dir. Try mkd on %s" % f)
-            ftp.mkd(f)
+                print("Finished mk-parent-dir. Try mkd on %s" % folder)
+                print("We're currently in %s" % ftp.pwd())
+            if DRY_RUN:
+                print("DRY_RUN: would make folder %s" % folder)
+            else:
+                ftp.mkd(folder)
 
 
 def ftp_files_to_target(local_dir, remote_dir, ftp, file_list):