From 425e9b0dd9fbdcd49a611afd9bd97a359f0cd499 Mon Sep 17 00:00:00 2001 From: IDzyre Date: Wed, 21 Jan 2026 12:48:10 -0800 Subject: [PATCH 1/7] partially switched to generate and return --- .setup/CONFIGURE_SUBMITTY.py | 370 +++-------------------------------- .setup/GENERATE_CONFIGS.py | 343 ++++++++++++++++++++++++++++++++ .setup/install_system.sh | 157 ++++++++------- 3 files changed, 456 insertions(+), 414 deletions(-) create mode 100644 .setup/GENERATE_CONFIGS.py diff --git a/.setup/CONFIGURE_SUBMITTY.py b/.setup/CONFIGURE_SUBMITTY.py index f39ff38d631..c928362c78c 100644 --- a/.setup/CONFIGURE_SUBMITTY.py +++ b/.setup/CONFIGURE_SUBMITTY.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - import argparse from collections import OrderedDict import grp @@ -9,50 +7,23 @@ import secrets import shutil import string -import tzlocal import tempfile +from GENERATE_CONFIGS import generate_config def get_uid(user): return pwd.getpwnam(user).pw_uid - def get_gid(user): return pwd.getpwnam(user).pw_gid - def get_ids(user): try: return get_uid(user), get_gid(user) except KeyError: raise SystemExit("ERROR: Could not find user: " + user) - -def get_input(question, default=""): - add = "[{}] ".format(default) if default != "" else "" - user = input("{}: {}".format(question, add)).strip() - if user == "": - user = default - return user - - -class StrToBoolAction(argparse.Action): - """ - Custom action that parses strings to boolean values. All values that come - from bash are strings, and so need to parse that into the appropriate - bool value. - """ - def __init__(self, option_strings, dest, nargs=None, **kwargs): - if nargs is not None: - raise ValueError("nargs not allowed") - super().__init__(option_strings, dest, **kwargs) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, values != '0' and values.lower() != 'false') - - ############################################################################## -# this script must be run by root or sudo if os.getuid() != 0: raise SystemExit('ERROR: This script must be run by root or sudo') @@ -70,30 +41,6 @@ def __call__(self, parser, namespace, values, option_string=None): parser.add_argument('--websocket-port', default=8443, type=int, help='Port to use for websocket') args = parser.parse_args() - -# determine location of SUBMITTY GIT repository -# this script (CONFIGURES_SUBMITTY.py) is in the top level directory of the repository -# (this command works even if we run configure from a different directory) -SETUP_SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) -SUBMITTY_REPOSITORY = os.path.dirname(SETUP_SCRIPT_DIRECTORY) - -# recommended (default) directory locations -# FIXME: Check that directories exist and are readable/writeable? -SUBMITTY_INSTALL_DIR = args.install_dir -if not os.path.isdir(SUBMITTY_INSTALL_DIR) or not os.access(SUBMITTY_INSTALL_DIR, os.R_OK | os.W_OK): - raise SystemExit('Install directory {} does not exist or is not accessible'.format(SUBMITTY_INSTALL_DIR)) - -SUBMITTY_DATA_DIR = args.data_dir -if not os.path.isdir(SUBMITTY_DATA_DIR) or not os.access(SUBMITTY_DATA_DIR, os.R_OK | os.W_OK): - raise SystemExit('Data directory {} does not exist or is not accessible'.format(SUBMITTY_DATA_DIR)) - -TAGRADING_LOG_PATH = os.path.join(SUBMITTY_DATA_DIR, 'logs') -AUTOGRADING_LOG_PATH = os.path.join(SUBMITTY_DATA_DIR, 'logs', 'autograding') - -WEBSOCKET_PORT = args.websocket_port - -############################################################################## - # recommended names for special users & groups related to the SUBMITTY system PHP_USER = 'submitty_php' PHP_GROUP = 'submitty_php' @@ -153,7 +100,10 @@ def __call__(self, parser, namespace, values, option_string=None): NUM_GRADING_SCHEDULER_WORKERS = 5 ############################################################################## - +SUBMITTY_INSTALL_DIR = args.install_dir +SUBMITTY_DATA_DIR = args.data_dir +SETUP_SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +SUBMITTY_REPOSITORY = os.path.dirname(SETUP_SCRIPT_DIRECTORY) SETUP_INSTALL_DIR = os.path.join(SUBMITTY_INSTALL_DIR, '.setup') SETUP_REPOSITORY_DIR = os.path.join(SUBMITTY_REPOSITORY, '.setup') @@ -165,235 +115,10 @@ def __call__(self, parser, namespace, values, option_string=None): EMAIL_JSON = os.path.join(CONFIG_INSTALL_DIR, 'email.json') AUTHENTICATION_JSON = os.path.join(CONFIG_INSTALL_DIR, 'authentication.json') -############################################################################## - -authentication_methods = [ - 'PamAuthentication', - 'DatabaseAuthentication', - 'LdapAuthentication', - 'SamlAuthentication' -] - -defaults = { - 'database_host': 'localhost', - 'database_port': 5432, - 'database_user': 'submitty_dbuser', - 'database_course_user': 'submitty_course_dbuser', - 'submission_url': '', - 'supervisor_user': 'submitty', - 'vcs_url': '', - 'authentication_method': 0, - 'institution_name' : '', - 'institution_homepage' : '', - 'user_create_account' : False, - 'timezone' : str(tzlocal.get_localzone()), - 'submitty_admin_username': '', - 'email_user': '', - 'email_password': '', - 'email_sender': 'submitty@myuniversity.edu', - 'email_reply_to': 'submitty_do_not_reply@myuniversity.edu', - 'email_server_hostname': 'mail.myuniversity.edu', - 'email_server_port': 25, - 'email_internal_domain': 'example.com', - 'course_code_requirements': "Please follow your school's convention for course code.", - 'sys_admin_email': '', - 'sys_admin_url': '', - 'ldap_options': { - 'url': '', - 'uid': '', - 'bind_dn': '' - }, - 'saml_options': { - 'name': '', - 'username_attribute': '' - }, - 'course_material_file_upload_limit_mb': 100 -} - -loaded_defaults = {} -if os.path.isfile(CONFIGURATION_JSON): - with open(CONFIGURATION_JSON) as conf_file: - loaded_defaults = json.load(conf_file) -if os.path.isfile(SUBMITTY_ADMIN_JSON): - with open(SUBMITTY_ADMIN_JSON) as submitty_admin_file: - loaded_defaults.update(json.load(submitty_admin_file)) -if os.path.isfile(EMAIL_JSON): - with open(EMAIL_JSON) as email_file: - loaded_defaults.update(json.load(email_file)) - -if os.path.isfile(AUTHENTICATION_JSON): - with open(AUTHENTICATION_JSON) as authentication_file: - loaded_defaults.update(json.load(authentication_file)) - -# no need to authenticate on a worker machine (no website) -if not args.worker: - if 'authentication_method' in loaded_defaults: - loaded_defaults['authentication_method'] = authentication_methods.index(loaded_defaults['authentication_method']) + 1 - -# grab anything not loaded in (useful for backwards compatibility if a new default is added that -# is not in an existing config file.) -for key in defaults.keys(): - if key not in loaded_defaults: - loaded_defaults[key] = defaults[key] -defaults = loaded_defaults - -print("\nWelcome to the Submitty Homework Submission Server Configuration\n") -DEBUGGING_ENABLED = args.debug is True - -if DEBUGGING_ENABLED: - print('!! DEBUG MODE ENABLED !!') - print() - -if args.worker: - print("CONFIGURING SUBMITTY AS A WORKER !!") - -print('Hit enter to use default in []') -print() - -if args.worker: - SUPERVISOR_USER = get_input('What is the id for your submitty user?', defaults['supervisor_user']) - print('SUPERVISOR USER : {}'.format(SUPERVISOR_USER)) -else: - DATABASE_HOST = get_input('What is the database host?', defaults['database_host']) - print() - - if not os.path.isdir(DATABASE_HOST): - DATABASE_PORT = int(get_input('What is the database port?', defaults['database_port'])) - print() - else: - DATABASE_PORT = defaults['database_port'] - - DATABASE_USER = get_input('What is the global database user/role?', defaults['database_user']) - print() - - default = '' - if 'database_password' in defaults and DATABASE_USER == defaults['database_user']: - default = '(Leave blank to use same password)' - DATABASE_PASS = get_input('What is the password for the global database user/role {}? {}'.format(DATABASE_USER, default)) - if DATABASE_PASS == '' and DATABASE_USER == defaults['database_user'] and 'database_password' in defaults: - DATABASE_PASS = defaults['database_password'] - print() - - DATABASE_COURSE_USER = get_input('What is the course database user/role?', defaults['database_course_user']) - print() - - default = '' - if 'database_course_password' in defaults and DATABASE_COURSE_USER == defaults['database_course_user']: - default = '(Leave blank to use same password)' - DATABASE_COURSE_PASSWORD = get_input('What is the password for the course database user/role {}? {}'.format(DATABASE_COURSE_USER, default)) - if DATABASE_COURSE_PASSWORD == '' and DATABASE_COURSE_USER == defaults['database_course_user'] and 'database_course_password' in defaults: - DATABASE_COURSE_PASSWORD = defaults['database_course_password'] - print() - - TIMEZONE = get_input('What timezone should Submitty use? (for a full list of supported timezones see http://php.net/manual/en/timezones.php)', defaults['timezone']) - print() - - DEFAULT_LOCALE = get_input('What default language should the Submitty site use?', 'en_US') - print() - - COURSE_MATERIAL_UPLOAD_LIMIT_MB = get_input('What is the maximum file upload size for course materials (in MB)?', defaults['course_material_file_upload_limit_mb']) - print() - - SUBMISSION_URL = get_input('What is the url for submission? (ex: http://192.168.56.101 or ' - 'https://submitty.cs.rpi.edu)', defaults['submission_url']).rstrip('/') - print() - - VCS_URL = get_input('What is the url for VCS? (Leave blank to default to submission url + {$vcs_type}) (ex: http://192.168.56.101/{$vcs_type} or https://submitty-vcs.cs.rpi.edu/{$vcs_type}', defaults['vcs_url']).rstrip('/') - print() - - INSTITUTION_NAME = get_input('What is the name of your institution? (Leave blank/type "none" if not desired)', - defaults['institution_name']) - print() - - if INSTITUTION_NAME == '' or INSTITUTION_NAME.isspace(): - INSTITUTION_HOMEPAGE = '' - else: - INSTITUTION_HOMEPAGE = get_input("What is the url of your institution\'s homepage? " - '(Leave blank/type "none" if not desired)', defaults['institution_homepage']) - if INSTITUTION_HOMEPAGE.lower() == "none": - INSTITUTION_HOMEPAGE = '' - print() - - SYS_ADMIN_EMAIL = get_input("What is the email for system administration?", defaults['sys_admin_email']) - SYS_ADMIN_URL = get_input("Where to report problems with Submitty (url for help link)?", defaults['sys_admin_url']) - - print('What authentication method to use:') - for i in range(len(authentication_methods)): - print(f"{i + 1}. {authentication_methods[i]}") - - while True: - try: - auth = int(get_input('Enter number?', defaults['authentication_method'])) - 1 - except ValueError: - auth = -1 - if auth in range(len(authentication_methods)): - break - print(f'Number must in between 1 - {len(authentication_methods)} (inclusive)!') - print() - - AUTHENTICATION_METHOD = authentication_methods[auth] - - default_auth_options = defaults.get('ldap_options', dict()) - LDAP_OPTIONS = { - 'url': default_auth_options.get('url', ''), - 'uid': default_auth_options.get('uid', ''), - 'bind_dn': default_auth_options.get('bind_dn', '') - } - USER_CREATE_ACCOUNT = False - if AUTHENTICATION_METHOD == 'DatabaseAuthentication': - user_create_account = get_input("Allow users to create their own accounts? [y/n]", 'n') - USER_CREATE_ACCOUNT = user_create_account.lower() in ['yes', 'y'] - print() - if AUTHENTICATION_METHOD == 'LdapAuthentication': - LDAP_OPTIONS['url'] = get_input('Enter LDAP url?', LDAP_OPTIONS['url']) - LDAP_OPTIONS['uid'] = get_input('Enter LDAP UID?', LDAP_OPTIONS['uid']) - LDAP_OPTIONS['bind_dn'] = get_input('Enter LDAP bind_dn?', LDAP_OPTIONS['bind_dn']) - - default_auth_options = defaults.get('saml_options', dict()) - SAML_OPTIONS = { - 'name': default_auth_options.get('name', ''), - 'username_attribute': default_auth_options.get('username_attribute', '') - } - - if AUTHENTICATION_METHOD == 'SamlAuthentication': - SAML_OPTIONS['name'] = get_input('Enter name you would like shown to user for authentication?', SAML_OPTIONS['name']) - SAML_OPTIONS['username_attribute'] = get_input('Enter SAML username attribute?', SAML_OPTIONS['username_attribute']) - - - CGI_URL = SUBMISSION_URL + '/cgi-bin' - - SUBMITTY_ADMIN_USERNAME = get_input("What is the submitty admin username (optional)?", defaults['submitty_admin_username']) - - while True: - is_email_enabled = get_input("Will Submitty use email notifications? [y/n]", 'y') - if (is_email_enabled.lower() in ['yes', 'y']): - EMAIL_ENABLED = True - EMAIL_USER = get_input("What is the email user?", defaults['email_user']) - EMAIL_PASSWORD = get_input("What is the email password",defaults['email_password']) - EMAIL_SENDER = get_input("What is the email sender address (the address that will appear in the From: line)?",defaults['email_sender']) - EMAIL_REPLY_TO = get_input("What is the email reply to address?", defaults['email_reply_to']) - EMAIL_SERVER_HOSTNAME = get_input("What is the email server hostname?", defaults['email_server_hostname']) - try: - EMAIL_SERVER_PORT = int(get_input("What is the email server port?", defaults['email_server_port'])) - except ValueError: - EMAIL_SERVER_PORT = defaults['email_server_port'] - EMAIL_INTERNAL_DOMAIN = get_input("What is the internal email address format?", defaults['email_internal_domain']) - break - - elif (is_email_enabled.lower() in ['no', 'n']): - EMAIL_ENABLED = False - EMAIL_USER = defaults['email_user'] - EMAIL_PASSWORD = defaults['email_password'] - EMAIL_SENDER = defaults['email_sender'] - EMAIL_REPLY_TO = defaults['email_reply_to'] - EMAIL_SERVER_HOSTNAME = defaults['email_server_hostname'] - EMAIL_SERVER_PORT = defaults['email_server_port'] - EMAIL_INTERNAL_DOMAIN = defaults['email_internal_domain'] - break - print() - +#### +generated_config = generate_config(SUBMITTY_INSTALL_DIR, SUBMITTY_DATA_DIR, args.worker, args.debug) ############################################################################## # make the installation setup directory @@ -406,8 +131,7 @@ def __call__(self, parser, namespace, values, option_string=None): ############################################################################## # WRITE CONFIG FILES IN ${SUBMITTY_INSTALL_DIR}/.setup - -config = OrderedDict() +# Prompt for user configurable values config['submitty_install_dir'] = SUBMITTY_INSTALL_DIR config['submitty_repository'] = SUBMITTY_REPOSITORY @@ -420,14 +144,11 @@ def __call__(self, parser, namespace, values, option_string=None): config['first_untrusted_gid'] = FIRST_UNTRUSTED_UID config['num_grading_scheduler_workers'] = NUM_GRADING_SCHEDULER_WORKERS - config['daemon_user'] = DAEMON_USER config['daemon_uid'] = DAEMON_UID config['daemon_gid'] = DAEMON_GID -if args.worker: - config['supervisor_user'] = SUPERVISOR_USER -else: +if not args.worker: config['php_user'] = PHP_USER config['cgi_user'] = CGI_USER config['daemonphp_group'] = DAEMONPHP_GROUP @@ -436,35 +157,7 @@ def __call__(self, parser, namespace, values, option_string=None): config['php_uid'] = PHP_UID config['php_gid'] = PHP_GID - config['database_host'] = DATABASE_HOST - config['database_port'] = DATABASE_PORT - config['database_user'] = DATABASE_USER - config['database_password'] = DATABASE_PASS - config['database_course_user'] = DATABASE_COURSE_USER - config['database_course_password'] = DATABASE_COURSE_PASSWORD - config['timezone'] = TIMEZONE - config['default_locale'] = DEFAULT_LOCALE - - config['authentication_method'] = AUTHENTICATION_METHOD - config['vcs_url'] = VCS_URL - config['submission_url'] = SUBMISSION_URL - config['cgi_url'] = CGI_URL - config['websocket_port'] = WEBSOCKET_PORT - - config['institution_name'] = INSTITUTION_NAME - config['institution_homepage'] = INSTITUTION_HOMEPAGE - config['debugging_enabled'] = DEBUGGING_ENABLED - config['user_create_account'] = USER_CREATE_ACCOUNT - -# site_log_path is a holdover name. This could more accurately be called the "log_path" -config['site_log_path'] = TAGRADING_LOG_PATH -config['autograding_log_path'] = AUTOGRADING_LOG_PATH - -if args.worker: - config['worker'] = 1 -else: - config['worker'] = 0 - + config['websocket_port'] = args.websocket_port with open(INSTALL_FILE, 'w') as open_file: def write(x=''): @@ -596,15 +289,16 @@ def write(x=''): # Write database json if not args.worker: - config = OrderedDict() - config['authentication_method'] = AUTHENTICATION_METHOD - config['database_host'] = DATABASE_HOST - config['database_port'] = DATABASE_PORT - config['database_user'] = DATABASE_USER - config['database_password'] = DATABASE_PASS - config['database_course_user'] = DATABASE_COURSE_USER - config['database_course_password'] = DATABASE_COURSE_PASSWORD - config['debugging_enabled'] = DEBUGGING_ENABLED + # config = OrderedDict() + config = generated_config['database'] + # config['authentication_method'] = AUTHENTICATION_METHOD + # config['database_host'] = DATABASE_HOST + # config['database_port'] = DATABASE_PORT + # config['database_user'] = DATABASE_USER + # config['database_password'] = DATABASE_PASS + # config['database_course_user'] = DATABASE_COURSE_USER + # config['database_course_password'] = DATABASE_COURSE_PASSWORD + # config['debugging_enabled'] = DEBUGGING_ENABLED with open(DATABASE_JSON, 'w') as json_file: json.dump(config, json_file, indent=2) @@ -615,9 +309,9 @@ def write(x=''): # Write authentication json if not args.worker: config = OrderedDict() - config['authentication_method'] = AUTHENTICATION_METHOD - config['ldap_options'] = LDAP_OPTIONS - config['saml_options'] = SAML_OPTIONS + # config['authentication_method'] = AUTHENTICATION_METHOD + # config['ldap_options'] = LDAP_OPTIONS + # config['saml_options'] = SAML_OPTIONS with open(AUTHENTICATION_JSON, 'w') as json_file: json.dump(config, json_file, indent=4) @@ -747,15 +441,15 @@ def write(x=''): # Write email json if not args.worker: - config = OrderedDict() - config['email_enabled'] = EMAIL_ENABLED - config['email_user'] = EMAIL_USER - config['email_password'] = EMAIL_PASSWORD - config['email_sender'] = EMAIL_SENDER - config['email_reply_to'] = EMAIL_REPLY_TO - config['email_server_hostname'] = EMAIL_SERVER_HOSTNAME - config['email_server_port'] = EMAIL_SERVER_PORT - config['email_internal_domain'] = EMAIL_INTERNAL_DOMAIN + # config = OrderedDict() + # config['email_enabled'] = EMAIL_ENABLED + # config['email_user'] = EMAIL_USER + # config['email_password'] = EMAIL_PASSWORD + # config['email_sender'] = EMAIL_SENDER + # config['email_reply_to'] = EMAIL_REPLY_TO + # config['email_server_hostname'] = EMAIL_SERVER_HOSTNAME + # config['email_server_port'] = EMAIL_SERVER_PORT + # config['email_internal_domain'] = EMAIL_INTERNAL_DOMAIN with open(EMAIL_JSON, 'w') as json_file: json.dump(config, json_file, indent=2) diff --git a/.setup/GENERATE_CONFIGS.py b/.setup/GENERATE_CONFIGS.py new file mode 100644 index 00000000000..8d220fa14a3 --- /dev/null +++ b/.setup/GENERATE_CONFIGS.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 + +import argparse +from collections import OrderedDict +import grp +import json +import os +import pwd +import secrets +import shutil +import string +import tzlocal +import tempfile + + +def get_input(question, default=""): + add = "[{}] ".format(default) if default != "" else "" + user = input("{}: {}".format(question, add)).strip() + if user == "": + user = default + return user + + +class StrToBoolAction(argparse.Action): + """ + Custom action that parses strings to boolean values. All values that come + from bash are strings, and so need to parse that into the appropriate + bool value. + """ + def __init__(self, option_strings, dest, nargs=None, **kwargs): + if nargs is not None: + raise ValueError("nargs not allowed") + super().__init__(option_strings, dest, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values != '0' and values.lower() != 'false') + +def generate_config(submitty_install_dir, submitty_data_dir, worker, debug): + # recommended (default) directory locations + # FIXME: Check that directories exist and are readable/writeable? + + SETUP_INSTALL_DIR = os.path.join(submitty_install_dir, '.setup') + CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') + CONFIG_INSTALL_DIR = os.path.join(submitty_install_dir, 'config') + SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') + EMAIL_JSON = os.path.join(CONFIG_INSTALL_DIR, 'email.json') + AUTHENTICATION_JSON = os.path.join(CONFIG_INSTALL_DIR, 'authentication.json') + SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') + + if not os.path.isdir(submitty_install_dir) or not os.access(submitty_install_dir, os.R_OK | os.W_OK): + raise SystemExit('Install directory {} does not exist or is not accessible'.format(submitty_install_dir)) + + if not os.path.isdir(submitty_data_dir) or not os.access(submitty_data_dir, os.R_OK | os.W_OK): + raise SystemExit('Data directory {} does not exist or is not accessible'.format(submitty_data_dir)) + + SETUP_INSTALL_DIR = os.path.join(submitty_install_dir, '.setup') + CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') + + ########################################################################## + + authentication_methods = [ + 'PamAuthentication', + 'DatabaseAuthentication', + 'LdapAuthentication', + 'SamlAuthentication' + ] + + defaults = { + 'database_host': 'localhost', + 'database_port': 5432, + 'database_user': 'submitty_dbuser', + 'database_course_user': 'submitty_course_dbuser', + 'submission_url': '', + 'supervisor_user': 'submitty', + 'vcs_url': '', + 'authentication_method': 0, + 'institution_name' : '', + 'institution_homepage' : '', + 'user_create_account' : False, + 'timezone' : str(tzlocal.get_localzone()), + 'submitty_admin_username': '', + 'email_user': '', + 'email_password': '', + 'email_sender': 'submitty@myuniversity.edu', + 'email_reply_to': 'submitty_do_not_reply@myuniversity.edu', + 'email_server_hostname': 'mail.myuniversity.edu', + 'email_server_port': 25, + 'email_internal_domain': 'example.com', + 'course_code_requirements': "Please follow your school's convention for course code.", + 'sys_admin_email': '', + 'sys_admin_url': '', + 'ldap_options': { + 'url': '', + 'uid': '', + 'bind_dn': '' + }, + 'saml_options': { + 'name': '', + 'username_attribute': '' + }, + 'course_material_file_upload_limit_mb': 100 + } + + loaded_defaults = {} + if os.path.isfile(CONFIGURATION_JSON): + with open(CONFIGURATION_JSON) as conf_file: + loaded_defaults = json.load(conf_file) + if os.path.isfile(SUBMITTY_ADMIN_JSON): + with open(SUBMITTY_ADMIN_JSON) as submitty_admin_file: + loaded_defaults.update(json.load(submitty_admin_file)) + if os.path.isfile(EMAIL_JSON): + with open(EMAIL_JSON) as email_file: + loaded_defaults.update(json.load(email_file)) + + if os.path.isfile(AUTHENTICATION_JSON): + with open(AUTHENTICATION_JSON) as authentication_file: + loaded_defaults.update(json.load(authentication_file)) + + # no need to authenticate on a worker machine (no website) + if not worker: + if 'authentication_method' in loaded_defaults: + loaded_defaults['authentication_method'] = authentication_methods.index(loaded_defaults['authentication_method']) + 1 + + # grab anything not loaded in (useful for backwards compatibility if a new default is added that + # is not in an existing config file.) + for key in defaults.keys(): + if key not in loaded_defaults: + loaded_defaults[key] = defaults[key] + defaults = loaded_defaults + + print("\nWelcome to the Submitty Homework Submission Server Configuration\n") + DEBUGGING_ENABLED = debug is True + + if DEBUGGING_ENABLED: + print('!! DEBUG MODE ENABLED !!') + print() + + if worker: + print("CONFIGURING SUBMITTY AS A WORKER !!") + + print('Hit enter to use default in []') + print() + + if worker: + SUPERVISOR_USER = get_input('What is the id for your submitty user?', defaults['supervisor_user']) + print('SUPERVISOR USER : {}'.format(SUPERVISOR_USER)) + else: + DATABASE_HOST = get_input('What is the database host?', defaults['database_host']) + print() + + if not os.path.isdir(DATABASE_HOST): + DATABASE_PORT = int(get_input('What is the database port?', defaults['database_port'])) + print() + else: + DATABASE_PORT = defaults['database_port'] + + DATABASE_USER = get_input('What is the global database user/role?', defaults['database_user']) + print() + + default = '' + if 'database_password' in defaults and DATABASE_USER == defaults['database_user']: + default = '(Leave blank to use same password)' + DATABASE_PASS = get_input('What is the password for the global database user/role {}? {}'.format(DATABASE_USER, default)) + if DATABASE_PASS == '' and DATABASE_USER == defaults['database_user'] and 'database_password' in defaults: + DATABASE_PASS = defaults['database_password'] + print() + + DATABASE_COURSE_USER = get_input('What is the course database user/role?', defaults['database_course_user']) + print() + + default = '' + if 'database_course_password' in defaults and DATABASE_COURSE_USER == defaults['database_course_user']: + default = '(Leave blank to use same password)' + DATABASE_COURSE_PASSWORD = get_input('What is the password for the course database user/role {}? {}'.format(DATABASE_COURSE_USER, default)) + if DATABASE_COURSE_PASSWORD == '' and DATABASE_COURSE_USER == defaults['database_course_user'] and 'database_course_password' in defaults: + DATABASE_COURSE_PASSWORD = defaults['database_course_password'] + print() + + TIMEZONE = get_input('What timezone should Submitty use? (for a full list of supported timezones see http://php.net/manual/en/timezones.php)', defaults['timezone']) + print() + + DEFAULT_LOCALE = get_input('What default language should the Submitty site use?', 'en_US') + print() + + COURSE_MATERIAL_UPLOAD_LIMIT_MB = get_input('What is the maximum file upload size for course materials (in MB)?', defaults['course_material_file_upload_limit_mb']) + print() + + SUBMISSION_URL = get_input('What is the url for submission? (ex: http://192.168.56.101 or ' + 'https://submitty.cs.rpi.edu)', defaults['submission_url']).rstrip('/') + print() + + VCS_URL = get_input('What is the url for VCS? (Leave blank to default to submission url + {$vcs_type}) (ex: http://192.168.56.101/{$vcs_type} or https://submitty-vcs.cs.rpi.edu/{$vcs_type}', defaults['vcs_url']).rstrip('/') + print() + + INSTITUTION_NAME = get_input('What is the name of your institution? (Leave blank/type "none" if not desired)', + defaults['institution_name']) + print() + + if INSTITUTION_NAME == '' or INSTITUTION_NAME.isspace(): + INSTITUTION_HOMEPAGE = '' + else: + INSTITUTION_HOMEPAGE = get_input("What is the url of your institution\'s homepage? " + '(Leave blank/type "none" if not desired)', defaults['institution_homepage']) + if INSTITUTION_HOMEPAGE.lower() == "none": + INSTITUTION_HOMEPAGE = '' + print() + + SYS_ADMIN_EMAIL = get_input("What is the email for system administration?", defaults['sys_admin_email']) + SYS_ADMIN_URL = get_input("Where to report problems with Submitty (url for help link)?", defaults['sys_admin_url']) + + print('What authentication method to use:') + for i in range(len(authentication_methods)): + print(f"{i + 1}. {authentication_methods[i]}") + + while True: + try: + auth = int(get_input('Enter number?', defaults['authentication_method'])) - 1 + except ValueError: + auth = -1 + if auth in range(len(authentication_methods)): + break + print(f'Number must in between 1 - {len(authentication_methods)} (inclusive)!') + print() + + AUTHENTICATION_METHOD = authentication_methods[auth] + + default_auth_options = defaults.get('ldap_options', dict()) + LDAP_OPTIONS = { + 'url': default_auth_options.get('url', ''), + 'uid': default_auth_options.get('uid', ''), + 'bind_dn': default_auth_options.get('bind_dn', '') + } + USER_CREATE_ACCOUNT = False + if AUTHENTICATION_METHOD == 'DatabaseAuthentication': + user_create_account = get_input("Allow users to create their own accounts? [y/n]", 'n') + USER_CREATE_ACCOUNT = user_create_account.lower() in ['yes', 'y'] + print() + if AUTHENTICATION_METHOD == 'LdapAuthentication': + LDAP_OPTIONS['url'] = get_input('Enter LDAP url?', LDAP_OPTIONS['url']) + LDAP_OPTIONS['uid'] = get_input('Enter LDAP UID?', LDAP_OPTIONS['uid']) + LDAP_OPTIONS['bind_dn'] = get_input('Enter LDAP bind_dn?', LDAP_OPTIONS['bind_dn']) + + default_auth_options = defaults.get('saml_options', dict()) + SAML_OPTIONS = { + 'name': default_auth_options.get('name', ''), + 'username_attribute': default_auth_options.get('username_attribute', '') + } + + if AUTHENTICATION_METHOD == 'SamlAuthentication': + SAML_OPTIONS['name'] = get_input('Enter name you would like shown to user for authentication?', SAML_OPTIONS['name']) + SAML_OPTIONS['username_attribute'] = get_input('Enter SAML username attribute?', SAML_OPTIONS['username_attribute']) + + + CGI_URL = SUBMISSION_URL + '/cgi-bin' + + SUBMITTY_ADMIN_USERNAME = get_input("What is the submitty admin username (optional)?", defaults['submitty_admin_username']) + + while True: + is_email_enabled = get_input("Will Submitty use email notifications? [y/n]", 'y') + if (is_email_enabled.lower() in ['yes', 'y']): + EMAIL_ENABLED = True + EMAIL_USER = get_input("What is the email user?", defaults['email_user']) + EMAIL_PASSWORD = get_input("What is the email password",defaults['email_password']) + EMAIL_SENDER = get_input("What is the email sender address (the address that will appear in the From: line)?",defaults['email_sender']) + EMAIL_REPLY_TO = get_input("What is the email reply to address?", defaults['email_reply_to']) + EMAIL_SERVER_HOSTNAME = get_input("What is the email server hostname?", defaults['email_server_hostname']) + try: + EMAIL_SERVER_PORT = int(get_input("What is the email server port?", defaults['email_server_port'])) + except ValueError: + EMAIL_SERVER_PORT = defaults['email_server_port'] + EMAIL_INTERNAL_DOMAIN = get_input("What is the internal email address format?", defaults['email_internal_domain']) + break + + elif (is_email_enabled.lower() in ['no', 'n']): + EMAIL_ENABLED = False + EMAIL_USER = defaults['email_user'] + EMAIL_PASSWORD = defaults['email_password'] + EMAIL_SENDER = defaults['email_sender'] + EMAIL_REPLY_TO = defaults['email_reply_to'] + EMAIL_SERVER_HOSTNAME = defaults['email_server_hostname'] + EMAIL_SERVER_PORT = defaults['email_server_port'] + EMAIL_INTERNAL_DOMAIN = defaults['email_internal_domain'] + break + print() + + full_config = OrderedDict() + config = OrderedDict() + if worker: + config['supervisor_user'] = SUPERVISOR_USER + else: + database_config = OrderedDict() + auth_config = OrderedDict() + email_config = OrderedDict() + + database_config['authentication_method'] = AUTHENTICATION_METHOD + database_config['database_host'] = DATABASE_HOST + database_config['database_port'] = DATABASE_PORT + database_config['database_user'] = DATABASE_USER + database_config['database_password'] = DATABASE_PASS + database_config['database_course_user'] = DATABASE_COURSE_USER + database_config['database_course_password'] = DATABASE_COURSE_PASSWORD + database_config['debugging_enabled'] = DEBUGGING_ENABLED + + full_config['database'] = database_config + + auth_config['authentication_method'] = AUTHENTICATION_METHOD + auth_config['ldap_options'] = LDAP_OPTIONS + auth_config['saml_options'] = SAML_OPTIONS + + full_config['authentication'] = auth_config + + email_config = OrderedDict() + email_config['email_enabled'] = EMAIL_ENABLED + email_config['email_user'] = EMAIL_USER + email_config['email_password'] = EMAIL_PASSWORD + email_config['email_sender'] = EMAIL_SENDER + email_config['email_reply_to'] = EMAIL_REPLY_TO + email_config['email_server_hostname'] = EMAIL_SERVER_HOSTNAME + email_config['email_server_port'] = EMAIL_SERVER_PORT + email_config['email_internal_domain'] = EMAIL_INTERNAL_DOMAIN + email_config['timezone'] = TIMEZONE + email_config['default_locale'] = DEFAULT_LOCALE + + full_config['email'] = email_config + + config['authentication_method'] = AUTHENTICATION_METHOD + config['vcs_url'] = VCS_URL + config['submission_url'] = SUBMISSION_URL + config['cgi_url'] = CGI_URL + + + config['institution_name'] = INSTITUTION_NAME + config['institution_homepage'] = INSTITUTION_HOMEPAGE + config['user_create_account'] = USER_CREATE_ACCOUNT + +# site_log_path is a holdover name. This could more accurately be called the "log_path" +config['site_log_path'] = TAGRADING_LOG_PATH +config['autograding_log_path'] = AUTOGRADING_LOG_PATH + +if args.worker: + config['worker'] = 1 +else: + config['worker'] = 0 \ No newline at end of file diff --git a/.setup/install_system.sh b/.setup/install_system.sh index 2263d1373c2..3a478b3b395 100644 --- a/.setup/install_system.sh +++ b/.setup/install_system.sh @@ -23,6 +23,10 @@ trap 'err_message' ERR # print commands as we execute and fail early set -ev +# Sources +source ${CURRENT_DIR}/bin/versions.sh +source ${CURRENT_DIR}/distro_setup/setup_distro.sh + # this script must be run by root or sudo if [[ "$UID" -ne "0" ]] ; then echo "ERROR: This script must be run by root or sudo" @@ -41,6 +45,12 @@ LICHEN_REPOSITORY=/usr/local/submitty/GIT_CHECKOUT/Lichen SUBMITTY_INSTALL_DIR=/usr/local/submitty SUBMITTY_DATA_DIR=/var/local/submitty +# create directory and fix permissions +mkdir -p ${SUBMITTY_DATA_DIR} + +INSTALL_SYS_DIR=$(mktemp -d) +chmod 777 "${INSTALL_SYS_DIR}" +pushd "${INSTALL_SYS_DIR}" > /dev/null # USERS / GROUPS DAEMON_USER=submitty_daemon @@ -50,12 +60,18 @@ PHP_GROUP=submitty_php CGI_USER=submitty_cgi CGI_GROUP=submitty_cgi + +COURSE_BUILDERS_GROUP=submitty_course_builders +DB_USER=submitty_dbuser +DATABASE_PASSWORD=submitty_dbuser +DB_COURSE_USER=submitty_course_dbuser +DB_COURSE_PASSWORD=submitty_dbuser + DAEMONPHP_GROUP=submitty_daemonphp DAEMONCGI_GROUP=submitty_daemoncgi DAEMONPHPCGI_GROUP=submitty_daemonphpcgi -# VERSIONS -source ${CURRENT_DIR}/bin/versions.sh + ################################################################# # PROVISION SETUP @@ -107,6 +123,69 @@ fi if [ ${UTM} == 1 ]; then mkdir ${SUBMITTY_REPOSITORY}/.utm fi +################################################################# +# DISTRO SETUP +################# + +bash "${SUBMITTY_REPOSITORY}/.setup/update_system.sh" "config=${SUBMITTY_DIRECTORY}" + +################################################################# +# SUBMITTY SETUP +################# +echo Beginning Submitty Setup +sudo apt-get update +sudo add-apt-repository universe +sudo apt update +sudo apt install python3-pip -y + +pip3 install tzlocal +#If in worker mode, run configure with --worker option. +if [ ${WORKER} == 1 ]; then + echo "Running configure submitty in worker mode" + if [ ${DEV_VM} == 1 ]; then + echo "submitty" | python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --worker + else + python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --worker + fi +else + if [ ${DEV_VM} == 1 ]; then + # This should be set by setup_distro.sh for whatever distro we have, but + # in case it is not, default to our primary URL + if [ -z "${SUBMISSION_URL}" ]; then + SUBMISSION_URL='http://192.168.56.101' + fi + echo -e "/var/run/postgresql +${DB_USER} +${DATABASE_PASSWORD} +${DB_COURSE_USER} +${DB_COURSE_PASSWORD} +America/New_York +en_US +100 +${SUBMISSION_URL} + + +sysadmin@example.com +https://example.com +1 +submitty-admin +y + + +submitty@vagrant +do-not-reply@vagrant +localhost +25 +" | python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --debug --setup-for-sample-courses --websocket-port ${WEBSOCKET_PORT} + + # Set these manually as they're not asked about during CONFIGURE_SUBMITTY.py + sed -i -e 's/"url": ""/"url": "ldap:\/\/localhost"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json + sed -i -e 's/"uid": ""/"uid": "uid"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json + sed -i -e 's/"bind_dn": ""/"bind_dn": "ou=users,dc=vagrant,dc=local"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json + else + python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py + fi +fi if [ ${DEV_VM} == 1 ] && [ ${WORKER} == 0 ]; then # Setting it up to allow SSH as root by default @@ -208,24 +287,6 @@ else fi -INSTALL_SYS_DIR=$(mktemp -d) -chmod 777 "${INSTALL_SYS_DIR}" -pushd "${INSTALL_SYS_DIR}" > /dev/null - -COURSE_BUILDERS_GROUP=submitty_course_builders -DB_USER=submitty_dbuser -DATABASE_PASSWORD=submitty_dbuser -DB_COURSE_USER=submitty_course_dbuser -DB_COURSE_PASSWORD=submitty_dbuser - -################################################################# -# DISTRO SETUP -################# - -source ${CURRENT_DIR}/distro_setup/setup_distro.sh - -bash "${SUBMITTY_REPOSITORY}/.setup/update_system.sh" "config=${SUBMITTY_DIRECTORY}" - ################################################################# # STACK SETUP ################# @@ -576,9 +637,6 @@ EOF sed -i -e "s/^disable_functions = .*/disable_functions = ${DISABLED_FUNCTIONS}/g" /etc/php/${PHP_VERSION}/fpm/php.ini fi -# create directories and fix permissions -mkdir -p ${SUBMITTY_DATA_DIR} - #Set up database and copy down the tutorial repo if not in worker mode if [ ${WORKER} == 0 ]; then # create the courses directory. This is needed for the first time we run @@ -667,59 +725,6 @@ if [ ! -d "${clangsrc}" ]; then echo 'DONE PREPARING CLANG INSTALLATION' fi -################################################################# -# SUBMITTY SETUP -################# -echo Beginning Submitty Setup - -#If in worker mode, run configure with --worker option. -if [ ${WORKER} == 1 ]; then - echo "Running configure submitty in worker mode" - if [ ${DEV_VM} == 1 ]; then - echo "submitty" | python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --worker - else - python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --worker - fi -else - if [ ${DEV_VM} == 1 ]; then - # This should be set by setup_distro.sh for whatever distro we have, but - # in case it is not, default to our primary URL - if [ -z "${SUBMISSION_URL}" ]; then - SUBMISSION_URL='http://192.168.56.101' - fi - echo -e "/var/run/postgresql -${DB_USER} -${DATABASE_PASSWORD} -${DB_COURSE_USER} -${DB_COURSE_PASSWORD} -America/New_York -en_US -100 -${SUBMISSION_URL} - - -sysadmin@example.com -https://example.com -1 -submitty-admin -y - - -submitty@vagrant -do-not-reply@vagrant -localhost -25 -" | python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --debug --setup-for-sample-courses --websocket-port ${WEBSOCKET_PORT} - - # Set these manually as they're not asked about during CONFIGURE_SUBMITTY.py - sed -i -e 's/"url": ""/"url": "ldap:\/\/localhost"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json - sed -i -e 's/"uid": ""/"uid": "uid"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json - sed -i -e 's/"bind_dn": ""/"bind_dn": "ou=users,dc=vagrant,dc=local"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json - else - python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py - fi -fi - if [ ${WORKER} == 1 ]; then #Add the submitty user to /etc/sudoers if in worker mode. SUPERVISOR_USER=$(jq -r '.supervisor_user' ${SUBMITTY_INSTALL_DIR}/config/submitty_users.json) From 2373fe747b1a4d8dd6cd9873be6457b0988dce78 Mon Sep 17 00:00:00 2001 From: IDzyre Date: Wed, 21 Jan 2026 19:38:37 -0800 Subject: [PATCH 2/7] More work --- .setup/CONFIGURE_SUBMITTY.py | 55 +++++++++++++++++++++-------- .setup/GENERATE_CONFIGS.py | 67 ++++-------------------------------- 2 files changed, 47 insertions(+), 75 deletions(-) diff --git a/.setup/CONFIGURE_SUBMITTY.py b/.setup/CONFIGURE_SUBMITTY.py index c928362c78c..b4d0586ce35 100644 --- a/.setup/CONFIGURE_SUBMITTY.py +++ b/.setup/CONFIGURE_SUBMITTY.py @@ -48,6 +48,43 @@ def get_ids(user): DAEMON_USER = 'submitty_daemon' DAEMON_GROUP = 'submitty_daemon' +SUBMITTY_INSTALL_DIR = args.install_dir +SUBMITTY_DATA_DIR = args.data_dir +SETUP_SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +SUBMITTY_REPOSITORY = os.path.dirname(SETUP_SCRIPT_DIRECTORY) +SETUP_INSTALL_DIR = os.path.join(SUBMITTY_INSTALL_DIR, '.setup') +SETUP_REPOSITORY_DIR = os.path.join(SUBMITTY_REPOSITORY, '.setup') + +INSTALL_FILE = os.path.join(SETUP_INSTALL_DIR, 'INSTALL_SUBMITTY.sh') +CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') +SITE_CONFIG_DIR = os.path.join(SUBMITTY_INSTALL_DIR, "site", "config") +CONFIG_INSTALL_DIR = os.path.join(SUBMITTY_INSTALL_DIR, 'config') +SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') +EMAIL_JSON = os.path.join(CONFIG_INSTALL_DIR, 'email.json') +AUTHENTICATION_JSON = os.path.join(CONFIG_INSTALL_DIR, 'authentication.json') + +#### +loaded_defaults = {} +if os.path.isfile(CONFIGURATION_JSON): + with open(CONFIGURATION_JSON) as conf_file: + loaded_defaults = json.load(conf_file) +if os.path.isfile(SUBMITTY_ADMIN_JSON): + with open(SUBMITTY_ADMIN_JSON) as submitty_admin_file: + loaded_defaults.update(json.load(submitty_admin_file)) +if os.path.isfile(EMAIL_JSON): + with open(EMAIL_JSON) as email_file: + loaded_defaults.update(json.load(email_file)) + +if os.path.isfile(AUTHENTICATION_JSON): + with open(AUTHENTICATION_JSON) as authentication_file: + loaded_defaults.update(json.load(authentication_file)) + +if not os.path.isdir(SUBMITTY_INSTALL_DIR) or not os.access(SUBMITTY_INSTALL_DIR, os.R_OK | os.W_OK): + raise SystemExit('Install directory {} does not exist or is not accessible'.format(SUBMITTY_INSTALL_DIR)) + +if not os.path.isdir(SUBMITTY_DATA_DIR) or not os.access(SUBMITTY_DATA_DIR, os.R_OK | os.W_OK): + raise SystemExit('Data directory {} does not exist or is not accessible'.format(SUBMITTY_DATA_DIR)) + if not args.worker: PHP_UID, PHP_GID = get_ids(PHP_USER) CGI_UID, CGI_GID = get_ids(CGI_USER) @@ -100,22 +137,7 @@ def get_ids(user): NUM_GRADING_SCHEDULER_WORKERS = 5 ############################################################################## -SUBMITTY_INSTALL_DIR = args.install_dir -SUBMITTY_DATA_DIR = args.data_dir -SETUP_SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) -SUBMITTY_REPOSITORY = os.path.dirname(SETUP_SCRIPT_DIRECTORY) -SETUP_INSTALL_DIR = os.path.join(SUBMITTY_INSTALL_DIR, '.setup') -SETUP_REPOSITORY_DIR = os.path.join(SUBMITTY_REPOSITORY, '.setup') -INSTALL_FILE = os.path.join(SETUP_INSTALL_DIR, 'INSTALL_SUBMITTY.sh') -CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') -SITE_CONFIG_DIR = os.path.join(SUBMITTY_INSTALL_DIR, "site", "config") -CONFIG_INSTALL_DIR = os.path.join(SUBMITTY_INSTALL_DIR, 'config') -SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') -EMAIL_JSON = os.path.join(CONFIG_INSTALL_DIR, 'email.json') -AUTHENTICATION_JSON = os.path.join(CONFIG_INSTALL_DIR, 'authentication.json') - -#### generated_config = generate_config(SUBMITTY_INSTALL_DIR, SUBMITTY_DATA_DIR, args.worker, args.debug) @@ -356,6 +378,9 @@ def write(x=''): config['submitty_install_dir'] = SUBMITTY_INSTALL_DIR config['submitty_repository'] = SUBMITTY_REPOSITORY config['submitty_data_dir'] = SUBMITTY_DATA_DIR +# site_log_path is a holdover name. This could more accurately be called the "log_path" +config['site_log_path'] = TAGRADING_LOG_PATH +config['autograding_log_path'] = AUTOGRADING_LOG_PATH config['autograding_log_path'] = AUTOGRADING_LOG_PATH if not args.worker: config['sys_admin_email'] = SYS_ADMIN_EMAIL diff --git a/.setup/GENERATE_CONFIGS.py b/.setup/GENERATE_CONFIGS.py index 8d220fa14a3..b7b4994ebfa 100644 --- a/.setup/GENERATE_CONFIGS.py +++ b/.setup/GENERATE_CONFIGS.py @@ -2,15 +2,8 @@ import argparse from collections import OrderedDict -import grp -import json import os -import pwd -import secrets -import shutil -import string import tzlocal -import tempfile def get_input(question, default=""): @@ -39,23 +32,6 @@ def generate_config(submitty_install_dir, submitty_data_dir, worker, debug): # recommended (default) directory locations # FIXME: Check that directories exist and are readable/writeable? - SETUP_INSTALL_DIR = os.path.join(submitty_install_dir, '.setup') - CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') - CONFIG_INSTALL_DIR = os.path.join(submitty_install_dir, 'config') - SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') - EMAIL_JSON = os.path.join(CONFIG_INSTALL_DIR, 'email.json') - AUTHENTICATION_JSON = os.path.join(CONFIG_INSTALL_DIR, 'authentication.json') - SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') - - if not os.path.isdir(submitty_install_dir) or not os.access(submitty_install_dir, os.R_OK | os.W_OK): - raise SystemExit('Install directory {} does not exist or is not accessible'.format(submitty_install_dir)) - - if not os.path.isdir(submitty_data_dir) or not os.access(submitty_data_dir, os.R_OK | os.W_OK): - raise SystemExit('Data directory {} does not exist or is not accessible'.format(submitty_data_dir)) - - SETUP_INSTALL_DIR = os.path.join(submitty_install_dir, '.setup') - CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') - ########################################################################## authentication_methods = [ @@ -101,33 +77,6 @@ def generate_config(submitty_install_dir, submitty_data_dir, worker, debug): 'course_material_file_upload_limit_mb': 100 } - loaded_defaults = {} - if os.path.isfile(CONFIGURATION_JSON): - with open(CONFIGURATION_JSON) as conf_file: - loaded_defaults = json.load(conf_file) - if os.path.isfile(SUBMITTY_ADMIN_JSON): - with open(SUBMITTY_ADMIN_JSON) as submitty_admin_file: - loaded_defaults.update(json.load(submitty_admin_file)) - if os.path.isfile(EMAIL_JSON): - with open(EMAIL_JSON) as email_file: - loaded_defaults.update(json.load(email_file)) - - if os.path.isfile(AUTHENTICATION_JSON): - with open(AUTHENTICATION_JSON) as authentication_file: - loaded_defaults.update(json.load(authentication_file)) - - # no need to authenticate on a worker machine (no website) - if not worker: - if 'authentication_method' in loaded_defaults: - loaded_defaults['authentication_method'] = authentication_methods.index(loaded_defaults['authentication_method']) + 1 - - # grab anything not loaded in (useful for backwards compatibility if a new default is added that - # is not in an existing config file.) - for key in defaults.keys(): - if key not in loaded_defaults: - loaded_defaults[key] = defaults[key] - defaults = loaded_defaults - print("\nWelcome to the Submitty Homework Submission Server Configuration\n") DEBUGGING_ENABLED = debug is True @@ -327,17 +276,15 @@ def generate_config(submitty_install_dir, submitty_data_dir, worker, debug): config['vcs_url'] = VCS_URL config['submission_url'] = SUBMISSION_URL config['cgi_url'] = CGI_URL - + config['duck_special_effects'] = False config['institution_name'] = INSTITUTION_NAME config['institution_homepage'] = INSTITUTION_HOMEPAGE config['user_create_account'] = USER_CREATE_ACCOUNT -# site_log_path is a holdover name. This could more accurately be called the "log_path" -config['site_log_path'] = TAGRADING_LOG_PATH -config['autograding_log_path'] = AUTOGRADING_LOG_PATH - -if args.worker: - config['worker'] = 1 -else: - config['worker'] = 0 \ No newline at end of file + + + if worker: + config['worker'] = 1 + else: + config['worker'] = 0 \ No newline at end of file From f40f05c93e799f25d66deb255048dd5dd0bd1ab3 Mon Sep 17 00:00:00 2001 From: IDzyre Date: Mon, 26 Jan 2026 17:12:20 -0800 Subject: [PATCH 3/7] work --- .setup/CONFIGURE_SUBMITTY.py | 307 +++++++++++++++++++++-------------- .setup/GENERATE_CONFIGS.py | 170 ++++++------------- 2 files changed, 243 insertions(+), 234 deletions(-) diff --git a/.setup/CONFIGURE_SUBMITTY.py b/.setup/CONFIGURE_SUBMITTY.py index b4d0586ce35..f06e0c172cd 100644 --- a/.setup/CONFIGURE_SUBMITTY.py +++ b/.setup/CONFIGURE_SUBMITTY.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import argparse from collections import OrderedDict import grp @@ -9,24 +11,114 @@ import string import tempfile -from GENERATE_CONFIGS import generate_config +from y import generate_config + def get_uid(user): return pwd.getpwnam(user).pw_uid + def get_gid(user): return pwd.getpwnam(user).pw_gid + def get_ids(user): try: return get_uid(user), get_gid(user) except KeyError: raise SystemExit("ERROR: Could not find user: " + user) +class StrToBoolAction(argparse.Action): + """ + Custom action that parses strings to boolean values. All values that come + from bash are strings, and so need to parse that into the appropriate + bool value. + """ + def __init__(self, option_strings, dest, nargs=None, **kwargs): + if nargs is not None: + raise ValueError("nargs not allowed") + super().__init__(option_strings, dest, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values != '0' and values.lower() != 'false') + + ############################################################################## +# this script must be run by root or sudo if os.getuid() != 0: raise SystemExit('ERROR: This script must be run by root or sudo') +authentication_methods = [ + 'PamAuthentication', + 'DatabaseAuthentication', + 'LdapAuthentication', + 'SamlAuthentication' +] + +defaults = { + 'database_host': 'localhost', + 'database_port': 5432, + 'database_user': 'submitty_dbuser', + 'database_course_user': 'submitty_course_dbuser', + 'submission_url': '', + 'supervisor_user': 'submitty', + 'vcs_url': '', + 'authentication_method': 0, + 'institution_name' : '', + 'institution_homepage' : '', + 'user_create_account' : False, + 'timezone' : str(tzlocal.get_localzone()), + 'submitty_admin_username': '', + 'email_user': '', + 'email_password': '', + 'email_sender': 'submitty@myuniversity.edu', + 'email_reply_to': 'submitty_do_not_reply@myuniversity.edu', + 'email_server_hostname': 'mail.myuniversity.edu', + 'email_server_port': 25, + 'email_internal_domain': 'example.com', + 'course_code_requirements': "Please follow your school's convention for course code.", + 'sys_admin_email': '', + 'sys_admin_url': '', + 'ldap_options': { + 'url': '', + 'uid': '', + 'bind_dn': '' + }, + 'saml_options': { + 'name': '', + 'username_attribute': '' + }, + 'course_material_file_upload_limit_mb': 100, + 'user_id_requirements': { + 'any_user_id': True, + 'require_name': False, + 'min_length': 6, + 'max_length': 25, + # Example for Alyssa Hacker : hackal -- Allows for shorter names. If they are shorter, then it will just take the entire name. + # Example for Joseph Wo : wojo + 'name_requirements': { + 'given_first': False, + 'given_name': 2, + 'family_name': 4 + }, + 'require_email': False, + # If the user_id must contain part of the email. If whole_email is true, it must match the email. + # If whole_prefix is true, then the user_id must equal everything before the final @ sign. + # Else, it must be a certain number of characters of the prefix. + # Examples for myemail@email.com: + # Whole email: myemail@gmail.com + # Whole prefix: myemail + # Part of prefix: myemai + 'email_requirements': { + 'whole_email': False, + 'whole_prefix': False, + 'prefix_count': 6 + }, + 'accepted_emails': [ + 'gmail.com' + ] + } +} parser = argparse.ArgumentParser(description='Submitty configuration script', formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -41,50 +133,38 @@ def get_ids(user): parser.add_argument('--websocket-port', default=8443, type=int, help='Port to use for websocket') args = parser.parse_args() -# recommended names for special users & groups related to the SUBMITTY system -PHP_USER = 'submitty_php' -PHP_GROUP = 'submitty_php' -CGI_USER = 'submitty_cgi' -DAEMON_USER = 'submitty_daemon' -DAEMON_GROUP = 'submitty_daemon' +full_config = generate_config(defaults, args.worker, authentication_methods) -SUBMITTY_INSTALL_DIR = args.install_dir -SUBMITTY_DATA_DIR = args.data_dir +# determine location of SUBMITTY GIT repository +# this script (CONFIGURES_SUBMITTY.py) is in the top level directory of the repository +# (this command works even if we run configure from a different directory) SETUP_SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) SUBMITTY_REPOSITORY = os.path.dirname(SETUP_SCRIPT_DIRECTORY) -SETUP_INSTALL_DIR = os.path.join(SUBMITTY_INSTALL_DIR, '.setup') -SETUP_REPOSITORY_DIR = os.path.join(SUBMITTY_REPOSITORY, '.setup') - -INSTALL_FILE = os.path.join(SETUP_INSTALL_DIR, 'INSTALL_SUBMITTY.sh') -CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') -SITE_CONFIG_DIR = os.path.join(SUBMITTY_INSTALL_DIR, "site", "config") -CONFIG_INSTALL_DIR = os.path.join(SUBMITTY_INSTALL_DIR, 'config') -SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') -EMAIL_JSON = os.path.join(CONFIG_INSTALL_DIR, 'email.json') -AUTHENTICATION_JSON = os.path.join(CONFIG_INSTALL_DIR, 'authentication.json') - -#### -loaded_defaults = {} -if os.path.isfile(CONFIGURATION_JSON): - with open(CONFIGURATION_JSON) as conf_file: - loaded_defaults = json.load(conf_file) -if os.path.isfile(SUBMITTY_ADMIN_JSON): - with open(SUBMITTY_ADMIN_JSON) as submitty_admin_file: - loaded_defaults.update(json.load(submitty_admin_file)) -if os.path.isfile(EMAIL_JSON): - with open(EMAIL_JSON) as email_file: - loaded_defaults.update(json.load(email_file)) - -if os.path.isfile(AUTHENTICATION_JSON): - with open(AUTHENTICATION_JSON) as authentication_file: - loaded_defaults.update(json.load(authentication_file)) +# recommended (default) directory locations +# FIXME: Check that directories exist and are readable/writeable? +SUBMITTY_INSTALL_DIR = args.install_dir if not os.path.isdir(SUBMITTY_INSTALL_DIR) or not os.access(SUBMITTY_INSTALL_DIR, os.R_OK | os.W_OK): - raise SystemExit('Install directory {} does not exist or is not accessible'.format(SUBMITTY_INSTALL_DIR)) + raise SystemExit('Install directory {} does not exist or is not accessible'.format(SUBMITTY_INSTALL_DIR)) +SUBMITTY_DATA_DIR = args.data_dir if not os.path.isdir(SUBMITTY_DATA_DIR) or not os.access(SUBMITTY_DATA_DIR, os.R_OK | os.W_OK): raise SystemExit('Data directory {} does not exist or is not accessible'.format(SUBMITTY_DATA_DIR)) +TAGRADING_LOG_PATH = os.path.join(SUBMITTY_DATA_DIR, 'logs') +AUTOGRADING_LOG_PATH = os.path.join(SUBMITTY_DATA_DIR, 'logs', 'autograding') + +WEBSOCKET_PORT = args.websocket_port + +############################################################################## + +# recommended names for special users & groups related to the SUBMITTY system +PHP_USER = 'submitty_php' +PHP_GROUP = 'submitty_php' +CGI_USER = 'submitty_cgi' +DAEMON_USER = 'submitty_daemon' +DAEMON_GROUP = 'submitty_daemon' + if not args.worker: PHP_UID, PHP_GID = get_ids(PHP_USER) CGI_UID, CGI_GID = get_ids(CGI_USER) @@ -138,9 +218,57 @@ def get_ids(user): ############################################################################## +SETUP_INSTALL_DIR = os.path.join(SUBMITTY_INSTALL_DIR, '.setup') +SETUP_REPOSITORY_DIR = os.path.join(SUBMITTY_REPOSITORY, '.setup') + +INSTALL_FILE = os.path.join(SETUP_INSTALL_DIR, 'INSTALL_SUBMITTY.sh') +CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') +SITE_CONFIG_DIR = os.path.join(SUBMITTY_INSTALL_DIR, "site", "config") +CONFIG_INSTALL_DIR = os.path.join(SUBMITTY_INSTALL_DIR, 'config') +SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') +EMAIL_JSON = os.path.join(CONFIG_INSTALL_DIR, 'email.json') +AUTHENTICATION_JSON = os.path.join(CONFIG_INSTALL_DIR, 'authentication.json') + +############################################################################## + +loaded_defaults = {} +if os.path.isfile(CONFIGURATION_JSON): + with open(CONFIGURATION_JSON) as conf_file: + loaded_defaults = json.load(conf_file) +if os.path.isfile(SUBMITTY_ADMIN_JSON): + with open(SUBMITTY_ADMIN_JSON) as submitty_admin_file: + loaded_defaults.update(json.load(submitty_admin_file)) +if os.path.isfile(EMAIL_JSON): + with open(EMAIL_JSON) as email_file: + loaded_defaults.update(json.load(email_file)) + +if os.path.isfile(AUTHENTICATION_JSON): + with open(AUTHENTICATION_JSON) as authentication_file: + loaded_defaults.update(json.load(authentication_file)) + +# no need to authenticate on a worker machine (no website) +if not args.worker: + if 'authentication_method' in loaded_defaults: + loaded_defaults['authentication_method'] = authentication_methods.index(loaded_defaults['authentication_method']) + 1 + +# grab anything not loaded in (useful for backwards compatibility if a new default is added that +# is not in an existing config file.) +for key in defaults.keys(): + if key not in loaded_defaults: + loaded_defaults[key] = defaults[key] +defaults = loaded_defaults + +print("\nWelcome to the Submitty Homework Submission Server Configuration\n") +DEBUGGING_ENABLED = args.debug is True + +if DEBUGGING_ENABLED: + print('!! DEBUG MODE ENABLED !!') + print() + +if args.worker: + print("CONFIGURING SUBMITTY AS A WORKER !!") -generated_config = generate_config(SUBMITTY_INSTALL_DIR, SUBMITTY_DATA_DIR, args.worker, args.debug) ############################################################################## # make the installation setup directory @@ -153,8 +281,8 @@ def get_ids(user): ############################################################################## # WRITE CONFIG FILES IN ${SUBMITTY_INSTALL_DIR}/.setup -# Prompt for user configurable values +config = full_config['general_config'] config['submitty_install_dir'] = SUBMITTY_INSTALL_DIR config['submitty_repository'] = SUBMITTY_REPOSITORY config['submitty_data_dir'] = SUBMITTY_DATA_DIR @@ -166,6 +294,7 @@ def get_ids(user): config['first_untrusted_gid'] = FIRST_UNTRUSTED_UID config['num_grading_scheduler_workers'] = NUM_GRADING_SCHEDULER_WORKERS + config['daemon_user'] = DAEMON_USER config['daemon_uid'] = DAEMON_UID config['daemon_gid'] = DAEMON_GID @@ -178,8 +307,18 @@ def get_ids(user): config['daemonphpcgi_group'] = DAEMONPHPCGI_GROUP config['php_uid'] = PHP_UID config['php_gid'] = PHP_GID + config['websocket_port'] = WEBSOCKET_PORT + config['debugging_enabled'] = DEBUGGING_ENABLED + +# site_log_path is a holdover name. This could more accurately be called the "log_path" +config['site_log_path'] = TAGRADING_LOG_PATH +config['autograding_log_path'] = AUTOGRADING_LOG_PATH + +if args.worker: + config['worker'] = 1 +else: + config['worker'] = 0 - config['websocket_port'] = args.websocket_port with open(INSTALL_FILE, 'w') as open_file: def write(x=''): @@ -311,96 +450,37 @@ def write(x=''): # Write database json if not args.worker: - # config = OrderedDict() - config = generated_config['database'] - # config['authentication_method'] = AUTHENTICATION_METHOD - # config['database_host'] = DATABASE_HOST - # config['database_port'] = DATABASE_PORT - # config['database_user'] = DATABASE_USER - # config['database_password'] = DATABASE_PASS - # config['database_course_user'] = DATABASE_COURSE_USER - # config['database_course_password'] = DATABASE_COURSE_PASSWORD - # config['debugging_enabled'] = DEBUGGING_ENABLED - + database_config = full_config['database'] + database_config['debugging_enabled'] = DEBUGGING_ENABLED with open(DATABASE_JSON, 'w') as json_file: - json.dump(config, json_file, indent=2) + json.dump(database_config, json_file, indent=2) shutil.chown(DATABASE_JSON, 'root', DAEMONPHP_GROUP) os.chmod(DATABASE_JSON, 0o440) ############################################################################## # Write authentication json if not args.worker: - config = OrderedDict() - # config['authentication_method'] = AUTHENTICATION_METHOD - # config['ldap_options'] = LDAP_OPTIONS - # config['saml_options'] = SAML_OPTIONS - + auth_config = full_config['authentication'] with open(AUTHENTICATION_JSON, 'w') as json_file: - json.dump(config, json_file, indent=4) + json.dump(auth_config, json_file, indent=4) shutil.chown(AUTHENTICATION_JSON, 'root', DAEMONPHP_GROUP) os.chmod(AUTHENTICATION_JSON, 0o440) ############################################################################## -# Write submitty json -# Full documentation at submitty.org/... -user_id_requirements = { - "any_user_id": True, - "require_name": False, - "min_length": 6, - "max_length": 25, - # Example for Alyssa Hacker : hackal -- Allows for shorter names. If they are shorter, then it will just take the entire name. - # Example for Joseph Wo : wojo - "name_requirements": { - "given_first": False, - "given_name": 2, - "family_name": 4 - }, - "require_email": False, - # If the user_id must contain part of the email. If whole_email is true, it must match the email. - # If whole_prefix is true, then the user_id must equal everything before the final @ sign. - # Else, it must be a certain number of characters of the prefix. - # Examples for myemail@email.com: - # Whole email: myemail@gmail.com - # Whole prefix: myemail - # Part of prefix: myemai - "email_requirements": { - "whole_email": False, - "whole_prefix": False, - "prefix_count": 6 - }, - "accepted_emails": [ - "gmail.com" - ] -} - config = submitty_config +generated_submitty_config = full_config['submitty_config'] +config = config + submitty_config config['submitty_install_dir'] = SUBMITTY_INSTALL_DIR config['submitty_repository'] = SUBMITTY_REPOSITORY config['submitty_data_dir'] = SUBMITTY_DATA_DIR -# site_log_path is a holdover name. This could more accurately be called the "log_path" -config['site_log_path'] = TAGRADING_LOG_PATH config['autograding_log_path'] = AUTOGRADING_LOG_PATH -config['autograding_log_path'] = AUTOGRADING_LOG_PATH -if not args.worker: - config['sys_admin_email'] = SYS_ADMIN_EMAIL - config['sys_admin_url'] = SYS_ADMIN_URL # site_log_path is a holdover name. This could more accurately be called the "log_path" config['site_log_path'] = TAGRADING_LOG_PATH if not args.worker: - config['submission_url'] = SUBMISSION_URL - config['vcs_url'] = VCS_URL - config['cgi_url'] = CGI_URL config['websocket_port'] = WEBSOCKET_PORT - config['institution_name'] = INSTITUTION_NAME - config['institution_homepage'] = INSTITUTION_HOMEPAGE - config['timezone'] = TIMEZONE - config['default_locale'] = DEFAULT_LOCALE config['duck_special_effects'] = False - config['course_material_file_upload_limit_mb'] = COURSE_MATERIAL_UPLOAD_LIMIT_MB - config['user_create_account'] = USER_CREATE_ACCOUNT - config['user_id_requirements'] = user_id_requirements config['worker'] = True if args.worker == 1 else False @@ -430,7 +510,7 @@ def write(x=''): config['daemoncgi_group'] = DAEMONCGI_GROUP config['daemonphpcgi_group'] = DAEMONPHPCGI_GROUP else: - config['supervisor_user'] = SUPERVISOR_USER + config['supervisor_user'] = full_config['general']['supervisor_user'] with open(SUBMITTY_USERS_JSON, 'w') as json_file: json.dump(config, json_file, indent=2) @@ -455,7 +535,7 @@ def write(x=''): if not args.worker: config = OrderedDict() - config['submitty_admin_username'] = SUBMITTY_ADMIN_USERNAME + config['submitty_admin_username'] = full_config['admin_username'] with open(SUBMITTY_ADMIN_JSON, 'w') as json_file: json.dump(config, json_file, indent=2) @@ -466,16 +546,7 @@ def write(x=''): # Write email json if not args.worker: - # config = OrderedDict() - # config['email_enabled'] = EMAIL_ENABLED - # config['email_user'] = EMAIL_USER - # config['email_password'] = EMAIL_PASSWORD - # config['email_sender'] = EMAIL_SENDER - # config['email_reply_to'] = EMAIL_REPLY_TO - # config['email_server_hostname'] = EMAIL_SERVER_HOSTNAME - # config['email_server_port'] = EMAIL_SERVER_PORT - # config['email_internal_domain'] = EMAIL_INTERNAL_DOMAIN - + config = full_config['email'] with open(EMAIL_JSON, 'w') as json_file: json.dump(config, json_file, indent=2) shutil.chown(EMAIL_JSON, 'root', DAEMONPHP_GROUP) @@ -487,4 +558,4 @@ def write(x=''): print(f' sudo {INSTALL_FILE}') print(' or') print(f' sudo {INSTALL_FILE} clean') -print("\n") +print("\n") \ No newline at end of file diff --git a/.setup/GENERATE_CONFIGS.py b/.setup/GENERATE_CONFIGS.py index b7b4994ebfa..7197ee7bde6 100644 --- a/.setup/GENERATE_CONFIGS.py +++ b/.setup/GENERATE_CONFIGS.py @@ -1,10 +1,5 @@ -#!/usr/bin/env python3 - -import argparse from collections import OrderedDict import os -import tzlocal - def get_input(question, default=""): add = "[{}] ".format(default) if default != "" else "" @@ -14,82 +9,11 @@ def get_input(question, default=""): return user -class StrToBoolAction(argparse.Action): - """ - Custom action that parses strings to boolean values. All values that come - from bash are strings, and so need to parse that into the appropriate - bool value. - """ - def __init__(self, option_strings, dest, nargs=None, **kwargs): - if nargs is not None: - raise ValueError("nargs not allowed") - super().__init__(option_strings, dest, **kwargs) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, values != '0' and values.lower() != 'false') - -def generate_config(submitty_install_dir, submitty_data_dir, worker, debug): - # recommended (default) directory locations - # FIXME: Check that directories exist and are readable/writeable? - - ########################################################################## - - authentication_methods = [ - 'PamAuthentication', - 'DatabaseAuthentication', - 'LdapAuthentication', - 'SamlAuthentication' - ] - - defaults = { - 'database_host': 'localhost', - 'database_port': 5432, - 'database_user': 'submitty_dbuser', - 'database_course_user': 'submitty_course_dbuser', - 'submission_url': '', - 'supervisor_user': 'submitty', - 'vcs_url': '', - 'authentication_method': 0, - 'institution_name' : '', - 'institution_homepage' : '', - 'user_create_account' : False, - 'timezone' : str(tzlocal.get_localzone()), - 'submitty_admin_username': '', - 'email_user': '', - 'email_password': '', - 'email_sender': 'submitty@myuniversity.edu', - 'email_reply_to': 'submitty_do_not_reply@myuniversity.edu', - 'email_server_hostname': 'mail.myuniversity.edu', - 'email_server_port': 25, - 'email_internal_domain': 'example.com', - 'course_code_requirements': "Please follow your school's convention for course code.", - 'sys_admin_email': '', - 'sys_admin_url': '', - 'ldap_options': { - 'url': '', - 'uid': '', - 'bind_dn': '' - }, - 'saml_options': { - 'name': '', - 'username_attribute': '' - }, - 'course_material_file_upload_limit_mb': 100 - } - - print("\nWelcome to the Submitty Homework Submission Server Configuration\n") - DEBUGGING_ENABLED = debug is True - - if DEBUGGING_ENABLED: - print('!! DEBUG MODE ENABLED !!') - print() - - if worker: - print("CONFIGURING SUBMITTY AS A WORKER !!") +def generate_config(defaults, worker, authentication_methods): print('Hit enter to use default in []') print() - + if worker: SUPERVISOR_USER = get_input('What is the id for your submitty user?', defaults['supervisor_user']) print('SUPERVISOR USER : {}'.format(SUPERVISOR_USER)) @@ -231,16 +155,35 @@ def generate_config(submitty_install_dir, submitty_data_dir, worker, debug): EMAIL_INTERNAL_DOMAIN = defaults['email_internal_domain'] break print() - full_config = OrderedDict() - config = OrderedDict() + general_config = OrderedDict() + database_config = OrderedDict() + authentication_config = OrderedDict() + submitty_config = OrderedDict() + email_config = OrderedDict() if worker: - config['supervisor_user'] = SUPERVISOR_USER - else: - database_config = OrderedDict() - auth_config = OrderedDict() - email_config = OrderedDict() - + general_config['supervisor_user'] = SUPERVISOR_USER + + else: + + general_config['database_host'] = DATABASE_HOST + general_config['database_port'] = DATABASE_PORT + general_config['database_user'] = DATABASE_USER + general_config['database_password'] = DATABASE_PASS + general_config['database_course_user'] = DATABASE_COURSE_USER + general_config['database_course_password'] = DATABASE_COURSE_PASSWORD + general_config['timezone'] = TIMEZONE + general_config['default_locale'] = DEFAULT_LOCALE + + general_config['authentication_method'] = AUTHENTICATION_METHOD + general_config['vcs_url'] = VCS_URL + general_config['submission_url'] = SUBMISSION_URL + general_config['cgi_url'] = CGI_URL + + general_config['institution_name'] = INSTITUTION_NAME + general_config['institution_homepage'] = INSTITUTION_HOMEPAGE + general_config['user_create_account'] = USER_CREATE_ACCOUNT + database_config['authentication_method'] = AUTHENTICATION_METHOD database_config['database_host'] = DATABASE_HOST database_config['database_port'] = DATABASE_PORT @@ -248,17 +191,25 @@ def generate_config(submitty_install_dir, submitty_data_dir, worker, debug): database_config['database_password'] = DATABASE_PASS database_config['database_course_user'] = DATABASE_COURSE_USER database_config['database_course_password'] = DATABASE_COURSE_PASSWORD - database_config['debugging_enabled'] = DEBUGGING_ENABLED - - full_config['database'] = database_config + + authentication_config['authentication_method'] = AUTHENTICATION_METHOD + authentication_config['ldap_options'] = LDAP_OPTIONS + authentication_config['saml_options'] = SAML_OPTIONS + + submitty_config['sys_admin_email'] = SYS_ADMIN_EMAIL + submitty_config['sys_admin_url'] = SYS_ADMIN_URL + submitty_config['submission_url'] = SUBMISSION_URL + submitty_config['vcs_url'] = VCS_URL + submitty_config['cgi_url'] = CGI_URL + submitty_config['institution_name'] = INSTITUTION_NAME + submitty_config['institution_homepage'] = INSTITUTION_HOMEPAGE + submitty_config['timezone'] = TIMEZONE + submitty_config['default_locale'] = DEFAULT_LOCALE + + submitty_config['course_material_file_upload_limit_mb'] = COURSE_MATERIAL_UPLOAD_LIMIT_MB + submitty_config['user_create_account'] = USER_CREATE_ACCOUNT + submitty_config['user_id_requirements'] = defaults['user_id_requirements'] - auth_config['authentication_method'] = AUTHENTICATION_METHOD - auth_config['ldap_options'] = LDAP_OPTIONS - auth_config['saml_options'] = SAML_OPTIONS - - full_config['authentication'] = auth_config - - email_config = OrderedDict() email_config['email_enabled'] = EMAIL_ENABLED email_config['email_user'] = EMAIL_USER email_config['email_password'] = EMAIL_PASSWORD @@ -267,24 +218,11 @@ def generate_config(submitty_install_dir, submitty_data_dir, worker, debug): email_config['email_server_hostname'] = EMAIL_SERVER_HOSTNAME email_config['email_server_port'] = EMAIL_SERVER_PORT email_config['email_internal_domain'] = EMAIL_INTERNAL_DOMAIN - email_config['timezone'] = TIMEZONE - email_config['default_locale'] = DEFAULT_LOCALE - - full_config['email'] = email_config - - config['authentication_method'] = AUTHENTICATION_METHOD - config['vcs_url'] = VCS_URL - config['submission_url'] = SUBMISSION_URL - config['cgi_url'] = CGI_URL - config['duck_special_effects'] = False - config['institution_name'] = INSTITUTION_NAME - config['institution_homepage'] = INSTITUTION_HOMEPAGE - config['user_create_account'] = USER_CREATE_ACCOUNT - - - - if worker: - config['worker'] = 1 - else: - config['worker'] = 0 \ No newline at end of file + full_config['database'] = database_config + full_config['authentication'] = authentication_config + full_config['submitty'] = submitty_config + full_config['general'] = general_config + full_config['email'] = email_config + full_config['admin_username'] = SUBMITTY_ADMIN_USERNAME + return full_config \ No newline at end of file From 29c8b2b6a067432a8a51b76f19427699e1842e2e Mon Sep 17 00:00:00 2001 From: IDzyre Date: Tue, 27 Jan 2026 19:26:00 -0800 Subject: [PATCH 4/7] Fix location of source --- .setup/CONFIGURE_SUBMITTY.py | 2 +- .setup/install_system.sh | 8 ++++---- Vagrantfile | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.setup/CONFIGURE_SUBMITTY.py b/.setup/CONFIGURE_SUBMITTY.py index f06e0c172cd..8375cb77a73 100644 --- a/.setup/CONFIGURE_SUBMITTY.py +++ b/.setup/CONFIGURE_SUBMITTY.py @@ -11,7 +11,7 @@ import string import tempfile -from y import generate_config +from GENERATE_CONFIGS import generate_config def get_uid(user): diff --git a/.setup/install_system.sh b/.setup/install_system.sh index 3a478b3b395..c24d84a2152 100644 --- a/.setup/install_system.sh +++ b/.setup/install_system.sh @@ -23,10 +23,6 @@ trap 'err_message' ERR # print commands as we execute and fail early set -ev -# Sources -source ${CURRENT_DIR}/bin/versions.sh -source ${CURRENT_DIR}/distro_setup/setup_distro.sh - # this script must be run by root or sudo if [[ "$UID" -ne "0" ]] ; then echo "ERROR: This script must be run by root or sudo" @@ -45,6 +41,10 @@ LICHEN_REPOSITORY=/usr/local/submitty/GIT_CHECKOUT/Lichen SUBMITTY_INSTALL_DIR=/usr/local/submitty SUBMITTY_DATA_DIR=/var/local/submitty +# Sources +source ${CURRENT_DIR}/bin/versions.sh +source ${CURRENT_DIR}/distro_setup/setup_distro.sh + # create directory and fix permissions mkdir -p ${SUBMITTY_DATA_DIR} diff --git a/Vagrantfile b/Vagrantfile index 388df422c72..a2cb32fab7f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -143,7 +143,7 @@ Vagrant.configure(2) do |config| apple_silicon = Vagrant::Util::Platform.darwin? && (arm || (`sysctl -n machdep.cpu.brand_string`.chomp.start_with? 'Apple M')) use_prebuilt_version = !ENV.fetch('PREBUILT_VERSION', '').empty? custom_box = ENV.has_key?('VAGRANT_BOX') - base_box = ENV.has_key?('BASE_BOX') || ENV.has_key?('FROM_SCRATCH') || apple_silicon || arm + base_box = ENV.has_key?('BASE_BOX') || ENV.has_key?('FROM_SCRATCH') || apple_silicon || arm || true # The time in seconds that Vagrant will wait for the machine to boot and be accessible. config.vm.boot_timeout = 600 From be5518cc0d6458bd556e4a9b005c4eb92621267c Mon Sep 17 00:00:00 2001 From: IDzyre Date: Wed, 28 Jan 2026 14:04:57 -0800 Subject: [PATCH 5/7] Separate users setup --- .setup/CONFIGURE_SUBMITTY.py | 167 +-------------- .setup/GENERATE_CONFIGS.py | 4 +- .setup/USERS_SETUP.py | 201 ++++++++++++++++++ .setup/distro_setup/ubuntu/22.04/variables.sh | 6 + .setup/distro_setup/variables.sh | 20 ++ .setup/install_system.sh | 31 +-- Vagrantfile | 2 +- 7 files changed, 256 insertions(+), 175 deletions(-) create mode 100644 .setup/USERS_SETUP.py create mode 100644 .setup/distro_setup/ubuntu/22.04/variables.sh create mode 100644 .setup/distro_setup/variables.sh diff --git a/.setup/CONFIGURE_SUBMITTY.py b/.setup/CONFIGURE_SUBMITTY.py index 8375cb77a73..a3ceb891c91 100644 --- a/.setup/CONFIGURE_SUBMITTY.py +++ b/.setup/CONFIGURE_SUBMITTY.py @@ -2,32 +2,15 @@ import argparse from collections import OrderedDict -import grp import json import os -import pwd import secrets import shutil import string -import tempfile +import tzlocal from GENERATE_CONFIGS import generate_config - -def get_uid(user): - return pwd.getpwnam(user).pw_uid - - -def get_gid(user): - return pwd.getpwnam(user).pw_gid - - -def get_ids(user): - try: - return get_uid(user), get_gid(user) - except KeyError: - raise SystemExit("ERROR: Could not find user: " + user) - class StrToBoolAction(argparse.Action): """ Custom action that parses strings to boolean values. All values that come @@ -133,7 +116,6 @@ def __call__(self, parser, namespace, values, option_string=None): parser.add_argument('--websocket-port', default=8443, type=int, help='Port to use for websocket') args = parser.parse_args() -full_config = generate_config(defaults, args.worker, authentication_methods) # determine location of SUBMITTY GIT repository # this script (CONFIGURES_SUBMITTY.py) is in the top level directory of the repository @@ -164,51 +146,7 @@ def __call__(self, parser, namespace, values, option_string=None): CGI_USER = 'submitty_cgi' DAEMON_USER = 'submitty_daemon' DAEMON_GROUP = 'submitty_daemon' - -if not args.worker: - PHP_UID, PHP_GID = get_ids(PHP_USER) - CGI_UID, CGI_GID = get_ids(CGI_USER) - # System Groups - DAEMONPHP_GROUP = 'submitty_daemonphp' - try: - grp.getgrnam(DAEMONPHP_GROUP) - except KeyError: - raise SystemExit("ERROR: Could not find group: " + DAEMONPHP_GROUP) - DAEMONCGI_GROUP = 'submitty_daemoncgi' - try: - grp.getgrnam(DAEMONCGI_GROUP) - except KeyError: - raise SystemExit("ERROR: Could not find group: " + DAEMONCGI_GROUP) - DAEMONPHPCGI_GROUP = 'submitty_daemonphpcgi' - try: - grp.getgrnam(DAEMONPHPCGI_GROUP) - except KeyError: - raise SystemExit("ERROR: Could not find group: " + DAEMONPHPCGI_GROUP) - -DAEMON_UID, DAEMON_GID = get_ids(DAEMON_USER) - COURSE_BUILDERS_GROUP = 'submitty_course_builders' -try: - grp.getgrnam(COURSE_BUILDERS_GROUP) -except KeyError: - raise SystemExit("ERROR: Could not find group: " + COURSE_BUILDERS_GROUP) - -############################################################################## - -# This is the upper limit of the number of parallel grading threads on -# this machine -NUM_UNTRUSTED = 60 - -FIRST_UNTRUSTED_UID, FIRST_UNTRUSTED_GID = get_ids('untrusted00') - -# confirm that the uid/gid of the untrusted users are sequential -for i in range(1, NUM_UNTRUSTED): - untrusted_user = "untrusted{:0=2d}".format(i) - uid, gid = get_ids(untrusted_user) - if uid != FIRST_UNTRUSTED_UID + i: - raise SystemExit('CONFIGURATION ERROR: untrusted UID not sequential: ' + untrusted_user) - elif gid != FIRST_UNTRUSTED_GID + i: - raise SystemExit('CONFIGURATION ERROR: untrusted GID not sequential: ' + untrusted_user) ############################################################################## @@ -225,6 +163,7 @@ def __call__(self, parser, namespace, values, option_string=None): CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') SITE_CONFIG_DIR = os.path.join(SUBMITTY_INSTALL_DIR, "site", "config") CONFIG_INSTALL_DIR = os.path.join(SUBMITTY_INSTALL_DIR, 'config') +SUBMITTY_USERS_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_users.json') SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') EMAIL_JSON = os.path.join(CONFIG_INSTALL_DIR, 'email.json') AUTHENTICATION_JSON = os.path.join(CONFIG_INSTALL_DIR, 'authentication.json') @@ -268,6 +207,7 @@ def __call__(self, parser, namespace, values, option_string=None): if args.worker: print("CONFIGURING SUBMITTY AS A WORKER !!") +full_config = generate_config(defaults, args.worker, authentication_methods) ############################################################################## # make the installation setup directory @@ -275,38 +215,28 @@ def __call__(self, parser, namespace, values, option_string=None): if os.path.isdir(SETUP_INSTALL_DIR): shutil.rmtree(SETUP_INSTALL_DIR) os.makedirs(SETUP_INSTALL_DIR, exist_ok=True) - -shutil.chown(SETUP_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) os.chmod(SETUP_INSTALL_DIR, 0o751) ############################################################################## # WRITE CONFIG FILES IN ${SUBMITTY_INSTALL_DIR}/.setup +config = full_config['general'] + +if args.worker: + config['supervisor_user'] = config['supervisor_user'] + with open(SUBMITTY_USERS_JSON, 'w') as json_file: + json.dump(config, json_file, indent=2) -config = full_config['general_config'] config['submitty_install_dir'] = SUBMITTY_INSTALL_DIR config['submitty_repository'] = SUBMITTY_REPOSITORY config['submitty_data_dir'] = SUBMITTY_DATA_DIR - config['course_builders_group'] = COURSE_BUILDERS_GROUP - -config['num_untrusted'] = NUM_UNTRUSTED -config['first_untrusted_uid'] = FIRST_UNTRUSTED_UID -config['first_untrusted_gid'] = FIRST_UNTRUSTED_UID config['num_grading_scheduler_workers'] = NUM_GRADING_SCHEDULER_WORKERS - - config['daemon_user'] = DAEMON_USER -config['daemon_uid'] = DAEMON_UID -config['daemon_gid'] = DAEMON_GID if not args.worker: config['php_user'] = PHP_USER config['cgi_user'] = CGI_USER - config['daemonphp_group'] = DAEMONPHP_GROUP - config['daemoncgi_group'] = DAEMONCGI_GROUP - config['daemonphpcgi_group'] = DAEMONPHPCGI_GROUP - config['php_uid'] = PHP_UID - config['php_gid'] = PHP_GID + config['websocket_port'] = WEBSOCKET_PORT config['debugging_enabled'] = DEBUGGING_ENABLED @@ -352,43 +282,6 @@ def write(x=''): except FileNotFoundError: pass -#Rescue the autograding_workers and _containers files if they exist. -rescued = list() -tmp_folder = tempfile.mkdtemp() -if not args.worker: - for full_file_name, file_name in [(WORKERS_JSON, 'autograding_workers.json'), (CONTAINERS_JSON, 'autograding_containers.json')]: - if os.path.isfile(full_file_name): - #make a tmp folder and copy autograding workers to it - tmp_file = os.path.join(tmp_folder, file_name) - shutil.move(full_file_name, tmp_file) - rescued.append((full_file_name, tmp_file)) - -IGNORED_FILES_AND_DIRS = ['saml', 'login.md'] - -if os.path.isdir(CONFIG_INSTALL_DIR): - for file in os.scandir(CONFIG_INSTALL_DIR): - if file.name not in IGNORED_FILES_AND_DIRS: - if file.is_file(): - os.remove(os.path.join(CONFIG_INSTALL_DIR, file.name)) - else: - os.rmdir(os.path.join(CONFIG_INSTALL_DIR, file.name)) -elif os.path.exists(CONFIG_INSTALL_DIR): - os.remove(CONFIG_INSTALL_DIR) -os.makedirs(CONFIG_INSTALL_DIR, exist_ok=True) -shutil.chown(CONFIG_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) -os.chmod(CONFIG_INSTALL_DIR, 0o755) - -# Finish rescuing files. -for full_file_name, tmp_file_name in rescued: - #copy autograding workers back - shutil.move(tmp_file_name, full_file_name) - #make sure the permissions are correct. - shutil.chown(full_file_name, 'root', DAEMON_GID) - os.chmod(full_file_name, 0o660) - -#remove the tmp folder -os.removedirs(tmp_folder) - ############################################################################## # WRITE CONFIG FILES IN ${SUBMITTY_INSTALL_DIR}/config @@ -443,9 +336,6 @@ def write(x=''): for file in [WORKERS_JSON, CONTAINERS_JSON]: os.chmod(file, 0o660) - shutil.chown(WORKERS_JSON, PHP_USER, DAEMON_GID) - shutil.chown(CONTAINERS_JSON, group=DAEMONPHP_GROUP) - ############################################################################## # Write database json @@ -454,7 +344,6 @@ def write(x=''): database_config['debugging_enabled'] = DEBUGGING_ENABLED with open(DATABASE_JSON, 'w') as json_file: json.dump(database_config, json_file, indent=2) - shutil.chown(DATABASE_JSON, 'root', DAEMONPHP_GROUP) os.chmod(DATABASE_JSON, 0o440) ############################################################################## @@ -463,13 +352,12 @@ def write(x=''): auth_config = full_config['authentication'] with open(AUTHENTICATION_JSON, 'w') as json_file: json.dump(auth_config, json_file, indent=4) - shutil.chown(AUTHENTICATION_JSON, 'root', DAEMONPHP_GROUP) os.chmod(AUTHENTICATION_JSON, 0o440) ############################################################################## config = submitty_config -generated_submitty_config = full_config['submitty_config'] +generated_submitty_config = full_config['submitty'] config = config + submitty_config config['submitty_install_dir'] = SUBMITTY_INSTALL_DIR config['submitty_repository'] = SUBMITTY_REPOSITORY @@ -488,36 +376,6 @@ def write(x=''): json.dump(config, json_file, indent=2) os.chmod(SUBMITTY_JSON, 0o444) -############################################################################## -# Write users json - -config = OrderedDict() -config['num_grading_scheduler_workers'] = NUM_GRADING_SCHEDULER_WORKERS -config['num_untrusted'] = NUM_UNTRUSTED -config['first_untrusted_uid'] = FIRST_UNTRUSTED_UID -config['first_untrusted_gid'] = FIRST_UNTRUSTED_UID -config['daemon_uid'] = DAEMON_UID -config['daemon_gid'] = DAEMON_GID -config['daemon_user'] = DAEMON_USER -config['course_builders_group'] = COURSE_BUILDERS_GROUP - -if not args.worker: - config['php_uid'] = PHP_UID - config['php_gid'] = PHP_GID - config['php_user'] = PHP_USER - config['cgi_user'] = CGI_USER - config['daemonphp_group'] = DAEMONPHP_GROUP - config['daemoncgi_group'] = DAEMONCGI_GROUP - config['daemonphpcgi_group'] = DAEMONPHPCGI_GROUP -else: - config['supervisor_user'] = full_config['general']['supervisor_user'] - -with open(SUBMITTY_USERS_JSON, 'w') as json_file: - json.dump(config, json_file, indent=2) -shutil.chown(SUBMITTY_USERS_JSON, 'root', DAEMON_GROUP if args.worker else DAEMONPHP_GROUP) - -os.chmod(SUBMITTY_USERS_JSON, 0o440) - ############################################################################## # Write secrets_submitty_php json @@ -527,7 +385,6 @@ def write(x=''): config['session'] = ''.join(secrets.choice(characters) for _ in range(64)) with open(SECRETS_PHP_JSON, 'w') as json_file: json.dump(config, json_file, indent=2) - shutil.chown(SECRETS_PHP_JSON, 'root', PHP_GROUP) os.chmod(SECRETS_PHP_JSON, 0o440) ############################################################################## @@ -539,7 +396,6 @@ def write(x=''): with open(SUBMITTY_ADMIN_JSON, 'w') as json_file: json.dump(config, json_file, indent=2) - shutil.chown(SUBMITTY_ADMIN_JSON, 'root', DAEMON_GROUP) os.chmod(SUBMITTY_ADMIN_JSON, 0o440) ############################################################################## @@ -549,7 +405,6 @@ def write(x=''): config = full_config['email'] with open(EMAIL_JSON, 'w') as json_file: json.dump(config, json_file, indent=2) - shutil.chown(EMAIL_JSON, 'root', DAEMONPHP_GROUP) os.chmod(EMAIL_JSON, 0o440) ############################################################################## diff --git a/.setup/GENERATE_CONFIGS.py b/.setup/GENERATE_CONFIGS.py index 7197ee7bde6..13b3894de97 100644 --- a/.setup/GENERATE_CONFIGS.py +++ b/.setup/GENERATE_CONFIGS.py @@ -8,8 +8,6 @@ def get_input(question, default=""): user = default return user - - def generate_config(defaults, worker, authentication_methods): print('Hit enter to use default in []') print() @@ -19,7 +17,7 @@ def generate_config(defaults, worker, authentication_methods): print('SUPERVISOR USER : {}'.format(SUPERVISOR_USER)) else: DATABASE_HOST = get_input('What is the database host?', defaults['database_host']) - print() + print(DATABASE_HOST) if not os.path.isdir(DATABASE_HOST): DATABASE_PORT = int(get_input('What is the database port?', defaults['database_port'])) diff --git a/.setup/USERS_SETUP.py b/.setup/USERS_SETUP.py new file mode 100644 index 00000000000..884c24b2a5c --- /dev/null +++ b/.setup/USERS_SETUP.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 + +import argparse +import grp +import json +import pwd +from collections import OrderedDict +import shutil +import tempfile +import os + +def get_uid(user): + return pwd.getpwnam(user).pw_uid + + +def get_gid(user): + return pwd.getpwnam(user).pw_gid + + +def get_ids(user): + try: + return get_uid(user), get_gid(user) + except KeyError: + raise SystemExit("ERROR: Could not find user: " + user) + +parser = argparse.ArgumentParser(description='Submitty configuration script', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--worker', action='store_true', default=False, help='Configure Submitty with autograding only') +parser.add_argument('--install-dir', default='/usr/local/submitty', help='Set the install directory for Submitty') +args = parser.parse_args() + +########################################################## + +# recommended names for special users & groups related to the SUBMITTY system +PHP_USER = 'submitty_php' +PHP_GROUP = 'submitty_php' +CGI_USER = 'submitty_cgi' +DAEMON_USER = 'submitty_daemon' +DAEMON_GROUP = 'submitty_daemon' +COURSE_BUILDERS_GROUP = 'submitty_course_builders' + +############################################################################## + +# adjust this number depending on the # of processors +# available on your hardware +NUM_GRADING_SCHEDULER_WORKERS = 5 + +############################################################################## + +SETUP_INSTALL_DIR = os.path.join(args.install_dir, '.setup') + +INSTALL_FILE = os.path.join(SETUP_INSTALL_DIR, 'INSTALL_SUBMITTY.sh') +CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') +CONFIG_INSTALL_DIR = os.path.join(args.install_dir, 'config') + +EMAIL_JSON = os.path.join(CONFIG_INSTALL_DIR, 'email.json') +AUTHENTICATION_JSON = os.path.join(CONFIG_INSTALL_DIR, 'authentication.json') +DATABASE_JSON = os.path.join(CONFIG_INSTALL_DIR, 'database.json') +SUBMITTY_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty.json') +SUBMITTY_USERS_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_users.json') +WORKERS_JSON = os.path.join(CONFIG_INSTALL_DIR, 'autograding_workers.json') +SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') +CONTAINERS_JSON = os.path.join(CONFIG_INSTALL_DIR, 'autograding_containers.json') +SECRETS_PHP_JSON = os.path.join(CONFIG_INSTALL_DIR, 'secrets_submitty_php.json') + +if not args.worker: + PHP_UID, PHP_GID = get_ids(PHP_USER) + CGI_UID, CGI_GID = get_ids(CGI_USER) + # System Groups + DAEMONPHP_GROUP = 'submitty_daemonphp' + try: + grp.getgrnam(DAEMONPHP_GROUP) + except KeyError: + raise SystemExit("ERROR: Could not find group: " + DAEMONPHP_GROUP) + DAEMONCGI_GROUP = 'submitty_daemoncgi' + try: + grp.getgrnam(DAEMONCGI_GROUP) + except KeyError: + raise SystemExit("ERROR: Could not find group: " + DAEMONCGI_GROUP) + DAEMONPHPCGI_GROUP = 'submitty_daemonphpcgi' + try: + grp.getgrnam(DAEMONPHPCGI_GROUP) + except KeyError: + raise SystemExit("ERROR: Could not find group: " + DAEMONPHPCGI_GROUP) + +DAEMON_UID, DAEMON_GID = get_ids(DAEMON_USER) + + +try: + grp.getgrnam(COURSE_BUILDERS_GROUP) +except KeyError: + raise SystemExit("ERROR: Could not find group: " + COURSE_BUILDERS_GROUP) + +############################################################################## + +# This is the upper limit of the number of parallel grading threads on +# this machine +NUM_UNTRUSTED = 60 + +FIRST_UNTRUSTED_UID, FIRST_UNTRUSTED_GID = get_ids('untrusted00') + +# confirm that the uid/gid of the untrusted users are sequential +for i in range(1, NUM_UNTRUSTED): + untrusted_user = "untrusted{:0=2d}".format(i) + uid, gid = get_ids(untrusted_user) + if uid != FIRST_UNTRUSTED_UID + i: + raise SystemExit('CONFIGURATION ERROR: untrusted UID not sequential: ' + untrusted_user) + elif gid != FIRST_UNTRUSTED_GID + i: + raise SystemExit('CONFIGURATION ERROR: untrusted GID not sequential: ' + untrusted_user) + +with open(SUBMITTY_JSON, 'r') as json_file: + config = json.load(json_file, object_pairs_hook=OrderedDict) + config['num_untrusted'] = NUM_UNTRUSTED + config['first_untrusted_uid'] = FIRST_UNTRUSTED_UID + config['first_untrusted_gid'] = FIRST_UNTRUSTED_UID + + config['daemon_uid'] = DAEMON_UID + config['daemon_gid'] = DAEMON_GID + + # not worker + config['daemonphp_group'] = DAEMONPHP_GROUP + config['daemoncgi_group'] = DAEMONCGI_GROUP + config['daemonphpcgi_group'] = DAEMONPHPCGI_GROUP + config['php_uid'] = PHP_UID + config['php_gid'] = PHP_GID + +#Rescue the autograding_workers and _containers files if they exist. +rescued = list() +tmp_folder = tempfile.mkdtemp() +if not args.worker: + for full_file_name, file_name in [(WORKERS_JSON, 'autograding_workers.json'), (CONTAINERS_JSON, 'autograding_containers.json')]: + if os.path.isfile(full_file_name): + #make a tmp folder and copy autograding workers to it + tmp_file = os.path.join(tmp_folder, file_name) + shutil.move(full_file_name, tmp_file) + rescued.append((full_file_name, tmp_file)) + +IGNORED_FILES_AND_DIRS = ['saml', 'login.md'] + +if os.path.isdir(CONFIG_INSTALL_DIR): + for file in os.scandir(CONFIG_INSTALL_DIR): + if file.name not in IGNORED_FILES_AND_DIRS: + if file.is_file(): + os.remove(os.path.join(CONFIG_INSTALL_DIR, file.name)) + else: + os.rmdir(os.path.join(CONFIG_INSTALL_DIR, file.name)) +elif os.path.exists(CONFIG_INSTALL_DIR): + os.remove(CONFIG_INSTALL_DIR) +os.makedirs(CONFIG_INSTALL_DIR, exist_ok=True) +shutil.chown(CONFIG_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) +os.chmod(CONFIG_INSTALL_DIR, 0o755) + +# Finish rescuing files. +for full_file_name, tmp_file_name in rescued: + #copy autograding workers back + shutil.move(tmp_file_name, full_file_name) + #make sure the permissions are correct. + shutil.chown(full_file_name, 'root', DAEMON_GID) + os.chmod(full_file_name, 0o660) + +#remove the tmp folder +os.removedirs(tmp_folder) + +############################################################################## +# Write users json + +config = OrderedDict() +config['num_grading_scheduler_workers'] = NUM_GRADING_SCHEDULER_WORKERS +config['num_untrusted'] = NUM_UNTRUSTED +config['first_untrusted_uid'] = FIRST_UNTRUSTED_UID +config['first_untrusted_gid'] = FIRST_UNTRUSTED_UID +config['daemon_uid'] = DAEMON_UID +config['daemon_gid'] = DAEMON_GID +config['daemon_user'] = DAEMON_USER +config['course_builders_group'] = COURSE_BUILDERS_GROUP + +if not args.worker: + config['php_uid'] = PHP_UID + config['php_gid'] = PHP_GID + config['php_user'] = PHP_USER + config['cgi_user'] = CGI_USER + config['daemonphp_group'] = DAEMONPHP_GROUP + config['daemoncgi_group'] = DAEMONCGI_GROUP + config['daemonphpcgi_group'] = DAEMONPHPCGI_GROUP +if os.path.exists(SUBMITTY_USERS_JSON): + with open(SUBMITTY_USERS_JSON, 'r') as json_file: + config_file = json.load(json_file) + config['supervisor_user'] = config_file['supervisor_user'] +with open(SUBMITTY_USERS_JSON, 'w') as json_file: + json.dump(config, json_file, indent=2) +shutil.chown(SUBMITTY_USERS_JSON, 'root', DAEMON_GROUP if args.worker else DAEMONPHP_GROUP) + +os.chmod(SUBMITTY_USERS_JSON, 0o440) +shutil.chown(WORKERS_JSON, PHP_USER, DAEMON_GID) +shutil.chown(CONTAINERS_JSON, group=DAEMONPHP_GROUP) +shutil.chown(DATABASE_JSON, 'root', DAEMONPHP_GROUP) +shutil.chown(AUTHENTICATION_JSON, 'root', DAEMONPHP_GROUP) +shutil.chown(EMAIL_JSON, 'root', DAEMONPHP_GROUP) +shutil.chown(SECRETS_PHP_JSON, 'root', PHP_GROUP) +shutil.chown(SUBMITTY_ADMIN_JSON, 'root', DAEMON_GROUP) +shutil.chown(SETUP_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) \ No newline at end of file diff --git a/.setup/distro_setup/ubuntu/22.04/variables.sh b/.setup/distro_setup/ubuntu/22.04/variables.sh new file mode 100644 index 00000000000..80edc76c777 --- /dev/null +++ b/.setup/distro_setup/ubuntu/22.04/variables.sh @@ -0,0 +1,6 @@ +if [ ${DEV_VM} == 1 ]; then + export SUBMISSION_URL='http://localhost:1511' + export SUBMISSION_PORT=1511 + export DATABASE_PORT=16442 + export WEBSOCKET_PORT=8443 +fi diff --git a/.setup/distro_setup/variables.sh b/.setup/distro_setup/variables.sh new file mode 100644 index 00000000000..42a8cf03d97 --- /dev/null +++ b/.setup/distro_setup/variables.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# this script must be run by root or sudo +if [[ "$UID" -ne "0" ]] ; then + echo "ERROR: This script must be run by root or sudo" + exit +fi + +# Note: This script is source by install_system.sh and so caution should be used on naming variables +# to ensure there is no collision. +SETUP_DISTRO_DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DISTRO=$(lsb_release -si | tr '[:upper:]' '[:lower:]') +VERSION=$(lsb_release -sr | tr '[:upper:]' '[:lower:]') + +if [ ! -d ${SETUP_DISTRO_DIR}/${DISTRO}/${VERSION} ]; then + (>&2 echo "Unknown distro: ${DISTRO} ${VERSION}") + exit 1 +fi + +source ${SETUP_DISTRO_DIR}/${DISTRO}/${VERSION}/variables.sh diff --git a/.setup/install_system.sh b/.setup/install_system.sh index c24d84a2152..2a11177eb55 100644 --- a/.setup/install_system.sh +++ b/.setup/install_system.sh @@ -41,13 +41,9 @@ LICHEN_REPOSITORY=/usr/local/submitty/GIT_CHECKOUT/Lichen SUBMITTY_INSTALL_DIR=/usr/local/submitty SUBMITTY_DATA_DIR=/var/local/submitty -# Sources -source ${CURRENT_DIR}/bin/versions.sh -source ${CURRENT_DIR}/distro_setup/setup_distro.sh - # create directory and fix permissions mkdir -p ${SUBMITTY_DATA_DIR} - +mkdir -p ${SUBMITTY_INSTALL_DIR}/config INSTALL_SYS_DIR=$(mktemp -d) chmod 777 "${INSTALL_SYS_DIR}" pushd "${INSTALL_SYS_DIR}" > /dev/null @@ -60,7 +56,6 @@ PHP_GROUP=submitty_php CGI_USER=submitty_cgi CGI_GROUP=submitty_cgi - COURSE_BUILDERS_GROUP=submitty_course_builders DB_USER=submitty_dbuser DATABASE_PASSWORD=submitty_dbuser @@ -126,18 +121,14 @@ fi ################################################################# # DISTRO SETUP ################# - -bash "${SUBMITTY_REPOSITORY}/.setup/update_system.sh" "config=${SUBMITTY_DIRECTORY}" - +# Sources +source ${CURRENT_DIR}/bin/versions.sh +source ${CURRENT_DIR}/distro_setup/variables.sh ################################################################# # SUBMITTY SETUP ################# -echo Beginning Submitty Setup -sudo apt-get update -sudo add-apt-repository universe -sudo apt update -sudo apt install python3-pip -y - +apt-get update +apt-get install -qqy python3 python3-pip pip3 install tzlocal #If in worker mode, run configure with --worker option. if [ ${WORKER} == 1 ]; then @@ -154,6 +145,7 @@ else if [ -z "${SUBMISSION_URL}" ]; then SUBMISSION_URL='http://192.168.56.101' fi + mkdir /var/run/postgresql echo -e "/var/run/postgresql ${DB_USER} ${DATABASE_PASSWORD} @@ -287,6 +279,8 @@ else fi +source ${CURRENT_DIR}/distro_setup/setup_distro.sh + ################################################################# # STACK SETUP ################# @@ -410,6 +404,13 @@ sudo chown "${DAEMON_USER}:${DAEMON_USER}" "$gitconfig_path" usermod -a -G docker "${DAEMON_USER}" +#If in worker mode, run configure with --worker option. +if [ ${WORKER} == 1 ]; then + echo "Using users to update config files" + python3 ${SUBMITTY_REPOSITORY}/.setup/USER_SETUP.py --worker +else + python3 ${SUBMITTY_REPOSITORY}/.setup/USER_SETUP.py +fi ################################################################# # JAR SETUP ################# diff --git a/Vagrantfile b/Vagrantfile index a2cb32fab7f..388df422c72 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -143,7 +143,7 @@ Vagrant.configure(2) do |config| apple_silicon = Vagrant::Util::Platform.darwin? && (arm || (`sysctl -n machdep.cpu.brand_string`.chomp.start_with? 'Apple M')) use_prebuilt_version = !ENV.fetch('PREBUILT_VERSION', '').empty? custom_box = ENV.has_key?('VAGRANT_BOX') - base_box = ENV.has_key?('BASE_BOX') || ENV.has_key?('FROM_SCRATCH') || apple_silicon || arm || true + base_box = ENV.has_key?('BASE_BOX') || ENV.has_key?('FROM_SCRATCH') || apple_silicon || arm # The time in seconds that Vagrant will wait for the machine to boot and be accessible. config.vm.boot_timeout = 600 From 09574050a9a74869f7922afe320e60546c9a819c Mon Sep 17 00:00:00 2001 From: peteca Date: Wed, 28 Jan 2026 14:43:18 -0800 Subject: [PATCH 6/7] Create config dir --- .setup/CONFIGURE_SUBMITTY.py | 12 ++++++++++++ .setup/USERS_SETUP.py | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.setup/CONFIGURE_SUBMITTY.py b/.setup/CONFIGURE_SUBMITTY.py index a3ceb891c91..16076253ff1 100644 --- a/.setup/CONFIGURE_SUBMITTY.py +++ b/.setup/CONFIGURE_SUBMITTY.py @@ -170,6 +170,18 @@ def __call__(self, parser, namespace, values, option_string=None): ############################################################################## +if os.path.isdir(CONFIG_INSTALL_DIR): + for file in os.scandir(CONFIG_INSTALL_DIR): + if file.name not in IGNORED_FILES_AND_DIRS: + if file.is_file(): + os.remove(os.path.join(CONFIG_INSTALL_DIR, file.name)) + else: + os.rmdir(os.path.join(CONFIG_INSTALL_DIR, file.name)) +elif os.path.exists(CONFIG_INSTALL_DIR): + os.remove(CONFIG_INSTALL_DIR) +os.makedirs(CONFIG_INSTALL_DIR, exist_ok=True) +os.chmod(CONFIG_INSTALL_DIR, 0o755) + loaded_defaults = {} if os.path.isfile(CONFIGURATION_JSON): with open(CONFIGURATION_JSON) as conf_file: diff --git a/.setup/USERS_SETUP.py b/.setup/USERS_SETUP.py index 884c24b2a5c..fa504c034fd 100644 --- a/.setup/USERS_SETUP.py +++ b/.setup/USERS_SETUP.py @@ -198,4 +198,5 @@ def get_ids(user): shutil.chown(EMAIL_JSON, 'root', DAEMONPHP_GROUP) shutil.chown(SECRETS_PHP_JSON, 'root', PHP_GROUP) shutil.chown(SUBMITTY_ADMIN_JSON, 'root', DAEMON_GROUP) -shutil.chown(SETUP_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) \ No newline at end of file +shutil.chown(SETUP_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) +shutil.chown(CONFIG_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) \ No newline at end of file From 07b9c5d98690c94d081b0c6990606488b9f65151 Mon Sep 17 00:00:00 2001 From: IDzyre Date: Wed, 28 Jan 2026 17:11:31 -0800 Subject: [PATCH 7/7] Fix concat array, and correct name of python script --- .setup/CONFIGURE_SUBMITTY.py | 2 +- .setup/install_system.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.setup/CONFIGURE_SUBMITTY.py b/.setup/CONFIGURE_SUBMITTY.py index 16076253ff1..f2b5cf53bad 100644 --- a/.setup/CONFIGURE_SUBMITTY.py +++ b/.setup/CONFIGURE_SUBMITTY.py @@ -370,7 +370,7 @@ def write(x=''): config = submitty_config generated_submitty_config = full_config['submitty'] -config = config + submitty_config +config.update(generated_submitty_config) config['submitty_install_dir'] = SUBMITTY_INSTALL_DIR config['submitty_repository'] = SUBMITTY_REPOSITORY config['submitty_data_dir'] = SUBMITTY_DATA_DIR diff --git a/.setup/install_system.sh b/.setup/install_system.sh index 2a11177eb55..0babff05a63 100644 --- a/.setup/install_system.sh +++ b/.setup/install_system.sh @@ -407,9 +407,9 @@ usermod -a -G docker "${DAEMON_USER}" #If in worker mode, run configure with --worker option. if [ ${WORKER} == 1 ]; then echo "Using users to update config files" - python3 ${SUBMITTY_REPOSITORY}/.setup/USER_SETUP.py --worker + python3 ${SUBMITTY_REPOSITORY}/.setup/USERS_SETUP.py --worker else - python3 ${SUBMITTY_REPOSITORY}/.setup/USER_SETUP.py + python3 ${SUBMITTY_REPOSITORY}/.setup/USERS_SETUP.py fi ################################################################# # JAR SETUP