diff --git a/.setup/CONFIGURE_SUBMITTY.py b/.setup/CONFIGURE_SUBMITTY.py index f39ff38d631..f2b5cf53bad 100644 --- a/.setup/CONFIGURE_SUBMITTY.py +++ b/.setup/CONFIGURE_SUBMITTY.py @@ -2,39 +2,14 @@ import argparse from collections import OrderedDict -import grp import json import os -import pwd import secrets import shutil import string import tzlocal -import tempfile - - -def get_uid(user): - return pwd.getpwnam(user).pw_uid - - -def get_gid(user): - return pwd.getpwnam(user).pw_gid - - -def get_ids(user): - try: - return get_uid(user), get_gid(user) - except KeyError: - raise SystemExit("ERROR: Could not find user: " + user) - - -def get_input(question, default=""): - add = "[{}] ".format(default) if default != "" else "" - user = input("{}: {}".format(question, add)).strip() - if user == "": - user = default - return user +from GENERATE_CONFIGS import generate_config class StrToBoolAction(argparse.Action): """ @@ -56,6 +31,77 @@ def __call__(self, parser, namespace, values, option_string=None): if os.getuid() != 0: raise SystemExit('ERROR: This script must be run by root or sudo') +authentication_methods = [ + 'PamAuthentication', + 'DatabaseAuthentication', + 'LdapAuthentication', + 'SamlAuthentication' +] + +defaults = { + 'database_host': 'localhost', + 'database_port': 5432, + 'database_user': 'submitty_dbuser', + 'database_course_user': 'submitty_course_dbuser', + 'submission_url': '', + 'supervisor_user': 'submitty', + 'vcs_url': '', + 'authentication_method': 0, + 'institution_name' : '', + 'institution_homepage' : '', + 'user_create_account' : False, + 'timezone' : str(tzlocal.get_localzone()), + 'submitty_admin_username': '', + 'email_user': '', + 'email_password': '', + 'email_sender': 'submitty@myuniversity.edu', + 'email_reply_to': 'submitty_do_not_reply@myuniversity.edu', + 'email_server_hostname': 'mail.myuniversity.edu', + 'email_server_port': 25, + 'email_internal_domain': 'example.com', + 'course_code_requirements': "Please follow your school's convention for course code.", + 'sys_admin_email': '', + 'sys_admin_url': '', + 'ldap_options': { + 'url': '', + 'uid': '', + 'bind_dn': '' + }, + 'saml_options': { + 'name': '', + 'username_attribute': '' + }, + 'course_material_file_upload_limit_mb': 100, + 'user_id_requirements': { + 'any_user_id': True, + 'require_name': False, + 'min_length': 6, + 'max_length': 25, + # Example for Alyssa Hacker : hackal -- Allows for shorter names. If they are shorter, then it will just take the entire name. + # Example for Joseph Wo : wojo + 'name_requirements': { + 'given_first': False, + 'given_name': 2, + 'family_name': 4 + }, + 'require_email': False, + # If the user_id must contain part of the email. If whole_email is true, it must match the email. + # If whole_prefix is true, then the user_id must equal everything before the final @ sign. + # Else, it must be a certain number of characters of the prefix. + # Examples for myemail@email.com: + # Whole email: myemail@gmail.com + # Whole prefix: myemail + # Part of prefix: myemai + 'email_requirements': { + 'whole_email': False, + 'whole_prefix': False, + 'prefix_count': 6 + }, + 'accepted_emails': [ + 'gmail.com' + ] + } +} parser = argparse.ArgumentParser(description='Submitty configuration script', formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -100,51 +146,7 @@ def __call__(self, parser, namespace, values, option_string=None): CGI_USER = 'submitty_cgi' DAEMON_USER = 'submitty_daemon' DAEMON_GROUP = 'submitty_daemon' - -if not args.worker: - PHP_UID, PHP_GID = get_ids(PHP_USER) - CGI_UID, CGI_GID = get_ids(CGI_USER) - # System Groups - DAEMONPHP_GROUP = 'submitty_daemonphp' - try: - grp.getgrnam(DAEMONPHP_GROUP) - except KeyError: - raise SystemExit("ERROR: Could not find group: " + DAEMONPHP_GROUP) - DAEMONCGI_GROUP = 'submitty_daemoncgi' - try: - grp.getgrnam(DAEMONCGI_GROUP) - except KeyError: - raise SystemExit("ERROR: Could not find group: " + DAEMONCGI_GROUP) - DAEMONPHPCGI_GROUP = 'submitty_daemonphpcgi' - try: - grp.getgrnam(DAEMONPHPCGI_GROUP) - except KeyError: - raise SystemExit("ERROR: Could not find group: " + DAEMONPHPCGI_GROUP) - -DAEMON_UID, DAEMON_GID = get_ids(DAEMON_USER) - COURSE_BUILDERS_GROUP = 'submitty_course_builders' -try: - grp.getgrnam(COURSE_BUILDERS_GROUP) -except KeyError: - raise SystemExit("ERROR: Could not find group: " + COURSE_BUILDERS_GROUP) - -############################################################################## - -# This is the upper limit of the number of parallel grading threads on -# this machine -NUM_UNTRUSTED = 60 - -FIRST_UNTRUSTED_UID, FIRST_UNTRUSTED_GID = get_ids('untrusted00') - -# confirm that the uid/gid of the untrusted users are sequential -for i in range(1, NUM_UNTRUSTED): - untrusted_user = "untrusted{:0=2d}".format(i) - uid, gid = get_ids(untrusted_user) - if uid != FIRST_UNTRUSTED_UID + i: - raise SystemExit('CONFIGURATION ERROR: untrusted UID not sequential: ' + untrusted_user) - elif gid != FIRST_UNTRUSTED_GID + i: - raise SystemExit('CONFIGURATION ERROR: untrusted GID not sequential: ' + untrusted_user) ############################################################################## @@ -161,54 +163,24 @@ def __call__(self, parser, namespace, values, option_string=None): CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') SITE_CONFIG_DIR = os.path.join(SUBMITTY_INSTALL_DIR, "site", "config") CONFIG_INSTALL_DIR = os.path.join(SUBMITTY_INSTALL_DIR, 'config') +SUBMITTY_USERS_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_users.json') SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') EMAIL_JSON = os.path.join(CONFIG_INSTALL_DIR, 'email.json') AUTHENTICATION_JSON = os.path.join(CONFIG_INSTALL_DIR, 'authentication.json') ############################################################################## -authentication_methods = [ - 'PamAuthentication', - 'DatabaseAuthentication', - 'LdapAuthentication', - 'SamlAuthentication' -] - -defaults = { - 'database_host': 'localhost', - 'database_port': 5432, - 'database_user': 'submitty_dbuser', - 'database_course_user': 'submitty_course_dbuser', - 'submission_url': '', - 'supervisor_user': 'submitty', - 'vcs_url': '', - 'authentication_method': 0, - 'institution_name' : '', - 'institution_homepage' : '', - 'user_create_account' : False, - 'timezone' : str(tzlocal.get_localzone()), - 'submitty_admin_username': '', - 'email_user': '', - 'email_password': '', - 'email_sender': 'submitty@myuniversity.edu', - 'email_reply_to': 'submitty_do_not_reply@myuniversity.edu', - 'email_server_hostname': 'mail.myuniversity.edu', - 'email_server_port': 25, - 'email_internal_domain': 'example.com', - 'course_code_requirements': "Please follow your school's convention for course code.", - 'sys_admin_email': '', - 'sys_admin_url': '', - 'ldap_options': { - 'url': '', - 'uid': '', - 'bind_dn': '' - }, - 'saml_options': { - 'name': '', - 'username_attribute': '' - }, - 'course_material_file_upload_limit_mb': 100 -} +if os.path.isdir(CONFIG_INSTALL_DIR): + for file in os.scandir(CONFIG_INSTALL_DIR): + if file.name not in IGNORED_FILES_AND_DIRS: + if file.is_file(): + os.remove(os.path.join(CONFIG_INSTALL_DIR, file.name)) + else: + os.rmdir(os.path.join(CONFIG_INSTALL_DIR, file.name)) +elif os.path.exists(CONFIG_INSTALL_DIR): + os.remove(CONFIG_INSTALL_DIR) +os.makedirs(CONFIG_INSTALL_DIR, exist_ok=True) +os.chmod(CONFIG_INSTALL_DIR, 0o755) loaded_defaults = {} if os.path.isfile(CONFIGURATION_JSON): @@ -247,152 +219,7 @@ def __call__(self, parser, namespace, values, option_string=None): if args.worker: print("CONFIGURING SUBMITTY AS A WORKER !!") -print('Hit enter to use default in []') -print() - -if args.worker: - SUPERVISOR_USER = get_input('What is the id for your submitty user?', defaults['supervisor_user']) - print('SUPERVISOR USER : {}'.format(SUPERVISOR_USER)) -else: - DATABASE_HOST = get_input('What is the database host?', defaults['database_host']) - print() - - if not os.path.isdir(DATABASE_HOST): - DATABASE_PORT = int(get_input('What is the database port?', defaults['database_port'])) - print() - else: - DATABASE_PORT = defaults['database_port'] - - DATABASE_USER = get_input('What is the global database user/role?', defaults['database_user']) - print() - - default = '' - if 'database_password' in defaults and DATABASE_USER == defaults['database_user']: - default = '(Leave blank to use same password)' - DATABASE_PASS = get_input('What is the password for the global database user/role {}? {}'.format(DATABASE_USER, default)) - if DATABASE_PASS == '' and DATABASE_USER == defaults['database_user'] and 'database_password' in defaults: - DATABASE_PASS = defaults['database_password'] - print() - - DATABASE_COURSE_USER = get_input('What is the course database user/role?', defaults['database_course_user']) - print() - - default = '' - if 'database_course_password' in defaults and DATABASE_COURSE_USER == defaults['database_course_user']: - default = '(Leave blank to use same password)' - DATABASE_COURSE_PASSWORD = get_input('What is the password for the course database user/role {}? {}'.format(DATABASE_COURSE_USER, default)) - if DATABASE_COURSE_PASSWORD == '' and DATABASE_COURSE_USER == defaults['database_course_user'] and 'database_course_password' in defaults: - DATABASE_COURSE_PASSWORD = defaults['database_course_password'] - print() - - TIMEZONE = get_input('What timezone should Submitty use? (for a full list of supported timezones see http://php.net/manual/en/timezones.php)', defaults['timezone']) - print() - - DEFAULT_LOCALE = get_input('What default language should the Submitty site use?', 'en_US') - print() - - COURSE_MATERIAL_UPLOAD_LIMIT_MB = get_input('What is the maximum file upload size for course materials (in MB)?', defaults['course_material_file_upload_limit_mb']) - print() - - SUBMISSION_URL = get_input('What is the url for submission? (ex: http://192.168.56.101 or ' - 'https://submitty.cs.rpi.edu)', defaults['submission_url']).rstrip('/') - print() - - VCS_URL = get_input('What is the url for VCS? (Leave blank to default to submission url + {$vcs_type}) (ex: http://192.168.56.101/{$vcs_type} or https://submitty-vcs.cs.rpi.edu/{$vcs_type}', defaults['vcs_url']).rstrip('/') - print() - - INSTITUTION_NAME = get_input('What is the name of your institution? (Leave blank/type "none" if not desired)', - defaults['institution_name']) - print() - - if INSTITUTION_NAME == '' or INSTITUTION_NAME.isspace(): - INSTITUTION_HOMEPAGE = '' - else: - INSTITUTION_HOMEPAGE = get_input("What is the url of your institution\'s homepage? " - '(Leave blank/type "none" if not desired)', defaults['institution_homepage']) - if INSTITUTION_HOMEPAGE.lower() == "none": - INSTITUTION_HOMEPAGE = '' - print() - - SYS_ADMIN_EMAIL = get_input("What is the email for system administration?", defaults['sys_admin_email']) - SYS_ADMIN_URL = get_input("Where to report problems with Submitty (url for help link)?", defaults['sys_admin_url']) - - print('What authentication method to use:') - for i in range(len(authentication_methods)): - print(f"{i + 1}. {authentication_methods[i]}") - - while True: - try: - auth = int(get_input('Enter number?', defaults['authentication_method'])) - 1 - except ValueError: - auth = -1 - if auth in range(len(authentication_methods)): - break - print(f'Number must in between 1 - {len(authentication_methods)} (inclusive)!') - print() - - AUTHENTICATION_METHOD = authentication_methods[auth] - - default_auth_options = defaults.get('ldap_options', dict()) - LDAP_OPTIONS = { - 'url': default_auth_options.get('url', ''), - 'uid': default_auth_options.get('uid', ''), - 'bind_dn': default_auth_options.get('bind_dn', '') - } - USER_CREATE_ACCOUNT = False - if AUTHENTICATION_METHOD == 'DatabaseAuthentication': - user_create_account = get_input("Allow users to create their own accounts? [y/n]", 'n') - USER_CREATE_ACCOUNT = user_create_account.lower() in ['yes', 'y'] - print() - if AUTHENTICATION_METHOD == 'LdapAuthentication': - LDAP_OPTIONS['url'] = get_input('Enter LDAP url?', LDAP_OPTIONS['url']) - LDAP_OPTIONS['uid'] = get_input('Enter LDAP UID?', LDAP_OPTIONS['uid']) - LDAP_OPTIONS['bind_dn'] = get_input('Enter LDAP bind_dn?', LDAP_OPTIONS['bind_dn']) - - default_auth_options = defaults.get('saml_options', dict()) - SAML_OPTIONS = { - 'name': default_auth_options.get('name', ''), - 'username_attribute': default_auth_options.get('username_attribute', '') - } - - if AUTHENTICATION_METHOD == 'SamlAuthentication': - SAML_OPTIONS['name'] = get_input('Enter name you would like shown to user for authentication?', SAML_OPTIONS['name']) - SAML_OPTIONS['username_attribute'] = get_input('Enter SAML username attribute?', SAML_OPTIONS['username_attribute']) - - - CGI_URL = SUBMISSION_URL + '/cgi-bin' - - SUBMITTY_ADMIN_USERNAME = get_input("What is the submitty admin username (optional)?", defaults['submitty_admin_username']) - - while True: - is_email_enabled = get_input("Will Submitty use email notifications? [y/n]", 'y') - if (is_email_enabled.lower() in ['yes', 'y']): - EMAIL_ENABLED = True - EMAIL_USER = get_input("What is the email user?", defaults['email_user']) - EMAIL_PASSWORD = get_input("What is the email password",defaults['email_password']) - EMAIL_SENDER = get_input("What is the email sender address (the address that will appear in the From: line)?",defaults['email_sender']) - EMAIL_REPLY_TO = get_input("What is the email reply to address?", defaults['email_reply_to']) - EMAIL_SERVER_HOSTNAME = get_input("What is the email server hostname?", defaults['email_server_hostname']) - try: - EMAIL_SERVER_PORT = int(get_input("What is the email server port?", defaults['email_server_port'])) - except ValueError: - EMAIL_SERVER_PORT = defaults['email_server_port'] - EMAIL_INTERNAL_DOMAIN = get_input("What is the internal email address format?", defaults['email_internal_domain']) - break - - elif (is_email_enabled.lower() in ['no', 'n']): - EMAIL_ENABLED = False - EMAIL_USER = defaults['email_user'] - EMAIL_PASSWORD = defaults['email_password'] - EMAIL_SENDER = defaults['email_sender'] - EMAIL_REPLY_TO = defaults['email_reply_to'] - EMAIL_SERVER_HOSTNAME = defaults['email_server_hostname'] - EMAIL_SERVER_PORT = defaults['email_server_port'] - EMAIL_INTERNAL_DOMAIN = defaults['email_internal_domain'] - break - print() - - +full_config = generate_config(defaults, args.worker, authentication_methods) ############################################################################## # make the installation setup directory @@ -400,61 +227,30 @@ def __call__(self, parser, namespace, values, option_string=None): if os.path.isdir(SETUP_INSTALL_DIR): shutil.rmtree(SETUP_INSTALL_DIR) os.makedirs(SETUP_INSTALL_DIR, exist_ok=True) - -shutil.chown(SETUP_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) os.chmod(SETUP_INSTALL_DIR, 0o751) ############################################################################## # WRITE CONFIG FILES IN ${SUBMITTY_INSTALL_DIR}/.setup +config = full_config['general'] -config = OrderedDict() +if args.worker: + config['supervisor_user'] = config['supervisor_user'] + with open(SUBMITTY_USERS_JSON, 'w') as json_file: + json.dump(config, json_file, indent=2) config['submitty_install_dir'] = SUBMITTY_INSTALL_DIR config['submitty_repository'] = SUBMITTY_REPOSITORY config['submitty_data_dir'] = SUBMITTY_DATA_DIR - config['course_builders_group'] = COURSE_BUILDERS_GROUP - -config['num_untrusted'] = NUM_UNTRUSTED -config['first_untrusted_uid'] = FIRST_UNTRUSTED_UID -config['first_untrusted_gid'] = FIRST_UNTRUSTED_UID config['num_grading_scheduler_workers'] = NUM_GRADING_SCHEDULER_WORKERS - - config['daemon_user'] = DAEMON_USER -config['daemon_uid'] = DAEMON_UID -config['daemon_gid'] = DAEMON_GID -if args.worker: - config['supervisor_user'] = SUPERVISOR_USER -else: +if not args.worker: config['php_user'] = PHP_USER config['cgi_user'] = CGI_USER - config['daemonphp_group'] = DAEMONPHP_GROUP - config['daemoncgi_group'] = DAEMONCGI_GROUP - config['daemonphpcgi_group'] = DAEMONPHPCGI_GROUP - config['php_uid'] = PHP_UID - config['php_gid'] = PHP_GID - - config['database_host'] = DATABASE_HOST - config['database_port'] = DATABASE_PORT - config['database_user'] = DATABASE_USER - config['database_password'] = DATABASE_PASS - config['database_course_user'] = DATABASE_COURSE_USER - config['database_course_password'] = DATABASE_COURSE_PASSWORD - config['timezone'] = TIMEZONE - config['default_locale'] = DEFAULT_LOCALE - - config['authentication_method'] = AUTHENTICATION_METHOD - config['vcs_url'] = VCS_URL - config['submission_url'] = SUBMISSION_URL - config['cgi_url'] = CGI_URL + config['websocket_port'] = WEBSOCKET_PORT - - config['institution_name'] = INSTITUTION_NAME - config['institution_homepage'] = INSTITUTION_HOMEPAGE config['debugging_enabled'] = DEBUGGING_ENABLED - config['user_create_account'] = USER_CREATE_ACCOUNT # site_log_path is a holdover name. This could more accurately be called the "log_path" config['site_log_path'] = TAGRADING_LOG_PATH @@ -498,43 +294,6 @@ def write(x=''): except FileNotFoundError: pass -#Rescue the autograding_workers and _containers files if they exist. -rescued = list() -tmp_folder = tempfile.mkdtemp() -if not args.worker: - for full_file_name, file_name in [(WORKERS_JSON, 'autograding_workers.json'), (CONTAINERS_JSON, 'autograding_containers.json')]: - if os.path.isfile(full_file_name): - #make a tmp folder and copy autograding workers to it - tmp_file = os.path.join(tmp_folder, file_name) - shutil.move(full_file_name, tmp_file) - rescued.append((full_file_name, tmp_file)) - -IGNORED_FILES_AND_DIRS = ['saml', 'login.md'] - -if os.path.isdir(CONFIG_INSTALL_DIR): - for file in os.scandir(CONFIG_INSTALL_DIR): - if file.name not in IGNORED_FILES_AND_DIRS: - if file.is_file(): - os.remove(os.path.join(CONFIG_INSTALL_DIR, file.name)) - else: - os.rmdir(os.path.join(CONFIG_INSTALL_DIR, file.name)) -elif os.path.exists(CONFIG_INSTALL_DIR): - os.remove(CONFIG_INSTALL_DIR) -os.makedirs(CONFIG_INSTALL_DIR, exist_ok=True) -shutil.chown(CONFIG_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) -os.chmod(CONFIG_INSTALL_DIR, 0o755) - -# Finish rescuing files. -for full_file_name, tmp_file_name in rescued: - #copy autograding workers back - shutil.move(tmp_file_name, full_file_name) - #make sure the permissions are correct. - shutil.chown(full_file_name, 'root', DAEMON_GID) - os.chmod(full_file_name, 0o660) - -#remove the tmp folder -os.removedirs(tmp_folder) - ############################################################################## # WRITE CONFIG FILES IN ${SUBMITTY_INSTALL_DIR}/config @@ -589,99 +348,39 @@ def write(x=''): for file in [WORKERS_JSON, CONTAINERS_JSON]: os.chmod(file, 0o660) - shutil.chown(WORKERS_JSON, PHP_USER, DAEMON_GID) - shutil.chown(CONTAINERS_JSON, group=DAEMONPHP_GROUP) - ############################################################################## # Write database json if not args.worker: - config = OrderedDict() - config['authentication_method'] = AUTHENTICATION_METHOD - config['database_host'] = DATABASE_HOST - config['database_port'] = DATABASE_PORT - config['database_user'] = DATABASE_USER - config['database_password'] = DATABASE_PASS - config['database_course_user'] = DATABASE_COURSE_USER - config['database_course_password'] = DATABASE_COURSE_PASSWORD - config['debugging_enabled'] = DEBUGGING_ENABLED - + database_config = full_config['database'] + database_config['debugging_enabled'] = DEBUGGING_ENABLED with open(DATABASE_JSON, 'w') as json_file: - json.dump(config, json_file, indent=2) - shutil.chown(DATABASE_JSON, 'root', DAEMONPHP_GROUP) + json.dump(database_config, json_file, indent=2) os.chmod(DATABASE_JSON, 0o440) ############################################################################## # Write authentication json if not args.worker: - config = OrderedDict() - config['authentication_method'] = AUTHENTICATION_METHOD - config['ldap_options'] = LDAP_OPTIONS - config['saml_options'] = SAML_OPTIONS - + auth_config = full_config['authentication'] with open(AUTHENTICATION_JSON, 'w') as json_file: - json.dump(config, json_file, indent=4) - shutil.chown(AUTHENTICATION_JSON, 'root', DAEMONPHP_GROUP) + json.dump(auth_config, json_file, indent=4) os.chmod(AUTHENTICATION_JSON, 0o440) ############################################################################## -# Write submitty json -# Full documentation at submitty.org/... -user_id_requirements = { - "any_user_id": True, - "require_name": False, - "min_length": 6, - "max_length": 25, - # Example for Alyssa Hacker : hackal -- Allows for shorter names. If they are shorter, then it will just take the entire name. - # Example for Joseph Wo : wojo - "name_requirements": { - "given_first": False, - "given_name": 2, - "family_name": 4 - }, - "require_email": False, - # If the user_id must contain part of the email. If whole_email is true, it must match the email. - # If whole_prefix is true, then the user_id must equal everything before the final @ sign. - # Else, it must be a certain number of characters of the prefix. - # Examples for myemail@email.com: - # Whole email: myemail@gmail.com - # Whole prefix: myemail - # Part of prefix: myemai - "email_requirements": { - "whole_email": False, - "whole_prefix": False, - "prefix_count": 6 - }, - "accepted_emails": [ - "gmail.com" - ] -} - config = submitty_config +generated_submitty_config = full_config['submitty'] +config.update(generated_submitty_config) config['submitty_install_dir'] = SUBMITTY_INSTALL_DIR config['submitty_repository'] = SUBMITTY_REPOSITORY config['submitty_data_dir'] = SUBMITTY_DATA_DIR config['autograding_log_path'] = AUTOGRADING_LOG_PATH -if not args.worker: - config['sys_admin_email'] = SYS_ADMIN_EMAIL - config['sys_admin_url'] = SYS_ADMIN_URL # site_log_path is a holdover name. This could more accurately be called the "log_path" config['site_log_path'] = TAGRADING_LOG_PATH if not args.worker: - config['submission_url'] = SUBMISSION_URL - config['vcs_url'] = VCS_URL - config['cgi_url'] = CGI_URL config['websocket_port'] = WEBSOCKET_PORT - config['institution_name'] = INSTITUTION_NAME - config['institution_homepage'] = INSTITUTION_HOMEPAGE - config['timezone'] = TIMEZONE - config['default_locale'] = DEFAULT_LOCALE config['duck_special_effects'] = False - config['course_material_file_upload_limit_mb'] = COURSE_MATERIAL_UPLOAD_LIMIT_MB - config['user_create_account'] = USER_CREATE_ACCOUNT - config['user_id_requirements'] = user_id_requirements config['worker'] = True if args.worker == 1 else False @@ -689,36 +388,6 @@ def write(x=''): json.dump(config, json_file, indent=2) os.chmod(SUBMITTY_JSON, 0o444) -############################################################################## -# Write users json - -config = OrderedDict() -config['num_grading_scheduler_workers'] = NUM_GRADING_SCHEDULER_WORKERS -config['num_untrusted'] = NUM_UNTRUSTED -config['first_untrusted_uid'] = FIRST_UNTRUSTED_UID -config['first_untrusted_gid'] = FIRST_UNTRUSTED_UID -config['daemon_uid'] = DAEMON_UID -config['daemon_gid'] = DAEMON_GID -config['daemon_user'] = DAEMON_USER -config['course_builders_group'] = COURSE_BUILDERS_GROUP - -if not args.worker: - config['php_uid'] = PHP_UID - config['php_gid'] = PHP_GID - config['php_user'] = PHP_USER - config['cgi_user'] = CGI_USER - config['daemonphp_group'] = DAEMONPHP_GROUP - config['daemoncgi_group'] = DAEMONCGI_GROUP - config['daemonphpcgi_group'] = DAEMONPHPCGI_GROUP -else: - config['supervisor_user'] = SUPERVISOR_USER - -with open(SUBMITTY_USERS_JSON, 'w') as json_file: - json.dump(config, json_file, indent=2) -shutil.chown(SUBMITTY_USERS_JSON, 'root', DAEMON_GROUP if args.worker else DAEMONPHP_GROUP) - -os.chmod(SUBMITTY_USERS_JSON, 0o440) - ############################################################################## # Write secrets_submitty_php json @@ -728,7 +397,6 @@ def write(x=''): config['session'] = ''.join(secrets.choice(characters) for _ in range(64)) with open(SECRETS_PHP_JSON, 'w') as json_file: json.dump(config, json_file, indent=2) - shutil.chown(SECRETS_PHP_JSON, 'root', PHP_GROUP) os.chmod(SECRETS_PHP_JSON, 0o440) ############################################################################## @@ -736,30 +404,19 @@ def write(x=''): if not args.worker: config = OrderedDict() - config['submitty_admin_username'] = SUBMITTY_ADMIN_USERNAME + config['submitty_admin_username'] = full_config['admin_username'] with open(SUBMITTY_ADMIN_JSON, 'w') as json_file: json.dump(config, json_file, indent=2) - shutil.chown(SUBMITTY_ADMIN_JSON, 'root', DAEMON_GROUP) os.chmod(SUBMITTY_ADMIN_JSON, 0o440) ############################################################################## # Write email json if not args.worker: - config = OrderedDict() - config['email_enabled'] = EMAIL_ENABLED - config['email_user'] = EMAIL_USER - config['email_password'] = EMAIL_PASSWORD - config['email_sender'] = EMAIL_SENDER - config['email_reply_to'] = EMAIL_REPLY_TO - config['email_server_hostname'] = EMAIL_SERVER_HOSTNAME - config['email_server_port'] = EMAIL_SERVER_PORT - config['email_internal_domain'] = EMAIL_INTERNAL_DOMAIN - + config = full_config['email'] with open(EMAIL_JSON, 'w') as json_file: json.dump(config, json_file, indent=2) - shutil.chown(EMAIL_JSON, 'root', DAEMONPHP_GROUP) os.chmod(EMAIL_JSON, 0o440) ############################################################################## @@ -768,4 +425,4 @@ def write(x=''): print(f' sudo {INSTALL_FILE}') print(' or') print(f' sudo {INSTALL_FILE} clean') -print("\n") +print("\n") \ No newline at end of file diff --git a/.setup/GENERATE_CONFIGS.py b/.setup/GENERATE_CONFIGS.py new file mode 100644 index 00000000000..13b3894de97 --- /dev/null +++ b/.setup/GENERATE_CONFIGS.py @@ -0,0 +1,226 @@ +from collections import OrderedDict +import os + +def get_input(question, default=""): + add = "[{}] ".format(default) if default != "" else "" + user = input("{}: {}".format(question, add)).strip() + if user == "": + user = default + return user + +def generate_config(defaults, worker, authentication_methods): + print('Hit enter to use default in []') + print() + + if worker: + SUPERVISOR_USER = get_input('What is the id for your submitty user?', defaults['supervisor_user']) + print('SUPERVISOR USER : {}'.format(SUPERVISOR_USER)) + else: + DATABASE_HOST = get_input('What is the database host?', defaults['database_host']) + print(DATABASE_HOST) + + if not os.path.isdir(DATABASE_HOST): + DATABASE_PORT = int(get_input('What is the database port?', defaults['database_port'])) + print() + else: + DATABASE_PORT = defaults['database_port'] + + DATABASE_USER = get_input('What is the global database user/role?', defaults['database_user']) + print() + + default = '' + if 'database_password' in defaults and DATABASE_USER == defaults['database_user']: + default = '(Leave blank to use same password)' + DATABASE_PASS = get_input('What is the password for the global database user/role {}? {}'.format(DATABASE_USER, default)) + if DATABASE_PASS == '' and DATABASE_USER == defaults['database_user'] and 'database_password' in defaults: + DATABASE_PASS = defaults['database_password'] + print() + + DATABASE_COURSE_USER = get_input('What is the course database user/role?', defaults['database_course_user']) + print() + + default = '' + if 'database_course_password' in defaults and DATABASE_COURSE_USER == defaults['database_course_user']: + default = '(Leave blank to use same password)' + DATABASE_COURSE_PASSWORD = get_input('What is the password for the course database user/role {}? {}'.format(DATABASE_COURSE_USER, default)) + if DATABASE_COURSE_PASSWORD == '' and DATABASE_COURSE_USER == defaults['database_course_user'] and 'database_course_password' in defaults: + DATABASE_COURSE_PASSWORD = defaults['database_course_password'] + print() + + TIMEZONE = get_input('What timezone should Submitty use? (for a full list of supported timezones see http://php.net/manual/en/timezones.php)', defaults['timezone']) + print() + + DEFAULT_LOCALE = get_input('What default language should the Submitty site use?', 'en_US') + print() + + COURSE_MATERIAL_UPLOAD_LIMIT_MB = get_input('What is the maximum file upload size for course materials (in MB)?', defaults['course_material_file_upload_limit_mb']) + print() + + SUBMISSION_URL = get_input('What is the url for submission? (ex: http://192.168.56.101 or ' + 'https://submitty.cs.rpi.edu)', defaults['submission_url']).rstrip('/') + print() + + VCS_URL = get_input('What is the url for VCS? (Leave blank to default to submission url + {$vcs_type}) (ex: http://192.168.56.101/{$vcs_type} or https://submitty-vcs.cs.rpi.edu/{$vcs_type}', defaults['vcs_url']).rstrip('/') + print() + + INSTITUTION_NAME = get_input('What is the name of your institution? (Leave blank/type "none" if not desired)', + defaults['institution_name']) + print() + + if INSTITUTION_NAME == '' or INSTITUTION_NAME.isspace(): + INSTITUTION_HOMEPAGE = '' + else: + INSTITUTION_HOMEPAGE = get_input("What is the url of your institution\'s homepage? " + '(Leave blank/type "none" if not desired)', defaults['institution_homepage']) + if INSTITUTION_HOMEPAGE.lower() == "none": + INSTITUTION_HOMEPAGE = '' + print() + + SYS_ADMIN_EMAIL = get_input("What is the email for system administration?", defaults['sys_admin_email']) + SYS_ADMIN_URL = get_input("Where to report problems with Submitty (url for help link)?", defaults['sys_admin_url']) + + print('What authentication method to use:') + for i in range(len(authentication_methods)): + print(f"{i + 1}. {authentication_methods[i]}") + + while True: + try: + auth = int(get_input('Enter number?', defaults['authentication_method'])) - 1 + except ValueError: + auth = -1 + if auth in range(len(authentication_methods)): + break + print(f'Number must in between 1 - {len(authentication_methods)} (inclusive)!') + print() + + AUTHENTICATION_METHOD = authentication_methods[auth] + + default_auth_options = defaults.get('ldap_options', dict()) + LDAP_OPTIONS = { + 'url': default_auth_options.get('url', ''), + 'uid': default_auth_options.get('uid', ''), + 'bind_dn': default_auth_options.get('bind_dn', '') + } + USER_CREATE_ACCOUNT = False + if AUTHENTICATION_METHOD == 'DatabaseAuthentication': + user_create_account = get_input("Allow users to create their own accounts? [y/n]", 'n') + USER_CREATE_ACCOUNT = user_create_account.lower() in ['yes', 'y'] + print() + if AUTHENTICATION_METHOD == 'LdapAuthentication': + LDAP_OPTIONS['url'] = get_input('Enter LDAP url?', LDAP_OPTIONS['url']) + LDAP_OPTIONS['uid'] = get_input('Enter LDAP UID?', LDAP_OPTIONS['uid']) + LDAP_OPTIONS['bind_dn'] = get_input('Enter LDAP bind_dn?', LDAP_OPTIONS['bind_dn']) + + default_auth_options = defaults.get('saml_options', dict()) + SAML_OPTIONS = { + 'name': default_auth_options.get('name', ''), + 'username_attribute': default_auth_options.get('username_attribute', '') + } + + if AUTHENTICATION_METHOD == 'SamlAuthentication': + SAML_OPTIONS['name'] = get_input('Enter name you would like shown to user for authentication?', SAML_OPTIONS['name']) + SAML_OPTIONS['username_attribute'] = get_input('Enter SAML username attribute?', SAML_OPTIONS['username_attribute']) + + + CGI_URL = SUBMISSION_URL + '/cgi-bin' + + SUBMITTY_ADMIN_USERNAME = get_input("What is the submitty admin username (optional)?", defaults['submitty_admin_username']) + + while True: + is_email_enabled = get_input("Will Submitty use email notifications? [y/n]", 'y') + if (is_email_enabled.lower() in ['yes', 'y']): + EMAIL_ENABLED = True + EMAIL_USER = get_input("What is the email user?", defaults['email_user']) + EMAIL_PASSWORD = get_input("What is the email password",defaults['email_password']) + EMAIL_SENDER = get_input("What is the email sender address (the address that will appear in the From: line)?",defaults['email_sender']) + EMAIL_REPLY_TO = get_input("What is the email reply to address?", defaults['email_reply_to']) + EMAIL_SERVER_HOSTNAME = get_input("What is the email server hostname?", defaults['email_server_hostname']) + try: + EMAIL_SERVER_PORT = int(get_input("What is the email server port?", defaults['email_server_port'])) + except ValueError: + EMAIL_SERVER_PORT = defaults['email_server_port'] + EMAIL_INTERNAL_DOMAIN = get_input("What is the internal email address format?", defaults['email_internal_domain']) + break + + elif (is_email_enabled.lower() in ['no', 'n']): + EMAIL_ENABLED = False + EMAIL_USER = defaults['email_user'] + EMAIL_PASSWORD = defaults['email_password'] + EMAIL_SENDER = defaults['email_sender'] + EMAIL_REPLY_TO = defaults['email_reply_to'] + EMAIL_SERVER_HOSTNAME = defaults['email_server_hostname'] + EMAIL_SERVER_PORT = defaults['email_server_port'] + EMAIL_INTERNAL_DOMAIN = defaults['email_internal_domain'] + break + print() + full_config = OrderedDict() + general_config = OrderedDict() + database_config = OrderedDict() + authentication_config = OrderedDict() + submitty_config = OrderedDict() + email_config = OrderedDict() + if worker: + general_config['supervisor_user'] = SUPERVISOR_USER + + else: + + general_config['database_host'] = DATABASE_HOST + general_config['database_port'] = DATABASE_PORT + general_config['database_user'] = DATABASE_USER + general_config['database_password'] = DATABASE_PASS + general_config['database_course_user'] = DATABASE_COURSE_USER + general_config['database_course_password'] = DATABASE_COURSE_PASSWORD + general_config['timezone'] = TIMEZONE + general_config['default_locale'] = DEFAULT_LOCALE + + general_config['authentication_method'] = AUTHENTICATION_METHOD + general_config['vcs_url'] = VCS_URL + general_config['submission_url'] = SUBMISSION_URL + general_config['cgi_url'] = CGI_URL + + general_config['institution_name'] = INSTITUTION_NAME + general_config['institution_homepage'] = INSTITUTION_HOMEPAGE + general_config['user_create_account'] = USER_CREATE_ACCOUNT + + database_config['authentication_method'] = AUTHENTICATION_METHOD + database_config['database_host'] = DATABASE_HOST + database_config['database_port'] = DATABASE_PORT + database_config['database_user'] = DATABASE_USER + database_config['database_password'] = DATABASE_PASS + database_config['database_course_user'] = DATABASE_COURSE_USER + database_config['database_course_password'] = DATABASE_COURSE_PASSWORD + + authentication_config['authentication_method'] = AUTHENTICATION_METHOD + authentication_config['ldap_options'] = LDAP_OPTIONS + authentication_config['saml_options'] = SAML_OPTIONS + + submitty_config['sys_admin_email'] = SYS_ADMIN_EMAIL + submitty_config['sys_admin_url'] = SYS_ADMIN_URL + submitty_config['submission_url'] = SUBMISSION_URL + submitty_config['vcs_url'] = VCS_URL + submitty_config['cgi_url'] = CGI_URL + submitty_config['institution_name'] = INSTITUTION_NAME + submitty_config['institution_homepage'] = INSTITUTION_HOMEPAGE + submitty_config['timezone'] = TIMEZONE + submitty_config['default_locale'] = DEFAULT_LOCALE + + submitty_config['course_material_file_upload_limit_mb'] = COURSE_MATERIAL_UPLOAD_LIMIT_MB + submitty_config['user_create_account'] = USER_CREATE_ACCOUNT + submitty_config['user_id_requirements'] = defaults['user_id_requirements'] + + email_config['email_enabled'] = EMAIL_ENABLED + email_config['email_user'] = EMAIL_USER + email_config['email_password'] = EMAIL_PASSWORD + email_config['email_sender'] = EMAIL_SENDER + email_config['email_reply_to'] = EMAIL_REPLY_TO + email_config['email_server_hostname'] = EMAIL_SERVER_HOSTNAME + email_config['email_server_port'] = EMAIL_SERVER_PORT + email_config['email_internal_domain'] = EMAIL_INTERNAL_DOMAIN + + full_config['database'] = database_config + full_config['authentication'] = authentication_config + full_config['submitty'] = submitty_config + full_config['general'] = general_config + full_config['email'] = email_config + full_config['admin_username'] = SUBMITTY_ADMIN_USERNAME + return full_config \ No newline at end of file diff --git a/.setup/USERS_SETUP.py b/.setup/USERS_SETUP.py new file mode 100644 index 00000000000..fa504c034fd --- /dev/null +++ b/.setup/USERS_SETUP.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 + +import argparse +import grp +import json +import pwd +from collections import OrderedDict +import shutil +import tempfile +import os + +def get_uid(user): + return pwd.getpwnam(user).pw_uid + + +def get_gid(user): + return pwd.getpwnam(user).pw_gid + + +def get_ids(user): + try: + return get_uid(user), get_gid(user) + except KeyError: + raise SystemExit("ERROR: Could not find user: " + user) + +parser = argparse.ArgumentParser(description='Submitty configuration script', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--worker', action='store_true', default=False, help='Configure Submitty with autograding only') +parser.add_argument('--install-dir', default='/usr/local/submitty', help='Set the install directory for Submitty') +args = parser.parse_args() + +########################################################## + +# recommended names for special users & groups related to the SUBMITTY system +PHP_USER = 'submitty_php' +PHP_GROUP = 'submitty_php' +CGI_USER = 'submitty_cgi' +DAEMON_USER = 'submitty_daemon' +DAEMON_GROUP = 'submitty_daemon' +COURSE_BUILDERS_GROUP = 'submitty_course_builders' + +############################################################################## + +# adjust this number depending on the # of processors +# available on your hardware +NUM_GRADING_SCHEDULER_WORKERS = 5 + +############################################################################## + +SETUP_INSTALL_DIR = os.path.join(args.install_dir, '.setup') + +INSTALL_FILE = os.path.join(SETUP_INSTALL_DIR, 'INSTALL_SUBMITTY.sh') +CONFIGURATION_JSON = os.path.join(SETUP_INSTALL_DIR, 'submitty_conf.json') +CONFIG_INSTALL_DIR = os.path.join(args.install_dir, 'config') + +EMAIL_JSON = os.path.join(CONFIG_INSTALL_DIR, 'email.json') +AUTHENTICATION_JSON = os.path.join(CONFIG_INSTALL_DIR, 'authentication.json') +DATABASE_JSON = os.path.join(CONFIG_INSTALL_DIR, 'database.json') +SUBMITTY_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty.json') +SUBMITTY_USERS_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_users.json') +WORKERS_JSON = os.path.join(CONFIG_INSTALL_DIR, 'autograding_workers.json') +SUBMITTY_ADMIN_JSON = os.path.join(CONFIG_INSTALL_DIR, 'submitty_admin.json') +CONTAINERS_JSON = os.path.join(CONFIG_INSTALL_DIR, 'autograding_containers.json') +SECRETS_PHP_JSON = os.path.join(CONFIG_INSTALL_DIR, 'secrets_submitty_php.json') + +if not args.worker: + PHP_UID, PHP_GID = get_ids(PHP_USER) + CGI_UID, CGI_GID = get_ids(CGI_USER) + # System Groups + DAEMONPHP_GROUP = 'submitty_daemonphp' + try: + grp.getgrnam(DAEMONPHP_GROUP) + except KeyError: + raise SystemExit("ERROR: Could not find group: " + DAEMONPHP_GROUP) + DAEMONCGI_GROUP = 'submitty_daemoncgi' + try: + grp.getgrnam(DAEMONCGI_GROUP) + except KeyError: + raise SystemExit("ERROR: Could not find group: " + DAEMONCGI_GROUP) + DAEMONPHPCGI_GROUP = 'submitty_daemonphpcgi' + try: + grp.getgrnam(DAEMONPHPCGI_GROUP) + except KeyError: + raise SystemExit("ERROR: Could not find group: " + DAEMONPHPCGI_GROUP) + +DAEMON_UID, DAEMON_GID = get_ids(DAEMON_USER) + + +try: + grp.getgrnam(COURSE_BUILDERS_GROUP) +except KeyError: + raise SystemExit("ERROR: Could not find group: " + COURSE_BUILDERS_GROUP) + +############################################################################## + +# This is the upper limit of the number of parallel grading threads on +# this machine +NUM_UNTRUSTED = 60 + +FIRST_UNTRUSTED_UID, FIRST_UNTRUSTED_GID = get_ids('untrusted00') + +# confirm that the uid/gid of the untrusted users are sequential +for i in range(1, NUM_UNTRUSTED): + untrusted_user = "untrusted{:0=2d}".format(i) + uid, gid = get_ids(untrusted_user) + if uid != FIRST_UNTRUSTED_UID + i: + raise SystemExit('CONFIGURATION ERROR: untrusted UID not sequential: ' + untrusted_user) + elif gid != FIRST_UNTRUSTED_GID + i: + raise SystemExit('CONFIGURATION ERROR: untrusted GID not sequential: ' + untrusted_user) + +with open(SUBMITTY_JSON, 'r') as json_file: + config = json.load(json_file, object_pairs_hook=OrderedDict) + config['num_untrusted'] = NUM_UNTRUSTED + config['first_untrusted_uid'] = FIRST_UNTRUSTED_UID + config['first_untrusted_gid'] = FIRST_UNTRUSTED_UID + + config['daemon_uid'] = DAEMON_UID + config['daemon_gid'] = DAEMON_GID + + # not worker + config['daemonphp_group'] = DAEMONPHP_GROUP + config['daemoncgi_group'] = DAEMONCGI_GROUP + config['daemonphpcgi_group'] = DAEMONPHPCGI_GROUP + config['php_uid'] = PHP_UID + config['php_gid'] = PHP_GID + +#Rescue the autograding_workers and _containers files if they exist. +rescued = list() +tmp_folder = tempfile.mkdtemp() +if not args.worker: + for full_file_name, file_name in [(WORKERS_JSON, 'autograding_workers.json'), (CONTAINERS_JSON, 'autograding_containers.json')]: + if os.path.isfile(full_file_name): + #make a tmp folder and copy autograding workers to it + tmp_file = os.path.join(tmp_folder, file_name) + shutil.move(full_file_name, tmp_file) + rescued.append((full_file_name, tmp_file)) + +IGNORED_FILES_AND_DIRS = ['saml', 'login.md'] + +if os.path.isdir(CONFIG_INSTALL_DIR): + for file in os.scandir(CONFIG_INSTALL_DIR): + if file.name not in IGNORED_FILES_AND_DIRS: + if file.is_file(): + os.remove(os.path.join(CONFIG_INSTALL_DIR, file.name)) + else: + os.rmdir(os.path.join(CONFIG_INSTALL_DIR, file.name)) +elif os.path.exists(CONFIG_INSTALL_DIR): + os.remove(CONFIG_INSTALL_DIR) +os.makedirs(CONFIG_INSTALL_DIR, exist_ok=True) +shutil.chown(CONFIG_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) +os.chmod(CONFIG_INSTALL_DIR, 0o755) + +# Finish rescuing files. +for full_file_name, tmp_file_name in rescued: + #copy autograding workers back + shutil.move(tmp_file_name, full_file_name) + #make sure the permissions are correct. + shutil.chown(full_file_name, 'root', DAEMON_GID) + os.chmod(full_file_name, 0o660) + +#remove the tmp folder +os.removedirs(tmp_folder) + +############################################################################## +# Write users json + +config = OrderedDict() +config['num_grading_scheduler_workers'] = NUM_GRADING_SCHEDULER_WORKERS +config['num_untrusted'] = NUM_UNTRUSTED +config['first_untrusted_uid'] = FIRST_UNTRUSTED_UID +config['first_untrusted_gid'] = FIRST_UNTRUSTED_UID +config['daemon_uid'] = DAEMON_UID +config['daemon_gid'] = DAEMON_GID +config['daemon_user'] = DAEMON_USER +config['course_builders_group'] = COURSE_BUILDERS_GROUP + +if not args.worker: + config['php_uid'] = PHP_UID + config['php_gid'] = PHP_GID + config['php_user'] = PHP_USER + config['cgi_user'] = CGI_USER + config['daemonphp_group'] = DAEMONPHP_GROUP + config['daemoncgi_group'] = DAEMONCGI_GROUP + config['daemonphpcgi_group'] = DAEMONPHPCGI_GROUP +if os.path.exists(SUBMITTY_USERS_JSON): + with open(SUBMITTY_USERS_JSON, 'r') as json_file: + config_file = json.load(json_file) + config['supervisor_user'] = config_file['supervisor_user'] +with open(SUBMITTY_USERS_JSON, 'w') as json_file: + json.dump(config, json_file, indent=2) +shutil.chown(SUBMITTY_USERS_JSON, 'root', DAEMON_GROUP if args.worker else DAEMONPHP_GROUP) + +os.chmod(SUBMITTY_USERS_JSON, 0o440) +shutil.chown(WORKERS_JSON, PHP_USER, DAEMON_GID) +shutil.chown(CONTAINERS_JSON, group=DAEMONPHP_GROUP) +shutil.chown(DATABASE_JSON, 'root', DAEMONPHP_GROUP) +shutil.chown(AUTHENTICATION_JSON, 'root', DAEMONPHP_GROUP) +shutil.chown(EMAIL_JSON, 'root', DAEMONPHP_GROUP) +shutil.chown(SECRETS_PHP_JSON, 'root', PHP_GROUP) +shutil.chown(SUBMITTY_ADMIN_JSON, 'root', DAEMON_GROUP) +shutil.chown(SETUP_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) +shutil.chown(CONFIG_INSTALL_DIR, 'root', COURSE_BUILDERS_GROUP) \ No newline at end of file diff --git a/.setup/distro_setup/ubuntu/22.04/variables.sh b/.setup/distro_setup/ubuntu/22.04/variables.sh new file mode 100644 index 00000000000..80edc76c777 --- /dev/null +++ b/.setup/distro_setup/ubuntu/22.04/variables.sh @@ -0,0 +1,6 @@ +if [ ${DEV_VM} == 1 ]; then + export SUBMISSION_URL='http://localhost:1511' + export SUBMISSION_PORT=1511 + export DATABASE_PORT=16442 + export WEBSOCKET_PORT=8443 +fi diff --git a/.setup/distro_setup/variables.sh b/.setup/distro_setup/variables.sh new file mode 100644 index 00000000000..42a8cf03d97 --- /dev/null +++ b/.setup/distro_setup/variables.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# this script must be run by root or sudo +if [[ "$UID" -ne "0" ]] ; then + echo "ERROR: This script must be run by root or sudo" + exit +fi + +# Note: This script is source by install_system.sh and so caution should be used on naming variables +# to ensure there is no collision. +SETUP_DISTRO_DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DISTRO=$(lsb_release -si | tr '[:upper:]' '[:lower:]') +VERSION=$(lsb_release -sr | tr '[:upper:]' '[:lower:]') + +if [ ! -d ${SETUP_DISTRO_DIR}/${DISTRO}/${VERSION} ]; then + (>&2 echo "Unknown distro: ${DISTRO} ${VERSION}") + exit 1 +fi + +source ${SETUP_DISTRO_DIR}/${DISTRO}/${VERSION}/variables.sh diff --git a/.setup/install_system.sh b/.setup/install_system.sh index 2263d1373c2..0babff05a63 100644 --- a/.setup/install_system.sh +++ b/.setup/install_system.sh @@ -41,6 +41,12 @@ LICHEN_REPOSITORY=/usr/local/submitty/GIT_CHECKOUT/Lichen SUBMITTY_INSTALL_DIR=/usr/local/submitty SUBMITTY_DATA_DIR=/var/local/submitty +# create directory and fix permissions +mkdir -p ${SUBMITTY_DATA_DIR} +mkdir -p ${SUBMITTY_INSTALL_DIR}/config +INSTALL_SYS_DIR=$(mktemp -d) +chmod 777 "${INSTALL_SYS_DIR}" +pushd "${INSTALL_SYS_DIR}" > /dev/null # USERS / GROUPS DAEMON_USER=submitty_daemon @@ -50,12 +56,17 @@ PHP_GROUP=submitty_php CGI_USER=submitty_cgi CGI_GROUP=submitty_cgi +COURSE_BUILDERS_GROUP=submitty_course_builders +DB_USER=submitty_dbuser +DATABASE_PASSWORD=submitty_dbuser +DB_COURSE_USER=submitty_course_dbuser +DB_COURSE_PASSWORD=submitty_dbuser + DAEMONPHP_GROUP=submitty_daemonphp DAEMONCGI_GROUP=submitty_daemoncgi DAEMONPHPCGI_GROUP=submitty_daemonphpcgi -# VERSIONS -source ${CURRENT_DIR}/bin/versions.sh + ################################################################# # PROVISION SETUP @@ -107,6 +118,66 @@ fi if [ ${UTM} == 1 ]; then mkdir ${SUBMITTY_REPOSITORY}/.utm fi +################################################################# +# DISTRO SETUP +################# +# Sources +source ${CURRENT_DIR}/bin/versions.sh +source ${CURRENT_DIR}/distro_setup/variables.sh +################################################################# +# SUBMITTY SETUP +################# +apt-get update +apt-get install -qqy python3 python3-pip +pip3 install tzlocal +#If in worker mode, run configure with --worker option. +if [ ${WORKER} == 1 ]; then + echo "Running configure submitty in worker mode" + if [ ${DEV_VM} == 1 ]; then + echo "submitty" | python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --worker + else + python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --worker + fi +else + if [ ${DEV_VM} == 1 ]; then + # This should be set by setup_distro.sh for whatever distro we have, but + # in case it is not, default to our primary URL + if [ -z "${SUBMISSION_URL}" ]; then + SUBMISSION_URL='http://192.168.56.101' + fi + mkdir /var/run/postgresql + echo -e "/var/run/postgresql +${DB_USER} +${DATABASE_PASSWORD} +${DB_COURSE_USER} +${DB_COURSE_PASSWORD} +America/New_York +en_US +100 +${SUBMISSION_URL} + + +sysadmin@example.com +https://example.com +1 +submitty-admin +y + + +submitty@vagrant +do-not-reply@vagrant +localhost +25 +" | python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --debug --setup-for-sample-courses --websocket-port ${WEBSOCKET_PORT} + + # Set these manually as they're not asked about during CONFIGURE_SUBMITTY.py + sed -i -e 's/"url": ""/"url": "ldap:\/\/localhost"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json + sed -i -e 's/"uid": ""/"uid": "uid"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json + sed -i -e 's/"bind_dn": ""/"bind_dn": "ou=users,dc=vagrant,dc=local"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json + else + python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py + fi +fi if [ ${DEV_VM} == 1 ] && [ ${WORKER} == 0 ]; then # Setting it up to allow SSH as root by default @@ -208,24 +279,8 @@ else fi -INSTALL_SYS_DIR=$(mktemp -d) -chmod 777 "${INSTALL_SYS_DIR}" -pushd "${INSTALL_SYS_DIR}" > /dev/null - -COURSE_BUILDERS_GROUP=submitty_course_builders -DB_USER=submitty_dbuser -DATABASE_PASSWORD=submitty_dbuser -DB_COURSE_USER=submitty_course_dbuser -DB_COURSE_PASSWORD=submitty_dbuser - -################################################################# -# DISTRO SETUP -################# - source ${CURRENT_DIR}/distro_setup/setup_distro.sh -bash "${SUBMITTY_REPOSITORY}/.setup/update_system.sh" "config=${SUBMITTY_DIRECTORY}" - ################################################################# # STACK SETUP ################# @@ -349,6 +404,13 @@ sudo chown "${DAEMON_USER}:${DAEMON_USER}" "$gitconfig_path" usermod -a -G docker "${DAEMON_USER}" +#If in worker mode, run configure with --worker option. +if [ ${WORKER} == 1 ]; then + echo "Using users to update config files" + python3 ${SUBMITTY_REPOSITORY}/.setup/USERS_SETUP.py --worker +else + python3 ${SUBMITTY_REPOSITORY}/.setup/USERS_SETUP.py +fi ################################################################# # JAR SETUP ################# @@ -576,9 +638,6 @@ EOF sed -i -e "s/^disable_functions = .*/disable_functions = ${DISABLED_FUNCTIONS}/g" /etc/php/${PHP_VERSION}/fpm/php.ini fi -# create directories and fix permissions -mkdir -p ${SUBMITTY_DATA_DIR} - #Set up database and copy down the tutorial repo if not in worker mode if [ ${WORKER} == 0 ]; then # create the courses directory. This is needed for the first time we run @@ -667,59 +726,6 @@ if [ ! -d "${clangsrc}" ]; then echo 'DONE PREPARING CLANG INSTALLATION' fi -################################################################# -# SUBMITTY SETUP -################# -echo Beginning Submitty Setup - -#If in worker mode, run configure with --worker option. -if [ ${WORKER} == 1 ]; then - echo "Running configure submitty in worker mode" - if [ ${DEV_VM} == 1 ]; then - echo "submitty" | python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --worker - else - python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --worker - fi -else - if [ ${DEV_VM} == 1 ]; then - # This should be set by setup_distro.sh for whatever distro we have, but - # in case it is not, default to our primary URL - if [ -z "${SUBMISSION_URL}" ]; then - SUBMISSION_URL='http://192.168.56.101' - fi - echo -e "/var/run/postgresql -${DB_USER} -${DATABASE_PASSWORD} -${DB_COURSE_USER} -${DB_COURSE_PASSWORD} -America/New_York -en_US -100 -${SUBMISSION_URL} - - -sysadmin@example.com -https://example.com -1 -submitty-admin -y - - -submitty@vagrant -do-not-reply@vagrant -localhost -25 -" | python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --debug --setup-for-sample-courses --websocket-port ${WEBSOCKET_PORT} - - # Set these manually as they're not asked about during CONFIGURE_SUBMITTY.py - sed -i -e 's/"url": ""/"url": "ldap:\/\/localhost"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json - sed -i -e 's/"uid": ""/"uid": "uid"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json - sed -i -e 's/"bind_dn": ""/"bind_dn": "ou=users,dc=vagrant,dc=local"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json - else - python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py - fi -fi - if [ ${WORKER} == 1 ]; then #Add the submitty user to /etc/sudoers if in worker mode. SUPERVISOR_USER=$(jq -r '.supervisor_user' ${SUBMITTY_INSTALL_DIR}/config/submitty_users.json)