diff --git a/.github/workflows/new-preview-container.yml b/.github/workflows/new-preview-container.yml new file mode 100644 index 00000000..676d7972 --- /dev/null +++ b/.github/workflows/new-preview-container.yml @@ -0,0 +1,35 @@ +name: Build and Push Docker Image + +on: + push: + branches: + - master + +jobs: + + build: + runs-on: ubuntu-latest + if: startsWith(github.repository, 'osg-htc/') + + steps: + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + ref: master # Needed or it will checkout the wrong commit + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to OSG Harbor + uses: docker/login-action@v1 + with: + registry: hub.opensciencegrid.org + username: ${{ secrets.OSG_HARBOR_ROBOT_USER }} + password: ${{ secrets.OSG_HARBOR_ROBOT_PASSWORD }} + + - name: Build and push Docker images + uses: docker/build-push-action@v2.2.0 + with: + context: . + push: true + tags: "hub.opensciencegrid.org/osg-portal-documentation:latest" \ No newline at end of file diff --git a/.github/workflows/publish-workflow.yml b/.github/workflows/publish-workflow.yml deleted file mode 100644 index c1e06d36..00000000 --- a/.github/workflows/publish-workflow.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Publish to Freshdesk -# This workflow is triggered on pushes to the repository. -on: [push] -jobs: - publish: - # Job name is Greeting - name: publish - # This job runs on Linux, but with our own container - runs-on: ubuntu-latest - container: - image: docker://opensciencegrid/osgvo-ubuntu-18.04:latest - steps: - # use a custom git clone so we can do sub modules - - name: Configuation file setup - env: - APIKEY: ${{ secrets.FRESHDESK_APIKEY }} - run: | - git clone --recursive https://github.com/OSGConnect/connectbook.git /tmp/connectbook - cd /tmp/connectbook - echo "[api]" >update/apikey.ini - echo "apikey = $APIKEY" >>update/apikey.ini - - name: Run the update script - run: | - cd /tmp/connectbook - ./update/update.sh - diff --git a/.github/workflows/update-documentation.yml b/.github/workflows/update-documentation.yml new file mode 100644 index 00000000..0f6e5542 --- /dev/null +++ b/.github/workflows/update-documentation.yml @@ -0,0 +1,83 @@ +name: Update the documentation website + +on: + workflow_dispatch: + push: + branches: + - master + +jobs: + + Update-Submodules: + runs-on: ubuntu-latest + if: startsWith(github.repository, 'osg-htc/') + steps: + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + fetch-depth: 0 + + - name: Pull, Merge, Push submodules + run: | + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + + git submodule update --remote --merge + + if [ -n "$(git status --porcelain)" ]; then + git add . + git commit -m "Update Documentation" + git push + else + echo "no changes"; + fi + + Build-Static-Site: + runs-on: ubuntu-latest + if: startsWith(github.repository, 'osg-htc/') + needs: Update-Submodules + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: 'recursive' + + - name: Initialize GH User + run: | + git config user.name "GitHub Actions" + git config user.email "actions@github.com" + + - name: Update the Submodule pulled + run: | + git submodule update --recursive --remote + + - name: Build MkDocs pages + if: startsWith(github.repository, 'osg-htc/') + uses: docker://squidfunk/mkdocs-material:8.2.8 + with: + args: >- + build + --verbose + + - name: Deploy to 'static' branch + run: | + # Commit the build then use 'git subtree' to create a branch with just the site contents + git add site -f + git commit -m "Build Static HTML" + git checkout -b static `git subtree split --prefix site master` + + # Push to Production + git push --set-upstream origin static --force + + Trigger-New-Documentation-Release: + runs-on: ubuntu-latest + if: startsWith(github.repository, 'osg-htc/') + needs: [Build-Static-Site, Update-Submodules] + env: + OWNER: "osg-htc" + REPO: "osg-portal" + WORKFLOW_ID: "release-documentation-update.yml" + steps: + - name: Trigger Documentation Release + run: | + curl -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.REPO_ACCESS_TOKEN }}" https://api.github.com/repos/${{env.OWNER}}/${{env.REPO}}/actions/workflows/${{env.WORKFLOW_ID}}/dispatches -d '{"ref":"master"}' diff --git a/.github/workflows/validate-mkdocs.yml b/.github/workflows/validate-mkdocs.yml new file mode 100644 index 00000000..01ff6708 --- /dev/null +++ b/.github/workflows/validate-mkdocs.yml @@ -0,0 +1,40 @@ +name: Validate static MkDocs pages +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + validate-mkdocs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Build MkDocs pages + uses: docker://squidfunk/mkdocs-material:8.2.8 + timeout-minutes: 1 + with: + args: >- + build + --verbose + --strict + + - id: format-github-repo + run: echo "::set-output name=repo-name::${GITHUB_REPOSITORY#*\/}" + + - name: Test links + timeout-minutes: 10 + uses: docker://klakegg/html-proofer:3.16.0 + env: + GITHUB_REPOSITORY: ${{ github.repository }} + with: + args: >- + --allow-hash-href + --check-html + --http-status-ignore 302 + ./documentation diff --git a/.gitmodules b/.gitmodules index 8d971b55..bfecdd39 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,96 +1,99 @@ [submodule "tutorials/tutorial-AutoDockVina"] - path = tutorials/tutorial-AutoDockVina + path = docs/software_examples_for_osg/drug_discovery/tutorial-AutoDockVina url = https://github.com/OSGConnect/tutorial-AutoDockVina [submodule "tutorials/tutorial-R"] - path = tutorials/tutorial-R + path = docs/software_examples_for_osg/r/tutorial-R url = https://github.com/OSGConnect/tutorial-R [submodule "tutorials/tutorial-R-addlibSNA"] - path = tutorials/tutorial-R-addlibSNA + path = docs/software_examples_for_osg/r/tutorial-R-addlibSNA url = https://github.com/OSGConnect/tutorial-R-addlibSNA [submodule "tutorials/tutorial-ScalingUp-R"] - path = tutorials/tutorial-ScalingUp-R + path = docs/software_examples_for_osg/r/tutorial-ScalingUp-R url = https://github.com/OSGConnect/tutorial-ScalingUp-R [submodule "tutorials/tutorial-blast"] - path = tutorials/tutorial-blast + path = docs/tutorials/tutorial-blast url = https://github.com/OSGConnect/tutorial-blast [submodule "tutorials/tutorial-error101"] - path = tutorials/tutorial-error101 + path = docs/managing_htc_workloads_on_osg_connect/submitting_htc_workloads_with_htcondor/tutorial-error101 url = https://github.com/OSGConnect/tutorial-error101 [submodule "tutorials/tutorial-exitcode"] - path = tutorials/tutorial-exitcode + path = docs/tutorials/tutorial-exitcode url = https://github.com/OSGConnect/tutorial-exitcode [submodule "tutorials/tutorial-gromacs"] - path = tutorials/tutorial-gromacs + path = docs/tutorials/tutorial-gromacs url = https://github.com/OSGConnect/tutorial-gromacs [submodule "tutorials/tutorial-htcondor-transfer"] - path = tutorials/tutorial-htcondor-transfer + path = docs/tutorials/tutorial-htcondor-transfer url = https://github.com/OSGConnect/tutorial-htcondor-transfer [submodule "tutorials/tutorial-makeflow-quickstart"] - path = tutorials/tutorial-makeflow-quickstart + path = docs/tutorials/tutorial-makeflow-quickstart url = https://github.com/OSGConnect/tutorial-makeflow-quickstart [submodule "tutorials/tutorial-matlab-HelloWorld"] - path = tutorials/tutorial-matlab-HelloWorld + path = docs/software_examples_for_osg/matlab_runtime/tutorial-matlab-HelloWorld url = https://github.com/OSGConnect/tutorial-matlab-HelloWorld [submodule "tutorials/tutorial-namd"] - path = tutorials/tutorial-namd + path = docs/tutorials/tutorial-namd url = https://github.com/OSGConnect/tutorial-namd [submodule "tutorials/tutorial-nelle-nemo"] - path = tutorials/tutorial-nelle-nemo + path = docs/tutorials/tutorial-nelle-nemo url = https://github.com/OSGConnect/tutorial-nelle-nemo [submodule "tutorials/tutorial-octave"] - path = tutorials/tutorial-octave + path = docs/tutorials/tutorial-octave url = https://github.com/OSGConnect/tutorial-octave [submodule "tutorials/tutorial-pegasus"] - path = tutorials/tutorial-pegasus + path = docs/managing_htc_workloads_on_osg_connect/automated_workflows/tutorial-pegasus url = https://github.com/OSGConnect/tutorial-pegasus [submodule "tutorials/tutorial-photodemo"] - path = tutorials/tutorial-photodemo + path = docs/tutorials/tutorial-photodemo url = https://github.com/OSGConnect/tutorial-photodemo [submodule "tutorials/tutorial-python-virtualenv"] - path = tutorials/tutorial-python-virtualenv + path = docs/tutorials/tutorial-python-virtualenv url = https://github.com/OSGConnect/tutorial-python-virtualenv [submodule "tutorials/tutorial-quickstart"] - path = tutorials/tutorial-quickstart + path = docs/managing_htc_workloads_on_osg_connect/submitting_htc_workloads_with_htcondor/tutorial-quickstart url = https://github.com/OSGConnect/tutorial-quickstart [submodule "tutorials/tutorial-root"] - path = tutorials/tutorial-root + path = docs/tutorials/tutorial-root url = https://github.com/OSGConnect/tutorial-root [submodule "tutorials/tutorial-scaling"] - path = tutorials/tutorial-scaling + path = docs/tutorials/tutorial-scaling url = https://github.com/OSGConnect/tutorial-scaling [submodule "tutorials/tutorial-scaling-up-resources"] - path = tutorials/tutorial-scaling-up-resources + path = docs/tutorials/tutorial-scaling-up-resources url = https://github.com/OSGConnect/tutorial-scaling-up-resources [submodule "tutorials/tutorial-software"] - path = tutorials/tutorial-software + path = docs/tutorials/tutorial-software url = https://github.com/OSGConnect/tutorial-software [submodule "tutorials/tutorial-stash-http"] - path = tutorials/tutorial-stash-http + path = docs/tutorials/tutorial-stash-http url = https://github.com/OSGConnect/tutorial-stash-http [submodule "tutorials/tutorial-stashcache-blast"] - path = tutorials/tutorial-stashcache-blast + path = docs/tutorials/tutorial-stashcache-blast url = https://github.com/OSGConnect/tutorial-stashcache-blast [submodule "tutorials/tutorial-swift"] - path = tutorials/tutorial-swift + path = docs/tutorials/tutorial-swift url = https://github.com/OSGConnect/tutorial-swift [submodule "tutorials/tutorial-osg-locations"] - path = tutorials/tutorial-osg-locations + path = docs/managing_htc_workloads_on_osg_connect/submitting_htc_workloads_with_htcondor/tutorial-osg-locations url = https://github.com/OSGConnect/tutorial-osg-locations [submodule "tutorials/tutorial-ScalingUp-Python"] - path = tutorials/tutorial-ScalingUp-Python + path = docs/software_examples_for_osg/python/tutorial-ScalingUp-Python url = https://github.com/OSGConnect/tutorial-ScalingUp-Python [submodule "tutorials/tutorial-blast-split"] - path = tutorials/tutorial-blast-split + path = docs/software_examples_for_osg/bioinformatics/tutorial-blast-split url = https://github.com/OSGConnect/tutorial-blast-split [submodule "tutorials/tutorial-tensorflow-containers"] - path = tutorials/tutorial-tensorflow-containers + path = docs/software_examples_for_osg/machine_learning/tutorial-tensorflow-containers url = https://github.com/OSGConnect/tutorial-tensorflow-containers.git [submodule "tutorials/tutorial-wordfreq"] - path = tutorials/tutorial-wordfreq + path = docs/software_examples_for_osg/python/tutorial-wordfreq url = https://github.com/OSGConnect/tutorial-wordfreq [submodule "tutorials/tutorial-bwa"] - path = tutorials/tutorial-bwa + path = docs/software_examples_for_osg/bioinformatics/tutorial-bwa url = https://github.com/OSGConnect/tutorial-bwa [submodule "tutorials/tutorial-organizing"] - path = tutorials/tutorial-organizing + path = docs/managing_htc_workloads_on_osg_connect/submitting_htc_workloads_with_htcondor/tutorial-organizing url = https://github.com/OSGConnect/tutorial-organizing +[submodule "user-documentation"] + path = user-documentation + url = https://github.com/osg-htc/user-documentation.git diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e3c0d63d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM squidfunk/mkdocs-material:latest + +COPY user-documentation/scripts/requirements.txt /tmp/requirements.txt +RUN pip3 install -r /tmp/requirements.txt + +ADD images/ /docs/images +ADD overrides/ /docs/overrides +COPY mkdocs.yml /docs/ + +COPY scripts/entrypoint.sh /entrypoint.sh +RUN chmod 755 /entrypoint.sh + +ENTRYPOINT ["sh", "/entrypoint.sh"] +CMD ["serve", "--dev-addr=0.0.0.0:8000"] diff --git a/README.md b/README.md index 677decd8..499bd41e 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,49 @@ -![](https://github.com/osgconnect/connectbook/workflows/Publish%20to%20Freshdesk/badge.svg) +[![Validate static MkDocs pages](https://github.com/osg-htc/osg-portal-documentation/actions/workflows/validate-mkdocs.yml/badge.svg)](https://github.com/osg-htc/osg-portal-documentation/actions/workflows/validate-mkdocs.yml) -The OSG ConnectBook +# Mkdocs Static Website -Be sure to clone recursively! +If you are here to edit something you are likely in the wrong place. -git clone --recursive https://github.com/OSGConnect/connectbook.git +To edit the OSG Portal look here -> https://github.com/osg-htc/osg-portal +To edit User Documentation look here -> https://github.com/osg-htc/user-documentation -To update the submodules run +## For the Minority -git submodule foreach git pull origin master +This repository hosts the code for building the static Mkdocs website that is hosted at portal.osg-htc.org/documentation. + +All new commits to master are built into a static site found on the ```static``` branch, which is then pulled into the osg-portal. + +Changes you should be making here include: +- HTML Templates + +**There is 0 reason to touch the ```./user-documentation``` folder from this repo** + +If you are looking to change the following look in [here](https://github.com/osg-htc/user-documentation): +- CSS +- Documentation +- Website Layout + +### Development + +```shell +docker run --rm -it -p 8001:8001 -v ${PWD}:/docs squidfunk/mkdocs-material +``` + +ARM +```shell +docker run --rm -it -p 8000:8000 -v ${PWD}:/docs ghcr.io/afritzler/mkdocs-material +``` + +### Test Links Locally + +```shell +# Build the site into /site directory +docker run --rm -v ${PWD}:/docs ghcr.io/afritzler/mkdocs-material build + +# Test the links +docker run --rm -it \ + -v $(pwd)/documentation:/src \ + klakegg/html-proofer:3.19.2 \ + --allow-hash-href --check-html --empty-alt-ignore --disable-external +``` diff --git a/admin/add-solution-2.jpg b/admin/add-solution-2.jpg deleted file mode 100644 index af0d077c..00000000 Binary files a/admin/add-solution-2.jpg and /dev/null differ diff --git a/admin/add-solution.jpg b/admin/add-solution.jpg deleted file mode 100644 index 8ce975d8..00000000 Binary files a/admin/add-solution.jpg and /dev/null differ diff --git a/admin/connect/approving-user-application.md b/admin/connect/approving-user-application.md deleted file mode 100644 index ae7a90e7..00000000 --- a/admin/connect/approving-user-application.md +++ /dev/null @@ -1,88 +0,0 @@ -[TOC] - -[title]: - "Approving User Applications" - - -## Requirements for Users - -Users need to satisfy the following in order to be approved: - -* Must provide an institutional email address (e.g. user@university.edu, etc.) -* Must apply with using globusid account -* Must be affiliated with a US institution or research group - -## On-Boarding - -Use the following on-boarding procedures for new users - -* Copy and paste the user's "Member Info" into a note in the account application ticket: - * Go to https://www.osgconnect.net and sign in. - * Go to Connect > My Projects and click on the osg group. - * On the "About" tab, select the name of the applicant and then click on "Member Info" - * Copy and paste the fields (Username, Full Name, E-mail Address, etc...) into the ticket as a note -* Call the user at the phone number provided and conduct the first [on-boarding consultation](https://docs.google.com/document/d/1Bpk9m7RUzqpccv8VoYpn12pFUjH5orGabN54SUYr74k/edit?usp=sharing). If unable to reach the user, leave a voice message and send a reply to the ticket with an invitation for them to call you or provide a good time for you to call. If the user doesn't respond, ping the user once or twice over the course of a week or two before rejecting the application (see below) and resolving the ticket. - * ***Sample email: "We received your application for a computing account on OSG Connect. We called you to learn about your computing needs on OSG Connect and confirm your identity, but were unable to reach you. Please specify a convenient time to contact you or call me at XXX-XXX-XXXX at your earliest convenience. -Agent"*** -* Once on-boarding consultation is complete, approve user to join the osg group: - * In the osgconnect.net portal go to My Projects and click on the osg group. - * On the "About" tab, select the name of the applicant and then click Accept. -* Invite the user to the group osg.ConnectTrain: - * In the osgconnect.net portal go to My Projects and click on osg.ConnectTrain. - * Click on the "Members" tab and select "Invite people to this group". - * Search for the user by globusID and send the invite. -* Reply to the ticket with the OSG Connect [Welcome email](https://docs.google.com/document/d/1z9MObqfeOypO7zSyuXe2T_KgMZUe2YIYo8hCHqklXVI/edit?usp=sharing). -* Add a note to the ticket with notes on the phone consultation (a short paragraph or so). -* If there are no outstanding issues, change the status of the ticket to "resolved" and the Type to "on-boarding X weeks". -* Add the user's information to Insightly: ***Update: Billing info in Insightly needs to be updated to continue using*** - * Go to https://opensciencegrid.insight.ly and log in. - * Click on the "+" icon at the top of the page and select "Add New Contact" from the drop-down menu. - * Fill in the following fields for the new member: First name, last name, Occupation, Organization, Email Address, Phone, Username (globusID), Field of Science, and Department. - * Click "Save" and then "Add Tags". Select the "osg-connect-users" tag, and then click "save". - -## Finishing On-Boarding - -The final on-boarding procedure for new users. - -* When the "finished on-boarding" ticket automatically re-opens, send a reply to the user asking if they have been able to get their workflow running and if they need any help. - * ***Sample email: "I was wondering if you have had a chance to get a workflow going on OSG yet? If you have come across any problems or have any questions for me, I would be happy to help. -Agent"*** -* Change ticket status to "waiting on customer". -* If there is no response within a few days to a week, resolve the ticket. If there is a response, try to resolve the issue or schedule a second phone consultation if needed. - -## Approving Users - -Do the following to add user to the newly created project - -* Go to https://www.osgconnect.net and login as the *connect* user -* Go to Connect > My Projects in the menu -* Go to osg group or relevant group if applying to another group -* Click on members and click on pencil icon next to user -* Click on Approve button -* Do the above for the connect group as well - -## Rejecting an Application - -* Go to https://www.osgconnect.net and login as the *connect* user -* Go to Connect > My Projects in the menu -* Go to osg group or relevant group if applying to another group -* Click on members and click on pencil icon next to user -* Click on Reject button -* Do the above for the connect group as well - -## On-Boarding Scenario: Username on application is not a GlobusID or Email is not a university/institutional email. - -* Copy and paste the user's "Member Info" into a note in the account application ticket. -* Send the user an email informing them of the issue and asking them to re-apply. - * ***Sample email (no GlobusID): "We got your application to join OSG Connect. In the application, the username is XXXXXXX. This username is based on an institute provided ID, but it needs to be a GlobusID. On our system it is difficult to create and maintain a unix account with a username other than a globusID. Could you please re-apply with a globusID as username as outlined here: https://osgconnect.net/signup. I will remove your current pending application so that you will be able to re-apply. -Agent"*** - * ***Sample email (no institutional email): "We received your application to join OSG Connect. The email you used for the account application is a XXXXX address. Could you please re-apply and this time supply your XXXXX University email address so that we can verify your affiliation? I will remove the current application so that you will be able to re-apply. -Agent"*** -* Reject the application for both the "osg" and "connect" groups so that they will be able to re-apply. -* Change ticket status to "waiting on customer". -* If user re-applies, merge the two tickets. If user does not re-apply, resolve the original ticket after waiting a week or so. - -## On-Boarding Scenario: User has no apparent affiliation with a non-profit U.S. research institution or project. - -* Copy and paste the user's "Member Info" into a note in the account application ticket. -* Send the user an email informing them of the issue and asking if they have a U.S. affiliation. - * ***Sample email: "We received your application for an OSG Connect account. One of the requirements to have an account on OSG Connect is that the applicant needs to have an affiliation with a US Institution. Do you have an affiliation with a U.S. institution, organization, or project that was not listed on your application? -Agent"*** -* Change ticket status to "waiting on customer". -* If user does not reply in a week or so, reject the application for both the "osg" and "connect" groups and resolve the ticket. If user provides an appropriate affiliation, contine with on-boarding process outlined above (make sure to still require institutional email or some other verification of affiliation). - - diff --git a/admin/connect/create-project.md b/admin/connect/create-project.md deleted file mode 100644 index f72d8c55..00000000 --- a/admin/connect/create-project.md +++ /dev/null @@ -1,59 +0,0 @@ -[TOC] - -[title]: - "Creating a new project" - -Creating a project involves two separate steps: creating the project in the Globus system and adding the user to the group in Globus. Note, it may take a few hours for changes to propagate to the login nodes. - -## Assumptions - -Creating a project requires the following two things: - -* An account on services.ci-connect.net -* Admin access on globus (i.e. access to the connect@globusid.net account) - -## Creating the project - -Do the following to create the project in Globus - -* Login to services.ci-connect.net -* Create a file in /tmp (e.g. /tmp/) with the text from the helpdesk ticket (see bottom of this section for a template) -* _cd /usr/local/gosync3_ -* run _./add_connect_group.py --projectfile /path/to/file --parent _ , is usually osg - -Project file template: - - Your Name: - Your Email Address: - Project Name: - Short Project Name: - Field of Science: - Field of Science (if Other): - PI Name: - PI Email: - PI Organization: - PI Department: - Join Date: - Sponsor: - OSG Sponsor Contact: - Project Contact: - Project Contact Email: - Telephone Number: - Project Description: - - - - -## Adding requester to project - -Do the following to add user to the newly created project - -* Go to https://www.osgconnect.com and login as the connect user -* Go to Connect > My Projects in the menu -* Scroll down to the project that was just generated -* Click on members, and then the "Invite people to this group" link -* Search for user, and then hit the send invitation button - -## Helpdesk responses - -There is a canned response that can be used to reply to the user when the group has been created. It's in the 'Research Facilitation > Project Created' response. - diff --git a/admin/connect/user-login-project-manual.md b/admin/connect/user-login-project-manual.md deleted file mode 100644 index 7120b593..00000000 --- a/admin/connect/user-login-project-manual.md +++ /dev/null @@ -1,430 +0,0 @@ -[TOC] - -[title]: - "Create user login and projects by manual intervention with gosync" - - -# GOSync3 - -GOSync3 is the replacement for the original GOSync. It is based on the [Globus SDK](http://globus-sdk-python.readthedocs.io/en/latest/), the [Globus SDK-based Globus Nexus Client](https://github.com/sirosen/globus-nexus-client/tree/master/globus_nexus_client), and puppet/hiera to create and manage UNIX users and groups. The main tasks of these classes and scripts are to interact with the GlobusID and Globus Groups database through their RESTful API and manage the JSON file that puppet/hiera requires for creating and managing user accounts. - -The original GOSync is based on the [Globus Nexus Python library](https://github.com/globusonline/python-nexus-client). The Globus Nexus Python library has been officially deprecated. An improved version of the original GOSync, i.e. GOSync2, was under development. It was still based on the Globus Nexus Python library. The development was abandoned as Globus has moved to a OAuth2-based authentication model and access to a user's GlobusID. - -Important notes READ BEFORE USING: - -* This is a BETA. It does not have all the necessary features to act as a full replacement yet. -* This version uses Globus Nexus client based on the Globus SDK created by Stephen Rosen. This is not an official product of the Globus team. It is maintained though. - -## Assumptions - -Following assumptions are made in the code: - -* All users are part of the `connect` Globus Group -* The `connect` user is an `admin` or `manager` in all relevant groups - -## Work flow - -The GOSync3 work flow is meant to operate without human intervention, i.e. as a `cron` job, besides the normal user approval process. At the moment, there is no connection between account applications and GOSync3. Hence, there is no way of knowing which user is new, updated his/her profile, or changed their group membership. This should change in the future, see the last section for details. - -In the current work flow, GOSync3 retrieves all groups in which the `connect` user is an `Administrator` or `Manager`. The `connect` user acts like the root user in a UNIX operating system. In addition to the group name and the UUID assigned by Globus Groups, the number of active members is being fetched by querying the group's summary from Globus. - -For creating and updating users the work flow is more complicated. First, GOSync3 retrieves all users and their profile associated with the root group, i.e. `connect`. It is necessary to fetch the user profile because it contains the user's SSH key. To determine the the user's group membership, the the group to users mapping is generated by looping through all groups getting their group members. FRom this mapping a user to groups mapping is generated. With the necessary information in hand, it the user information in the JSON object is created or updated. - -## Prerequisites - -GOSync3 requires at least Python 2.7 and Python packages: - -``` -globus-sdk[jwt]>=1.0,<2.0 -globus-nexus-client>=0.2.5 -``` - -`globus-sdk[jwt]>=1.0,<2.0` is the Globus SDK including the [JSON Web Token (JWT)](https://jwt.io/) library. JWT is necessary to interact with Globus Auth and be able to do token introspection. `globus-nexus-client>=0.2.5` is an implementation of the Nexus client using the Globus SDK. It is not an official Globus product, but supported by one of the authors (Stephen Rosen) of the Globus SDK. - -In addition to the Python packages, one will need a Globus Confidential application, see [here](https://docs.globus.org/api/auth/developer-guide/) for details, that: - -* Includes the user-granted permissions ("scopes"): - - `openid` - - `profile` - - `email` - - `urn:globus:auth:scope:auth.globus.org:view_identity_set` - - `urn:globus:auth:scope:auth.globus.org:view_identities` - - `urn:globus:auth:scope:transfer.api.globus.org:all` - - `urn:globus:auth:scope:auth.globus.org:view_ssh_public_keys` -* Is allowed to use the Group scopes. This requires filing a ticket with Globus to get the app ID added to the system -* Has the correct redirection URLs (this depends on the website you are running) -* Requires the GlobusID as an identity provider -* Has a secret associated with the application - -For more details, please see the [Globus SDK documentation](http://globus-sdk-python.readthedocs.io/en/latest/tutorial/#step-1-get-a-client). - -## Configuration - -The configuration is a JSON file for ease of parsing it as a dictionary. The minimal configuration needed is: - -``` -{ - "users": { - "passwd_file": - "default_group": - }, - "groups": { - "group_file": - }, - "globus": { - "groups": { - "root_group": , - "root_group_uuid": - }, - "root_user": { - // root user should only have admin or manager roles in the groups - "roles": [ - "admin", - "manager" - ], - "username": , - "secret": , - "auth_refresh_token": , - "nexus_refresh_token": - }, - "user": { - // regular user may have any role in a group - "roles": [ - "member", - "admin", - "manager" - ] - }, - "app": { - "scopes": [ - "openid", - "profile", - "email", - "urn:globus:auth:scope:auth.globus.org:view_identities", - "urn:globus:auth:scope:transfer.api.globus.org:all", - "urn:globus:auth:scope:auth.globus.org:view_identity_set", - "urn:globus:auth:scope:nexus.api.globus.org:groups" - ], - "client_id": , - "client_secret": - } - }, - "connect_db": { - "db_file": - } -} -``` - -This JSON object will be parsed into a Python dictionary and will be passed to the various classes. - -## Execute code - -### Syncing User and Groups - `gosync_globus_auth.py` - -Executing `gosync_globus_auth.py` will sync the users and groups from Globus. If you want to run with you own config: `./gosync_globus_auth.py --config /path/to/config`. If you want to increase the verbosity, default the program will not print out anything to screen, simply add the `-v` flag. To increase the verbosity, just add more `v`s, i.e. `-vvvv`. - -### Adding Group - `add_connect_group.py` - -To add a group to Globus Groups requires, a file formatted as provided by the OSG Connect Website, details below. With the groups project file, a group is added through executing `./add_connect_group.py --projectfile /path/to/file --parent `. The `--parent ` is optional, but necessary to maintain the group tree structure and determine the correct group name in Globus. To pass your own configuration you will need to the `--config /path/to/new/config` option. There is also a verbosity flag, i.e. `-v`, `-vv`, and `-vvv`. - -## Globus Interface - `globus_db.py` - -The class `globus_db` is meant as an interface to the Globus ID and Globus Groups services. Please note that being able to `PUT` and `PUSH` information into Globus Groups is possible through the Nexus interface. Currently, only the `PUT` for Globus Groups is supported. - -The class requires a configuration dictionary and an `connect_db` object. The configuration dictionary is explained above. The `connect_db` is needed to retrieve the the refresh tokens for users and allow of to check for changes in group membership, i.e. if users were added or removed from a group. - -The class is split into four sections: client methods, group methods, group membership methods, and user methods. - -### Client Methods and some explanation about various Globus tokens - -The client methods are for getting the different types of clients needed to interact with GlobusID through Globus Auth and Globus Groups through Globus Nexus. There are two main clients used: Globus Auth client and Globus Nexus client. The Globus Auth client is for interacting with the user's GlobusID, i.e get the user profile in GlobusID. The Globus Nexus client is for interacting with a user's Globus Groups and their Globus Groups's profile. - -The clients can be authenticated using Globus Auth tokens or Globus Online Auth legacy tokens. Tokens can be thought of as randomly generated passwords that encode the user's identity and the application's permission level. Globus Online Auth legacy tokens will be referred to as legacy tokens from here on out. Legacy tokens should be avoided at all cost. They may not work down the road and are bad practice. - -Globus Auth tokens are OAuth2 tokens. OAuth2 gives the user and the authorization server the explicit power to reject or limit (either in time or scope) an application's access, to the user's information. It also moves large parts of the authentication process from the resource provider to an authentication provider, which allows for better separation between resources and authentication. For more details please visit [An introduction to OAuth2](https://www.digitalocean.com/community/tutorials/an-introduction-to-oauth-2). - -The Globus Auth tokens are split into three different types: Auth, Transfer, and Nexus. One will receive one, two, or all three, when a user authenticates against the app depending on an app's scopes, i.e. requested permissions. With the app created in the prerequisites one will receive all three tokens. Auth tokens are meant for retrieving a user's information from Globus ID, i.e. linked identities, SSH keys, etc. Transfer tokens are for initiating Globus transfers on behalf of the user. Nexus tokens are for authorizing against the Globus Groups service to allow viewing a user's group membership. The Nexus tokens do not allow to view a user's Globus groups profile through a call to the user's Globus Nexus profile, i.e. the group-specific custom fields and the user identity, directly. This due to the Nexus group scope not having the permissions to view the user's GlobusID. This can be circumvented by through accessing the profile through the groups interface instead. I know... Please note that Nexus tokens are special. They are not officially available, one has to request access to the "group" scopes from Globus. - -Globus Auth tokens expire after some time, usually within 10 minutes, i.e. you as the application only have a limited amount of time to retrieve the desired information out of Globus. To be able to repeatedly authenticate with Globus Auth, one can request "refresh tokens". These tokens are valid until the user revokes an app's permission. These are required for GOSync3. - -The individual methods are self-explanatory: - -* `get_tokens`: Returns a user's tokens from the `connect_db` or retrieves the tokens from the configuration file for the root user. -* `get_legacy_client`: Returns a user's Nexus client that has been authenticated using legacy Globus Online Auth token, i.e. the user's username/password. -* `get_globus_client`: Returns the user's Globus Auth and Globus Nexus client authenticated using the user's refresh tokens. - -### Group Methods - -#### Retrieving Group Infomation - `GET` Methods -There are two ways to access all groups associated with `connect`. The first is to the retrieve the group tree for the root group, named `connect`. The second is to retrieve all groups associated with the root user, also named `connect`. - -In GOSyn3, we are using the second method. There is a method to retrieve the group tree, but it is unused at the moment. The `get_group` and `get_groups` method simply filter the `all_groups` list for the desired group(s). `get_all_groups` is the method that uses the root user's credentials to determine groups in which the root user, `connect`, is an `admin` or `manager`. The distinction between between being `admin`, `manager`, or `member` is important here. It filters the groups that are returned by Globus Nexus. The `connect` user will always be an `admin` or `manager` in a subgroup, while a user might be a `admin`, `manager`, or `member`. The root user is a `member` of some groups that are not associated with the Connect instances. - -In addition there is a `check_new_members` methods at is currently used. It allows to filter the group list to those groups that have recently added members. - -#### Adding Group Information - `POST` Methods - -GOSync3 has the ability to add groups to Globus Groups. This is done through the `add_group` and `parse_project_file` methods. Adding a group is done through the `add_groups` method. It calls the `globus-nexus-client`'s `create_group` method to create the group with name and description provided through the project description text file, details on this below. Optionally, one can pass a parent group to the method. It is strongly recommended to provide a parent group, without a group the group will be assumed to be a top-level group below the root `connect` group. - -The project description text file should follow the format of the form on the [New Project section on the OSG Connect website](http://osgconnect.net/newproject). This will provide a text file of the following format: - -``` -Your Name: -Your Email Address: -Project Name: -Short Project Name: -Field of Science: Evolutionary -Field of Science (if Other): -PI Name: -PI Email: -PI Organization: -PI Department: -Join Date: -Sponsor: -OSG Sponsor Contact: -Project Contact: -Project Contact Email: -Telephone Number: -Project Description: -``` - -From this the only required field is the "Short Project Name". The value will be used as the group name in Globus Groups. - -`parse_project_file` parses the project file, determines the expected name of the group, and converts the plain text to HTML-formatted text. The project name is determined from the "Short Project Name" in the project file and the parent group. The format of the Globus Groups name is `.`. - -### Group Membership Methods - -The work flow for retrieving a user's group membership depends on the authentication method used. In a purely Globus Auth-based workflow, one would retrieve a user's group membership by using their tokens and calling `list_groups` method from the Nexus client. This is done in the function `get_user_groups_auth`. - -At the moment, we only have tokens for the `connect` user. To work around this, GOSync3 tries to generate a mapping of group to users first and then inverts that mapping, see function `get_user_groups_no_tokens`. It retrieves the list of all groups associated with the `connect` user and then determines the group members for every group. Using `_invert_dict_list_values`, the group-to-users mapping is then inverted to the user-to-groups mapping. - -`get_group_members` simply returns the users for a given group. This has to be done using the group's Globus UUID. A mapping of Globus group name to Globus group UUID is provided by the `connect_db`. - -### User Methods - -The user methods allow the user to retrieve more information about user, i.e. query Globus for the user's "user information" and manipulate the Globus output in a more easily digestible patterns. - -`get_user_info` is a specialization of `get_user_groups_profile`. It allows to fetch the a user's profile, i.e. username, SSH key, group-specific information, full name, e-mail, through Globus Groups. `get_user_info` is specialization in the sense that it uses the root group user profile rather than specific group's profile as needed by `get_user_groups_profile`. - -`get_all_users` retrieves all the users in the root group and then queries Globus Groups for the user's profile. `_invert_dict_list_values` allows you to invert the group to users mapping to a user to groups mapping. - -## Puppet/Hiera Interface - `connect_db.py` - -In this case, the JSON file used by puppet/hiera is used as a user database. This is suboptimal. It will allow us to quickly deploy GOSync3. The information source for the puppet/hiera JSON file can be replaced by a real DB later on. Some of the information, i.e. UNIX ids, stored in the JSON file will be needed to populate a replacement database. - -The puppet/hiera interface, i.e `connect_db`, is a thin layer of the JSON object that puppet/hiera uses to provision user accounts. It reads a previous version of the JSON object, and produces a `users` and `groups` dictionary and a `uids` and `gids` list. These four objects contain all the necessary information to be able add new groups and users to the JSON object passed to puppet/hiera. - -The `users` and `groups` dictionaries are made up of sub-dictionaries. Holding the information for each user and group, respectively. The `users` dictionary is a mapping of username to user information, such that: - -``` -{ - "auth_refresh_token": # user's Globus Auth refresh token - "comment": # user's name - "email": # user's emails - "gid": # default group for passwd file - "manage_group": # puppet/hiera config parameter - "nexus_refresh_token": # user's Globus Nexus refresh token - "shell": # default user shell - "ssh_keys": # SSH key dictionary, explained below - "uid": # user's UNIX id - "groups": # list of user's groups, - "connect_project": # Initial connect project, typically osg.ConnectTrain - "condor_schedd": # The condor schedd to pick on the login host -} -``` - -The `groups` dictionary follows a similar pattern. Mapping a group name to: - -``` -{ - "gid": # group UNIX ID - "num_members": # Number if active user according to Globus - "globus_uuid": # Groups Globus UUID -} -``` - -Some of the methods in this class are self-explanatory: - -* `add_group`: Add a new group to the `groups` dictionary -* `add_user`: Add a new user to the `users` dictionary -* `get_user`: Retrieve the user information by username -* `get_group`: Retrieve the group by group name -* `new_unix_id`: This will generate a new UNIX id by incrementing the maximum ID or setting it to 100000 for both groups and users -* `get_member_count`: Retrieve the group's active member count -* `get_auth_token`: Retrieve user's Globus Auth refresh token -* `get_nexus_token`: Retrieve user's Globus Nexus refresh token -* `get_globus_tokens`: Retrieves user's Globus Auth and Nexus refresh tokens -* `remove_unicode`: Remove unicode characters from a user's name. This can cause problems when generating a passwd file or trying to serialize a JSON file. -* `commit_old_version`: In the spirit of old GOSync, we commit the JSON file to Gitlab, so puppet/hiera can grab it from there -* `write_db`: Write the JSON object out. If `email_file` is supplied in the config, it will also create a directory with json files that maps the group users to their email addresses. Similarly, if `mailchimp_file` is supplied, it will also create a directory with json files that maps the group users to their information needed for mailchimp. -* `set_user_nologin`: Set a user's shell to nologin, used in case they are no longer "active" in a Globus group -* `get_emails`: Get email for everyone or optionally for a given group -* `get_email`: Get email for a specific user -* `get_mailchimp_info`: Gather information about users for mailchimp. Mapping users to their, first name, last name, and email address. - -The `get_default_project` method is tries to guess a user's first OSG project for account reasons. If the user is a member of more than more than one sub-project we need to filter out any of the default ones. First, "osg.ConnectTrain" is removed. If there are still more than one projects, we filter out any project associated with a user school and any OSG project, if the user is a member of the other connect instances, i.e. SPT, ATLAS, CMS, and Duke. - -The `decompose_sshkey` method is necessary because of the format that puppet/hiera wants the SSH key in. A typical SSH key is formatted as follows: - -``` - - -``` - -where the `` is the type of SSH key, i.e. ssh-rsa, `` is the actual key portion of an SSH key, and `` is an optional identifier that is usually `@` of the machine the key pair was generated on. - -Puppet/hiera wants the key in this JSON object: - -``` -{ - "": - { - "type": # Encryption type, i.e. ssh-rsa, ssh-dsa - "key": # Key part - } -} -``` - -This requires to split the key according to the spaces it in. Unfortunately, not all keys have the identifier. In those cases it is replaced with the user's email address. This will not affect operations. There is a question though about overriding SSH keys. - -## Future Plans - -### Work flow Improvements - -There are three slow processes in the current work flow: - -* Getting the group summary -* Getting the user summary -* Getting the user group memberships - -The first processes are slow because we have to query the Globus Groups database separately each piece of the group information: group profile, number of members, and group members, respectively. In addition, queries on the Globus side slow down as the group tree grows in size and as we add more groups the more queries we have to perform. Getting the user summary is a similarly expensive process because we have to query Globus for every user twice, once to get their general information, and another to get their SSH key. - -One of the main steps to improve the efficiency requires to change the website to use OAuth2. This would allow us to operate on a per-user basis rather than on a per-group basis. In an idealized work flow, a user would sign up on the website. This sign up process would trigger the ability to retrieve the users identity (including their SSH key)and their Globus OAuth tokens, see below for more details on Globus OAuth tokens. With the identity and Globus token in hand, we can then query Globus for just the new user's group memberships. The first query would happen be default at sign up, while the group membership query would come after they are approved. The second query may have to be repeated several time. This is not wasted effort though, since we are waiting for human intervention. Similarly, we could trigger a Globus query of a given user's profile once they login. This would make updating the user information on our end dependent on user actions rather than us having to repeatedly query Globus for their information. Given that we most likely will never have tokens for all users, we will need operate in a hybrid mode, where the new user's are handled solely through the tokens, while older users will have still have to be kept up to date through the above described work flow. - -### `PUT` and `PUSH` Methods for Globus Interface - -To create groups in Globus Groups, we will need to implement `PUSH` methods. The `NexusClient` already has these, but they are untested. I will need to test them before I can sanction them. I also want to standardize all information that is stored in Globus Groups for all the groups. Currently there are several different formats. - -### Moving connect DB to a real DB - -The data volume is not that large and the JSON file is sufficient to store all the information. Down the road, we might want split from Globus and at this point we need to retrieve all the data from Globus. Storing this data would need a database. - -### Multiple Connect Globus Apps - -Branded websites in Globus are mapped one-to-one to a specific Globus App. To have different branded website for the various connect instances, we would need multiple Globus Apps. Tokens are app-specific, so for users that are members of multiple connect instances, for example CI Connect and OSG Connect, we will need to keep track of different refresh tokens and apps that they are associated with. - - - - - - diff --git a/admin/connectbook high level.graffle b/admin/connectbook high level.graffle deleted file mode 100644 index a21ab582..00000000 --- a/admin/connectbook high level.graffle +++ /dev/null @@ -1,1102 +0,0 @@ - - - - - ActiveLayerIndex - 0 - ApplicationVersion - - com.omnigroup.OmniGrafflePro - 139.18.0.187838 - - AutoAdjust - - BackgroundGraphic - - Bounds - {{0, 0}, {756, 553.00002670288086}} - Class - SolidGraphic - ID - 2 - Style - - shadow - - Draws - NO - - stroke - - Draws - NO - - - - BaseZoom - 0 - CanvasOrigin - {0, 0} - ColumnAlign - 1 - ColumnSpacing - 36 - CreationDate - 2015-06-08 17:53:11 +0000 - Creator - dgc - DisplayScale - 1 0/72 in = 1.0000 in - ExportShapes - - - InspectorGroup - 255 - ShapeImageRect - {{2, 2}, {22, 22}} - ShapeName - 5E7BC683-58AD-47BE-A289-900DC9FBE24C-84496-000206453406DB7D - ShouldExport - YES - StrokePath - - elements - - - element - MOVETO - point - {0.0071754499999999999, -0.5} - - - element - LINETO - point - {0.0070476499999999999, -0.5} - - - control1 - {-0.12950300000000001, -0.5} - control2 - {-0.258102, -0.44202200000000003} - element - CURVETO - point - {-0.35053600000000001, -0.34291100000000002} - - - element - LINETO - point - {-0.49962400000000001, -0.43038700000000002} - - - element - LINETO - point - {-0.5, 0.122889} - - - element - LINETO - point - {-0.027294200000000001, -0.15340200000000001} - - - element - LINETO - point - {-0.18032999999999999, -0.24310100000000001} - - - control1 - {-0.12737499999999999, -0.28518900000000003} - control2 - {-0.061824799999999999, -0.30928} - element - CURVETO - point - {0.0071124999999999999, -0.30928} - - - element - LINETO - point - {0.0071754499999999999, -0.30928} - - - control1 - {0.17525099999999999, -0.30928} - control2 - {0.31199300000000002, -0.17050199999999999} - element - CURVETO - point - {0.31199300000000002, 6.2942500000000003e-05} - - - control1 - {0.31199300000000002, 0.170567} - control2 - {0.17525099999999999, 0.309284} - element - CURVETO - point - {0.0071754499999999999, 0.309284} - - - control1 - {-0.088207199999999999, 0.309284} - control2 - {-0.176317, 0.265289} - element - CURVETO - point - {-0.23466300000000001, 0.18849399999999999} - - - control1 - {-0.26631199999999999, 0.14691499999999999} - control2 - {-0.325405, 0.13903399999999999} - element - CURVETO - point - {-0.36651600000000001, 0.171267} - - - control1 - {-0.40762900000000002, 0.203371} - control2 - {-0.41527399999999998, 0.26325599999999999} - element - CURVETO - point - {-0.38356200000000001, 0.30496000000000001} - - - control1 - {-0.28937099999999999, 0.42892599999999997} - control2 - {-0.146927, 0.5} - element - CURVETO - point - {0.0071754499999999999, 0.5} - - - control1 - {0.27890599999999999, 0.5} - control2 - {0.5, 0.27571699999999999} - element - CURVETO - point - {0.5, 6.2942500000000003e-05} - - - control1 - {0.5, -0.27571499999999999} - control2 - {0.27890599999999999, -0.5} - element - CURVETO - point - {0.0071754499999999999, -0.5} - - - element - CLOSE - - - element - MOVETO - point - {0.0071754499999999999, -0.5} - - - - TextBounds - {{0, 0}, {1, 1}} - - - InspectorGroup - 255 - ShapeImageRect - {{2, 2}, {22, 22}} - ShapeName - 8B8844FF-D926-4D08-972B-50D9BAEB7C28-12200-0000ED5871BBCBFE - ShouldExport - YES - StrokePath - - elements - - - element - MOVETO - point - {-0.5, 0.5} - - - control1 - {-0.5, 0.5} - control2 - {-0.48802200000000001, 0.40712100000000001} - element - CURVETO - point - {-0.47006199999999998, 0.376162} - - - control1 - {-0.452096, 0.34520499999999998} - control2 - {-0.29417599999999999, 0.28667700000000002} - element - CURVETO - point - {-0.23262099999999999, 0.29577199999999998} - - - control1 - {-0.171068, 0.304865} - control2 - {-0.12709999999999999, 0.19575000000000001} - element - CURVETO - point - {-0.12709999999999999, 0.19575000000000001} - - - element - LINETO - point - {-0.14468800000000001, 0.177564} - - - control1 - {-0.14468800000000001, 0.177564} - control2 - {-0.13589699999999999, 0.18665799999999999} - element - CURVETO - point - {-0.171068, 0.19575000000000001} - - - control1 - {-0.19745099999999999, 0.19575000000000001} - control2 - {-0.223827, 0.18665799999999999} - element - CURVETO - point - {-0.223827, 0.18665799999999999} - - - control1 - {-0.223827, 0.18665799999999999} - control2 - {-0.17986199999999999, 0.15937799999999999} - element - CURVETO - point - {-0.17986199999999999, 0.113916} - - - control1 - {-0.19744700000000001, 0.12300700000000001} - control2 - {-0.17986199999999999, 0.1321} - element - CURVETO - point - {-0.25020999999999999, 0.1321} - - - control1 - {-0.26779399999999998, 0.1321} - control2 - {-0.276588, 0.12300700000000001} - element - CURVETO - point - {-0.276588, 0.12300700000000001} - - - element - LINETO - point - {-0.18865599999999999, 0.059356899999999997} - - - control1 - {-0.18865599999999999, 0.059356899999999997} - control2 - {-0.276588, 0.059355999999999999} - element - CURVETO - point - {-0.25020999999999999, 0.041170400000000003} - - - control1 - {-0.223827, 0.022983400000000001} - control2 - {-0.215032, 0.0047991300000000004} - element - CURVETO - point - {-0.215032, 0.0047991300000000004} - - - element - LINETO - point - {-0.215032, -0.013386800000000001} - - - element - LINETO - point - {-0.20624400000000001, -0.067944400000000002} - - - control1 - {-0.20624400000000001, -0.067944400000000002} - control2 - {-0.215032, -0.113409} - element - CURVETO - point - {-0.19745099999999999, -0.17705899999999999} - - - control1 - {-0.17986199999999999, -0.195245} - control2 - {-0.215032, -0.37710100000000002} - element - CURVETO - point - {-0.0743399, -0.46803099999999997} - - - control1 - {0.066359000000000001, -0.55896000000000001} - control2 - {0.21584700000000001, -0.43165999999999999} - element - CURVETO - point - {0.242232, -0.38619500000000001} - - - control1 - {0.25981399999999999, -0.34072400000000003} - control2 - {0.27739799999999998, -0.29526000000000002} - element - CURVETO - point - {0.27740399999999998, -0.25889099999999998} - - - control1 - {0.27740399999999998, -0.222523} - control2 - {0.26860699999999998, -0.149779} - element - CURVETO - point - {0.26860699999999998, -0.149779} - - - element - LINETO - point - {0.27739799999999998, -0.040666000000000001} - - - control1 - {0.27739799999999998, -0.040666000000000001} - control2 - {0.28619499999999998, -0.022481000000000001} - element - CURVETO - point - {0.29498600000000003, -0.0042953499999999999} - - - control1 - {0.30378100000000002, 0.0138911} - control2 - {0.27740399999999998, 0.013892399999999999} - element - CURVETO - point - {0.27740399999999998, 0.013892399999999999} - - - element - LINETO - point - {0.25981399999999999, 0.0047969800000000002} - - - element - LINETO - point - {0.26860699999999998, 0.0502627} - - - element - LINETO - point - {0.25102200000000002, 0.059355999999999999} - - - element - LINETO - point - {0.25102200000000002, 0.077542899999999998} - - - control1 - {0.25102200000000002, 0.077542899999999998} - control2 - {0.22464300000000001, 0.095728199999999999} - element - CURVETO - point - {0.22464300000000001, 0.1321} - - - control1 - {0.22464300000000001, 0.16847100000000001} - control2 - {0.20705599999999999, 0.23212099999999999} - element - CURVETO - point - {0.18946499999999999, 0.22302900000000001} - - - control1 - {0.171879, 0.21393400000000001} - control2 - {0.26860699999999998, 0.28667700000000002} - element - CURVETO - point - {0.26860699999999998, 0.28667700000000002} - - - control1 - {0.26860699999999998, 0.28667700000000002} - control2 - {0.30378100000000002, 0.277586} - element - CURVETO - point - {0.32136399999999998, 0.28667700000000002} - - - control1 - {0.33895199999999998, 0.29577199999999998} - control2 - {0.43413499999999999, 0.33282} - element - CURVETO - point - {0.46407500000000002, 0.35758699999999999} - - - control1 - {0.49401400000000001, 0.38235400000000003} - control2 - {0.5, 0.49898599999999999} - element - CURVETO - point - {0.5, 0.49898599999999999} - - - element - CLOSE - - - element - MOVETO - point - {-0.5, 0.5} - - - - TextBounds - {{0, 0}, {1, 1}} - - - GraphDocumentVersion - 8 - GraphicsList - - - Bounds - {{171, 373.5}, {90, 45}} - Class - ShapedGraphic - ID - 808 - Rotation - 90 - Shape - AdjustableArrow - ShapeData - - width - 18 - - Style - - shadow - - Draws - NO - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1347\cocoasubrtf570 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GothamBook;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs18 \cf0 API push} - - - - Bounds - {{9, 441}, {423, 90}} - Class - ShapedGraphic - ID - 807 - Shape - Rectangle - Style - - shadow - - Draws - NO - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1347\cocoasubrtf570 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GothamBook;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs36 \cf0 support.opensciencegrid.org -\fs18 \ - -\fs24 Freshdesk\ -knowledgebase} - - TextPlacement - 0 - - - Bounds - {{27, 288}, {333, 72}} - Class - ShapedGraphic - ID - 806 - Shape - Rectangle - Style - - shadow - - Draws - NO - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1347\cocoasubrtf570 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GothamBook;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs36 \cf0 cron\ - -\fs20 \ -periodically converts git clone to html (also using makomate's markdown handler) and pushes to Freshdesk via API} - - TextPlacement - 0 - - - Bounds - {{288, 81}, {108, 108}} - Class - ShapedGraphic - ID - 804 - Shape - Rectangle - Style - - shadow - - Draws - NO - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1347\cocoasubrtf570 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GothamBook;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs18 \cf0 clone of connect book + tutorial subrepos} - - - - Bounds - {{27, 63}, {270, 198}} - Class - ShapedGraphic - ID - 805 - Shape - Rectangle - Style - - shadow - - Draws - NO - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1347\cocoasubrtf570 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GothamBook;\f1\fnil\fcharset0 LucidaGrande;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs28 \cf0 Apache\ - -\fs18 with makomate input filter\ -\ -\ -Makomate is the templating engine that builds the current site. It can read markdown files directly and present them as HTML in a template page.\ -\ -Does not require static conversion; renders in real time. Content is as current as the git clone is. -\f1 \uc0\u8594 -\f0 \ -\ -\ -[osg connect header]\ -[markdown to html]\ -[osg connect footer]} - - TextPlacement - 0 - - - Bounds - {{9, 9}, {369, 369}} - Class - ShapedGraphic - ID - 803 - Shape - Rectangle - Style - - shadow - - Draws - NO - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1347\cocoasubrtf570 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GothamBook;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs48 \cf0 osgconnect.net} - - TextPlacement - 0 - - - Bounds - {{405, 90}, {90, 90}} - Class - ShapedGraphic - ID - 802 - Shape - 5E7BC683-58AD-47BE-A289-900DC9FBE24C-84496-000206453406DB7D - Style - - shadow - - Draws - NO - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1347\cocoasubrtf570 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GothamBook;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs18 \cf0 push (git hook) or pull (cron)} - - - - Bounds - {{563.30379894941325, 462.69619857518728}, {52.522319793701172, 63.129920959472656}} - Class - ShapedGraphic - ID - 801 - Rotation - 270 - Shape - AdjustableArrow - ShapeData - - width - 15 - - Style - - shadow - - Draws - NO - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1347\cocoasubrtf570 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GothamBook;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs18 \cf0 push update to github} - - - - Bounds - {{639, 486}, {81, 54}} - Class - ShapedGraphic - ID - 800 - Shape - 8B8844FF-D926-4D08-972B-50D9BAEB7C28-12200-0000ED5871BBCBFE - Style - - shadow - - Draws - NO - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1347\cocoasubrtf570 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GothamBook;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs18 \cf0 Documentation developer} - - - - Bounds - {{504, 45}, {216, 405}} - Class - ShapedGraphic - ID - 799 - Shape - Rectangle - Style - - shadow - - Draws - NO - - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1347\cocoasubrtf570 -\cocoascreenfonts1{\fonttbl\f0\fnil\fcharset0 GothamBook;\f1\fmodern\fcharset0 Courier;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs48 \cf0 Github -\fs18 \ - -\fs28 OSGConnect organization\ -\ -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f1\fs20 \cf0 \ul \ulc0 connectbook, containing:\ -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc -\cf0 \ulnone tutorial-R\ -tutorial-ScalingUp-R\ -tutorial-VinaAutodock\ -tutorial-blast\ -tutorial-cp2k\ -tutorial-dagman-namd\ -tutorial-error101\ -tutorial-exitcode\ -tutorial-htcondor-transfer\ -tutorial-namd\ -tutorial-nelle-nemo\ -tutorial-oasis-parrot\ -tutorial-octave\ -tutorial-pegasus\ -tutorial-pegasus-namd\ -tutorial-pegasus-vina\ -tutorial-photodemo\ -tutorial-quickstart\ -tutorial-root\ -tutorial-scaling\ -tutorial-scaling-up-resources\ -tutorial-software\ -tutorial-stash-chirp\ -tutorial-stash-http\ -tutorial-stash-namd\ -tutorial-swift\ -} - - TextPlacement - 0 - - - GridInfo - - MajorGridColor - - a - 0.3 - b - 0.75 - g - 0.75 - r - 0.75 - - MinorGridColor - - a - 0.3 - w - 0.821168 - - ShowsGrid - YES - SnapsToGrid - YES - - GuidesLocked - NO - GuidesVisible - YES - HPages - 1 - ImageCounter - 1 - KeepToScale - - Layers - - - Lock - NO - Name - Layer 1 - Print - YES - View - YES - - - LayoutInfo - - Animate - NO - circoMinDist - 18 - circoSeparation - 0.0 - layoutEngine - dot - neatoSeparation - 0.0 - twopiSeparation - 0.0 - - LinksVisible - NO - MagnetsVisible - NO - MasterSheets - - ModificationDate - 2015-06-08 18:11:18 +0000 - Modifier - dgc - NotesVisible - NO - Orientation - 2 - OriginVisible - NO - PageBreaks - YES - PrintInfo - - NSBottomMargin - - float - 41 - - NSHorizonalPagination - - coded - BAtzdHJlYW10eXBlZIHoA4QBQISEhAhOU051bWJlcgCEhAdOU1ZhbHVlAISECE5TT2JqZWN0AIWEASqEhAFxlwCG - - NSLeftMargin - - float - 18 - - NSOrientation - - coded - BAtzdHJlYW10eXBlZIHoA4QBQISEhAhOU051bWJlcgCEhAdOU1ZhbHVlAISECE5TT2JqZWN0AIWEASqEhAFxlwGG - - NSPaperSize - - size - {792, 612.00002670288086} - - NSPrintReverseOrientation - - int - 0 - - NSRightMargin - - float - 18 - - NSTopMargin - - float - 18 - - - PrintOnePage - - ReadOnly - NO - RowAlign - 1 - RowSpacing - 36 - SheetTitle - Canvas 1 - SmartAlignmentGuidesActive - YES - SmartDistanceGuidesActive - YES - UniqueID - 1 - UseEntirePage - - UserInfo - - kMDItemAuthors - - David Champion <dgc@uchicago.edu> - - kMDItemCopyright - University of Chicago - kMDItemOrganizations - - IT Services, University of Chicago - - - VPages - 1 - WindowInfo - - CurrentSheet - 0 - ExpandedCanvases - - - name - Canvas 1 - - - Frame - {{242, 204}, {1295, 973}} - ListView - - OutlineWidth - 142 - RightSidebar - - ShowRuler - - Sidebar - - SidebarWidth - 147 - VisibleRegion - {{0, 0}, {755.33333333333337, 554}} - Zoom - 1.5 - ZoomValues - - - Canvas 1 - 1.5 - 1 - - - - - diff --git a/admin/connectbook-high-level.png b/admin/connectbook-high-level.png deleted file mode 100644 index 458916aa..00000000 Binary files a/admin/connectbook-high-level.png and /dev/null differ diff --git a/admin/content/support-add-non-tutorial.md b/admin/content/support-add-non-tutorial.md deleted file mode 100644 index 77cde32c..00000000 --- a/admin/content/support-add-non-tutorial.md +++ /dev/null @@ -1,125 +0,0 @@ -[title]: - "Add a topic to the knowledge base" - -[TOC] - -# General - -The Connect documentation is kept in the [ConnectBook repository -in GitHub][connectbook]. Most documents are in Markdown format, -and are published to the [support site][support] when changed. - -[connectbook]: https://github.com/OSGConnect/connectbook/tree/master/ -[support]: https://support.opensciencegrid.org/ - -To add a new document to the Knowledge Base, several steps must be -taken: - -1. *create the document* and add it to the `connectbook` repo -2. *create a template* article in Freshdesk -3. *map* the new file in the `freshpush.ini` file. - -For discussion's sake, let's say we want to create an article named -"Contacting us during a natural disaster." We'll explain each of the -above steps in detail below. This guide assumes that you're using the -GitHub web UI for all changes. (You can alternatively use the git -client, but you'll need to translate directions accordingly.) - -# Create the document - -Begin by creating a new file in GitHub. For our example, we might name -it `contacting-disaster.md`. The file name is not significant. - -The file might begin as follows: - - [title]: - "Contacting us during a natural disaster" - - [TOC] - - # What events are natural disasters? - -The `[title]` line sets the article's title. If you don't have one of -these, then the first headline will be used instead. That's not usually -what you want. - -Continue editing until your document is as you want it. Remember that -it's a Markdown document, and its filename should end in `.md`. When -you're done, save/commit the new file. - - -# Create a template article - -To publish our Markdown content into Freshdesk, we need an article -to send it to. Unfortunately there's no way (at present) to have -`freshpush` just make one. We need to make one directly in the -Freshdesk Agent environment, then record its article number. - -To begin, go to the category that you want your new article to -appear in. Let's say you want to add an article in the -Additional Support & Training -> Education and Training -category. - -Click the folder, then click "add solution": - -![add solution screenshot](https://raw.githubusercontent.com/OSGConnect/connectbook/master/admin/add-solution-2.jpg) - -Give the article any title at all, it doesn't matter. Don't add any -content either. Just save it, and note the number that appears in -the URL bar. For example, I just got this: - -http://support.opensciencegrid.org/solution/articles/5000641902-hfghf - -That `5000641902` is the article number. Save it for the next step. -Leave the article. - - -# Map the file in freshpush.ini - -There's still one more piece to put in place, though. For `freshpush` -to be able to update a document in Freshdesk, it needs to know the -article number for each document it should care about. We give it that -through a mapping table that's kept in the [freshpush.ini][freshpush-ini] -file. - -[freshpush-ini]: https://github.com/OSGConnect/connectbook/blob/master/update/freshpush.ini - -To make your new tutorial active, you'll need to add a line to the -`[filemap]` section of this file. Just click over there and edit the -file in GitHub, then commit -- it's simple enough, there's no need to -get fancy. Just add another row mapping `tutorial-folding/README.md` -to the article number for the new article. Above, we pretended it was -5000641902, so it would look like this: - - [filemap] - ... - client-using.md = 5000621976 - client-whatis.md = 5000640327 - communicate.md = 5000632439 - contact-information.md = 5000634383 - ... - - -Commit this change. It will be sent immediately to the server that -handles publishing from GitHub to Freshdesk. You should see the update -within a minute. - - -# Test - -That's all there is. The next change committed to -`tutorial-folding/README.md` should be published to Freshdesk -straightaway, so commit a change to the file and test it out! - - -# Images and "attachments" - -If you need images or other "attachments" in your document, just make -them part of the `connectbook` repository. You can put them into a -subdirectory or not. Your Markdown content will refer to the file's -location using a specially constructed URL. For example, the URL to -the screenshot above is: - - https://raw.githubusercontent.com/OSGConnect/connectbook/master/admin/add-solution-2.jpg - -Everything _up to and including_ `admin` is the special construction. -The folder and file names following `admin` are relative to the -`connectbook` repository root. diff --git a/admin/content/support-add-tutorial.md b/admin/content/support-add-tutorial.md deleted file mode 100644 index d3b2f04d..00000000 --- a/admin/content/support-add-tutorial.md +++ /dev/null @@ -1,157 +0,0 @@ -[title]: - "Add a tutorial to the knowledge base" - -[TOC] - -# General - -Tutorials are separate repositories in GitHub, so that each can be -cloned independently on demand. (This is the main thing that the -`tutorial` command does when you use it to set up a tutorial.) Each -tutorial has, at minimum, a `README.md` file that describes the -tutorial, the steps to be taken, and the lessons learned. - -To add this tutorial file to the Knowledge Base, several steps -must be taken: - -1. add the tutorial as a *submodule* to the `connectbook` repo -2. add a *webhook* to the repository on GitHub -3. *create a template* article in Freshdesk -4. *map* the README.md file in the `freshpush.ini` file. - -For discussion's sake, let's say we've already created a tutorial named -`tutorial-folding`, that it has a `README.md` file and all its requisite -materials, and that it works with the `tutorial` command already. We'll -explain each of the above steps in detail below. - -# Add the tutorial as a submodule - -Since it's a tutorial, `tutorial-folding` exists as a completely -separate repository in GitHub. This makes sense: changes to one -tutorial rarely affect others, and users are usually not interested in -_all_ tutorials -- just a handful. But for documentation's sake we -want to track all tutorials with the `connectbook`, which serves as an -"umbrella" for all the Help Desk documentation. We do that by making -each tutorial a _submodule_, which means that the `connectbook` repo has -a reference to the tutorial repo, and can incorporate changes by pulling -from the submodule. Each of the tutorials is incorporated by reference -like this. [You can see how it looks in GitHub][ghtutorials]. - -[ghtutorials]: https://github.com/OSGConnect/connectbook/tree/master/tutorials - -Adding a new submodule is easy, but it must be done from the git -client -- it can't be done in the GitHub web UI. Let's say that -our new tutorial is hosted on GitHub at -https://github.com/OSGConnect/tutorial-folding : - - $ git clone --recursive https://github.com/OSGConnect/connectbook - $ cd connectbook/tutorials - $ git submodule add https://github.com/OSGConnect/tutorial-folding - $ git commit .gitmodules tutorial-folding - $ git push - -That's it! The new tutorial is now part of `connectbook`, incorporated -by reference. Future pulls and clones will copy it alongside the other -tutorials. - - -# Add a webhook - -We've established `tutorial-folding` as a part of the overall -connectbook repository, but what happens when someone pushes changes -to the tutorial? Those changes still belong to the `tutorial-folding` -repo, and that repo doesn't know about `connectbook`. We need to create -some trigger so that when changes arrive for `tutorial-folding`, they -cause the connectbook material to be updated on the Help Desk. - -That trigger is called a _webhook_, and we set it up in GitHub. All -our webhooks look the same! The `connectbook` repo has one, and -each tutorial repo has one too, and each of them is identical. -You can load the webhook settings for `tutorial-folding` here: -https://github.com/OSGConnect/tutorial-folding/settings/hooks - -Here's what that screen looks like for `tutorial-R`: - -![Webhook Setup](https://raw.githubusercontent.com/OSGConnect/connectbook/master/admin/webhook.jpg) - -To add a webhook for `tutorial-folding`, click *Add Webhook*. In the -new screen, paste https://ci-connect.net/github.wsgi into the *Payload -URL* input. Ensure that the *content type* is _application/json_. No -*secret* is needed. Finally, for the "which events" radio, select -*just the push event*. Then click *Add webhook* to save your changes. - -Now, whenever a change to this repo is received by GitHub, it will -notify https://ci-connect.net/github.wsgi and an update to the Help Desk -will be triggered (using the `freshpush` program). - - -# Create a template article - -To publish our Markdown content into Freshdesk, we need an article -to send it to. Unfortunately there's no way (at present) to have -`freshpush` just make one. We need to make one directly in the -Freshdesk Agent environment, then record its article number. - -To begin, go to the category that you want your new article to -appear in. For a tutorial, this is likely the [HTC on OSG][htccategory] -category. - -[htccategory]: http://support.opensciencegrid.org/solution/categories/5000161171 - -Choose a folder, then click "add solution": - -![add solution screenshot](https://raw.githubusercontent.com/OSGConnect/connectbook/master/admin/add-solution.jpg) - -Give the article any title at all, it doesn't matter. Don't add any -content either. Just save it, and note the number that appears in -the URL bar. For example, I just got this: - -http://support.opensciencegrid.org/solution/articles/5000641902-hfghf - -That `5000641902` is the article number. Save it for the next step. -Leave the article. - - -# Map the file in freshpush.ini - -There's still one more piece to put in place, though. For `freshpush` -to be able to update a document in Freshdesk, it needs to know the -article number for each document it should care about. We give it that -through a mapping table that's kept in the [freshpush.ini][freshpush-ini] -file. - -[freshpush-ini]: https://github.com/OSGConnect/connectbook/blob/master/update/freshpush.ini - -To make your new tutorial active, you'll need to add a line to -the `[filemap]` section of this file. Just click over there and -edit the file in GitHub, then commit -- it's simple enough, there's -no need to get fancy. - - [filemap] - ... - tutorial-R/README.md = 5000634364 - tutorial-ScalingUp-R/README.md = 5000639781 - tutorial-VinaAutodock/README.md = 5000634379 - ... - -That's a snippet of the `[filemap]` section. Just add another row -mapping `tutorial-folding/README.md` to the article number for the new -tutorial. Above, we pretended it was 5000641902, so it would look -like this: - - [filemap] - ... - tutorial-R/README.md = 5000634364 - tutorial-ScalingUp-R/README.md = 5000639781 - tutorial-VinaAutodock/README.md = 5000634379 - ... - tutorial-folding/README.md = 5000641902 - -Commit this change. It will be sent immediately to the server that -handles publishing from GitHub to Freshdesk. - - -# Test - -That's all there is. The next change committed to -`tutorial-folding/README.md` should be published to Freshdesk -straightaway, so commit a change to the file and test it out! diff --git a/admin/content/support-editing-tutorial.md b/admin/content/support-editing-tutorial.md deleted file mode 100644 index ed542e66..00000000 --- a/admin/content/support-editing-tutorial.md +++ /dev/null @@ -1,31 +0,0 @@ -[title]: - "Editing a tutorial in the knowledge base" - -[TOC] - -Because tutorials are separate GitHub repositories that are included in the connectbook repo as submodules, the correct workflow for editing/updating a tutorial is to edit the main (upstream) repository and then pull those changes into the connectbook tutorial submodule. For example, if we are editing the “quickstart” tutorial, proceed as follows: - -1) Clone the main/upstream tutorial repository locally. - - $ git clone https://github.com/OSGConnect/tutorial-quickstart.git - $ cd tutorial-quickstart - -Make your changes, commit them, and then push those changes to GitHub. - - $ git push origin master - -2) Next, update the tutorial-quickstart submodule in the connectbook repo. Start by first cloning the connectbook repository locally (recursively to include submodules). Make sure not to clone it inside the tutorial-quickstart repo from step 1. - - $ cd ~ - $ git clone --recursive https://github.com/OSGConnect/connectbook - -Then, update the submodule from the upstream remote repo. Commit the update and push to GitHub. - - $ cd connectbook - $ git submodule update --remote tutorials/tutorial-quickstart - $ git add tutorials/tutorial-quickstart - $ git commit -m “Pulling in changes from upstream OSGConnect/tutorial-quckstart repo” - $ git push origin master - -That's it. Now check the tutorial article in Freshdesk to make sure that any updates to the README file have propagated. - - diff --git a/admin/content/support-editing.md b/admin/content/support-editing.md deleted file mode 100644 index 44b2e57c..00000000 --- a/admin/content/support-editing.md +++ /dev/null @@ -1,96 +0,0 @@ -[title]: - "Editing content (topics)" - -[TOC] - -# General - -All Knowledge Base articles on the [helpdesk] are maintained -in GitHub. This provides us version tracking and co-authoring capabilities, -as well as allowing us to maintain documents in a neutral format (we chose -Markdown) that can be converted to HTML or to any other presentation. - -[helpdesk]: https://support.opensciencegrid.org/ - -# Reviewing content - -For reviewing content, it's _critical_ to view the [Support Knowledge Base][kbase] -(formerly the ConnectBook) *as a user or customer would see it*. - -If you are a help desk *agent*, meaning you have advanced privileges in the -system for supporting users and organizing content, your default view is -the agent view. To see the *customer portal*, choose an article or category -in the Solutions tab ([kbase]). In the upper right hand corner is a small box with an arrow -which will take you to the user's view of the article. - -Alternatively you can visit the site in another browser, or using your browser's "Incognito", -"Private Browsing", etc. mode. - -[kbase]: http://support.opensciencegrid.org/solution/categories - -At the end of any article that you want to edit, you'll find a statement like this one: - -> This page was updated on Jul 01, 2015 at 12:33 from [example.md][example-link]. - -[example-link]: https://github.com/OSGConnect/connectbook/blob/master/example.md - -This is a link directly to the source document in GitHub. Click it, or copy its link -and paste into another browser, and you'll see the document rendered by GitHub. Take -note: this view is probably slightly different from what you see in the Knowledge Base. -This is because there's a difference between regular Markdown (with extensions) and -the "[GitHub-Flavored Markdown][gfm]" that GitHub uses. Don't worry how it looks -in GitHub. - -[gfm]: https://help.github.com/articles/github-flavored-markdown/ - -# Editing content in the browser - -Once you see the document in GitHub, find the "edit" button in the action bar: -![editbar][editbar] - -[editbar]: https://raw.githubusercontent.com/OSGConnect/connectbook/master/admin/github-edit.jpg - -The edit button is the pencil icon near the right edge. Click it, and you'll enter -a text editor where you can edit the Markdown. - -You can directly edit Markdown text if you're comfortable with it. If you'd like -a different experience, you also can copy/paste the entire document into one of the -Markdown editors shown below, make your edits, and then copy/paste it back to Github. -When you finish, add a commit changelog note and click "Commit changes". Your updates -will be published to the Knowledge Base within 15-20 seconds. - -# Editing content on your computer - -You also can edit content on your own computer. This requires a working understanding -of git which we won't get into here. Just clone the connectbook repository: - - $ git clone --recursive https://github.com/OSGConnect/connectbook - -(The `--recursive` option is critical for getting copies of all tutorials.) - -Make your changes, then commit and push from your computer. Again, changes -will be published within 15-20 seconds, provided that no merge was necessary -at the time you pushed. (If one was, there may be further delay.) - - -# About Markdown - -You can learn more about Markdown here: - -* [daringfireball.net] -* http://whatismarkdown.com/ -* http://www.squarespace.com/display/ShowHelp?section=Markdown - -[daringfireball.net]: http://daringfireball.net/projects/markdown/ - - -Tools for editing Markdown: - -* Web - * http://daringfireball.net/projects/markdown/dingus - * http://dillinger.io/ - * https://stackedit.io/ -* MacOS - * http://mouapp.com/ - * http://texts.io/ - - diff --git a/admin/content/support-freshpush.md b/admin/content/support-freshpush.md deleted file mode 100644 index 27389492..00000000 --- a/admin/content/support-freshpush.md +++ /dev/null @@ -1,165 +0,0 @@ -[title]: - "How knowledge base synchronization works (technical reference)" - -[TOC] - -# General - -All Knowledge Base articles at http://support.osgconnect.net/ are maintained -in GitHub. This provides us version tracking and co-authoring capabilities, -as well as allowing us to maintain documents in a neutral format (we chose -Markdown) that can be converted to HTML or to any other presentation. - - -# Data flow - -Here is a high-level view of the data flow: - -![architecture](https://raw.githubusercontent.com/OSGConnect/connectbook/master/admin/connectbook-high-level.png) - -(This image is a bit outdated: in place of `cron`, we should have `GitHub webhook`. -This will be updated.) - -The piece of this diagram that concerns us right now is at the lower left -corner. Inside the `connectbook` repository are three files: - -* [freshpush](https://github.com/OSGConnect/connectbook/blob/master/update/freshpush) -* [freshpush.ini](https://github.com/OSGConnect/connectbook/blob/master/update/freshpush.ini) -* [update.sh](https://github.com/OSGConnect/connectbook/blob/master/update/update.sh) - -When GitHub receives an update to any of the `tutorial-` repos, or to the `connectbook` -repo, it sends a signal via webhook to a WSGI program running on `www.osgconnect.net`. -(Specifically this WSGI is at `/web/prod/ci-connect.net/github.wsgi`.) That in turn -runs the `update.sh` script. `Update.sh` `git pull`s all pending changes to -`connectbook` and all its submodules (the `tutorial-` repos), then runs `freshpush` -to send updates to Freshdesk. `Update.sh` receives as arguments the filenames -of any files that `github.wsgi` knows have changed, and passes these on to `freshpush`. - -`Freshpush` loads the configuration data in `freshpush.ini`. The main items of -interest in `freshpush.ini` are the mappings in the `[filemap]` section: - - [filemap] - acknowledgOSG.md = 5000640421 - client-install-rp.md = 5000640325 - client-install-user.md = 5000640326 - ... - tutorial-namd/README.md = 5000634378 - ... - admin/support-freshpush.md = 5000641634 - ... - -The numbers on the right are _article numbers_ in Freshdesk. They are used to construct -a URL for a Freshdesk API call. That API call is to update the article's content, and -the content is HTML that `freshpush` generates from the Markdown data in the corresponding -filename on the left-hand side. Thus any changes to the `acknowledgeOSG.md` file in the -`connectbook` repo are turned into HTML, and become part of the new content that is -pushed upstream to Freshdesk. Likewise, any Markdown change in the `README.md` file -in the `tutorial-namd` (sub-)repo becomes part of article `5000634378`. So long as the -changed files' filenames can be detected by `github.wsgi`, this entire procedure takes -downwards of 15 seconds from the time you `git push` your changes. - -# Specifics of editing articles - -There are several things to keep in mind as you write Markdown content for Freshdesk -publication. - -* The title of the article in Freshdesk is generated on the fly from metadata in the Markdown content. -* We use "real" Markdown, not [GitHub-Flavored Markdown (GFM)](https://help.github.com/articles/github-flavored-markdown/). -* There are several Markdown extensions available, notable Tables and Table of Contents. - -Let's look at each point. - -## Article titling - -There are three ways that `freshpush` can infer a title for your article: - -1. from the `[title]` "tag" -2. from the first heading in the article -3. from the file name - -### `[title]` - -The first is preferable. It (ab)uses the link by reference notation: - - \[linkname\]: url "alt text for url" - -to stand as invisible metadata within the document. "title" is the magic value for -"linkname". Inside your document, the first thing that `freshpush` sees that matches -this pattern will be used as the article's title: - - \[title\]: - "Article title" - -Note that "-" is used for the URL. - -### first heading - -If no `[title]:` appears in the article, then `freshpush` finds the first heading -in the article that is denoted by some number of leading `#` symbols. The entire -heading (after the hash marks) becomes the article title. - -### file name - -Finally, of neither previous method is successful, then the _local_ file name of the -article -- e.g., `support-freshpush.md` -- becomes the article name. However, the -`.md` at the end is removed, and all hyphens in the remainder become spaces: -`support freshpush`. - - -## Markdown flavor - -This is a requirement of the software libraries we use for publication, and a good idea for -portability. GFM is GitHub-specific. Do not assume that if it looks fine in GitHub, it -will be OK everywhere! The most common portability problem is that GFM allows "fenced" block -quotations: - - Regular text - ``` - blockquote content - ``` - More regular text - -Markdown does not allow this. It requires just a simple indentation for block quoting: - - Regular text - - blockquote content - - More regular text - -The second most common problem also has to do with block quotes: in Markdown, you need -a blank line between your regular content and the block-quoted content. Not including -one **will** mess up your rendering. - -## Markdown extensions - -Lastly, we support both the table of contents extension for Markdown and the tables -extension. The table of contents extension is very straightforward. It simply notes -all headings in the markdown content, and turns them into a table of contents with -internal anchor links to the associated section. This ToC is inserted wherever the -original document contains the simple line: - - \[TOC\] - -Upper case is required. - -The table extension allows you to construct simple tables. By way of example: - -| President | Year | Vice President | -|--|--|--| -| Washington | 1789 | Adams | -| Adams | 1797 | Jefferson | -| Jefferson | 1801 | Burr, Clinton | -| Madison | 1809 | Clinton, Gerry | - -(View this document's source to see the Markdown.) - -## Caveats - -Note that GitHub does not support either of these extensions, and your edits **will** -look wrong in GitHub's Markdown viewer. - - -# Provenance - -On publication, each document will contain a link at the very end -to the location on GitHub where it can be edited. - diff --git a/admin/content/support-remove-docs.md b/admin/content/support-remove-docs.md deleted file mode 100644 index d460f948..00000000 --- a/admin/content/support-remove-docs.md +++ /dev/null @@ -1,35 +0,0 @@ -[title]: - "Removing Content From the Helpdesk" - -[TOC] - -## Remove or hide the FreshDesk article - -Go to the Freshdesk portal, click on the solutions option on the left, and -find the relevant article. Click "Edit" in the top right and then either delete -the article (button on the left) or save it as a draft. - -## Remove the `freshpush.ini` entry - -In the [connectbook](https://github.com/OSGConnect/connectbook) repository, edit the `update/freshpush.ini` file by -either removing the referenced page or tutorial or commenting it out (if you -think it will be updated and restored later). - -## Remove the source - -### For non-tutorials - -Go to the [connectbook](https://github.com/OSGConnect/connectbook) repository and -either remove the source `.md` file for the guide, or move it to the `archive/` -folder. Commit changes and push to the central Github repository. - -### For tutorials - -If you want to revisit the tutorial later, change its name on GitHub to something like -`TOREVIEW-tutorial-name`. - -If you want to completely remove the tutorial: -- optional: download a zip copy first -- delete it from Github -- remove its submodule from the connectbook repository by following these instructions: -[removing a git submodule](https://gist.github.com/myusuf3/7f645819ded92bda6677) - diff --git a/admin/github-edit.jpg b/admin/github-edit.jpg deleted file mode 100644 index 058a9382..00000000 Binary files a/admin/github-edit.jpg and /dev/null differ diff --git a/admin/webhook.jpg b/admin/webhook.jpg deleted file mode 100644 index ce6c9eb1..00000000 Binary files a/admin/webhook.jpg and /dev/null differ diff --git a/alphalist-of-apps.md b/alphalist-of-apps.md deleted file mode 100644 index 282f7b2a..00000000 --- a/alphalist-of-apps.md +++ /dev/null @@ -1,212 +0,0 @@ -[title]: - "Software modules catalog" - -[TOC] - - -## Physics and Engineering -* **CDO** version (1.6.4) — CDO is a collection of command line Operators to manipulate and analyse Climate and NWP model Data. -* **CLHEP** version (2.1.0.1,2.2.0.8,2.3.1.0,2.3.1.1,...) — Set of HEP-specific foundation and utility classes such as random generators, physics vectors, geometry and linear algebra. -* **CP2K** version (2.5.1) — CP2K is the state-of-the-art method for accurate atomistic simulations. -* **EEMT** version (0.1) — EEMT stack which includes a bunch of GIS tools -* **GDAL** version (2.0.0) — GDAL is a translator library for raster and vector geospatial data formats that is released by the Open Source Geospatial Foundation. -* **GEANT4** version (9.4p02,10.02,10.3p01) — toolkit for the simulation of the passage of particles through matter. -* **GLPK** version (4.54) — glpk is the GNU's linear programming took kit. -* **GNUPLOT** version (4.6.5) — Gnuplot is an application for generating plots. -* **GRAPHVIZ** version (2.38.0) — Graphviz is an application for visualizing networks and graphs. -* **GRASS** version (6.4.4) — GRASS (Geographic Resources Analysis Support System) -* **JULIA** version (0.6.0) — Julia is a high-level, high-performance dynamic programming language for technical computing, with syntax that is familiar to users of other technical computing environments. -* **JULIA** version (0.6.0) — Julia is a high-level, high-performance dynamic programming language for technical computing, with syntax that is familiar to users of other technical computing environments. -* **LAMMPS** version (2.0,15May15) — LAMMPS is a particle simulator code. -* **LMOD** version (5.6.2) — Lmod is a Lua based module system that easily handles the MODULEPATH Hierarchical problem. -* **LMOD** version (AnEnvironmentModuleSystem) — Lmod is a Lua based module system that easily handles the MODULEPATH Hierarchical problem. -* **MADGRAPH** version (2.1.2,2.2.2) — MadGraph is a monte carlo generator for particle physics interactions -* **MATLAB** version (2013b,2014a,2014b,2015a,...) — Runtime libraries. The MATLAB Runtime is a standalone set of shared libraries that enables the execution of compiled MATLAB applications or components on computers that do not have MATLAB installed. -* **OCTAVE** version (3.8.1) — Octave is a high-level interpreted language -* **OPENSEES** version (6482) — OpenSees is a software framework for developing applications to simulate the performance of structural and geotechnical sys tems subjected to earthquakes. -* **PANDANA** version (-) — PANDANA performs hundreds of thousands of network queries in under a second (for walking-scale distances) -* **PSIMS** version (-) — A set of packages for needed for the PSIMS (numpy, h5py, netcdf4) -* **PYTORCH** version (-) — PYTORCH Tensors and Dynamic neural networks in Python with strong GPU acceleration. -* **QHULL** version (2012.1) — qhull libraries provide functions for computing convex hull. -* **R** version (3.1.1,3.2.0,3.2.1,3.2.2,3.3.1,3.3.2) — R is a free software environment for statistical computing and graphics. -* **ROOT** version (5.34-32-py34,5.34-32,6.06-02-py34) — ROOT is a high energy physics data analysis framework. -* **ROOT_NUMPY** version (-) — ROOT_NUMPY is a Python extension module that provides an efficient interface between ROOT and NumPy -* **SIESTA** version (3.2) — SIESTA performs efficient electronic structure calculations of molecules and solids. -* **UDUNITS** version (2.2.17) — UDUNITS package supports units of physical quantities. - - - -## Chemistry and Biochemistry -* **ARC-LITE** version (2015) — arc-lite (ARCIMBOLDO) is a package for the protein structure refinement. -* **AUTODOCK** version (4.2.6) — Autodock is a suite of automated docking tools to predict how small molecules bind to a receptor. -* **CASINO** version (2.13.211) — CASINO performs quantum Monte Carlo (QMC) electronic structure calculations. -* **CCP4** version (2015) — arc-lite (ARCIMBOLDO) is a package for the protein structure refinement. -* **ESPRESSO** version (5.1,5.2) — Quantum Espresso is an integrated suite of Open-Source computer codes for electronic-structure calculations and materials modeling at the nanoscale. -* **GAMESS** version (2013) — GAMESS is a general ab initio quantum chemistry package. -* **GROMACS** version (4.6.5,5.0.0,5.0.5.cuda,5.0.5,...) — Gromacs molecular dynamics simulator. Requires fftw/fftw-3.3.4-gromacs -* **NAMD** version (2.9,2.10.cuda,2.10) — NAnoscale Molecular Dynamics (NAMD) program. -* **OPENBABEL** version (2.3.2) — Open Babel is a chemical toolbox designed to speak the many languages of chemical data. -* **OPENBUGS** version (3.2.3) — OpenBUGS is a software package for performing Bayesian inference Using Gibbs Sampling. -* **ORCA** version (3.0.3,4.0.0) — ORCA is a general-purpose quantum chemistry program package. -* **PHENIX** version (1.10) — software suite for the determination of molecular structures from X-Ray and other methods. -* **PSI4** version (0.3.74,1.1) — Psi4 is an open-source suite of ab initio quantum chemistry programs designed for efficient, high-accuracy simulations of a variety of molecular properties. -* **ROSETTA** version (2015,2016-02,2016-32) — The Rosetta software suite includes algorithms for computational modeling and analysis of protein structures. -* **SHELX** version (2015) — SHELX is a set of programs for the determination of small and macromolecular crystal structures. -* **SHELX** version (2015) — SHELX is a set of programs for the determination of small and macromolecular crystal structures. -* **SIMBODY** version (3.5.3) — Simbody is an open source software system for biomechanical modeling, simulation and analysis. -* **SQLITE** version (3.8.11.1) — SQLite is a software library that implements a self-contained, serverless, zero-configuration, transactional SQL database engine. -* **VMD** version (1.9.1) — VMD is a molecular dynamics visualization application. - - - -## Image Analysis -* **ANTS** version (1.9.4,2.1.0) — ANTs is a popular toolkit for medical image registration and segmentation. -* **BLENDER** version (-) — Blender 3D rendering software. -* **ELASTIX** version (2015) — consists of a collection of algorithms that are commonly used to solve (medical) image registration problems -* **FIJI** version (2.0) — Fiji is an image processing package. It can be described as a "batteries-included" distribution of ImageJ (and ImageJ2) -* **FREESURFER** version (5.1.0,5.3.0,6.0.0) — Freesurfer is an open source software suite for processing and analyzing (human) brain MRI images. -* **FSL** version (5.0.8) — FSL is a comprehensive library of analysis tools for FMRI, MRI and DTI brain imaging data -* **GATE** version (7.2) — Gate does simulations of preclinical and clinical scans in Emission Tomography, Transmission Tomography a nd Radiation Therapy. -* **IGRAPH** version (0.7.1) — igraph is a library for creating and manipulating graphs. -* **IMAGEMAGICK** version (7.0.2) — Imagemagick is a set of image manipulation tools. -* **IMAGE_MODULES** version (-) — Image_modules includes a set of Python packages for needed for the image processing - - - -## Bioinformatics -* **ABYSS** version (2.0.2) — ABySS is a de novo, parallel, paired-end sequence assembler that is designed for short reads. -* **BEDTOOLS** version (2.21) — A set of tools for a wide-range of genomics analysis tasks. -* **BLASR** version (1.3.1) — Blasr is a tool to search and align sequences with refinements. -* **BLAST** version (-) — Blast is a tool to search and align gene and protein sequences. -* **BOWTIE** version (2.2.3,2.2.9) — Bowtie is an ultrafast and memory-efficient tool for aligning sequencing reads to long reference sequences. -* **BWA** version (0.7.12,2014) — BWA is a package for mapping low-divergent sequences against a large reference genome. -* **CUFFLINKS** version (2.2.1) — Cufflinks assembles transcripts -* **ENTROPY** version (2017.03.16) — Admixture and the organization of genetic diversity in a butterfly species complex revealed through common and rare genetic variants. -* **ETE2** version (2.3.8) — ETE is a toolkit based on Python framework for the analysis and visualization of trees. -* **HISAT2** version (2.0.3-beta) — HISAT2 is a fast and sensitive alignment program for mapping next-generation sequencing reads (both DNA and RNA) against th e general human population (as well as against a single reference genome). -* **HMMER** version (3.1) — HMMER searches and aligns homologs of protein sequences using hidden Markov models. -* **MOTHUR** version (1.39.0) — mothur is a project that seeks to develop a single piece of open-source, expandable software to fill the bioinformatics nee ds of the microbial ecology community. -* **MRBAYES** version (3.2.2) — MrBayes is a program for Bayesian inference of phylogenetic and evolutionary models. -* **MUMMER** version (3.23) — MUMmer is a system for rapidly aligning entire genomes, whether in complete or draft form -* **MUSCLE** version (3.8.31) — MUSCLE stands for MUltiple Sequence Comparison by Log-Expectation -* **NGSTOOLS** version (2017.03.16) — ngsTools is a collection of programs for population genetics analyses from NGS data, taking into account its statistical un certainty. -* **PBSUITE** version (14.9.9) — PBSuite is a software for long-read sequencing data from PacBio. -* **PRODIGAL** version (2.6.3) — Fast, reliable protein-coding gene prediction for prokaryotic genomes. -* **RAXML** version (8.2.9) — A tool for Phylogenetic Analysis and Post-Analysis of Large Phylogenies -* **RAXML-NG** version (0.5.0beta) — Phylogenetic tree inference tool which uses maximum-likelihood (ML) optimality criterion. -* **SAMTOOLS** version (0.1.17,1.3.1) — Samtools is a set of utilities for working with data in sequence alignment map format. -* **SEQGEN** version (1.3.3) — Seq-Gen is a program that will simulate the evolution of nucleotide or amino acid sequences along a phylogeny -* **SHRIMP** version (2.2.3) — SHRiMP is a software package for aligning genomic reads against a target genome. -* **SRA** version (2.5.4,2.8.0) — The NCBI SRA Toolkit enables reading ("dumping") of sequencing files from the SRA database and writing ("loading") files into the .sra format -* **STRINGTIE** version (1.1.2,1.2.2) — StringTie is a fast and highly efficient assembler of RNA-Seq alignments into potential transcripts. -* **TASSEL** version (5.0) — TASSEL has functionality for association study, evaluating evolutionary relationships, analysis of linkage disequilibrium, principal component analysis, cluster analysis, missing data imputation and data visualization. -* **TOPHAT** version (2.0.13,2.1.1) — TopHat is a fast splice junction mapper for RNA-Seq reads. -* **TRANSABYSS** version (1.5.5) — de novo assembly of RNA-Seq data using ABySS -* **UCLUST** version (2.22) — Uclust offers search and clustering algorithms that are often orders of magnitude faster than BLAST. -* **VIENNARNA** version (2.2) — ViennaRNA Package consists of a C code library and several stand-alone programs for the prediction and comparison of RNA secondary structures. - - - -## Numerical Libraries -* **ATLAS** version (3.10.1,3.10.2) — The ATLAS (Automatically Tuned Linear Algebra Software) provides optimzied libraries for Linear Algebra. -* **BOOST** version (1.50.0,1.56,1.57.0,1.62.0-cxx11,...) — Boost provides free peer-reviewed portable C++ source libraries. -* **CBLOSC** version (1.7.1) — A blocking, shuffling and loss-less compression library -* **EIGEN** version (3.2.10) — Eigen is a C++ template library for linear algebra: matrices, vectors, numerical solvers, and related algorithms. -* **FFTW** version (3.3.4-gromacs,3.3.4) — FFTW is a library for fast fourier transforms. FFTW 3.3.4 library compiled for general use. -* **GEOS** version (3.4.2) — GEOS (Geometry Engine - Open Source) is an API for processing spatial geometry. -* **GMP** version (6.0.0) — Gnu Multi Precision Libraries -* **GSL** version (1.16,2.3) — The GNU Scientific Library (GSL) is a numerical library for C and C++ programmers. -* **IPOPT** version (3.12.6) — Ipopt (Interior Point OPTimizert) is a software package for large-scale nonlinear optimization -* **LAPACK** version (3.5.0,3.6.1) — LAPACK (Linear Algebra Package) is a standard software library for numerical linear algebra. -* **MIXMODLIB** version (3.1) — Mixmod statistical libraries -* **MPC** version (1.0.3) — Gnu Mpc is a C library for arithmatics with arbitrarily high precision of complex numbers. -* **MPFR** version (3.1.3) — MPFR library is a C library for multiple-precision floating-point computations. -* **NCO** version (4.3.0) — The NCO toolkit manipulates and analyzes data stored in netCDF-accessible formats. -* **NETCDF** version (4.2.0) — NetCDF is a set of software libraries supporting array-oriented scientific data. -* **PARI** version (2.7.5) — PARI is a widely used computer algebra system designed for fast computations in number theory -* **SAGA** version (2.2.0) — SAGA is a GIS application -* **SDPA** version (7.3.8) — SDPA (Semi Definite Programming Algorithm) is one of the efficient software package for solving SDPs. -* **SPARSE** version (-) — Sparse works with Python 2.7 and offers a set of packages to deal with sparse matrices -* **SPARSESUITE** version (4.2.1) — SuiteSparse is a suite of sparse matrix algorithms. -* **SUNDIALS** version (2.5) — SUNDIALS (SUite of Nonlinear and DIfferential/ALgebraic equation Solvers) - - - -## Software Libraries, Languages, and Tools -* **ALL-PKGS** version (-) — all-pkgs loads Python with a set of popular packages for scientific computing (Numpy, matplotlib, Scipy, pandas, nltk) -* **ANT** version (1.9.4) — Ant is a Java library and command-line tool is to drive processes described in build files. -* **APR** version (1.5.1) — The Apache Portable Runtime (APR) libraries provide a interface to platform-specific implementations. -* **APRUTIL** version (1.5.3) — APR Util provides cross platform libraries for handling OS related operations. -* **BBFTP** version (3.2.1) — BBFTP is a file transfer software. -* **BINUTILS** version (2.26) — GNU Binutils are a collection of binary tools that do low level operations on binaries. -* **BOTO** version (-) — Boto is a Python package that provides interfaces to Amazon Web Services. -* **BZIP2** version (1.0.6) — bzip2 is a freely available, patent free, high-quality data compressor. -* **CANOPY** version (1.4.1) — Canopy is the next generation of the Enthought Python Distribution (EPD) -* **CCTOOLS** version (4.4.2,5.2.3,5.4.7,6.0.7) — Cooperative Computing Lab is software designed to easily harness large scale distributed systems such as clusters, clouds, and grids. -* **CFITSIO** version (3.37) — CFITSIO is a library of C and Fortran subroutines for reading and writing data files in F ITS (Flexible Image Transport System) data format. -* **CIRCOS** version (0.68) — Circos is a software package for visualizing data and information in a circular layout. -* **CMAKE** version (3.0.1,3.4.1,3.8.0) — CMake is a cross platform make utility -* **CONNECT-CLIENT** version (0.2.1,0.3.0,...) — connect-client is command line tool to use OSG from your local machine. -* **CPAN** version (perl-5.10) — The Comprehensive Perl Archive Network (CPAN) is a repository for Perl programming language -* **CURL** version (7.37.1) — Curl is a command line HTTP client -* **DAKOTA** version (6.4.0) — The Dakota toolkit provides a flexible, extensible interface between analysis codes and iterative systems analysis methods. -* **DENDROPY** version (-) — DendroPy is a Python library for phylogenetic computing -* **DMTCP** version (2.5.0) — DMTCP (Distributed MultiThreaded Checkpointing) transparently checkpoints a single-host or distributed computation in user-space -* **ECTOOLS** version (-) — Ectools is a set of tools for correcting long read assembly data -* **EXPAT** version (2.1.0) — Expat is an XML parser library written in C. -* **FFMPEG** version (0.10.15,2.5.2) — ffmpeg is a complete -* **FPC** version (2.6.4) — Free pascal Free Pascal is a 32 -* **FREETYPE** version (2.5.5) — FreeType 2 provides a simple and easy-to-use API to access font content in a uniform way -* **GCC** version (4.6.2,4.6.4,4.8.1,4.9.2,4.9.3,6.2.0) — GCC is a set of compilers for c -* **GD** version (2.1.1) — GD is an open source code library for the dynamic creation of images by programmers. -* **GFAL** version (7.20) — GFAL2 is a set of file access utilities. -* **GIT** version (1.9.0) — Git is a software change management tool. -* **GLOBUS-CLI** version (-) — globus-cli is a free command line tool that provides an interface to Globus services -* **GNOME_LIBS** version (1.0) — GNOME desktop libraries. -* **HDF5** version (1.8.5-cxx11,1.8.9,1.8.12-cxx11,1.8.12,...) — HDF5 is a library for storing and managing data in flexible -* **IRODS** version (4.2.2) — The Integrated Rule-Oriented Data System (iRODS) is open source data management software. -* **JASPER** version (1.900.1) — reference implementation of the codec specified in the JPEG -* **JAVA** version (7u71,8u25,8u131) — Java -* **JPEG** version (6b,9a) — Libjpeg is a widely used C library for reading and writing jpeg image files. -* **LIBGFORTRAN** version (4.4.7) — -* **LIBICU** version (4.2.1) — International Components for Unicode (ICU) libraries. -* **LIBTIFF** version (4.0.4) — -* **LIBXPM** version (3.5.10) — LibXPM is a package with X.Org, X11, and libXpm runtime libraries -* **LLVM** version (3.6,3.7,3.8.0,5.0.0) — LLVM is a toolkit for the construction of highly optimized compilers, optimizers, and runtime environments. -* **MERCURIAL** version (1.9.1) — Mercurial is a source code management tool. -* **MONO** version (4.2.1) — Mono is a software platform designed to allow developers to easily create cross platform applications. -* **MPLAYER** version (1.1) — Mplayer is an image viewer and editor. -* **MYSQL** version (5.1.73) — mysql client tool that provides command line interface to the mysql database. -* **OPENCV** version (2.4.10) — OpenCV (Open Source Computer Vision Library) is an open source computer vision and machine learning software library. -* **OPENSIM** version (3.3) — OpenSim is an open source software system for biomechanical modeling, simulation and analysis. -* **PAPI** version (5.3.2) — PAPI is a performance profiler. -* **PARSL** version (-) — Parsl is a parallel scripting library that enables easy parallelism and workflow design. -* **PBSUITE_PKGS** version (-) — PBSuite loads python and adds numpy, h5py, and pysam packages. -* **PCRE** version (8.35) — PCRE is perl compatible regular expression library. -* **PEGASUS** version (4.4.2-image_tools,4.5.3,4.6.0dev,...) — Pegasus is a workflow management system to automate, recover, and debug scientific computations. -* **POPPLER** version (0.24.1,0.32) — Poppler is a PDF rendering library based on the xpdf-3.0 code base -* **POVRAY** version (3.7) — The Persistence of Vision Raytracer is a high-quality, Free Software tool for creating stunning three-dimensional graphics. -* **PROJ** version (4.9.1) — Cartographic projection library -* **PROOT** version (2014) — PRoot is a user-space implementation of chroot -* **PROTOBUF** version (2.5) — Protocol buffers are Google's language-neutral, platform-neutral, extensible mechanism for serializing structured data – think XML, but smaller, faster, and simpler. -* **PYTHON** version (2.7,3.4,3.5.2) — Python is a programming language that lets you work quickly and integrate systems more effectively. -* **RUBY** version (2.1) — Ruby is a dynamic -* **RUCIO** version (1.6.6) — Distributed Data Management (DDM) system services -* **S3CMD** version (-) — S3cmd is a free command line tool and client for uploading, retrieving and managing data in Amazon S3 and other cloud stora ge service providers that use the S3 protocol -* **SCONS** version (2.3.4) — SCons is a software construction tool and is a superior alternative to the classic "Make" build tool. -* **SERF** version (1.37) — Serf is a library that provides high performance http client functions. -* **SETTARG** version (5.6.2) — The settarg module dynamically and automatically updates "$TARG" and a host of other environment variables. -* **SNAPPY** version (1.1.3) — Snappy is a compression and decompression library. -* **STASHCP** version (2.6,4.3.0,4.3.1) — Stashcp is a front-end to XRootD caching for OSG Connect Stash. -* **SUBVERSION** version (1.8.10) — Subversion is a source control revision tool. -* **SWIFT** version (0.94.1,0.96.2) — Swift is a parallel scripting language. -* **TCL** version (8.6.2) — Tcl (Tool Command Language) is a very powerful but easy to learn dynamic programming language. -* **TCSH** version (6.20.00) — tcsh is a Unix shell based on and compatible with the C shell (csh). It is essentially the C shell with programmable command-line completion, command-line editing, and a few other features. -* **UNIXODBC** version (2.3.2) — The unixODBC Project goals are to develop and promote unixODBC to be the definitive standard for ODBC on non MS Windows platforms. -* **VALGRIND** version (3.10) — Valgrind can detect memory management and threading bugs -* **WGET** version (1.15) — Wget is a command line HTTP client -* **WXGTK** version (3.0.2) — wxWidgets is a C++ library that lets developers create applications for Windows, Mac OS X, and Linux. -* **XROOTD** version (4.1.1,4.2.1) — The XROOTD project aims at giving high performance, scalable fault tolerant access to data repositories of many kinds. -* **XZ** version (5.2.2) — XZ Utils is free general-purpose data compression software with a high compression ratio. -* **ZLIB** version (1.2.8) — Zlib is a library that provides functions for compression and decompression of data - - diff --git a/archive/2020-transition-guide.md b/archive/2020-transition-guide.md deleted file mode 100644 index 3522db10..00000000 --- a/archive/2020-transition-guide.md +++ /dev/null @@ -1,81 +0,0 @@ -[title]: - "Transitioning to a New Login Node" - -[TOC] - -## Summary - -In January 2020, OSG Connect will transition to a new user management system, new login nodes, and new filesystem layout. The changes are detailed below, but in summary: - -1. **New Website** The OSG Connect website, where you log in to manage your account details will look -different and present information in a clearer way. -1. **New Login Node** Instead of using a generic login node name (`login.osgconnect.net`), you will be assigned a -specific, numbered login node to use, which will appear on your OSG Connect website profile. We’ve given you time to transition to the new login nodes, and recommend migrating workloads and data as soon as possible in light of the deadlines listed further below. -1. **New Filesystem** The filesystem layout is different on the new login nodes. `$HOME` is no longer a shared filesystem, `/local-scratch` no longer exists (use `$HOME` instead, for all job submission), and /public is a large shared space for larger job data that will need to be to be accessible to jobs via stashcp, http, and CVMFS. - -## Deadlines - -Access to the previous login nodes (`login02`, `login03`) will still be possible, however, we -recommend migrating workloads and data as soon as possible in light of the following -deadlines: - -* **January 9, 2020**: stashcp will no longer be able to access files in the old /stash folder, files must be in /public -* **January 22, 2020**: Submission of new jobs will be disabled on the old login nodes (login, login02, login03) to encourage users to transition to the new login nodes. (Already-submitted jobs and DAGs will be able to continue and complete up until March 31.) -* **March 31, 2020**: Data will be deleted from the old login nodes and the previous Stash data locations. -* **After March 31, the existing login nodes and all data in their /home, /local-scratch, and previous Stash locations will be removed.** Because it may take time to move your data, it is important to begin transitioning as soon as possible! - -## Action Items - -Follow these steps to ensure you have your data and can continue to submit jobs: - -1. Log into https://osgconnect.net. Confirm that your information is still correct, -and find the name of the login node that you've been assigned. The login node -information can be seen on your profile here: -![Identify Login Node](https://raw.githubusercontent.com/OSGConnect/connectbook/master/images/find_osgconnect_login_node.png "OSG Connect Profile") - - > **IMPORTANT** If you log into the OSG Connect website with your institutional credentials - and it asks you to sign up for OSG / create a profile, do NOT create a new profile. Instead, do the following: - > * close the OSG Connect page and go to http://globus.org. - > * Log in with your Globus ID and go to "Account" on the left-hand menu. - > * If your institutional identity is already listed on that Account page, [email the OSG Connect support team](mailto:support@opensciencegrid.org) - > * If your institutional identity is NOT listed on the account page, select "Link Another Identity" on the - > right and add your institutional identity. Then go back to the OSG Connect website and try to - > log in again. - -1. Log into the new login node and set your primary project. Follow the instructions -at the bottom of the OSG project guide: [Join and Use a Project in OSG Connect](5000634360) - - $ connect project - -1. Retrieve all the data from your old home directory. Migrate any files that you are still using to run jobs to your `/home` directory on the new assigned login node. Any other (older) files can be transferred back to your own personal computer / storage. The old $HOME directory is available under `/old-home/$USERNAME/`. For example: - - $ mv /old-home/$USERNAME/example-data $HOME/ - -1. Similarly, retrieve your data from Stash on the old login nodes. Note that the Stash equivalent on the new login nodes, `/public`, is all publicly readable - files can be downloaded from that folder by any person who has the correct link. For example, moving data from the old stash to the new public directory: - - $ mv /stash/user/$USERNAME/example-data /public/$USERNAME/ - - This is a good time to clean up! Please use `mv` when moving the data from the old stash location and `rm` do remove any data you no longer need. If you leave data in the old stash location, we might contact you in the future to have it cleaned up. - -1. Update your workloads for the new filesystem layout. Due to the change in paths (no -more `/stash`, no `/local-scratch`), you may need to modify submit files and scripts -to work correctly on the new log in nodes. If you are using stashcp, paths will need to be updated. Please read our updated guide on the new filesystem layout, how the different filesystems should be used, and quotas. - - * [New Data Management Guide](12000002985) - - In short, jobs should run from `$HOME` and can use `/public` for data access. - -## Other Questions - -### Will there be GlobusOnline access? - -There will be no GlobusOnline access in the new set up. - -### Special projects (XENON, SPT, VERITAS, ...) - are they affected? - -Not at this time. - -### Why are we transitioning? - -There are two drivers for this transition. The first one the new user management system which should make it much easier to sign up for OSG Connect and use it. A second reason is to simplify data access and increase the performance of the submit nodes. - - diff --git a/archive/accessing-data-overview.md b/archive/accessing-data-overview.md deleted file mode 100644 index ab9cdecf..00000000 --- a/archive/accessing-data-overview.md +++ /dev/null @@ -1,31 +0,0 @@ -[title]: - "Accessing Data - When to Use Which Tool?" - -Job Input Data --------------- - -There are multiple solutions for accessing input data, and which solution to use -depends on factors such as data size, if the same data is used by multiple jobs, and -if you prefer POSIX-like file access or not. Here is a list of options, and links -to details: - -- **HTCondor file transfers** - This is by far the simplest solution, and can be - used for small to moderate file sizes (up to 1 GB total). - [Details](/solution/articles/5000639787) - -- **Accessing Stash over HTTP** - This is a good solution for moderate to large - filesizes (0 - 100 GB), - [Details](/solution/articles/5000639798) - -- **StashCache** - This has similar characteristics as the Stash solution, but - with the additional benefit of transparent data caching making the solution - even more attractive if you have many job using the same data inputs. - [Details](/solution/articles/12000002775) - - -Job Output Data ---------------- - -The solutions for data transfers from your job back to OSG Connect are more limited. -At this point, we recommend that you use the built-in HTCondor file transfer mechanism -(*transfer_output_files=...* in your job submit file). - diff --git a/archive/communicate.md b/archive/communicate.md deleted file mode 100644 index eece6056..00000000 --- a/archive/communicate.md +++ /dev/null @@ -1,17 +0,0 @@ -[title]: - "Communicate with us via Twitter" - -## Receive status updates - -The OSG Research Facilitation Twitter account is -[@osgusers](http://twitter.com/osgusers). We will report issues in -near real time using that feed so you have one place to look for late -breaking news or updates. - -## Send a direct message - -Additionally, you can report problems or questions directly -from your Twitter account by sending a direct message (DM) to -[@osgusers](http://twitter.com/osgusers). If you link your Twitter -account with your OSG help desk account, we'll be able to track and -follow-up more efficiently. - diff --git a/archive/hpcadmin/campus/hosted-ce.md b/archive/hpcadmin/campus/hosted-ce.md deleted file mode 100644 index dd4ee0d1..00000000 --- a/archive/hpcadmin/campus/hosted-ce.md +++ /dev/null @@ -1,3 +0,0 @@ -[title]: - "OSG Managed Services" - -This article has been permanently moved: https://opensciencegrid.org/docs/compute-element/hosted-ce/ diff --git a/archive/hpcadmin/campus/htcondor-compute-element.md b/archive/hpcadmin/campus/htcondor-compute-element.md deleted file mode 100644 index 27736fd7..00000000 --- a/archive/hpcadmin/campus/htcondor-compute-element.md +++ /dev/null @@ -1,25 +0,0 @@ -[title]: - "HTCondor Compute Element" - -[TOC] - -# What is a Compute Element? - -An OSG Compute Element (CE) is the entry point for the OSG to your local resources: a layer of software -that you install on a machine that can submit jobs into your local batch system. At the heart of the CE is the -job gateway software, which is responsible for handling incoming jobs, authorizing them, and delegating them to -your batch system for execution. Historically, the OSG only had one option for a job gateway solution, Globus -Toolkit’s GRAM-based gatekeeper, but now offers the HTCondor CE as an alternative. - -Today in OSG, most jobs that arrive at a CE (called grid jobs) are not end-user jobs, but rather pilot jobs -submitted from factories. Successful pilot jobs create and make available an environment for actual end-user jobs -to match and ultimately run within the pilot job container. Eventually pilot jobs remove themselves, typically -after a period of inactivity. - -# What is HTCondor CE? - -HTCondor CE is a special configuration of the HTCondor software designed to be a job gateway solution for the -OSG. It is configured to use the [JobRouter daemon](http://research.cs.wisc.edu/htcondor/manual/v8.2/5_4HTCondor_Job.html) to delegate jobs by transforming and submitting them to the site’s -batch system. - -# More information -The [OSG twiki](https://twiki.opensciencegrid.org/bin/view/Documentation/Release3/HTCondorCEOverview) has more information about the specifics of how the HTCondor CE operates and information on installing and configuring one. diff --git a/archive/hpcadmin/campus/overview.md b/archive/hpcadmin/campus/overview.md deleted file mode 100644 index 99f33aad..00000000 --- a/archive/hpcadmin/campus/overview.md +++ /dev/null @@ -1,32 +0,0 @@ -[title]: - "Overview" - -## OSG Managed Services - -There are two main ways to provide computing resources to the OSG. The -first method, which is appropriate for smaller sites or those with -limited effort to deploy and operate the OSG software stack, is -called the **OSG Managed Service**. Here the access is provided through an ssh -account to the cluster, much like a normal user account. Jobs from the -OSG virtual organization are delivered from a factory service provided -by the OSG Connect system. Once commissioned, very little manpower is -required to maintain the connection. - -## HTCondor Compute Element - -The standard method is to deploy the OSG software stack, the most relevant -component being the **Compute Element (CE)**. The method is based on -standard grid technology and is how the vast majority of OSG computing -centers deliver resources to users. This is the preferred choice -for large sites aiming to support all of the virtual organizations -participating in OSG. There is more up-front learning to deploy this -software, but once operational the effort required to keep it running is -manageable. In OSG we have recently developed a new version of the the -CE based on HTCondor. - -## Talk to us! - -If you are interested in having your campus HPC -center join the Open Science Pool, contact us at -[support@opensciencegrid.org](mailto:support@opensciencegrid.org) -and we'll help you determine the best option and help you get -started. diff --git a/archive/hpcadmin/flocking/osg-flock.md b/archive/hpcadmin/flocking/osg-flock.md deleted file mode 100644 index 20b2eb0c..00000000 --- a/archive/hpcadmin/flocking/osg-flock.md +++ /dev/null @@ -1,3 +0,0 @@ -[title]: - "Submit Node Flocking to OSG" - -This documentation has been moved to https://opensciencegrid.org/docs/submit/osg-flock/ diff --git a/archive/intro-to-r.md b/archive/intro-to-r.md deleted file mode 100644 index 769b8e4e..00000000 --- a/archive/intro-to-r.md +++ /dev/null @@ -1,254 +0,0 @@ -[title]: - "Run R scripts on OSG" - -[TOC] - -## Overview -This tutorial describes how to run R scripts on the OSG. We'll first run the program locally as a test. After that we'll create a submit file, submit it to OSG using OSG Connect, and look at the results when the jobs finish. Finally, we will talk about how to use custom R libraries on OSG Connect. - -## Run R scripts on OSG -### Access R on the submit host - -First we'll need to create a working directory, you can either run `$ tutorial R` or type the following: - - $ mkdir tutorial-R; cd tutorial-R - -R is installed using modules on OSG. To load this modules and access R, enter: - - $ module load r/3.5.1-py2.7 - - -Now, we can try to run R: - - $ R - - R version 3.5.1 (2018-07-02) -- "Feather Spray" - Copyright (C) 2018 The R Foundation for Statistical Computing - Platform: x86_64-pc-linux-gnu (64-bit) - - R is free software and comes with ABSOLUTELY NO WARRANTY. - You are welcome to redistribute it under certain conditions. - Type 'license()' or 'licence()' for distribution details. - - Natural language support but running in an English locale - - R is a collaborative project with many contributors. - Type 'contributors()' for more information and - 'citation()' on how to cite R or R packages in publications. - - Type 'demo()' for some demos, 'help()' for on-line help, or - 'help.start()' for an HTML browser interface to help. - Type 'q()' to quit R. - - > - -Great! R works. You can quit out with `q()`. - - > q() - Save workspace image? [y/n/c]: n - $ - -### Run R code - -Now that we can run R, let's create a small script. Create the file `hello_world.R` that contains the following: - - print("Hello World!") - -R normally runs as an interactive shell, but it is easy to run in batch mode too. - - $ Rscript --no-save hello_world.R - [1] "Hello World!" - -Notice here that we're using Rscript (equivalent to `R CMD BATCH`) which accepts the script as command line argument. This approach makes `R` much less verbose, and it's easier to parse the output later. If you run it at the command line, you should get similar output as above. - -### Build the HTCondor job - -To prepare our R job to run on OSG, we need to create a wrapper for our R environment, based on the setup we did in previous sections. Create the file `R-wrapper.sh`: - - #!/bin/bash - - module load r/3.5.1-py2.7 - Rscript --no-save hello_world.R - -Change the permissions on the wrapper script so it is executable and then test it for correct output: - - $ chmod +x R-wrapper.sh - $ ./R-wrapper.sh - [1] "Hello World!" - -Now that we've created a wrapper, let's build a HTCondor submit file around it. We'll call this one `R.submit`: - - universe = vanilla - log = R.log.$(Cluster).$(Process) - error = R.err.$(Cluster).$(Process) - output = R.out.$(Cluster).$(Process) - - executable = R-wrapper.sh - transfer_input_files = hello_world.R - - request_cpus = 1 - request_memory = 1GB - request_disk = 1GB - - requirements = OSGVO_OS_STRING == "RHEL 7" && Arch == "X86_64" && HAS_MODULES == True - queue 1 - - -The `R.submit` file may have included a few lines that you are unfamiliar with. For example, `$(Cluster)` and `$(Process)` are variables that will be replaced with the job's cluster and process id. This is useful when you have many jobs submitted in the same file. Any output and errors will be placed in a separate file for each job. - -Notice the requirements line? You'll need to put `HAS_MODULES == True` any time you need software that is loaded via modules. - -Also, did you see the transfer_input_files line? This tells HTCondor what files to transfer with the job to the worker node. You don't have to tell it to transfer the executable, HTCondor is smart enough to know that the job will need that. But any extra files, such as our R script file, will need to be explicitly listed to be transferred with the job. You can use transfer_input_files for input data to the job, as shown in [Transferring data with HTCondor](https://github.com/OSGConnect/tutorial-htcondor_transfer). - - -### Submit and analyze the output - -Finally, submit the job to OSG Connect! - - $ condor_submit R.submit - Submitting job(s).......... - 1 job(s) submitted to cluster 3796250. - $ condor_q user - - $ condor_q - -- Schedd: login03.osgconnect.net : <192.170.227.22:9618?... @ 05/13/19 09:51:04 - OWNER BATCH_NAME SUBMITTED DONE RUN IDLE TOTAL JOB_IDS - user ID: 3796250 5/13 09:50 _ _ 1 1 3796250.0 - ... - -You can follow the status of your job cluster with the `connect watch` command, which shows `condor_q` output that refreshes each 5 seconds. Press `control-C` to stop watching. - -Since our jobs prints to standard out, we can check the output files. Let's see what one looks like: - - $ cat R.out.3796250.0 - [1] "Hello World!" - -## Use custom R libraries on OSG - -Often we may need to add R external libraries that are not part of standard R installation. As a user, we could add the libraries in our home (or stash) directory and make the libraries available on remote machines for job executions. - -### Build external packages for R under userspace - -It is helpful to create a dedicated directory to install the package into. This will facilitate zipping the library so it can be transported with the job. Say, you decided to built the library in the path `/home/username/R_libs/lubridate_R.3.5`. If it does not already exist, make the necessary directory by typing the following in your shell prompt: - - $ mkdir -p ~/R_libs/lubridate_R.3.5 - -After defining the path, we set the `R_LIBS` environment variable so R knows to use our custom library directory: - - $ export R_LIBS=~/R_libs/lubridate_R.3.5 - -Now we can run R and check that our library location is being used (here the `>` is the R-prompt): - - $ module load r/3.5.1-py2.7 - $ R - ... - > .libPaths() - [1] "/home/user/R_libs/lubridate_R.3.5" - [2] "/cvmfs/connect.opensciencegrid.org/modules/packages/linux-rhel7-x86_64/gcc-6.4.0spack/r-3.5.1-eoot7bzcbxp3pwf4dxlqrssdk7clylwd/rlib/R/library" - -Excellent. We can see the location listed as library path `[1]`. We can also check for available libraries within R. - - > library() - -Press `q` to close that display. - -If you want to install the package “XYZ”, within R do - - > install.packages("XYZ", repos = "http://cloud.r-project.org/", dependencies = TRUE) - -To install `lubridate`, enter this command: - - > install.packages("lubridate", repos="http://cloud.r-project.org/", dependencies=TRUE) - -### Install multiple packages at once - -If you have multiple packages to be added, it may be better to list each of the `install.packages()` commands within a separate R script and source the file to R. For example, if we needed to install `ggplot2`, `dplyr`, and `tidyr`, we can list them to be installed in a script called `setup_packages.R` which would contain the following: - - install.packages("ggplot2", repos="http://cloud.r-project.org/", dependencies=TRUE) - install.packages("dplyr", repos="http://cloud.r-project.org/", dependencies = TRUE) - install.packages("tidyr", repos="http://cloud.r-project.org/", dependencies = TRUE) - -Run the setup file within R. - - > source(`setup_packages.R`) - -### Prepare a tarball of the add-on packages - -Proceeding with the `lubridate` package, the next step is create a tarball of the package so we can send the tarball along with the job. - -Exit from the R prompt by typing: - - > quit() - -or: - - >q() - -In either case, be sure to say `n` when prompted to `Save workspace image? [y/n/c]:`. - -To tar the package directory, type the following at the shell prompt: - - $ cd /home/user/R_libs - $ tar -cvzf lubridate_R.3.5.tar.gz lubridate_R.3.5 - -Now copy the tarball to the job directory where the R program, job wrapper script and condor job description file are. - -### Use the packages in your OSG job - -Now, let's change the `hello_world` job to use the new package. First, modify the R script `hello_world.R` by adding the following lines: - - library(lubridate) - print(today()) - -This will add a print out of the local date to the output of the job. - -### Define the libPaths() in the wrapper script - -R library locations are set upon launch and can be modified using the `R_LIBS` environmental variable. To set this correctly, we need to modify the wrapper script. Change the file `R-wrapper.sh` so it matches the following: - - #!/bin/bash - - module load r/3.5.1-py2.7 - - # Uncompress the tarball - tar -xzf lubridate_R.3.5.tar.gz - - # Set the library location - export R_LIBS="$PWD/lubridate_R.3.5" - - # run the R program - Rscript --no-save hello_world.R - -Next, we need to modify the submit script so that the package tarball is transferred correctly with the job. Change the submit script `R.submit` so that `transfer_input_files` and `arguments` are set correctly. Here's what the completed file should look like: - - universe = vanilla - log = R.log.$(Cluster).$(Process) - error = R.err.$(Cluster).$(Process) - output = R.out.$(Cluster).$(Process) - - executable = R-wrapper.sh - transfer_input_files = lubridate_R.3.5.tar.gz, hello_world.R - - request_cpus = 1 - request_memory = 1GB - request_disk = 1GB - - requirements = OSGVO_OS_STRING == "RHEL 7" && Arch == "X86_64" && HAS_MODULES == True - queue 1 - -### Job submission and output -Now we are ready to submit the job: - - $ condor_submit R.submit - -and check the job status: - - $ condor_q username - -Once the job finished running, check the output files as before. They should now look like this: - - $ cat R.out.3796676.0 - [1] "2019-05-13" - [1] "Hello World!" - -## Getting Help -For assistance or questions, please email the OSG Research Facilitation team at or visit the [help desk and community forums](http://support.opensciencegrid.org). diff --git a/archive/list-of-applications.md b/archive/list-of-applications.md deleted file mode 100644 index 28fd33a0..00000000 --- a/archive/list-of-applications.md +++ /dev/null @@ -1,212 +0,0 @@ -[title]: - "Software Modules Catalog" - -[TOC] - - -## Physics and Engineering -* **CDO** version (1.6.4) — CDO is a collection of command line Operators to manipulate and analyse Climate and NWP model Data. -* **CLHEP** version (2.1.0.1,2.2.0.8,2.3.1.0,2.3.1.1,...) — Set of HEP-specific foundation and utility classes such as random generators, physics vectors, geometry and linear algebra. -* **CP2K** version (2.5.1) — CP2K is the state-of-the-art method for accurate atomistic simulations. -* **EEMT** version (0.1) — EEMT stack which includes a bunch of GIS tools -* **GDAL** version (2.0.0) — GDAL is a translator library for raster and vector geospatial data formats that is released by the Open Source Geospatial Foundation. -* **GEANT4** version (9.4p02,10.02,10.3p01) — toolkit for the simulation of the passage of particles through matter. -* **GLPK** version (4.54) — glpk is the GNU's linear programming took kit. -* **GNUPLOT** version (4.6.5) — Gnuplot is an application for generating plots. -* **GRAPHVIZ** version (2.38.0) — Graphviz is an application for visualizing networks and graphs. -* **GRASS** version (6.4.4) — GRASS (Geographic Resources Analysis Support System) -* **JULIA** version (0.6.0) — Julia is a high-level, high-performance dynamic programming language for technical computing, with syntax that is familiar to users of other technical computing environments. -* **JULIA** version (0.6.0) — Julia is a high-level, high-performance dynamic programming language for technical computing, with syntax that is familiar to users of other technical computing environments. -* **LAMMPS** version (2.0,15May15) — LAMMPS is a particle simulator code. -* **LMOD** version (5.6.2) — Lmod is a Lua based module system that easily handles the MODULEPATH Hierarchical problem. -* **LMOD** version (AnEnvironmentModuleSystem) — Lmod is a Lua based module system that easily handles the MODULEPATH Hierarchical problem. -* **MADGRAPH** version (2.1.2,2.2.2) — MadGraph is a monte carlo generator for particle physics interactions -* **MATLAB** version (2013b,2014a,2014b,2015a,...) — Runtime libraries. The MATLAB Runtime is a standalone set of shared libraries that enables the execution of compiled MATLAB applications or components on computers that do not have MATLAB installed. -* **OCTAVE** version (3.8.1) — Octave is a high-level interpreted language -* **OPENSEES** version (6482) — OpenSees is a software framework for developing applications to simulate the performance of structural and geotechnical sys tems subjected to earthquakes. -* **PANDANA** version (-) — PANDANA performs hundreds of thousands of network queries in under a second (for walking-scale distances) -* **PSIMS** version (-) — A set of packages for needed for the PSIMS (numpy, h5py, netcdf4) -* **PYTORCH** version (-) — PYTORCH Tensors and Dynamic neural networks in Python with strong GPU acceleration. -* **QHULL** version (2012.1) — qhull libraries provide functions for computing convex hull. -* **R** version (3.1.1,3.2.0,3.2.1,3.2.2,3.3.1,3.3.2) — R is a free software environment for statistical computing and graphics. -* **ROOT** version (5.34-32-py34,5.34-32,6.06-02-py34) — ROOT is a high energy physics data analysis framework. -* **ROOT_NUMPY** version (-) — ROOT_NUMPY is a Python extension module that provides an efficient interface between ROOT and NumPy -* **SIESTA** version (3.2) — SIESTA performs efficient electronic structure calculations of molecules and solids. -* **UDUNITS** version (2.2.17) — UDUNITS package supports units of physical quantities. - - - -## Chemistry and Biochemistry -* **ARC-LITE** version (2015) — arc-lite (ARCIMBOLDO) is a package for the protein structure refinement. -* **AUTODOCK** version (4.2.6) — Autodock is a suite of automated docking tools to predict how small molecules bind to a receptor. -* **CASINO** version (2.13.211) — CASINO performs quantum Monte Carlo (QMC) electronic structure calculations. -* **CCP4** version (2015) — arc-lite (ARCIMBOLDO) is a package for the protein structure refinement. -* **ESPRESSO** version (5.1,5.2) — Quantum Espresso is an integrated suite of Open-Source computer codes for electronic-structure calculations and materials modeling at the nanoscale. -* **GAMESS** version (2013) — GAMESS is a general ab initio quantum chemistry package. -* **GROMACS** version (4.6.5,5.0.0,5.0.5.cuda,5.0.5,...) — Gromacs molecular dynamics simulator. Requires fftw/fftw-3.3.4-gromacs -* **NAMD** version (2.9,2.10.cuda,2.10) — NAnoscale Molecular Dynamics (NAMD) program. -* **OPENBABEL** version (2.3.2) — Open Babel is a chemical toolbox designed to speak the many languages of chemical data. -* **OPENBUGS** version (3.2.3) — OpenBUGS is a software package for performing Bayesian inference Using Gibbs Sampling. -* **ORCA** version (3.0.3,4.0.0) — ORCA is a general-purpose quantum chemistry program package. -* **PHENIX** version (1.10) — software suite for the determination of molecular structures from X-Ray and other methods. -* **PSI4** version (0.3.74,1.1) — Psi4 is an open-source suite of ab initio quantum chemistry programs designed for efficient, high-accuracy simulations of a variety of molecular properties. -* **ROSETTA** version (2015,2016-02,2016-32) — The Rosetta software suite includes algorithms for computational modeling and analysis of protein structures. -* **SHELX** version (2015) — SHELX is a set of programs for the determination of small and macromolecular crystal structures. -* **SHELX** version (2015) — SHELX is a set of programs for the determination of small and macromolecular crystal structures. -* **SIMBODY** version (3.5.3) — Simbody is an open source software system for biomechanical modeling, simulation and analysis. -* **SQLITE** version (3.8.11.1) — SQLite is a software library that implements a self-contained, serverless, zero-configuration, transactional SQL database engine. -* **VMD** version (1.9.1) — VMD is a molecular dynamics visualization application. - - - -## Image Analysis -* **ANTS** version (1.9.4,2.1.0) — ANTs is a popular toolkit for medical image registration and segmentation. -* **BLENDER** version (-) — Blender 3D rendering software. -* **ELASTIX** version (2015) — consists of a collection of algorithms that are commonly used to solve (medical) image registration problems -* **FIJI** version (2.0) — Fiji is an image processing package. It can be described as a "batteries-included" distribution of ImageJ (and ImageJ2) -* **FREESURFER** version (5.1.0,5.3.0,6.0.0) — Freesurfer is an open source software suite for processing and analyzing (human) brain MRI images. -* **FSL** version (5.0.8) — FSL is a comprehensive library of analysis tools for FMRI, MRI and DTI brain imaging data -* **GATE** version (7.2) — Gate does simulations of preclinical and clinical scans in Emission Tomography, Transmission Tomography a nd Radiation Therapy. -* **IGRAPH** version (0.7.1) — igraph is a library for creating and manipulating graphs. -* **IMAGEMAGICK** version (7.0.2) — Imagemagick is a set of image manipulation tools. -* **IMAGE_MODULES** version (-) — Image_modules includes a set of Python packages for needed for the image processing - - - -## Bioinformatics -* **ABYSS** version (2.0.2) — ABySS is a de novo, parallel, paired-end sequence assembler that is designed for short reads. -* **BEDTOOLS** version (2.21) — A set of tools for a wide-range of genomics analysis tasks. -* **BLASR** version (1.3.1) — Blasr is a tool to search and align sequences with refinements. -* **BLAST** version (-) — Blast is a tool to search and align gene and protein sequences. -* **BOWTIE** version (2.2.3,2.2.9) — Bowtie is an ultrafast and memory-efficient tool for aligning sequencing reads to long reference sequences. -* **BWA** version (0.7.12,2014) — BWA is a package for mapping low-divergent sequences against a large reference genome. -* **CUFFLINKS** version (2.2.1) — Cufflinks assembles transcripts -* **ENTROPY** version (2017.03.16) — Admixture and the organization of genetic diversity in a butterfly species complex revealed through common and rare genetic variants. -* **ETE2** version (2.3.8) — ETE is a toolkit based on Python framework for the analysis and visualization of trees. -* **HISAT2** version (2.0.3-beta) — HISAT2 is a fast and sensitive alignment program for mapping next-generation sequencing reads (both DNA and RNA) against th e general human population (as well as against a single reference genome). -* **HMMER** version (3.1) — HMMER searches and aligns homologs of protein sequences using hidden Markov models. -* **MOTHUR** version (1.39.0) — mothur is a project that seeks to develop a single piece of open-source, expandable software to fill the bioinformatics nee ds of the microbial ecology community. -* **MRBAYES** version (3.2.2) — MrBayes is a program for Bayesian inference of phylogenetic and evolutionary models. -* **MUMMER** version (3.23) — MUMmer is a system for rapidly aligning entire genomes, whether in complete or draft form -* **MUSCLE** version (3.8.31) — MUSCLE stands for MUltiple Sequence Comparison by Log-Expectation -* **NGSTOOLS** version (2017.03.16) — ngsTools is a collection of programs for population genetics analyses from NGS data, taking into account its statistical un certainty. -* **PBSUITE** version (14.9.9) — PBSuite is a software for long-read sequencing data from PacBio. -* **PRODIGAL** version (2.6.3) — Fast, reliable protein-coding gene prediction for prokaryotic genomes. -* **RAXML** version (8.2.9) — A tool for Phylogenetic Analysis and Post-Analysis of Large Phylogenies -* **RAXML-NG** version (0.5.0beta) — Phylogenetic tree inference tool which uses maximum-likelihood (ML) optimality criterion. -* **SAMTOOLS** version (0.1.17,1.3.1) — Samtools is a set of utilities for working with data in sequence alignment map format. -* **SEQGEN** version (1.3.3) — Seq-Gen is a program that will simulate the evolution of nucleotide or amino acid sequences along a phylogeny -* **SHRIMP** version (2.2.3) — SHRiMP is a software package for aligning genomic reads against a target genome. -* **SRA** version (2.5.4,2.8.0) — The NCBI SRA Toolkit enables reading ("dumping") of sequencing files from the SRA database and writing ("loading") files into the .sra format -* **STRINGTIE** version (1.1.2,1.2.2) — StringTie is a fast and highly efficient assembler of RNA-Seq alignments into potential transcripts. -* **TASSEL** version (5.0) — TASSEL has functionality for association study, evaluating evolutionary relationships, analysis of linkage disequilibrium, principal component analysis, cluster analysis, missing data imputation and data visualization. -* **TOPHAT** version (2.0.13,2.1.1) — TopHat is a fast splice junction mapper for RNA-Seq reads. -* **TRANSABYSS** version (1.5.5) — de novo assembly of RNA-Seq data using ABySS -* **UCLUST** version (2.22) — Uclust offers search and clustering algorithms that are often orders of magnitude faster than BLAST. -* **VIENNARNA** version (2.2) — ViennaRNA Package consists of a C code library and several stand-alone programs for the prediction and comparison of RNA secondary structures. - - - -## Numerical Libraries -* **ATLAS** version (3.10.1,3.10.2) — The ATLAS (Automatically Tuned Linear Algebra Software) provides optimzied libraries for Linear Algebra. -* **BOOST** version (1.50.0,1.56,1.57.0,1.62.0-cxx11,...) — Boost provides free peer-reviewed portable C++ source libraries. -* **CBLOSC** version (1.7.1) — A blocking, shuffling and loss-less compression library -* **EIGEN** version (3.2.10) — Eigen is a C++ template library for linear algebra: matrices, vectors, numerical solvers, and related algorithms. -* **FFTW** version (3.3.4-gromacs,3.3.4) — FFTW is a library for fast fourier transforms. FFTW 3.3.4 library compiled for general use. -* **GEOS** version (3.4.2) — GEOS (Geometry Engine - Open Source) is an API for processing spatial geometry. -* **GMP** version (6.0.0) — Gnu Multi Precision Libraries -* **GSL** version (1.16,2.3) — The GNU Scientific Library (GSL) is a numerical library for C and C++ programmers. -* **IPOPT** version (3.12.6) — Ipopt (Interior Point OPTimizert) is a software package for large-scale nonlinear optimization -* **LAPACK** version (3.5.0,3.6.1) — LAPACK (Linear Algebra Package) is a standard software library for numerical linear algebra. -* **MIXMODLIB** version (3.1) — Mixmod statistical libraries -* **MPC** version (1.0.3) — Gnu Mpc is a C library for arithmatics with arbitrarily high precision of complex numbers. -* **MPFR** version (3.1.3) — MPFR library is a C library for multiple-precision floating-point computations. -* **NCO** version (4.3.0) — The NCO toolkit manipulates and analyzes data stored in netCDF-accessible formats. -* **NETCDF** version (4.2.0) — NetCDF is a set of software libraries supporting array-oriented scientific data. -* **PARI** version (2.7.5) — PARI is a widely used computer algebra system designed for fast computations in number theory -* **SAGA** version (2.2.0) — SAGA is a GIS application -* **SDPA** version (7.3.8) — SDPA (Semi Definite Programming Algorithm) is one of the efficient software package for solving SDPs. -* **SPARSE** version (-) — Sparse works with Python 2.7 and offers a set of packages to deal with sparse matrices -* **SPARSESUITE** version (4.2.1) — SuiteSparse is a suite of sparse matrix algorithms. -* **SUNDIALS** version (2.5) — SUNDIALS (SUite of Nonlinear and DIfferential/ALgebraic equation Solvers) - - - -## Software Libraries, Languages, and Tools -* **ALL-PKGS** version (-) — all-pkgs loads Python with a set of popular packages for scientific computing (Numpy, matplotlib, Scipy, pandas, nltk) -* **ANT** version (1.9.4) — Ant is a Java library and command-line tool is to drive processes described in build files. -* **APR** version (1.5.1) — The Apache Portable Runtime (APR) libraries provide a interface to platform-specific implementations. -* **APRUTIL** version (1.5.3) — APR Util provides cross platform libraries for handling OS related operations. -* **BBFTP** version (3.2.1) — BBFTP is a file transfer software. -* **BINUTILS** version (2.26) — GNU Binutils are a collection of binary tools that do low level operations on binaries. -* **BOTO** version (-) — Boto is a Python package that provides interfaces to Amazon Web Services. -* **BZIP2** version (1.0.6) — bzip2 is a freely available, patent free, high-quality data compressor. -* **CANOPY** version (1.4.1) — Canopy is the next generation of the Enthought Python Distribution (EPD) -* **CCTOOLS** version (4.4.2,5.2.3,5.4.7,6.0.7) — Cooperative Computing Lab is software designed to easily harness large scale distributed systems such as clusters, clouds, and grids. -* **CFITSIO** version (3.37) — CFITSIO is a library of C and Fortran subroutines for reading and writing data files in F ITS (Flexible Image Transport System) data format. -* **CIRCOS** version (0.68) — Circos is a software package for visualizing data and information in a circular layout. -* **CMAKE** version (3.0.1,3.4.1,3.8.0) — CMake is a cross platform make utility -* **CONNECT-CLIENT** version (0.2.1,0.3.0,...) — connect-client is command line tool to use OSG from your local machine. -* **CPAN** version (perl-5.10) — The Comprehensive Perl Archive Network (CPAN) is a repository for Perl programming language -* **CURL** version (7.37.1) — Curl is a command line HTTP client -* **DAKOTA** version (6.4.0) — The Dakota toolkit provides a flexible, extensible interface between analysis codes and iterative systems analysis methods. -* **DENDROPY** version (-) — DendroPy is a Python library for phylogenetic computing -* **DMTCP** version (2.5.0) — DMTCP (Distributed MultiThreaded Checkpointing) transparently checkpoints a single-host or distributed computation in user-space -* **ECTOOLS** version (-) — Ectools is a set of tools for correcting long read assembly data -* **EXPAT** version (2.1.0) — Expat is an XML parser library written in C. -* **FFMPEG** version (0.10.15,2.5.2) — ffmpeg is a complete -* **FPC** version (2.6.4) — Free pascal Free Pascal is a 32 -* **FREETYPE** version (2.5.5) — FreeType 2 provides a simple and easy-to-use API to access font content in a uniform way -* **GCC** version (4.6.2,4.6.4,4.8.1,4.9.2,4.9.3,6.2.0) — GCC is a set of compilers for c -* **GD** version (2.1.1) — GD is an open source code library for the dynamic creation of images by programmers. -* **GFAL** version (7.20) — GFAL2 is a set of file access utilities. -* **GIT** version (1.9.0) — Git is a software change management tool. -* **GLOBUS-CLI** version (-) — globus-cli is a free command line tool that provides an interface to Globus services -* **GNOME_LIBS** version (1.0) — GNOME desktop libraries. -* **HDF5** version (1.8.5-cxx11,1.8.9,1.8.12-cxx11,1.8.12,...) — HDF5 is a library for storing and managing data in flexible -* **IRODS** version (4.2.2) — The Integrated Rule-Oriented Data System (iRODS) is open source data management software. -* **JASPER** version (1.900.1) — reference implementation of the codec specified in the JPEG -* **JAVA** version (7u71,8u25,8u131) — Java -* **JPEG** version (6b,9a) — Libjpeg is a widely used C library for reading and writing jpeg image files. -* **LIBGFORTRAN** version (4.4.7) — -* **LIBICU** version (4.2.1) — International Components for Unicode (ICU) libraries. -* **LIBTIFF** version (4.0.4) — -* **LIBXPM** version (3.5.10) — LibXPM is a package with X.Org, X11, and libXpm runtime libraries -* **LLVM** version (3.6,3.7,3.8.0,5.0.0) — LLVM is a toolkit for the construction of highly optimized compilers, optimizers, and runtime environments. -* **MERCURIAL** version (1.9.1) — Mercurial is a source code management tool. -* **MONO** version (4.2.1) — Mono is a software platform designed to allow developers to easily create cross platform applications. -* **MPLAYER** version (1.1) — Mplayer is an image viewer and editor. -* **MYSQL** version (5.1.73) — mysql client tool that provides command line interface to the mysql database. -* **OPENCV** version (2.4.10) — OpenCV (Open Source Computer Vision Library) is an open source computer vision and machine learning software library. -* **OPENSIM** version (3.3) — OpenSim is an open source software system for biomechanical modeling, simulation and analysis. -* **PAPI** version (5.3.2) — PAPI is a performance profiler. -* **PARSL** version (-) — Parsl is a parallel scripting library that enables easy parallelism and workflow design. -* **PBSUITE_PKGS** version (-) — PBSuite loads python and adds numpy, h5py, and pysam packages. -* **PCRE** version (8.35) — PCRE is perl compatible regular expression library. -* **PEGASUS** version (4.4.2-image_tools,4.5.3,4.6.0dev,...) — Pegasus is a workflow management system to automate, recover, and debug scientific computations. -* **POPPLER** version (0.24.1,0.32) — Poppler is a PDF rendering library based on the xpdf-3.0 code base -* **POVRAY** version (3.7) — The Persistence of Vision Raytracer is a high-quality, Free Software tool for creating stunning three-dimensional graphics. -* **PROJ** version (4.9.1) — Cartographic projection library -* **PROOT** version (2014) — PRoot is a user-space implementation of chroot -* **PROTOBUF** version (2.5) — Protocol buffers are Google's language-neutral, platform-neutral, extensible mechanism for serializing structured data – think XML, but smaller, faster, and simpler. -* **PYTHON** version (2.7,3.4,3.5.2) — Python is a programming language that lets you work quickly and integrate systems more effectively. -* **RUBY** version (2.1) — Ruby is a dynamic -* **RUCIO** version (1.6.6) — Distributed Data Management (DDM) system services -* **S3CMD** version (-) — S3cmd is a free command line tool and client for uploading, retrieving and managing data in Amazon S3 and other cloud stora ge service providers that use the S3 protocol -* **SCONS** version (2.3.4) — SCons is a software construction tool and is a superior alternative to the classic "Make" build tool. -* **SERF** version (1.37) — Serf is a library that provides high performance http client functions. -* **SETTARG** version (5.6.2) — The settarg module dynamically and automatically updates "$TARG" and a host of other environment variables. -* **SNAPPY** version (1.1.3) — Snappy is a compression and decompression library. -* **STASHCP** version (2.6,4.3.0,4.3.1) — Stashcp is a front-end to XRootD caching for OSG Connect Stash. -* **SUBVERSION** version (1.8.10) — Subversion is a source control revision tool. -* **SWIFT** version (0.94.1,0.96.2) — Swift is a parallel scripting language. -* **TCL** version (8.6.2) — Tcl (Tool Command Language) is a very powerful but easy to learn dynamic programming language. -* **TCSH** version (6.20.00) — tcsh is a Unix shell based on and compatible with the C shell (csh). It is essentially the C shell with programmable command-line completion, command-line editing, and a few other features. -* **UNIXODBC** version (2.3.2) — The unixODBC Project goals are to develop and promote unixODBC to be the definitive standard for ODBC on non MS Windows platforms. -* **VALGRIND** version (3.10) — Valgrind can detect memory management and threading bugs -* **WGET** version (1.15) — Wget is a command line HTTP client -* **WXGTK** version (3.0.2) — wxWidgets is a C++ library that lets developers create applications for Windows, Mac OS X, and Linux. -* **XROOTD** version (4.1.1,4.2.1) — The XROOTD project aims at giving high performance, scalable fault tolerant access to data repositories of many kinds. -* **XZ** version (5.2.2) — XZ Utils is free general-purpose data compression software with a high compression ratio. -* **ZLIB** version (1.2.8) — Zlib is a library that provides functions for compression and decompression of data - - diff --git a/archive/osg-quick-connect.md b/archive/osg-quick-connect.md deleted file mode 100644 index c3030379..00000000 --- a/archive/osg-quick-connect.md +++ /dev/null @@ -1,18 +0,0 @@ -[title]: - "OSG Quick Connect" - -[TOC] - -The requirements to connect your cluster to the OSG using this method are minimal: - - 1. Create a normal user account for the OSG Connect service. - 2. Provide ssh access to this account via public ssh key. - 3. Add the OSG Connect user to the FUSE user group (this is to provide access to the network-based OSG software repository). - 4. (Optional) Install a local http proxy service (Squid) providing about 1 TB of disk storage to cache software libraries locally at your site. - 5. (Optional) Install a data cache service on a node providing 10-48 TB of local cache for data intensive workloads. - -Usually a couple of phone calls between the technical administrators from both teams is needed to complete the task. We have successfully used this approach to connect to several campus based clusters, including those at Syracuse University (OrangeGrid), Clemson University (Palmetto). We have also applied this method for the [ATLAS Connect campus grid][atlas], which routes ATLAS Monte Carlo simulations of collisions at the CERN LHC to clusters at Indiana University (Karst), TACC (Stampede), Cal State Fresno (ATLAS Tier3 Center), University of Illinois (Illinois Campus Cluster), and Harvard University (Odyssey). - -Drop us a note at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) if this option is of interest to you. - -[atlas]: http://connect.usatlas.org - diff --git a/archive/overview/welcome.md b/archive/overview/welcome.md deleted file mode 100644 index c5432cfd..00000000 --- a/archive/overview/welcome.md +++ /dev/null @@ -1,52 +0,0 @@ -[title]: - "A Welcome from the Research Facilitation Team" - -Greetings from the OSG Research Facilitation Team ------------------------------------------------------- - -On behalf of the OSG Research Facilitation team, it is a pleasure to welcome -you to the help desk portal! - - -The help desk is primarily intended for users, i.e. students, postdocs, -faculty, and staff, doing science using the OSG. We also provide technical -documentation and discussion forums for campus HPC resource providers and -systems administrators. In particular, we support campus-based [cyberinfrastructure -facilitators](), including [options for connecting -university-based HPC centers]() to -OSG, and [XSede champions]() on -all matters relating to OSG, [OSG Connect](), [CI -Connect](), and distributed high throughput computation. - - -Our extended support team consists of systems experts, computer scientists, and -Ph.D. level scientists experienced in distributed high throughput scientific -computation, data-intensive computing, and wide area networking.  There is very -little we won't be able to handle, or point you to the right expert who can. - - -We hope that you find our [knowledge -base]() and educational -materials a useful resource. You can [engage with -us]() and others in -the [community -forums](). Most -importantly, please [report any -problems]() you -encounter, and feel free to send questions or feature requests. If needed our -support team can schedule a consultation with you, your research team or campus -HPC center support staff.   - - -We look forward to hearing from you! - - -Warm regards, - -Rob Gardner - -[Rob Gardner]() - - -OSG Research Facilitation and Campus Grids Area Coordinator - -[@rwg]() diff --git a/archive/scaling-up.md b/archive/scaling-up.md deleted file mode 100644 index 36d4156d..00000000 --- a/archive/scaling-up.md +++ /dev/null @@ -1,89 +0,0 @@ -[title]: - "Scale Up your Workflow on OSG Connect" -[TOC] - -## Background - -Much of OGS's computing power comes from the ability to run a large number -of jobs simulateously. Many HTC-friendly workflows involve running the same -job multiple times with no or minor changes between them. Here, we will -talk about how to submit multiple jobs at once using the `queue` command as -well as some important considerations and tips for scaling up your HTCondor jobs. -For a hands-on example of these concepts, checkout the [OSG Connect Quickstart] -(https://support.opensciencegrid.org/support/solutions/articles/5000633410-osg-connect-quickstart). - -## Things to Consider - -In order to avoid confusion, let's review some HTCondor terminiology: - - When you queue a single submit script using the `condor_submit` command, this is -called a **cluster** and can consist of one more individual jobs. - - Each individual job within a cluster is called a **process**. Therefore, each -*cluster* can be made up of one or more *processees*. Each proceess lands on its -own job slot and is handled individually from the other processes in the cluster. - -### Handling Output - -HTCondor has built in features that make submitting multiple jobs with the same -submit script possible and easy. However, this means that you will need to ensure -that the output created by each individual process will be uniquely identified so -it is not overwritten by the output from other processes. - -In order to uniquely identify each job output, you can use cluster and process IDs -in the submit script. For example, to route each job log into it's own file, the -following line would ensure that each job's log file would end up with a unique file -name: - - log = job.$(Cluster).$(Process).log - -For a working example on how to use this in a submit file, and additional applications, -please see [Job 3 of the OSG Connect Quickstart guide] -(https://support.opensciencegrid.org/support/solutions/articles/5000633410-osg-connect-quickstart#job-3-submitting-jobs-concurrently) - -### Submitting Multiple Jobs with a Single Script - -The easiest way to submit multiple jobs from a single submit script is to follow -queue line in the submit script with a number. For example, if we edit the `queue` -line to read: - - queue 100 - -HTCondor will then submit 100 **processes** within 1 **cluster**. Each of these -*processes* will have the same *cluster* ID. but will each have their own unique -*process* ID. Checkout the Handling Output section for more information on how -to use these IDs to uniquely tag output and job logs. - -The queue command can also be used to queue a list of values from an input file -or list. For more details on the different possibilities for the `queue` command, -visit [UW-Madison CHTC's Documentation](http://chtc.cs.wisc.edu/multiple-jobs.shtml) - - -### Disk Usage - -Each OSG Connect user is granted 100 GB of storage in their `/home` directory and -500 GB of storage in their `/public` directories. This may seem like a lot, but -when running many jobs this can fill quickly. As the number of jobs submitted -simultaneously increases, so will storage usage on the connect login nodes. For -example, if a single job creates 1 GB of output, running 1000 such jobs will -generate approximately 1 TB of output. - -To prevent errors or workflow interruption, carefully consider disk usage for your jobs. -By default, HTCondor will transfer back any new files created inside the jobs working -directory. Adding clean up steps to remove intermediate and/or unnecessary files after -your analysis is complete will help reduce the amount of space used and help prevent -running into that quota limit. - -### Debugging Jobs - -Testing your jobs and workflows is always strongly recommended, but it is of even -more importance when submitting a large number of jobs. Start out with running a -single job. When it completes with no errors, then try a small number - about 10. -Use this to ensure that the jobs complete successfully, produce the desired -output, and do not conflict with each other. Once you are confident that the jobs -will complete as desired, submit your large workflow. - - -### Workflow Management - -To help manage complicated workflows, consider a workflow manager such as [Pegasus] -(https://support.opensciencegrid.org/support/solutions/articles/5000639789-pegasus) or -[DAGman](https://research.cs.wisc.edu/htcondor/dagman/dagman.html), HTCondor's built -in meta-scheduler. diff --git a/archive/software-transfer-via-htcondor-or-http.md b/archive/software-transfer-via-htcondor-or-http.md deleted file mode 100644 index 77c82880..00000000 --- a/archive/software-transfer-via-htcondor-or-http.md +++ /dev/null @@ -1,64 +0,0 @@ - -[title]: - "Software Transfer via HTCondor or HTTP" - - -# Software Transfer via HTCondor - -## Overview - -For some OSG Connect users, it may be necessary or advatageous to install your own software. This guide will -describe steps and important considerations for transferring your installed software via the HTCondor submit file. - -## Important Considerations - -How you transfer your software via HTCondor will depend on the size and location of the executable binary or tar archive -that needs to be transferred along with your jobs. Only software files <1GB should be transferred via HTCondor using the steps described below. Please see [Introduction to Data Management on OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/12000002985) for more information regarding our policies for staging files (data, software, etc.) in OSG Connect. - -Keep in mind that HTCondor will always transfer the script or binary that is specified as the `executable` in your submit -file. Thus, it is not necessary to follow the steps below if your software is the designated executable in your submit file. - -The steps and considerations described below also apply to transferring data, and other files, as described in the [Transferring Data With HTCondor](https://support.opensciencegrid.org/support/solutions/articles/5000639787) guide. - -## Transferring Software From `/home` - -Software executables and tar archives that are <100MB should be staged in your `/home` directory. To transfer software -from your `/home` directory use the `transfer_input_files` statement in your HTCondor submit file. For example: - - transfer_input_files = path/to/my_software.tar.gz - -> When using `transfer_input_files` to transfer files located in `/home`, keep in mind that the path to the file is -> relative to the location of the submit file. If you have software located in a different `/home` subdirectory, -> we recommend specifying the full path those files, which is also a matter of good practice, for exmaple: -> ``` -> transfer_input_files = /home/username/path/to/my_software.tar.gz -> ``` -> Where `username` refers to your OSG Connect username. - -In addition to any software needed for your jobs, be sure to also include other required input files, for example: - - transfer_input_files = path/to/my_software.tar.gz, path/to/my_input.csv - -## Transferring Software from `/public` - -Software executables and tar archives that are >100MB should be staged in your `/public` directory. Only software -located in `/public` that is <1GB should be transferred using `transfer_input_files` in your submit file. If you have software that is >1GB please plan to use [StashCache](https://support.opensciencegrid.org/support/solutions/articles/12000002775). - -Transferring software from `/public` using the `transfer_input_files` statement in your HTCondor submit file will occur via -an HTTP connection, for example: - - transfer_input_files = http://stash.osgconnect.net/public/username/path/my_software.tar.gz - -Where `username` refers to your OSG Connect username. - -## Transferring Precompiled Binaries From The Web - -Precompiled binaries that are available on the web, and are <1GB in size, may also be transferred via HTTP using `transfer_input_files`. For example, Blast precompiled binaries are available from the NCBI at https://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/LATEST/ncbi-blast-2.10.0+-x64-linux.tar.gz. We can use HTCondor to transfer this tar archive of precompiled binaries (which is 222MB in size): - - transfer_input_files = https://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/LATEST/ncbi-blast-2.10.0+-x64-linux.tar.gz - -> If using precompiled binaries, be sure to download those that have been precompiled for a Linux operating system. It will -> also be necessary to first confirm that these precompiled binaries will work in OSG before scaling up to multiple job -> submissions. - - - diff --git a/archive/xsede/osg-xsede.md b/archive/xsede/osg-xsede.md deleted file mode 100644 index 2bd4f668..00000000 --- a/archive/xsede/osg-xsede.md +++ /dev/null @@ -1,353 +0,0 @@ -## Overview - -The [OSG](https://osg-htc.org/) promotes science by: - - * enabling a framework of distributed computing and storage resources - - * making available a set of services and methods that enable better access - to ever increasing computing resources for researchers and communities - - * providing resource sharing principles and software that enable distributed - high throughput computing (DHTC) for users and communities at all scales. - -The OSG facilitates access to DHTC for scientific research -in the US. The resources accessible through the OSG are -contributed by the community, organized by the OSG, and -governed by the [OSG -Consortium](http://www.opensciencegrid.org); an overview is available at -[An Introduction to OSG](http://osg-docdb.opensciencegrid.org/0008/000839/004/OSG%20Intro%2 -0v23.pdf). In 2017, OSG is comprised -of about 126 institutions with ~120 active sites that collectively -support usage of ~4,000,000 core hours per day. Up-to-date usage metrics -are available at the [OSG Usage Display](https://display.opensciencegrid.org/). - -Cores that are not being used at any point in time by -the owning communities are made available for shared use by other -researchers; this usage mode is called opportunistic access. OSG -supports XSEDE users by providing a Virtual Cluster that forms an -abstraction layer to access the opportunistic cores in the distributed -OSG infrastructure. This interface allows XSEDE users to view the OSG as -a single cluster to manage their jobs, provide the inputs and retrieve -the outputs. XSEDE users access the OSG via the OSG-XSEDE login host -that appears as a resource in the XSEDE infrastructure. - -## Computation that is a good match for OSG - -High throughput workflows with simple system and -data dependencies are a good fit for OSG. The -[Condor manual](http://research.cs.wisc.edu/condor/manual/current/) has an overview of [high throughput computing](http://research.cs.wisc.edu/condor/manual/current/1_Overview.html). - -Jobs submitted into the OSG Virtual Cluster will be executed on machines -at several remote physical clusters. These machines may differ in terms -of computing environment from the submit node. Therefore it is important -that the jobs are as self-contained as possible by generic binaries -and data that can be either carried with the job, or staged on demand. -Please consider the following guidelines: - - - * Software should preferably be **single threaded**, using - **less than 2 GB memory** and each invocation should - **run for 1-12 hours**. Please contact the support listed - below for more information about these capabilities. System level check - pointing, such as the HTCondor standard universe, is not available. - Application level check pointing, for example applications writing out - state and restart files, can be made to work on the system. - - * Compute sites in the OSG can be configured to use pre-emption, which - means jobs can be automatically killed if higher priority jobs enter - the system. Pre-empted jobs will restart on another site, but it is - important that the jobs can handle multiple restarts. - - * Binaries should preferably be statically linked. However, dynamically - linked binaries with standard library dependencies, built for a 64-bit - Red Hat Enterprise Linux (RHEL) 6 machines will also work. Also, - interpreted languages such as Python or Perl will work as long as there - are no special module requirements. - - * Input and output data for each job should be < 10 GB to allow them - to be pulled in by the jobs, processed and pushed back to the submit - node. Note that the OSG Virtual Cluster does not currently have a global - shared file system, so jobs with such dependencies will not work. - - * Software dependencies can be difficult to accommodate unless the software - can be staged with the job. - - -The following are examples of computations that are **not** good -matches for OSG: - - * Tightly coupled computations, for example MPI based communication, will - not work well on OSG due to the distributed nature of the infrastructure. - - * Computations requiring a shared file system will not work, as there is - no shared filesystem between the different clusters on OSG. - - * Computations requiring complex software deployments are not a good fit. - There is limited support for distributing software to the compute - clusters, but for complex software, or licensed software, deployment - can be a major task. - - -## System Configuration - -The OSG Virtual Cluster is a Condor pool overlay on top of OSG -resources. The pool is dynamically sized based on the demand, the -number of jobs in the queue, and supply, resource availability at the -OSG resources. It is expected that the average number of resources, -on average, available to XSEDE users will be in the order of 1,000 -cores. - -One important difference between the OSG Virtual Cluster and most of -the other XSEDE resources is that the OSG Virtual Cluster does not have -a shared file system. This means that your jobs will have to bring -executables and input data. Condor can transfer the files for you, -but you will have to identify and list the files in your Condor job -description file. - -Local storage space at the submission site is controlled by -quota. Your home directory has a quota of 10 GBs and your work directory -`/local-scratch/$USER` has a quota of 1 TB. There are no -global quotas on the remote compute nodes, but expect that about 10 GBs -are available as scratch space for each job. - - -## System Access - -The preferred method to access the system is via the XSEDE Single -Sign On (SSO) Hub. Please see the [sso documentation](https://portal.xsede.org/single-sign-on-hub) -for details. - -A secondary access methor is to use SSH public key authentication. -Secure shell users should feel free to append their public RSA key -to their `~/.ssh/authorized_keys` file to enable access -from their own computer. Please login once via the SSO Hub to install your -key. Please make sure that the permissions on the .ssh directory and -the authorized_keys file have appropiate permissions. For example - - $ chmod 755 ~/.ssh - $ chmod 644 ~/.ssh/authorized_keys - - -## Application Development< - -Most of the clusters in OSG are running Red Hat Enterprise Linux (RHEL) -6 or 7, or some derivative thereof, on an x86_64 architecture. For -your application to work well in this environment, it is recommend -that the application is compiled on a similar system, for example on -the OSG Virtual Cluster login system: `submit-1.osg.xsede.org -`. It is also recommended that the application be statically -linked, or alternatively dynamically linked against just a few standard -libraries. What libraries a binary depends on can be checked using the -Unix `ldd` command-line utility: - - $ ldd a.out - a.out is a static executable - - -In the case of interpreted languages like Python and Perl, applications -have to either use only standard modules, or be able to ship the modules -with the jobs. Please note that different compute nodes might have -different versions of these tools installed. - -A good solution to complex software stack is Singularity containers -which are described below. - - -## Running Your Application - -The OSG Virtual Cluster is based on Condor and the -[Condor manual](http://research.cs.wisc.edu/condor/manual/current/) -provides a reference for command line tools. The commonly used tools -are: - - * `**condor_submit**` - Takes a Condor submit file and adds the job to the queue - - * `**condor_q**` - Lists the jobs in the queue. Can be invoked with your - username to limit the list of jobs to your jobs: `condor_q $USER` - - * `**condor_status**` - Lists the available slots in the system. - Note that this is a dynamic list and if there are no jobs in the system, - `condor_status` may return an empty list - - * `**condor_rm**` - Remove a job from the queue. If you are running a DAG, - please `condor_rm` the id of the DAG to remove the whole workflow. - -### Submitting a Simple Job - -Below is a basic job description for the Virtual Cluster. - - universe = vanilla - - # specifies the XSEDE project to charge the job usage to - this is a - # required attribute for all jobs submitted on the OSG-XSEDE resource - +ProjectName = "TG-NNNNNN" - - # requirements is an expression to specify machines that can run jobs - requirements = OSGVO_OS_STRING == "RHEL 6" && Arch == "X86_64" && HAS_MODULES == True - request_cpus = 1 - request_memory = 2 GB - request_disk = 10 GB - - executable = /bin/hostname - - arguments = -f - - should_transfer_files = YES - WhenToTransferOutput = ON_EXIT - - output = job.out - error = job.err - log = job.log - - notification = NEVER - - queue - - - Create a file named `job.condor` containing the above text and then run: - - $ condor_submit job.condor - -You can check the status of the job using the `condor_q` command. - -**Note:** The Open Science Pool is a distributed resource, and there -will be minor differences in the compute nodes, for example in what -system libraries and tools are installed. Therefore, when running -a large number of jobs, it is important to detect and handle job -failures correctly in an automatic fashion. It is recommended that your -application uses non-zero exit code convention to indicate failures, and -that you enable retries in your Condor submit files. For example: - - # stay in queue on failures - on_exit_hold = (ExitBySignal == True) || (ExitCode != 0) - - # retry job 3 times, pause 1 hour between retries - periodic_release = (NumJobStarts < 3) && ((CurrentTime - EnteredCurrentStatus) > (60*60)) - - -### Job Example: Java with a job wrapper - -The following is an example on how to run Java code on Open -Science Grid. The job requirements specifies that the job requires -Java, and a wrapper script is used to invoke Java. - -File: `condor.sub` - - universe = vanilla - - # specifies the XSEDE project to charge the job usage to - this is a - # required attribute for all jobs submitted on the OSG-XSEDE resource - +ProjectName = "TG-NNNNNN" - - # requirements is an expression to specify machines that can run jobs - requirements = HAS_JAVA == True - - # stay in queue on failures on_exit_hold = (ExitBySignal == True) || (ExitCode != 0) - - # retry job 3 times, pause 1 hour between retries - periodic_release = (NumJobStarts < 3) && ((CurrentTime - EnteredCurrentStatus) > (60*60)) - - executable = wrapper.sh - - should_transfer_files = YES - WhenToTransferOutput = ON_EXIT - - # a list of files that the job needs - transfer_input_files = HelloWorld.jar - - output = job.out - error = job.err - log = job.log - - notification = NEVER - - queue - -File: `wrapper.sh` - - #!/bin/bash - - set -e - - java HelloWorld - - -## Sample Jobs and Workflows - -A set of sample jobs and workflows can be found under -`/opt/sample-jobs` on the submit-1.osg.xsede.org host. README -files are included with details for each sample. - -`/opt/sample-jobs/single/` contains a single Condor job -example. Single jobs can be used for smaller set of jobs or if the job -structure is simple, such as parameter sweeps. - -A sample-app package -([sampleapp.tgz](http://www.ncsa.illinois.edu/People/yanliu/codes/sampleapp.tgz)) -is available in the `/opt/sample-jobs/sampleapp/` directory. This package shows -how to build a library and an executable, both with dynamic and static -linking, and submit the job to a set of different schedulers available -on XSEDE. The package includes submit files for PBS, SGE and Condor. - -[DAGMan](http://research.cs.wisc.edu/condor/manual/current/2_10DAGMan_Applications.html) -is a HTCondor workflow tool. It allows the -creation of a directed acyclic graph of jobs to be run, and then DAGMan -submits and manages the jobs. DAGMan is also useful if you have a -large number of jobs, even if there are no job inter-dependencies, as -DAGMan can keep track of failures and provide a restart mechanism if -part of the workflow fails. A sample DAGMan workflow can be found in -`/opt/sample-jobs/dag/` - -[Pegasus](https://pegasus.isi.edu) is a workflow system that -can be used for more complex workflows. It plans abstract workflow -representations down to an executable workflow and uses Condor DAGMan -as a workflow executor. Pegasus also provides debugging and monitoring -tools that allow users to easily track failures in their workflows. -Workflow provenance information is collected and can be summarized with -the provided statistical and plotting tools. A sample Pegasus workflow -can be found in `/opt/sample-jobs/pegasus/` . - - -## Singularity Containers - -Singularity containers provide a great solution for complex software -stacks or OS requirements, and OSG has easy to use integrated support -for such containers. Full details can be found in the -[Singularity Containers](https://support.opensciencegrid.org/support/solutions/articles/12000024676). - - -## Distribute data with Stash - -Stash is a data solution used under -[OSGConnect](https://osgconnect.net/), but is partly also available -for OSG XSEDE users. Files under `/local-scratch/public_stash/` will -automatically synchronize to the globally available -`/cvmfs/stash.osgstorage.org/user/xd-login/public/` file system, which -is available to the majority of OSG connected compute nodes. This is a great -way to distribute software and commonly used data sets. To get started, create -your own sub directory: - - $ mkdir -p /local-scratch/public_stash/$USER - -Now, populate that directory with the software and data required for your jobs. -The synchronization can take couple of hours. You can verify the data has -reached the /cvmfs system by using `ls`: - - $ ls /cvmfs/stash.osgstorage.org/user/xd-login/public/ - -To steer your jobs to compute nodes which can access the file system, add -`HAS_CVMFS_stash_osgstorage_org == True` to your job -requirements. For example: - - requirements = OSGVO_OS_STRING == "RHEL 6" && Arch == "X86_64" && HAS_MODULES == True && HAS_CVMFS_stash_osgstorage_org == True - -Once a job starts up on such a compute node, the job can read directly -from `/cvmfs/stash.osgstorage.org/user/xd-login/public/` - - -## How to get help using OSG - -XSEDE users of OSG may get technical support by -contacting OSG Research Facilitation staff at email -. -Users may also contact the [XSEDE helpdesk](https://portal.xsede.org/help-desk). diff --git a/assets/ConnectBook.png b/assets/ConnectBook.png deleted file mode 100644 index 50033558..00000000 Binary files a/assets/ConnectBook.png and /dev/null differ diff --git a/assets/osg-swc-300ppi.jpg b/assets/osg-swc-300ppi.jpg deleted file mode 100644 index 071bd4f1..00000000 Binary files a/assets/osg-swc-300ppi.jpg and /dev/null differ diff --git a/assets/rob-signature.jpg b/assets/rob-signature.jpg deleted file mode 100644 index 1fcd1249..00000000 Binary files a/assets/rob-signature.jpg and /dev/null differ diff --git a/attachments/osg.bib b/attachments/osg.bib deleted file mode 100644 index 9c5db704..00000000 --- a/attachments/osg.bib +++ /dev/null @@ -1,45 +0,0 @@ -@inproceedings{osg07, - title = {The open science grid}, - author = { - Pordes, Ruth - and Petravick, Don - and Kramer, Bill - and Olson, Doug - and Livny, Miron - and Roy, Alain - and Avery, Paul - and Blackburn, Kent - and Wenaus, Torre - and W{\"u}rthwein, Frank - and Foster, Ian - and Gardner, Rob - and Wilde, Mike - and Blatecky, Alan - and McGee, John - and Quick, Rob - }, - doi = {10.1088/1742-6596/78/1/012057}, - booktitle = {J. Phys. Conf. Ser.}, - volume = {78}, - series = {78}, - pages = {012057}, - year = {2007}, -} - -@inproceedings{osg09, - title = {The pilot way to grid resources using glideinWMS}, - author = { - Sfiligoi, Igor - and Bradley, Daniel C - and Holzman, Burt - and Mhashilkar, Parag - and Padhi, Sanjay - and Wurthwein, Frank - }, - doi = {10.1109/CSIE.2009.950}, - booktitle = {2009 WRI World Congress on Computer Science and Information Engineering}, - volume = {2}, - series = {2}, - pages = {428--432}, - year = {2009}, -} diff --git a/drafts/StashCache-DataDelivery.md b/drafts/StashCache-DataDelivery.md deleted file mode 100644 index 0a0faceb..00000000 --- a/drafts/StashCache-DataDelivery.md +++ /dev/null @@ -1,127 +0,0 @@ -[title]: - "StashCache: a Data Delivery Service for OSG" -[TOC] - -## Introduction - -StashCache is a solution under development which transparently caches data near compute sites. It is available for beta testers. - -The basic idea is one places data into an origin server (for OSG Connect users, this is Stash) and uses a command stash-cp within the job to access data from the remote site worker node. - -## Usage - - stashcp [-d] [-r] [-h] -s [-l ] - - -d: show debugging information - -r: recursively copy - -h: show this help text - - --closest: return closest cache location - - Exit status 4 indicates that at least one file did not successfully copy over. - Exit status 1 indicates that the WantsStashCache classad was not present in job environment. - -## Overview of algorithm - -All the functions are defined above everything else, so the code is not simple to read. STASHCP itself starts "running" after the comment line. - -## LOGIC TO RUN STASHCP - - -### Startup -Before any downloading happens, STASHCP checks for relevant classads, loads xrootd, initializes information variables and processes arguments. It also determines the closest local cache. - -### Classad -In order to make sure that StashCache jobs are only sent to those sites that can handle them, users are required to add a StashCache classad to their jobs: +WantsStashCache = true -If that classad is not present, STASHCP will stop, return 1, and print out an error message. -Note: This classad is not required if STASHCP is being run interactively, i.e., outside of a job environment. -Information variables -The information variables are shell arrays holding strings corresponding to the start and end times of downloads, as well as the file or folder name and the size. At the end of STASHCP, the information variables will be turned into strings and set as classads for the job. Right now, because HTCondor limits classads to 1024 characters, the strings are truncated. -When a directory is downloaded, the information variables will be updated as if the directory were a single unit - the filename variable will hold just the directory path, the size will reflect the size of the directory, and the download times will reflect the downloading time for the directory. This improves legibility and reduces space requirements. If a user downloads a directory mydir, the information variable for filename will hold mydir+. If a user downloads mydir/, the information variable for filename will hold mydir/+. - -### Arguments -STASHCP only requires a single argument, the source. Every other argument is optional. - -s : is the comma-delimited list of files and/or directories that the user wishes to download. The path of a given file will be of the form user//public/. - -l : is the location within the job directory that the user wishes to download their files/directories to. This can only be a single location. If the directory does not exist when STASHCP is run, STASHCP will fail and return 1. - -d : if this flag is present, print debugging information. - -r : if this flag is present, download recursively (all subfolders). - -### Local cache -Simply calls setStashCache.sh and holds the result. The called code uses geoip information and the caches.json file to determine to closest active cache. - -### Main Loop -This loop iterates over every file/directory that the user wishes to download. Each of these items is referred to as a source item. -Before any downloading occurs, STASHCP checks to see if the source currently being examined is a file or a directory. If the source is a file, doStashCpSingle is called; otherwise, doStashCpDirectory is called. The source item is downloaded to the user-specified directory. In both cases, a flag is set to indicate that the information variables are to be updated with respect to the source being downloaded. - -### Location Logic -Due to numerous problems with trying to do this recursively, I decided to take a more direct approach and have STASHCP use the full source file path to direct location logic. -Simply speaking, STASHCP swaps out the prefix in the full path of the source item (the source prefix, denoted in the code as $prefixRm) for the prefix that the downloaded item should have in the job space). That prefix is set per original source argument in the main loop. The job prefix is a little more complicated, consisting of a base directory (the user-specified location) and a local path (defined as the source path with $prefixRm removed from the beginning). The $prefixRm and $baseDir variables are defined at the source level within the main loop, and the $localPath variable is defined individually for each file within doStashCpSingle or doStashCpDirectory. The ultimate command is this: xrdcp $xrdargs -f $sourcePrefix://$downloadFile $baseDir/$localPath -This can be better understood in the examination of a few different cases. In every case, the user-specified location is the home directory, and so $loc=. $sourcePrefix is just the location of the closest cache. The xrdcp commands are simplified for legibility. -The source item is a file user/jsmith/public/data.dat. - - $downloadFile = user/jsmith/public/data.dat - $prefixRm = user/jsmith/public/ - $baseDir = . - $localPath = data.dat - Command: xrdcp $sourcePrefix://user/jsmith/public/data.dat ./data.dat -The source item is a directory user/jsmith/public/folderA, and STASHCP is downloading a file A1.dat from folderA. - - $downloadFile = user/jsmith/public/folderA/A1.dat - $prefixRm = user/jsmith/public/folderA/ - $baseDir = ./folderA - $localPath = A1.dat - Command: xrdcp $sourcePrefix://user/jsmith/public/folderA/A1.dat folderA/A1.dat -The source item is a directory user/jsmith/public/folderB, and STASHCP is downloading a file beta1.dat from one of its subdirectories beta. - - $downloadFile = user/jsmith/public/folderB/beta/beta1.dat - $prefixRm = user/jsmith/public/folderB - $baseDir = ./folderB - $localPath = beta/beta1.dat - Command: xrdcp $sourcePrefix://user/jsmith/public/folderB/beta/beta1.dat folderB/beta/beta1.dat - -### doStashCpSingle -This is where all the downloading actually happens. -This function can take two arguments. The first one, which is required, is the name of the file to be downloaded. If the second argument is present, the function will update the information variables with information about this particular file download. If the second argument is not present, no updating of information variables occurs (such as when the file being downloaded is but one member of a larger directory being downloaded). -doStashCpSingle depends on another script, downloading_timeout.sh for all timeout logic. -doStashCpSingle attempts to run xrdcp from the local cache, keeping track of start and end time. If this pull is not successful, a second xrdcp from local is attempted. Should that pull fail, STASHCP fails over to pulling from the trunk, and failover information is updated. If the final pull is not successful, failure information is updated. However, if any pull is successful, the usual information variables are updated. Furthermore, information about the successful download is uploaded to our Hadoop machine. - -### doStashCpDirectory -Like doStashCpSingle, this function can take two arguments - the first is the directory to be downloaded, and the second is a flag to let the function know if it should update information. Information should not be updated if the directory being currently downloaded is a subdirectory of a larger directory being downloaded. -doStashCpDirectory iterates through the contents of the input directory. If an item is a file, doStashCpSingle is called on the item. If the item is a directory, and the recursive flag -r has been set, then the appropriate directory is created (look for the ## creating local directory for subfolder comment) and doStashCpDirectory is called on the item recursively. The time it takes to iterate and download all of the contents is recorded and, if appropriate, updated to the information variables. -Finishing -The information variables are chirped, as described above. -If any single download failed, STASHCP itself has failed. In this case, STASHCP returns 4. - -### downloading_timeout.sh -This is a separate script that tracks the size of a file being downloaded, and cuts off the download command if the file's size does not change enough in a given period of time. This helps to tell the difference between a stalled download and a slow download. - - Usage: downloading_timeout.sh -t -d -f -s - -The script waits until $file exists, at which point it stores `$prevSize`, the size of the file in bytes. Every `$timeoutseconds`, the script computes the expected size of the file using `$prevSize, $expSize and $diff: $wantSize := min($prevSize + $diff, $expSize)`. Thus, the script is asking for at least an increase of `$diff bytes`, `unless $prevSize + $diff > $expSize`. If the file size has not increased appropriately, the script shuts down the downloading command. -It is recommended that $timeout not be set to 1 second, as tests showed that download times varied on a second-by-second level. A better value is in the range of 3-10 seconds. These variables are set in STASHCP. - -## Known issues and concerns - Relies on GEOIP to find closest cache - GEOIP doesn't always work - Closest cache isn't necessarily the best cache - No checking for "next-closest" cache if closest cache is temporarily down and status is not yet reflected in caches.json - Call could be simpler, without requiring the use of flags for every argument - Want: stashcp - Have: stashcp -s -l - Static hard-coded number of attempts to pull from cache (2) and trunk (1) - Does not take type of error/failure into account - If the closest cache is the trunk, then the algorithm will attempt to pull from the trunk 3 times, instead of 1 or 2 - Relies explicitly on the trunk being up in order to run critical steps - In particular, relies on trunk being up in order to get size of file or to get contents of directory - Could lead to unnecessary failure when the trunk is down but files are already present and accounted for on closest cache - Does not do anything else if STASHCP fails - Maybe if STASHCP from the local cache and from the trunk fail, should try wget? Hard to think of a situation when wget would work but stashcp from the trunk would not. - Relatedly, STASHCP returns failure even if only one file isn't downloaded. We should consider breaking out of STASHCP immediately upon failure. - Error messages are not informative for users - Messages written only for the coder to use - Hadoop messages still rely on timeout utility, which may not be available on all sites. - On the other hand, the sites that don't have timeout are likely to be troublemakers in other ways, and would not support STASHCP for many other reasons. - STASHCP assumes the following: - The Owner ClassAd is present and not Undefined - The ProjectName ClassAd exists - STASHCP does not transfer files back from the worker node - No record of whether the file was new to the cache it was pulled from or not diff --git a/drafts/htcondor-file-transfer.md b/drafts/htcondor-file-transfer.md deleted file mode 100644 index e69de29b..00000000 diff --git a/drafts/molecular-dynamics-workflow-namd.md b/drafts/molecular-dynamics-workflow-namd.md deleted file mode 100644 index e69de29b..00000000 diff --git a/drafts/namd-workflow.md b/drafts/namd-workflow.md deleted file mode 100644 index e69de29b..00000000 diff --git a/drafts/pegasus-tutorials.md b/drafts/pegasus-tutorials.md deleted file mode 100644 index e69de29b..00000000 diff --git a/drafts/signup.md b/drafts/signup.md deleted file mode 100644 index 402f04d9..00000000 --- a/drafts/signup.md +++ /dev/null @@ -1 +0,0 @@ -Test 4 diff --git a/drafts/software-access-via-parrot.md b/drafts/software-access-via-parrot.md deleted file mode 100644 index e69de29b..00000000 diff --git a/drafts/stash-http-server.md b/drafts/stash-http-server.md deleted file mode 100644 index be3a120d..00000000 --- a/drafts/stash-http-server.md +++ /dev/null @@ -1,29 +0,0 @@ -[title]: - "Accessing Stash via the web" - -Making data accessible over HTTP --------------------------------- - -All user accounts on OSG-Connect have a directory that is automatically web accessible. This directory is located at -~/data/public. To make a file or directory accessible, copy it to this directory or a subdirectory of this directory -and give files permissions of 644 and directories permissions of 755. E.g. : - - $ cd ~/osg-stash_http - $ cp random_words ~/data/public - $ chmod 644 ~/data/public/random_words - $ cp -a test_directory ~/data/public/test_directory - $ chmod 755 ~/data/public/test_directory - $ chmod 644 ~/data/public/test_directory/test_file - - -Manually Accessing Stash using HTTP ------------------------------------ - -All the contents of the public directory are made available over HTTP. Go to http://stash.osgconnect.net/+username to view -the files and directory that you just made available in the previous section. You can also use wget to retrieve the files, -e.g: - - $ cd ~/osg-stash_http - $ mkdir tmp - $ cd tmp - $ wget --no-check-certificate http://stash.osgconnect.net/+username/test_directory/test_file - diff --git a/drafts/stash-http-worker.md b/drafts/stash-http-worker.md deleted file mode 100644 index e69de29b..00000000 diff --git a/drafts/stash-intro.md b/drafts/stash-intro.md deleted file mode 100644 index 100eeb40..00000000 --- a/drafts/stash-intro.md +++ /dev/null @@ -1,16 +0,0 @@ -[title]: - "Introduction to Stash" - -Overview --------- - -Stash (http://stash.osgconnect.net/) is a temporary data storage solution for your data workflow. Use Stash to pre-stage job input datasets or write output files when they in aggregate they exceed more than what can be comfortably handled out of your home directory (e.g. on the `login.osgconnect.net` host). This will vary depending on the job type, but typically this occurs in the range of a few hundred megabytes to gigabytes. You can retrieve your job output from Stash at a later time, transferring the data wherever you store your precious datasets. - -Usage ------ -If you have datasets on your campus' storage system, by far the easiest method is to use Globus as the file transfer service. - -Stash is mounted on all OSG Connect submit nodes (`login.osgconnect.net`) at `~/data`, and is visible to workers via Parrot/Chirp. The portion of your Stash designated public is also available directly on the WWW as `http://stash.osgconnect.net/~yourusername`. - -Policy ------- -At present, the storage service is offered as a free scratch-like storage service and there are no user quotas. When space on the system becomes tight, files will be removed on a simple least-recently-used basis. diff --git a/drafts/swift-basic-workflow.md b/drafts/swift-basic-workflow.md deleted file mode 100644 index e69de29b..00000000 diff --git a/drafts/swift-parallel-scripting-language.md b/drafts/swift-parallel-scripting-language.md deleted file mode 100644 index e69de29b..00000000 diff --git a/drafts/switching-modules.md b/drafts/switching-modules.md deleted file mode 100644 index 7afe0c61..00000000 --- a/drafts/switching-modules.md +++ /dev/null @@ -1,66 +0,0 @@ -[title]: - "Switching between OASIS and local modules" - -[TOC] - -# Switching between module systems - -Your local cluster may already have a modules system setup. If this is -the case, then you'll need to take some extra setup in order to access OASIS modules -in other to test your scripts. The switching process is specific to each site, -so you should find your institution below and use the instructions there. - - -## Clemson Palmetto - -These instructions indicate how to switch between the modules on the Palmetto -cluster at Clemson and OASIS. - -First, run -``` -source /cvmfs/oasis.opensciencegrid.org/osg/modules/switch-modules/rcc/switch-modules.sh -``` - -### Switching to OASIS modules - -In order to deactivate the local modules and activate the OASIS modules run -``` -switch_modules oasis -``` - -### Switching to local modules -In order to deactivate the OASIS modules and activate the local modules run -``` -switch_modules local -``` - -## UChicago RCC Midway - -These instructions indicate how to switch between OASIS and local modules on the midway cluster -at UChicago. - -First, run -``` -source /cvmfs/oasis.opensciencegrid.org/osg/modules/switch-modules/rcc/switch-modules.sh -``` - -### Switching to OASIS modules - -In order to deactivate the local modules and activate the OASIS modules run -``` -switch_modules oasis -``` - -### Switching to local modules -In order to deactivate the OASIS modules and activate the local modules run -``` -switch_modules local -``` - -## Others -These instructions are for sites without modules installed - -### Activating to OASIS modules -To activate OASIS modules run -``` -source source /cvmfs/oasis.opensciencegrid.org/osg/modules/lmod/current/init/$SHELL -``` diff --git a/drafts/word-search-workflow.md b/drafts/word-search-workflow.md deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/.DS_Store b/examples/.DS_Store deleted file mode 100644 index bb8d2ea1..00000000 Binary files a/examples/.DS_Store and /dev/null differ diff --git a/examples/FreeSurfer/Introduction.md b/examples/FreeSurfer/Introduction.md deleted file mode 100644 index 9806d5e6..00000000 --- a/examples/FreeSurfer/Introduction.md +++ /dev/null @@ -1,35 +0,0 @@ -[title]: - "Introduction to FreeSurfer on the OSPool" -[TOC] - -## Overview - -[FreeSurfer](http://freesurfer.net/) is a software package to analyze MRI scans -of human brains. The OSG used to have a service called -Fsurf, which is now discontinued. Instead we have community supported -FreeSurfer container image and workflow. Please see: - -* [https://github.com/pegasus-isi/freesurfer-osg-workflow](https://github.com/pegasus-isi/freesurfer-osg-workflow) - scroll down to see the documentaion on this page. -* Container image: `/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-freesurfer:latest` and defined at [https://github.com/opensciencegrid/osgvo-freesurfer](https://github.com/opensciencegrid/osgvo-freesurfer) - -## Prerequisites - -To use the FreeSurfer on OSG, you need: - -* Your own FreeSurfer license file (see: [https://surfer.nmr.mgh.harvard.edu/fswiki/DownloadAndInstall#License](https://surfer.nmr.mgh.harvard.edu/fswiki/DownloadAndInstall#License)) -* A regular OSG Connect account. - -## Privacy and Confidentiality of Subjects - -In order to protect the privacy of your participants’ scans, we request that you -submit only defaced and fully deidentified scans for processing. - -## Getting Help - -For assistance or questions, please email the OSG Research Facilitation team at -[support@opensciencegrid.org](mailto:support@opensciencegrid.org) or -visit the [help desk and community forums](http://support.opensciencegrid.org). - -## Acknowledging the OSG Consortium - -We gratefully request your acknowledgement of the OSG in publications benefiting from this service as described [here](https://support.opensciencegrid.org/support/solutions/articles/5000640421-acknowledging-the-open-science-grid). - diff --git a/examples/conda-on-osg.md b/examples/conda-on-osg.md deleted file mode 100644 index 13e4f7fd..00000000 --- a/examples/conda-on-osg.md +++ /dev/null @@ -1,149 +0,0 @@ -[title]: - "Using conda to Run Python on the OSPool" - -[TOC] - -The Anaconda/Miniconda distribution of Python is a common tool for installing and managing Python-based software and other tools. - -# Overview -When should you use Miniconda as an installation method in OSG? -* Your software has specific conda-centric installation instructions. -* The above is true and the software has a lot of dependencies. -* You mainly use Python to do your work. - -Notes on terminology: - -* **conda** is a Python package manager and package ecosystem that exists in parallel with `pip` and [PyPI](https://pypi.org/). -* **Miniconda** is a slim Python distribution, containing the minimum amount of packages necessary for a Python installation that can use conda. -* **Anaconda** is a pre-built scientific Python distribution based on Miniconda that has many useful scientific packages pre-installed. - -To create the smallest, most portable Python installation possible, we recommend starting with Miniconda and installing only the packages you actually require. - -To use a Miniconda installation on OSG, create your installation environment on the submit server and send a zipped version to your jobs. - -* [Pre-Install Miniconda and Transfer to Jobs](#pre-install-miniconda-and-transfer-to-jobs) -This guide also discusses how to [“pin” your conda environment](#specifying-exact-dependency-versions) to create a more consistent and reproducible environment with specified versions of packages. - -# Install Miniconda and Package for Jobs -In this approach, we will create an entire software installation inside Miniconda and then use a tool called `conda pack` to package it up for running jobs. - -## 1. Create a Miniconda Installation - -On the submit server, download the latest Linux [miniconda installer](https://docs.conda.io/en/latest/miniconda.html) and run it. - - [alice@login05]$ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh - [alice@login05]$ sh Miniconda3-latest-Linux-x86_64.sh - -Accept the license agreement and default options. At the end, you can choose whether or not to “initialize Miniconda3 by running conda init?” The default is no; you would then run the **eval** command listed by the installer to “activate” Miniconda. If you choose “no” you’ll want to save this command so that you can reactivate the Miniconda installation when needed in the future. - -## 2. Create a conda "Environment" With Your Packages - -(If you are using an `environment.yml` file as described [later](#specifying-exact-dependency-versions), you should instead create the environment from your `environment.yml` file. If you don’t have an `environment.yml` file to work with, follow the install instructions in this section. We recommend switching to the `environment.yml` method of creating environments once you understand the “manual” method presented here.) - -Make sure that you’ve activated the base Miniconda environment if you haven’t already. Your prompt should look like this: - - (base)[alice@login05]$ -To create an environment, use the `conda create` command and then activate the environment: - - (base)[alice@login05]$ conda create -n env-name - (base)[alice@login05]$ conda activate env-name -Then, run the `conda install` command to install the different packages and software you want to include in the installation. How this should look is often listed in the installation examples for software (e.g. [Qiime2](https://docs.qiime2.org/2020.2/install/native/#install-qiime-2-within-a-conda-environment), [Pytorch](https://pytorch.org/get-started/locally/)). - - (env-name)[alice@login05]$ conda install pkg1 pkg2 -Some Conda packages are only available via specific Conda channels which serve as repositories for hosting and managing packages. If Conda is unable to locate the requested packages using the example above, you may need to have Conda search other channels. More detail are available at [https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/channels.html.](https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/channels.html) - -Packages may also be installed via `pip`, but you should only do this when there is no `conda` package available. - -Once everything is installed, deactivate the environment to go back to the Miniconda “base” environment. - - (env-name)[alice@login05]$ conda deactivate -For example, if you wanted to create an installation with `pandas` and `matplotlib` and call the environment `py-data-sci`, you would use this sequence of commands: - - (base)[alice@login05]$ conda create -n py-data-sci - (base)[alice@login05]$ conda activate py-data-sci - (py-data-sci)[alice@login05]$ conda install pandas matplotlib - (py-data-sci)[alice@login05]$ conda deactivate - (base)[alice@login05]$ - -> ### More About Miniconda -> See the [official conda documentation](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) for more information on creating and managing environments with **conda**. - -## 3. Create Software Package -Make sure that your job’s Miniconda environment is created, but deactivated, so that you’re in the “base” Miniconda environment: - - (base)[alice@login05]$ -Then, run this command to install the `conda pack` tool: - - (base)[alice@login05]$ conda install -c conda-forge conda-pack -Enter `y` when it asks you to install. - -Finally, use `conda pack` to create a zipped tar.gz file of your environment (substitute the name of your conda environment where you see `env-name`), set the proper permissions for this file using `chmod`, and check the size of the final tarball: - - (base)[alice@login05]$ conda pack -n env-name - (base)[alice@login05]$ chmod 644 env-name.tar.gz - (base)[alice@login05]$ ls -sh env-name.tar.gz -When this step finishes, you should see a file in your current directory named `env-name.tar.gz`. - -## 4. Check Size of Conda Environment Tar Archive -The tar archive, `env-name.tar.gz`, created in the previous step will be used as input for subsequent job submission. As with all job input files, you should check the size of this Conda environment file. **If >100MB in size, you should NOT transfer the tar ball using `transfer_input_files` from your home directory**. Instead, you should plan to use OSG Connect's `/public` folder, and a `stash:///` link, as described in [this guide](https://support.opensciencegrid.org/support/solutions/articles/12000002775-transfer-larger-input-and-output-files). Please contact a research computing facilitator at support@opensciencegrid.org if you have questions about the best option for your jobs. - -More information is available at [File Availability with Squid Web Proxy](https://support.opensciencegrid.org/support/solutions/articles/5000633467-steer-your-jobs-with-htcondor-job-requirements) and [Managing Large Data in HTC Jobs](https://support.opensciencegrid.org/support/solutions/articles/12000002985-overview-data-staging-and-transfer-to-jobs). - -## 5. Create a Job Executable -The job will need to go through a few steps to use this “packed” conda environment; first, setting the `PATH`, then unzipping the environment, then activating it, and finally running whatever program you like. The script below is an example of what is needed (customize as indicated to match your choices above). - - #!/bin/bash - - # have job exit if any command returns with non-zero exit status (aka failure) - set -e - - # replace env-name on the right hand side of this line with the name of your conda environment - ENVNAME=env-name - - # if you need the environment directory to be named something other than the environment name, change this line - ENVDIR=$ENVNAME - - # these lines handle setting up the environment; you shouldn't have to modify them - export PATH - mkdir $ENVDIR - tar -xzf $ENVNAME.tar.gz -C $ENVDIR - . $ENVDIR/bin/activate - - # modify this line to run your desired Python script and any other work you need to do - python3 hello.py - -## 6. Submit Jobs -In your submit file, make sure to have the following: - -* Your executable should be the the bash script you created in [step 5](#5-create-a-job-executable). -* Remember to transfer your Python script and the environment `tar.gz` file to the job. If the `tar.gz` file is larger than 100MB, please use the `stash:///` file delivery mechanism as described above. - -# Specificying Exact Dependency Versions - -An important part of improving reproducibility and consistency between runs is to ensure that you use the correct/expected versions of your dependencies. - -When you run a command like `conda install numpy` `conda` tries to install the most recent version of `numpy` For example, `numpy` version `1.22.3` was released on Mar 7, 2022. To install exactly this version of numpy, you would run `conda install numpy=1.22.3` (the same works for `pip` if you replace `=` with `==`). We recommend installing with an explicit version to make sure you have exactly the version of a package that you want. This is often called “pinning” or “locking” the version of the package. - -If you want a record of what is installed in your environment, or want to reproduce your environment on another computer, conda can create a file, usually called `environment.yml`, that describes the exact versions of all of the packages you have installed in an environment. This file can be re-used by a different conda command to recreate that exact environment on another computer. - -To create an `environment.yml` file from your currently-activated environment, run - - [alice@login05]$ conda env export > environment.yml -This `environment.yml` will pin the exact version of every dependency in your environment. This can sometimes be problematic if you are moving between platforms because a package version may not be available on some other platform, causing an “unsatisfiable dependency” or “inconsistent environment” error. A much less strict pinning is - - [alice@login05]$ conda env export --from-history > environment.yml -which only lists packages that you installed manually, and **does not pin their versions unless you yourself pinned them during installation**. If you need an intermediate solution, it is also possible to manually edit `environment.yml` files; see the [conda environment documentation](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#) for more details about the format and what is possible. In general, exact environment specifications are simply not guaranteed to be transferable between platforms (e.g., between Windows and Linux). **We strongly recommend using the strictest possible pinning available to you**. - -To create an environment from an `environment.yml` file, run - - [alice@login05]$ conda env create -f environment.yml -By default, the name of the environment will be whatever the name of the source environment was; you can change the name by adding a `-n \` option to the `conda env create` command. - -If you use a source control system like `git`, we recommend checking your `environment.yml` file into source control and making sure to recreate it when you make changes to your environment. Putting your environment under source control gives you a way to track how it changes along with your own code. - -If you are developing software on your local computer for eventual use on the Open Science pool, your workflow might look like this: - -1. Set up a conda environment for local development and install packages as desired (e.g., `conda create -n science; conda activate science; conda install numpy`). -2. Once you are ready to run on the Open Science pool, create an `environment.yml` file from your local environment (e.g., `conda env export > environment.yml`). -3. Move your `environment.yml` file from your local computer to the submit machine and create an environment from it (e.g., `conda env create -f environment.yml`), then pack it for use in your jobs, as per [Create Software Package](#3-create-software-package). -More information on conda environments can be found in [their documentation](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#). - diff --git a/examples/java-on-osg.md b/examples/java-on-osg.md deleted file mode 100644 index 9e8a3763..00000000 --- a/examples/java-on-osg.md +++ /dev/null @@ -1,49 +0,0 @@ -[title]: - "Using Java in Jobs" - -[TOC] - -## Overview - -If your code uses Java via a `.jar` file, it is easy to bring along your -own copy of the Java Development Kit (JDK) which allows you to run -your `.jar` file anywhere on the Open Science Pool. - -## Steps to Use Java in Jobs - -1. **Get a copy of Java/JDK.** You can access the the Java Development Kit (JDK) from -the [JDK website](https://jdk.java.net/). First select the link to the -JDK that is listed as "Ready for Use" and then download the Linux/x64 -version of the tar.gz file using a Unix command such as `wget` from your `/home` directory. -For example, - - $ wget https://download.java.net/java/GA/jdk17.0.1/2a2082e5a09d4267845be086888add4f/12/GPL/openjdk-17.0.1_linux-x64_bin.tar.gz - -The downloaded file should end up in your -home directory on the OSG Connect access point. - -2. **Include Java in Input Files.** Add the downloaded tar file to the `transfer_input_files` line of your -submit file, along with the `.jar` file and any other input files the job needs: - - transfer_input_files = openjdk-17.0.1_linux-x64_bin.tar.gz, program.jar, other_input - -3. **Setup Java inside the job.** Write a script that unpacks the JDK tar file, sets -the environment to -find the java software, and then runs your program. This script will be -your job\'s executable. See this example for what the script should look -like: - - #!/bin/bash - - # unzip the JDK - tar -xzf openjdk-17.0.1_linux-x64_bin.tar.gz - # Add the unzipped JDK folder to the environment - export PATH=$PWD/jdk-17.0.1/bin:$PATH - export JAVA_HOME=$PWD/jdk-17.0.1 - - # run your .jar file - java -jar program.jar - - **Note that the exact name of the unzipped JDK folder and the JDK tar.gz file will - vary depending on the version you downloaded.** You should unzip the JDK tar.gz - file in your home directory to find out the correct directory name to add to - the script. \ No newline at end of file diff --git a/examples/julia-on-osg.md b/examples/julia-on-osg.md deleted file mode 100644 index 822caa8b..00000000 --- a/examples/julia-on-osg.md +++ /dev/null @@ -1,226 +0,0 @@ -[title]: - "Using Julia on the OSPool" - -[TOC] - -# Overview - -This guide provides an introduction to running Julia code on the Open -Science Pool. The [Quickstart Instructions](#quickstart-instructions) provide -an outline of job submission. The following sections provide more details about -installing Julia packages ([Install Julia Packages](#install-julia-packages)) and creating a complete -job submission ([Submit Julia Jobs](#submit-julia-jobs)). This guide assumes that -you have a script written in Julia and can identify the additional Julia packages -needed to run the script. - -If you are using many Julia packages or have other software dependencies as -part of your job, you may want to manage your software via a container instead -of using the tar.gz file method described in this guide. The OSG Connect team -maintains a [Julia container](12000073449) that can be used as a starting point -for creating a customized container with added packages. See -our [Docker and Singularity Guide](12000024676) for more details. - -# Quickstart Instructions - -1. Download the precompiled Julia software from . -You will need the 64-bit, tarball compiled for general use on a Linux x86 system. The -file name will resemble something like `julia-#.#.#-linux-x86_64.tar.gz`. - - * Tip: use `wget` to download directly to your `/home` directory on the -login node, **OR** use `transfer_input_files = url` in your HTCondor submit files. - -1. Install your Julia packages on the login node, else skip to the next step. - - * For more details, see the section on installing Julia - packages below: [Installing Julia Packages](#install-julia-packages) - -1. Submit a job that executes a Julia script using the Julia precompiled binary -with base Julia and Standard Library, via a shell script like the following as -the job's executable: - - #!/bin/bash - - # extract Julia tar.gz file - tar -xzf julia-#.#.#-linux-x86_64.tar.gz - - # add Julia binary to PATH - export PATH=$_CONDOR_SCRATCH_DIR/julia-#-#-#/bin:$PATH - - # run Julia script - julia my-script.jl - - * For more details on the job submission, see the section - below: [Submit Julia Jobs](#submit-julia-jobs) - -# Install Julia Packages - -If your work requires additional Julia packages, you will need to peform a one-time -installation of these packages within a Julia project. A copy of the project -can then be saved for use in subsequent job submissions. For more details, -please see Julia's documentation at [Julia Pkg.jl](https://julialang.github.io/Pkg.jl). - -## Download Julia and set up a "project" - -If you have not already downloaded a copy of Julia, download the -precompiled Julia software from . -You will need the 64-bit, tarball compiled for general use on a Linux x86 system. The -file name will resemble something like `julia-#.#.#-linux-x86_64.tar.gz`. - -We will need a copy of the original tar.gz file for running jobs, but to install -packages, we also need an unpacked version of the software. Run the following commands -to extract the Julia software and add Julia to your `PATH`: - - $ tar -xzf julia-#.#.#-linux-x86_64.tar.gz - $ export PATH=$PWD/julia-#.#.#/bin:$PATH - -After these steps, you should be able to run Julia from the command line, e.g. - - $ julia --version - -Now create a project directory to install your packages (we've called -it `my-project/` below) and tell Julia its name: - - $ mkdir my-project - $ export JULIA_DEPOT_PATH=$PWD/my-project - -> If you already have a directory with Julia packages on the login node, you can -> add to it by skipping the `mkdir` step above and going straight to setting the -> `JULIA_DEPOT_PATH` variable. - -You can choose whatever name to use for this directory \-- if you have -different projects that you use for different jobs, you could -use a more descriptive name than "my-project". - -## Install Packages - -We will now use Julia to install any needed packages to the project directory -we created in the previous step. - -Open Julia with the `--project` option set to the project directory: - - $ julia --project=my-project - -Once you've started up the Julia REPL (interpreter), start the Pkg REPL, used to -install packages, by typing `]`. Then install and test packages by using -Julia's `add Package` syntax. - - _ - _ _ _(_)_ | Documentation: https://docs.julialang.org - (_) | (_) (_) | - _ _ _| |_ __ _ | Type "?" for help, "]?" for Pkg help. - | | | | | | |/ _` | | - | | |_| | | | (_| | | Version 1.0.5 (2019-09-09) - _/ |\__'_|_|_|\__'_| | Official https://julialang.org/ release - |__/ | - - julia> ] - (my-project) pkg> add Package - (my-project) pkg> test Package - -If you have multiple packages to install they can be combined -into a single command, e.g. `(my-project) pkg> add Package1 Package2 Package3`. - -**If you encounter issues getting packages to install successfully, please -contact us at support@opensciencegrid.org** - -Once you are done, you can exit the Pkg REPL by typing the `DELETE` key and then -typing `exit()` - - (my-project) pkg> - julia> exit() - -Your packages will have been installed to the `my_project` directory; we want -to compress this folder so that it is easier to copy to jobs. - - $ tar -czf my-project.tar.gz my-project/ - -# Submit Julia Jobs - -To submit a job that runs a Julia script, create a bash -script and HTCondor submit file following the examples in this section. These -example assume that you have downloaded a copy of Julia for Linux as a `tar.gz` -file and if using packages, you have gone through the steps above to install them -and create an additional `tar.gz` file of the installed packages. - -## Create Executable Bash Script - -Your job will use a bash script as the HTCondor `executable`. This script -will contain all the steps needed to unpack the Julia binaries and -execute your Julia script (`script.jl` below). What follows are two example bash scripts, -one which can be used to execute a script with base Julia only, and one that -will use packages you installed to a project directory (see [Install Julia Packages](#install-julia-packages)). - -### Example Bash Script For Base Julia Only - -If your Julia script can run without additional packages (other than base Julia and -the Julia Standard library) use the example script directly below. - - #!/bin/bash - - # julia-job.sh - - # extract Julia tar.gz file - tar -xzf julia-#.#.#-linux-x86_64.tar.gz - - # add Julia binary to PATH - export PATH=$_CONDOR_SCRATCH_DIR/julia-#.#.#/bin:$PATH - - # run Julia script - julia script.jl - - -### Example Bash Script For Julia With Installed Packages - - #!/bin/bash - - # julia-job.sh - - # extract Julia tar.gz file and project tar.gz file - tar -xzf julia-#.#.#-linux-x86_64.tar.gz - tar -xzf my-project.tar.gz - - # add Julia binary to PATH - export PATH=$_CONDOR_SCRATCH_DIR/julia-#.#.#/bin:$PATH - # add Julia packages to DEPOT variable - export JULIA_DEPOT_PATH=$_CONDOR_SCRATCH_DIR/my-project - - # run Julia script - julia --project=my-project script.jl - - -## Create HTCondor Submit File - -After creating a bash script to run Julia, then create a submit file -to submit the job. - -More details about setting up a submit file, including a submit file template, -can be found in our quickstart guide: [Quickstart Tutorial](5000633410) - - # julia-job.sub - - executable = julia-job.sh - - transfer_input_files = julia-#.#.#-linux-x86_64.tar.gz, script.jl - should_transfer_files = Yes - when_to_transfer_output = ON_EXIT - - output = job.$(Cluster).$(Process).out - error = job.$(Cluster).$(Process).error - log = job.$(Cluster).$(Process).log - - requirements = (OSGVO_OS_STRING == "RHEL 7") - request_cpus = 1 - request_memory = 2GB - request_disk = 2GB - - queue 1 - -If your Julia script needs to use packages installed for a project, -be sure to include `my-project.tar.gz` as an input file in `julia-job.sub`. -For project tarballs that are <100MB, you can follow the below example: - - transfer_input_files = julia-#.#.#-linux-x86_64.tar.gz, script.jl, my-project.tar.gz - -Modify the CPU/memory request lines to match what is needed by the job. -Test a few jobs for disk space/memory usage in order to make sure your -requests for a large batch are accurate! Disk space and memory usage can be found in the -log file after the job completes. diff --git a/examples/manage-python-packages.md b/examples/manage-python-packages.md deleted file mode 100644 index 466f4d6d..00000000 --- a/examples/manage-python-packages.md +++ /dev/null @@ -1,188 +0,0 @@ -[title]: - "Run Python Scripts on the OSPool" - -# Run Python Scripts on the OSPool - -[TOC] - -## Overview - -This guide will show you two examples of how to run jobs that use Python in the Open Science Pool. -The first example will demonstrate how to submit a job that uses base Python. -The second example will demonstrate the workflow for jobs that use specific Python packages, including -how to install a custom set of Python packages to your home directory and how to add them to a Python job submission. - -Before getting started, you should know which Python packages you need to run your job. - -## Running Base Python on the Open Science Pool -Several installations of base Python are available via the [Open Science Pool's Software -Module System][module-guide]. To see what Python versions are available on the Open Science Pool -run `module avail` while connected to our login node. - -### Create a bash script to run Python -To submit jobs that use a module to run base Python, first create a bash executable - for -this example we'll call it `run_py.sh` - which will include commands to first -load the appropriate Python module and then run our Python script called `myscript.py`. - -For example, `run_py.sh`: - - #!/bin/bash - - # Load Python - module load python/3.7.0 - - # Run the Python script - python3 myscript.py - - -> If you need to use Python 2, load the appropriate module and -> replace the `python3` above with `python2`. - -### Create an HTCondor submit file -In order to submit `run_py.sh` as part of a job, we need to create an HTCondor -submit file. This should include the following: - -* `run_py.sh` specified as the executable -* use `transfer_input_files` to bring our Python script `myscript.py`to wherever the job runs -* include requirements that request OSG nodes with access to base Python modules - -All together, the submit file will look something like this: - - universe = vanilla - executable = run_py.sh - - transfer_input_files = myscript.py - - log = job.log - output = job.out - error = job.error - - # Require nodes that can access the correct OSG modules - Requirements = (HAS_MODULES =?= true) && (OSGVO_OS_STRING == "RHEL 7") - - request_cpus = 1 - request_memory = 2GB - request_disk = 2GB - - queue 1 - -Once everything is set up, the job can be submitted in the usual way, by running -the `condor_submit` command with the name of the submit file. - -## Running Python Jobs That Use Additional Packages -It's likely that you'll need additional Python packages (aka libraries) that are not -present in the base Python installations made available via modules. This portion of the -guide describes how to create a Python "virtual environment" that contains your packages -and which can be included as part of your jobs. - -### Install Python packages -While connected to your login node, load the Python module that you want to use to run jobs: - - $ module load python/3.7.0 - -Next, create a virtual environment. The first command creates a base environment: - - $ python3 -m venv my_env - -> You can swap out `my_env` for a more descriptive name like `scipy` or `word-analysis`. - -This creates a directory `my_env` in the current working directory -with sub-directories `bin/`, `include/`, and `lib/`. - -Then _activate_ the environment and install packages to it. - - $ source my_env/bin/activate - -Notice how our command line prompt changes to: - - (my_env)$ - -The activation process redefines some of the shell variables -such as PYTHON_PATH, LIBRARY_PATH etc. - -After activation, packages can be installed using `pip` -which is a tool to install Python packages. - - (my_env)$ pip install numpy - ......some download message... - Installing collected packages: numpy - Installing collected packages: numpy - Successfully installed numpy-1.16.3 - -Install each package that you need for your job using the `pip install` command. Once -you are done, you can leave the virtual environment: - - (my_env)$ deactivate - -The above command resets the shell environmental variables and returns you to the -normal shell prompt (with the prefix `my_env` removed). - -All of the packages that were just installed should be contained in a sub-directory -of the `my_env` directory. To use these packages in a job, the entire `my_env` directory -will be transfered as a tar.gz file. So our final step is to compress the -directory, as follows: - - $ tar czf my_env.tar.gz my_env - - -### Create executable script to use installed packages -In addition to loading the appropriate Python module, we will need to add a few -steps to our bash executable to set-up the virtual environment we -just created. That will look something like this: - - #!/bin/bash - - # Load Python - # (should be the same version used to create the virtual environment) - module load python/3.7.0 - - # Unpack your envvironment (with your packages), and activate it - tar -xzf my_env.tar.gz - python3 -m venv my_env - source my_env/bin/activate - - # Run the Python script - python3 myscript.py - - # Deactivate environment - deactivate - -### Modify the HTCondor submit file to transfer Python packages -The submit file for this job will be similar to the base Python job submit file shown above -with one addition - we need to include `my_env.tar.gz` in the list of files specified by `transfer_input_files`. -As an example: - - universe = vanilla - executable = run_py.sh - - transfer_input_files = myscript.py, my_env.tar.gz - - log = job.log - output = job.out - error = job.error - - # Require nodes that can access the correct OSG modules - Requirements = (HAS_MODULES =?= true) && (OSGVO_OS_STRING == "RHEL 7") - - request_cpus = 1 - request_memory = 2GB - request_disk = 2GB - - queue 1 - -## Other Considerations -This guide mainly focuses on the nuts and bolts of running Python, but it's important -to remember that additional files needed for your jobs (input data, setting files, etc.) -need to be transferred with the job as well. See our [Introduction to Data Management -on OSG][data-intro] for details on the different ways to deliver inputs to your jobs. - -When you've prepared a real job submission, make sure to run a test job and then check -the `log` file for disk and memory usage; if you're using significantly more or less -than what you requested, make sure you adjust your requests. - -## Getting Help - -For assistance or questions, please email the OSG Research Facilitation -team at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) or visit the [help desk and community forums](http://support.opensciencegrid.org). - -[module-guide]: 12000048518 -[data-intro]: 12000002985 diff --git a/hpcadmin/osg-flock.md b/hpcadmin/osg-flock.md deleted file mode 100644 index a010c465..00000000 --- a/hpcadmin/osg-flock.md +++ /dev/null @@ -1,5 +0,0 @@ -[title]: - "Submit Node Flocking to OSG" - -This page has moved to [https://opensciencegrid.org/docs/submit/osg-flock/](https://opensciencegrid.org/docs/submit/osg-flock/) - - diff --git a/hpcadmin/osg-xsede.md b/hpcadmin/osg-xsede.md deleted file mode 100644 index e2e16c47..00000000 --- a/hpcadmin/osg-xsede.md +++ /dev/null @@ -1,355 +0,0 @@ -[title]: Running OSG jobs on XSEDE -[TOC] - -## Overview - -The [OSG](http://www.osg-htc.org/) promotes science by: - - * enabling a framework of distributed computing and storage resources - - * making available a set of services and methods that enable better access - to ever increasing computing resources for researchers and communities - - * providing resource sharing principles and software that enable distributed - High Throughput Computing (dHTC) for users and communities at all scales. - -The OSG facilitates access to dHTC for scientific research -in the US. The resources accessible through the OSG are -contributed by the community, organized by the OSG, and -governed by the [OSG -Consortium](http://www.osg-htc.org); an overview is available at -[An Introduction to OSG](http://osg-docdb.opensciencegrid.org/0008/000839/004/OSG%20Intro%2 -0v23.pdf). In 2017, OSG is comprised -of about 126 institutions with ~120 active sites that collectively -support usage of ~4,000,000 core hours per day. Up-to-date usage metrics -are available at the [OSG Usage Display](https://display.opensciencegrid.org/). - -Cores that are not being used at any point in time by -the owning communities are made available for shared use by other -researchers; this usage mode is called opportunistic access. OSG -supports XSEDE users by providing a Virtual Cluster that forms an -abstraction layer to access the opportunistic cores in the distributed -OSG infrastructure. This interface allows XSEDE users to view the OSG as -a single cluster to manage their jobs, provide the inputs and retrieve -the outputs. XSEDE users access the OSG via the OSG-XSEDE login host -that appears as a resource in the XSEDE infrastructure. - -## Computation that is a good match for OSG - -High throughput workflows with simple system and -data dependencies are a good fit for OSG. The -[Condor manual](http://research.cs.wisc.edu/condor/manual/current/) has an overview of [high throughput computing](http://research.cs.wisc.edu/condor/manual/current/1_Overview.html). - -Jobs submitted into the OSG Virtual Cluster will be executed on machines -at several remote physical clusters. These machines may differ in terms -of computing environment from the submit node. Therefore it is important -that the jobs are as self-contained as possible by generic binaries -and data that can be either carried with the job, or staged on demand. -Please consider the following guidelines: - - - * Software should preferably be **single threaded**, using - **less than 2 GB memory** and each invocation should - **run for 1-12 hours**. Please contact the support listed - below for more information about these capabilities. System level check - pointing, such as the HTCondor standard universe, is not available. - Application level check pointing, for example applications writing out - state and restart files, can be made to work on the system. - - * Compute sites in the OSG can be configured to use pre-emption, which - means jobs can be automatically killed if higher priority jobs enter - the system. Pre-empted jobs will restart on another site, but it is - important that the jobs can handle multiple restarts. - - * Binaries should preferably be statically linked. However, dynamically - linked binaries with standard library dependencies, built for a 64-bit - Red Hat Enterprise Linux (RHEL) 6 machines will also work. Also, - interpreted languages such as Python or Perl will work as long as there - are no special module requirements. - - * Input and output data for each job should be < 10 GB to allow them - to be pulled in by the jobs, processed and pushed back to the submit - node. Note that the OSG Virtual Cluster does not currently have a global - shared file system, so jobs with such dependencies will not work. - - * Software dependencies can be difficult to accommodate unless the software - can be staged with the job. - - -The following are examples of computations that are **not** good -matches for OSG: - - * Tightly coupled computations, for example MPI based communication, will - not work well on OSG due to the distributed nature of the infrastructure. - - * Computations requiring a shared file system will not work, as there is - no shared filesystem between the different clusters on OSG. - - * Computations requiring complex software deployments are not a good fit. - There is limited support for distributing software to the compute - clusters, but for complex software, or licensed software, deployment - can be a major task. - - -## System Configuration - -The OSG Virtual Cluster is a Condor pool overlay on top of OSG -resources. The pool is dynamically sized based on the demand, the -number of jobs in the queue, and supply, resource availability at the -OSG resources. It is expected that the average number of resources, -on average, available to XSEDE users will be in the order of 1,000 -cores. - -One important difference between the OSG Virtual Cluster and most of -the other XSEDE resources is that the OSG Virtual Cluster does not have -a shared file system. This means that your jobs will have to bring -executables and input data. Condor can transfer the files for you, -but you will have to identify and list the files in your Condor job -description file. - -Local storage space at the submission site is controlled by -quota. Your home directory has a quota of 10 GBs and your work directory -`/local-scratch/$USER` has a quota of 1 TB. There are no -global quotas on the remote compute nodes, but expect that about 10 GBs -are available as scratch space for each job. - - -## System Access - -The preferred method to access the system is via the XSEDE Single -Sign On (SSO) Hub. Please see the [sso documentation](https://portal.xsede.org/single-sign-on-hub) -for details. - -A secondary access methor is to use SSH public key authentication. -Secure shell users should feel free to append their public RSA key -to their `~/.ssh/authorized_keys` file to enable access -from their own computer. Please login once via the SSO Hub to install your -key. Please make sure that the permissions on the .ssh directory and -the authorized_keys file have appropiate permissions. For example - - $ chmod 755 ~/.ssh - $ chmod 644 ~/.ssh/authorized_keys - - -## Application Development< - -Most of the clusters in OSG are running Red Hat Enterprise Linux (RHEL) -6 or 7, or some derivative thereof, on an x86_64 architecture. For -your application to work well in this environment, it is recommend -that the application is compiled on a similar system, for example on -the OSG Virtual Cluster login system: `submit-1.osg.xsede.org -`. It is also recommended that the application be statically -linked, or alternatively dynamically linked against just a few standard -libraries. What libraries a binary depends on can be checked using the -Unix `ldd` command-line utility: - - $ ldd a.out - a.out is a static executable - - -In the case of interpreted languages like Python and Perl, applications -have to either use only standard modules, or be able to ship the modules -with the jobs. Please note that different compute nodes might have -different versions of these tools installed. - -A good solution to complex software stack is Singularity containers -which are described below. - - -## Running Your Application - -The OSG Virtual Cluster is based on Condor and the -[Condor manual](http://research.cs.wisc.edu/condor/manual/current/) -provides a reference for command line tools. The commonly used tools -are: - - * `**condor_submit**` - Takes a Condor submit file and adds the job to the queue - - * `**condor_q**` - Lists the jobs in the queue. Can be invoked with your - username to limit the list of jobs to your jobs: `condor_q $USER` - - * `**condor_status**` - Lists the available slots in the system. - Note that this is a dynamic list and if there are no jobs in the system, - `condor_status` may return an empty list - - * `**condor_rm**` - Remove a job from the queue. If you are running a DAG, - please `condor_rm` the id of the DAG to remove the whole workflow. - -### Submitting a Simple Job - -Below is a basic job description for the Virtual Cluster. - - universe = vanilla - - # specifies the XSEDE project to charge the job usage to - this is a - # required attribute for all jobs submitted on the OSG-XSEDE resource - +ProjectName = "TG-NNNNNN" - - # requirements is an expression to specify machines that can run jobs - requirements = OSGVO_OS_STRING == "RHEL 6" && Arch == "X86_64" && HAS_MODULES == True - request_cpus = 1 - request_memory = 2 GB - request_disk = 10 GB - - executable = /bin/hostname - - arguments = -f - - should_transfer_files = YES - WhenToTransferOutput = ON_EXIT - - output = job.out - error = job.err - log = job.log - - notification = NEVER - - queue - - - Create a file named `job.condor` containing the above text and then run: - - $ condor_submit job.condor - -You can check the status of the job using the `condor_q` command. - -**Note:** The Open Science Pool is a distributed resource, and there -will be minor differences in the compute nodes, for example in what -system libraries and tools are installed. Therefore, when running -a large number of jobs, it is important to detect and handle job -failures correctly in an automatic fashion. It is recommended that your -application uses non-zero exit code convention to indicate failures, and -that you enable retries in your Condor submit files. For example: - - # stay in queue on failures - on_exit_hold = (ExitBySignal == True) || (ExitCode != 0) - - # retry job 3 times, pause 1 hour between retries - periodic_release = (NumJobStarts < 3) && ((CurrentTime - EnteredCurrentStatus) > (60*60)) - - -### Job Example: Java with a job wrapper - -The following is an example on how to run Java code on Open Science Pool. The job requirements specifies that the job requires -Java, and a wrapper script is used to invoke Java. - -File: `condor.sub` - - universe = vanilla - - # specifies the XSEDE project to charge the job usage to - this is a - # required attribute for all jobs submitted on the OSG-XSEDE resource - +ProjectName = "TG-NNNNNN" - - # requirements is an expression to specify machines that can run jobs - requirements = HAS_JAVA == True - - # stay in queue on failures on_exit_hold = (ExitBySignal == True) || (ExitCode != 0) - - # retry job 3 times, pause 1 hour between retries - periodic_release = (NumJobStarts < 3) && ((CurrentTime - EnteredCurrentStatus) > (60*60)) - - executable = wrapper.sh - - should_transfer_files = YES - WhenToTransferOutput = ON_EXIT - - # a list of files that the job needs - transfer_input_files = HelloWorld.jar - - output = job.out - error = job.err - log = job.log - - notification = NEVER - - queue - -File: `wrapper.sh` - - #!/bin/bash - - set -e - - java HelloWorld - - -## Sample Jobs and Workflows - -A set of sample jobs and workflows can be found under -`/opt/sample-jobs` on the submit-1.osg.xsede.org host. README -files are included with details for each sample. - -`/opt/sample-jobs/single/` contains a single Condor job -example. Single jobs can be used for smaller set of jobs or if the job -structure is simple, such as parameter sweeps. - -A sample-app package -([sampleapp.tgz](http://www.ncsa.illinois.edu/People/yanliu/codes/sampleapp.tgz)) -is available in the `/opt/sample-jobs/sampleapp/` directory. This package shows -how to build a library and an executable, both with dynamic and static -linking, and submit the job to a set of different schedulers available -on XSEDE. The package includes submit files for PBS, SGE and Condor. - -[DAGMan](http://research.cs.wisc.edu/condor/manual/current/2_10DAGMan_Applications.html) -is a HTCondor workflow tool. It allows the -creation of a directed acyclic graph of jobs to be run, and then DAGMan -submits and manages the jobs. DAGMan is also useful if you have a -large number of jobs, even if there are no job inter-dependencies, as -DAGMan can keep track of failures and provide a restart mechanism if -part of the workflow fails. A sample DAGMan workflow can be found in -`/opt/sample-jobs/dag/` - -[Pegasus](https://pegasus.isi.edu) is a workflow system that -can be used for more complex workflows. It plans abstract workflow -representations down to an executable workflow and uses Condor DAGMan -as a workflow executor. Pegasus also provides debugging and monitoring -tools that allow users to easily track failures in their workflows. -Workflow provenance information is collected and can be summarized with -the provided statistical and plotting tools. A sample Pegasus workflow -can be found in `/opt/sample-jobs/pegasus/` . - - -## Singularity Containers - -Singularity containers provide a great solution for complex software -stacks or OS requirements, and OSG has easy to use integrated support -for such containers. Full details can be found in the -[Singularity Containers](https://support.opensciencegrid.org/support/solutions/articles/12000024676). - - -## Distribute data with Stash - -Stash is a data solution used under -[OSGConnect](https://osgconnect.net/), but is partly also available -for OSG XSEDE users. Files under `/local-scratch/public_stash/` will -automatically synchronize to the globally available -`/cvmfs/stash.osgstorage.org/user/xd-login/public/` file system, which -is available to the majority of OSG connected compute nodes. This is a great -way to distribute software and commonly used data sets. To get started, create -your own sub directory: - - $ mkdir -p /local-scratch/public_stash/$USER - -Now, populate that directory with the software and data required for your jobs. -The synchronization can take couple of hours. You can verify the data has -reached the /cvmfs system by using `ls`: - - $ ls /cvmfs/stash.osgstorage.org/user/xd-login/public/ - -To steer your jobs to compute nodes which can access the file system, add -`HAS_CVMFS_stash_osgstorage_org == True` to your job -requirements. For example: - - requirements = OSGVO_OS_STRING == "RHEL 6" && Arch == "X86_64" && HAS_MODULES == True && HAS_CVMFS_stash_osgstorage_org == True - -Once a job starts up on such a compute node, the job can read directly -from `/cvmfs/stash.osgstorage.org/user/xd-login/public/` - - -## How to get help using OSG - -XSEDE users of OSG may get technical support by -contacting OSG Research Facilitation staff at email -. -Users may also contact the [XSEDE helpdesk](https://portal.xsede.org/help-desk). diff --git a/index.md b/index.md deleted file mode 100644 index e5ef18b0..00000000 --- a/index.md +++ /dev/null @@ -1,114 +0,0 @@ -
![ConnectBook Logo](assets/ConnectBook.png)
- -# Getting Started - -* [Sign-up and login information][signup] -* [Quickstart guideB ][quickstart] -* [Starting a project][projects] - - -# Tutorials - -* [The tutorial command][tutorial-command] -* Math and statistics ([R][R], [Octave][octave]) -* Molecular science ([NAMD][namd], [Autodock-Vina][vina-autodock]) -* Bioinformatics ([BLAST]([blast]) -* Image analysis ([photodemo][photodemo]) - - -# Moving data to/from OSG Connect - -* [Stash: the OSG Connect storage service][stash] -* [Data transfer with Globus][globus] -* [Moving files onto login.osgconnect.net using scp][scp] -* [Accessing Stash files via the Web][stash-web] - - -# Accessing data from jobs running on the OSG - -* [Web access of files on Stash from jobs running on the OSG][stash-web-2] -* [Using built-in HTCondor file transfer][htcondor-data] -* Using StashCache (future) - - -# Workflow Solutions - -* [DAGMan](http://research.cs.wisc.edu/htcondor/dagman/dagman.html) tutorials - * [Molecular dynamics workflow (NAMD)][namd] -* [Pegasus](http://pegasus.isi.edu/) tutorials - * [Word search workflow][pegasus] - * [NAMD workflow][pegasus-namd] - * [Autodock-Vina workflow][pegasus-vina] -* [Swift](http://swift-lang.org/main/index.php) parallel scripting language - * [Swift basic workflow][swift] - - -# Software Access - -* [Setting up and using Environment Modules][modules] -* [Software transfer via HTCondor or HTTP][software-htcondor-http] -* [Software access via Parrot][software-parrot] -* [List of Applications][module-list] - - -# Additional Support - -* [Workshops and training events][workshops] -* [Contact information][osg-community] -* [Frequently Asked Questions (FAQ)][faq] -* [Prior topics][obsolete] - - -[blast]: tutorials/tutorial-blast/README.md -[cp2k]: tutorials/tutorial-cp2k/README.md -[dagman-namd]: tutorials/tutorial-dagman-namd/README.md -[error101]: tutorials/tutorial-error101/README.md -[exitcode]: tutorials/tutorial-exitcode/README.md -[htcondor-transfer]: tutorials/tutorial-htcondor-transfer/README.md -[namd]: tutorials/tutorial-namd/README.md -[nelle-nemo]: tutorials/tutorial-nelle-nemo/README.md -[oasis-parrot]: tutorials/tutorial-oasis-parrot/README.md -[octave]: tutorials/tutorial-octave/README.md -[pegasus]: tutorials/tutorial-pegasus/README.md -[pegasus-namd]: tutorials/tutorial-pegasus-namd/README.md -[pegasus-vina]: tutorials/tutorial-pegasus-vina/README.md -[photodemo]: tutorials/tutorial-photodemo/README.md -[quickstart]: tutorials/tutorial-quickstart/README.md -[R]: tutorials/tutorial-R/README.md -[root]: tutorials/tutorial-root/README.md -[scaling]: tutorials/tutorial-scaling/README.md -[scaling-up-resources]: tutorials/tutorial-scaling-up-resources/README.md -[ScalingUp-R]: tutorials/tutorial-ScalingUp-R/README.md -[software]: tutorials/tutorial-software/README.md -[stash-chirp]: tutorials/tutorial-stash-chirp/README.md -[stash-http]: tutorials/tutorial-stash-http/README.md -[stash-namd]: tutorials/tutorial-stash-namd/README.md -[swift]: tutorials/tutorial-swift/README.md -[vina-autodock]: tutorials/tutorial-VinaAutodock/README.md -[locations]: tutorials/tutorial-osg-locations/README.md - -[DAGMan]: needed -[Pegasus]: needed -[Swift]: needed -[faq]: needed -[globus]: needed -[htcondor-data]: needed -[module-list]: needed -[modules]: needed -[obsolete]: needed -[osg-community]: needed -[projects]: needed -[scp]: needed -[signup]: needed -[software-htcondor-http]: needed -[software-parrot]: needed -[stash]: needed -[stash-web]: needed -[stash-web-2]: needed -[swift]: needed -[tutorial-command]: needed -[workshops]: needed - - - - diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..ed7ad3b1 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,31 @@ +INHERIT: ./user-documentation/data/configs/ospool.yml + +docs_dir: ./user-documentation/data/docs/ospool + +site_name: OSPool Documentation +site_url: "https://portal.osg-htc.org/documentation/" +repo_name: osg-htc/user-documentation +theme: + features: + - navigation.tabs + name: material + logo: assets/OSG_Logo.svg + favicon: assets/OSG_Logo.svg + custom_dir: overrides + +markdown_extensions: + - admonition + - codehilite: + guess_lang: False + - meta + - pymdownx.details + - toc: + permalink: True + - attr_list + - pymdownx.emoji: + emoji_index: !!python/name:materialx.emoji.twemoji + emoji_generator: !!python/name:materialx.emoji.to_svg + +extra_css: + - stylesheets/code-highlight.css + - stylesheets/osg.css diff --git a/overrides/home.html b/overrides/home.html new file mode 100644 index 00000000..b9780845 --- /dev/null +++ b/overrides/home.html @@ -0,0 +1,247 @@ + + +{% extends "main.html" %} +{% block tabs %} +{{ super() }} + + + + +
+
+
+
+ +
+
+

OSPool Documentation

+

Documentation and Support hub for researchers using OSPool's compute and storage resources to help with their research.

+ + Get started + + + Sign Up + +
+
+
+
+ + + +{% if nav|length>1 %} +
+ {% for nav_item in nav %} + {% if not loop.first %} +
+

{{ nav_item.title }}

+
+ {% for nav_item in nav_item.children %} +
+

{{ nav_item.title }}

+ +
+ {% endfor %} +
+
+ {% endif %} + {% endfor %} +
+{% endif %} + +{% endblock %} + +{% block content %}{% endblock %} \ No newline at end of file diff --git a/overrides/main.html b/overrides/main.html new file mode 100644 index 00000000..35bdb294 --- /dev/null +++ b/overrides/main.html @@ -0,0 +1,5 @@ +{% extends "base.html" %} + +{% block site_name %} + Lorem ipsum dolor sit amet +{% endblock %} \ No newline at end of file diff --git a/drafts/autodock-vina-workflow.md b/scripts/docker.py similarity index 100% rename from drafts/autodock-vina-workflow.md rename to scripts/docker.py diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100644 index 00000000..84d582f2 --- /dev/null +++ b/scripts/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +python3 user-documentation/scripts/link-docs.py + +echo "I have built the User Documentation" + +mkdocs "$@" \ No newline at end of file diff --git a/start/.DS_Store b/start/.DS_Store deleted file mode 100644 index 4c70f398..00000000 Binary files a/start/.DS_Store and /dev/null differ diff --git a/start/account/.DS_Store b/start/account/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/start/account/.DS_Store and /dev/null differ diff --git a/start/account/generate-add-sshkey.md b/start/account/generate-add-sshkey.md deleted file mode 100644 index 4eb0119d..00000000 --- a/start/account/generate-add-sshkey.md +++ /dev/null @@ -1,169 +0,0 @@ -[title]: - "Generate SSH Keys and Activate Your OSG Login" - -[TOC] - -## Overview - -OSG Connect requires SSH-key-based logins. You need to follow -a two-step process to set up the SSH key to your account. - -1. Generate a SSH key pair. - -2. Add your public key to the submit host by uploading it to -your OSG Connect user profile (via the OSG Connect website). - -After completing the process, you can log in from a local computer -(your laptop or desktop) to the OSG Connect login node assigned -using either ssh or an ssh program like Putty -- see below for -more details on logging in. - -NOTE: Please do not edit the authorized keys file on the login node. - -## Step 1: Generate SSH Keys - -We will discuss how to generate a SSH key pair for two cases: - -* "Unix" systems (Linux, Mac) and certain, latest versions of Windows -* Older Windows systems - -Please note: The key pair consist of a private key and a public key. You will upload the -public key to OSG Connect, but you also need to keep a copy of the private key to log in! -You should keep the private key on machines that you have -direct access to, i.e. your local computer (your laptop or desktop). - -### Unix-based operating system (Linux/Mac) or latest Windows 10 versions - -We will create a key in the .ssh directory of your computer. Open a terminal on your local computer and run the following commands: - - mkdir ~/.ssh - chmod 700 ~/.ssh - ssh-keygen -t rsa - -For the newer OS versions the .ssh directory is already created and the first command is redundant. The last command will produce a prompt similar to - - Generating public/private rsa key pair. - Enter file in which to save the key (/home//.ssh/id_rsa): - -Unless you want to change the location of the key, continue by pressing enter. -Now you will be asked for a passphrase. Enter a passphrase that you will be -able to remember and which is secure: - - Enter passphrase (empty for no passphrase): - Enter same passphrase again: - -When everything has successfully completed, the output should resemble the -following: - - Your identification has been saved in /home//.ssh/id_rsa. - Your public key has been saved in /home//.ssh/id_rsa.pub. - The key fingerprint is: - ... - -The part you want to upload is the content of the `.pub` file (~/.ssh/id_rsa.pub) - -### Windows, using Putty to log in - -If you can connect using the `ssh` command within the Command Prompt (Windows 10 build version 1803 and later), please follow the Mac/Linux directions above. If not, -continue with the directions below. - -1. Open the `PuTTYgen` program. You can download `PuttyGen` -here: [PuttyGen Download Page](https://www.chiark.greenend.org.uk/~sgtatham/putty/latest.html), -scroll down until you see the `puttygen.exe` file. - -2. For Type of key to generate, select RSA or SSH-2 RSA. - -2. Click the "Generate" button. - -3. **Move your mouse in the area below the progress bar.** -When the progress bar is full, PuTTYgen generates your key pair. - -4. Type a passphrase in the "Key passphrase" field. Type the same passphrase in the "Confirm passphrase" field. You -can use a key without a passphrase, but this is not recommended. - -5. Click the "Save private key" button to save the private key. **You must save the private key.** You will need it to connect to your machine. - -6. Right-click in the text field labeled "Public key for pasting into OpenSSH authorized_keys file" and choose Select All. - -7. Right-click again in the same text field and choose Copy. - -![alt text](https://raw.githubusercontent.com/OSGConnect/connectbook/master/images/puttygen_ssh_key.png "PuttyGen Window") - -## Step 2: Add the public SSH key to login node - -To add your public key to the OSG Connect log in node: - -1. Go to www.osgconnect.net and sign in with the institutional identity you used when requesting an OSG Connect account. - -2. Click "Profile" in the top right corner. - -3. Click the "Edit Profile" button located after the user information in the left hand box. - -4. Copy/paste the public key which is found in the `.pub` file into the "SSH Public Key" text box. -The expected key is a single line, with three fields looking something like -`ssh-rsa ASSFFSAF... user@host`. If you used the first set of key-generating -instructions it is the content of `~/.ssh/id_rsa.pub` and for the second (using -PuTTYgen), it is the content from step 7 above. - -6. Click "Update Profile" - -The key is now added to your profile in the OSG Connect website. This will automatically -be added to the login nodes within a couple hours. - -> ### Can I Use Multiple Keys? -> Yes! If you want to log into OSG Connect from multiple computers, you can do so by generating -> a keypair on each computer you want to use, and then adding the public key to your OSG -> Connect profile. - -## Logging In - -After following the steps above to upload your key and it's been a few hours, you should -be able to log in to OSG Connect. - -### Determine which login node to use - -Before you can connect, you will need to know which login node your account is assigned to. You can find -this information on your profile from the OSG Connect website. - -1. Go to www.osgconnect.net and sign in with your CILogin. - -2. Click "Profile" in the top right corner. - -3. The assigned login nodes are listed in the left side box. Make note of the address of -your assigned login node as you will use this to connect to OSG Connect. - -![Identify Login Node](https://raw.githubusercontent.com/OSGConnect/connectbook/master/images/find_osgconnect_login_node.png "OSG Connect Profile") - -### For Mac, Linux, or newer versions of Windows - -Open a terminal and type in: - - ssh @ - -It will ask for the passphrase for your ssh key (if you set one) and then you -should be logged in. - -### For older versions of Windows - -On older versions of Windows, you can use the [Putty program](https://www.chiark.greenend.org.uk/~sgtatham/putty/latest.html) to log in. - -PuTTY Intructions Screenshot - -1. Open the `PutTTY` program. If necessary, you can download PuTTY from the website here [PuTTY download page](https://www.chiark.greenend.org.uk/~sgtatham/putty/latest.html). - -2. Type the address of your assigned login node as the hostname (see "Determine which login node to use" above). - -3. In the left hand menu, click the "+" next to "SSH" to expand the menu. - -4. Click "Auth" in the "SSH" menu. - -5. Click "Browse" and specify the private key file you saved in step 5 above. - -6. Return to "Session". -    a. Name your session -    b. Save session for future use -7. Click "Open" to launch shell. Provide your ssh-key passphrase (created at Step 4 in PuTTYgen) when prompted to do so. - -## Getting Help - -For assistance or questions, please email the OSG Research Facilitation team at or visit the [help desk and community forums](http://support.opensciencegrid.org). - diff --git a/start/account/registration-and-login.md b/start/account/registration-and-login.md deleted file mode 100644 index 96b5deda..00000000 --- a/start/account/registration-and-login.md +++ /dev/null @@ -1,123 +0,0 @@ -[title]: - "Registration and Login for OSG Connect" - -[TOC] - -## Registration and Login for OSG Connect - -The major steps to getting started on OSG Connect are: - -* apply for an OSG Connect account -* meet with an OSG Connect staff member for an short consultation and orientation. -* join and set your default "project" -* upload `.ssh` keys to the OSG Connect website - -Each of these is detailed in the guide below. - -Once you've gone through these steps, you should be able to login to the OSG Connect -submit node. - -## Account Creation - -### Sign in to or create an account - -Start by creating an OSG Connect account. Visit the [OSG Connect web -site](), then click on the **Sign Up** button. You -will need to agree to our Acceptable Use Policy in order to get to the -main log in screen. - -The main log in screen will prompt you to sign in via your primary institutional -affiliation. You'll be directed to a discovery -service which asks you what your home institution is. (If you've used CILogon -before it may already know your home institution and skip this step.) Locate -your institution in the list, or type its name to find matches. - -> **No Institutional Identity?** -> -> If you don't have an institutional identity or can't find your institution -> on the provided list, either (separately) sign up for a Globus ID and follow -> the link for that option on this page, or contact the OSG Connect support -> team for guidance at support@opensciencegrid.org - -Note that this is the identity that will get linked to your OSG Connect account, -so be sure to pick the institution (if you have multiple affiliations) that -you would like to associate with your OSG Connect account. - -After selecting your institution in the discovery service, you'll be taken to -your own institution's local sign-in screen. You've probably used it before, -and if it looks familiar that's because it's exactly the same web site.  Sign in -using your campus credentials. When done, you'll return automatically to the -OSG Connect portal and can carry on with signup. - -> **Returning to OSG Connect**? -> -> If you already have an OSG Connect account and are not being taken to your -> profile page after logging in with your institution's credentials, -> see our [transition guide](12000065909#action-items) for -> how to proceed. - -After continuing, and allowing certain permissions in the next screen, you'll be -asked to create a profile and save changes. If this works successfully, you should -see that your membership to OSG is "pending" on the right hand side of the screen. - -### Orientation Meeting - -Once you've applied to join OSG Connect as described above, an OSG Connect support -team member will contact you to arrange an initial orientation meeting. This meeting -generally takes about 20-30 minutes and is a chance to talk about your work, how it will -fit on the OSG, and some practical next steps for getting started. Some of these -next steps are also listed below. - -### Join a Project - -As part of the sign up and meeting process, you'll be asked for information related -to your research group so that you can be assigned to an accounting project. For -more information about this process, see this guide: [Start or Join a Project in OSG Connect][projects] - -### Generate and Add an SSH key - -Once your account is created and you're approved, you can generate and upload an -SSH key to the OSG Connect website; this key will be duplicated on the OSG Connect -login node so that you're able to log in there and submit jobs. - -To see how to generate and add an SSH key, please visit this -page: [Step by step instructions to generate and adding an SSH key][ssh-key] - -## Log In - -Once you've gone through the steps above, you should be able to log in to the OSG Connect -login node. See the second half of the SSH key guide for details: -[How to Log Into the OSG Connect Login Node][ssh-key] - -## Overview of access procedure and accounting - -For those interested in details, this section describes some of the background -information behind projects. - -The OSG governs access to grid resources through an accounting -framework that assigns each user's *jobs* to an *accounting group* or *project*. -As a new user of the OSG, one of the first things to iron out is what -project or projects best describe your work.  This is more a matter of -accountability than of entitlement: it concerns how organizations report to -their sponsors and funding agencies on the utilization of resources placed under -their administration. - -To assist in this, OSG Connect uses a group management tool that places users -into one or more groups with names such as *osg.RDCEP* or *osg.Extenci*. -The *osg* portion of this name differentiates our groups from those of -other organizations in the same group management facility. The latter portion -identifies the specific project, each with a Principal Investigator or other -administrator, that oversees access to resources. Within our web tools, these -names are often in mixed case, though you may see them uppercased in some -reporting/accounting software. - -The first step in registration is to create a user account and *bind* it to -other identity information. After that you will enroll in a project group. - -Once you've enrolled in a group, you'll have the requisite rights to log in to -the submit node for the OSG Connect job scheduler, or to transfer data in and -out of Stash. Submit node logins are typically via Secure Shell (SSH) using a -password or a public key. We'll discuss how to connect further on. - -[ssh-key]: 12000027675 -[projects]: 5000634360 - diff --git a/start/account/starting-project.md b/start/account/starting-project.md deleted file mode 100644 index ca441861..00000000 --- a/start/account/starting-project.md +++ /dev/null @@ -1,73 +0,0 @@ -[title]: - "Join and Use a Project in OSG Connect" -[TOC] - -## Background - -The OSG Connect team assigns individual user accounts to "projects". These projects -are a way to track usage hours and capture information about the types of -research using OSG Connect. - -A project typically corresponds to a research group headed by a single PI, but can -sometimes represent a long-term multi-institutional project or some other grouping. - -You must be a member of a project before you can use OSG Connect to submit jobs. -The next section of this guide describes the process for joining an OSG Connect project. - -## Joining a Project - -### Project Membership via Account Creation Process (Default) - -You will be added to a project when going through the typical -OSG Connect account setup process. After applying for an OSG Connect account, -you will receive an email to set up a consultation meeting and confirm which -'OSG Project' your usage should be associated with. You will be prompted to provide information -based on the following two scenarios: - -- **If you are the first member of your research group / team to use the OSG through -OSG Connect**, a new project will be created for you. You will need to provide the following information to do so: - - Project Name - - PI Name - - PI Email - - PI Organization - - PI Department - - Field of Science: (out of https://osp.unm.edu/pi-resources/nsf-research-classifications.html) - - Project Description -- **If you know that other members of your research group have used OSG Connect** in the past, -you can likely join a pre-existing group. Provide the name of your institution and PI -to the OSG Connect team (if you haven't already) and we can confirm. - -Based on this information, OSG Connect support staff will either create a project and -add you to it, or add you to an existing project when your account is approved. - -### Join a Project - -If you need to join an existing project (you can be a member of more than one), please email the -OSG team (support@opensciencegrid.org) with your name and the project -you wish to join, with PI in CC to confirm. - -## "Set" your OSG Connect project - -Job submission on OSG Connect requires a project be assigned to your account -on the login node. This can be set after you have been added to a project as -described above. - -* **Option 1 (preferred)**: To set your default project, sign in to your login node and type - - $ connect project - -You should see a list of projects that you have joined. Most often there will -only be one option! Make sure the right project is highlighted and press "enter" -to save that choice. - -* **Option 2**: If need to run jobs under a different project you are a member of (not your default), you can manually -set the project for those jobs by putting this option in the submit file: - - +ProjectName="ProjectName" - -## View Metrics For Your Project - -The project's resource usage appears in the OSG accounting system, [GRACC](https://gracc.opensciencegrid.org/d/000000033/osg-project-accounting?orgId=1). -You can see the main OSG Connect dashboard here: [Link to OSG Connect Dashboard](https://gracc.opensciencegrid.org/d/000000099/osg-connect-summary-osgconnect-net-submit-hosts-only?orgId=1) - -At the top of that dashboard, there is a set of filters that you can use to examine -the number of hours used by your project, specific users, or your institution. diff --git a/start/data/file-transfer-via-htcondor.md b/start/data/file-transfer-via-htcondor.md deleted file mode 100644 index 44bd8143..00000000 --- a/start/data/file-transfer-via-htcondor.md +++ /dev/null @@ -1,59 +0,0 @@ -[title]: - "Transfer Input Files Up To 100MB In Size" - -[TOC] - -# Overview - -Due to the distributed configuration of the OSG, more often than not, -your jobs will need to bring along a copy (i.e. transfer a copy) of -data, code, packages, software, etc. from the login node where the job -is submitted to the execute node where the job will run. This requirement -applies to any and all files that are needed to successfully execute and -complete your job that do not otherwise exist on OSG execute servers. - -**This guide will describe steps and important considerations for transferring -input files that are <100MB in size via the HTCondor submit file.** - -# Important Considerations - -As described in the [Introduction to Data Management on OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/12000002985) -any data, files, or even software that is <100MB should be staged in -your `/home` directory on your login node. Files in your -`/home` directory can be transferred to jobs via your HTCondor submit file. - -# Transfer Files From `/home` Using HTCondor - -To transfer files from your `/home` directory use the `transfer_input_files` -statement in your HTCondor submit file. For example: - - # submit file example - - log = my_job.$(Cluster).$(Process).log - error = my_job.$(Cluster).$(Process).err - output = my_job.$(Cluster).$(Process).out - - # transfer small file from /home - transfer_input_files = my_data.csv - - ...other submit file details... - -Multiple files can be specified using a comma-separated list, for example: - - # transfer multiple files from /home - transfer_input_files = my_data.csv, my_software.tar.gz, my_script.py - -When using `transfer_input_files` to transfer files located in `/home`, -keep in mind that the path to the file is relative to the location of -the submit file. If you have files located in a different `/home` subdirectory, -we recommend specifying the full path to those files, which is also a matter -of good practice, for example: - - transfer_input_files = /home/username/path/to/my_software.tar.gz - -Where `username` refers to your OSG Connect username. - -# Get Help - -For assistance or questions, please email the OSG Research Facilitation team -at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) or visit the -[help desk and community forums](http://support.opensciencegrid.org). diff --git a/start/data/file-transfer-via-http.md b/start/data/file-transfer-via-http.md deleted file mode 100644 index 7282fdca..00000000 --- a/start/data/file-transfer-via-http.md +++ /dev/null @@ -1,52 +0,0 @@ -[title]: - "Transfer HTTP-available Files up to 1GB In Size" - -[TOC] - -# Overview - -If some of the data or software your jobs depend on is available via the web, -you can have such files transferred by HTCondor using the appropriate HTTP address! - -# Important Considerations - -While our [Overview of Data Mangement on OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/12000002985) -describes how you can stage data, files, or even software in OSG Connect locations, -any web-accessible file can be transferred directly to your jobs **IF**: - -- the file is accessible via an HTTP address -- the file is less than 1GB in size (if larger, you'll need to pre-stage them for [stash-based transfer](12000002775) -- the server or website they're on can handle large numbers of your jobs accessing them simultaneously - -Importantly, you'll also want to make sure your job executable knows how to handle the file -(un-tar, etc.) from within the working directory of the job, just like it would for any other input file. - -# Transfer Files via HTTP - -Use an HTTP URL in -combination with the `transfer_input_files` statement in your HTCondor submit file. - -For example: - - # submit file example - - log = my_job.$(Cluster).$(Process).log - error = my_job.$(Cluster).$(Process).err - output = my_job.$(Cluster).$(Process).out - - # transfer software tarball from public via http - transfer_input_files = http://www.website.com/path/file.tar.gz - - ...other submit file details... - -Multiple URLs can -be specified using a comma-separated list, and a combination of URLs and -files from `/home` directory can be provided in a comma separated list. For example, - - # transfer software tarball from public via http - # transfer input data from home via htcondor file transfer - transfer_input_files = http://www.website.com/path/file1.tar.gz, http://www.website.com/path/file2.tar.gz, my_data.csv - -# Get Help - -For assistance or questions, please email the OSG Research Facilitation team at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) or visit the [help desk and community forums](http://support.opensciencegrid.org). - diff --git a/start/data/osgconnect-storage.md b/start/data/osgconnect-storage.md deleted file mode 100644 index 1eca6847..00000000 --- a/start/data/osgconnect-storage.md +++ /dev/null @@ -1,145 +0,0 @@ -[title]: - "Overview: Data Staging and Transfer to Jobs" - -[TOC] - -# Overview - -OSG Connect provides two locations for uploading files (data and software) that are -needed for running jobs: - - home: /home/ (general storage from which ALL jobs should be submitted) - public: /public/ (only for large job input and output using stash links (input) and stashcp (output)) - -In general, OSG Connect users are responsible for managing data in -these folders and for using appropriate mechanisms for delivering -data to/from jobs, as detailed below. Each is controlled with a -quota and should be treated as temporary storage for _active_ -job execution. OSG Connect has no routine backup of data in these -locations, and users should remove old data after jobs complete, -in part, to make room for future submissions. **If you think you'll -need more space for a set of concurrently-queued jobs, even after -cleaning up old data, please send a request to -[support@opensciencegrid.org](mailto:support@opensciencegrid.org)!** - -For additional data information, see also the "Data Storage and Transfer" section of -our [FAQ](5000634384#data-storage-and-transfer). - -**Note: OSG Connect staff reserve the right to monitor and/or remove -data without notice to the user IF doing so is necessary for ensuring -proper use or to quickly fix a performance or security issue.** - -# Data Locations and Quotas - -Your OSG Connect account includes access to two data storage locations: -`/home` and `/public`. **Where you store your files and how your files -are made accessible to your jobs depends on the size of the file and how -much data is needed or produced by your jobs.** - -| **Location** | **Storage Needs** | **Network mounted** | **Backed Up?** | **Initial Quota** | -| :-----: | :------: | :------: | :------: | :------: | -| `/home` | Storage of submit files, input files <100MB each, and per-job output
up to a 1GB. Jobs should ONLY be submitted from this folder. | No | No | 50 GB | -| `/public` | Staging ONLY for large input files (100MB-50GB, each) for publicly-accessible
download into jobs (using `stash:///` links or `stashcp` see below) and large output files (1-10GB). | Yes | No | 500 GB | - -
- -Your quota status will be displayed when you connect to your OSG Connect login node: - - Disk utilization for username: - /public : [ ] 0% (0/500000 MB) - /home : [ # ] 4% (2147/53687 MB) - -You can also display your quota usage at any time using the command -`quota` while connected to your login node. - -**Don't hesitate to contact us at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) if you -think you need a quota increase! We can support very large amounts of data.** - -## `/home` Usage And Policies - -User directories within `/home` are meant for general-use storage of your files -needed for job submission. The initial quota per user is 50 GBs, and can be increased by request -to support@opensciencegrid.org when a user needs more space for appropriately-sized files. - -**ALL JOBS MUST BE SUBMITTED FROM WITHIN `/home`**. Users are also prohibited -from making their `/home` directory world-readable due to security concerns. See -[Policies for Using OSG via OSG Connect Submit Servers](https://support.opensciencegrid.org/support/solutions/articles/12000074852) for more details. - -If you're unable to submit jobs or your jobs are going on hold because you've -reached your `/home` quota, please contact us at -[support@opensciencegrid.org](mailto:support@opensciencegrid.org) about a quota increase. - -## `/public` Usage and Policies - -User directories within `/public` are meant **ONLY** for staging job files too large for regular -HTCondor file transfer (per-job input greater than 100MB, or per-job output greater than 1GB), and should only OSG caching -mechanisms (see tables for input and output, further below). - -**JOBS MUST NEVER BE SUBMITTED FROM WITHIN `/public`**, and should not list `/public` files in the -`transfer_input_files` or other lines of a submit file, unless as a `stash:///` address (see tables further below). -Files placed in `/public` should only be accessed by jobs using the below tools (see -[Transferring Data To/From Jobs](#transferring-data-tofrom-jobs)). Users violating these policies may -lose the ability to submit jobs until their submissions are corrected. - -The initial disk quota of `/public` is 500 GBs. Contact [support@opensciencegrid.org](mailto:support@opensciencegrid.org) -if you will need an increase for concurrently running work, after cleaning up all -data from past jobs. **Given that users should not be storing long-term data (like submit files, -software, etc.) in `/public`, files and directories that have not been accessed for over six -months may be deleted by OSG Connect staff with or without notifying the user.** - -Files placed within a user's `/public` directory **are publicly accessible**, -discoverable and readable by anyone. Data is made public via the `stash` transfer mechanisms (which also make data public via http/https), and mirrored -to a shared data repository which is available on a large number of systems around the world. - -### Is there any support for private data? - -If you do not want your data to be downloadable by anyone, and it's small enough for -HTCondor file transfer, then it should be staged in your `/home` directory and -transferred to jobs with HTCondor file transfer (`transfer_input_files` in the submit -file). If it cannot be public (cannot use http or stash for job delivery), and is too -large for HTCondor file transfer, then it's not a good fit for the open environment of -the Open Science Pool, and another resource will likely be more appropriate. As a -reminder, if the data is not being used for active computing work on OSG Connect, it -should not be stored on OSG Connect systems, and our data storage locations are not -backed up or suitable for project-duration storage. - -## External Data Transfer to/from OSG Connect Login Nodes - -In general, common Unix tools such as `rsync`, `scp`, Putty, WinSCP, `gFTP`, etc. can be used to upload data from your computer to the OSG Connect login node, or to download files from the OSG Connect login node. - -
- -# Transferring Data To/From Jobs - -## Transferring Input Data to Jobs - -This table summarizes the options for sending input files from the OSG Connect -login node to the execution node where a job is running. This assumes that you -have already uploaded these input files from your own computer to your OSG Connect login node. - -| **Transfer Method** | **File Sizes**| **File Location** | **Command** | **More Info** | -| :--------: | :------: | :-----: | :-----: | :--------: | -| **regular HTCondor
file transfer** | <100 MB;
<500 MB total per job | `/home` | `transfer_input_files` | [HTCondor File Transfer](https://support.opensciencegrid.org/support/solutions/articles/5000639787)| -| **OSG's
transfer tools** | >1GB;
<10 GB per job | `/public` | `stash` address in `transfer_input_files` | [Stash Transfer](https://support.opensciencegrid.org/support/solutions/articles/12000002775)| -| **HTTP** | <1GB | non-OSG web server | `http` address in `transfer_input_files` |[HTTP Access](https://support.opensciencegrid.org/support/solutions/articles/5000639798)| -| **GridFTP** | > 10 GB | `/public` or non-OSG data server | `gfal-copy` | [contact us](mailto:support@opensciencegrid.org) | - -
- -## Transferring Output Data from Jobs - -This table summarizes a job's options for returning output files generated by the job back to the OSG Connect login node. - -| **Transfer Method** | **File Sizes** | **Transfer To** | **Command** | **More Info** | -| :---------: | :------: | :-----: | :--------: | :------: | -| **regular HTCondor file transfer** | < 1 GB | `/home` | HTCondor default output transfer or `transfer_output_files` | [HTCondor Transfer](https://support.opensciencegrid.org/support/solutions/articles/5000639787)| -| **OSG's transfer tools** | >1GB and
< 10 GB | `/public` | `stashcp` in job executable | [stashcp](https://support.opensciencegrid.org/support/solutions/articles/12000002775) | -| **GridFTP** | > 10 GB | `/public` | `gfal-copy` | [contact us](mailto:support@opensciencegrid.org) | - -
- -**Watch this video from the 2021 OSG Virtual School** for more information about Handling Data on OSG: - -[](https://www.youtube.com/embed/YBGWycYZRD4) - -# Get Help -For assistance or questions, please email the OSG Research Facilitation team at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) or visit the [help desk and community forums](http://support.opensciencegrid.org). diff --git a/start/data/output-file-transfer-via-htcondor.md b/start/data/output-file-transfer-via-htcondor.md deleted file mode 100644 index 547e61e6..00000000 --- a/start/data/output-file-transfer-via-htcondor.md +++ /dev/null @@ -1,103 +0,0 @@ -[title]: - "Transfer Job Output Files Up To 1GB In Size" - -[TOC] - -# Overview - -When your OSG Connect jobs run, any output that gets generated is specifically written to -the execute node on which the job ran. In order to get access to your output files, a copy of -the output must be transferred back to your OSG Connect login node. - -This guide will describe the necessary steps, along with important considerations, for transferring your -output files back to your `/home` directory on your OSG Connect login node. - -# Important Considerations - -As described in the [Introduction to Data Management on OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/12000002985), -any output <1GB should be staged in your `/home` directory. For output files >1GB, -please refer to our [Transfer Large Input and Output Files >1GB In Size](https://support.opensciencegrid.org/support/solutions/articles/12000002775) guide. - -**If your jobs use any input files >1GB that are transferred from your `/public` directory -using StacheCash, it is important that these files get deleted from the job's working directory or moved to a -subdirectory so that HTCondor will not transfer these large files back to your `/home` directory.** - -# Use HTCondor To Transfer Output <1GB - -By default, HTCondor will transfer any new or modified files in the job's -top-level directory back to your `/home` directory location from which -the `condor_submit` command was performed. **This behavior only applies -to files in the top-level directory of -where your job executes, meaning HTCondor will ignore any files -created in subdirectories of the job's top-level directory.** Several -options exist for modifying this default output file transfer behavior, including -those described in this guide. To learn more, please contact us -at [support@opensciencegrid.org](mailto:support@opensciencegrid.org). - -*What is the top-level directory of a job?* - -Before executing a job, HTCondor will create a new directory on the execute -node just for your job - this is the top-level directory of the job and the -path is stored in the environment variable `_CONDOR_SCRATCH_DIR`. All of the -input files transferred via `transfer_input_files` will first be written to -this directory and it is from this path that a job starts to execute. After -a job has completed the top-level directory and all of it's contents are -deleted. - -*What if my output file(s) are not written to the top-level directory?* - -If your output files are written to a subdirectory, use the steps described -[below](#group-multiple-output-files-for-convenience) to convert the output -directory to a "tarball" that is written to the top-level directory. - -Alternatively, you can include steps in the executable bash script of -your job to move (i.e. `mv`) output files from a subdirectory to -the top-level directory. For example, if there is an output file that -needs to be transferred back to the login node named `job_output.txt` -written to `job_output/`: - - #! /bin/bash - - # various commands needed to run your job - - # move csv files to scratch dir - mv job_output/job_output.txt $_CONDOR_SCRATCH_DIR - -## Group Multiple Output Files For Convenience - -If your jobs will generate multiple output files, we recommend combining all output into a compressed -tar archive for convenience, particularly when transferring your results to your local computer from -your login node. To create a compressed tar archive, include commands in your your bash executable script -to create a new subdirectory, move all of the output to this new subdirectory, and create a tar archive. -For example: - - #! /bin/bash - - # various commands needed to run your job - - # create output tar archive - mkidr my_output - mv my_job_output.csv my_job_output.svg my_output/ - tar -czf my_job.output.tar.gz my_ouput/ - -The example above will create a file called `my_job.output.tar.gz` that contains all the output that -was moved to `my_output`. Be sure to create `my_job.output.tar.gz` in the top-level directory of where -your job executes and HTCondor will automatically transfer this tar archive back to your `/home` -directory. - -## Select Specific Output Files To Transfer to `/home` Using HTCondor - -As described above, HTCondor will, by default, transfer any files that are generated during the -execution of your job(s) back to your `/home` directory. If your job(s) will produce multiple output -files but you only need to retain a subset of these output files, we recommend deleting the unrequired -output files or moving them to a subdirectory as a step in the bash -executable script of your job - only the output files that remain in the top-level -directory will be transferred back to your `/home` directory. - -In cases where a bash script is not used as the excutable of your job and you wish to have only specific -output files transferred back, please contact us at [support@opensciencegrid.org](mailto:support@opensciencegrid.org). - -# Get Additional Options For Managing Job Output - -Several options exist for managing output file transfers back to your `/home` directory and we -encourage you to get in touch with us at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) to -help identify the best solution for your needs. diff --git a/start/data/scp.md b/start/data/scp.md deleted file mode 100644 index dadafe23..00000000 --- a/start/data/scp.md +++ /dev/null @@ -1,73 +0,0 @@ -[title]: - "Use scp To Transfer Files To and From OSG Connect" - -[TOC] - -# Overview - -This tutorial assumes that you will be using a command line application -for performing file transfers instead of a GUI-based application such as WinSCP. - -We can transfer files to and from the OSG Connect login node using the -`scp` command. Note `scp` is a counterpart to the secure shell -command,`ssh`, that allows for secure, encrypted file transfers between -systems using your ssh credentials. - -When using `scp`, you will always need to specify both the source of the -content that you wish to copy and the destination of where you would like -the copy to end up. For example: - - $ scp - -Files on remote systems (like an OSG Connect login node) are indicated using -`username@machine:/path/to/file`. - -# Transfer Files To OSG Connect - -Let's say you have a file you wish to transfer to OSG Connect named `my_file.txt`. - -Using the terminal application on your computer, navigate to the location of `my_file.txt`. - -Then use the following `scp` command to tranfer `my_file.txt` to your `/home` on OSG Connect. Note -that you will **not** be logged into OSG Connect when you perform this step. - - $ scp my_file.txt username@loginNN.osgconnect.net:/home/username/ - -Where **NN** is the specific number of your assigned login node (i.e. `04` or `05`). - -Large files (>100MB in size) can be uploaded to your `/public` directory also using `scp`: - - $ scp my_large_file.gz username@loginNN.osgconnect.net:/public/username/ - -## Transfer Directories To OSG Connect - -To copy directories using `scp`, add the (recursive) `-r` option to your scp command. - -For example: - - $ scp -r my_Dir username@loginNN.osgconnect.net:/home/username/ - -# Transfer Files From OSG Connect - -To transfer files from OSG Connect back to your laptop or desktop you can use the `scp` as shown above, -but with the source being the copy that is located on OSG Connect: - - $ scp username@loginNN.osgconnect.net:/home/username/my_file.txt ./ - -where `./` sets the destination of the copy to your current location on your computer - -# Transfer Files Between OSG Connect and Another Server - -`scp` can be used to transfer files between OSG Connect and another server that you have -`ssh` access to. This means that files don't have to first be transferred to your -personal computer which can save a lot of time and effort! For example, to transfer -a file from another server and your OSG Connect login node `/home` directory: - - $ scp username@serverhostname:/path/to/my_file.txt username@loginNN.osgconnect.net:/home/username - -Be sure to use the username assigned to you on the other server and to provide the -full path on the other server to your file. - -# Get Help - -For assistance or questions, please email the OSG Research Facilitation team at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) or visit the [help desk and community forums](http://support.opensciencegrid.org). - diff --git a/start/data/stashcache.md b/start/data/stashcache.md deleted file mode 100644 index d72efe40..00000000 --- a/start/data/stashcache.md +++ /dev/null @@ -1,192 +0,0 @@ -[title]: - "Transfer Larger Input and Output Files" - -[TOC] - -# Overview - -For input files >100MB and output files >1GB in size, the default HTCondor file transfer mechanisms -run the risk of over-taxing the login nodes and their network capacity. And this is exactly why the -**[OSG Data Federation](https://opensciencegrid.org/about/osdf/)** exists for researchers with larger -per-job data! - -Users on an -OSG Connect login node can handle such files via the OSG Connect **data caching origin** -(mounted and visible as the `/public` location) and use OSG's caching tools to -scalably transfer them between the running jobs and the origin. -The OSG caching tools ensure faster delivery to and from execute nodes by taking adantage of -regional data caches in the OSG Data Federation, while preserving login node performance. - -# Important Considerations and Best Practices - -1. As described in OSG Connect's [Introduction to Data Management on OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/12000002985), -the `/public` location **must** be used for: - - - Any **input data or software larger than 100MB** for - transfer to jobs using OSG caching tools - - Any per-job **output >1GB and <10GB**, which - should ONLY be transferred back to the origin using a `stashcp` command within the job executable. - -2. **User must never submit jobs from the `/public` location,** and should continue to -ONLY submit jobs from within their `/home` directory. All `log`, `error`, `output` -files and any other files smaller than the above values should ONLY ever -exist within the user's /home directory, unless otherwise directed by an OSG staff member. - - Thus, **files within the `/public` location should only be referenced within - the submit file by using the methods described further below**, and should - never be listed for direct HTCondor transfer via `transfer_input_files`, - `transfer_output_files`, or `transfer_output_remaps`. - - The `/public` location is a mount of the OSG Connect origin filesystem. It is mounted to the - OSG Connect login nodes only so that users can appropriately stage large job inputs or retrieve outputs via - the login nodes. - -3. Because of impacts to the filesystem of the data origin, **files in the data origin (`/public`) should -be organized in one or very few files per job.** The filesystem is likely to encounter performance issues -if/when files accumulated there are highly numerous and/or small. - -4. **The `/public` location is otherwise unnecessary for smaller files** (which can and should be served -via the user's /home directory and regular HTCondor file transfer). Smaller files should only be handled -via `/public` with explicit instruction from an OSG staff member. - -5. **Files placed within a user's `/public` directory are publicly accessible**, -discoverable and readable by anyone, via the web. Data is made public via `stash` -transfer (and, thus, via http addresses), and mirrored to a shared data repository -which is available on a large number of systems around the world. - - -# Use a 'stash' URL to Transfer Large Input Files to Jobs - -Jobs submitted from the OSG Connect login nodes will transfer data from the origin -when files are indicated with an appropriate `stash:///` URL in the `transfer_input_files` line -of the submit file: - -1. Upload your larger input and/or software files to your `/public` directory -which is accessible via your OSG Connect login node at `/public/username` -for which our -[Using scp To Transfer Files To OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/5000634376) -guide may be helpful. - - **Because of the way your files in `/public` are cached across the Open Science Pool, - any changes or modifications that you make after placing a file in `/public` - will not be propagated.** This means that if you add a new version - of a file to your `/public` directory, it must first be given a unique name (or directory path) - to distinguish it from previous versions of that file. Adding a date or - version number to directories or file names is strongly encouraged to manage your files in - `/public`. - -2. Add the necessary details to your HTCondor submit file to tell -HTCondor which files to transfer, and that your jobs must run on executes nodes that -have access to the Open Science Data Federation. - - # Submit file example of large input/software transfer - - log = my_job.$(Cluster).$(Process).log - error = my_job.$(Cluster).$(Process).err - output = my_job.$(Cluster).$(Process).out - - #Transfer input files - transfer_input_files = stash:///osgconnect/public///, - - ...other submit file details... - - - **Note how the `/public` mount (visible on the login node) corresponds to the `/osgconnect/public` namespace - across the Open Science Federation.** For example, if the data file is located at - `/public//samples/sample01.dat`, then the `stash:///` URL to - transfer this file to the job's working directory on the execute point would be: - - stash:///osgconnect/public//samples/sample01.dat - -# Use `stashcp` to Transfer Larger Job Outputs to the Data Origin - -For output, users should use the **`stashcp`** command within their job executable, -which will transfer the user's specified file to the specific location in the data origin. - -[Remember that you should NEVER list a `/public` location -within the submit file (e.g. in 'transfer_output_remaps`) or submit jobs from within `/public`](https://support.opensciencegrid.org/support/solutions/articles/12000002985). - -1. Add the necessary details to your HTCondor submit file to tell -HTCondor that your jobs must run on executes nodes that -have access to the `stashcp` module (among other --supported modules). Note that the output files -are NOT listed anywhere in the submit file for transfer purposes. - - # submit file example for large output - - log = my_job.$(Cluster).$(Process).log - error = my_job.$(Cluster).$(Process).err - output = my_job.$(Cluster).$(Process).out - - requirements = (OSGVO_OS_STRING =?= "RHEL 7") && (HAS_MODULES =?= true) - - ...other submit file details... - -2. Add a `stashcp` command at the end of your executable to transfer the data files back to the OSG Connect data origin (within `/public`). You will -need to prepend your `/public` directory path with `stash:///osgconnect` as follows: - - #!/bin/bash - - # other commands to be executed in job: - - # transfer large output to public - stashcp stash:///osgconnect/public/username/path/ - - For example, if you wish to transfer `output.dat` to the directory - `/public//output/` then the `stash` command would be: - - stashcp output.dat stash:///osgconnect/public//output/output.dat - - **Note that the output file name must also be included at the end of the - `/public` path where the file will be transferred, which also allows you to rename the file.** - - - -# Stachcp Command Manual - -More usage options are described in the stashcp help message: - - $ stashcp -h - Usage: stashcp [options] source destination - - Options: - -h, --help show this help message and exit - -d, --debug debug - -r recursively copy - --closest Return the closest cache and exit - -c CACHE, --cache=CACHE - Cache to use - -j CACHES_JSON, --caches-json=CACHES_JSON - The JSON file containing the list of caches - --methods=METHODS Comma separated list of methods to try, in order. - Default: cvmfs,xrootd,http - -t TOKEN, --token=TOKEN - Token file to use for reading and/or writing - -# Get Help - -For assistance or questions, please email the OSG Research Facilitation team -at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) or visit -the [help desk and community forums](http://support.opensciencegrid.org). diff --git a/start/jobdurationcategory.md b/start/jobdurationcategory.md deleted file mode 100644 index c37abdf9..00000000 --- a/start/jobdurationcategory.md +++ /dev/null @@ -1,59 +0,0 @@ -[title]: - "Indicate the Duration Category of Your Jobs" - -[TOC] -## Why Job Duration Categories? -To maximize the value of the capacity contributed by the different organizations to the Open Science Pool (OSPool), -users are requested to identify one of three duration categories for their jobs. These categories should be selected based upon test -jobs (run on the OSPool) and allow for more effective scheduling of the capacity contributed to the pool, -honoring the community’s shared responsibility for efficient use of the contributed resources. As a reminder, -[jobs with single executions longer than 20 hours in tests on the OSPool should not be submitted](5000632058), without -self-checkpointing (see further below). - -Every job submitted via an OSG Connect access point must -be labeled with a **Job Duration Category** upon submission. -By knowing the expected duration, the OSG is working to be able to direct longer-running jobs to resources that are -faster and are interrupted less, while shorter jobs can run across more of the OSPool for better overall throughput. - -## Specify a Job Duration Category -The JobDurationCategory must be listed anywhere prior to the final ‘queue’ statement of the submit file, as below: - - +JobDurationCategory = “Long” - -| **JobDurationCategory** | **Expected Job Duration** | Maximum Allowed Duration | -|:---------|:------------|:-------------| -| Medium (default) | <10 hrs | 20 hrs | -| Long | <20 hrs | 40 hrs | - - -If the user does not indicate a JobDurationCategory in the submit file, the relevant job(s) will be -labeled as **Medium** by default. **Batches with jobs that individually execute for longer than 20 hours -are not a good fit for the OSPool**. *If your jobs are self-checkpointing, -see “Self-Checkpointing Jobs”, further below.* - -## Test Jobs for Expected Duration -As part of the [preparation for running a full-scale job batch](https://support.opensciencegrid.org/support/solutions/articles/12000076552-always-test-a-few-jobs-before-submitting-many), -users should test a subset (first ~10, then 100 or 1000) of their jobs with the **Medium** or **Long** categories, -and then review actual job execution durations in the job log files. -If the user expects potentially significant variation in job durations within a single batch, a longer JobDurationCategory may be warranted relative to the duration of test jobs. Or, if variations in job duration may be predictable, the user may choose to submit different -subsets of jobs with different Job Duration Categories. - -**OSG Facilitators have a lot of experience with approaches for achieving shorter jobs (e.g. breaking up work into shorter, more numerous jobs; self-checkpointing; automated sequential job submissions; etc.) Get in touch, and we'll help you work through a solution!! support@opensciencegrid.org** - -## Maximum Allowed Duration -Jobs in each category will be placed on hold in the queue if they run longer than their Maximum Allowed Duration -(starting Tuesday, Nov 16, 2021). In that case, the user may remove and resubmit the jobs, identifying a longer category. - -**Jobs that test as longer than 20 hours are not a good fit for the OSPool resources, and should not be submitted prior to contacting** -[support@opensciencegrid.org](mailto:support@opensciencegrid.org) **to discuss options**. The Maximum Allowed Durations -are longer than the Expected Job Durations in order to accommodate CPU speed variations across OSPool computing resources, -as well as other contributions to job duration that may not be apparent in smaller test batches. -Similarly, **Long** jobs held after running longer -than 40 hours represent significant wasted capacity and should never be released or resubmitted by the user without -first taking steps to modify and test the jobs to run shorter. - -## Self-Checkpointing Jobs -Jobs that [self-checkpoint](https://htcondor.readthedocs.io/en/latest/users-manual/self-checkpointing-applications.html) -at least every 10 hours are an excellent way for users to run jobs that would otherwise be longer in total execution time -than the durations listed above. Jobs that complete a checkpoint at least as often as allowed for their JobDurationCategory will not be held. - -We are excited to help you think through and implement self-checkpointing. Get in touch via support@opensciencegrid.org if you have questions. :) diff --git a/start/jobs/tutorial-command.md b/start/jobs/tutorial-command.md deleted file mode 100644 index ccd79c78..00000000 --- a/start/jobs/tutorial-command.md +++ /dev/null @@ -1,69 +0,0 @@ -[title]: - "Use OSG Connect Tutorials" -[TOC] - -OSG Connect tutorials on Github -------------------------------- - -All of the OSG Connect tutorials are available as repositories on -[Github](). These -tutorials are tested regularly and should work as is, but if -you experience any issues please contact us. - -Tutorial commands ------------------ - -From the OSG Connect login node, the following tutorial -commands are available: - - $ tutorial - $ tutorial list - $ tutorial info - $ tutorial - -Available tutorials -------------------- - -The OSG Connect login nodes have the following tutorials -pre-installed. To see what is available: - - $ tutorial list - Currently available tutorials: - R ...................... Estimate Pi using the R programming language - R-addlibSNA ............ Shows how to add R external libraries for the R jobs on the OSG - ScalingUp-Python ....... Scaling up compute resources - Python example to optimize a function on grid points - annex .................. None - blast-split ............ How to run BLAST on the OSG by splitting a large input file - connect-client ......... Demonstrates how to use the connect client for remote job submission - dagman-wordfreq ........ DAGMan based wordfreq example - dagman-wordfreq-stash .. DAGMan based wordfreq - data from stash - error101 ............... Use condor_q -better-analyze to analyze stuck jobs - exitcode ............... Use HTCondor's periodic_release to retry failed jobs - htcondor-transfer ...... Transfer data via HTCondor's own mechanisms - matlab-HelloWorld ...... Creating standalone MATLAB application - Hello World - nelle-nemo ............. Running Nelle Nemo's goostats on the grid - oasis-parrot ........... Software access with OASIS and Parrot - octave ................. Matrix manipulation via the Octave programming language - osg-locations .......... Tutorial based on OSG location exercise from the User School - pegasus ................ An introduction to the Pegasus job workflow manager - photodemo .............. A complete analysis workflow using HTTP transfer - quickstart ............. How to run your first OSG job - root ................... Inspect ntuples using the ROOT analysis framework - scaling ................ Learn to steer jobs to particular resources - scaling-up-resources ... A simple multi-job demonstration - software ............... Software access tutorial - stash-cvmfs ............ Shows how to use stash-cvmfs for input data transfer - stash-http ............. Retrieve job input files from Stash via HTTP - tensorflow-matmul ...... Tensorflow math operations as a singularity container job on the OSG - matrix multiplication - -Install and setup a tutorial ----------------------------- - -On the OSG Connect login node, create a directory, `cd` -to it, and invoke the command: - - $ tutorial - -This command will clone the tutorial repository to your current working directory. -`cd` to the repository directory and follow the steps described in the `readme.md` file. -Alternatively, you can view the `readme.md` file at the tutorial's corresponding GitHub page. - diff --git a/start/resources/gpu-jobs.md b/start/resources/gpu-jobs.md deleted file mode 100644 index 60b5e103..00000000 --- a/start/resources/gpu-jobs.md +++ /dev/null @@ -1,84 +0,0 @@ -[title]: - "Using GPUs on the OSPool" - -[TOC] - -The Open Science Pool has an increasing number of GPUs available to -run jobs. - -# Requesting GPUs - -To request a GPU for your HTCondor job, you can use the -HTCondor *request_gpus* attribute in your submit file (along -with the usual *request_cpus*, *request_memory*, and *request_disk* -attributes). For example: - - request_gpus = 1 - request_cpus = 1 - request_memory = 4 GB - request_disk = 2 GB - -Currently, a job can only use 1 GPU at the time. - -You should only request a GPU if your software has been written to use a GPU. It is -also worth running test jobs on a GPU versus CPUs-only, to observe the amount of -speed up. - -## Specific GPU Requests - -HTCondor records different GPU attributes that can be used to select -specific types of GPU devices. A few attributes that may be useful: - -* `CUDACapability`: this is NOT the CUDA library, but rather a measure of the GPU's "Compute Capability" -* `CUDADriverVersion`: maximum version of the CUDA libraries that can be supported on the GPU -* `CUDAGlobalMemoryMb`: amount of GPU memory available on the GPU device - -Any of the attributes above can be used in the submit file's `requirements` line to -select a specific kind of GPU. For -example, to request a GPU with more than 8GB of GPU memory, one could use: - - requirements = (CUDAGlobalMemoryMb >= 8192) - -If you want a certain type or family of GPUs, we usually recommend using the GPU's -'Compute Capability', known as the `CUDACapability` by HTCondor. An A100 GPU has a -Compute Capability of 8.0, so if you wanted to run on an A100 GPU specifically, -the submit file requirement would be: - - requirements = (CUDACapability == 8.0) - -**Note that the more requirements you include, the fewer resources will be available -to you! It's always better to set the minimal possible requirements (ideally, none!) -in order to access the greatest amount of computing capacity.** - -# Available GPUs - -Just like CPUs, GPUs are shared with the OSG community only when the -resource is idle. Therefore, we do not know exactly what resources are -available at what time. When requesting a GPU job, you might land on one -of the following types of GPUs: - -* Tesla K20m, K40m (CUDACapability: 3.5) -* Quadro M5000 (CUDACapability: 5.2) -* GeForce GTX 1080 Ti (CUDACapability: 6.1) -* V100 (CUDACapability: 7.0) -* Quadro RTX 6000 (CUDACapability: 7.5) -* A100 (CUDACapability: 8.0) -* A40 (CUDACapability: 8.6) - -# Software and Data Considerations - -## Software for GPUs - -For GPU-enabled machine learning libraries, we recommend using -containers to set up your software for jobs: - - * [Using Containers on the OSPool](https://support.opensciencegrid.org/solution/articles/12000024676-singularity-containers) - * [Sample TensorFlow GPU Container Image Definition](https://github.com/opensciencegrid/osgvo-tensorflow-gpu/blob/master/Dockerfile) - * [TensorFlow Example Job](https://support.opensciencegrid.org/solution/articles/12000028940-tensorflow) - -## Data Needs for GPU Jobs - -As with any kind of job submission, check your data sizes (per job) before submitting -jobs and choose the appropriate file transfer method for your data. - -See our [Data Staging and Transfer guide](https://support.opensciencegrid.org/support/solutions/articles/12000002985-overview-data-staging-and-transfer-to-jobs#transferring-data-tofrom-jobs) for -details and contact the Research Computing Facilitation team with questions. diff --git a/start/resources/large-memory-jobs.md b/start/resources/large-memory-jobs.md deleted file mode 100644 index 6a917d05..00000000 --- a/start/resources/large-memory-jobs.md +++ /dev/null @@ -1,23 +0,0 @@ -[title]: - "Large Memory Jobs" - -[TOC] - -By default, 2 GB of RAM (aka memory) will be assigned to your jobs. However, some jobs will require -additional memory to complete successfully. To request more memory, use the HTCondor *request_memory* -attribute in your submit file. The default unit is MB. For example, the following will request 12 GB: - - request_memory = 12228 - -You might be wondering why the above is requesting 12228 MB for 12 GB. That's because byte units don't -actually scale by 1000 (10^10) like the metric system, but instead scale by 1024 (2^10) due to the binary -nature of bytes. - -Alternatively, you can define a memory request using standard units - - request_memory = 12GB - -We recommend always explictly defining the byte units in your *request_memory* statement. - -Please note that the OSG has limited resources available for large memory jobs. Requesting jobs with -higher memory needs will results in longer than average queue times for these jobs. - diff --git a/start/resources/multicore-jobs.md b/start/resources/multicore-jobs.md deleted file mode 100644 index 3cdf5ebc..00000000 --- a/start/resources/multicore-jobs.md +++ /dev/null @@ -1,20 +0,0 @@ -[title]: - "Multicore Jobs" - -[TOC] - -Please note, the OSG has limited support for multicore jobs. Multicore jobs -can be submitted for threaded or OpenMP applications. To request multiple cores -(aka cpus) use the HTCondor *request_cpus* attribute in your submit file. - -Example: - - request_cpus = 8 - -We recommend requesting a maximum of 8 cpus. - -**Important considerations** -When submitting multicore jobs please note that you will also have to tell -your code or application to use the number of cpus requested in your submit -file. Do not use core auto-detection as it might detect more cores than what -were actually assigned to your job. - diff --git a/start/resources/openmpi-jobs.md b/start/resources/openmpi-jobs.md deleted file mode 100644 index d0144612..00000000 --- a/start/resources/openmpi-jobs.md +++ /dev/null @@ -1,60 +0,0 @@ -[title]: - "OpenMPI Jobs" - -[TOC] - -Even though the Open Science Pool is a high throughput computing system, sometimes -there is a need to run small OpenMPI based jobs. OSG has limited support for -this, as long as the core count is small (4 is known to work well, 8 and 16 -becomes more difficult due to the limited number of resources). - - -To get started, first compile your code using the OpenMPI in the modules -system. For example: - - $ module load openmpi - $ mpicc -o hello hello.c - - -You can test the executable locally using `mpiexec`: - - $ mpiexec -n 4 hello - Hello world from process 1 of 4 - Hello world from process 3 of 4 - Hello world from process 0 of 4 - Hello world from process 2 of 4 - - -To run your code as a job on the Open Science Pool, first create a `wrapper.sh`. Example: - - #!/bin/bash - - set -e - - module load openmpi - - mpiexec -n 4 hello - - -Then, a job submit file: - - Requirements = OSGVO_OS_STRING == "RHEL 7" && TARGET.Arch == "X86_64" && HAS_MODULES == True - request_cpus = 4 - request_memory = 4 GB - - executable = wrapper.sh - - transfer_input_files = hello - - output = job.out.$(Cluster).$(Process) - error = job.error.$(Cluster).$(Process) - log = job.log.$(Cluster).$(Process) - - queue 1 - - -Note how the executable is the `wrapper.sh` script, and that the real executable `hello` is -transferred using the `transfer_input_files` mechanism. - -Please make sure that the number of cores specified in the submit file via -`request_cpus` match the `-n` argument in the `wrapper.sh` file. - diff --git a/start/resources/requirements.md b/start/resources/requirements.md deleted file mode 100644 index 46616cfd..00000000 --- a/start/resources/requirements.md +++ /dev/null @@ -1,117 +0,0 @@ -[title]: - "Control Where Your Jobs Run / Job Requirements" - -[TOC] - -By default, your jobs will match any available spot in the OSG. This is fine -for very generic jobs. However, in some cases a job may have one or more system -requirements in order to complete successfully. For instance, your job may need to run -on a node with a specific operating system. - -HTCondor provides several options for "steering" your jobs to appropriate -nodes and system environments. The `request_cpus`, `request_gpus`, `request_memory`, and `request_disk` -submit file attributes should be used to specify the hardware needs of your jobs. -Please see our guides [Multicore Jobs](https://support.opensciencegrid.org/support/solutions/articles/5000653862-multicore-jobs) and [Large Memory Jobs](https://support.opensciencegrid.org/support/solutions/articles/5000652304-large-memory-jobs) -for more details. - -HTCondor also provides a `requirements` attribute and feature-specific -attributes that can be added to your submit files to target specific environments in -which to run your jobs. - -Lastly, there are some custom attributes you can add to your submit file to -either focus on, or avoid, certain execution sites. - -# Requirements - -The `requirements` attribute is formatted as an expression, so you can use logical -operators to combine multiple requirements where `&&` is used for AND and -`||` used for OR. For example, the following `requirements` statement will direct -jobs only to 64 bit RHEL (Red Hat Enterprise Linux) 8 nodes. - - requirements = OSGVO_OS_STRING == "RHEL 8" && Arch == "X86_64" - -Alternatively, if you have code which can run on either RHEL 7 or 8, you can use OR: - - requirements = (OSGVO_OS_STRING == "RHEL 7" || OSGVO_OS_STRING == "RHEL 8") && Arch == "X86_64" - -Note that parentheses placement is important for controling how the logical operations -are interpreted by HTCondor. - -Another common requirement is to land on a node which has CVMFS. -Then the `requirements` would be: - - requirements = HAS_oasis_opensciencegrid_org == True - -## Additional Feature-Specific Attributes - -There are many attributes that you can use with `requirements`. To see what values -you can specify for a given attribute you can run the following command while -connected to your login node: - - $ condor_status -af {ATTR_NAME} | sort -u - -For example, to see what values you can specify for the OSGVO_OS_STRING attribute run: - - $ condor_status -af OSGVO_OS_STRING | sort -u - RHEL 7 - RHEL 8 - -This means that we can specify an OS version of `RHEL 7` or `RHEL 8`. Alternatively -you will find many attributes will take the boolean values `true` or `false`. - -Below is a list of common attributes that you can include in your submit file `requirements` statement. - -- **HAS_SINGULARITY** - Boolean specifying the need to use Singularity containers in your job. - -- **HAS_MODULES** - Boolean specifying the need to use modules in your job. - _module load ..._ or not. - -- **OSGVO_OS_NAME** - The name of the operating system of the compute node. - The most common name is _RHEL_ - -- **OSGVO_OS_VERSION** - Version of the operating system - -- **OSGVO_OS_STRING** - Combined OS name and version. Common values are - _RHEL 7_ and _RHEL 8_. Please see the requirements string above on the - recommended setup. - -- **OSGVO_CPU_MODEL** - The CPU model identifier string as presented in - /proc/cpuinfo - -- **HAS_CVMFS_oasis_opensciencegrid_org** - Attribute specifying - the need to access specific oasis /cvmfs file system repositories. - -- **CUDACapability** - For GPU jobs, specifies the CUDA compute capability. - See our [GPU guide](5000653025) for more details. - -# Specifying Sites / Avoiding Sites - -To run your jobs on a list of specific execution sites, or avoid a set of -sites, use the `+DESIRED_Sites`/`+UNDESIRED_Sites` attributes in your job -submit file. **These attributes should only be used as a last resort.** For -example, it is much better to use feature attributes (see above) to make -your job go to nodes matching what you really require, than to broadly -allow/block whole sites. We encourage you to contact the facilitation team before taking this action, to make sure it is right for you. - -To avoid certain sites, first find the site names. You can find a -current list by querying the pool: - - condor_status -af GLIDEIN_Site | sort -u - -In your submit file, add a comma separated list of sites like: - - +UNDESIRED_Sites = "ISI,SU-ITS" - -Those sites will now be exluded from the set of sites your job can -run at. - -Similarly, you can use `+DESIRED_Sites` to list a subset of sites -you want to target. For example, to run your jobs at the SU-ITS site, -and only at that site, use: - - - +DESIRED_Sites = "ISI,SU-ITS" - -Note that you should only specify one of `+DESIRED_Sites`/`+UNDESIRED_Sites` -in the submit file. Using both at the same time will prevent the job from -running. - diff --git a/start/roadmap.md b/start/roadmap.md deleted file mode 100644 index 2455c4a5..00000000 --- a/start/roadmap.md +++ /dev/null @@ -1,135 +0,0 @@ -[title]: - "Roadmap to HTC Workload Submission via OSG Connect" - -[TOC] - -# Overview - -This guide lays out the steps needed to go from logging in to an OSG -Connect login node to running a full scale high throughput computing -(HTC) workload on OSG's [Open Science Pool (OSPool)](https://opensciencegrid.org/about/open_science_pool/). -The steps listed here apply to any new workload -submission, whether you are a long-time OSG user or just getting -started with your first workload, with helpful links to our documentation pages. - -This guide assumes that you have applied for an account on the OSG Connect service and -have been approved after meeting with an OSG Research Computing Facilitator. -If you don't yet have an account, you can apply for one at -or [contact us](mailto:support@opensciencegrid.org) with any questions you have. - -Learning how to get started on the OSG does not need to end with this document or -our guides! Learn about our training opportunities and personal facilitation support -in the [Getting Help](#getting-help) section below. - -# 1. Introduction to the OSPool and OSG Connect - -The OSG's Open Science Pool is best-suited for computing work that can be run as many, independent -tasks, in an approach called "high throughput computing." For more information -on what kind of work is a good fit for the OSG, -see [Is the Open Science Pool for You?](5000632058). - -Learn more about the services provided by the OSG that can support your HTC workload: - -OSG Introduction - - - -# 2. Get on OSG Connect - -After your OSG account has been approved, go through the following guides to -complete your access to the login node and to enable your account to submit jobs. - -- [Generate ssh keys and login](https://support.opensciencegrid.org/support/solutions/articles/12000027675-generate-ssh-keys-and-activate-your-osg-login) -- [Set your default project](https://support.opensciencegrid.org/support/solutions/articles/5000634360-join-and-use-a-project-in-osg-connect) - -# 3. Learn to Submit HTCondor Jobs - -Computational work is run on the OSPool by submitting it as “jobs” to the -HTCondor scheduler. Jobs submitted to HTCondor are then scheduled and -run on different resources that are part of the Open Science Pool. -Before submitting your own computational work, it is important to -understand how HTCondor job submission works. The following guides show -how to submit basic HTCondor jobs. The second example allows you to see -where in the OSPool your jobs run. - -- [OSG Connect Quickstart](https://support.opensciencegrid.org/support/solutions/articles/5000633410-osg-connect-quickstart) -- [Finding OSG Locations](https://support.opensciencegrid.org/support/solutions/articles/12000061978-finding-osg-locations) - -# 4. Test a First Job - -After learning about the basics of HTCondor job submission, you will -need to generate your own HTCondor job -- including the software needed -by the job and the appropriate mechanism to handle the data. We -recommend doing this using a single test job. - -## Prepare your software - -Software is an integral part of your HTC workflow. Whether you’ve written it yourself, inherited it from your research group, or use common open-source packages, any required executables and libraries will need to be made available to your jobs if they are to run on the OSPool. - -Read through [this overview of Using Software in OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/5000634395-using-software-in-osg-connect) to help you determine the best way to provide your software. We also have the following guides/tutorials for each major software portability approach: - -- To **install your own software**, begin with the guide on [Compiling Software for OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/5000652099) and then complete the [Example Software Compilation tutorial](https://support.opensciencegrid.org/support/solutions/articles/12000074984). -- To **use precompiled binaries**, try the example presented in the [AutoDock Vina tutorial](https://support.opensciencegrid.org/support/solutions/articles/5000634379-running-a-molecule-docking-job-with-autodock-vina) and/or the [Julia tutorial](https://support.opensciencegrid.org/support/solutions/articles/12000078187-using-julia-on-the-osg). -- To **use Docker containers** for your jobs, start with the [Docker and Singularity Containers guide](https://support.opensciencegrid.org/support/solutions/articles/12000024676), and (optionally) work through the [Tensorflow tutorial](https://support.opensciencegrid.org/support/solutions/articles/12000028940-working-with-tensorflow-gpus-and-containers) (which uses Docker/Singularity) -- To **use Distributed Environment Modules** for your jobs, start with [this Modules guide](https://support.opensciencegrid.org/support/solutions/articles/12000048518) and then complete the Module example in [this R tutorial](https://support.opensciencegrid.org/support/solutions/articles/5000674219-run-r-scripts-on-osg) - -Finally, here are some additional guides specific to some of the most common scripting languages and software tools used on OSG\*\*: - -- [Python](https://support.opensciencegrid.org/support/solutions/articles/12000058785-run-python-scripts-on-osg) -- [R](https://support.opensciencegrid.org/support/solutions/articles/5000674218-use-external-packages-in-your-r-jobs) -- [Machine Learning](https://support.opensciencegrid.org/support/solutions/articles/12000028940-working-with-tensorflow-gpus-and-containers) -- [BLAST](https://support.opensciencegrid.org/support/solutions/articles/12000062020-running-a-blast-workflow) - -\*\*This is not a complete list. Feel free to search for your software in our [Knowledge base](https://support.opensciencegrid.org/support/solutions/). - -## Manage your data - -The data for your jobs will need to be transferred to each job that runs in the OSPool, -and HTCondor has built-in features for getting data to jobs. Our [Data Management Policies](https://support.opensciencegrid.org/support/solutions/articles/12000002985-data-management-and-policies) guide -discussed the relevant approaches, when to use them, and where to stage data for each. - - - - -## Assign the Appropriate Job Duration Category - -Jobs running in the OSPool may be interrupted at any time, and will be re-run by HTCondor, unless a single execution of a job exceeds the allowed duration. Jobs expected to take longer than 10 hours will need to identify themselves as 'Long' according to our [Job Duration policies](12000083468). Remember that jobs expected to take longer than 20 hours are not a good fit for the OSPool (see [Is the Open Science Pool for You?](5000632058)) without implementing self-checkpointing (further below). - -# 5. Scale Up - -After you have a sample job running successfully, you’ll want to scale -up in one or two steps (first run several jobs, before running ALL of them). -HTCondor has many useful features that make it easy to submit -multiple jobs with the same submit file. - -- [Easily submit multiple jobs](https://support.opensciencegrid.org/support/solutions/articles/12000073165-easily-submit-multiple-jobs) -- [Scaling up after success with test jobs](https://support.opensciencegrid.org/support/solutions/articles/12000076552-scaling-up-after-success-with-test-jobs) discusses how to test your jobs for duration, memory and disk usage, and the total amount of space you might need on the - - - -# 6. Special Use Cases - -If you think any of the below applies to you, -please [get in touch](mailto:support@opensciencegrid.org) -and our facilitation team will be happy to discuss your individual case. - -- Run sequential workflows of jobs: [Workflows with HTCondor's DAGMan](12000079038) -- Implement self-checkpointing for long jobs: [HTCondor Checkpointing Guide](https://htcondor.readthedocs.io/en/latest/users-manual/self-checkpointing-applications.html) -- Build your own Docker container: [Creating a Docker Container Image](12000058245) -- Submit more than 10,000 jobs at once: [FAQ, search for 'max_idle'](5000634384) -- Larger or speciality resource requests: - - GPUs: [GPU Jobs](5000653025) - - Multiple CPUs: [Multicore Jobs](5000653862) - - Large Memory: [Large Memory Jobs](5000652304) - -# Getting Help - -The OSG Facilitation team is here to help with questions and issues that come up as you work -through these roadmap steps. We are available via email, office hours, appointments, and offer -regular training opportunities. See our [Get Help page](12000084585) and [OSG Training page](12000084444) -for all the different ways you can reach us. Our purpose -is to assist you with achieving your computational goals, so we want to hear from you! diff --git a/start/scaling-up/checkpointing-on-OSPool.md b/start/scaling-up/checkpointing-on-OSPool.md deleted file mode 100644 index d67cc6db..00000000 --- a/start/scaling-up/checkpointing-on-OSPool.md +++ /dev/null @@ -1,137 +0,0 @@ -[title]: - "Checkpointing Jobs" - -[TOC] - -# What is Checkpointing? - -Checkpointing is a technique that provides fault tolerance for a user's analysis. It consists of saving snapshots of a job's progress so the job can be restarted without losing its progress and having to restart from the beginning. We highly encourage checkpointing as a solution for jobs that will exceed the 10 hour maximum suggested runtime on the OSPool. - -This section is about jobs capable of periodically saving checkpoint information, and how to make HTCondor store that information safely, in case it's needed to continue the job on another machine or at a later time. - -There are two types of checkpointing: exit driven and eviction driven. In a vast majority of cases, **exit driven checkpointing** is preferred over eviction driven checkpointing. Therefore, this guide will focus on how to utilize exit driven checkpointing for your analysis. - -Note that not all software, programs, or code are capable of creating checkpoint files and knowing how to resume from them. Consult the manual for your software or program to determine if it supports checkpointing features. Some manuals will refer this ability as "checkpoint" features, as the ability to "resume" mid-analysis if a job is interrupted, or as "checkpoint/restart" capabilities. Contact a Research Computing Facilitator if you would like help determining if your software, program, or code is able to checkpoint. - - -# Why Checkpoint? - -Checkpointing allows a job to automatically resume from approximately where it left off instead of having to start over if interrupted. This behavior is advantageous for jobs limited by a maximum runtime policy. It is also advantageous for jobs submitted to backfill resources with no runtime guarantee (i.e. jobs on the OSPool) where the compute resources may also be more prone to hardware or networking failures. - -For example, checkpointing jobs that are limited by a runtime policy can enable HTCondor to exit a job and automatically requeue it to avoid hitting the maximum runtime limit. By using checkpointing, jobs circumvent hitting the maximum runtime limit and can run for extended periods of time until the completion of the analysis. This behavior avoids costly setbacks that may be caused by loosing results mid-way through an analysis due to hitting a runtime limit. - -# Process of Exit Driven Checkpointing - -Using exit driven checkpointing, a job is specified to time out after a user-specified amount of time with an exit code value of 85 (more on this below). Upon hitting this time limit, HTCondor transfers any checkpoint files listed in the submit file attribute `transfer_checkpoint_files` to a directory called `/spool`. This directory acts as a storage location for these files in case the job is interrupted. HTCondor then knows that jobs with exit code `85` should be automatically requeued, and will transfer the checkpoint files in `/spool` to your job's working directory prior to restarting your executable. - -The process of exit driven checkpointing relies heavily on the use of exit codes to determine the next appropriate steps for HTCondor to take with a job. In general, exit codes are used to report system responses, such as when an analysis is running, encountered an error, or successfully completes. HTCondor recognizes exit code `85` as checkpointing jobs and therefore will know to handle these jobs differently than non-checkpoiting jobs. - - -# Requirements for Exit Driven Checkpointing - -Requirements for your code or software: - -- *Checkpoint*: The software, program, or code you are using must be able to capture checkpoint files (i.e. snapshots of the progress made thus far) and know how to resume from them. -- *Resume*: This means your code must be able to recognize checkpoint files and know to resume from them instead of the original input data when the code is restarted. -- *Exit*: Jobs should exit with an exit code value of `85` after successfully creating checkpoint files. Additionally, jobs need to be able to exit with a non-`85` value if they encounter an error or write the writing the final outputs. - -**In some cases, these requirements can be achieved by using a wrapper script.** This means that your executable may be a script, rather than the code that is writing the checkpoint. An example wrapper script that enables some of these behaviors is below. - -Contact a Research Computing Facilitator for help determining if your job is capable of using checkpointing. - - -# Changes to the Submit File - -Several modifications to the submit file are needed to enable HTCondor's checkpointing feature. - -- The line `checkpoint_exit_code = 85` must be added. HTCondor recognizes code `85` as a checkpoint job. This means HTCondor knows to end a job with this code but to then to requeue it repeatedly until the analysis completes. -- The value of `when_to_transfer_output` should be set to `ON_EXIT`. -- The name of the checkpoint files or directories to be transferred to `/spool` should be specified using `transfer_checkpoint_files`. - -**Optional** -In some cases, it is necessary to write a wrapper script to tell a job when to timeout and exit. In cases such as this, the executable will need to be changed to the name of that wrapper script. An example of a wrapper script that enables a job to checkout and exit with the proper exit codes can be found below. - -An example submit file for an exit driven checkpointing job looks like: - - # exit-driven-example.submit - - executable = exit-driven.sh - arguments = argument1 argument2 - - checkpoint_exit_code = 85 - transfer_checkpoint_files = my_output.txt, temp_dir, temp_file.txt - - should_transfer_files = yes - when_to_transfer_output = ON_EXIT - - output = example.out - error = example.err - log = example.log - - cpu = 1 - request_disk = 2 GB - request_memory = 2 GB - - queue 1 - - -# Example Wrapper Script for Checkpointing Job - -As previously described, it may be necessary to use a wrapper script to tell your job when and how to exit as it checkpoints. An example of a wrapper script that tells a job to exit every 4 hours looks like: - - #!/bin/bash - - timeout 4h do_science arg1 arg2 - - timeout_exit_status=$? - - if [ $timeout_exit_status -eq 124 ]; then - exit 85 - fi - - exit $timeout_exit_status - -Let's take a moment to understand what each section of this wrapper script is doing: - - #!/bin/bash - - timeout 4h do_science argument1 argument2 - # The `timeout` command will stop the job after 4 hours (4h). - # This number can be increased or decreased depending on how frequent your code/software/program - # is creating checkpoint files and how long it takes to create/resume from these files. - # Replace `do_science argument1 argument2` with the execution command and arguments for your job. - - timeout_exit_status=$? - # Uses the bash notation of `$?` to call the exit value of the last executed command - # and to save it in a variable called `timeout_exit_status`. - - - - if [ $timeout_exit_status -eq 124 ]; then - exit 85 - fi - - exit $timeout_exit_status - - # Programs typically have an exit code of `124` while they are actively running. - # The portion above replaces exit code `124` with code `85`. HTCondor recognizes - # code `85` and knows to end a job with this code once the time specified by `timeout` - # has been reached. Upon exiting, HTCondor saves the files from jobs with exit code `85` - # in the temporary directory within `/spool`. Once the files have been transferred, - # HTCondor automatically requeues that job and fetches the files found in `/spool`. - # If an exit code of `124` is not observed (for example if the program is done running - # or has encountered an error), HTCondor will end the job and will not automaticlally requeue it. - - -The ideal timeout frequency for a job is every 1-5 hours with a maximum of 10 hours. For jobs that checkpoint and timeout in under an hour, it is possible that a job may spend more time with checkpointing procedures than moving forward with the analysis. After 10 hours, the likelihood of a job being inturrupted on the OSPool is higher. - - -# Checking the Progress of Checkpointing Jobs - -It is possible to investigate checkpoint files once they have been transferred to `/spool`. - -You can explore the checkpointed files in `/spool` by navigating to `/home/condor/spool` on an OSG Connect login node. The directories in this folder are the last four digits of a job's cluster ID with leading zeros removed. Sub folders are labeled with the process ID for each job. For example, to investigate the checkpoint files for `17870068.220`, the files in `/spool` would be found in folder `68` in a subdirectory called `220`. - - -# More Information - -More information on checkpointing HTCondor jobs can be found in HTCondor's manual: https://htcondor.readthedocs.io/en/latest/users-manual/self-checkpointing-applications.html This documentation contains additional features available to checkpointing jobs, as well as additional examples such as a python checkpointing job. diff --git a/start/scaling-up/dagman-workflows.md b/start/scaling-up/dagman-workflows.md deleted file mode 100644 index a2c1f8c2..00000000 --- a/start/scaling-up/dagman-workflows.md +++ /dev/null @@ -1,24 +0,0 @@ -[title]: - "Submit Workflows with HTCondor's DAGMan" - -[TOC] - -# Overview - -If your work requires jobs that run in a particular sequence, you may benefit -from a workflow tool that submits and monitors jobs for you in the correct -order. A simple workflow manager that integrates with HTCondor is DAGMan, -or "DAG Manager" where DAG stands for the typical picture of a workflow, a -directed acyclic graph. - -# Learning Resources - -This talk (originally presented at HTCondor Week 2020) gives a good overview of -when to use DAGMan and its most useful features: - - -DAGMan Talk - - -For full details on various DAGMan features, see the HTCondor manual page: - -* [DAGMan Manual Page](https://htcondor.readthedocs.io/en/latest/users-manual/dagman-workflows.html) diff --git a/start/scaling-up/preparing-to-scale-up.md b/start/scaling-up/preparing-to-scale-up.md deleted file mode 100644 index f4b9e9da..00000000 --- a/start/scaling-up/preparing-to-scale-up.md +++ /dev/null @@ -1,202 +0,0 @@ -[title]: - "Determining the Amount of Resources to Request in a Submit File" - -# Learning Objectives -This guide discuses the following: -- Best practices for testing jobs and scaling up your analysis. -- How to determine the amount of resources (CPU, memory, disk space) to request in a submit file. - -[TOC] - -# Overview - -Much of HTCondor's power comes from the ability to run a large number -of jobs simultaneously. To optimize your work with a high-throughput computing (HTC) -approach, you will need to test and optimize the resource requests of those jobs to -only request the amount of memory, disk, and cpus truly needed. -This is an important practice that will maximize your throughput by optimizing the -number of potential 'slots' in the OSPool that your jobs can match to, reducing the overall -turnaround time for completing a whole batch. - -If you have questions -or are unsure if and how your work can be broken up, please contact us at -[support@opensciencegrid.org](mailto:support@opensciencegrid.org) - -This guide will describe best practices and general tips for testing -your job resource requests **before** scaling up to submit your full set of jobs. -Additional information is also available from the following "Introduction to High Throughput Computing with HTCondor" 2020 OSG Virtual -Pilot School lecture video: - - - 2020 VSP dHTC with HTCondor - -# Always Start With Test Jobs - -Submitting test jobs is an important first step for optimizing -the resource requests of your jobs. We always recommend submitting a few (3-10) -test jobs first before scaling up, whether this is your first time -using OSG or you're an experienced user starting a new workflow. If you plan to submit -thousands of jobs, you may even want to run an intermediate test of 100-1,000 jobs to catch any -failures or holds that mean your jobs have additional `requirements` they need to specify -(and which OSG staff can help you to identify, based upon your tests). - -Some general tips for test jobs: - -- Select smaller data sets or subsets of data for your first test jobs. Using -smaller data will keep the resource needs of your jobs low which will help get -test jobs to start and complete sooner, when you're just making sure that your submit file -and other logistical aspects of jobs submission are as you want them. - -- If possible, submit test jobs that will reproduce results you've gotten -using another system. This approach can be used as a good "sanity check" as you'll be able -to compare the results of the test to those previously obtained. - -- After initial tests complete successfully, scale up to larger or full-size -data sets; if your jobs span a range of input file sizes, submit tests using the smallest -and largest inputs to examine the range of resources that these jobs may need. - -- Give your test jobs and associated HTCondor `log`, `error`, `output`, -and `submit` files meaningful names so you know which results refer to which tests. - -# Requesting CPUs, Memory, and Disk Space in the HTCondor Submit File - -In the HTCondor submit file, you must explicitly request the number of -CPUs (i.e. cores), and the amount of disk and memory that the job needs -to complete successfully, and you may need to identify a [JobDurationCategory](12000083468). -When you submit a job for the -first time, you may not know just how much to request and that's OK. -Below are some suggestions for making resource requests for initial test -jobs. - -- For **requesting CPU cores** start by requesting a single cpu. With single-cpu jobs, you will see -your jobs start sooner. Ultimately you will be able to achieve -greater throughput with single cpus jobs compared to jobs that request -and use multiple cpus. - - - **Keep in mind, requesting more CPU cores for a job - does not mean that your jobs will use more cpus.** Rather, you want to make sure - that your CPU request matches the number of cores (i.e. 'threads' or 'processes') - that you expect your software to use. (Most softwares only use 1 CPU core, by default.) - - - There is limited support for multicore work in OSG. To learn more, - see our guide on - [Multicore Jobs](https://support.opensciencegrid.org/support/solutions/articles/5000653862) - - - Depending on how long you expect your test jobs to take on a single core, you may need to identify a - non-default [JobDurationCategory](12000083468), or consider implementing self-checkpointing (email us!). - -- To inform initial **disk requests** always look at the size of your input -files. At a minimum, you need to request enough disk to support all -of the input files, executable, and the output you expect, but don't forget that the standard 'error' and 'output' -files you specify will capture 'terminal' output that may add up, too. - - - If many of your input and output files are compressed -(i.e. zipped or tarballs) you will need to factor that into your -estimates for disk usage as these files will take up additional space once uncompressed -in the job. - - - For your initial tests it is OK to request more disk than -your job may need so that the test completes successfully. **The key -is to adjust disk requests for subsequent jobs based on the results -of these test jobs.** - -- Estimating **memory requests** can sometimes be tricky. If you've performed the -same or similar work on another computer, consider using the amount of -memory (i.e. RAM) from that computer as a starting point. For instance, -most laptop computers these days will have 8 or 16 GB of memory, which is okay to start -with if you know a single job will succeed on your laptop. - - - For your initial tests it is OK to request more memory than -your job may need so that the test completes successfully. **The key -is to adjust memory requests for subsequent jobs based on the results -of these test jobs.** - - - If you find that memory usage will vary greatly across a -batch of jobs, we can assist you with creating dynamic memory requests -in your submit files. - -# Optimize Job Resource Requests For Subsequent Jobs - -**As always, reviewing the HTCondor `log` file from past jobs is -a great way to learn about the resource needs of your jobs.** Optimizing the resources requested for each job may help your job run faster and achieve more throughput. - -Save the HTCondor `log` files from your jobs. HTCondor will report -the memory, disk, and cpu usage of your jobs at the end of this file. The amount of each resource requested in the submit file is listed under the "Request" column and information about the amount of each resource actually utilized to complete the job is provided in the "Usage" column. - -For example: - -``` - Partitionable Resources : Usage Request Allocated - Cpus : 1 1 - Disk (KB) : 12 1000000 26703078 - Memory (MB) : 0 1000 1000 -``` - - -- One quick option to query your `log` files is to use the Unix tool `grep`. For example: - ``` - [user@login]$ grep "Disk (KB)" my-job.log - ``` - The above will return all lines in `my-job.log` that report the disk - usage, request, and allocation of all jobs reported in that log file. - - Alternatively, `condor_history` can be used to query details from - recently completed job submissions. HTCondor's history is continuously updating with information from new jobs, so `condor_history` is best performed shortly after the jobs of interest enter/leave the queue. - -# Submit Multiple Jobs Using A Single Submit File - -Once you have a single test job that completes successfully, the next -step is to submit a small batch of test jobs (e.g. 5 or 10 jobs) -[**using a single submit file**](https://support.opensciencegrid.org/support/solutions/articles/12000073165). Use this small-scale -multi-job submission test to ensure that all jobs complete successfully, produce the -desired output, and do not conflict with each other when submitted together. Once -you are confident that the jobs will complete as desired, then scale up to submitting -the entire set of jobs. - -# Monitoring Job Status and Obtaining Run Information - -Gathering information about how, what, and where a job ran can be important for both troubleshooting and optimizing a workflow. The following commands are a great way to learn more about your jobs: - -| Command | Description | -| ----------- | ----------- | -| `condor_q` | Shows the queue information for your jobs. Includes information such as batch name and total jobs. | -| `condor_q -l` | Prints all information related to a job including attributes and run information about a job in the queue. Output includes `JobDurationCategory`, `ServerTime`, `SubmitFile`, etc. Also works with `condor_history`. | -| `condor_q -af ` | Prints information about an attribute or list of attributes for a single job using the autoformat `-af` flag. The list of possible attributes can be found using `condor_q -l`. Also works with `condor_history`. | -| `condor_q -constraint ' == ""' ` | The `-constraint` flag allows users to find all jobs with a certain value for a given parameter. This flag supports searching by more than one parameter and different operators (e.g. `=!=`). Also works with `condor_history`. | -| `condor_q -better-analyze -pool ` | Shows a list of the number of slots matching a job's requirements. For more information, see [Troubleshooting Job Errors](https://support.opensciencegrid.org/support/solutions/articles/5000639785-troubleshooting-job-errors). | - - -Additional `condor_q` flags involved in optimizing and troubleshooting jobs include: -| Flag | Description | -| ----------- | ----------- | -| -nobatch | Combined with `condor_q`, this flag will list jobs individually and not by batch. | -| -hold | Show only jobs in the "on hold" state and the reason for that. An action from the user is expected to solve the problem. | -| -run | Show your running jobs and related info, like how much time they have been running, where they are running, etc. | -| -dag | Organize `condor_q` output by DAG. | - -More information about the commands and flags above can be found in the [HTCondor manual](https://htcondor.readthedocs.io/en/latest/). - -## Avoid Exceeding Disk Quotas in /home and /public - -Each OSG Connect user is granted 50 GB of storage in their `/home` directory and - -500 GB of storage in their `/public` directory. This may seem like a lot, but -when running 100's or 1,000's of jobs, even small output can add up quickly. If -these quotas are exceeded, jobs will fail or go on hold when attempting returning output. - -To prevent errors or workflow interruption, be sure to estimate the -input and output needed for all of your concurrently running -jobs. By default, after your job terminates HTCondor will transfer back -any new or modified files from the top-level directory where the job ran, -back to your `/home` directory. Efficiently manage output by including steps -to remove intermediate and/or unnecessary files as part of your job. - -# Workflow Management - -To help manage complicated workflows, consider a workflow manager such -as HTCondor's built-in [DAGman](https://research.cs.wisc.edu/htcondor/dagman/dagman.html) -or the HTCondor-compatible [Pegasus](https://support.opensciencegrid.org/support/solutions/articles/5000639789-pegasus) -workflow tool. - -# Get Help - -For assistance or questions, please email the OSG Research Facilitation team at [support@opensciencegrid.org](mailto:support@opensciencegrid.org). diff --git a/start/scaling-up/submit-multiple-jobs.md b/start/scaling-up/submit-multiple-jobs.md deleted file mode 100644 index d4833522..00000000 --- a/start/scaling-up/submit-multiple-jobs.md +++ /dev/null @@ -1,287 +0,0 @@ -[title]: - "Easily Submit Multiple Jobs" - -[TOC] - -# Overview - -HTCondor has several convenient features for streamlining high-throughput -job submission. This guide provides several examples -of how to leverage these features to **submit multiple jobs with a -single submit file.** - -*Why submit multiple jobs with a single submit file?* - -As described in our [Policies for using an OSG Connect submit server](https://support.opensciencegrid.org/support/solutions/articles/12000074852), -users should submit multiple jobs using a single submit file, or where applicable, as few -separate submit files as needed. Using HTCondor multi-job submission features is more -efficient for users and will help ensure reliable operation of the the login nodes. - -Many options exist for streamlining your submission of multiple jobs, -and this guide only covers a few examples of what is truly possible with -HTCondor. If you are interested in a particular approach that isn't described here, -please contact [OSG Connect support](mailto:support@opensciencegrid.org) and we will -work with you to identify options to meet the needs of your work. - -# Submit Multiple Jobs Using `queue` - -All HTCondor submit files require a `queue` attribute (which must also be -the last line of the submit file). By default, `queue` will submit one job, but -users can also configure the `queue` attribute to behave like a for loop -that will submit multiple jobs, with each job varying as predefined by the user. - -Below are different HTCondor submit file examples for submitting batches of multiple -jobs and, where applicable, how to indicate the differences between jobs in a batch -with user-defined variables. Additional examples and use cases are provided further below: - -1. ***[queue ](#process)*** - will submit *N* number of jobs. Examples - include performing replications, where the same job must be repeated *N* number - of times, looping through files named with numbers, and looping through - a matrix where each job uses information from a specific row or column. -2. ***[queue from ](#foreach)*** - will loop through a - list of file names, parameters, etc. as defined in separate text file (i.e. **). - This `queue` option is very flexible and provides users with many options for - submitting multiple jobs. -3. **[Organizing Jobs Into Individual Directories](#initialdir)** - - another option that can be helpful in organizing multi-job submissions. - -These `queue` options are also described in the following video from HTCondor Week 2020: - - 2020 HTCondor Week Presentation - -[Submitting Multiple Jobs Using HTCondor Video](https://www.youtube.com/watch?v=m7dQChJH5LU) - -What makes these `queue` options powerful is the ability to use user-defined -variables to specify details about your jobs in the HTCondor submit file. The -examples below will include the use of `$(variable_name)` to specify details -like input file names, file locations (aka paths), etc. **When selecting a -variable name, users must avoid bespoke HTCondor submit file variables -such as `Cluster`, `Process`, `output`, and `input`, `arguments`, etc.** - -## 1. Use `queue N` in you HTCondor submit files - -When using `queue N`, HTCondor will submit a total of *N* -jobs, counting from 0 to *N* - 1 and each job will be assigned -a unique `Process` id number spanning this range of values. Because -the `Process` variable will be unique for each job, it can be used in -the submit file to indicate unique filenames and filepaths for each job. - -The most straightforward example of using `queue N` is to submit -*N* number of identical jobs. The example shown below demonstrates -how to use the `Cluster` and `Process` variables to assign unique names -for the HTCondor `error`, `output`, and `log` files for each job in the batch: - - # 100jobs.sub - # submit 100 identical jobs - - log = job_$(Cluster)_$(Process).log - error = job_$(Cluster)_$(Process).err - output = job_$(Cluster)_$(Process).out - - ... remaining submit details ... - - queue 100 - -For each job, the appropriate number, `0, 1, 2, ... 99` will replace `$(Process)`. -`$(Cluster)` will be a unique number assigned to the entire 100 job batch. Each -time you run `condor_submit job.sub`, you will be provided -with the `Cluster` number which you will also see in the output produced by -the command `condor_q`. - -If a uniquely named results file needs to be returned by each job, -`$(Process)` and `$(Cluster)` can also be used as `arguments`, and anywhere -else as needed, in the submit file: - - arguments = $(Cluster)_$(Process).results - - ... remaining submit details ... - - queue 100 - -Be sure to properly format the `arguments` statement according to the -executable used by the job. - -***What if my jobs are not identical?*** `queue N` may still be a great -option! Additional examples for using this option include: - -### A. Use integer numbered input files - - [user@login]$ ls *.data - 0.data 1.data 2.data 3.data - ... 97.data 98.data 99.data - -In the submit file, use: - - transfer_input_files = $(Process).data - - ... remaining submit details ... - - queue 100 - -### B. Specify a row or column number for each job - -$(Process) can be used to specify a unique row or column of information in a -matrix to be used by each job in the batch. The matrix needs to then be transferred -with each job as input. For exmaple: - - transfer_input_files = matrix.csv - arguments = $(Process) - - ... remaining submit details ... - - queue 100 - -The above exmaples assumes that your job is set up to use an argument to -specify the row or column to be used by your software. - -### C. Need *N* to start at 1 - -If your input files are numbered 1 - 100 instead of 0 - 99, or your matrix -row starts with 1 instead of 0, you can perform basic arithmetic in the submit -file: - - plusone = $(Process) + 1 - NewProcess = $INT(plusone, %d) - arguments = $(NewProcess) - - ... remaining submit details ... - - queue 100 - -Then use `$(NewProcess)` anywhere in the submit file that you would -have otherwise used `$(Process)`. Note that there is nothing special about the -names `plusone` and `NewProcess`, you can use any names you want as variables. - -## 2. Submit multiple jobs with one or more distinct variables per job - -Think about what's different between each job that needs to be submitted. -Will each job use a different input file or combination of software parameters? Do -some of the jobs need more memory or disk space? Do you want to use a different -software or script on a common set of input files? Using `queue from ` -in your submit files can make that possible! `` can be a single user-defined -variable or comma-separated list of variables to be used anywhere in the submit file. -`` is a plain text file that defines `` for each individual job to be submitted in the batch. - -Suppose you need to run a program called `compare_states` that will run on -on the following set of input files: `illinois.data`, `nebraska.data`, and -`wisconsin.data` and each input file can analyzed as a separate job. - -To create a submit file that will submit all three jobs, first create a -text file that lists each `.data` file (one file per line). -This step can be performed directly on the login node, for example: - - [user@state-analysis]$ ls *.data > states.txt - [user@state-analysis]$ cat states.txt - illinois.data - nebraska.data - wisconsin.data - -Then, in the submit file, following the pattern `queue from `, -replace `` with a variable name like `state` and replace `` -with the list of `.data` files saved in `states.txt`: - - queue state from states.txt - -For each line in `states.txt`, HTCondor will submit a job and the variable -`$(state)` can be used anywhere in the submit file to represent the name of the `.data` file -to be used by that job. For the first job, `$(state)` will be `illinois.data`, for the -second job `$(state)` will be `nebraska.data`, and so on. For example: - - # run_compare_states_per_state.sub - - transfer_input_files = $(state) - arguments = $(state) - executable = compare_states - - ... remaining submit details ... - - queue state from states.txt - -For a working example of this kind of job submission, see our [Word Frequency Tutorial](12000079856). - -### Use multiple variables for each job - -Let's imagine that each state `.data` file contains data spanning several -years and that each job needs to analyze a specific year of data. Then -the states.txt file can be modified to specify this information: - - [user@state-analysis]$ cat states.txt - illinois.data, 1995 - illinois.data, 2005 - nebraska.data, 1999 - nebraska.data, 2005 - wisconsin.data, 2000 - wisconsin.data, 2015 - -Then modify the `queue` to define two `` named `state` and `year`: - - queue state,year from states.txt - -Then the variables `$(state)` and `$(year)` can be used in the submit file: - - # run_compare_states_by_year.sub - arguments = $(state) $(year) - transfer_input_files = $(state) - executable = compare_states - - ... remaining submit details ... - - queue state,year from states.txt - -## 3. Organizing Jobs Into Individual Directories - -One way to organize jobs is to assign each job to its own directory, -instead of putting files in the same directory with unique names. To -continue our \"compare\_states\" example, suppose there\'s a directory -for each state you want to analyze, and each of those directories has -its own input file named `input.data`: - - [user@state-analysis]$ ls -F - compare_states illinois/ nebraska/ wisconsin/ - - [user@state-analysis]$ ls -F illinois/ - input.data - - [user@state-analysis]$ ls -F nebraska/ - input.data - - [user@state-analysis]$ ls -F wisconsin/ - input.data - -The HTCondor submit file attribute `initialdir` can be used -to define a specific directory from which each job in the batch will be -submitted. The default `initialdir` location is the directory from which the -command `condor_submit myjob.sub` is executed. - -Combining `queue var from list` with `initiadir`, each line of ** will include -the path to each state directory and `initialdir` set to this path for -each job: - - #state-per-dir-job.sub - initial_dir = $(state_dir) - transfer_input_files = input.data - executable = compare_states - - ... remaining submit details ... - - queue state_dir from state-dirs.txt - -Where `state-dirs.txt` is a list of each directory with state data: - - [user@state-analysis]$ cat state-dirs.txt - illinois - nebraska - wisconsin - -**Notice that `executable = compare_states` has remained unchanged in the above example. -When using `initialdir`, only the input and output file path (including the HTCondor log, error, and -output files) will be changed by `initialdir`**. - -In this example, HTCondor will create a job for each directory in `state-dirs.txt` and use -that state\'s directory as the `initialdir` from which the job will be submitted. -Therefore, `transfer_input_files = input.data` can be used without specifying -the path to this `input.data` file. Any output generated by the job will then be returned to the `initialdir` -location. - -# Get Help - -For assistance or questions, please email the OSG Research Facilitation team at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) or visit the [help desk and community forums](http://support.opensciencegrid.org). diff --git a/start/software/available-containers-list.md b/start/software/available-containers-list.md deleted file mode 100644 index 92e22924..00000000 --- a/start/software/available-containers-list.md +++ /dev/null @@ -1,57 +0,0 @@ -[title]: - "View Existing OSPool-Supported Containers" - -[TOC] - -This is list of commonly used containers in the Open Science Pool. These can be used -directly in your jobs or as base images if you want to define your own. Please -see the pages on [container overview][container-intro] and [creating containers][container-howto] -for detailed instructions on how to use containers and/or have Docker containers added to the OSPool's approved list. - -Also note that this list is not complete. There are many images under -`/cvmfs/singularity.opensciencegrid.org/` which are either project specific -or not described well enough to make this list. - - -## Base - -| **Name** | **CVMFS Locations** | **Description** | -|:---------|:--------------------|:----------------| -| EL 6 | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el6:latest | Enterprise Linux (CentOS) 6 base image
[Project Website](https://www.centos.org/)
[Container Definition](https://github.com/opensciencegrid/osgvo-el6) | -| EL 7 | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el7:latest | Enterprise Linux (CentOS) 7 base image
[Project Website](https://www.centos.org/)
[Container Definition](https://github.com/opensciencegrid/osgvo-el7) | -| EL 8 | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el8:latest | Enterprise Linux (CentOS) 8 base image
[Project Website](https://www.centos.org/)
[Container Definition](https://github.com/opensciencegrid/osgvo-el8) | -| Ubuntu 16.04 | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-ubuntu-xenial:latest | Ubuntu 16.04 (Xenial) base image
[Project Website](https://www.ubuntu.com)
[Container Definition](https://github.com/opensciencegrid/osgvo-ubuntu-xenial) | -| Ubuntu 18.04 | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-ubuntu-18.04:latest | Ubuntu 18.04 (Bionic) base image
[Project Website](https://www.ubuntu.com)
[Container Definition](https://github.com/opensciencegrid/osgvo-ubuntu-18.04) | -| Ubuntu 20.04 | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-ubuntu-20.04:latest | Ubuntu 20.04 (Focal) base image
[Project Website](https://www.ubuntu.com)
[Container Definition](https://github.com/opensciencegrid/osgvo-ubuntu-20.04) | - -## Languages - -| **Name** | **CVMFS Locations** | **Description** | -|:---------|:--------------------|:----------------| -| Julia | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-julia:latest | Ubuntu based image with Julia
[Project Website](https://julialang.org/)
[Container Definition](https://github.com/opensciencegrid/osgvo-julia) | -| Matlab Runtime | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-matlab-runtime:R2018b
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-matlab-runtime:R2019a
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-matlab-runtime:R2019b
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-matlab-runtime:R2020a
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-matlab-runtime:R2020b | This is the Matlab runtime component you can use to execute compiled Matlab codes
[Project Website](https://www.mathworks.com/products/compiler/matlab-runtime.html)
[Container Definition](https://github.com/opensciencegrid/osgvo-matlab-runtime) | -| R | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-r:3.5.0
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-r:4.0.2 | Example for building R images
[Project Website](https://www.r-project.org/)
[Container Definition](https://github.com/opensciencegrid/osgvo-r) | - -## Project - -| **Name** | **CVMFS Locations** | **Description** | -|:---------|:--------------------|:----------------| -| XENONnT | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-xenon:2020.11.06
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-xenon:2020.11.25
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-xenon:development
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-xenon:latest
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-xenon:py38 | Base software environment for XENONnT, including Python 3.6 and data management tools
[Project Website](http://www.xenon1t.org/)
[Container Definition](https://github.com/XENONnT/base_environment) | -| XENONnT | /cvmfs/singularity.opensciencegrid.org/xenonnt/osg_dev:latest | Base software environment for XENONnT, including Python 3.6 and data management tools
[Project Website](http://www.xenon1t.org/)
[Container Definition](https://github.com/XENONnT/base_environment) | -| XENONnT | /cvmfs/singularity.opensciencegrid.org/xenonnt/base-environment:2020.11.06
/cvmfs/singularity.opensciencegrid.org/xenonnt/base-environment:2020.11.25
/cvmfs/singularity.opensciencegrid.org/xenonnt/base-environment:development
/cvmfs/singularity.opensciencegrid.org/xenonnt/base-environment:latest
/cvmfs/singularity.opensciencegrid.org/xenonnt/base-environment:py38 | Base software environment for XENONnT, including Python 3.6 and data management tools
[Project Website](http://www.xenon1t.org/)
[Container Definition](https://github.com/XENONnT/base_environment) | -| XENONnT | /cvmfs/singularity.opensciencegrid.org/xenonnt/montecarlo:development
/cvmfs/singularity.opensciencegrid.org/xenonnt/montecarlo:latest | Base software environment for XENONnT, including Python 3.6 and data management tools
[Project Website](http://www.xenon1t.org/)
[Container Definition](https://github.com/XENONnT/base_environment) | - -## Tools - -| **Name** | **CVMFS Locations** | **Description** | -|:---------|:--------------------|:----------------| -| FreeSurfer | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-freesurfer:6.0.0
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-freesurfer:6.0.1
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-freesurfer:7.0.0 | A software package for the analysis and visualization of structural and functional neuroimaging data from cross-sectional or longitudinal studies
[Project Website](https://surfer.nmr.mgh.harvard.edu/fswiki/FreeSurferWiki)
[Container Definition](https://github.com/opensciencegrid/osgvo-freesurfer) | -| GROMACS | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-gromacs:latest | A versatile package to perform molecular dynamics, i.e. simulate the Newtonian equations of motion for systems with hundreds to millions of particles.
[Project Website](http://www.gromacs.org/)
[Container Definition](https://github.com/opensciencegrid/osgvo-gromacs) | -| GROMACS GPU | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-gromacs-gpu:latest | A versatile package to perform molecular dynamics, i.e. simulate the Newtonian equations of motion for systems with hundreds to millions of particles. This is a GPU enabled version.
[Project Website](http://www.gromacs.org/)
[Container Definition](https://github.com/opensciencegrid/osgvo-gromacs-gpu) | -| Quantum Espresso | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-quantum-espresso:6.6 | A suite for first-principles electronic-structure calculations and materials modeling
[Project Website](https://www.quantum-espresso.org/)
[Container Definition](https://github.com/opensciencegrid/osgvo-quantum-espresso) | -| TensorFlow | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/tensorflow:2.3
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/tensorflow:latest | TensorFlow image (CPU only)
[Project Website](https://www.tensorflow.org/)
[Container Definition](https://github.com/opensciencegrid/osgvo-tensorflow) | -| TensorFlow GPU | /cvmfs/singularity.opensciencegrid.org/opensciencegrid/tensorflow-gpu:2.2-cuda-10.1
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/tensorflow-gpu:2.3-cuda-10.1
/cvmfs/singularity.opensciencegrid.org/opensciencegrid/tensorflow-gpu:latest | TensorFlow image with GPU support
[Project Website](https://www.tensorflow.org/)
[Container Definition](https://github.com/opensciencegrid/osgvo-tensorflow-gpu) | - - -[container-intro]: 12000024676 -[container-howto]: 12000058245 - diff --git a/start/software/compiling-applications.md b/start/software/compiling-applications.md deleted file mode 100644 index 4a480470..00000000 --- a/start/software/compiling-applications.md +++ /dev/null @@ -1,135 +0,0 @@ -[title]: - "Compiling Software for OSG Connect" -[TOC] - -# Compiling Software for OSG Connect - -## Introduction - -Due to the distributed nature of the Open Science Pool, you will always need to -ensure that your jobs have access to the software that will be executed. This guide provides -useful information for compiling and using your software in OSG Connect. A detailed -example of performing software compilation is additionally available -at [OSG Connect Example Compilation Guide](https://support.opensciencegrid.org/support/solutions/articles/12000074984). - - -> *What is compiling?* -> The process of compiling converts human readable code into binary, -> machine readable code that will execute the steps of the program. - -## Get software source code - -The first step to compiling your software is to locate and download the source -code, being sure to select the version that you want. Source code -will often be made available as a compressed tar archive which will need to be -extracted for before compilation. - -You should also carefully review the installation instructions provided by the -software developers. The installation instructions should include important -information regarding various options for configuring and performing the compilation. -Also carefully note any system dependencies (hardware, other software, and libraries) that are -required for your software. - -## Select the appropriate compiler and compilation options - -A compiler is a program that is used to peform source code compilation. The GNU Compiler -Collection (GCC) is a common, open source collection of compilers with support for C, C++, -fotran, and other languages, and includes important libraries for supporting your compilation -and sometimes software execution. Your software compilation may require certain versions -of a compiler which should be noted in the installation instructions or system dependencies -documention. Currently the login nodes have `GCC 4.8.5` as the default version, but newer -versions of GCC may also be available - to learn more please contact . - -CMake is a commonly used compilation platform. Your software may have dependencies for -specific `cmake` versions. Currently the login nodes have two versions of CMake, `3.12.3` -and `3.13.0` available as [modules](https://support.opensciencegrid.org/support/solutions/articles/12000048518). - -### Static versus dynamic linking during compilation - -Binary code often depends on additional information (i.e. instructions) from other software, -known as libraries, for proper execution. The default behavior when compiling, is for the -final binary to be "dynamically linked" to libraries that it depends on, such that when -the binary is executed, it will look for these library files on the system that it is -running on. Thus a copy of the appropriate library files will need to be available to your -software wherever it runs. OSG Connect users can transfer a copy of the necessary -libraries along with with their jobs to manage such dependencies if not supported by the -execute node that your jobs run on. - -However, the option exists to "statically link" the library dependencies of your software. -By statically linking libraries during compilation, the library code will be -directly packaged with your software binary meaning the libraries will always be -available to your software which your software to run on more execute nodes. - -To statically link libraries during compilation, use the `-static` flag when running `gcc`, -use `--enable-static` when running a `configure` script, or set your `LD_FLAGS` -environment variable to `--enable-static` (e.g. `export LD_FLAGS="--enable-static"`). - -### Get access to libraries needed for your software - -As described above, your software may require additional software, known as libraries, for -compilation and execution. For greatest portability of your software, we recommend installing -the libraries needed for your software and transferring a copy of the libraries along with -your subsequent jobs. When using libraries that you have installed yourself, you will likely -need to add these libraries to your `LIBRARY_PATH` environment variable before compiling -your software. There may also be additional environment variables that will need to be -defined or modified for software compilation, this information should be provided -in the installtion instructions of your software. For any libraries added to `LIBRARY_PATH` -before software compilation, you'll also need to add these same libraries to -your `LD_LIBRARY_PATH` as a step in your job's executable bash script before executing -your software. - -The [distributed environment modules system](https://support.opensciencegrid.org/support/solutions/articles/12000048518) -available on OSG Connect also provides several commonly used software libraries -(lapack, atlas, hdf5, netcdf, etc.) that can be used for your software compilation and -execution. The appropriate modules should be loaded before performing your software -compilation. The process of loading a module will modify all appropriate -environment variables (e.g. `CPATH` and `LIBRARY_PATH`) for you. If you do use modules -from the modules system, you will need to modify your job scripts to load the appropriate -modules before running you software. - -## Perform your compilation - -Software compilation is easiest to perform interactively, and OSG Connect users are -welcome to compile software directly on their assigned login node. This will ensure -that your application is built on an environment that is similar to the majority -of the compute nodes on OSG. Because OSG Connect login nodes currently use the -Red Hat Enterprise Linux 7 operating system, your software will, generally, only be -compatible for execution on RHEL 7 or similar operating systems. You can use the -`requirements` statement of your HTCondor submit file to direct your jobs to execute -nodes with specific operating systems, for instance: - - requirements = (OSGVO_OS_STRING == "RHEL 7") - -Software installation typically includes three steps: 1.) configuration, 2.) compilation, and 3.) -"installation" which places the compiled code in a specific location. In most cases, -these steps will be achieved with the following commands: - - ./configure - make - make install - -**Most software is written to install to a default location, however your OSG Connect -account is not authorized to write to these default system locations.** Instead, you will want to -create a folder for your software installation in your `home` directory and use an option in the -configuration step that will install the software to this folder: - - ./configure --prefix=/home/username/path - -where `username` should be replaced with your OSG Connect username and `path` replaced with the -path to the directory you created for your software installation. - -## Use Your Software - -When submitting jobs, you will need to transfer a copy of your compiled software, -and any dynamically-linked dependencies that you also installed. Our -[Introduction to Data Management on OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/12000002985) -guide is a good starting point for more information for selecting the appropriate -methods for transferring you software. Depending on your job workflow, it may be possible -to directly specify your executable binary as the `executable` in your HTCondor -submit file. - -When using your software in subsequent job submissions, be sure to add additional -commands to the executable bash script to define evironment variables, like -for instance `LD_LIBRARY_PATH`, that may be needed to properly execute your software. - -## Get Additional Assistance -If you have questions or need assistance, please contact . diff --git a/start/software/containers-docker.md b/start/software/containers-docker.md deleted file mode 100644 index b1dd79bf..00000000 --- a/start/software/containers-docker.md +++ /dev/null @@ -1,199 +0,0 @@ -[title]: - "Create/Register a Docker Container Image" - -[TOC] - -This guide is meant to accompany the instructions for using containers -in the Open Science Pool. You can use your own custom container to run jobs in the -Open Science Pool, and we assume that those containers are built using Docker. This -guide describes how to create your own Docker container "image" (the blueprint for -the container). Once you have created your custom image, -you will need to register the image as described further down in this guide. - -For an overview and how to execute images on OSG, please see -[Use Containers on the OSG][osg-containers] - -## Install Docker and Get a Docker Hub Account - -You'll need a Docker Hub account in order to download Docker and share your -Docker container images with the OSG: [DockerHub](https://hub.docker.com/) - -Install Docker Desktop to your computer using the appropriate version for your -operating system. Note that OSG does not provide any container build hosts. - -## Identify Components - -What software do you want to install? Make sure that you have either the source -code or a command that can be used to install it through Linux (like `apt-get` or -`yum`). - -You'll also need to choose a "base" container, on which to add your particular -software or tools. *We strongly recommend using one of the OSG's published containers -as your starting point.* See the available containers on Docker Hub here: -[OSG Docker Containers](https://hub.docker.com/u/opensciencegrid) -The best candidates for you will be containers that have "osgvo" in the name. - -If you prefer, you can base your image on images not already published -by OSG, but if you do this, we recommend that as one of the creation steps you -create the `/cvmfs` directory. See [Special Cases](#special-cases) below. - -## Build a Container - -There are two main methods for generating your own container image. - -1. Editing the `Dockerfile` -2. Editing the default image using local Docker - -We recommend the first option, as it is more reproducible, but the second option -can be useful for troubleshooting or especially tricky installs. - -### Editing the `Dockerfile` - -Create a folder on your computer and inside it, create a blank text file -called `Dockerfile`. - -The first line of this file should include the keyword `FROM` and then -the name of a Docker image (from Docker Hub) you want -to use as your starting point. If using the OSG's Ubuntu Xenial image that -would look like this: - - FROM opensciencegrid/osgvo-ubuntu-xenial - -Then, for each command you want to run to add libraries or software, use the -keyword `RUN` and then the command. Sometimes it makes sense to string -commands together using the `&&` operator and line breaks `\`, like so: - - RUN apt-get update && \ - apt-get install -yy build-essentials - -or - - RUN wget https://cran.r-project.org/src/base/R-3/R-3.6.0.tar.gz && \ - tar -xzf R-3.6.0.tar.gz && \ - cd R-3.6.0 && \ - ./configure && \ - make && \ - make install - -Typically it's good to group together commands installing the same kind of thing -(system libraries, or software packages, or an installation process) under one `RUN` command, -and then have multiple `RUN` commands, one for each of the different type of -software or package you're installing. - -(For all the possible Dockerfile keywords, see the [Docker Documentation](https://docs.docker.com/engine/reference/builder/)) - -Once your Dockerfile is ready, you can "build" the container image by running this command: - - $ docker build -t namespace/repository_name . - -Note that the naming convention for Docker images is your Docker Hub username and then -a name you choose for that particular container image. So if my Docker Hub username -is `alice` and I created an image with the NCBI `blast` tool, I might use this name: - - $ docker build -t alice/NCBI-blast . - - -### Editing the default image using local Docker - -You can also build an image interactively, without a Dockerfile. First, get -the desired starting image from Docker Hub. Again, we will -look at the OSG Ubuntu Xenial image. - - $ docker pull opensciencegrid/osgvo-ubuntu-xenial - -We will run the image in a docker interactive session - - $ docker run -it --name opensciencegrid/osgvo-ubuntu-xenial /bin/bash - -Giving the session a name is important because it will make it easier to -reattach the session later and commit the changes later on. Now you will -be greeted by a new command line prompt that will look something like this - - [root@740b9db736a1 /]# - -You can now install the software that you need through the default package -manager, in this case `apt-get`. - - [root@740b9db736a1 /]# apt-get install build-essentials - -Once you have installed all the software, you simply `exit` - - [root@740b9db736a1 /]# exit - -Now you can commit the changes to the image and give it a name: - - docker commit namespace/repository_name - -You can also use the session's hash as found in the command prompt (`740b9db736a1` -in the above example) in place of the docker session name. - -## Upload Docker Container to Docker Hub - -Once your container is complete and tagged, it should appear in the list of local Docker -container images, which you can see by running: - - $ docker images - -From there, you need to put it in Docker Hub, which can be done via the `docker push` -command: - - $ docker push namespace/repository_name - -From here, if you're planning to use this container in OSG, return to our -[Containers in OSG Guide][osg-containers] to learn how to upload your container to the OSG's container repository. - - -## Submit your Docker Container to the OSG Repository - -Once your Docker image has been published on Docker Hub, -it needs to be submitted to the OSG Singularity repository -(`/cvmfs/singularity.opensciencegrid.org/`), which also hosts the -OSG-provided default images. - -To get your images included, please create a git pull request with the -container identifier in `docker_images.txt` in the -[cvmfs-singularity-sync repository](https://github.com/opensciencegrid/cvmfs-singularity-sync), -or contact -[support@opensciencegrid.org](mailto:support@opensciencegrid.org) -and we can help you. - -Once your submission has been accepted, it will be automatically -converted to a Singularity image and pushed to the OSG Singularity -repository (see further above). Note: some common Dockerfile features, -like ENV and ENTRYPOINT, are ignored when the Docker image is converted -to a Singularity image. - -Once your container has been added to CVMFS, if you update your original -Docker image, new versions pushed to Docker Hub will automatically be -detected and the version on the OSG (in the CVMFS filesystem) will be -updated accordingly. - -## Special Cases - -### Accessing CVMFS - -If you want your jobs to access CVMFS, make sure that you either: - -1. Use one of the base containers provided by the Open Science Pool - -or - -2. Add a `/cvmfs` folder to your container: - 1. If using a Dockerfile, you can do this with the line `RUN mkdir /cvmfs` - 2. If building your container interactively, run `$ mkdir -p /cvmfs` - -This will enable the container to access -tools and data published on `/cvmfs`. - -If you do not want `/cvmfs` mounted -in the container, please add `+SingularityBindCVMFS = False` to your job submit file. - -### ENTRYPOINT and ENV - -Two options that can be used in the Dockerfile to set the environment or -default command are `ENTRYPOINT` and `ENV`. Unfortunately, both of these -aspects of the Docker container are deleted when it is converted to a -Singularity image in the Open Science Pool. [Email us](mailto:support@opensciencegrid.org) if you would like -to preserve these attributes. - -[osg-containers]: 12000024676 - diff --git a/start/software/containers-singularity.md b/start/software/containers-singularity.md deleted file mode 100644 index 2389cb5e..00000000 --- a/start/software/containers-singularity.md +++ /dev/null @@ -1,97 +0,0 @@ -[title]: - "Create a Singularity Container Image" - -[TOC] - -**NOTE:** Building Singularity containers are currently not supported -on the OSGConnect access point. The guide assumes that you have -your own Linux machine where you can install Singularity and -execute commands via sudo. - -This guide is meant to accompany the instructions for using containers -in the Open Science Pool. You can use your own custom container to run -jobs in the Open Science Pool. This guide describes how to create your -own Singularity container "image" (the blueprint for the container). - -For an overview and how to execute images on OSG, please see -[Use Containers on the OSG][osg-containers] - -## Install Singularity - -Install Singularity to your computer using the appropriate version for -your operating system. Note that OSG does not provide any container -build hosts. sudo is required to build the images. - -## Identify Components - -What software do you want to install? Make sure that you have either the source -code or a command that can be used to install it through Linux (like `apt-get` or -`yum`). - -You'll also need to choose a "base" container, on which to add your particular -software or tools. *We strongly recommend using one of the OSG's published containers -as your starting point.* See the available containers on Docker Hub here: -[OSG Docker Containers](https://hub.docker.com/u/opensciencegrid) -The best candidates for you will be containers that have "osgvo" in the name. - -### Editing the Build Spec - -Create a folder on your computer and inside it, create a blank text file -called `image.def`. - -The first lines of this file should include where to get the base image -from. If using the OSG's Ubuntu 20.04 image that would look like this: - - Bootstrap: docker - From: opensciencegrid/osgvo-ubuntu-20.04:latest - -Then there is a section called `%post` where you put the additional -commands to make the image just like you need it. For example: - - %post - apt-get update -y - apt-get install -y \ - build-essential \ - libbz2-dev \ - libcurl4-gnutls-dev - wget https://cran.r-project.org/src/base/R-3/R-3.6.0.tar.gz - tar -xzf R-3.6.0.tar.gz - cd R-3.6.0 - ./configure - make - make install - -See the [Singularity documentation](https://apptainer.org/user-docs/master/definition_files.html) -for a full reference on how to specify build specs. Note that the `%runscript` -section is ignored when the container is executed on OSG. - -The final `image.def` looks like: - - Bootstrap: docker - From: opensciencegrid/osgvo-ubuntu-20.04:latest - - %post - apt-get update -y - apt-get install -y \ - build-essential \ - libbz2-dev \ - libcurl4-gnutls-dev - wget https://cran.r-project.org/src/base/R-3/R-3.6.0.tar.gz - tar -xzf R-3.6.0.tar.gz - cd R-3.6.0 - ./configure - make - make install - - -Once your build spec is ready, you can "build" the container image by running this command: - - $ sudo singularity build my-container.sif image.def - -Note that `sudo` here is currently required. - -Once the image is built, you can upload it to Stash, test it on OSGConenct, -and use it in your HTCondor jobs. This is all described in the -[container guide][osg-containers]. - -[osg-containers]: 12000024676 - diff --git a/start/software/containers.md b/start/software/containers.md deleted file mode 100644 index b9582f39..00000000 --- a/start/software/containers.md +++ /dev/null @@ -1,188 +0,0 @@ -[title]: - "Use Containers on the OSG" - -[TOC] - -Docker and Singularity are container systems that allow users full -control over their software environment. You can create your own -container image or choose from a set of pre-defined images, and specify -that your submitted jobs run within one of these. - -For jobs on OSG, it does not matter whether you provide a Docker or -Singularity image. Either is compatible with our system and can be -used with little to no modification. Determining factors on when to -use Singularity images over Docker images include if an image already -exists, external to OSG distribution preferences, and if you have -experience building images in one for format and not the other. - -Because OSG is a distributed infrastructure and workloads consists -of a large number jobs (and there container executions), it is -important to consider how the container image is transferred to -the execution nodes. The instructions below contain best practices -when it comes to access both Singularity and Docker images. - -When using a container for your jobs, the container image is -automatically started up when HTCondor matches your job to a slot. The -executable provided in the submit script will be run within the context -of the container image, having access to software and libraries that -were installed to the image, as if they were already on the server where -the job is running. Job executables need not (and should not) run any -commands to start the container. Nor should the container image -contain any entrypoint/cmd - the job is the command to be run in the -container. - -### Exploring Images on the Access Points - -Just like it is important to test your codes and jobs at a small scale, -you should make sure that your container is working correctly. One way -to explore how OSG sees your container images, is to explore them on -the OSG Connect access points. Start an interactive session with the -Singularity "shell" mode. The recommended command line, similar to how -containers are started for jobs, is: - - singularity shell \ - --home $PWD:/srv \ - --pwd /srv \ - --bind /cvmfs \ - --scratch /var/tmp \ - --scratch /tmp \ - --contain --ipc --pid \ - /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-ubuntu-20.04:latest/ - -This will give you an interactive shell in an Ubuntu 20.04 container, -with your current working directory mounted under /srv. You can explore -the container and test your code with for example your own inputs from -your home directory. Once you are down exploring, exit the container -by running `exit` or with `CTRL+D` - -## OSG-Provided Images - -The OSG Team maintains a set of images that are already in the OSG -Singularity repository. *[A list of available containers can be found on this page][container-list].* - -If the software you need isn't already supported in a listed container, -you can use your own container or any container image in Docker Hub -(see sections further below). Once the container you need is in the -OSG Singularity repository, your can submit jobs that run within a -particular container by listing the container image in the submit file. - -For example, this is what a submit file might look like to run your job -within our EL8 container: - - +SingularityImage = "/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el8:latest" - - - queue - -## Custom Singularity Images - -If you already have software in the form of a `.sif` Singuilarity file, -and that file is within the [supported data sizes][data-staging], you -can stage the .sif file with your job. The image will be resused for -each job, and thus the preferred transfer method is [Stash][stash]. -Store the .sif file under `/public/$USERNAME/`, and then use the stash -url directly in the `+SingularityImage` attribute. Note that you can not -use shell variable expansion in the submit file - be sure to replace the -username with your actual OSG Connect username. Example: - - +SingularityImage = "stash:///osgconnect/public/USERNAME/my-custom-image-v1.sif" - - - queue - -Be aware that Stash aggressively caches the image based on file naming. -If you need to do quick changes, please use versioning of the .sif file -so that the caches see a "new" name. In this example, replacing -`my-custom-image-v1.sif` with new content will probably mean that some -nodes get the old version and some nodes the new version. Prevent this -by creating a new file named with v2. - -More information on how to create Singularity images can be found -in the [Singularity Images Guide][singularity-guide]. - -## Custom Docker Images - -If you would prefer to create or use an existing Docker Hub container, -for example an authoritative container for your software which -already exists in Docker Hub, OSG can distribute the image for you -via CVMFS. The result is a synchronized copy of the image under -`/cvmfs/singularity.opensciencegrid.org/` which is cached and available -to the execution nodes. Creating and/or registering a Docker -image is described in the [Docker Images Guide][docker-guide]. - -To run a job with a Docker image, use the `+SingularityImage` to -specify the image the job should be using. Example: - - +SingularityImage = "/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el8:latest" - - - queue - -Another example would be if your Docker Hub username is `alice` and you -created a container called `ncbi-blast` added to the OSG Singularity -repository, your submit file will include: - - +SingularityImage = "/cvmfs/singularity.opensciencegrid.org/alice/ncbi-blast" - - - queue - -## Using Containers from Non-OSG Connect Access Points - -Users on non-OSG Connect access points can use all the container -functionality described above, but will have to use slightly more -complex job submit files. This is because the OSG Connect access points -uses job transforms to update the jobs based on the `+SingularityImage` -attribute, and OSG Connect users also have direct access to Stash. - -To run a Singularity image from a non-OSG Connect access point, include -a job `requirements`, and specify a method for image transfer. For example: - - Requirements = HAS_SINGULARITY == TRUE && SINGULARITY_CAN_USE_SIF = TRUE - transfer_input_files = http://datastore.host/mycontainer.sif - +SingularityImage = "./mycontainer.sif" - - - queue - -For images available on CVMFS, just add job `requirements`: - - Requirements = HAS_SINGULARITY == TRUE - +SingularityImage = "/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el8:latest" - - - queue - -## Frequently Asked Questions / Common Issues - -### FATAL: kernel too old - -If you get a *FATAL: kernel too old* error, it means that the glibc version in the -image is too new for the kernel on the host. You can work around this problem by -specifying the minimum host kernel. For example, if you want to run the Ubuntu 18.04 -image, specfy a minimum host kernel of 3.10.0, formatted as 31000 -(major * 10000 + minor * 100 + patch): - - Requirements = HAS_SINGULARITY == True && OSG_HOST_KERNEL_VERSION >= 31000 - - -### Learning More - -For more information about Docker, please see: - -* [Docker Home Page](https://www.docker.com/) - -and Singularity, please see: - - * [Singularity Home Page](http://singularity.lbl.gov/) - - Singularity has become the preferred containerization method in scientific computing. The following talk describes Singularity for scientific computing: - - - -[container-howto]: 12000058245 -[container-list]: 12000073449 -[data-staging]: 12000002985 -[stash]: 12000002775 -[docker-guide]: 12000058245 -[singularity-guide]: 12000086275 - diff --git a/start/software/example-compilation.md b/start/software/example-compilation.md deleted file mode 100644 index 6df0ef35..00000000 --- a/start/software/example-compilation.md +++ /dev/null @@ -1,485 +0,0 @@ -[title]: - "Example Software Compilation" -[TOC] - -# Example of Compilng Software For Use In OSG Connect - -## Introduction - -Due to the distributed nature of the Open Science Pool, you will always need to -ensure that your jobs have access to the software that will be executed. This guide provides -a detailed example of compiling software for use in OSG Connect. - -For this example, we will be compiling Samtools which is a very common bioinformatics -software for working with aligned sequencing data. However, the steps described in this -example are applicable to many other software. For a general introduction to software -compilation in OSG Connect, please see [Compiling Software for OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/5000652099). - -Specifically, this guide provides two examples of compiling Samtools, [one without CRAM file -support](#compile-samtools-without-cram-support) and [one with CRAM file support](#compile-samtools-with-cram-support). -*Why two examples?* Currently, to install Samtools -with CRAM support requires additional dependencies (aka libraries) that will also need to be -installed and most Samtools users are only working with BAM files which does not require CRAM -support. - -*Do I need CRAM support for my work?* CRAM is an alternative compressed sequence alignment -file format to BAM. Learn more at [https://www.sanger.ac.uk/tool/cram/](https://www.sanger.ac.uk/tool/cram/). - -## Compile Samtools Without CRAM Support - -### Step 1. Acquire Samtools source code - -Samtools source code is available at [http://www.htslib.org/download/](http://www.htslib.org/download/). The -development code is also available via GitHub at [https://github.com/samtools/samtools](https://github.com/samtools/samtools). On the download page is some important information to make note of: - -> "\[Samtools\] uses HTSlib internally \[and\] these source packages contain their own copies of htslib" - -What this means is 1.) HTSlib is a dependency of Samtools and 2.) the HTSlib source code is included -with the Samtools source code. - -Either download the Samtools source code to your computer and upload to your login node, or -right-click on the Samtools source code link and copy the link location. Login in to your OSG Connect login node -and use `wget` to download the source code directly and extract the tarball: - - [user@login ~]$ wget https://github.com/samtools/samtools/releases/download/1.10/samtools-1.10.tar.bz2 - [user@login ~]$ tar -xjf samtools-1.10.tar.bz2 - -The above two commands will create a directory named `samtools-1.10` which contains all the code -and instructions needed for compiling Samtools and HTSlib. Take a moment to look at the content available -in this new directory. - -### Step 2. Read through installation instructions - -*What steps need to be performed for our compilation? What system dependencies exist for our -software?* Answers to these questions, and other important information, should be available -in the installation instructions for your software which will be available online and/or -included in the source code. - -The HTSlib website where the Samtools source code is hosted provides basic installation instructions -and refers users to `INSTALL` (which is a plain text file that can be found in `samtools-1.10/`) for -more information. You will also see a `README` file in the source code directory which will provide -important information. `README` files will always be included with your source code and we -recommend reviewing before compiling software. There is also a `README` and `INSTALL` file available -for HTSlib in the source code directory `samtools-1.10/htslib-1.10/`. - -`cd` to samtools-1.10 and read through `README` and `INSTALL`. As described in `INSTALL`, -the Samtools installation will follow the common `configure`, `make`, `make install` process: - - Basic Installation - ================== - - To build and install Samtools, 'cd' to the samtools-1.x directory containing - the package's source and type the following commands: - - ./configure - make - make install - - The './configure' command checks your build environment and allows various - optional functionality to be enabled (see Configuration below). - -Also described in `INSTALL` are a number of required and optional system dependencies -for installing Samtools and HTSlib (which is itself a dependency of Samtools): - - System Requirements - =================== - - Samtools and HTSlib depend on the following libraries: - - Samtools: - zlib - curses or GNU ncurses (optional, for the 'tview' command) - - - HTSlib: - zlib - libbz2 - liblzma - libcurl - (optional but strongly recommended, for network access) - libcrypto - (optional, for Amazon S3 support; not needed on MacOS) - - ... - - The bzip2 and liblzma dependencies can be removed if full CRAM support - is not needed - see HTSlib's INSTALL file for details. - -Some dependencies are needed to support certain features from Samtools (such as `tview` and -CRAM compression). You will not need `tview` as this is intended for interactive work which is -not currently supported in OSG Connect. For this specific compilation example, we will disable -both `tview` and CRAM support - see [below](#compile-samtools-with-cram-support) for our -compilation example that will provide CRAM file support. - -Following the suggestion in the Samtools `INSTALL` file, we can view the HTSlib `INSTALL` -file at `samtools-1.10/htslib-1.10/INSTALL`. Here we will find the necessary -information for disabling bzip2 and liblzma dependencies: - - --disable-bz2 - Bzip2 is an optional compression codec format for CRAM, included - in HTSlib by default. It can be disabled with --disable-bz2, but - be aware that not all CRAM files may be possible to decode. - - --disable-lzma - LZMA is an optional compression codec for CRAM, included in HTSlib - by default. It can be disabled with --disable-lzma, but be aware - that not all CRAM files may be possible to decode. - -These are two flags that will need to be used when performing our installation. - -To determine what libraries are available on our OSG Connect login node, we can look at `/usr/lib` -and `/usr/lib64` for the various Samtools library dependencies, for example: - - [user@login ~]$ ls /usr/lib* | grep libcurl - [user@login ~]$ ls /usr/lib* | grep htslib - -Although we will find matches for `libcurl`, we will not find any `htslib` files meaning that -HTSlib is not currently installed on the login node, nor is it currently available -as a module. This means that HTSlib will also need to be compiled. Luckly, the Samtools -developers have conveniently included the HTSlib source code with the Samtools source code -and have made it possible to compile both Samtools and HTSlib at the same time. From the -Samtools `INSTALL` file, is the following: - - By default, configure looks for an HTSlib source tree within or alongside - the samtools source directory; if there are several likely candidates, - you will have to choose one via this option. - -This mean that we don't have to do anything extra to get HTSlib installed because -the Samtools installation will do it by default. - -> When performing your compilation, if your compiler is unable to locate the necessary -> libraries, or if newer versions of libraries are needed, it will result in an error - this -> makes for an alternative method of determining whether your system has the appropriate -> libraries for your software and more often than not, installation by trial and error is -> a common approach. However, taking a little bit of time before hand -> and looking for library files can save you time and frustration during software compilation. - -### Step 3. Perform Samtools compilation - -We now have all of the information needed to start our compilation of Samtools without CRAM support. - -First, we will create a new directory in our `home` directory that will store the -Samtools compiled software. The example here will use a directory, called `my-software`, -for organizing all compiled software in the `home` directory: - - [user@login ~]$ mkdir $HOME/my-software - [user@login ~]$ mkdir $HOME/my-software/samtools-1.10 - -> As a best practice, always include the version name of your software in the directory name. - -Next we'll change to the Samtools source code direcory that was created in -[Step 1](#step-1-acquire-samtools-source-code). You should see the `INSTALL` and `README` files -as well as a file called `configure`. - -The first command we will run is `./configure` - this step will execute the configure script -and allows us to modify various details about our Samtools installation. We will be executing `configure` -with several flags: - - [user@login samtools-1.10]$ ./configure --prefix=$HOME/my-software/samtools-1.10 --disable-bz2 --disable-lzma --without-curses - -Here we used `--prefix` to specify where we would like the final Samtools software -to be installed, `--disable-bz2` and `--disable-lzma` to disable `lzma` -and `bzip2` dependencies for CRAM, and `--without-curses` to disable `tview` support. - -Next run the final two commands: - - [user@login samtools-1.10]$ make - [user@login samtools-1.10]$ make install - -Once `make install` has finished running, the compilation is complete. We can -also confirm this by looking at the content of `~/my-software/samtools-1.10/` where -we had Samtools installed: - - [user@login samtools-1.10]$ cd ~ - [user@login ~]$ ls -F my-software/samtools-1.10/ - bin/ share/ - -There will be two directories present in `my-software/samtools-1.10`, one named `bin` and -another named `share`. The Samtools executable will be located in `bin` and we can give -it a quick test to make sure it runs as expected: - - [user@login ~]$ ./my-software/samtools-1.10/bin/samtools view - -which will return the Samtools `view` usage statement. - -### Step 4. Make our software portable - -Our subsequent job submissions on OSG Connect will need a copy of our software. For -convenience, we recommend converting your software directory to a tar archive. -First move to `my-software/`, then create the tar archive: - - [user@login ~]$ mv my-software/ - [user@login my-software]$ tar -czf samtools-1.10.tar.gz samtools-1.10/ - [user@login my-software]$ ls samtools-1.10* - samtools-1.10/ samtools-1.10.tar.gz - [user@login my-software]$ du -h samtools-1.10.tar.gz - 2.0M samtools-1.10.tar.gz - -The last command in the above example returns the size of our tar archive. This is -important for determine the appropriate method that we should use for transferring -this file along with our subsequent jobs. To learn more, please see -[Introduction to Data Management on OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/12000002985). - -To clean up and clear out space in your home directory, we recommend deleting the Samtools source -code directory. - -### Step 5. Use Samtools in our jobs - -Now that Samtools has been compiled we can submit jobs that use this software. Below is an example submit file -for a job that will use Samtools with a BAM file named `my-sample.bam` which is <100MB in size: - - #samtools.sub - log = samtools.$(Cluster).log - error = samtools.$(Cluster)_$(Process).err - output = samtools.$(Cluster)_$(Process).out - - executable = samtools.sh - - transfer_input_files = /home/username/my-software/samtools-1.10.tar.gz, my-sample.bam - - should_transfer_files = YES - when_to_transfer_output = ON_EXIT - - requirements = (OSGVO_OS_STRING == "RHEL 7") - request_memory = 1.3GB - request_disk = 1.5GB - request_cpus = 1 - - queue 1 - -The above submit file will transfer a complete copy of the Samtools tar archive -created in [Step 4](#step-4-make-our-software-portable) and also includes an important -`requirements` attribute which tells HTCondor to run our job on -execute nodes running Red Hat Linux version 7 operating system. - -> The resource requests for your jobs may differ from what is shown in the -> above example. Always run tests to determine the appropriate requests for your jobs. - -Some additional steps are then needed in the executable bash script used by this job -to "untar" the Samtools and add this software to the `PATH` enviroment variable: - - #!/bin/bash - # samtools.sh - - # untar software - tar -xzf samtools-1.10.tar.gz - - # modify environment variables - export PATH=$_CONDOR_SCRATCH_DIR/samtools-1.10/bin:$PATH - - # run samtools commands - ... - -## Compile Samtools With CRAM Support - -This example includes steps to install and use a library and to use a module, -which are both currently needed for compiling Samtools with CRAM support. - -The steps in this example assume that you have performed -[Step 1](#step-1-acquire-samtools-source-code) and -[Step 2](#step-2-read-through-installation-instructions) in the above example for -compiling Samtools without CRAM support. - -### Step 2. Read through installation instructions, continued - -From both the Samtools and HTSlib `INSTALL` files, we know that both bzip2 and -libzlma are required for CRAM support. We can check our system for these libraries: - - [user@login ~]$ ls /usr/lib* | grep libz - [user@login ~]$ ls /usr/lib* | grep libbz2 - -which will reveal that both sets of libraries are available on the login. **However** -if we were to attempt Samtools installation with CRAM support right now -we would find that this results in an error when performing the `configure` step. - -*If the libraries are present, why do we get this error?* This error is due to -differences between types of library files. For example, running -`ls /usr/lib* | grep libbz2` will return two matches, `libbz2.so.1` and `libbz2.so.1.0.6`. -But running `ls /usr/lib* | grep liblz` will return four matches including three -`.so` and one `.a` files. Our Samtools compilation specifically requires -the `.a` type of library file for both `libbz2` and `liblzma` and the absence of -this type of library file in `/usr/lib64` is why compilation -will fail without additional steps. - -Luckily for us, bzip2 version 1.0.6 is available as a module and this module -includes access to a `.a` library file. We will use this module for our Samtools compilation. -To learn more about using modules, please see -[Accessing Software using Distributed Environment Modules](https://support.opensciencegrid.org/support/solutions/articles/12000048518). -`liblzma` however is not currently available as a module and our next step -will be to install `liblzma`. - -### Step 3. Compile liblzma - -To compile Samtools with CRAM support requires that we first compile -`liblzma`. Following the same approach as we did for Samtools, first we -acquire a copy of the the latest liblzma source code, then review the installation instructions. -From our online search we will that `liblzma` -is availble from the XZ Utils library package. - - [user@login ~]$ wget https://tukaani.org/xz/xz-5.2.5.tar.gz - [user@login ~]$ tar -xzf xz-5.2.5.tar.gz - -Then review the installation instructions and check for dependencies. Everything -that is needed for the default installation of XZ utils is currently available on the login node. - - [user@login ~]$ cd xz-5.2.5/ - [user@login xz-5.2.5]$ less INSTALL - -Perform the XZ Utils compilation: - - [user@login xz-5.2.5]$ mkdir $HOME/my-software/xz-5.2.5 - [user@login xz-5.2.5]$ ./configure --prefix=$HOME/my-software/xz-5.2.5 - [user@login xz-5.2.5]$ make - [user@login xz-5.2.5]$ make install - [user@login xz-5.2.5]$ ls -F $HOME/my-software/xz-5.2.5 - /bin /include /lib /share - -Success! - -Lastly we need to set some environment variables so that Samtools knows where to -find this library: - - [user@login xz-5.2.5]$ export PATH=$HOME/my-software/xz-5.2.5/bin:$PATH - [user@login xz-5.2.5]$ export LIBRARY_PATH=$HOME/my-software/xz-5.2.5/lib:$LIBRARY_PATH - [user@login xz-5.2.5]$ export LD_LIBRARY_PATH=$LIBRARY_PATH - -### Step 4. Load bzip2 module - -After installing XZ Utils and setting our environment variable, next we will -load the bzip2 module: - - [user@login xz-5.2.5]$ module load bzip2/1.0.6 - -Loading this module will further modify some of your environment variables -so that Samtools is able to locate the bzip2 library files. - -To learn more about using modules, please see -[Accessing Software using Distributed Environment Modules](https://support.opensciencegrid.org/support/solutions/articles/12000048518). - -### Step 5. Compile Samtools - -After compiling XZ Utils (which provides `liblzma`) and loading the `bzip2 1.0.6` module, -we are now ready to compile Samtools with CRAM support. - -First, we will create a new directory in our `home` directory that will store the -Samtools compiled software. The example here will use a common directory, called `my-software`, -for organizing all compiled software in the `home` directory: - - [user@login ~]$ mkdir $HOME/my-software - [user@login ~]$ mkdir $HOME/my-software/samtools-1.10 - -> As a best practice, always include the version name of your software in the directory name. - -Next, we will change our directory to the Samtools source code direcory that was created in -[Step 1](#step-1-acquire-samtools-source-code). You should see the `INSTALL` and `README` files -as well as a file called `configure`. - -The first command we will run is `./configure` - this file is a script that allows us -to modify various details about our Samtools installation and we will be executing `configure` -with a flag that disables `tview`: - - [user@login samtools-1.10]$ ./configure --prefix=$HOME/my-software/samtools-1.10 --without-curses - -Here we used `--prefix` to specify where we would like the final Samtools software -to be installed and `--without-curses` to disable `tview` support. - -Next run the final two commands: - - [user@login samtools-1.10]$ make - [user@login samtools-1.10]$ make install - -Once `make install` has finished running, the compilation is complete. We can -also confirm this by looking at the content of `~/my-software/samtools-1.10/` where -we had Samtools installed: - - [user@login samtools-1.10]$ cd ~ - [user@login ~]$ ls -F my-software/samtools-1.10/ - bin/ share/ - -There will be two directories present in `my-software/samtools-1.10`, one named `bin` and -another named `share`. The Samtools executable will be located in `bin` and we can give -it a quick test to make sure it runs as expected: - - [user@login ~]$ ./my-software/samtools-1.10/bin/samtools view - -which will return the Samtools `view` usage statement. - -### Step 6. Make our software portable - -Our subsequent job submissions on OSG Connect will need a copy of our software. For -convenience, we recommend converting your software directory to a tar archive. -First move to `my-software/`, then create the tar archive: - - [user@login ~]$ cd my-software/ - [user@login my-software]$ tar -czf samtools-1.10.tar.gz samtools-1.10/ - [user@login my-software]$ ls samtools-1.10* - samtools-1.10/ samtools-1.10.tar.gz - [user@login my-software]$ du -h samtools-1.10.tar.gz - 2.0M samtools-1.10.tar.gz - -The last command in the above example returns the size of our tar archive. This is -important for determine the appropriate method that we should use for transferring -this file along with our subsequent jobs. To learn more, please see -[Introduction to Data Management on OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/12000002985). - -Follow the these same steps for creating a tar archive of the xz-5.2.5 library as well. - -To clean up and clear out space in your home directory, we recommend deleting the Samtools source -code directory. - -### Step 7. Use Samtools in our jobs - -Now that Samtools has been compiled we can submit jobs that use this software. For Samtools -with CRAM we will also need to bring along a copy of XZ Utils (which includes the `liblzma` library) -and ensure that our jobs have access to the `bzip2 1.0.6` module. Below is an example submit file -for a job that will use Samtools with a Fasta file `genome.fa' and CRAM file named `my-sample.cram` which is <100MB in size: - - #samtools-cram.sub - log = samtools-cram.$(Cluster).log - error = samtools-cram.$(Cluster)_$(Process).err - output = samtools-cram.$(Cluster)_$(Process).out - - executable = samtools-cram.sh - - transfer_input_files = /home/username/my-software/samtools-1.10.tar.gz, /home/username/my-software/xz-5.2.5.tar.gz, genome.fa, my-sample.cram - - should_transfer_files = YES - when_to_transfer_output = ON_EXIT - - requirements = (OSGVO_OS_STRING == "RHEL 7") && (HAS_MODULES =?= TRUE) - request_memory = 1.3GB - request_disk = 1.5GB - request_cpus = 1 - - queue 1 - -The above submit file will transfer a complete copy of the Samtools tar archive -created in [Step 6](#step-6-make-our-software-portable) as well as a copy of XZ Utils installation from -[Step 3](#step-3-compile-liblzma). This submit file -also includes two important `requirements` which tell HTCondor to run our job on -execute nodes running Red Hat Linux version 7 operating system and which has -access to OSG Connect software modules. - -> The resource requests for your jobs may differ from what is shown in the -> above example. Always run tests to determine the appropriate requests for your jobs. - -Some additional steps are then needed in the executable bash script used by this job -to "untar" the Samtools and XZ Util tar archives, modify the `PATH` and -`LD_LIBRARY_PATH` enviroments of our job, and load the `bzip2` module: - - #!/bin/bash - # samtools-cram.sh - - # untar software and libraries - tar -xzf samtools-1.10.tar.gz - tar -xzf xz-5.2.5.tar.gz - - # modify environment variables - export LD_LIBRARY_PATH=$_CONDOR_SCRATCH_DIR/xz-5.2.5/lib:$LD_LIBRARY_PATH - export PATH=$_CONDOR_SCRATCH_DIR/samtools-1.10/bin:$_CONDOR_SCRATCH_DIR/xz-5.2.5/bin:$PATH - - # load bzip2 module - module load bzip2/1.0.6 - - # run samtools commands - ... - diff --git a/start/software/new_modules_list.md b/start/software/new_modules_list.md deleted file mode 100644 index 75349ef1..00000000 --- a/start/software/new_modules_list.md +++ /dev/null @@ -1,93 +0,0 @@ - -[title]: - "Access Software using Distributed Environment Modules" - -[TOC] - -# Introduction - -This page covers the use of distributed environment modules on RHEL7 compute nodes in the OSG computing environment. Environment modules provide users with an easy way to access different versions of software, libraries, and compilers. - -Currently, OSG Connect login nodes and a majority of OSG compute nodes use the RHEL7 operating system. However, given the distributed nature of the OSG, there are at times a number of RHEL6 and RHEL8 compute nodes also available to users. For users that require, or who can also use, RHEL6 compute nodes for their applications, please [contact us](mailto:support@opensciencegrid.org) to learn more about modules specifically available to RHEL6 compute nodes. - -# List Available Modules on OSG Connect - -First sign in to your OSG Connect login node. Then use `module avail` to see all available software applications and libraries: - - $ module avail - - ------------ /cvmfs/connect.opensciencegrid.org/modules/modulefiles/linux-rhel7-x86_64/Core ------------ - autoconf/2.69 libiconv/1.15 py-idna/2.5-py3.7 (D) - automake/1.16.1 libjpeg-turbo/1.5.3 py-ipaddress/1.0.18-py2.7 - binutils/2.31.1-py2.7 libjpeg-turbo/1.5.90 (D) py-kiwisolver/1.0.1-py2.7 - binutils/2.31.1-py3.7 (D) libpciaccess/0.13.5 py-kiwisolver/1.0.1-py3.7 (D) - bison/3.0.5-py2.7 libpng/1.6.34 py-lit/0.5.0-py2.7 - bison/3.0.5 (D) libpthread-stubs/0.4 py-mako/1.0.4-py2.7 - boost/1.68.0-py2.7 libsigsegv/2.11 py-markupsafe/1.0-py2.7 - boost/1.68.0-py3.7 (D) libsm/1.2.2 py-matplotlib/2.2.3-py2.7 - bowtie2/2.3.4.1-py2.7 libtiff/4.0.9 py-matplotlib/3.0.0-py3.7 (D) - bullet3/2.87 libtool/2.4.6 py-nose/1.3.7-py2.7 - bwa/0.7.17 libx11/1.6.5 py-numexpr/2.6.5-py2.7 - bzip2/1.0.6 libxau/1.0.8 py-numexpr/2.6.5-py3.7 (D) - cairo/1.14.12-py2.7 libxcb/1.13 py-numpy/1.15.2-py2.7 - cctools/7.0.8-py2.7 libxdamage/1.1.4 py-numpy/1.15.2-py3.7 (D) - cfitsio/3.450 libxdmcp/1.1.2 py-pandas/0.23.4-py2.7 - charmpp/6.8.2 libxext/1.3.3 py-pandas/0.23.4-py3.7 (D) - - ... more modules ... - - Where: - D: Default Module - - Use "module spider" to find all possible modules. - Use "module keyword key1 key2 ..." to search for all possible modules matching any of the "keys". - -Please know, there are more than 200 modules available to OSG Connect users. The above is **not** a complete list of all available modules as the list is quite long and subject to change as new sofware and libraries get added. - -# How To Use Environment Modules - -In order to use the software or libraries provided in a module, you will need to 'load' that particular module. To load a particular module, use `module load [modulename]`. - -For example: - - $ module load python/2.7.14-k - -To see currently loaded modules, use `module list`. - -For example: - - $ module list - Currently Loaded Modules: - 1) python/2.7.14-k - -To unload a package and remove a module from your environment, use `module unload [modulename]`. - -For example: - - $ module unload python/2.7.14-k - -Finally, `module help` will give you more detailed information. - -# Use Environment Modules in Jobs - -## Add module commands to executable script - -To use environment modules within a job, use the same `module load` command -described above inside your job's main "executable" to load your software -and then run it. For example: - - #!/bin/bash - - module load python/2.7.14-k - python myscript.py - - -## Set appropriate submit file requirements - -Not all resources available through OSG Connect support distributed environment modules. In order to ensure that your -jobs will have access to OSG Connect modules, you will need to add the following to your HTCondor submit file: - - Requirements = (HAS_MODULES =?= true) && (OSGVO_OS_STRING == "RHEL 7") && (OpSys == "LINUX") - -or append the above requirements if you already have other requirements specified: - - Requirements = [Other requirements ] && (HAS_MODULES =?= true) && (OSGVO_OS_STRING == "RHEL 7") && (OpSys == "LINUX") diff --git a/start/software/singularity-containers.md b/start/software/singularity-containers.md deleted file mode 100644 index 631033a3..00000000 --- a/start/software/singularity-containers.md +++ /dev/null @@ -1,155 +0,0 @@ -[title]: - "Use Containers on the OSG" - -[TOC] - -Docker and Singularity are container systems that allow users full control -over their software environment. You can create your own container image or choose from a set of pre-defined images, -and specify that your submitted jobs run within one of these. - -For jobs on OSG, it does not matter whether you provide a Docker or Singularity -image. Either is compatible with our system and can be used with little to -no modification, though *may* be dependent on your software. Please -feel free to contact us at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) if you have any questions. - -## Use an OSG-Provided Singularity Image - -The OSG Team maintains a set of images that are already in the OSG Singularity repository. A list of available containers can be -found on [this page][container-list]. - -If the software you need isn't already supported in a listed container, you can use your -own container or any container image in Docker Hub (see sections further below). Once the container you -need is in the OSG Singularity repository, your can submit jobs that run within a particular container -by listing the container image in the submit file, along with a requirement to only run on sites in the -Open Science Pool that support Singularity. - -For example, this is what a submit file might look like to run your job within our EL7 container: - - Requirements = HAS_SINGULARITY == TRUE - +SingularityImage = "/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el7:latest" - - - queue - -> **When using a container for your jobs, as in the above example, the container image is automatically started up when HTCondor matches your job to a slot. The executable provided in the submit script will be run within the context of the container image, having access to software and libraries that were installed to the image, as if they were already on the server where the job is running. Job executables need not (and should not) run any singularity or docker commands to start the container.** - -## Use a Custom Singularity Image - -If you already have software in the form of a `.sif` Singuilarity file, and that file -is within the [supported data sizes](https://support.opensciencegrid.org/support/solutions/articles/12000002985-overview-data-staging-and-transfer-to-jobs), you can stage the .sif file in your `/home` or `/public` -location, and list it in the submit file to be transfered and used by -the job (from within the working directory of the job, i.e. `./`): - - transfer_input_files = path/to/mycontainer.sif, - +SingularityImage = "./mycontainer.sif" - Requirements = HAS_SINGULARITY == TRUE - - - queue - -**Note that if your container is large enough to stage in `/public` that you will -need to specify the appropriate `stash:///` address in the `transfer_input_files` line. - -## Use Docker Images via Docker Hub - -If you would prefer to create or use an existing Docker Hub container -(especially if an authoritative container for your software already exists in DockerHub), you will: - -1. Create your own Docker container image, and push it to Docker Hub. (if a container for your software doesn't already exist in Docker Hub) -2. Add a Docker Hub image to the OSG Singularity repository. -3. Use the container image in jobs. - -We will expand on each of these steps below. - -### Create a Container - -If you want to use an container image you have created yourself, the image -should be defined as a Docker image and published in [Docker -Hub](https://hub.docker.com/). Using Docker Hub allows us to easily import the images into -our the OSG Singularity repository (see further above). - -See [this page][container-howto] for how to create a Docker image on your own computer and -push it to Docker Hub so it can be used by the Open Science Pool. - -### Submit your Docker Container to the OSG Repository - -Once your Docker image has been published on Docker Hub, it needs to be -submitted to the OSG Singularity repository (`/cvmfs/singularity.opensciencegrid.org/`), -which also hosts the OSG-provided default images (described further above). - -To get your images included, please create a git pull request with the container -identifier in `docker_images.txt` in the -[cvmfs-singularity-sync repository](https://github.com/opensciencegrid/cvmfs-singularity-sync), -or contact -[support@opensciencegrid.org](mailto:support@opensciencegrid.org) -and we can help you. - -Once your submission has been accepted, it will be automatically converted to a Singularity -image and pushed to the OSG Singularity repository (see further above). Note: some -common Dockerfile features, like ENV and ENTRYPOINT, are ignored when the Docker -image is converted to a Singularity image. See our the "Special Cases" section of our -[how to guide][container-howto] for more details -of how to deal with this. - -Once your container has been added to CVMFS, -if you update your original Docker image, new versions pushed to Docker Hub will -automatically be detected and the version on the OSG (in the CVMFS filesystem) -will be updated accordingly. - -### Use the Docker Container in a Job - -To use your container to run jobs, you will specify the container path within CVMFS, as well as a requirement to run -only on execute points that support Singularity. For example, if your Docker Hub username is `alice` and you created a container called -`ncbi-blast` added to the OSG Singularity repository, your submit file will include: - - Requirements = HAS_SINGULARITY == TRUE - +SingularityImage = "/cvmfs/singularity.opensciencegrid.org/alice/ncbi-blast" - - - queue - - - -## Frequently Asked Questions / Common Issues - -### I already have a Singularity container, not a Docker one - -Email the OSG Connect team: support@opensciencegrid.org - -### FATAL: kernel too old - -If you get a *FATAL: kernel too old* error, it means that the glibc version in the -image is too new for the kernel on the host. You can work around this problem by -specifying the minimum host kernel. For example, if you want to run the Ubuntu 18.04 -image, specfy a minimum host kernel of 3.10.0, formatted as 31000 -(major * 10000 + minor * 100 + patch): - - Requirements = HAS_SINGULARITY == True && OSG_HOST_KERNEL_VERSION >= 31000 - -### Exploring Images on the Submit Host - -Images can be explored interactively on the submit hosts by starting it -in "shell" mode. The recommended command line, similar to how containers -are started for jobs, is: - - singularity shell \ - --home $PWD:/srv \ - --pwd /srv \ - --bind /cvmfs \ - --scratch /var/tmp \ - --scratch /tmp \ - --contain --ipc --pid \ - /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-ubuntu-xenial:latest - -### Learning More - -For more information about Docker, please see: - -* [Docker Home Page](https://www.docker.com/) - -and Singularity, please see: - - * [Singularity Home Page](http://singularity.lbl.gov/) - - -[container-howto]: 12000058245 -[container-list]: 12000073449 diff --git a/start/software/software-overview.md b/start/software/software-overview.md deleted file mode 100644 index 68661ab6..00000000 --- a/start/software/software-overview.md +++ /dev/null @@ -1,73 +0,0 @@ -[title]: - "Using Software on the Open Science Pool" - -[TOC] - -# Overview of Software Options - -There are several options available for managing the software needs of your work within the Open Science Pool (OSPool). -In general, the OSPool can support most popular, open source software that fit the distributed -high throughput computing model. At present, we do not have or support most commercial software -due to licensing issues. - -Here we review options, and provide links to additonal information, for using software -installed by users, software available as precompiled binaries or via containers, and -preinstalled software via modules. - -## Install Your Own Software - -For most cases, it will be advantageous for you to install the software needed for your jobs. -This not only gives you the greatest control over your computing environment, but will also -make your jobs more distributable, allowing you to run jobs at more locations. - -Installing your own software can be performed interactively on your assigned login server. More -information about how to install your own software from source code can be found at -[Compiling Software for the OSPool](https://support.opensciencegrid.org/support/solutions/articles/5000652099). -When installing software on an OSG Connect login node, your software will be specifically compiled against -the Red Hat Enterprise Linux (RHEL) 7 OS used on these nodes. In most cases, subsequent -jobs that use this software will also need to run on a RHEL 7 OS, which can be specified by the -`requirements` attribute of your HTCondor submit files - an example is provided in the software -compilation guide linked above. Be sure to also review our -[Introduction to Data Management on OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/12000002985) -guide to determine the appropriate method for transferring software with your jobs. - -## Use Precompiled Binaries and Prebuilt Executables - -Some software may be available as a precompiled binary or prebuilt executable -which provides a quick and easy way to run a program without the need for installation -from source code. Binaries and executables are software files that are ready to -run as is, however binaries should always be tested beforehand. There are several -important considerations for using precompiled binaries on the OSPool: -1.) only binary files compiled against a Linux operating system are suitable -for use on the OSPool, 2.) some softwares have system and hardware dependencies that must -be met in order to run properly, and 3.) the available binaries may not have been -compiled with the feaures or configuration needed for your work. - -## Use Docker and Singularity Containers - -Container systems provide users with customizable and reproducable computing and software -environments. The Open Science Pool is compatible with both Singularity and Docker containers - the -latter will be converted to a Singularity image and added to the OSG container image -repository. Users can choose from a set of pre-defined containers already available within OSG, -or can use published or custom made containers. More details on how to use containers on the OSPool can be found in our -[Docker and Singularity Containers](https://support.opensciencegrid.org/support/solutions/articles/12000024676) guide. - -## Access Software In Distributed Modules - -Currently, the OSPool provides over 200 preinstalled software, libraries, and -compiliers via distributed environment modules. Modules provide a streamlined option -for working with different software versions as well as managing library dependencies for -software. To run jobs that use modules, your HTCondor submit files must include additional -`requirements` to direct your jobs to appropriate compute nodes within OSG. - -More details about using modules can be found -[here](https://support.opensciencegrid.org/support/solutions/articles/12000048518). - -## Ask for Helpk - -If you are not sure which of the above options might be best for your software, we're happy to help! Just contact us at -[support@opensciencegrid.org](mailto:support@opensciencegrid.org). - -**Watch this video from the 2021 OSG Virtual School** for more information about using software on OSG: - -[](https://www.youtube.com/embed/xUeIQbVXOMQ) - diff --git a/start/software/software-request.md b/start/software/software-request.md deleted file mode 100644 index c4922fc7..00000000 --- a/start/software/software-request.md +++ /dev/null @@ -1,17 +0,0 @@ -[title]: - "Request Help with Your Software" - -A large number of software packages can be used by compiling a portable installation or using a container -(many community sofwares are already available in authoritative containers). If you believe none of -these options ([described here](5000634395)) are applicable for your software, please get in touch with a simple email to -[support@opensciencegrid.org][support] that describes: -1. the software name, version, and/or website with download and install instructions -2. what science each job does, using the software -3. what you've tried so far (if anything), and what indications of issues you've experienced - -As long as this code is: - -1. available to the public in source form (e.g. open source) -2. licensed to all users, and does not require a license key -3. would not be better supported by another approach (which are usually preferable) - -we should be able to help you create a portable installation with the 'right' solution. diff --git a/training/resources/contact-information.md b/training/resources/contact-information.md deleted file mode 100644 index cddb646b..00000000 --- a/training/resources/contact-information.md +++ /dev/null @@ -1,9 +0,0 @@ -[title]: - "Contact OSG for non-Support Inquiries" - -For media contact, leadership, or general questions about OSG, please see our -[main website](https://osg-htc.org/contact) - -To request a user account on OSG Connect, please visit the [website](http://osgconnect.net). - -For any assistance or technical questions regarding jobs or data, please see our page on how to [Get Help](https://support.opensciencegrid.org/support/solutions/articles/12000084585) -and/or contact the OSG Research Facilitation team at [support@opensciencegrid.org](mailto:support@opensciencegrid.org) diff --git a/training/resources/frequently-asked-questions-faq-.md b/training/resources/frequently-asked-questions-faq-.md deleted file mode 100644 index bb6d89ef..00000000 --- a/training/resources/frequently-asked-questions-faq-.md +++ /dev/null @@ -1,127 +0,0 @@ -[title]: - "Frequently Asked Questions" -[TOC] - -## Getting Started - -**Who is eligible to become the user of OSG Connect?** - -Any researcher affiliated with a U.S. institution (college, university, national laboratory or research foundation) is eligible to become an OSG Connect user. Researchers outside of the U.S. with affiliations to U.S. groups may be eligible for membership if they are sponsored by a collaborator within the U.S. Researchers outside of the U.S. are asked to first -[contact us](mailto:support@opensciencegrid.org) directly to discuss membership. - -**How do I become an user of OSG Connect?** - -Please follow the steps outlined in the [Sign Up process](http://osgconnect.net/signup). - -## Software - -**What software packages are available?** - -In general, we support most software that fit the distributed high throughput computing model. Users are encouraged to download and install their own software. For some software, we support distributed software modules listed [here](https://support.opensciencegrid.org/support/solutions/articles/12000048518). Software can be added to the modules upon request. - -Additionally, users may install their software into a Docker container which can run on OSG as a Singularity image. See [this guide](https://support.opensciencegrid.org/support/solutions/articles/12000024676-docker-and-singularity-containers) for more information. - -**How do I access a specific software application?** - -We have implemented modules within OSG Connect to manage the software that is available to users. Modules allow for easy access to a number of software and version options. Our [Accessing Software using Distributed Environment Modules](https://support.opensciencegrid.org/support/solutions/articles/12000048518) page provides more details on how to use modules in OSG Connect. - -**Are there any restrictions on installing commercial softwares?** - -We can only *directly* support software that is freely distributable. At present, we do not have or support most commercial software due to licensing issues. (One exception is running [MATLAB standalone executables](https://support.opensciencegrid.org/support/solutions/articles/5000660751-basics-of-compiled-matlab-applications-hello-world-example) which have been compiled with the MATLAB Compiler Runtime). Software that is licensed to individual users (and not to be shared between users) can be staged within the user's /home directory with HTCondor transferring to jobs, but should not be staged in OSG's public data staging locations (see https://support.opensciencegrid.org/support/solutions/articles/12000002985-data-management-and-policies). Please get in touch with any questions about licensed software. - - -**Can I request for system wide installation of the open source software useful for my research?** - -Yes. Please contact . - -## Running Jobs - -**What type of computation is a good match or NOT a good match for OSG Connect?** - -OSG Connect is a high throughput computing system. You can get the most of out OSG Connect resources by breaking up a -single large computational task into many smaller tasks for the fastest overall turnaround. This approach can be -invaluable in accelerating your computational work and thus your research. Please see our ["Is OSG for You?"](https://support.opensciencegrid.org/support/solutions/articles/5000632058) page for more details on how to determine if your work matches up well with OSG Connect's model. - -**What job scheduler is being used on OSG Connect?** - -We use use the task scheduling software called HTCondor to schedule and run jobs. - -**How do I submit a computing job?** - -Jobs are submitted via HTCondor scheduler. Please see our [QuickStart](http://support.opensciencegrid.org/support/solutions/articles/5000633410) guide for more details on submitting and managing jobs. - -**How many jobs can I have in the queue?** - -The number of jobs that are submitted to the queue by any one user should not exceed 10,000. If you have more jobs than that, we ask that you include the following statement in your submit file: - -`max_idle = 2000` - -This is the maximum number of jobs that you will have in the "Idle" or "Held" state for the submitted batch of jobs at any given time. Using a value of 2000 will ensure that your jobs continue to apply a constant pressure on the queue, but will not fill up the queue unnecessarily (which helps the scheduler to perform optimally). - -## Data Storage and Transfer - -**What is the best way to process large volume of data?** - -Use the Stash data system to stage large volumes of data. Please refer the section [Data Solutions](http://support.opensciencegrid.org/support/solutions/folders/5000262152) for more details. - -**How do I transfer my data to and from OSG Connect?** - -You can transfer data using scp or rsync. See [Using scp To Transfer Files To OSG Connect](https://support.opensciencegrid.org/support/solutions/articles/5000634376) for more details. - -**How public is /public?** - -The data under your `/public` location is discoverable and readable by anyone in the world. Data in `/public` is made public over http/https (via `https://stash.osgconnect.net/public/`) and mirrored to `/cvmfs/stash.osgstorage.org/osgconnect/public/` (for use with `stashcp`) which is mounted on a large number of systems around the world. - -**Is there any support for private data?** - ->**OSG currently has no storage appropriate for HIPAA data.** - -If you do not want your data to be downloadable by anyone, and it’s small enough for HTCondor file transfer (i.e. <100MB per file and <500MB total per job), then it should be staged in your `/home` directory and transferred to jobs with HTCondor file transfer (`transfer_input_files`, in the submit file). If your data must remain private and is too large for HTCondor file transfer, then it’s not a good fit for the “open” environment of the Open Science Pool, and another resource will likely be more appropriate. As a reminder, if the data is not being used for active computing work on OSG Connect, it should not be stored on OSG Connect systems. Lastly, our data storage locations are not backed up nor are they intended for long-term storage. - -**Can I get a quota increase?** - -[Contact us](mailto:support@opensciencegrid.org) if you think you'll need a quota increase for `/home` or `/public` to accommodate a set of concurrently-running jobs. We can suppport very large amounts of data, the default quotas are just a starting point. - -**Will I get notified about hitting quota limits?** - -The only place you can currently see your quota status is in the login messages. - -## Workflow Management - -**How do I run and manage complex workflows?** - -For workflows that have multiple steps and/or multiple files to, we advise using a workflow management system. A workflow management system allows you to define different computational steps in your workflow and indicate how inputs and outputs should be transferred between these steps. Once you define a workflow, the workflow management system will then run your workflow, automatically retrying failed jobs and transferrring files between different steps. - -**What workflow management systems are recommended on OSG?** - -We support and distribute DAGMan, Pegasus, and Swift for workflow management. - -## Workshops and Training - -**Do you plan to offer training sessions and workshop?** - -We plan to offer workshops for the researchers on multiple locations, including an annual, week-long summer school for OSG users. Please check our [events page](https://support.opensciencegrid.org/support/solutions/5000161177) for further information about workshop dates and locations. - -**Who may attend OSG workshops?** - -Workshops are typically open to students, post docs, staff and faculty. - -**What are the topics covered in a typical workshop?** - -We typically cover shell scripting, python (or R) programming, version control with git and distributed high throughout computing. - -**How to cite or acknowledge OSG?** - -Whenever you make use of OSG resources, services or tools, we would be grateful to have you acknowledge OSG in your presentations and publications. - -For example, you can add the following in your acknowledgements section: - -> "This research was done using resources provided by the OSG, which is supported by the National Science Foundation and the U.S. Department of Energy's Office of Science." - -We recommend the following references for citations - -> 1) Pordes, R. et al. (2007). "The Open Science Grid", J. Phys. Conf. Ser. 78, 012057.doi:10.1088/1742-6596/78/1/012057. - -> 2) Sfiligoi, I., Bradley, D. C., Holzman, B., Mhashilkar, P., Padhi, S. and Wurthwein, F. (2009). "The Pilot Way to Grid Resources Using glideinWMS", 2009 WRI World Congress on Computer Science and Information Engineering, Vol. 2, pp. 428–432. doi:10.1109/CSIE.2009.950. - - - diff --git a/training/training/Joint-SWC-OSG.md b/training/training/Joint-SWC-OSG.md deleted file mode 100644 index e49072d8..00000000 --- a/training/training/Joint-SWC-OSG.md +++ /dev/null @@ -1,17 +0,0 @@ - -[title]: - "Joint Software Carpentry and OSG Workshops" - - - -## Overview - -We offer training and tutorials for the scientists and researchers new to high throughput computing. As part of our training service we are offering an extended Software Carpentry workshop which augments instruction on basic Linux tools and programming with tutorials on using distributed high throughput computing workflows. These workshops are run by the OSG staff and instructors from Software Carpentry. - -## Joint OSG & Software Carpentry Workshops - - * [Joint OSG - Software Carpentry Workshop at the Thomas Jefferson National Accelerator Facility (JLAB), May 17-19, 2017](https://swc-osg-workshop.github.io/2017-05-17-JLAB/) - * [Joint OSG - Software Carpentry Workshop at the University of Nebraska, Lincoln, January 6-8, 2016](http://swc-osg-workshop.github.io/2016-01-06-UNL/) - * [Joint OSG - Software Carpentry Workshop at Duke University, October 27-29th 2015](http://swc-osg-workshop.github.io/2015-10-27-duke/index.html) - * [Joint OSG - Software Carpentry Workshop at IUPUI, Indianapolis. March 3-6th 2015](http://swc-osg-workshop.github.io/2015-03-03-iupui/index.html) - * [Joint OSG - Software Carpentry Workshop at The University of Chicago, December 15-17th 2014](http://swc-osg-workshop.github.io/2014-12-15-UChicago/) - diff --git a/training/training/osg-user-school.md b/training/training/osg-user-school.md deleted file mode 100644 index b0dad9bc..00000000 --- a/training/training/osg-user-school.md +++ /dev/null @@ -1,26 +0,0 @@ - -[title]: - "Annual, Week-Long OSG User School" -[TOC] - -## Overview - -During this week-long training event held at the University of Wisconsin-Madison every summer, students learn to use high-throughput computing (HTC) systems — at their own campus or using the OSG — to run large-scale computing applications that are at the heart of today’s cutting-edge science. Through lectures, discussions, and lots of hands-on activities with experienced OSG staff, students will learn how HTC systems work, how to run and manage lots of jobs and huge datasets, to implement a scientific computing workflow, and where to turn for more information and help. - -The School is ideal for graduate students in any science or research domain where large-scale computing is a vital part of the research process, plus we will consider applications from advanced undergraduates, post-doctoral students, faculty, and staff. Students accepted to this program will receive financial support for basic travel and local costs associated with the School. - -### Open Materials and Recordings - -The OSG User School want virtual in 2020 and 2021, which means that we were able to record lectures to complement lecture and exercise materials! - -* [OSG Virtual School Pilot, August 2021](https://osg-htc.org/virtual-school-2021/materials/) -* [OSG Virtual School Pilot, July 2020](https://osg-htc.org/virtual-school-pilot-2020/#materials/) - -### Past OSG User Schools - -* [OSG User School, July 15-19, 2019](https://opensciencegrid.org/user-school-2019/) -* [OSG User School, July 9-13, 2018](https://opensciencegrid.org/user-school-2018/) -* [OSG User School, July 17-21, 2017](https://opensciencegrid.org/user-school-2017/) -* [OSG User School, July 25-29, 2016](https://opensciencegrid.org/user-school-2016/) -* [OSG User School, July 27-31, 2015](https://opensciencegrid.org/user-school-2015/) -* [OSG User School, July 7-10, 2014](https://opensciencegrid.org/user-school-2014/) - diff --git a/training/training/osgusertraining.md b/training/training/osgusertraining.md deleted file mode 100644 index 41fcb469..00000000 --- a/training/training/osgusertraining.md +++ /dev/null @@ -1,36 +0,0 @@ -[title]: - "OSG User Training (regular/monthly)" - -## Sign Up for Upcoming Trainings! - -All User Training sessions are offered from 2:30-4pm ET (and usually on Tuesdays). New User Training is offered monthly, generally on the first Tuesday of the month, and training on various additional topics happens on the third Tuesday of the month. It's best to already have a user account on OSG Connect (or another access point that submits to the Open Science Pool) to follow along with hands-on examples, but anyone can listen in by registering. - -* Tuesday, May 3: New User Training, [Register here](https://docs.google.com/forms/d/e/1FAIpQLSdj3XT7I0SM4k9jBvST7YX5wsCH_er1HLA7VqRj9ICoEvf2GA/viewform) -* Tuesday, May 17: Using Containerized Software on the Open Science Pool, [Register here](https://docs.google.com/forms/d/e/1FAIpQLSdj3XT7I0SM4k9jBvST7YX5wsCH_er1HLA7VqRj9ICoEvf2GA/viewform) -* Tuesday, June 7: New User Training, [Register here](https://docs.google.com/forms/d/e/1FAIpQLSdj3XT7I0SM4k9jBvST7YX5wsCH_er1HLA7VqRj9ICoEvf2GA/viewform) -* Tuesday, June 21: Software Portability on the Open Science Pool, [Register here](https://docs.google.com/forms/d/e/1FAIpQLSdj3XT7I0SM4k9jBvST7YX5wsCH_er1HLA7VqRj9ICoEvf2GA/viewform) - -## Training Materials by Topic - -All of our training materials are public, and with recent video recordings available: - -### New User Training (monthly) - -The most recent version of our New User Training materials are here: - -* [Slides](https://docs.google.com/presentation/d/1z-f81xtk_ZXeJcA1kX60JoScXdGfe-xgsB9g5YemrqI/edit#slide=id.g10662d3fe4f_0_0), [Video](https://www.youtube.com/watch?v=D14eMrkZ2gQ) -* [Wordcount Frequency Tutorial](https://support.opensciencegrid.org/support/solutions/articles/12000079856) - -### Organizing and Submitting HTC Workloads - -The most recent version of these training materials are here: - -* [Slides](https://docs.google.com/presentation/d/1xYVp8NgiFSUdda2yD19HTLaXgH3HAPbVc1NASaG7Q74) -* [Wordcount Frequency Tutorial](https://github.com/OSGConnect/tutorial-organizing) - -### Using Containerized Software on the Open Science Pool - -In development (see above, and register!). - -### Additional Topics - -As we introduce new training topics, we will add materials to this page. diff --git a/training/training/previous-training-events.md b/training/training/previous-training-events.md deleted file mode 100644 index 1e67ea6f..00000000 --- a/training/training/previous-training-events.md +++ /dev/null @@ -1,26 +0,0 @@ -[title]: - "Other Past Training Events" -[TOC] - -## Overview - -We offer on-site training and tutorials on a periodic basis, usually at conferences (including the annual OSG All Hands Meeting) where many researchers and/or research computing staff are gathered. Below are some trainings for which the materials were public. (Apologies if any links/materials aren't accessible anymore, as some of these are external to our own web location. Feel free to let us know via support@opensciencegrid.org, in case we can fix/remove them.) - -## Workshops/Tutorials - - * [Empowering Research Computing at Your Organization Through the OSG (PEARC 21)](https://docs.google.com/document/d/1q8ylhZ88NkDhQjgbCvCB0Atc9R1w7YRhCZOmD9F2IVc) - * [Organizing and Submitting HTC Workloads (OSG User Training pilot, June 2021)](https://docs.google.com/document/d/1h7GXxs6mobNmzDl4_oBh2_Jnie5wYCQ1uj5ParvQ65c) - * [Empower Research Computing at your Organization Through the OSG (RMACC 2021)](https://docs.google.com/document/d/19i79nEOQJDBSfvxuSRF2HXhrFxTbUHJe6AxUyWNKfTw) - * [dHTC Campus Workshop (February 2021)](https://indico.fnal.gov/event/46925/) - * [Empowering Research Computing at Your Campus Through the OSG (PEARC 20)](https://opensciencegrid.org/Tutorial-PEARC-2020/) - * [Deploy jobs on the Open Science Grid (Gateways/eScience 2019)](https://swc-osg-workshop.github.io/OSG-UserTraining-Gateways-2019/) - * [High Throughput Computation on the Open Science Grid (Internet2 2018 Technology Exchange)](https://meetings.internet2.edu/2018-technology-exchange/program-guide/tutorials-workshops/#OSG) - * [Open Science Grid Workshop (The Quilt 2018)](https://www.thequilt.net/public-event/osg-pre-workshop-session/) - * [High Throughput Computation on the Open Science Grid (RMACC 18)](https://rmacc2018hpcsymposium.sched.com/event/EbOT/high-throughput-computation-on-the-open-science-grid) - -## Tutorials at Recent OSG All-Hands Meetings - -The below were offered on-site at OSG All-Hands Meetings. Note that the last on-site AHM in 2020 was canceled due to the pandemic, though we've linked to the materials. - - * [User/Facilitator Training at the OSG All Hands Meeting, University of Oklahoma (OU), March 2020](https://opensciencegrid.org/UserTraining-AHM-2020/) - * [User Training at the OSG All Hands Meeting, Thomas Jefferson National Accelerator Facility (JLAB), March 2019](https://swc-osg-workshop.github.io/OSG-UserTraining-JLab-2019/) - * [User Training at the OSG All Hands Meeting, University of Utah, March 2018](https://swc-osg-workshop.github.io/OSG-UserTraining-AHM18/) diff --git a/tutorials/tutorial-AutoDockVina b/tutorials/tutorial-AutoDockVina deleted file mode 160000 index c4b92832..00000000 --- a/tutorials/tutorial-AutoDockVina +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c4b92832f5fd13543fced127fa6a5122aec087c2 diff --git a/tutorials/tutorial-R b/tutorials/tutorial-R deleted file mode 160000 index 25a641fc..00000000 --- a/tutorials/tutorial-R +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 25a641fc97bd64decd3879708664fc2f3a24bf29 diff --git a/tutorials/tutorial-R-addlibSNA b/tutorials/tutorial-R-addlibSNA deleted file mode 160000 index 643ab9ad..00000000 --- a/tutorials/tutorial-R-addlibSNA +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 643ab9ad1fb7c581536e351f879c619c63b73682 diff --git a/tutorials/tutorial-ScalingUp-Python b/tutorials/tutorial-ScalingUp-Python deleted file mode 160000 index d8689666..00000000 --- a/tutorials/tutorial-ScalingUp-Python +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d8689666249a9fdc309ec1d80b9e869d5119b14a diff --git a/tutorials/tutorial-ScalingUp-R b/tutorials/tutorial-ScalingUp-R deleted file mode 160000 index ba94e2e5..00000000 --- a/tutorials/tutorial-ScalingUp-R +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ba94e2e5aae61e62f5c7c6493ea95e18c39397d9 diff --git a/tutorials/tutorial-blast b/tutorials/tutorial-blast deleted file mode 160000 index 94d91415..00000000 --- a/tutorials/tutorial-blast +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 94d91415c7c9b7931943c8255bda56eaf05c9e3f diff --git a/tutorials/tutorial-blast-split b/tutorials/tutorial-blast-split deleted file mode 160000 index 4dac0f10..00000000 --- a/tutorials/tutorial-blast-split +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4dac0f1062f024a4aa5d43e7e7b1bb5f086091ff diff --git a/tutorials/tutorial-bwa b/tutorials/tutorial-bwa deleted file mode 160000 index f37097e7..00000000 --- a/tutorials/tutorial-bwa +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f37097e70311ebb0681ec4e51f7d1eff25e8fb99 diff --git a/tutorials/tutorial-error101 b/tutorials/tutorial-error101 deleted file mode 160000 index 57f40e9e..00000000 --- a/tutorials/tutorial-error101 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 57f40e9ea22c52d75f677b5c91ab684578ce66c8 diff --git a/tutorials/tutorial-exitcode b/tutorials/tutorial-exitcode deleted file mode 160000 index 271c8580..00000000 --- a/tutorials/tutorial-exitcode +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 271c858035521a4730fb715dc1f5f2bf07780db6 diff --git a/tutorials/tutorial-gromacs b/tutorials/tutorial-gromacs deleted file mode 160000 index c2b68252..00000000 --- a/tutorials/tutorial-gromacs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c2b68252080cc6d5fee14afcc5766efe664bf1dd diff --git a/tutorials/tutorial-htcondor-transfer b/tutorials/tutorial-htcondor-transfer deleted file mode 160000 index d9dd3a07..00000000 --- a/tutorials/tutorial-htcondor-transfer +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d9dd3a07b1c7467e41e7c4b937b88170f3f67179 diff --git a/tutorials/tutorial-makeflow-quickstart b/tutorials/tutorial-makeflow-quickstart deleted file mode 160000 index b4eaeb79..00000000 --- a/tutorials/tutorial-makeflow-quickstart +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b4eaeb795792eedf964a299460d21e120a0ee604 diff --git a/tutorials/tutorial-matlab-HelloWorld b/tutorials/tutorial-matlab-HelloWorld deleted file mode 160000 index 80b4bedb..00000000 --- a/tutorials/tutorial-matlab-HelloWorld +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 80b4bedb21c0c48f0272db9bb923aa81c8d8f9bd diff --git a/tutorials/tutorial-namd b/tutorials/tutorial-namd deleted file mode 160000 index 4c77fcc6..00000000 --- a/tutorials/tutorial-namd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4c77fcc6de890eb9b96559226d56e4e6483702f6 diff --git a/tutorials/tutorial-nelle-nemo b/tutorials/tutorial-nelle-nemo deleted file mode 160000 index c77798ef..00000000 --- a/tutorials/tutorial-nelle-nemo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c77798ef11de1dbd948d38a72954d807230d36c9 diff --git a/tutorials/tutorial-octave b/tutorials/tutorial-octave deleted file mode 160000 index 24b36d6d..00000000 --- a/tutorials/tutorial-octave +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 24b36d6d652ae09504b62352e1c0cba4ae9133ce diff --git a/tutorials/tutorial-organizing b/tutorials/tutorial-organizing deleted file mode 160000 index cda44823..00000000 --- a/tutorials/tutorial-organizing +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cda44823b9221e1627e7ffafeaa3e98147fc4438 diff --git a/tutorials/tutorial-osg-locations b/tutorials/tutorial-osg-locations deleted file mode 160000 index c0109ecc..00000000 --- a/tutorials/tutorial-osg-locations +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c0109ecc3e662596dfcd0e477394dfab00221377 diff --git a/tutorials/tutorial-pegasus b/tutorials/tutorial-pegasus deleted file mode 160000 index d4f5c4bd..00000000 --- a/tutorials/tutorial-pegasus +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d4f5c4bd8dc3a217ea551a0cba125001bddc091a diff --git a/tutorials/tutorial-photodemo b/tutorials/tutorial-photodemo deleted file mode 160000 index 06432333..00000000 --- a/tutorials/tutorial-photodemo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 064323332ae8f6af04669bb2ab2a48f54028acd2 diff --git a/tutorials/tutorial-python-virtualenv b/tutorials/tutorial-python-virtualenv deleted file mode 160000 index a0b8a7b6..00000000 --- a/tutorials/tutorial-python-virtualenv +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a0b8a7b6f844d002cf1d9a23e695f18217d86a30 diff --git a/tutorials/tutorial-quickstart b/tutorials/tutorial-quickstart deleted file mode 160000 index 91c0c794..00000000 --- a/tutorials/tutorial-quickstart +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 91c0c794469a5df665ce10484c5ded369e3deb65 diff --git a/tutorials/tutorial-root b/tutorials/tutorial-root deleted file mode 160000 index 5608427e..00000000 --- a/tutorials/tutorial-root +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5608427efc5d3eb00c293b69aa0e346c0c3b51c7 diff --git a/tutorials/tutorial-scaling b/tutorials/tutorial-scaling deleted file mode 160000 index 494dbfcb..00000000 --- a/tutorials/tutorial-scaling +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 494dbfcb115ff79082f841bb14a350e334acdb5c diff --git a/tutorials/tutorial-scaling-up-resources b/tutorials/tutorial-scaling-up-resources deleted file mode 160000 index da3fe5e3..00000000 --- a/tutorials/tutorial-scaling-up-resources +++ /dev/null @@ -1 +0,0 @@ -Subproject commit da3fe5e3887d605ab444844e84a0210f9d1f9fac diff --git a/tutorials/tutorial-software b/tutorials/tutorial-software deleted file mode 160000 index 1ac04076..00000000 --- a/tutorials/tutorial-software +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1ac040769de4364f04b0bc61222e7942f215181b diff --git a/tutorials/tutorial-stash-http b/tutorials/tutorial-stash-http deleted file mode 160000 index ae38b5cb..00000000 --- a/tutorials/tutorial-stash-http +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ae38b5cbc01fe8e6d336b85d6625083b7682f3cf diff --git a/tutorials/tutorial-stashcache-blast b/tutorials/tutorial-stashcache-blast deleted file mode 160000 index 9fd3807d..00000000 --- a/tutorials/tutorial-stashcache-blast +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9fd3807d170222acfce7414377f5aa233992ca92 diff --git a/tutorials/tutorial-swift b/tutorials/tutorial-swift deleted file mode 160000 index f79861e4..00000000 --- a/tutorials/tutorial-swift +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f79861e46b16025f64511c5cfa5f04323ed3c7c0 diff --git a/tutorials/tutorial-tensorflow-containers b/tutorials/tutorial-tensorflow-containers deleted file mode 160000 index 83046b9b..00000000 --- a/tutorials/tutorial-tensorflow-containers +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 83046b9b6cc93bcd5cd2cb852b40a9d48620c03c diff --git a/tutorials/tutorial-wordfreq b/tutorials/tutorial-wordfreq deleted file mode 160000 index e37dcdd4..00000000 --- a/tutorials/tutorial-wordfreq +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e37dcdd4e4fe3d4026364f292d5af947dfc25697 diff --git a/update/README.md b/update/README.md deleted file mode 100644 index d9a40024..00000000 --- a/update/README.md +++ /dev/null @@ -1,2 +0,0 @@ -* update.sh -- called by github.wsgi webhook and pulls git updates and calls freshpush to update documents. -* freshpush -- pushes file to freshdesk api, needs a valid api key (per person). diff --git a/update/container-list/container-list.template b/update/container-list/container-list.template deleted file mode 100644 index 11734f43..00000000 --- a/update/container-list/container-list.template +++ /dev/null @@ -1,18 +0,0 @@ -[title]: - "Available Containers List" - -[TOC] - -This is list of commonly used containers in the Open Science Pool. These can be used -directly in your jobs or as base images if you want to define your own. Please -see the pages on [container overview][container-intro] and [defining containers][container-howto] -for detailed instructions. - -Also note that this list is not complete. There are many images under -`/cvmfs/singularity.opensciencegrid.org/` which are either project specific -or not described well enough to make this list. - -$table - -[container-intro]: 12000024676 -[container-howto]: 12000058245 - diff --git a/update/container-list/update-container-list b/update/container-list/update-container-list deleted file mode 100755 index 64287d3b..00000000 --- a/update/container-list/update-container-list +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/python3 - -''' -walk /cvmfs/singularity.opensciencegrid.org/ and find container metadata -''' - -import os -import json -import re -from string import Template -from pprint import pprint - -metadata = {} -md_sorted = [] - - -def sort_key(entry): - key = '' - if 'category' in entry: - key += entry['category'] - if 'name' in entry: - key += entry['name'] - return key - - -def import_metadata(top_dir, identifier, tag): - - global metadata - - json_file = os.path.join(top_dir, identifier + ':' + tag, '.singularity.d/labels.json') - if os.path.getsize(json_file) < 10: - return - - print('Processing {}...'.format(json_file)) - - # first ensure it has the required metadata - with open(json_file) as f: - fdata = json.load(f) - - if 'opensciencegrid.name' in fdata: - if identifier not in metadata: - data = {} - data['identifier'] = identifier - for k, v in fdata.items(): - newk = re.sub('opensciencegrid\.', '', k) - data[newk] = v - data['cvmfs_path'] = os.path.join(top_dir, identifier) - data['tags'] = [] - - # in the base area, only allow images from /opensciencegrid/ - # this will filter out derived images which did not update - # the labels - if data['category'] == 'Base' and \ - not re.search('singularity.opensciencegrid.org/opensciencegrid', json_file): - return - - metadata[identifier] = data - metadata[identifier]['tags'].append(tag) - - -def main(): - - top_dir = '/cvmfs/singularity.opensciencegrid.org' - - # we have two levels, first one is org - orgs = os.listdir(top_dir) - for org in orgs: - try: - images = os.listdir(os.path.join(top_dir, org)) - except: - continue - - for image in images: - if not os.path.isfile(os.path.join(top_dir, org, image, '.singularity.d/labels.json')): - continue - identifier, tag = image.split(':') - identifier = org + '/' + identifier - import_metadata(top_dir, identifier, tag) - - # convert the dict to a list and sort it - for k, v in metadata.items(): - md_sorted.append(v) - md_sorted.sort(key=sort_key) - - pprint(md_sorted) - - # write the tables - table = '' - curr_category = None - for entry in md_sorted: - if entry['category'] != curr_category: - curr_category = entry['category'] - table += '\n## {}\n\n'.format(entry['category']) - table += '| **Name** | **CVMFS Locations** | **Description** |\n' - table += '|:---------|:--------------------|:----------------|\n' - locations = '' - name = '{}'.format(entry['name']) - for tag in sorted(entry['tags']): - if locations != '': - locations += '
' - locations += '{}:{}'.format(entry['cvmfs_path'], tag) - desc = entry['description'] - if 'url' in entry: - desc += '
[Project Website]({})'.format(entry['url']) - if 'definition_url' in entry: - desc += '
[Container Definition]({})'.format(entry['definition_url']) - table += '| {} | {} | {} |\n'.format(name, locations, desc) - - - # the page is a simple template - with open('container-list.template') as f: - template = Template(f.read()) - - with open('../../start/software/available-containers-list.md', 'w') as f: - f.write(template.substitute(table=table)) - - -if __name__ == '__main__': - main() - diff --git a/update/freshpush b/update/freshpush deleted file mode 100755 index ed0a40bf..00000000 --- a/update/freshpush +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env python2.7 - -# Python 2.7 is required because we want Markdown tables, which appear to -# be available only in releases of python-markdown that don't run under -# Python 2.7. - -import json -import os -import sys -import markdown -import requests -import re -import ConfigParser -import time - -def tohtml(md, url=None, fn=None, alt=None): - extensions=['markdown.extensions.tables', - 'markdown.extensions.toc'] - html = markdown.markdown(md.decode('utf-8'), extensions=extensions) - - if html == '': - if alt: - html = '

' + alt + '

' - else: - html = '

' - else: - date = time.strftime('on %b %d, %Y at %H:%M') - html += '\n

 

' - html += '

This page was updated %s from %s.

' % \ - (date, url, fn) - - return html - - -class main(object): - - def send(self, title, html, uri=None, endpoint=None, category=None, folder=None, article=None, fn=''): - if not uri: - if not endpoint: - raise ValueError, 'uri or endpoint is required' - - if category and folder and article: - # update existing - path = ['solutions', 'categories', category, - 'folders', folder, 'articles', article] - - elif category and folder: - # update existing - path = ['solutions', 'categories', category, - 'folders', folder, 'articles'] - - else: - raise ValueError, 'at least category and folder are required' - - uri = endpoint + '/'.join([str(x) for x in path]) - - headers = { - 'Content-Type': 'application/json', - } - - data = { - 'status': 2, - 'title': title, - 'description': html - } - - sys.stdout.write('Sending %s (%s) to %s (%s)...' % (title, fn, article, uri)) - r = requests.put(uri, - data=json.dumps(data), - auth=requests.auth.HTTPBasicAuth(self.apikey, 'X'), - headers=headers) - sys.stdout.write('%d\n' % r.status_code) - if r.status_code != 200: - sys.stdout.write(str(r.headers)) - - - def readmap(file): - mappings = [] - fp = open(file, 'r') - for line in fp: - match, mapto = line.strip().split() - mappings.append([re.compile(match, re.I), mapto]) - fp.close() - return mappings - - def mapfilename(file): - mapfile = os.path.join(os.path.dirname(sys.argv[0]), 'articles.map') - mappings = readmap(mapfile) - for r, mapto in mappings: - m = r.search(file) - if m: - return m.expand(mapto) - return None - - def __call__(self, args): - cfg = ConfigParser.RawConfigParser() - cfg.read(sys.argv[0] + '.ini') - if cfg.has_section('include'): - for option, value in cfg.items('include'): - cfg.read(value) or cfg.read(option) - - self.apikey = cfg.get('api', 'apikey') - - mapping = [] - for option, value in cfg.items('filemap'): - #mapping.append([re.compile(option, re.I), value]) - mapping.append([option, value]) - - def mapfile(f): - for r, value in mapping: - #if r.search(f): - # return value - if f.lower().endswith(r.lower()): - return [x.strip() for x in value.split(',')] - return [] - - def gettitle(fn, lines, article=None): - # Four ways to derive a title - if article: - # find lines like this: - # [title]: articlenumber "title of document here" - for line in lines: - if line.startswith('[title]: %s' % str(article)): - off = line.find('"') - return line[off:].strip().strip('"') - - # find lines like this: - # [title]: - "title of document here" - for line in lines: - if line.startswith('[title]:'): - off = line.find('"') - if off < 0: - off = 8 # just remove the [title]: part - return line[off:].strip().strip('"') - - headlines = filter(lambda line: line.startswith('#'), lines) - if headlines: - return headlines[0].strip('#').strip() - - # fallback - return os.path.basename(fn).replace('.md', '').replace('-', ' ') + ' *' - - for file in args: - fn = os.path.normpath(file) - fn = fn.replace(os.getcwd(), '') - fn = fn.lstrip('/') - title = None - - paths = mapfile(fn) - if not paths: - print 'no mapping for', fn - continue - - # read content and parse out title - fp = open(file, 'r') - data = fp.read() - fp.close() - - # derive the source info and github link - origin = os.path.join(os.path.basename(os.getcwd()), fn) - origin = 'sourced from ' + origin - url = os.environ['upstream'] + '/blob/master/' + os.environ['relpath'] - alt = 'This document has not been created. Please edit ' + \ - 'the source to create it.' - - # then generate HTML - lines = data.split('\n') - html = tohtml(data, fn=fn, url=url, alt=alt) - - # Post the html once for each path - for path in paths: - try: - category, folder, article = path.split('/') - except ValueError: - category = '0' - folder = '0' - article = path - - if cfg.has_option('categories', category): - category = cfg.get('categories', category) - if cfg.has_option('folders', folder): - folder = cfg.get('folders', folder) - if cfg.has_option('articles', article): - article = cfg.get('articles', article) - - uri = cfg.get('api', 'uri') % locals() - if uri is None: - print 'no uri mapped for %s' % fn - continue - - title = gettitle(fn, lines, article=article) - - self.send(title, html, uri, fn=fn, article=article) - - -if __name__ == '__main__': - sys.exit(main()(sys.argv[1:])) diff --git a/update/freshpush.ini b/update/freshpush.ini deleted file mode 100644 index 8bd6d1c7..00000000 --- a/update/freshpush.ini +++ /dev/null @@ -1,171 +0,0 @@ -[include] -# Any file listed here is read into the configuration after this -# config file is processed. We include the apikey.ini file here -# so that it's not part of the git repository -- it contains -# secret data. -apikey_local = update/apikey.ini -apikey_global = /usr/local/connectbook/update/apikey.ini - -[api] -uri = https://opensciencegrid.freshdesk.com/api/v2/solutions/articles/%(article)s - - -[filemap] - -# Overview - -## Overview/Welcome -welcome/overview/getting-help-from-RCFs.md = 12000084585 -welcome/overview/acknowledgOSG.md = 5000640421 -welcome/overview/is-it-for-you.md = 5000632058 -welcome/overview/gracc.md = 12000028939 -welcome/overview/policy.md = 12000074852 -training/resources/frequently-asked-questions-faq-.md = 5000634384 - -# Getting Started on OSG Connect - -## Roadmap draft -start/roadmap.md = 12000081596 -start/jobdurationcategory.md = 12000083468 - -## Set Up Your Account -start/account/registration-and-login.md = 5000632072 -start/account/generate-add-sshkey.md = 12000027675 -start/account/starting-project.md = 5000634360 - -## Learn to Submit HTCondor Jobs -tutorials/tutorial-quickstart/README.md = 5000633410 -start/scaling-up/submit-multiple-jobs.md = 12000073165 -start/jobs/tutorial-command.md = 5000634361 -tutorials/tutorial-organizing/README.md = 12000086547 -tutorials/tutorial-error101/README.md = 5000639785 -tutorials/tutorial-osg-locations/README.md = 12000061978 - -## Using Software on OSG Connect -start/software/new_modules_list.md = 12000048518 -start/software/compiling-applications.md = 5000652099 -start/software/containers.md = 12000024676 -start/software/containers-docker.md = 12000058245 -start/software/containers-singularity.md = 12000086275 -start/software/available-containers-list.md = 12000073449 -start/software/software-overview.md = 5000634395 -start/software/software-request.md = 5000649173 -start/software/example-compilation.md = 12000074984 - -## Data management -start/data/osgconnect-storage.md = 12000002985 -start/data/scp.md = 5000634376 -start/data/file-transfer-via-http.md = 5000639798 -start/data/file-transfer-via-htcondor.md = 5000639787 -start/data/stashcache.md = 12000002775 -start/data/output-file-transfer-via-htcondor.md = 12000072729 - -## Choosing Resources for Jobs -start/resources/requirements.md = 5000633467 -start/resources/large-memory-jobs.md = 5000652304 -start/resources/gpu-jobs.md = 5000653025 -start/resources/multicore-jobs.md = 5000653862 -start/resources/openmpi-jobs.md = 12000054297 - -## Scaling Up and Automated Workflow Tools -start/scaling-up/preparing-to-scale-up.md = 12000076552 -tutorials/tutorial-pegasus/README.md = 5000639789 -start/scaling-up/dagman-workflows.md = 12000079038 -tutorials/tutorial-wordfreq/README.md = 12000079856 -start/scaling-up/checkpointing-on-OSPool.md = 12000086989 - -# Software Examples for OSG - -## Bioinformatics -tutorials/tutorial-blast-split/README.md = 12000062020 -tutorials/tutorial-bwa/README.md = 12000085928 - -## R -#examples/intro-to-r.md = 12000056217 -tutorials/tutorial-R/README.md = 5000674219 -tutorials/tutorial-R-addlibSNA/README.md = 5000674218 -tutorials/tutorial-ScalingUp-R/README.md = 5000674221 - -## Using FreeSurfer on OSG -examples/FreeSurfer/Introduction.md = 12000008483 - -## Matlab -tutorials/tutorial-matlab-HelloWorld/README.md = 5000660751 - -## Python -examples/manage-python-packages.md = 12000058785 -tutorials/tutorial-ScalingUp-Python/README.md = 12000062019 - -## Machine Learning -tutorials/tutorial-tensorflow-containers/README.md = 12000028940 - -## Drug Discovery -tutorials/tutorial-AutoDockVina/README.md = 5000634379 - -## Other Software -examples/julia-on-osg.md = 12000078187 -examples/java-on-osg.md = 12000084557 -examples/conda-on-osg.md = 12000086988 - -# Additional Support and Training - -## Education and Training -training/training/previous-training-events.md = 12000037330 -training/training/osg-user-school.md = 12000002319 -training/training/osgusertraining.md = 12000084444 -# training/training/osgnewusertraining.md = 12000084445 - -## Additional Resources -training/resources/contact-information.md = 5000634383 - -# HPC Administration - -## OSG for HPC Administrators -hpcadmin/osg-flock.md = 12000056360 -hpcadmin/osg-xsede.md = 12000056361 - -# Administration - -## OSG Connect -admin/connect/user-login-project-manual.md = 5000641628 -admin/connect/create-project.md = 12000047899 -admin/connect/approving-user-application.md = 12000047900 - -## Content Management -admin/content/support-freshpush.md = 5000641634 -admin/content/support-add-tutorial.md = 5000641633 -admin/content/support-add-non-tutorial.md = 5000641632 -admin/content/support-editing.md = 5000641843 -admin/content/support-editing-tutorial.md = 12000052289 -admin/content/support-remove-docs.md = 12000068777 - -# Drafts / Unpublished pages -ways-to-connect.md = 5000639193 -# start/account/2020-transition-guide.md = 12000065909 - -# Tutorials -## other - -# TO BE REVIEWED LATER (currently drafts) -#tutorials/tutorial-software/README.md = 5000639796 -#tutorial-dagman-wordfreq - -#tutorials/tutorial-blast/README.md = 5000634380 -#tutorials/tutorial-makeflow-quickstart/README.md = 12000007096 -#tutorials/tutorial-exitcode/README.md = 5000639786 -#tutorials/tutorial-gromacs/README.md = 5000645191 -#tutorials/tutorial-namd/README.md = 5000634378 -#tutorials/tutorial-python-virtualenv/README.md = 12000021525 -#tutorials/tutorial-root/README.md = 5000639793 -#tutorials/tutorial-stashcache-blast/README.md = 12000003724 -#tutorials/tutorial-swift/README.md = 5000639800 -#tutorials/tutorial-dagman-namd/README.md = 5000634386, 5000639932 -#tutorials/tutorial-octave/README.md = 5000634365 -#tutorials/tutorial-scaling-up-resources/README.md = 5000639795 -#tutorials/tutorial-scaling/README.md = 5000639794 - -## unlinked tutorials -# tutorial-nelle-nemo -# tutorial-stash-cvmfs -# tutorial-photodemo -# tutorial-annex diff --git a/update/mdcleanup b/update/mdcleanup deleted file mode 100755 index eed6b94a..00000000 --- a/update/mdcleanup +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh -# -# Transforms github-style block quotes into plain Markdown block quotes. -# - -dequote () { - awk ' - BEGIN { block = 0; } - block && /^```/ { block = 0; next; } - !block && /^```/ { block = 1; next; } - block { print "\t" $0; next; } - { print; } - ' "$1" >tmp && - mv -f tmp "$1" -} - -dequote2 () { - awk ' - BEGIN { block = 0; } - block && /^~~~/ { block = 0; next; } - !block && /^~~~/ { block = 1; next; } - block { print "\t" $0; next; } - { print; } - ' "$1" >tmp && - mv -f tmp "$1" -} - -denbsp () { - tr '\240' '\040' <"$1" >tmp && - mv -f tmp "$1" -} - -if [ $# -eq 0 ]; then - while read file; do - dequote "$file" - dequote2 "$file" - denbsp "$file" - done -else - for file in "$@"; do - dequote "$file" - dequote2 "$file" - denbsp "$file" - done -fi diff --git a/update/update.sh b/update/update.sh deleted file mode 100755 index 0f95e267..00000000 --- a/update/update.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh - -[ -n "$LOGFD" ] && { - exec >&${LOGFD} - exec 2>&${LOGFD} -} - -echo child pid: $$ -echo dir: $(pwd) - -if [ $# -eq 0 ]; then - echo 'files: (discovered)' - generator () { - find . -name \*.md -print - } -else - echo files: "$@" - generator () { - for file in "$@"; do - echo "$file" - done - } -fi - -git checkout master -git pull origin master -git submodule foreach git pull origin master - -generator "$@" | while read file; do - dir=$(dirname "$file") - upstream=$(cd "$dir"; git config --get remote.origin.url | sed -e 's/.git$//') - relpath=$(cd "$dir"; git ls-files --full-name $(basename "$file")) - env "upstream=$upstream" "relpath=$relpath" $(dirname $0)/freshpush $(pwd)/$file -done - diff --git a/user-documentation b/user-documentation new file mode 160000 index 00000000..be39978c --- /dev/null +++ b/user-documentation @@ -0,0 +1 @@ +Subproject commit be39978cb46bd4a2f6c5b4378facb7ec23055d95 diff --git a/welcome/overview/acknowledgOSG.md b/welcome/overview/acknowledgOSG.md deleted file mode 100644 index a62dfd94..00000000 --- a/welcome/overview/acknowledgOSG.md +++ /dev/null @@ -1,3 +0,0 @@ -[title]:- "Acknowledge the OSG" - -This page has been moved to the [OSG Website](https://osg-htc.org/acknowledging). diff --git a/welcome/overview/getting-help-from-RCFs.md b/welcome/overview/getting-help-from-RCFs.md deleted file mode 100644 index ce3af18a..00000000 --- a/welcome/overview/getting-help-from-RCFs.md +++ /dev/null @@ -1,51 +0,0 @@ -[title]:- "Email, Office Hours, and 1-1 Meetings" - -[TOC] - -*There are multiple ways to get help from OSG’s Research Computing Facilitators. Get in touch anytime!* - - -## Research Computing Facilitators -To help researchers effectively utilize computing resources, our Research Computing Facilitators (RCFs) not only assist you in implementing your computational work on OSG compute resources, but can also point you to other services related to research computing and data needs. For example, RCFs can: - -- Assist with planning your computational approach for a research problem -- Teach you to submit jobs via the OSG Connect service -- Help you with troubleshooting on OSG systems -- Connect you with other researchers using similar software or methods -- Point to learning materials for programming and software development -- Help you identify applicable non-OSG data storage options -- Find someone who knows the answer to your question, even if the RCF doesn’t -- … and other helpful activities to facilitate your use of cyberinfrastructure - -We don’t expect that you should be able to address all of your questions by consulting our [documentation](https://support.opensciencegrid.org/support/home), [user training](https://support.opensciencegrid.org/support/solutions/5000161177), or web searches. RCFs are here to help! - - -## Get an Account -If you don’t have an account yet, please [Sign Up](https://www.osgconnect.net/), and we’ll follow up quickly to set up a meeting time and create accounts. If you don’t have an account but just have general questions, feel free to send an email to support@opensciencegrid.org (see below). - - -## Help via Email -We provide ongoing support via email to support@opensciencegrid.org, and it’s never a bad idea to start by sending questions or issues via email. You can typically expect a first response within a few business hours. - - -## Virtual Office Hours -Drop-in for live help, starting January 11, 2022! - -- Tuesdays, 4-5:30pm ET / 1-2:30pm PT -- Thursdays, 11:30am-1pm ET / 8:30-10am PT - -You can find the URL to the Virtual Office Hours meeting room when you log into an OSG Connect login node, or in the signature of a support email from an RCF. - -Click [here](https://docs.google.com/forms/d/e/1FAIpQLSd3K78Xx1Vo-KjqW_2y0YKcUMXrEsKXWk3I1Aww64RL22QpnQ/viewform) to sign-in for office hours, once you arrive in the room. - -Cancellations will be announced via email. As always, if the times above don’t work for you, please email us at our usual support address to schedule a separate meeting. - - -## Make an Appointment -We are happy to arrange meetings outside of designated Office Hours, per your preference. Simply email us at support@opensciencegrid.org, and we will set up a time to meet! - - -## Training Opportunities - -The RCF team runs regular new user training on the first Tuesday of the month. See upcoming -training dates, registration information, and materials on the [OSG Training page](12000084444). diff --git a/welcome/overview/gracc.md b/welcome/overview/gracc.md deleted file mode 100644 index d083a599..00000000 --- a/welcome/overview/gracc.md +++ /dev/null @@ -1,19 +0,0 @@ -[title]: - "OSG Accounting (GRACC)" - -[TOC] - -GRACC is the Open Science Pool's accounting system. If you need graphs or high level statistics -on your OSG usage, please go to: - -[https://gracc.opensciencegrid.org/](https://gracc.opensciencegrid.org/) - -To drill down on your own project and/or user, go to _Payload Jobs_. - -Under the _Project_ or _User_ drop-downs, find your project/user. You can select multiple -ones. - -In the upper right corner, you can select a different time period. You can then select a -different _Group By_ time range. For example, if you want data for the last year grouped -monthly, select "Last 1 year" for the _Period_, and "1M" for the _Group By_. - - diff --git a/welcome/overview/is-it-for-you.md b/welcome/overview/is-it-for-you.md deleted file mode 100644 index 5d964f51..00000000 --- a/welcome/overview/is-it-for-you.md +++ /dev/null @@ -1,66 +0,0 @@ -[title]: - "Computation on the Open Science Pool" - - -The [OSG][osg] is a nationally-funded consortium of computing resources -at more than one hundred institutional partners that, together, offer a strategic -advantage for computing work that can be run as numerous short tasks that can execute -independent of one another. For researchers -who are not part of an organization with their own pool in the OSG, we offer the -[Open Science Pool (OSPool)](https://opensciencegrid.org/about/open_science_pool/), with dozens -of campuses contributing excess computing capacity in support of open science. The OSPool is available -for US-affiliated research projects and groups via the [OSG Connect](https://www.osgconnect.net/) service, which the documentation -on this site is specific to. - -Learn more about the services provided by the OSG that can support your HTC workload: - -OSG Introduction - -For problems that can be run as numerous independent jobs (a high-throughput approach) and have requirements -represented in the first two columns -of the table below, the significant capacity of the OSPool can transform the types of -questions that researchers are able to tackle. **Importantly, -many compute tasks that may appear to not be a good fit _can_ be modified in simple ways -to take advantage, and we'd love to discuss options with you!** - -| | **Ideal jobs!** | **Still advantageous** | **Maybe not, but get in touch!** | -|:--------|:--------------|:--------------|:--------------| -| Expected Throughput: | 1000s concurrent jobs | 100s concurrent jobs | let's discuss! | -| **Per-Job Requirements** | | | | -| CPU cores | 1 | < 8 | > 8 (or MPI)| -| GPUs | 0 | 1 | > 1 | -| Walltime | < 10 hrs* | < 20 hrs* | > 20 hrs (not a good fit) | -| RAM | < few GB | < 40 GB | > 40 GB | -| Input | < 500 MB | < 10 GB | > 10 GB** | -| Output | < 1GB | < 10 GB | > 10 GB** | -| Software | pre-compiled binaries, containers | Most other than ---> | Licensed software, non-Linux | - -\* or checkpointable - -\** per job; you can work with a multi-TB dataset on the OSPool if it can be split into pieces! - -Some examples of work that have been a good fit for the OSPool and benefited from -using its resources include: - -- image analysis (including MRI, GIS, etc.) -- text-based analysis, including DNA read mapping and other bioinformatics -- hyper/parameter sweeps -- Monte Carlo methods and other model optimization - -**Learn more and chat with a Research Computing Facilitator by [signing up for OSG Connect][account-request]** - - -## Resources to Quickly Learn More - -**Introduction to OSG the Distributed High Throughput Computing framework** from the annual [OSG User School](https://opensciencegrid.org/outreach/): - -[](https://www.youtube.com/embed/vpJPPjoQ3QU) - -**[Full OSG User Documentation](https://support.opensciencegrid.org/support/home)** including our [Roadmap to HTC Workload Submission](12000081596-roadmap-to-htc-workload-submission-via-osg-connect) - -**[OSG User Training materials](https://support.opensciencegrid.org/support/solutions/articles/12000084444-osg-user-training)** . Live training sessions are advertised/open to those with active accounts on OSG Connect. - - -**Learn more and chat with a Research Computing Facilitator by [signing up for OSG Connect][account-request]** - -[osg]: https://opensciencegrid.org/ -[account-request]: https://osgconnect.net/signup diff --git a/welcome/overview/policy.md b/welcome/overview/policy.md deleted file mode 100644 index 8a010482..00000000 --- a/welcome/overview/policy.md +++ /dev/null @@ -1,27 +0,0 @@ -[title]: - "Policies for Using OSG Connect and the OSPool" - -Access to OSG Connect and the Open Science Pool (OSPool) is contingent on compliance with the below and with any requests from OSG staff to change practices that cause issues for OSG systems and/or users. **Please contact us if you have any questions! We can often help with exceptions to default policies and/or identify available alternative approaches to help you with a perceived barrier.** - -As the below do not cover every possible scenario of potentially disruptive practices, OSG staff reserve the right to take any necessary corrective actions to ensure performance and resource availability for all OSG Connect users. This may include the hold or removal of jobs, deletion of user data, deactivation of accounts, etc. In some cases, these actions may need to be taken without notifying the user. - -1. **By using the OSG Connect service, users are expected to follow the Open Science Pool [acceptable use policy](https://github.com/opensciencegrid/osgconnect-portal-markdowns/blob/master/signup_content/signup_modal.md)**, which includes appropriate scope of use and common user security practices. OSG Connect is only available to individuals affiliated with a US-based academic, government, or non-profit organization, or with a research project led by an affiliated sponsor. - -2. **Users can have up to 10,000 jobs queued, without taking additional steps**, and should submit multiple jobs via a single submit file, according to our online guides. Please write to us if you’d like to easily submit more! - -3. **Do not run computationally-intensive or persistent processes on the access points (login nodes).** Exceptions include single-threaded software compilation and data management tasks (transfer to/from the login nodes, directory creation, file moving/renaming, untar-ing, etc.). The execution of multi-threaded tasks for job setup or post-processing or software testing will almost certainly cause performance issues and may result in loss of access. Software testing should be executed from within submitted jobs, where job scheduling also provides a more accurate test environment to the user without compromising performance of the login nodes. OSG staff reserve the right to kill any tasks running on the login nodes, in order to ensure performance for all users. Similarly, **please contact us to discuss appropriate features and options, rather than running scripts (including `cron`) to automate job submission**, throttling, resubmission, or ordered execution (e.g. workflows), even if these are executed remotely to coordinate work on the OSG Connect login nodes. These almost always end up causing significant issues and/or wasted computing capacity, and we're happy to help you to implement automation tools the integrate with HTCondor. - -4. **Data Policies**: *OSG Connect filesystems are not backed up and should be treated as temporary (“scratch”-like) space for active work, only*, following [OSG Connect policies for data storage and per-job transfers](https://support.opensciencegrid.org/support/solutions/articles/12000002985-introduction-to-data-management-on-osg-connect). Some OSG Connect storage spaces are truly ‘open’ with data available to be downloaded publicly. Of note: - - Users should keep copies of essential data and software in non-OSG locations, as OSG staff reserve the right to remove data at any time in order to ensure and/or restore system availability, and without prior notice to users. - - Proprietary data, HIPAA, and data with any other privacy concerns should not be stored on any OSG Connect filesystems or computed in OSG. Similarly, users should follow all licensing requirements when storing and executing software via OSG Connect submit servers. (See “Software in OSG Connect”.) - - Users should keep their /home directory privileges restricted to their user or group, and should not add ‘global’ permissions, which will allow other users to potentially make your data public. - - User-created ‘open’ network ports are [disallowed](https://github.com/opensciencegrid/security/blob/master/docs/policy/OSG_Connect_Login_Server_Open_Port_Policy.md), unless explicitly permitted following an accepted justification to support@opensciencegrid.org. (If you’re not sure whether something you want to do will open a port, just get in touch!) - -5. The following actions may be taken automatedly or by OSG staff to stop or prevent jobs from causing problems. Please contact us if you’d like help understanding why your jobs were held or removed, and so we can help you avoid problems in the future. - - **Jobs using more memory or disk than requested** may be automatically held (see [Scaling Up after Test Jobs](https://support.opensciencegrid.org/support/solutions/articles/12000076552-scaling-up-after-success-with-test-jobs) for tips on requesting the ‘right’ amount of job resources in your submit file). - - **Jobs running longer than their JobDurationCategory allows for** will be held (see [Indicate the Job Duration Category of Your Jobs](12000083468)). - - **Jobs that have executed more than 30 times without completing** may be automatically held (likely because [they’re too long](https://support.opensciencegrid.org/support/solutions/articles/5000632058-is-the-open-science-grid-for-you-) for OSG). - - **Jobs that have been held more than 14 days** may be automatically removed. - - **Jobs queued for more than three months** may be automatically removed. - - **Jobs otherwise causing known problems** may be held or removed, without prior notification to the user. - - **Held jobs may also be edited to prevent automated release/retry** - - NOTE: in order to respect user email clients, job holds and removals do not come with specific notification to the user, unless configured by the user at the time of submission using HTCondor’s ‘notification’ feature. diff --git a/welcome/xsede/osg-xsede.md b/welcome/xsede/osg-xsede.md deleted file mode 100644 index 2bd4f668..00000000 --- a/welcome/xsede/osg-xsede.md +++ /dev/null @@ -1,353 +0,0 @@ -## Overview - -The [OSG](https://osg-htc.org/) promotes science by: - - * enabling a framework of distributed computing and storage resources - - * making available a set of services and methods that enable better access - to ever increasing computing resources for researchers and communities - - * providing resource sharing principles and software that enable distributed - high throughput computing (DHTC) for users and communities at all scales. - -The OSG facilitates access to DHTC for scientific research -in the US. The resources accessible through the OSG are -contributed by the community, organized by the OSG, and -governed by the [OSG -Consortium](http://www.opensciencegrid.org); an overview is available at -[An Introduction to OSG](http://osg-docdb.opensciencegrid.org/0008/000839/004/OSG%20Intro%2 -0v23.pdf). In 2017, OSG is comprised -of about 126 institutions with ~120 active sites that collectively -support usage of ~4,000,000 core hours per day. Up-to-date usage metrics -are available at the [OSG Usage Display](https://display.opensciencegrid.org/). - -Cores that are not being used at any point in time by -the owning communities are made available for shared use by other -researchers; this usage mode is called opportunistic access. OSG -supports XSEDE users by providing a Virtual Cluster that forms an -abstraction layer to access the opportunistic cores in the distributed -OSG infrastructure. This interface allows XSEDE users to view the OSG as -a single cluster to manage their jobs, provide the inputs and retrieve -the outputs. XSEDE users access the OSG via the OSG-XSEDE login host -that appears as a resource in the XSEDE infrastructure. - -## Computation that is a good match for OSG - -High throughput workflows with simple system and -data dependencies are a good fit for OSG. The -[Condor manual](http://research.cs.wisc.edu/condor/manual/current/) has an overview of [high throughput computing](http://research.cs.wisc.edu/condor/manual/current/1_Overview.html). - -Jobs submitted into the OSG Virtual Cluster will be executed on machines -at several remote physical clusters. These machines may differ in terms -of computing environment from the submit node. Therefore it is important -that the jobs are as self-contained as possible by generic binaries -and data that can be either carried with the job, or staged on demand. -Please consider the following guidelines: - - - * Software should preferably be **single threaded**, using - **less than 2 GB memory** and each invocation should - **run for 1-12 hours**. Please contact the support listed - below for more information about these capabilities. System level check - pointing, such as the HTCondor standard universe, is not available. - Application level check pointing, for example applications writing out - state and restart files, can be made to work on the system. - - * Compute sites in the OSG can be configured to use pre-emption, which - means jobs can be automatically killed if higher priority jobs enter - the system. Pre-empted jobs will restart on another site, but it is - important that the jobs can handle multiple restarts. - - * Binaries should preferably be statically linked. However, dynamically - linked binaries with standard library dependencies, built for a 64-bit - Red Hat Enterprise Linux (RHEL) 6 machines will also work. Also, - interpreted languages such as Python or Perl will work as long as there - are no special module requirements. - - * Input and output data for each job should be < 10 GB to allow them - to be pulled in by the jobs, processed and pushed back to the submit - node. Note that the OSG Virtual Cluster does not currently have a global - shared file system, so jobs with such dependencies will not work. - - * Software dependencies can be difficult to accommodate unless the software - can be staged with the job. - - -The following are examples of computations that are **not** good -matches for OSG: - - * Tightly coupled computations, for example MPI based communication, will - not work well on OSG due to the distributed nature of the infrastructure. - - * Computations requiring a shared file system will not work, as there is - no shared filesystem between the different clusters on OSG. - - * Computations requiring complex software deployments are not a good fit. - There is limited support for distributing software to the compute - clusters, but for complex software, or licensed software, deployment - can be a major task. - - -## System Configuration - -The OSG Virtual Cluster is a Condor pool overlay on top of OSG -resources. The pool is dynamically sized based on the demand, the -number of jobs in the queue, and supply, resource availability at the -OSG resources. It is expected that the average number of resources, -on average, available to XSEDE users will be in the order of 1,000 -cores. - -One important difference between the OSG Virtual Cluster and most of -the other XSEDE resources is that the OSG Virtual Cluster does not have -a shared file system. This means that your jobs will have to bring -executables and input data. Condor can transfer the files for you, -but you will have to identify and list the files in your Condor job -description file. - -Local storage space at the submission site is controlled by -quota. Your home directory has a quota of 10 GBs and your work directory -`/local-scratch/$USER` has a quota of 1 TB. There are no -global quotas on the remote compute nodes, but expect that about 10 GBs -are available as scratch space for each job. - - -## System Access - -The preferred method to access the system is via the XSEDE Single -Sign On (SSO) Hub. Please see the [sso documentation](https://portal.xsede.org/single-sign-on-hub) -for details. - -A secondary access methor is to use SSH public key authentication. -Secure shell users should feel free to append their public RSA key -to their `~/.ssh/authorized_keys` file to enable access -from their own computer. Please login once via the SSO Hub to install your -key. Please make sure that the permissions on the .ssh directory and -the authorized_keys file have appropiate permissions. For example - - $ chmod 755 ~/.ssh - $ chmod 644 ~/.ssh/authorized_keys - - -## Application Development< - -Most of the clusters in OSG are running Red Hat Enterprise Linux (RHEL) -6 or 7, or some derivative thereof, on an x86_64 architecture. For -your application to work well in this environment, it is recommend -that the application is compiled on a similar system, for example on -the OSG Virtual Cluster login system: `submit-1.osg.xsede.org -`. It is also recommended that the application be statically -linked, or alternatively dynamically linked against just a few standard -libraries. What libraries a binary depends on can be checked using the -Unix `ldd` command-line utility: - - $ ldd a.out - a.out is a static executable - - -In the case of interpreted languages like Python and Perl, applications -have to either use only standard modules, or be able to ship the modules -with the jobs. Please note that different compute nodes might have -different versions of these tools installed. - -A good solution to complex software stack is Singularity containers -which are described below. - - -## Running Your Application - -The OSG Virtual Cluster is based on Condor and the -[Condor manual](http://research.cs.wisc.edu/condor/manual/current/) -provides a reference for command line tools. The commonly used tools -are: - - * `**condor_submit**` - Takes a Condor submit file and adds the job to the queue - - * `**condor_q**` - Lists the jobs in the queue. Can be invoked with your - username to limit the list of jobs to your jobs: `condor_q $USER` - - * `**condor_status**` - Lists the available slots in the system. - Note that this is a dynamic list and if there are no jobs in the system, - `condor_status` may return an empty list - - * `**condor_rm**` - Remove a job from the queue. If you are running a DAG, - please `condor_rm` the id of the DAG to remove the whole workflow. - -### Submitting a Simple Job - -Below is a basic job description for the Virtual Cluster. - - universe = vanilla - - # specifies the XSEDE project to charge the job usage to - this is a - # required attribute for all jobs submitted on the OSG-XSEDE resource - +ProjectName = "TG-NNNNNN" - - # requirements is an expression to specify machines that can run jobs - requirements = OSGVO_OS_STRING == "RHEL 6" && Arch == "X86_64" && HAS_MODULES == True - request_cpus = 1 - request_memory = 2 GB - request_disk = 10 GB - - executable = /bin/hostname - - arguments = -f - - should_transfer_files = YES - WhenToTransferOutput = ON_EXIT - - output = job.out - error = job.err - log = job.log - - notification = NEVER - - queue - - - Create a file named `job.condor` containing the above text and then run: - - $ condor_submit job.condor - -You can check the status of the job using the `condor_q` command. - -**Note:** The Open Science Pool is a distributed resource, and there -will be minor differences in the compute nodes, for example in what -system libraries and tools are installed. Therefore, when running -a large number of jobs, it is important to detect and handle job -failures correctly in an automatic fashion. It is recommended that your -application uses non-zero exit code convention to indicate failures, and -that you enable retries in your Condor submit files. For example: - - # stay in queue on failures - on_exit_hold = (ExitBySignal == True) || (ExitCode != 0) - - # retry job 3 times, pause 1 hour between retries - periodic_release = (NumJobStarts < 3) && ((CurrentTime - EnteredCurrentStatus) > (60*60)) - - -### Job Example: Java with a job wrapper - -The following is an example on how to run Java code on Open -Science Grid. The job requirements specifies that the job requires -Java, and a wrapper script is used to invoke Java. - -File: `condor.sub` - - universe = vanilla - - # specifies the XSEDE project to charge the job usage to - this is a - # required attribute for all jobs submitted on the OSG-XSEDE resource - +ProjectName = "TG-NNNNNN" - - # requirements is an expression to specify machines that can run jobs - requirements = HAS_JAVA == True - - # stay in queue on failures on_exit_hold = (ExitBySignal == True) || (ExitCode != 0) - - # retry job 3 times, pause 1 hour between retries - periodic_release = (NumJobStarts < 3) && ((CurrentTime - EnteredCurrentStatus) > (60*60)) - - executable = wrapper.sh - - should_transfer_files = YES - WhenToTransferOutput = ON_EXIT - - # a list of files that the job needs - transfer_input_files = HelloWorld.jar - - output = job.out - error = job.err - log = job.log - - notification = NEVER - - queue - -File: `wrapper.sh` - - #!/bin/bash - - set -e - - java HelloWorld - - -## Sample Jobs and Workflows - -A set of sample jobs and workflows can be found under -`/opt/sample-jobs` on the submit-1.osg.xsede.org host. README -files are included with details for each sample. - -`/opt/sample-jobs/single/` contains a single Condor job -example. Single jobs can be used for smaller set of jobs or if the job -structure is simple, such as parameter sweeps. - -A sample-app package -([sampleapp.tgz](http://www.ncsa.illinois.edu/People/yanliu/codes/sampleapp.tgz)) -is available in the `/opt/sample-jobs/sampleapp/` directory. This package shows -how to build a library and an executable, both with dynamic and static -linking, and submit the job to a set of different schedulers available -on XSEDE. The package includes submit files for PBS, SGE and Condor. - -[DAGMan](http://research.cs.wisc.edu/condor/manual/current/2_10DAGMan_Applications.html) -is a HTCondor workflow tool. It allows the -creation of a directed acyclic graph of jobs to be run, and then DAGMan -submits and manages the jobs. DAGMan is also useful if you have a -large number of jobs, even if there are no job inter-dependencies, as -DAGMan can keep track of failures and provide a restart mechanism if -part of the workflow fails. A sample DAGMan workflow can be found in -`/opt/sample-jobs/dag/` - -[Pegasus](https://pegasus.isi.edu) is a workflow system that -can be used for more complex workflows. It plans abstract workflow -representations down to an executable workflow and uses Condor DAGMan -as a workflow executor. Pegasus also provides debugging and monitoring -tools that allow users to easily track failures in their workflows. -Workflow provenance information is collected and can be summarized with -the provided statistical and plotting tools. A sample Pegasus workflow -can be found in `/opt/sample-jobs/pegasus/` . - - -## Singularity Containers - -Singularity containers provide a great solution for complex software -stacks or OS requirements, and OSG has easy to use integrated support -for such containers. Full details can be found in the -[Singularity Containers](https://support.opensciencegrid.org/support/solutions/articles/12000024676). - - -## Distribute data with Stash - -Stash is a data solution used under -[OSGConnect](https://osgconnect.net/), but is partly also available -for OSG XSEDE users. Files under `/local-scratch/public_stash/` will -automatically synchronize to the globally available -`/cvmfs/stash.osgstorage.org/user/xd-login/public/` file system, which -is available to the majority of OSG connected compute nodes. This is a great -way to distribute software and commonly used data sets. To get started, create -your own sub directory: - - $ mkdir -p /local-scratch/public_stash/$USER - -Now, populate that directory with the software and data required for your jobs. -The synchronization can take couple of hours. You can verify the data has -reached the /cvmfs system by using `ls`: - - $ ls /cvmfs/stash.osgstorage.org/user/xd-login/public/ - -To steer your jobs to compute nodes which can access the file system, add -`HAS_CVMFS_stash_osgstorage_org == True` to your job -requirements. For example: - - requirements = OSGVO_OS_STRING == "RHEL 6" && Arch == "X86_64" && HAS_MODULES == True && HAS_CVMFS_stash_osgstorage_org == True - -Once a job starts up on such a compute node, the job can read directly -from `/cvmfs/stash.osgstorage.org/user/xd-login/public/` - - -## How to get help using OSG - -XSEDE users of OSG may get technical support by -contacting OSG Research Facilitation staff at email -. -Users may also contact the [XSEDE helpdesk](https://portal.xsede.org/help-desk).