diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..1df2fcfd Binary files /dev/null and b/.DS_Store differ diff --git a/README.rst b/README.rst index 39bb79ee..3e8f3cbc 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,927 @@ +======= MPWorks ======= -merges pymatgen, custodian, and FireWorks into a custom workflow for Materials Project. See mpworks/docs directory for some basic explanation of the tasks. \ No newline at end of file +MPWorks merges pymatgen, custodian, and FireWorks into a custom workflow for Materials Project. It is very powerful in that it is used for all the calculations performed for the Materials Project database; however, it is also quite complicated and not completely flexible. This guide will try to explain the operation of the MPWorks system of running calculations + +0. Installation +=============== + +MPWorks is in essence a system of running calculations. Thus, in addition to the code you need to have several MongoDB databases and environment variables set. The easiest way to install all of this is to use (or modify) `the MPenv code `_, which will install all the necessary dependencies of MPWorks, build the appropriate databases, and set the environment variables. Unfortunately, MPenv only works on a few systems such as NERSC and (soon) ALCF. + +There are also some example workflows in MPWorks that can be attempted with much less complication. This guide will also cover those, but you will still need to learn the fundamentals and will still need a FireWorks database. It is important to note that MPWorks is not intended to be a "general purpose" code at this time, and is mainly used internally by the Materials Project team. + +1. Introduction and Pre-requisites +================================== + +This document is a guide for designing and running materials science and chemistry workflows using the Materials Project codebases (pymatgen, FireWorks, custodian, etc.) and NERSC resources. + +The advantage of learning the infrastructure is that once you gain familiarity, you will be able to very easily run and manage your calculations. If you need to perform a set of computations over a new compound, you will simply need to execute a command rather than editing input files, ssh’ing them to NERSC, running qsub, etc…This is crucial when running hundreds of thousands of jobs, but you’ll probably find it very nice to have in your day-to-day work as well. Whenever you want to compute a new structure, you can do it in almost no time. + +In addition, the infrastructure is meant to help you rigorously test your workflows over test sets of compounds and rapidly analyze the results. + +However, before taking advantage of this infrastructure you must take a little time to learn it. In particular, there are many components to making the Materials Project high-throughput project function and you will need to learn at least a bit about all of them: + +* The **pymatgen** codebase (http://pymatgen.org) is used to read and write input and output files for various computational codes (like VASP or NWChem) given arbitrary structures and for different computational types (like static or structure optimization). The **pymatgen-db** codebase (http://pythonhosted.org/pymatgen-db/) is used to parse output files. +* The **custodian** codebase (http://pythonhosted.org/custodian/) is used to execute the desired code in a way that can fix most errors that might be encountered during the run +* The **FireWorks** codebase (http://pythonhosted.org/FireWorks/) allows us to automically run and track many thousands of jobs (such as custodian jobs) over supercomputing resources. FireWorks can also fix more complicated errors that may arise (like server crashes) and help design dynamic workflows. + +This documentation focuses on how these codebases work together and is *not* intended to teach you how to use the codebases individually. Before starting, it is therefore crucial that you review and have a basic understanding of each codebase in isolation, so things will make sense when we start putting things together. + +1.1 FireWorks prerequisites +--------------------------- + +.. pull-quote:: | FireWorks documentation can be found at http://pythonhosted.org/FireWorks/ + +Before starting this documentation, make sure you have read through all the documentation on FireWorks and have a basic understanding of at least the following tutorials: + +* Quickstart +* Defining Jobs using FireTasks +* Creating Workflows +* Dynamic Workflows +* Tips for designing FireTasks, FireWorks, and Workflows + +This documentation assumes that you have at least a basic grasp of the concepts of those tutorials. If you are interested in not only designing and submitting workflows, but running testing or production jobs at NERSC, you should also review the following FireWorks documentation (you should read the tutorials, but you don’t have to actually follow the instructions to install anything at NERSC; remember, MPEnv will do that for you): + +* Worker Tutorial +* Launch Rockets through a queue +* Reserving FireWorks upon queue submission +* Installation Notes on various clusters / supercomputing centers + +1.2 custodian prerequisites +--------------------------- + +.. pull-quote:: | custodian documentation can be found at http://pythonhosted.org/custodian/ + +Additionally, you should read all the documentation on custodian (it is a single page) + +1.3 pymatgen and pymatgen-db prerequisites + +.. pull-quote:: | pymatgen documentation can be found at http://pythonhosted.org/pymatgen/ +.. pull-quote:: | pymatgen db documentation is at http://pythonhosted.org/pymatgen-db/ + +Before starting this documentation, you should have enough familiarity with pymatgen to: + +* write input files for the code you want to execute (e.g., VASP or NWChem) and +* parse output files for the code you want to execute into a MongoDB (JSON) format + +For the latter functionality, you might need to consult the pymatgen-db codebase. + +1.4 Other prerequisites +----------------------- + +You should have the **MPWorks** codebase downloaded (so you can see the +source code), e.g. through MPenv at NERSC. We’ll be referring to source +code in MPWorks during the course of this guide, so it’s important you +have it available. + +2. Designing Scientific Workflows for running at NERSC +====================================================== + +2.1 How workflow design fits in +------------------------------- + +If you recall the FireWorks documentation, there are essentially two steps to running scientific workflows at NERSC (Figure 1): + +.. image:: mpworks/docs/a.png + +**Figure 1 FireWorks operation** + +The steps are: + +1. Defining Workflows in FireWorks format and putting them in the + “LaunchPad” (a MongoDB database that is hosted “somewhere”) + +2. Running Workflows on NERSC (a FireWorker) by pulling jobs from the + LaunchPad + +This section concentrates only on part 1 – defining workflows and +putting them in the LaunchPad. This part does not actually require +logging into a NERSC machine and can be done from your laptop at home. +Running jobs will be covered later on. + +2.2 Designing Workflows for Materials Science +--------------------------------------------- + +The goal of this section is to explain how, given any crystal or +molecule, one might construct a FireWorks Workflow for computing its +properties (Figure 2): + +.. image:: mpworks/docs/d.png + +**Figure 2 Goal of this section – automatically create a FireWorks +workflow for any crystal or molecule** + +Once we can write some code that will take any compound and construct a +workflow, automation is easy. Those workflows can be run very easily at +NERSC using the job execution features of FireWorks. + +The issue then is how to write code that will take in a compound and +create a Workflow. If you recall the FireWorks documentation, a Workflow +is composed of multiple FireWorks, which are in turn composed of +multiple FireTasks. Each FireTask is meant to be an atomic “action”. In +our case, a single FireTask might: + +- write some input files for a code + +- execute code (e.g. VASP or NWChem), either directly in Python or + within a **custodian** + +- move files around + +- parse some files and enter them in a database + +- decide whether more computations are needed and add them to the + database if needed (using dynamic workflow features built into + FireWorks) + +By putting together multiple FireTasks and FireWorks, we’ll end up with +a Workflow that suits our needs. So we need to do two things: + +- write the code for each individual FireTask + +- meld them into an appropriate Workflow sequence, splitting some of + them up into different FireWorks + +Although these topics are somewhat related, we’ll try to cover some +examples of writing FireTasks first, and then move onto constructing +Workflows that tie them together. + +2.3 Writing FireTasks: a few examples +------------------------------------- + +It is somewhat difficult to create a guide for writing FireTasks – a +FireTask can really be arbitrary code. Therefore, we’ll just point to a +few case studies in the MPWorks and Rubicon codebases. These codebases +contain specific implementations of FireTasks for the Materials Project +and JCESR project, respectively. They depend on **pymatgen**, +**custodian**, and **FireWorks** in order to work. + +Note that these FireTasks change from time to time, so use this as a +rough guide. Also, don’t worry about understanding every detail of these +FireTasks – just get a rough sense for what they’re doing and try to get +through this section with a basic understanding the first time. Ask an +MPWorks expert (e.g., Wei or Anubhav) when you really need to understand +the fine details. + +**Important note:** Recall from the FireTasks documentation that the +*run\_task()* method of a FireTask is what gets executed. You should +concentrate on this method for each FireTask. + +2.3.1 VaspWriterTask +~~~~~~~~~~~~~~~~~~~ + + VaspWriterTask is located in + **/mpworks/firetasks/vasp\_io\_tasks.py** + +The VaspWriterTask is about as simple as it gets – it is just a few +lines of code. It reads information from the *fw\_spec* and uses that +information to write INCAR, KPOINTS, POSCAR, and POTCAR files to the +current directory. The expectation is that the next FireTask in the +sequence will run VASP. + +More specifically, this task is reading in the “vasp” key of the +*fw\_spec* that was stored by the person creating the FireWork. This key +contains the information needed generate the input files. The format of +the “vasp” key is pymatgen dictionary representations of the INCAR, +POSCAR, etc objects. Given these dictionary objects, this FireTask will +write the input files. + +Therefore, if you are trying to write some VASP input files, you can +just create a FireWork with the appropriate *spec* (a “vasp” key with +Pymatgen dictionary representations of input files) and then add the +*VaspWriterTask* as one of your FireTasks. + +Note that you might wonder why the specification expects pymatgen +representations of these files, rather than just the raw String content. +Either would work; the pymatgen dictionary representations are much +easier to query and explore with MongoDB. For example, you can very +easily search for all the FireWorks where the INCAR parameter has NSW +set to 0, which is harder (and slower) to do via String matching over +the database. + +2.3.2 VaspCustodianTask +~~~~~~~~~~~~~~~~~~~~~~~ + + A simplified version of VaspCustodianTask (called + VaspCustodianTaskEx) is located in + **/mpworks/examples/firetasks\_ex.py.** The actual VaspCustodianTask + is located in **/mpworks/firetasks/custodian\_task.py**. We will + discuss the simple version. + +The VaspCustodianTaskEx uses custodian to run an executable such as +VASP. It expects that all input files for VASP are already written in +the directory (e.g., via a VaspWriterTask). The job of +VaspCustodianTaskEx is to execute a **custodian** to call the VASP +executable. + +This is the core code that loads a custodian and runs it:: + + c = Custodian(self.handlers, self.jobs, self.max_errors) + custodian_out = c.run() + +You might notice that this code has nothing to do with VASP. The +parameters *self.handlers* and *self.jobs* contain **custodian** objects +that represent VASP jobs. The user passes these in via the FireTask’s +*parameters*. In the constructor for VaspCustodianTask, you’ll notice a +line of code that looks like this:: + + self.jobs = map(VaspJob.from_dict, parameters['jobs']) + +This is what is loading the Vasp Jobs based on the parameters of the +FireTask. To use VaspCustodianTaskEx, you must therefore create this +FireTask with the *job* and *handlers* parameters set to dictionary +representations of VaspJob and VasprunHandler objects from custodian. + +There is some extra code in this task regarding a choice between “aprun” +and “mpirun” because the command to execute VASP depends on the machine +we are running on. But the core of this method is to load custodian Job +and Handler objects in the constructor, and then instantiate and run the +custodian in the run() method. + +Note that another way to run VASP is to simply use the command:: + + import subprocess + subprocess.check_call([“vasp”]) + +inside the run() method, and do away with complicated constructors, +reading parameters like *job* or *handlers*, and make life simple. +However, if we did this we would not be able to use the error-correction +features of custodian. + +2.3.3 VaspCopyTask +~~~~~~~~~~~~~~~~~~~ + + VaspCopyTask is located in **/mpworks/firetasks/vasp\_io\_tasks.py** + +The VaspCopyTask in MPWorks is also very simple. All it’s doing is +copying a bunch of files from some directory to the current directory. +This FireTask is used, for example, to copy output files from the +structure optimization run to the static run. The directory containing +the previous run must be defined in the FireWork specification under the +“prev\_vasp\_dir” key. Other than that, there are some options for +choosing what files to move and dealing with tricky things like +“.relax#” extensions to output files added by certain types of VASP +custodian runs. + +2.3.4. VaspToDBTask +~~~~~~~~~~~~~~~~~~~~ + + A simplified version of VaspToDBTask called VaspToDBTaskEx is + located in **/mpworks/examples/firetasks\_ex.py.** The actual + VaspToDBTask is located in + **/mpworks/firetasks/vasp\_io\_tasks.py**. We will discuss the + simple version first, then the more complex version. + +The VASPtoDBTaskEx uses the pymatgen-db codebase to enter the output of +a VASP run into the database. + +First, it loads the VASP output directory from the “prev\_dir” +parameter. Then, it instantiates a *VaspToDBTaskDrone* object which, +given database credentials, can parse the output directory and enter the +results into the database. The actual database insertion is done via the +command:: + +t_id = drone.assimilate(prev_dir) + +At this point, we are largely done with the simplified VaspToDBTaskEx. + +The more complex VaspToDBTask (without the Ex) uses MPVaspDrone and does +the following (feel free to skip these details if you’re just getting +started): + +- Before database insertion, this task is also moving files from + NERSC’s **$SCRATCH** filesystem to NERSC’s **$PROJECT** filesystem + (we refer to it as the “garden”). All runs need to be moved from + SCRATCH to PROJECT after completion, due to limited space (but better + disk performance) on SCRATCH. + +- After database insertion, there is a lot of complicated code + determining whether this task should be rerun using a new Workflow + step. Feel free to ignore this detail for the moment. Normally, this + whole ordeal would be handled by **custodian** in a cleaner way. + However, the limitation of **custodian** is that all job restarts + occur within the same walltime at NERSC. If we have a 7-day walltime, + and the job fails after 6.5 days, a **custodian**-based restart would + only give the job’s reincarnation 0.5 days to complete. Most of the + time, jobs fail early and it’s OK to use **custodian** and restart + within the same walltime limit. However, some errors (like those + detected by UnconvergedHandler) fail very late or at the end of the + job, and they need to be wrapped in a new FireWork that will allow + the reincarnation of the job to run with a brand new 7-day walltime. + +2.3.4. Other MPWorks tasks, e.g. “Setup”-style tasks and Controller tasks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Setup-style tasks are located in + **/mpworks/firetasks/vasp\_setup\_tasks.py**. Controller tasks are + located in **/mpworks/firetasks/controller\_tasks.py** + +There are many MPWorks tasks that take the output of a previous VASP +directory and modify some of the inputs for the next step in the +workflow. For example, the final structure and run parameters of a +structure optimization run is used to create the input parameters of a +static run (with just a few parameters changed). The “Setup” style tasks +will read in the output files of the previous run (after they are moved +using VaspCopyTask), and perform the necessary operations to create +input files for the current run. + +The “Controller Task” is more complicated in that it reads in data from +a previous VASP run and dynamically creates new jobs as needed. At the +time of this writing, the controller task will create more VASP jobs if +initial calculations demonstrate the material to be an insulator with +gap > 0.5 eV. + +You can review these tasks on your own and contact the MP development +list if you have questions. In our example FireTask, we won’t be using +some of these FireTasks. + +2.4 Organizing FireTasks into Workflows +--------------------------------------- + +After studying the previous section, you should have a good idea of what +FireTasks are and some idea of how you might write some simple +FireTasks. The next step is to organize FireTasks into Workflows. Recall +that there are multiple ways to do this, in particular whether to put +many FireTasks in a single FireWork or to use multiple FireWorks (Figure +3): + +.. image:: mpworks/docs/h.png + +**Figure 3 Put all FireTasks in a single FireWork, or split the tasks +amongst multiple FireWorks?** + +The FireWorks documentation, in particular the tutorial on “Tips for +designing FireTasks, FireWorks, and Workflows”, contains many details on +how to do the design. In this section, we’ll follow one of the +recommendations in that tutorial and begin by putting all the FireTasks +in a single FireWork (left side of the diagram), and then iterating on +that design to see where multiple FireWorks are needed. + +2.4.1 A prototypical Materials Science workflow – iteration 1 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In Figure 4, we draw a prototypical materials science workflow, where +FireTasks are in purple and all are within a single FireWork: + +.. image:: mpworks/docs/e.png + +**Figure 4 Initial draft of how a Workflow could be written. This is not +the suggested way to do things.** + +The Workflow in Figure 4 runs two types of calculations and two database +insertions (one for each calculation). Each calculation might represent +a VASP run, GULP calculation, NWChem calculation, or MD simulation. + +While putting everything in a single FireWork is an easy way to design a +Workflow, it suffers from several limitations: + +- Recall that each FireWork gets its own job at NERSC, with a single + walltime. The walltime at NERSC (7 days) might not be enough to + confidently finish both calculations. We might want to give each + calculation its own 7-day walltime + +- We might want to run the calculations on different machines or with + different parameters. e.g., if calculation 1 requires using 2 nodes + with low memory but calculation 2 requires using 100 nodes with high + memory, you would need to put the jobs in different FireWorks so they + can use different queue settings or run on different machines + altogether. + +- Recall that if you want to rerun a job, you’ll need to rerun the + entire FireWork from scratch. This means that if you embed the + Workflow within a single FireWork, and a server crashes or memory + error occurs during calculation 2, FireWorks must rerun everything + from scratch in this design. + +- Similarly, duplicate checking occurs at the FireWork level. So if + you’ve already run calculation 1 in the past but not calculation 2, + FireWorks cannot do atomistic duplicate checking and only run + calculation 2 (unless you split the Workflow into 2 FireWorks). + +- We might want to do some branching operation in between the + calculations. For example, calculation 2 might not be necessary if + calculation 1 finishes with an error. It is then more natural to + program this using 2 FireWorks, and have the first FireWork send an + instruction to quit or branch the workflow after its execution + through the FWAction object. + +- We might want the FireWorks codebase to compile runtime statistics + for us, and get separate reports for calculation 1 and calculation 2. + This cannot be done if everything is within a single FireWork – only + the overall stats for that FireWork will be reported. + +All these considerations lead to the conclusion that each executable job +should probably be run within its own FireWork. Let’s consider this +option in the next iteration of our Workflow. + +2.4.2 A prototypical Materials Science workflow – iteration 2 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If we put each of the two calculations within its own FireWork, the +Workflow will look like Figure 5: + +.. image:: mpworks/docs/f.png + +**Figure 5 Second iteration of how a Workflow could be written. This is +not the suggested way to do things.** + +In Figure 5, each calculation gets its own FireWork, which solves most +of the issues in our initial draft. For example, now each calculation +will get its own job at NERSC and its own walltime. If something goes +wrong during the second calculation, we can rerun just that calculation +without repeating calculation 1 using FireWork’s rerun features. + +However, splitting the Workflow also adds some complications: + +- The second calculation might need to know some information from the + first one. For example, it might need to know some of the results, or + might even need to know the directory where it output files so it can + copy some of them. As a user, you will need to explictly pass the + information you need between FireWorks using the FWAction object. + This complicates things. Two key pieces of information passed between + FireWorks in the MPWorks codebase are: + + - the directory where the previous job ran + + - the type of task of the previous job (“structure optimization”, + “static”, etc…) + +- By default, the calculations may run on different machines: the + FireWorks codebase runs each FireWork on whatever machine is + available. If you want to run the jobs on the same machine, or on + specific machines, you as a user will need to setup FireWorks to do + this explicitly (see the docs). Of course, you now also gain the + freedom to run the jobs on different machines (or the first available + machine) if this is what you’d like to do. + +These issues are all solvable, but require extra effort on the part of +the user to setup FireWorks correctly. In addition, our second iteration +has more pressing problems: + +- File movement and database insertion are performed at the end of a + FireWork. If the calculation doesn’t leave enough walltime for these + operations to complete, you might end up with an incomplete state + where file movement or database insertion is incomplete. + +- If database insertion fails due to a parse error, you cannot rerun + only database insertion (e.g., with a patched code). You must rerun + the entire FireWork (including the calculation part) + +- We might want to track stats like database insertion time or + calculation time separately within FireWorks + +For these reasons, it might make sense to separate these steps into +their own FireWork, so that you can be confident that these operations +will have their own walltime that you can set as high as you need and so +you can rerun these steps atomically as needed. This leads us to the +next (and final iteration) of the workflow. + +2.4.3 A prototypical Materials Science workflow – iteration 3 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this third (and final) iteration of the Workflow, both calculations +and file movement/database steps are given their own FireWorks (Figure +6): + +.. image:: mpworks/docs/g.png + +**Figure 6 Third iteration of how a Workflow could be written – both +calculations and database insertions are given their own FireWorks.** + +Now we finally have a situation where all our major tasks have their own +FireWork. They can be rerun independently, all get their own walltime +and resources, and are tracked and monitored independently. This is all +quite helpful, but we did add some complications: + +- We still need to make sure we pass all the necessary data between + FireWorks, and there is even more data passing going on now + +- We again need to set things up so each job runs on the appropriate + machine and with the right resources. For example, we might want the + database insertions to occur on a different machine (maybe even on + regular server without walltime rather than a supercomputer). If you + want to do anything other than “run any job on any machine”, you’ll + need to provide specifics to FireWorks + +- Each FireWork carries some overhead. For example, each FireWork has + its own run directory on the filesystem, and typically writes a few + files like FW.json in that directory. More FireWorks means more run + directories and more files written for this overhead (you can turn + off certain things like the FW.json in the configuration). + +All of these bullet points have solutions, but you may need to send a +message to the development list if you get stuck. + +2.5 Workflow implementations +---------------------------- + +Recall in the beginning of this Chapter, we stated that the goal was to +start with a crystal or molecule and end up with a Workflow (Figure 2). +Throughout this chapter, we’ve provided an introduction to some concepts +on how to achieve this. Now that you’re familiar with these concepts, +the next step is to look at some of the actual code used in MPWorks and +Rubicon to construct these workflows. + +2.5.1 A simple workflow to start with – Structure to Workflow +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + structure\_to\_wf() is located in **/mpworks/examples/wf\_ex.py** + +Recall that the goal is to start with a pymatgen *Structure* object and +transform it to a FireWorks *Workflow* object. You can then add that +*Workflow* to the FireWorks database and run it. + +A simple example to get started with is wf\_ex.py. You can try running +this method and seeing that a Workflow object is indeed produced +(“Si\_wf.json”). The key method is *structure\_to\_wf()*, which +transforms a structure into a workflow. This workflow uses the concepts +of the previous sections to build a simple Workflow that will optimize a +structure with VASP and then perform a static run on it, along with +database insertion after each step. Later in this manual, we will add +this Workflow to the FireWorks database and run it. Note that instead of +generating a file, we could also have directly used Python code to enter +the Workflow in the FireWorks database, but for simplicity in this +tutorial we will use the file method. + +Note that anytime you have *any* pymatgen Structure, you can now use the +s\ *tructure\_to\_wf()* to transform that into a Workflow file! + +2.5.2 MPWorks – SNL to Workflow +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + snl\_to\_wf() is located in **/mpworks/workflows/snl\_to\_wf.py** + +If you want to graduate to the big leagues, you can take a look at +*snl\_to\_wf()*. However, this method is significantly more complicated +than the previous one so we suggest that you skip it the first time +around. + +The MPWorks code that takes any crystal and creates a Workflow is +*snl\_to\_wf()*. Recall that SNL (or StructureNL) is just a pymatgen +Structure with some additional information attached (like authors, +references, tags, etc.). We suggest that you take a look at +*snl\_to\_wf()* now. + +Here is a rough guide to the current workflow: + +- The initial step adds the submission to an “SNL” database and does + some duplicate checking on the structure alone (insufficient to do + real duplicate checks which also involve VASP parameters) – *for now, + just ignore this step* + +- A GGA structure optimization job is created as the first step, just + like Figure 6 + +- A database insertion job is created as the second step, just like + Figure 6 + +- The third step is different; it is a “Controller” job that can + dynamically create more calculations depending on the output of the + first calculation. The current Controller creates static, Uniform, + and band structure calculations if the calculated gap from the + structure optimization is greater than 0.5 eV (see the code for + ControllerTask). + +- Finally, the second step forks into another branch for transition + metal oxides. For these systems, we run a GGA+U calculation that uses + the optimized structure of the GGA calculation as a starting point. + After the GGA+U calculation is another DB step, and another + controller step that creates GGA+U static, Uniform, and band + structure calculations. This GGA+U “branch” acts independently and + parallel to the GGA branch, after the initial structure optimization. + +3. Running Workflows at NERSC +============================= + +The main purpose of writing workflow code and using FireWorks was to +make it easy to run your jobs at supercomputing centers (there are also +other benefits, like having a database and built in FW web tool in which +you can query your jobs). This section will show you how to run your +jobs at NERSC. In particular, it will show you how to run your jobs in a +*personal environment* that will let you test and run your workflows. +While users have used the *personal environment* to do their own +research, one of the main goals of a personal environment is to allow +you to develop and test workflows so that they can be integrated into +the Materials Project production environment. + +3.1 Installing MPenv +-------------------- + +The MPenv codebase is used to automatically install one or more personal +environments into your user account at NERSC. Once you are up and +running with your environment, it is your job to manage it. Note that +your environment is a sandbox that does not interact with other +environments or the Materials Project production environment. This is +because it uses separate codebase copies and separate databases from +other environments. So, you can develop confidently and not be afraid of +messing something up in production. + + Note: make sure you’ve set up you’re environment to use 2 nodes + (mppwidth=48 on Hopper) in order to run VASP jobs. + +3.2 Environment system +---------------------- + +An *environment* is combination of: + +- A particular version of all the Materials Project codebases + (pymatgen, FireWorks, rubicon, etc) + +- A set of databases + +- Settings files, e.g. config files for FireWorks that choose a queue + to run on, default walltimes, etc. + +A user might have several environments, e.g. one for testing and one for +production, or one for each separate projects. Each environment is a +standalone sandbox for storing and running FireWorks computations with +the Materials Project infrastructure (e.g. pymatgen, custodian, etc). + +You can switch between environments by “activating” them through the +command “use\_\ **\ ”. For example, suppose you have two +environments, “test” and “prod”. If you type “use\_test” you will be in +the testing environment. Now, if you clear the FireWorks database, +nothing in the production environment will be affected. Only the test +environment will be affected. Similarly, if you change queue config +parameters in your test environment, your production environment will +continue running as before. + +.. image:: mpworks/docs/c.png + +**Figure 7 Activating an environment on the matcomp user chooses a set of +codebases (left) and databases (right) to use. You can then operate on +one environment without affecting the others.** + +When you *activate* an environment, the following happens (Figure 7): + +- A Python virtual environment is chosen and activated. If you activate + the test environment, for example, the code copies in the directory + *test/codes* will be used. (There are separate copies of the code for + each environment) + +- A set of standard environment variables are set, which give the paths + to database credentials and settings files. Because codes like + MPWorks use environment variables to determine what database to + connect to, by switching the environment variables we can connect to + different versions of a database. These settings files are located in + the “config” directory of your environment (e.g., *test/config*). + They contain credentials for the databases to connect to (e.g., + *test/config/dbs*) as well as the FireWorks settings to use (e.g., + *test/config/config\_Hopper*). + +To see how the environment variables are being modified to point to +different database and settings files based on the environment, look +inside your “.bashrc.ext” file. You’ll see a line like:: + + alias use_aj='source /global/u2/a/ajain/aj/virtenv_aj/bin/activate; + export FW_CONFIG_FILE=$FW_CONFIG_aj; export + DB_LOC=/global/u2/a/ajain/aj/config/dbs; export + VENV_LOC=/global/u2/a/ajain/aj/virtenv_aj/bin/activate; export + SCRIPT_LOC=/global/u2/a/ajain/aj/config/scripts; echo "You are in + environment aj."' + +As you can see, this line of code is for activating the “aj” environment +and sets several environment variables like FW\_CONFIG\_FILE, DB\_LOC, +VENV\_LOC, and SCRIPT\_LOC to environment-specific locations. The +MPWorks code uses these environment variables to dynamically shift what +databases and queue parameters are being used. + +3.3 Running workflows within the environment system +--------------------------------------------------- + +Now that you are all set up, the next step is to add some Workflows to +your personal FireWorks database and run it. + +3.3.1 Adding Workflows to your LaunchPad +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +3.3.1.1 Option 1: Add a simple MPWorks workflow +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Recall that we earlier created a simple Workflow object called +“Si\_wf.json” in Section 2.5.1. We can navigate to this file in the +MPWorks code on the NERSC machine by typing the following your prompt:: + + cd /codes/MPWorks/mpworks/examples + lpad add Si_wf.json + +Note: Make sure that the “mpworks.examples” directory is in your ADD\_USER\_PACKAGES option in your FWConfig.yaml file in/ configs/config\_XXXX) + +3.3.1.2 Option 2 : Add a workflow from your laptop +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you have a Workflow object already (called, for example, *my\_wf*), +you can add it to your testing environment using the following Python +code:: + + lp = LaunchPad.from_file(“my_launchpad.yaml”) + lp.add_wf(my_wf) + +where *my\_wf* is your workflow and my\_launchpad.yaml is the db file +from MPenv located in in */config/config\_Hopper*. You might +have to delete a few lines in my\_launchpad.yaml, e.g. ones relating to +log directories on your remote machine, to get it working. + +3.3.1.3 Option 3 : Use the submissions framework +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The MPWorks submissions framework will add a production-level workflow +from an SNL object using the snl\_to\_wf() function discussed in 2.5.2. +The sequence of events is as follows: + +- You use the SubmissionMongoAdapter to submit raw SNL objects to a + submissions database + +- You use the SubmissionProcessor to turn those submitted SNL to + Workflows that get added to the LaunchPad (this uses snl\_to\_wf() + under the hood) + +A schematic is shown below + +.. image:: mpworks/docs/b.png + +**Figure 8 Submissions framework** + +The nice thing about this method is that, once set up, all you need to +do is submit StructureNL or molecule objects and not worry about +Workflows or FireWorks. + +There are 4(!) main databases that interact within MPenv in this framework. You have credentials for these 4 databases in the MPenv files sent to you by the MPenv admin. + +1. The **FireWorks** database contains all the workflows that you want to run. + +2. The **VASP** database contains the results of your calculations + +3. There is also a **submissions** database where you can submit Structure objects (actually SNL objects) for computation. Using this database is optional but (as demonstrated later) can be simpler than trying to create FireWorks directly. + +4. Finally, there is an **SNL** database that contains all the structures you've submitted and relaxed. It is used for duplicate checking as well as record-keeping. Generally speaking, you do not need to do worry that this database exists. + +In the procedure above, you submit Structures to the **submissions** database, then use an *automated* command to convert those submissions into **FireWorks** workflows and run them. The results are checked via the **VASP** database. The order of operations is **submissions** -> **FireWorks** --> **VASP**, but your interaction is only with **submissions** and **VASP** databases. + +Here is some code you can use to submit a custom Structure to the **submissions** database (you will need to copy your ``/configs/db/submission_db.yaml`` file to the location you run this code, and also have set up your MPRester API key if you want to grab a structure from Materials Project as in this example):: + + from mpworks.submission.submission_mongo import SubmissionMongoAdapter + from pymatgen import MPRester + from pymatgen.matproj.snl import StructureNL + + submissions_file = 'submission_db.yaml' + sma = SubmissionMongoAdapter.from_file(submissions_file) + + # get a Structure object + mpr = MPRester() + s = mpr.get_structure_by_material_id("mp-149") # this is Silicon + + # At this point, you could modify the structure if you want. + + # create an SNL object and submit to your submissions database + snl = StructureNL(s, 'John Doe ') + sma.submit_snl(snl, 'my_email@gmail.com', parameters=None) + +This will only add a compound to the submissions database. It does not +yet create FireWorks to run. To create the FireWorks, you must: + +1. Log into NERSC + +2. Activate your desired test environment using ACTIVATE\_CMD, e.g. + “use\_test” + +3. Run the command: ``go_submissions`` + +The go\_submissions command will use snl\_to\_wf() to convert all your +SNL into FireWork workflows. After this, follow steps 6-10 in the previous section to complete running your workflows using FireWorks. + +There are many advanced options for setting priority, basic WF tailoring, auto-setting the submission database based on environment, etc. Consult the email list if you need help with a specific problem. + +3.3.1.4 Option 4 : Use the submissions framework + built-in test set +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Using the built in test set, you can create Workflows for 45 “test” +materials automatically. Like in the previous section, this method uses +the snl\_to\_wf() method to create Workflows. The difference is that a +set of about 45 compounds are pre-chosen and you don’t need to do any +work to create SNL files or Workflow files. This test follows the **submissions** -> **FireWorks** --> **VASP** paradigm. + +To use this method: + +1. Log into NERSC + +2. Activate your desired test environment using ACTIVATE\_CMD , e.g. + “use\_test” + +3. Note the following re-initializes/clears your database and adds a bunch of new submissions - run the command: ``go_testing --clear (!!warning, this clears your databases!!)``. (Note: You can run this command without the --clear option. +There is also a --name option to submit only single compound rather than all ~40 compounds.) + +4. If you want, you can at this point try connecting to your **submissions** database (e.g. via MongoHub) and confirm that you see compounds there. + +5. Run the command: ``go_submissions``. + + **Important:** Never run “go\_testing –clear” when in a production + environment! You will destroy all your results. + +6. You will see output saying that you have new workflows. This command *automatically* turned the new submissions into workflows in the **FireWorks** database that can can be run at NERSC. If you want, you can at this point try connecting to your **FireWorks** database (e.g. via MongoHub) and confirm that you see Workflows there. Or you can type ``lpad get_wflows -d less`` as another option to see what's in the FireWorks database. + +7. Let's run our FireWorks by navigating to a scratch directory and using the ``qlaunch`` command of FireWorks:: + + cd $GSCRATCH2 + mkdir first_tests + cd first_tests + qlaunch -r rapidfire --nlaunches infinite -m 50 --sleep 100 -b 10000 + +8. This should have submitted some jobs to the queues at NERSC. You should keep the qlaunch command running (or run it periodically) so that as workflow steps complete, new jobs can be submitted. + +9. You can check progress of your workflows using the built-in FireWorks monitoring tools. Several such tools, including a web gui, are documented in the FW docs. If you want to be efficient, you will actually look this up (as well as how to rerun jobs, detect failures, etc.). Here is a simple command you can use for basic checking:: + + lpad get_wflows -d more + +10. When your workflows complete, you should see the results in the **VASP** database (e.g. connect via MongoHub or via pymatgen-db frontend). + +3.3.1.5 - Running custom workflows +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The previous sections were about running custom *structures* through a typical MP workflow. If you want to run custom workflows (new types of calculations not coded in MP), you have a couple of options. You can either learn a bit more about MPWorks and try to code your workflow so that it can be run as in Part 3, but submitted with certain parameters (e.g., ``sma.submit_snl(snl, 'my_email@gmail.com', parameters={"calculation_type":"CUSTOM_STUFF"})``). This requires modifying the code that turns StructureNL into Workflows. In this case you are still following the **submissions** -> **FireWorks** --> **VASP** paradigm. + +The alternate strategy is to create Workflow objects directly and put them in the **FireWorks** database, bypassing the submissions database entirely. Then you are just doing **FireWorks** --> **VASP**. Once the Workflow objects are in the **FireWorks** database, you can run them by following steps 7-10 in Part 2 of this guide (i.e., basically you just need to run the ``qlaunch`` command. + +One code in development to create basic workflows that can run VASP is the **fireworks-vasp** repository (https://github.com/materialsvirtuallab/fireworks-vasp). This code can create Workflow objects that you can directly enter into your FireWorks database (the credentials for your FW database is in the ``my_launchpad.yaml`` given to you by the MPenv admin). This is not the code used by Materials Project for running workflows (MPWorks does that), but is considerably simpler to understand and modify for your needs. You can probably get started with custom workflows much more quickly with this strategy. + +3.3.2 Verifying your workflows +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To verify your workflows got entered: + +1. Log into NERSC + +2. Activate your desired test environment using ACTIVATE\_CMD , e.g. ``use_test`` + +3. Run the command ``lpad get_fws –d less`` + +(this will print all your workflows) + +For a more comprehensive list of query commands, see the FireWorks +documentation on Querying FireWorks and Workflows. For example, you can +count the number of Workflows completed over time. + +http://pythonhosted.org/FireWorks/query_tutorial.html + +3.3.3 Running the workflows +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Once your Workflows are in the LaunchPad, running them is really simple + +- Log into NERSC + +- Navigate to a SCRATCH directory, e.g. ``cd $SCRATCH/`` + +where is the directory you created when setting up your +environment (see Section 3.3.1) + +- Now that you are in the correct directory, type the command:: + + qlaunch -r rapidfire --nlaunches infinite -m 20 --sleep 100 -b 1000 + +(type qlaunch rapidfire –h if you want help on what the options mean). +This will start launching jobs to the queue . You will want to keep this +window alive as long as possible (or until all your workflows complete). +Unfortunately, this is difficult to do at NERSC as NERSC will timeout a +terminal after inactivity and close the connection. Another option +employed by Materials Project is to coordinate setting up a crontab with +NERSC to periodically run jobs. In this case you should set --nlaunches +to be 0 as this will prevent infinite looping of many queue launchers. + +3.3.4 Updating code +~~~~~~~~~~~~~~~~~~~ + +After activating the new environment, you can update all codes with the +command:: + + update_codes + +You can also update individual codebases:: + + cd /codes + cd MPWorks + git pull + python setup.py develop (this command is needed when version changes) + +3.4 Looking under the hood and concluding remarks +------------------------------------------------- + +You’ve now created a personal execution environment, added Workflows +using one of several methods, and executed the Workflowsusing FireWorks. +With this infrastructure, you should be able to automate as many jobs as +you need. + +However, even with this lengthy tutorial, there are many aspects of the +MP workflow that were not covered. The best way to ask questions is +through the Materials Project development group: + +https://groups.google.com/forum/#!forum/matproj-develop + +4. Give feedback! +================= + +This documentation, and the testing environment in general, are works in +progress. Despite best efforts, there might be typos and topics or +commands left out. Please give your feedback to improve this as a +reference for yourself and others. diff --git a/mpworks/.DS_Store b/mpworks/.DS_Store new file mode 100644 index 00000000..36d8a7f1 Binary files /dev/null and b/mpworks/.DS_Store differ diff --git a/mpworks/docs/MPWorks_docs_old.docx b/mpworks/docs/MPWorks_docs_old.docx new file mode 100755 index 00000000..b6dd29fe Binary files /dev/null and b/mpworks/docs/MPWorks_docs_old.docx differ diff --git a/mpworks/docs/MPWorks_docs_old.pdf b/mpworks/docs/MPWorks_docs_old.pdf new file mode 100755 index 00000000..cafb6e95 Binary files /dev/null and b/mpworks/docs/MPWorks_docs_old.pdf differ diff --git a/mpworks/docs/a.png b/mpworks/docs/a.png new file mode 100755 index 00000000..bb17188e Binary files /dev/null and b/mpworks/docs/a.png differ diff --git a/mpworks/docs/b.png b/mpworks/docs/b.png new file mode 100755 index 00000000..56e1a089 Binary files /dev/null and b/mpworks/docs/b.png differ diff --git a/mpworks/docs/c.png b/mpworks/docs/c.png new file mode 100755 index 00000000..3d83d8bd Binary files /dev/null and b/mpworks/docs/c.png differ diff --git a/mpworks/docs/d.png b/mpworks/docs/d.png new file mode 100755 index 00000000..961cbad8 Binary files /dev/null and b/mpworks/docs/d.png differ diff --git a/mpworks/docs/e.png b/mpworks/docs/e.png new file mode 100755 index 00000000..45aafc92 Binary files /dev/null and b/mpworks/docs/e.png differ diff --git a/mpworks/docs/f.png b/mpworks/docs/f.png new file mode 100755 index 00000000..8bcfdd0e Binary files /dev/null and b/mpworks/docs/f.png differ diff --git a/mpworks/docs/g.png b/mpworks/docs/g.png new file mode 100755 index 00000000..4c8da527 Binary files /dev/null and b/mpworks/docs/g.png differ diff --git a/mpworks/docs/h.png b/mpworks/docs/h.png new file mode 100755 index 00000000..25061772 Binary files /dev/null and b/mpworks/docs/h.png differ diff --git a/mpworks/docs/original/a.png b/mpworks/docs/original/a.png new file mode 100755 index 00000000..bb17188e Binary files /dev/null and b/mpworks/docs/original/a.png differ diff --git a/mpworks/docs/original/b.png b/mpworks/docs/original/b.png new file mode 100755 index 00000000..64c5fe24 Binary files /dev/null and b/mpworks/docs/original/b.png differ diff --git a/mpworks/docs/original/c.png b/mpworks/docs/original/c.png new file mode 100755 index 00000000..10a7bfea Binary files /dev/null and b/mpworks/docs/original/c.png differ diff --git a/mpworks/docs/original/d.png b/mpworks/docs/original/d.png new file mode 100755 index 00000000..7e66e102 Binary files /dev/null and b/mpworks/docs/original/d.png differ diff --git a/mpworks/docs/original/e.png b/mpworks/docs/original/e.png new file mode 100755 index 00000000..3f1cd71e Binary files /dev/null and b/mpworks/docs/original/e.png differ diff --git a/mpworks/docs/original/f.png b/mpworks/docs/original/f.png new file mode 100755 index 00000000..0ac39ab3 Binary files /dev/null and b/mpworks/docs/original/f.png differ diff --git a/mpworks/docs/original/g.png b/mpworks/docs/original/g.png new file mode 100755 index 00000000..404c0542 Binary files /dev/null and b/mpworks/docs/original/g.png differ diff --git a/mpworks/drones/README.md b/mpworks/drones/README.md new file mode 100755 index 00000000..8b16d2a4 --- /dev/null +++ b/mpworks/drones/README.md @@ -0,0 +1,9 @@ +# Drones package + + The MPVaspDrone is currently used by the production workflow. + +The drones package is an extension of the pymatgen-db drone, which converts a VASP directory into a database dictionary. The MPVaspDrone adds a "post_process" method and modifies some of the default drone behavior. It would be better if this could extend the existing drone rather than repeat a lot of the pymatgen-db drone, but it was not workable at the time of its creation. + +For example, the signal detectors help tag extra things that have might gone wrong with the run, and put it in the key analysis.signals and analysis.critical_signals. + +Another thing the custom drone does is SNL management. In particular, for structure optimizations it adds a new SNL to the SNL database (the newly optimized structure). For static runs (where the structure doesn't change), a new SNL is not added. The packages also add keys like "snlgroup_changed" which check whether the new and old SNL match after the relaxation run. \ No newline at end of file diff --git a/mpworks/drones/mp_vaspdrone.py b/mpworks/drones/mp_vaspdrone.py index 6660f5cc..34188f29 100644 --- a/mpworks/drones/mp_vaspdrone.py +++ b/mpworks/drones/mp_vaspdrone.py @@ -21,7 +21,7 @@ from pymatgen.entries.compatibility import MaterialsProjectCompatibility from pymatgen.entries.computed_entries import ComputedEntry from pymatgen.matproj.snl import StructureNL -from pymatgen.io.vaspio.vasp_output import Vasprun, Outcar +from pymatgen.io.vasp.outputs import Vasprun, Outcar from pymatgen.analysis.structure_analyzer import oxide_type @@ -245,7 +245,13 @@ def process_fw(self, dir_name, d): d['snlgroup_id'] = fw_dict['spec']['snlgroup_id'] d['vaspinputset_name'] = fw_dict['spec'].get('vaspinputset_name') d['task_type'] = fw_dict['spec']['task_type'] - + # Process data for deformed structures + if 'deformed' in d['task_type']: + d['deformation_matrix'] = fw_dict['spec']['deformation_matrix'] + d['original_task_id'] = fw_dict['spec']['original_task_id'] + if 'modified volume' in d['task_type']: + d['strainfactor'] = fw_dict['spec']['strainfactor'] + d['original_task_id'] = fw_dict['spec']['original_task_id'] if not self.update_duplicates: if 'optimize structure' in d['task_type'] and 'output' in d: # create a new SNL based on optimized structure diff --git a/mpworks/dupefinders/README.md b/mpworks/dupefinders/README.md new file mode 100755 index 00000000..90f8ffda --- /dev/null +++ b/mpworks/dupefinders/README.md @@ -0,0 +1,3 @@ +# Dupefinders package + + The Dupefinders are currently used by the production workflow. They are used to implement the duplicate checking within the production FireWorks. \ No newline at end of file diff --git a/mpworks/examples/README.md b/mpworks/examples/README.md new file mode 100755 index 00000000..a27225f9 --- /dev/null +++ b/mpworks/examples/README.md @@ -0,0 +1,3 @@ +# Examples package + + These are meant to be more simple examples of workflows than the production MP Workflows. They are referred to in the MPWorks docs. These are only examples meant as learning tools and are not used in production. The "fireworks-vasp" repo is a different attempt at this. \ No newline at end of file diff --git a/mpworks/examples/Si_wf.json b/mpworks/examples/Si_wf.json index f18c4748..702c9d88 100644 --- a/mpworks/examples/Si_wf.json +++ b/mpworks/examples/Si_wf.json @@ -1 +1 @@ -{"updated_on": "2013-11-06T00:46:51.196825", "metadata": {}, "name": "Si2", "links": {"1": [2], "3": [4], "2": [3], "4": []}, "fws": [{"fw_id": 1, "spec": {"vaspinputset_name": "DictVaspInputSet", "vasp": {"incar": {"MAGMOM": [0.6, 0.6], "NELM": 100, "ENCUT": 520, "@module": "pymatgen.io.vaspio.vasp_input", "PREC": "Accurate", "ISIF": 3, "ICHARG": 1, "IBRION": 2, "ALGO": "Fast", "LREAL": "Auto", "ISMEAR": -5, "ISPIN": 2, "EDIFF": 0.0001, "LWAVE": false, "NPAR": 2, "SIGMA": 0.2, "LORBIT": 11, "@class": "Incar", "NSW": 99}, "kpoints": {"comment": "pymatgen generated KPOINTS with grid density = 1000 / atom", "usershift": [0, 0, 0], "kpoints": [[8, 8, 8]], "@module": "pymatgen.io.vaspio.vasp_input", "nkpoints": 0, "generation_style": "Monkhorst", "@class": "Kpoints"}, "poscar": {"comment": "Si2", "selective_dynamics": null, "velocities": null, "@module": "pymatgen.io.vaspio.vasp_input", "@class": "Poscar", "predictor_corrector": null, "structure": {"lattice": {"a": 3.866, "c": 3.866, "b": 3.866, "matrix": [[3.3480542110306395, 0.0, 1.9330000000000005], [1.1160180703435467, 3.1565757818665885, 1.9330000000000005], [0.0, 0.0, 3.866]], "volume": 40.85738351924835, "beta": 59.99999999999999, "alpha": 59.99999999999999, "gamma": 59.99999999999999}, "sites": [{"label": "Si", "xyz": [0.5580090351717732, 0.39457197273332356, 0.9665000000000001], "abc": [0.125, 0.125, 0.125], "properties": {}, "species": [{"occu": 1, "element": "Si"}]}, {"label": "Si", "xyz": [3.9060632462024127, 2.762003809133265, 6.765500000000001], "abc": [0.875, 0.875, 0.875], "properties": {}, "species": [{"occu": 1, "element": "Si"}]}], "@class": "Structure", "@module": "pymatgen.core.structure"}, "true_names": true}, "potcar": {"symbols": ["Si"], "functional": "PBE", "@class": "Potcar", "@module": "pymatgen.io.vaspio.vasp_input"}}, "task_type": "GGA optimize structure (2x) example", "_tasks": [{"_fw_name": "Vasp Writer Task"}, {"jobs": [{"settings_override": null, "suffix": ".relax1", "auto_gamma": true, "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", "default_vasp_input_set": {"sort_structure": true, "hubbard_off": false, "constrain_total_magmom": false, "name": "MIT", "config_dict": {"INCAR": {"NELM": 100, "IBRION": 2, "LDAUTYPE": 2, "LWAVE": false, "SIGMA": 0.05, "MAGMOM": {"Mo": 5, "Ni": 2, "Mn4+": 3, "Co": 5, "Ni3+": 1, "Fe2+": 4, "Mn": 5, "Co3+": 0.6, "Fe3+": 5, "Mn2+": 5, "Fe": 5, "W": 5, "Mn3+": 4, "Ni4+": 0.6, "Cr": 5, "V": 5, "Fe4+": 4, "Co4+": 1, "Ta": 5}, "NELMIN": 6, "LDAUL": {"S": {"Mn": 2.5, "Fe": 2}, "O": {"Ni": 2, "Co": 2, "Ag": 2, "Mo": 2, "Mn": 2, "Re": 2, "Fe": 2, "W": 2, "V": 2, "Cr": 2, "Nb": 2, "Cu": 2, "Ta": 2}, "F": {"Ni": 2, "Co": 2, "Ag": 2, "Mo": 2, "Mn": 2, "Re": 2, "Fe": 2, "W": 2, "V": 2, "Cr": 2, "Nb": 2, "Cu": 2, "Ta": 2}}, "LDAUJ": {"S": {"Mn": 0, "Fe": 0}, "O": {"Ni": 0, "Co": 0, "Ag": 0, "Mo": 0, "Mn": 0, "Re": 0, "Fe": 0, "W": 0, "V": 0, "Cr": 0, "Nb": 0, "Cu": 0, "Ta": 0}, "F": {"Ni": 0, "Co": 0, "Ag": 0, "Mo": 0, "Mn": 0, "Re": 0, "Fe": 0, "W": 0, "V": 0, "Cr": 0, "Nb": 0, "Cu": 0, "Ta": 0}}, "ENCUT": 520, "ISIF": 3, "ICHARG": 1, "LDAUU": {"S": {"Mn": 2.5, "Fe": 1.9}, "O": {"Ni": 6, "Co": 3.4, "Ag": 1.5, "Mo": 4.38, "Mn": 3.9, "Re": 2, "Fe": 4.0, "W": 4.0, "V": 3.1, "Cr": 3.5, "Nb": 1.5, "Cu": 4, "Ta": 2}, "F": {"Ni": 6, "Co": 3.4, "Ag": 1.5, "Mo": 4.38, "Mn": 3.9, "Re": 2, "Fe": 4.0, "W": 4.0, "V": 3.1, "Cr": 3.5, "Nb": 1.5, "Cu": 4, "Ta": 2}}, "EDIFF": 5e-05, "NSW": 99, "LREAL": "AUTO", "ISMEAR": -5, "NPAR": 1, "PREC": "Accurate", "LDAU": true, "ALGO": "FAST", "ISPIN": 2, "LORBIT": "11"}, "KPOINTS": {"grid_density": 1000}, "POTCAR": {"Ru": "Ru_pv", "Re": "Re", "Rb": "Rb_pv", "Rh": "Rh", "Be": "Be", "Ba": "Ba_sv", "Bi": "Bi", "Br": "Br", "H": "H", "P": "P", "Os": "Os", "Ge": "Ge", "Gd": "Gd", "Ga": "Ga", "Pr": "Pr", "Pt": "Pt", "Pu": "Pu", "C": "C", "Pb": "Pb", "Pa": "Pa", "Pd": "Pd", "Cd": "Cd", "Pm": "Pm", "Ho": "Ho_3", "Hf": "Hf", "Hg": "Hg", "He": "He", "Mg": "Mg", "K": "K_sv", "Mn": "Mn", "O": "O", "S": "S", "W": "W_pv", "Zn": "Zn", "Eu": "Eu", "Zr": "Zr", "Er": "Er_3", "Ni": "Ni", "Na": "Na", "Nb": "Nb_pv", "Nd": "Nd", "Ne": "Ne", "Np": "Np", "Fe": "Fe", "B": "B", "F": "F", "Sr": "Sr_sv", "N": "N", "Kr": "Kr", "Si": "Si", "Sn": "Sn_d", "Sm": "Sm_3", "V": "V_pv", "Sc": "Sc_sv", "Sb": "Sb", "Se": "Se", "Co": "Co", "Cl": "Cl", "Ca": "Ca_sv", "Ce": "Ce", "Xe": "Xe", "Tm": "Tm_3", "Cs": "Cs_sv", "Cr": "Cr", "Cu": "Cu", "La": "La", "Li": "Li", "Tl": "Tl", "Lu": "Lu_3", "Th": "Th", "Ti": "Ti", "Te": "Te", "Tb": "Tb_3", "Tc": "Tc", "Ta": "Ta", "Yb": "Yb", "Dy": "Dy_3", "I": "I", "U": "U", "Y": "Y_sv", "Ac": "Ac", "Ag": "Ag", "Ir": "Ir", "Al": "Al", "As": "As", "Ar": "Ar", "Au": "Au", "In": "In", "Mo": "Mo_pv"}}, "@module": "pymatgen.io.vaspio_set", "@class": "DictVaspInputSet"}, "gamma_vasp_cmd": null, "vasp_cmd": "", "gzipped": false, "backup": true, "final": false, "@class": "VaspJob"}, {"settings_override": [{"action": {"_set": {"ISTART": 1}}, "dict": "INCAR"}, {"action": {"_file_copy": {"dest": "POSCAR"}}, "filename": "CONTCAR"}], "suffix": ".relax2", "auto_gamma": true, "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", "default_vasp_input_set": {"sort_structure": true, "hubbard_off": false, "constrain_total_magmom": false, "name": "MIT", "config_dict": {"INCAR": {"NELM": 100, "IBRION": 2, "LDAUTYPE": 2, "LWAVE": false, "SIGMA": 0.05, "MAGMOM": {"Mo": 5, "Ni": 2, "Mn4+": 3, "Co": 5, "Ni3+": 1, "Fe2+": 4, "Mn": 5, "Co3+": 0.6, "Fe3+": 5, "Mn2+": 5, "Fe": 5, "W": 5, "Mn3+": 4, "Ni4+": 0.6, "Cr": 5, "V": 5, "Fe4+": 4, "Co4+": 1, "Ta": 5}, "NELMIN": 6, "LDAUL": {"S": {"Mn": 2.5, "Fe": 2}, "O": {"Ni": 2, "Co": 2, "Ag": 2, "Mo": 2, "Mn": 2, "Re": 2, "Fe": 2, "W": 2, "V": 2, "Cr": 2, "Nb": 2, "Cu": 2, "Ta": 2}, "F": {"Ni": 2, "Co": 2, "Ag": 2, "Mo": 2, "Mn": 2, "Re": 2, "Fe": 2, "W": 2, "V": 2, "Cr": 2, "Nb": 2, "Cu": 2, "Ta": 2}}, "LDAUJ": {"S": {"Mn": 0, "Fe": 0}, "O": {"Ni": 0, "Co": 0, "Ag": 0, "Mo": 0, "Mn": 0, "Re": 0, "Fe": 0, "W": 0, "V": 0, "Cr": 0, "Nb": 0, "Cu": 0, "Ta": 0}, "F": {"Ni": 0, "Co": 0, "Ag": 0, "Mo": 0, "Mn": 0, "Re": 0, "Fe": 0, "W": 0, "V": 0, "Cr": 0, "Nb": 0, "Cu": 0, "Ta": 0}}, "ENCUT": 520, "ISIF": 3, "ICHARG": 1, "LDAUU": {"S": {"Mn": 2.5, "Fe": 1.9}, "O": {"Ni": 6, "Co": 3.4, "Ag": 1.5, "Mo": 4.38, "Mn": 3.9, "Re": 2, "Fe": 4.0, "W": 4.0, "V": 3.1, "Cr": 3.5, "Nb": 1.5, "Cu": 4, "Ta": 2}, "F": {"Ni": 6, "Co": 3.4, "Ag": 1.5, "Mo": 4.38, "Mn": 3.9, "Re": 2, "Fe": 4.0, "W": 4.0, "V": 3.1, "Cr": 3.5, "Nb": 1.5, "Cu": 4, "Ta": 2}}, "EDIFF": 5e-05, "NSW": 99, "LREAL": "AUTO", "ISMEAR": -5, "NPAR": 1, "PREC": "Accurate", "LDAU": true, "ALGO": "FAST", "ISPIN": 2, "LORBIT": "11"}, "KPOINTS": {"grid_density": 1000}, "POTCAR": {"Ru": "Ru_pv", "Re": "Re", "Rb": "Rb_pv", "Rh": "Rh", "Be": "Be", "Ba": "Ba_sv", "Bi": "Bi", "Br": "Br", "H": "H", "P": "P", "Os": "Os", "Ge": "Ge", "Gd": "Gd", "Ga": "Ga", "Pr": "Pr", "Pt": "Pt", "Pu": "Pu", "C": "C", "Pb": "Pb", "Pa": "Pa", "Pd": "Pd", "Cd": "Cd", "Pm": "Pm", "Ho": "Ho_3", "Hf": "Hf", "Hg": "Hg", "He": "He", "Mg": "Mg", "K": "K_sv", "Mn": "Mn", "O": "O", "S": "S", "W": "W_pv", "Zn": "Zn", "Eu": "Eu", "Zr": "Zr", "Er": "Er_3", "Ni": "Ni", "Na": "Na", "Nb": "Nb_pv", "Nd": "Nd", "Ne": "Ne", "Np": "Np", "Fe": "Fe", "B": "B", "F": "F", "Sr": "Sr_sv", "N": "N", "Kr": "Kr", "Si": "Si", "Sn": "Sn_d", "Sm": "Sm_3", "V": "V_pv", "Sc": "Sc_sv", "Sb": "Sb", "Se": "Se", "Co": "Co", "Cl": "Cl", "Ca": "Ca_sv", "Ce": "Ce", "Xe": "Xe", "Tm": "Tm_3", "Cs": "Cs_sv", "Cr": "Cr", "Cu": "Cu", "La": "La", "Li": "Li", "Tl": "Tl", "Lu": "Lu_3", "Th": "Th", "Ti": "Ti", "Te": "Te", "Tb": "Tb_3", "Tc": "Tc", "Ta": "Ta", "Yb": "Yb", "Dy": "Dy_3", "I": "I", "U": "U", "Y": "Y_sv", "Ac": "Ac", "Ag": "Ag", "Ir": "Ir", "Al": "Al", "As": "As", "Ar": "Ar", "Au": "Au", "In": "In", "Mo": "Mo_pv"}}, "@module": "pymatgen.io.vaspio_set", "@class": "DictVaspInputSet"}, "gamma_vasp_cmd": null, "vasp_cmd": "", "gzipped": false, "backup": false, "final": true, "@class": "VaspJob"}], "max_errors": 10, "_fw_name": "Vasp Custodian Task (Example)", "handlers": [{"output_filename": "vasp.out", "@class": "VaspErrorHandler", "@module": "custodian.vasp.handlers"}, {"timeout": 3600, "output_filename": "vasp.out", "@class": "FrozenJobErrorHandler", "@module": "custodian.vasp.handlers"}, {"output_vasprun": "vasprun.xml", "output_filename": "vasp.out", "@class": "MeshSymmetryErrorHandler", "@module": "custodian.vasp.handlers"}, {"change_algo": false, "nionic_steps": 10, "output_filename": "OSZICAR", "@class": "NonConvergingErrorHandler", "@module": "custodian.vasp.handlers"}]}]}, "created_on": "2013-11-06T00:46:51.195644", "name": "Si2--GGA_optimize_structure_(2x)_example"}, {"fw_id": 2, "spec": {"_tasks": [{"_fw_name": "Vasp to Database Task (Example)"}], "task_type": "VASP db insertion example"}, "created_on": "2013-11-06T00:46:51.195737", "name": "Si2--VASP_db_insertion_example"}, {"fw_id": 3, "spec": {"_tasks": [{"_fw_name": "Vasp Copy Task", "use_CONTCAR": true, "skip_CHGCAR": true}, {"_fw_name": "Setup Static Task"}, {"jobs": [{"settings_override": null, "suffix": "", "auto_gamma": true, "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", "default_vasp_input_set": {"sort_structure": true, "hubbard_off": false, "constrain_total_magmom": false, "name": "MIT", "config_dict": {"INCAR": {"NELM": 100, "IBRION": 2, "LDAUTYPE": 2, "LWAVE": false, "SIGMA": 0.05, "MAGMOM": {"Mo": 5, "Ni": 2, "Mn4+": 3, "Co": 5, "Ni3+": 1, "Fe2+": 4, "Mn": 5, "Co3+": 0.6, "Fe3+": 5, "Mn2+": 5, "Fe": 5, "W": 5, "Mn3+": 4, "Ni4+": 0.6, "Cr": 5, "V": 5, "Fe4+": 4, "Co4+": 1, "Ta": 5}, "NELMIN": 6, "LDAUL": {"S": {"Mn": 2.5, "Fe": 2}, "O": {"Ni": 2, "Co": 2, "Ag": 2, "Mo": 2, "Mn": 2, "Re": 2, "Fe": 2, "W": 2, "V": 2, "Cr": 2, "Nb": 2, "Cu": 2, "Ta": 2}, "F": {"Ni": 2, "Co": 2, "Ag": 2, "Mo": 2, "Mn": 2, "Re": 2, "Fe": 2, "W": 2, "V": 2, "Cr": 2, "Nb": 2, "Cu": 2, "Ta": 2}}, "LDAUJ": {"S": {"Mn": 0, "Fe": 0}, "O": {"Ni": 0, "Co": 0, "Ag": 0, "Mo": 0, "Mn": 0, "Re": 0, "Fe": 0, "W": 0, "V": 0, "Cr": 0, "Nb": 0, "Cu": 0, "Ta": 0}, "F": {"Ni": 0, "Co": 0, "Ag": 0, "Mo": 0, "Mn": 0, "Re": 0, "Fe": 0, "W": 0, "V": 0, "Cr": 0, "Nb": 0, "Cu": 0, "Ta": 0}}, "ENCUT": 520, "ISIF": 3, "ICHARG": 1, "LDAUU": {"S": {"Mn": 2.5, "Fe": 1.9}, "O": {"Ni": 6, "Co": 3.4, "Ag": 1.5, "Mo": 4.38, "Mn": 3.9, "Re": 2, "Fe": 4.0, "W": 4.0, "V": 3.1, "Cr": 3.5, "Nb": 1.5, "Cu": 4, "Ta": 2}, "F": {"Ni": 6, "Co": 3.4, "Ag": 1.5, "Mo": 4.38, "Mn": 3.9, "Re": 2, "Fe": 4.0, "W": 4.0, "V": 3.1, "Cr": 3.5, "Nb": 1.5, "Cu": 4, "Ta": 2}}, "EDIFF": 5e-05, "NSW": 99, "LREAL": "AUTO", "ISMEAR": -5, "NPAR": 1, "PREC": "Accurate", "LDAU": true, "ALGO": "FAST", "ISPIN": 2, "LORBIT": "11"}, "KPOINTS": {"grid_density": 1000}, "POTCAR": {"Ru": "Ru_pv", "Re": "Re", "Rb": "Rb_pv", "Rh": "Rh", "Be": "Be", "Ba": "Ba_sv", "Bi": "Bi", "Br": "Br", "H": "H", "P": "P", "Os": "Os", "Ge": "Ge", "Gd": "Gd", "Ga": "Ga", "Pr": "Pr", "Pt": "Pt", "Pu": "Pu", "C": "C", "Pb": "Pb", "Pa": "Pa", "Pd": "Pd", "Cd": "Cd", "Pm": "Pm", "Ho": "Ho_3", "Hf": "Hf", "Hg": "Hg", "He": "He", "Mg": "Mg", "K": "K_sv", "Mn": "Mn", "O": "O", "S": "S", "W": "W_pv", "Zn": "Zn", "Eu": "Eu", "Zr": "Zr", "Er": "Er_3", "Ni": "Ni", "Na": "Na", "Nb": "Nb_pv", "Nd": "Nd", "Ne": "Ne", "Np": "Np", "Fe": "Fe", "B": "B", "F": "F", "Sr": "Sr_sv", "N": "N", "Kr": "Kr", "Si": "Si", "Sn": "Sn_d", "Sm": "Sm_3", "V": "V_pv", "Sc": "Sc_sv", "Sb": "Sb", "Se": "Se", "Co": "Co", "Cl": "Cl", "Ca": "Ca_sv", "Ce": "Ce", "Xe": "Xe", "Tm": "Tm_3", "Cs": "Cs_sv", "Cr": "Cr", "Cu": "Cu", "La": "La", "Li": "Li", "Tl": "Tl", "Lu": "Lu_3", "Th": "Th", "Ti": "Ti", "Te": "Te", "Tb": "Tb_3", "Tc": "Tc", "Ta": "Ta", "Yb": "Yb", "Dy": "Dy_3", "I": "I", "U": "U", "Y": "Y_sv", "Ac": "Ac", "Ag": "Ag", "Ir": "Ir", "Al": "Al", "As": "As", "Ar": "Ar", "Au": "Au", "In": "In", "Mo": "Mo_pv"}}, "@module": "pymatgen.io.vaspio_set", "@class": "DictVaspInputSet"}, "gamma_vasp_cmd": null, "vasp_cmd": "", "gzipped": false, "backup": true, "final": true, "@class": "VaspJob"}], "max_errors": 10, "_fw_name": "Vasp Custodian Task (Example)", "handlers": [{"output_filename": "vasp.out", "@class": "VaspErrorHandler", "@module": "custodian.vasp.handlers"}, {"timeout": 3600, "output_filename": "vasp.out", "@class": "FrozenJobErrorHandler", "@module": "custodian.vasp.handlers"}, {"output_vasprun": "vasprun.xml", "output_filename": "vasp.out", "@class": "MeshSymmetryErrorHandler", "@module": "custodian.vasp.handlers"}, {"change_algo": false, "nionic_steps": 10, "output_filename": "OSZICAR", "@class": "NonConvergingErrorHandler", "@module": "custodian.vasp.handlers"}]}], "task_type": "GGA static example"}, "created_on": "2013-11-06T00:46:51.196656", "name": "Si2--GGA_static_example"}, {"fw_id": 4, "spec": {"_tasks": [{"_fw_name": "Vasp to Database Task (Example)"}], "task_type": "VASP db insertion example"}, "created_on": "2013-11-06T00:46:51.196737", "name": "Si2--VASP_db_insertion_example"}]} \ No newline at end of file +{"name": "Si2", "links": {"1": [2], "3": [4], "2": [3], "4": []}, "created_on": "2016-07-22T18:16:08.259942", "updated_on": "2016-07-22T18:16:08.259945", "fws": [{"updated_on": "2016-07-22T18:16:08.258840", "fw_id": 1, "spec": {"vaspinputset_name": "MPRelaxSet", "vasp": {"incar": {"MAGMOM": [0.6, 0.6], "ENCUT": 520, "NELM": 100, "NSW": 99, "PREC": "Accurate", "@module": "pymatgen.io.vasp.inputs", "ISIF": 3, "ICHARG": 1, "IBRION": 2, "GGAU": false, "LREAL": "Auto", "EDIFF": 0.0001, "ISPIN": 2, "ISMEAR": -5, "LWAVE": false, "NPAR": 2, "SIGMA": 0.05, "LORBIT": 11, "@class": "Incar", "ALGO": "Fast"}, "kpoints": {"comment": "Si2", "selective_dynamics": null, "velocities": null, "@module": "pymatgen.io.vasp.inputs", "@class": "Poscar", "predictor_corrector": null, "structure": {"lattice": {"a": 3.866, "c": 3.866, "b": 3.866, "matrix": [[3.3480542110306395, 0.0, 1.9330000000000005], [1.1160180703435467, 3.1565757818665885, 1.9330000000000005], [0.0, 0.0, 3.866]], "volume": 40.85738351924835, "beta": 59.99999999999999, "alpha": 59.99999999999999, "gamma": 59.99999999999999}, "sites": [{"xyz": [0.5580090351717732, 0.39457197273332356, 0.9665000000000001], "abc": [0.125, 0.125, 0.125], "species": [{"occu": 1, "element": "Si"}], "label": "Si"}, {"xyz": [3.9060632462024127, 2.762003809133265, 6.765500000000001], "abc": [0.875, 0.875, 0.875], "species": [{"occu": 1, "element": "Si"}], "label": "Si"}], "@class": "Structure", "@module": "pymatgen.core.structure"}, "true_names": true}, "poscar": {"comment": "Si2", "selective_dynamics": null, "velocities": null, "@module": "pymatgen.io.vasp.inputs", "@class": "Poscar", "predictor_corrector": null, "structure": {"lattice": {"a": 3.866, "c": 3.866, "b": 3.866, "matrix": [[3.3480542110306395, 0.0, 1.9330000000000005], [1.1160180703435467, 3.1565757818665885, 1.9330000000000005], [0.0, 0.0, 3.866]], "volume": 40.85738351924835, "beta": 59.99999999999999, "alpha": 59.99999999999999, "gamma": 59.99999999999999}, "sites": [{"xyz": [0.5580090351717732, 0.39457197273332356, 0.9665000000000001], "abc": [0.125, 0.125, 0.125], "species": [{"occu": 1, "element": "Si"}], "label": "Si"}, {"xyz": [3.9060632462024127, 2.762003809133265, 6.765500000000001], "abc": [0.875, 0.875, 0.875], "species": [{"occu": 1, "element": "Si"}], "label": "Si"}], "@class": "Structure", "@module": "pymatgen.core.structure"}, "true_names": true}, "potcar": {"comment": "Si2", "selective_dynamics": null, "velocities": null, "@module": "pymatgen.io.vasp.inputs", "@class": "Poscar", "predictor_corrector": null, "structure": {"lattice": {"a": 3.866, "c": 3.866, "b": 3.866, "matrix": [[3.3480542110306395, 0.0, 1.9330000000000005], [1.1160180703435467, 3.1565757818665885, 1.9330000000000005], [0.0, 0.0, 3.866]], "volume": 40.85738351924835, "beta": 59.99999999999999, "alpha": 59.99999999999999, "gamma": 59.99999999999999}, "sites": [{"xyz": [0.5580090351717732, 0.39457197273332356, 0.9665000000000001], "abc": [0.125, 0.125, 0.125], "species": [{"occu": 1, "element": "Si"}], "label": "Si"}, {"xyz": [3.9060632462024127, 2.762003809133265, 6.765500000000001], "abc": [0.875, 0.875, 0.875], "species": [{"occu": 1, "element": "Si"}], "label": "Si"}], "@class": "Structure", "@module": "pymatgen.core.structure"}, "true_names": true}}, "task_type": "GGA optimize structure (2x) example", "_tasks": [{"_fw_name": "Vasp Writer Task"}, {"jobs": [{"settings_override": null, "suffix": ".relax1", "auto_gamma": true, "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", "gamma_vasp_cmd": null, "vasp_cmd": "", "backup": true, "final": false, "@class": "VaspJob"}, {"settings_override": [{"action": {"_set": {"ISTART": 1, "EDIFFG": -0.05}}, "dict": "INCAR"}, {"action": {"_file_copy": {"dest": "POSCAR"}}, "file": "CONTCAR"}], "suffix": ".relax2", "auto_gamma": true, "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", "gamma_vasp_cmd": null, "vasp_cmd": "", "backup": false, "final": true, "@class": "VaspJob"}], "max_errors": 5, "_fw_name": "Vasp Custodian Task (Example)", "handlers": [{"output_filename": "vasp.out", "@class": "VaspErrorHandler", "@module": "custodian.vasp.handlers"}, {"timeout": 21600, "output_filename": "vasp.out", "@class": "FrozenJobErrorHandler", "@module": "custodian.vasp.handlers"}, {"output_vasprun": "vasprun.xml", "output_filename": "vasp.out", "@class": "MeshSymmetryErrorHandler", "@module": "custodian.vasp.handlers"}, {"change_algo": false, "nionic_steps": 10, "output_filename": "OSZICAR", "@class": "NonConvergingErrorHandler", "@module": "custodian.vasp.handlers"}]}]}, "created_on": "2016-07-22T18:16:08.258829", "name": "Si2--GGA_optimize_structure_(2x)_example"}, {"updated_on": "2016-07-22T18:16:08.259015", "fw_id": 2, "spec": {"_tasks": [{"_fw_name": "Vasp to Database Task (Example)"}], "task_type": "VASP db insertion example"}, "created_on": "2016-07-22T18:16:08.259011", "name": "Si2--VASP_db_insertion_example"}, {"updated_on": "2016-07-22T18:16:08.259659", "fw_id": 3, "spec": {"_tasks": [{"_fw_name": "Vasp Copy Task", "use_CONTCAR": true, "skip_CHGCAR": true}, {"_fw_name": "Setup Static Task"}, {"jobs": [{"settings_override": null, "suffix": "", "auto_gamma": true, "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", "gamma_vasp_cmd": null, "vasp_cmd": "", "backup": true, "final": true, "@class": "VaspJob"}], "max_errors": 5, "_fw_name": "Vasp Custodian Task (Example)", "handlers": [{"output_filename": "vasp.out", "@class": "VaspErrorHandler", "@module": "custodian.vasp.handlers"}, {"timeout": 21600, "output_filename": "vasp.out", "@class": "FrozenJobErrorHandler", "@module": "custodian.vasp.handlers"}, {"output_vasprun": "vasprun.xml", "output_filename": "vasp.out", "@class": "MeshSymmetryErrorHandler", "@module": "custodian.vasp.handlers"}, {"change_algo": false, "nionic_steps": 10, "output_filename": "OSZICAR", "@class": "NonConvergingErrorHandler", "@module": "custodian.vasp.handlers"}]}], "task_type": "GGA static example"}, "created_on": "2016-07-22T18:16:08.259656", "name": "Si2--GGA_static_example"}, {"updated_on": "2016-07-22T18:16:08.259800", "fw_id": 4, "spec": {"_tasks": [{"_fw_name": "Vasp to Database Task (Example)"}], "task_type": "VASP db insertion example"}, "created_on": "2016-07-22T18:16:08.259797", "name": "Si2--VASP_db_insertion_example"}], "metadata": {}} \ No newline at end of file diff --git a/mpworks/examples/firetasks_ex.py b/mpworks/examples/firetasks_ex.py index 18bb8823..39d6068a 100644 --- a/mpworks/examples/firetasks_ex.py +++ b/mpworks/examples/firetasks_ex.py @@ -26,9 +26,9 @@ def __init__(self, parameters): parameters = parameters if parameters else {} self.update(parameters) # get VaspJob objects from 'jobs' parameter in Firework - self.jobs = map(VaspJob.from_dict, parameters['jobs']) + self.jobs = parameters['jobs'] # get VaspHandler objects from 'handlers' parameter in Firework - self.handlers = map(MontyDecoder().process_decoded, parameters['handlers']) + self.handlers = parameters['handlers'] self.max_errors = parameters['max_errors'] def run_task(self, fw_spec): diff --git a/mpworks/examples/wf_ex.py b/mpworks/examples/wf_ex.py index 8e3fc88b..e28858be 100644 --- a/mpworks/examples/wf_ex.py +++ b/mpworks/examples/wf_ex.py @@ -9,7 +9,7 @@ from mpworks.firetasks.vasp_setup_tasks import SetupStaticRunTask from pymatgen import Composition, Lattice from pymatgen.core.structure import Structure -from pymatgen.io.vaspio_set import MPGGAVaspInputSet +from pymatgen.io.vasp.sets import MPRelaxSet __author__ = 'Anubhav Jain' __copyright__ = 'Copyright 2013, The Materials Project' @@ -34,11 +34,12 @@ def structure_to_wf(structure): connections = defaultdict(list) # dependencies between FireWorks # generate VASP input objects for 1st VASP run - this is put in the FW spec - mpvis = MPGGAVaspInputSet(user_incar_settings={'NPAR': 2}) - incar = mpvis.get_incar(structure) - poscar = mpvis.get_poscar(structure) - kpoints = mpvis.get_kpoints(structure) - potcar = mpvis.get_potcar(structure) + mpvis = MPRelaxSet(structure, user_incar_settings={'NPAR': 2, + "GGAU":False}) + incar = mpvis.incar + poscar = mpvis.poscar + kpoints = mpvis.poscar + potcar = mpvis.poscar # serialize the VASP input objects to the FW spec spec = {} @@ -51,7 +52,7 @@ def structure_to_wf(structure): spec['task_type'] = 'GGA optimize structure (2x) example' # set up the custodian that we want to run - jobs = VaspJob.double_relaxation_run('', gzipped=False) + jobs = VaspJob.double_relaxation_run('') for j in jobs: # turn off auto npar, it doesn't work for >1 node j.auto_npar = False handlers = [VaspErrorHandler(), FrozenJobErrorHandler(), MeshSymmetryErrorHandler(), diff --git a/mpworks/firetasks/README.md b/mpworks/firetasks/README.md new file mode 100755 index 00000000..2a1bc096 --- /dev/null +++ b/mpworks/firetasks/README.md @@ -0,0 +1,5 @@ +# Firetasks package + +This package is used by the production workflow. + +It contains implementations of FireTasks that are connected into workflows. \ No newline at end of file diff --git a/mpworks/firetasks/boltztrap_tasks.py b/mpworks/firetasks/boltztrap_tasks.py index dbf305f4..9538fdad 100644 --- a/mpworks/firetasks/boltztrap_tasks.py +++ b/mpworks/firetasks/boltztrap_tasks.py @@ -104,7 +104,7 @@ def get_extreme(self, d, target, maximize=True, iso_cutoff=0.05, max_didx=None): max_val = val max_temp = float(t) max_dope = d['doping'][te_type][didx] - max_mu = d['mu_doping'][te_type][t][didx] + max_mu = d['mu_doping'][te_type][str(t)][didx] isotropic = evs['isotropic'] data[te_type] = {'value': max_val, 'temperature': max_temp, 'doping': max_dope, 'mu': max_mu, 'isotropic': isotropic} @@ -121,7 +121,6 @@ def get_extreme(self, d, target, maximize=True, iso_cutoff=0.05, max_didx=None): def run_task(self, fw_spec): # import here to prevent import errors in bigger MPCollab - from mpcollab.thermoelectrics.boltztrap_TE import BoltztrapAnalyzerTE, BoltzSPB # get the band structure and nelect from files """ prev_dir = get_loc(fw_spec['prev_vasp_dir']) @@ -132,7 +131,7 @@ def run_task(self, fw_spec): bs = vr.get_band_structure(kpoints_filename=kpoints_loc) """ filename = get_slug( - 'JOB--' + fw_spec['mpsnl']['reduced_cell_formula_abc'] + '--' + 'JOB--' + fw_spec['mpsnl'].structure.composition.reduced_formula + '--' + fw_spec['task_type']) with open(filename, 'w+') as f: f.write('') @@ -177,6 +176,10 @@ def run_task(self, fw_spec): # put the data in the database bta = BoltztrapAnalyzer.from_files(dir) + + # 8/21/15 - Anubhav removed fs_id (also see line further below, ted['boltztrap_full_fs_id'] ...) + # 8/21/15 - this is to save space in MongoDB, as well as non-use of full Boltztrap output (vs rerun) + """ data = bta.as_dict() data.update(get_meta_from_structure(bs._structure)) data['snlgroup_id'] = fw_spec['snlgroup_id'] @@ -188,26 +191,25 @@ def run_task(self, fw_spec): del data['hall'] # remove because it is too large and not useful fs = gridfs.GridFS(tdb, "boltztrap_full_fs") btid = fs.put(json.dumps(jsanitize(data))) + """ # now for the "sanitized" data - te_analyzer = BoltztrapAnalyzerTE.from_BoltztrapAnalyzer(bta) - - ted = te_analyzer.as_dict() + ted = bta.as_dict() del ted['seebeck'] del ted['hall'] del ted['kappa'] del ted['cond'] - ted['boltztrap_full_fs_id'] = btid + # ted['boltztrap_full_fs_id'] = btid ted['snlgroup_id'] = fw_spec['snlgroup_id'] ted['run_tags'] = fw_spec['run_tags'] - ted['snl'] = fw_spec['mpsnl'] + ted['snl'] = fw_spec['mpsnl'].as_dict() ted['dir_name_full'] = dir ted['dir_name'] = get_block_part(dir) ted['task_id'] = m_task['task_id'] - ted['pf_doping'] = te_analyzer.get_power_factor(tau=self.TAU).as_dict() - ted['zt_doping'] = te_analyzer.get_ZT(kappal=self.KAPPAL, tau=self.TAU).as_dict() + ted['pf_doping'] = bta.get_power_factor(output='tensor', relaxation_time=self.TAU) + ted['zt_doping'] = bta.get_zt(output='tensor', relaxation_time=self.TAU, kl=self.KAPPAL) ted['pf_eigs'] = self.get_eigs(ted, 'pf_doping') ted['pf_best'] = self.get_extreme(ted, 'pf_eigs') @@ -231,7 +233,8 @@ def run_task(self, fw_spec): ted['kappa_best_dope19'] = self.get_extreme(ted, 'kappa_eigs', maximize=False, max_didx=4) try: - bzspb = BoltzSPB(te_analyzer) + from mpcollab.thermoelectrics.boltztrap_TE import BoltzSPB + bzspb = BoltzSPB(ted) maxpf_p = bzspb.get_maximum_power_factor('p', temperature=0, tau=1E-14, ZT=False, kappal=0.5,\ otherprops=('get_seebeck_mu_eig', 'get_conductivity_mu_eig', \ 'get_thermal_conductivity_mu_eig', 'get_average_eff_mass_tensor_mu')) @@ -278,8 +281,8 @@ def run_task(self, fw_spec): update_spec = {'prev_vasp_dir': fw_spec['prev_vasp_dir'], 'boltztrap_dir': os.getcwd(), 'prev_task_type': fw_spec['task_type'], - 'mpsnl': fw_spec['mpsnl'], + 'mpsnl': fw_spec['mpsnl'].as_dict(), 'snlgroup_id': fw_spec['snlgroup_id'], 'run_tags': fw_spec['run_tags'], 'parameters': fw_spec.get('parameters')} - return FWAction(update_spec=update_spec) \ No newline at end of file + return FWAction(update_spec=update_spec) diff --git a/mpworks/firetasks/controller_tasks.py b/mpworks/firetasks/controller_tasks.py index 1a83177f..55425211 100644 --- a/mpworks/firetasks/controller_tasks.py +++ b/mpworks/firetasks/controller_tasks.py @@ -55,7 +55,7 @@ def run_task(self, fw_spec): type_name = 'GGA+U' if 'GGA+U' in fw_spec['prev_task_type'] else 'GGA' - snl = StructureNL.from_dict(fw_spec['mpsnl']) + snl = fw_spec['mpsnl'] f = Composition(snl.structure.composition.reduced_formula).alphabetical_formula fws = [] @@ -154,7 +154,7 @@ def run_task(self, fw_spec): print 'Adding more runs...' type_name = 'GGA+U' if 'GGA+U' in fw_spec['prev_task_type'] else 'GGA' - snl = StructureNL.from_dict(fw_spec['mpsnl']) + snl = fw_spec['mpsnl'] f = Composition(snl.structure.composition.reduced_formula).alphabetical_formula fws = [] @@ -223,4 +223,4 @@ class DummyLegacyTask(FireTaskBase, FWSerializable): _fw_name = "Dummy Legacy Task" def run_task(self, fw_spec): - pass \ No newline at end of file + pass diff --git a/mpworks/firetasks/custodian_task.py b/mpworks/firetasks/custodian_task.py index c5a5e0e7..c48f11da 100644 --- a/mpworks/firetasks/custodian_task.py +++ b/mpworks/firetasks/custodian_task.py @@ -1,6 +1,8 @@ from gzip import GzipFile import logging import socket + +from fireworks.fw_config import FWData from monty.os.path import which from custodian.vasp.handlers import VaspErrorHandler, NonConvergingErrorHandler, \ FrozenJobErrorHandler, MeshSymmetryErrorHandler, PositiveEnergyErrorHandler @@ -12,9 +14,9 @@ import shlex import os from fireworks.utilities.fw_utilities import get_slug -from mpworks.workflows.wf_utils import j_decorate -from pymatgen.io.vaspio.vasp_input import Incar -from pymatgen.serializers.json_coders import MontyDecoder +from mpworks.workflows.wf_utils import j_decorate, ScancelJobStepTerminator +from pymatgen.io.vasp.inputs import Incar +from monty.json import MontyDecoder __author__ = 'Anubhav Jain' __copyright__ = 'Copyright 2013, The Materials Project' @@ -23,10 +25,15 @@ __email__ = 'ajain@lbl.gov' __date__ = 'Mar 15, 2013' + def check_incar(task_type): errors = [] incar = Incar.from_file("INCAR") + if 'deformed' in task_type: + if incar['ISIF'] != 2: + errors.append("Deformed optimization requires ISIF = 2") + if 'static' in task_type or 'Uniform' in task_type or 'band structure' in task_type: if incar["IBRION"] != -1: errors.append("IBRION should be -1 for non structure optimization runs") @@ -34,14 +41,14 @@ def check_incar(task_type): if "NSW" in incar and incar["NSW"] != 0: errors.append("NSW must be 0 for non structure optimization runs") - if 'static' in task_type and not incar["LCHARG"]: + if 'static' in task_type and not incar.get("LCHARG", True): errors.append("LCHARG must be True for static runs") - if 'Uniform' in task_type and incar["ICHARG"]!=11: - errors.append("ICHARG must be 11 for Uniform runs") + if 'Uniform' in task_type and incar["ICHARG"] != 11: + errors.append("ICHARG must be 11 for Uniform runs") - if 'band structure' in task_type and incar["ICHARG"]!=11: - errors.append("ICHARG must be 11 for band structure runs") + if 'band structure' in task_type and incar["ICHARG"] != 11: + errors.append("ICHARG must be 11 for band structure runs") if 'GGA+U' in task_type: # check LDAU @@ -62,7 +69,7 @@ class VaspCustodianTask(FireTaskBase, FWSerializable): def __init__(self, parameters): self.update(parameters) - self.jobs = map(VaspJob.from_dict, self['jobs']) + self.jobs = self['jobs'] dec = MontyDecoder() self.handlers = map(dec.process_decoded, self['handlers']) self.max_errors = self.get('max_errors', 1) @@ -85,23 +92,47 @@ def run_task(self, fw_spec): else: raise ValueError("No MPI command found!") - nproc = os.environ['PBS_NP'] - - v_exe = shlex.split('{} -n {} {}'.format(mpi_cmd, nproc, fw_env.get("vasp_cmd", "vasp"))) - gv_exe = shlex.split('{} -n {} {}'.format(mpi_cmd, nproc, fw_env.get("gvasp_cmd", "gvasp"))) + # TODO: last two env vars, i.e. SGE and LoadLeveler, are untested + env_vars = ['PBS_NP', 'SLURM_NTASKS', 'NSLOTS', 'LOADL_TOTAL_TASKS'] + nproc = None + for env_var in env_vars: + nproc = os.environ.get(env_var, None) + if nproc is not None: + break + if nproc is None: + raise ValueError("None of the env vars {} found to set nproc!".format(env_vars)) + + fw_data = FWData() + if (not fw_data.MULTIPROCESSING) or (fw_data.NODE_LIST is None): + if "srun" in mpi_cmd: + mpi_cmd += " -v" + v_exe = shlex.split('{} -n {} {}'.format(mpi_cmd, nproc, fw_env.get("vasp_cmd", "vasp"))) + gv_exe = shlex.split('{} -n {} {}'.format(mpi_cmd, nproc, fw_env.get("gvasp_cmd", "gvasp"))) + else: + v_exe, gv_exe = self._get_vasp_cmd_in_job_packing(fw_data, fw_env, mpi_cmd) - print 'host:', os.environ['HOSTNAME'] + print('host:', os.environ['HOSTNAME']) + stderr_file = "std_err.txt" for job in self.jobs: job.vasp_cmd = v_exe job.gamma_vasp_cmd = gv_exe + job.stderr_file = stderr_file + if v_exe[0] == "srun": + scancel_terminator = ScancelJobStepTerminator(stderr_file) + terminate_func = scancel_terminator.cancel_job_step + else: + terminate_func = None incar_errors = check_incar(fw_spec['task_type']) if incar_errors: raise ValueError("Critical error: INCAR does not pass checks: {}".format(incar_errors)) logging.basicConfig(level=logging.DEBUG) - c = Custodian(self.handlers, self.jobs, max_errors=self.max_errors, gzipped_output=False, validators=[VasprunXMLValidator()]) # manual gzip + + c = Custodian(self.handlers, self.jobs, max_errors=self.max_errors, gzipped_output=False, + validators=[VasprunXMLValidator()], + terminate_func=terminate_func) # manual gzip custodian_out = c.run() if self.gzip_output: @@ -127,10 +158,50 @@ def run_task(self, fw_spec): return FWAction(stored_data=stored_data, update_spec=update_spec) - def _write_formula_file(self, fw_spec): + @staticmethod + def _get_vasp_cmd_in_job_packing(fw_data, fw_env, mpi_cmd): + tasks_per_node_flag = {"srun": "--ntasks-per-node", + "mpirun": "--npernode", + "aprun": "-N"} + nodelist_flag = {"srun": "--nodelist", + "mpirun": "--host", + "aprun": "-L"} + ranks_num_flag = {"srun": "--ntasks", + "mpirun": "-n", + "aprun": "-n"} + nodes_spec = {"srun": "--nodes {}".format(len(fw_data.NODE_LIST)), + "mpirun": "", + "aprun": ""} + verbose_flag = {"srun": "-v", + "mpirun": "", + "aprun": ""} + mpirun = mpi_cmd.split()[0] + fw_data = FWData() + # Don't honor the SLURM_NTASKS in case of job packing, Because SLURM_NTASKS is referring + # to total number of processes of the parent job + sub_nproc = fw_data.SUB_NPROCS + vasp_cmds = [fw_env.get("vasp_cmd", "vasp"), fw_env.get("gvasp_cmd", "gvasp")] + vasp_exes = [shlex.split('{mpi_cmd} {verbose_flag} {nodes_spec} {ranks_flag} {nproc} {tpn_flag} {tpn} ' + '{nl_flag} {nl} {vasp_cmd}'. + format(mpi_cmd=mpi_cmd, + verbose_flag=verbose_flag, + nodes_spec=nodes_spec[mpirun], + ranks_flag=ranks_num_flag[mpirun], + nproc=sub_nproc, + tpn_flag=tasks_per_node_flag[mpirun], + tpn=int(fw_data.SUB_NPROCS) / len(fw_data.NODE_LIST), + nl_flag=nodelist_flag[mpirun], + nl=','.join(fw_data.NODE_LIST), + vasp_cmd=vasp_cmd)) + for vasp_cmd in vasp_cmds] + v_exe, gv_exe = vasp_exes + return v_exe, gv_exe + + @staticmethod + def _write_formula_file(fw_spec): filename = get_slug( - 'JOB--' + fw_spec['mpsnl']['reduced_cell_formula_abc'] + '--' - + fw_spec['task_type']) + 'JOB--' + fw_spec['mpsnl'].structure.composition.reduced_formula + + '--' + fw_spec['task_type']) with open(filename, 'w+') as f: f.write('') @@ -142,8 +213,8 @@ def get_custodian_task(spec): MeshSymmetryErrorHandler(), NonConvergingErrorHandler(), PositiveEnergyErrorHandler()] if 'optimize structure (2x)' in task_type: - jobs = VaspJob.double_relaxation_run(v_exe, gzipped=False) - elif 'static' in task_type: + jobs = VaspJob.double_relaxation_run(v_exe) + elif 'static' in task_type or 'deformed' in task_type: jobs = [VaspJob(v_exe)] else: # non-SCF runs diff --git a/mpworks/firetasks/elastic_tasks.py b/mpworks/firetasks/elastic_tasks.py new file mode 100755 index 00000000..33bd1a4f --- /dev/null +++ b/mpworks/firetasks/elastic_tasks.py @@ -0,0 +1,340 @@ +from monty.os.path import zpath +import os +import json +from pymongo import MongoClient +import numpy as np +from decimal import Decimal + +__author__ = 'Wei Chen' +__credits__ = 'Joseph Montoya' +__maintainer__ = 'Joseph Montoya ', + projects=["Elasticity"]) + tasks = [AddSNLTask()] + snl_priority = fw_spec.get('priority', 1) + spec = {'task_type': 'Add Deformed Struct to SNL database', + 'snl': snl.as_dict(), + '_queueadapter': QA_DB, + '_priority': snl_priority} + if 'snlgroup_id' in fw_spec and isinstance(snl, MPStructureNL): + spec['force_mpsnl'] = snl.as_dict() + spec['force_snlgroup_id'] = fw_spec['snlgroup_id'] + del spec['snl'] + fws.append(Firework(tasks, spec, + name=get_slug(f + '--' + spec['task_type']), + fw_id=-1000+i*10)) + connections[-1000+i*10] = [-999+i*10] + spec = snl_to_wf._snl_to_spec(snl, + parameters={'exact_structure':True}) + spec = update_spec_force_convergence(spec) + spec['deformation_matrix'] = d_struct_set.deformations[i].tolist() + spec['original_task_id'] = fw_spec["task_id"] + spec['_priority'] = fw_spec['_priority']*2 + #Turn off dupefinder for deformed structure + del spec['_dupefinder'] + spec['task_type'] = "Optimize deformed structure" + fws.append(Firework([VaspWriterTask(), SetupElastConstTask(), + get_custodian_task(spec)], + spec, + name=get_slug(f + '--' + spec['task_type']), + fw_id=-999+i*10)) + + priority = fw_spec['_priority']*3 + spec = {'task_type': 'VASP db insertion', + '_priority': priority, + '_allow_fizzled_parents': True, + '_queueadapter': QA_DB, + 'elastic_constant':"deformed_structure", + 'clean_task_doc':True, + 'deformation_matrix':d_struct_set.deformations[i].tolist(), + 'original_task_id':fw_spec["task_id"]} + fws.append(Firework([VaspToDBTask(), AddElasticDataToDBTask()], spec, + name=get_slug(f + '--' + spec['task_type']), + fw_id=-998+i*10)) + connections[-999+i*10] = [-998+i*10] + wf.append(Workflow(fws, connections)) + return FWAction(additions=wf) + + +class AddElasticDataToDBTask(FireTaskBase, FWSerializable): + _fw_name = "Add Elastic Data to DB" + + def run_task(self, fw_spec): + db_dir = os.environ['DB_LOC'] + db_path = os.path.join(db_dir, 'tasks_db.json') + i = fw_spec['original_task_id'] + + with open(db_path) as f: + db_creds = json.load(f) + connection = MongoClient(db_creds['host'], db_creds['port']) + tdb = connection[db_creds['database']] + tdb.authenticate(db_creds['admin_user'], db_creds['admin_password']) + tasks = tdb[db_creds['collection']] + elasticity = tdb['elasticity'] + ndocs = tasks.find({"original_task_id": i, + "state":"successful"}).count() + existing_doc = elasticity.find_one({"relaxation_task_id" : i}) + if existing_doc: + print "Updating: " + i + else: + print "New material: " + i + d = {"analysis": {}, "error": [], "warning": []} + d["ndocs"] = ndocs + o = tasks.find_one({"task_id" : i}, + {"pretty_formula" : 1, "spacegroup" : 1, + "snl" : 1, "snl_final" : 1, "run_tags" : 1}) + if not o: + raise ValueError("Cannot find original task id") + # Get stress from deformed structure + d["deformation_tasks"] = {} + ss_dict = {} + for k in tasks.find({"original_task_id": i}, + {"deformation_matrix":1, + "calculations.output":1, + "state":1, "task_id":1}): + defo = k['deformation_matrix'] + d_ind = np.nonzero(defo - np.eye(3)) + delta = Decimal((defo - np.eye(3))[d_ind][0]) + # Normal deformation + if d_ind[0] == d_ind[1]: + dtype = "_".join(["d", str(d_ind[0][0]), + "{:.0e}".format(delta)]) + # Shear deformation + else: + dtype = "_".join(["s", str(d_ind[0] + d_ind[1]), + "{:.0e}".format(delta)]) + sm = IndependentStrain(defo) + if dtype in d["deformation_tasks"].keys(): + print "old_task: {}".format(d["deformation_tasks"][dtype]["task_id"]) + print "new_task: {}".format(k["task_id"]) + raise ValueError("Duplicate deformation task in database.") + d["deformation_tasks"][dtype] = {"state" : k["state"], + "deformation_matrix" : defo, + "strain" : sm.tolist(), + "task_id": k["task_id"]} + if k["state"] == "successful": + st = Stress(k["calculations"][-1]["output"] \ + ["ionic_steps"][-1]["stress"]) + ss_dict[sm] = st + d["snl"] = o["snl"] + if "run_tags" in o.keys(): + d["run_tags"] = o["run_tags"] + for tag in o["run_tags"]: + if isinstance(tag, dict): + if "input_id" in tag.keys(): + d["input_mp_id"] = tag["input_id"] + d["snl_final"] = o["snl_final"] + d["pretty_formula"] = o["pretty_formula"] + + # Old input mp-id style + if o["snl"]["about"].get("_mp_id"): + d["material_id"] = o["snl"]["about"]["_mp_id"] + + # New style + elif "input_mp_id" in d: + d["material_id"] = d["input_mp_id"] + else: + d["material_id"] = None + d["relaxation_task_id"] = i + + calc_struct = Structure.from_dict(o["snl_final"]) + # TODO: + # JHM: This test is unnecessary at the moment, but should be redone + """ + conventional = is_conventional(calc_struct) + if conventional: + d["analysis"]["is_conventional"] = True + else: + d["analysis"]["is_conventional"] = False + """ + d["spacegroup"]=o.get("spacegroup", "Unknown") + + if ndocs >= 20: + # Perform Elastic tensor fitting and analysis + result = ElasticTensor.from_stress_dict(ss_dict) + d["elastic_tensor"] = result.voigt.tolist() + kg_average = result.kg_average + d.update({"K_Voigt":kg_average[0], "G_Voigt":kg_average[1], + "K_Reuss":kg_average[2], "G_Reuss":kg_average[3], + "K_Voigt_Reuss_Hill":kg_average[4], + "G_Voigt_Reuss_Hill":kg_average[5]}) + d["universal_anisotropy"] = result.universal_anisotropy + d["homogeneous_poisson"] = result.homogeneous_poisson + if ndocs < 24: + d["warning"].append("less than 24 tasks completed") + + # Perform filter checks + symm_t = result.voigt_symmetrized + d["symmetrized_tensor"] = symm_t.voigt.tolist() + d["analysis"]["not_rare_earth"] = True + for s in calc_struct.species: + if s.is_rare_earth_metal: + d["analysis"]["not_rare_earth"] = False + eigvals = np.linalg.eigvals(symm_t.voigt) + eig_positive = np.all((eigvals > 0) & np.isreal(eigvals)) + d["analysis"]["eigval_positive"] = bool(eig_positive) + c11 = symm_t.voigt[0][0] + c12 = symm_t.voigt[0][1] + c13 = symm_t.voigt[0][2] + c23 = symm_t.voigt[1][2] + d["analysis"]["c11_c12"]= not (abs((c11-c12)/c11) < 0.05 + or c11 < c12) + d["analysis"]["c11_c13"]= not (abs((c11-c13)/c11) < 0.05 + or c11 < c13) + d["analysis"]["c11_c23"]= not (abs((c11-c23)/c11) < 0.1 + or c11 < c23) + d["analysis"]["K_R"] = not (d["K_Reuss"] < 2) + d["analysis"]["G_R"] = not (d["G_Reuss"] < 2) + d["analysis"]["K_V"] = not (d["K_Voigt"] < 2) + d["analysis"]["G_V"] = not (d["G_Voigt"] < 2) + filter_state = np.all(d["analysis"].values()) + d["analysis"]["filter_pass"] = bool(filter_state) + d["analysis"]["eigval"] = list(eigvals) + + # TODO: + # JHM: eventually we can reintroduce the IEEE conversion + # but as of now it's not being used, and it should + # be in pymatgen + """ + # IEEE Conversion + try: + ieee_tensor = IEEE_conversion.get_ieee_tensor(struct_final, result) + d["elastic_tensor_IEEE"] = ieee_tensor[0].tolist() + d["analysis"]["IEEE"] = True + except Exception as e: + d["elastic_tensor_IEEE"] = None + d["analysis"]["IEEE"] = False + d["error"].append("Unable to get IEEE tensor: {}".format(e)) + """ + # Add thermal properties + nsites = calc_struct.num_sites + volume = calc_struct.volume + natoms = calc_struct.composition.num_atoms + weight = calc_struct.composition.weight + num_density = 1e30 * nsites / volume + mass_density = 1.6605e3 * nsites * volume * weight / \ + (natoms * volume) + tot_mass = sum([e.atomic_mass for e in calc_struct.species]) + avg_mass = 1.6605e-27 * tot_mass / natoms + y_mod = 9e9 * result.k_vrh * result.g_vrh / \ + (3. * result.k_vrh * result.g_vrh) + trans_v = 1e9 * result.k_vrh / mass_density**0.5 + long_v = 1e9 * result.k_vrh + \ + 4./3. * result.g_vrh / mass_density**0.5 + clarke = 0.87 * 1.3806e-23 * avg_mass**(-2./3.) * \ + mass_density**(1./6.) * y_mod**0.5 + cahill = 1.3806e-23 / 2.48 * num_density**(2./3.) * long_v + \ + 2 * trans_v + snyder_ac = 0.38483 * avg_mass * \ + (long_v + 2./3.*trans_v)**3. / \ + (300. * num_density**(-2./3.) * nsites**(1./3.)) + snyder_opt = 1.66914e-23 * (long_v + 2./3.*trans_v) / \ + num_density**(-2./3.) * \ + (1 - nsites**(-1./3.)) + snyder_total = snyder_ac + snyder_opt + debye = 2.489e-11 * avg_mass**(-1./3.) * \ + mass_density**(-1./6.) * y_mod**0.5 + + d["thermal"]={"num_density" : num_density, + "mass_density" : mass_density, + "avg_mass" : avg_mass, + "num_atom_per_unit_formula" : natoms, + "youngs_modulus" : y_mod, + "trans_velocity" : trans_v, + "long_velocity" : long_v, + "clarke" : clarke, + "cahill" : cahill, + "snyder_acou_300K" : snyder_ac, + "snyder_opt" : snyder_opt, + "snyder_total" : snyder_total, + "debye": debye + } + else: + d['state'] = "Fewer than 20 successful tasks completed" + return FWAction() + + if o["snl"]["about"].get("_kpoint_density"): + d["kpoint_density"]= o["snl"]["about"].get("_kpoint_density") + + if d["error"]: + raise ValueError("Elastic analysis failed: {}".format(d["error"])) + elif d["analysis"]["filter_pass"]: + d["state"] = "successful" + else: + d["state"] = "filter_failed" + elasticity.update({"relaxation_task_id": d["relaxation_task_id"]}, + d, upsert=True) + return FWAction() diff --git a/mpworks/firetasks/eos_thermal_tasks.py b/mpworks/firetasks/eos_thermal_tasks.py new file mode 100644 index 00000000..b6c1d1c6 --- /dev/null +++ b/mpworks/firetasks/eos_thermal_tasks.py @@ -0,0 +1,232 @@ +from monty.os.path import zpath +import os +import json +from pymongo import MongoClient +import numpy as np +from decimal import Decimal + +__author__ = 'Cormac Toher' + + +from fireworks.utilities.fw_serializers import FWSerializable +from fireworks.core.firework import FireTaskBase, FWAction +from pymatgen.io.vasp.inputs import Incar, Poscar, Kpoints +from pymatgen.analysis.eqn_of_state_thermal.eos_vasp_setup import ModifiedVolumeStructureSet +from pymatgen.analysis.eqn_of_state_thermal.eos_wf_run import eos_thermal_properties +from fireworks.core.firework import Firework, Workflow +from mpworks.firetasks.vasp_io_tasks import VaspWriterTask, VaspToDBTask +from mpworks.firetasks.custodian_task import get_custodian_task +from fireworks.utilities.fw_utilities import get_slug +from pymatgen import Composition +from pymatgen.matproj.snl import StructureNL +from mpworks.workflows import snl_to_wf +from mpworks.firetasks.snl_tasks import AddSNLTask +from mpworks.snl_utils.mpsnl import get_meta_from_structure, MPStructureNL +from pymatgen.core.structure import Structure +from mpworks.workflows.wf_settings import QA_VASP, QA_DB, QA_VASP_SMALL + +def update_spec_force_convergence(spec, user_vasp_settings=None): + fw_spec = spec + update_set = {"ENCUT": 700, "EDIFF": 0.000001, "ALGO":"N", "NPAR":2, "NSW":0} + if user_vasp_settings and user_vasp_settings.get("incar"): + update_set.update(user_vasp_settings["incar"]) + fw_spec['vasp']['incar'].update(update_set) + old_struct=Poscar.from_dict(fw_spec["vasp"]["poscar"]).structure + if user_vasp_settings and user_vasp_settings.get("kpoints"): + kpoints_density = user_vasp_settings["kpoints"]["kpoints_density"] + else: + kpoints_density = 7000 + k=Kpoints.automatic_density(old_struct, kpoints_density) + fw_spec['vasp']['kpoints'] = k.as_dict() + return fw_spec + + +class SetupFConvergenceTask(FireTaskBase, FWSerializable): + _fw_name = "Setup Force Convergence Task" + + def run_task(self, fw_spec): + incar = fw_spec['vasp']['incar'] + update_set = {"ENCUT": 600, "EDIFF": 0.00005} + incar.update(update_set) + kpoints = fw_spec['vasp']['kpoints'] + k = [int(round(2.5*k)) if int(round(2.5*k))%2 else int(round(2.5*k))+1 for k in kpoints['kpoints'][0]] + kpoints['kpoints'] = [k] + return FWAction() + +class SetupEoSThermalTask(FireTaskBase, FWSerializable): + _fw_name = "Setup EoS Thermal Task" + + def run_task(self, fw_spec): + incar = Incar.from_file(zpath("INCAR")) + incar.update({"NSW": 0}) + incar.update({"LCHARG": True}) + incar.update({"IBRION": -1}) + incar.write_file("INCAR") + return FWAction() + +class SetupModifiedVolumeStructTask(FireTaskBase, FWSerializable): + _fw_name = "Setup Modified Volume Struct Task" + + def run_task(self, fw_spec): + # Read structure from previous relaxation + relaxed_struct = fw_spec['output']['crystal'] + modified_struct_set = ModifiedVolumeStructureSet(relaxed_struct) + poisson_val = fw_spec['poisson_ratio'] + wf=[] + task_id_list=[] + for i, mod_struct in enumerate(modified_struct_set.modvol_structs): + fws=[] + connections={} + f = Composition(mod_struct.formula).alphabetical_formula + snl = StructureNL(mod_struct, 'Cormac Toher ',projects=["Thermal"]) + tasks = [AddSNLTask()] + snl_priority = fw_spec.get('priority', 1) + spec = {'task_type': 'Add Modified Struct to SNL database', 'snl': snl.as_dict(), + '_queueadapter': QA_DB, '_priority': snl_priority} + if 'snlgroup_id' in fw_spec and isinstance(snl, MPStructureNL): + spec['force_mpsnl'] = snl.as_dict() + spec['force_snlgroup_id'] = fw_spec['snlgroup_id'] + del spec['snl'] + fws.append(Firework(tasks, spec, name=get_slug(f + '--' + spec['task_type']), fw_id=-1000+i*10)) + connections[-1000+i*10] = [-999+i*10] + spec = snl_to_wf._snl_to_spec(snl, parameters={'exact_structure':True}) + spec = update_spec_force_convergence(spec) + spec['strainfactor'] = modified_struct_set.strainfactors[i] + spec['original_task_id']=fw_spec["task_id"] + spec['_priority'] = fw_spec['_priority']*2 + # Turn off dupefinder for modified structure + del spec['_dupefinder'] + spec['task_type'] = "Calculate static modified volume structure" + fws.append(Firework([VaspWriterTask(), SetupEoSThermalTask(), get_custodian_task(spec)], + spec, name=get_slug(f + '--' + fw_spec['task_type']), fw_id=-999+i*10)) + + priority = fw_spec['_priority']*3 + spec = {'task_type': 'VASP db insertion', '_priority': priority, + '_allow_fizzled_parents': True, '_queueadapter': QA_DB, 'poisson_ratio': poisson_val, + 'eqn_of_state_thermal':"modified_structure", 'clean_task_doc':True, + 'strainfactor':modified_struct_set.strainfactors[i], 'original_task_id':fw_spec["task_id"], 'original_task_id_list':task_id_list} + fws.append(Firework([VaspToDBTask(), AddEoSThermalDataToDBTask()], spec, name=get_slug(f + '--' + spec['task_type']), fw_id=-998+i*10)) + connections[-999+i*10] = [-998+i*10] + wf.append(Workflow(fws, connections)) + return FWAction(additions=wf) + + +class AddEoSThermalDataToDBTask(FireTaskBase, FWSerializable): + _fw_name = "Add EoS Thermal Data to DB Task" + + def run_task(self, fw_spec): + db_dir = os.environ['DB_LOC'] + db_path = os.path.join(db_dir, 'tasks_db.json') + poisson_val = fw_spec['poisson_ratio'] + strainfactor_val = fw_spec['strainfactor'] + task_id_list = fw_spec['original_task_id_list'] + i = fw_spec['original_task_id'] + j = fw_spec['task_id'] + + with open(db_path) as f: + db_creds = json.load(f) + connection = MongoClient(db_creds['host'], db_creds['port']) + tdb = connection[db_creds['database']] + tdb.authenticate(db_creds['admin_user'], db_creds['admin_password']) + tasks = tdb[db_creds['collection']] + eos_thermal = tdb['eos_thermal'] + ndocs = tasks.find({"original_task_id": i, + "state":"successful"}).count() + existing_doc = eos_thermal.find_one({"relaxation_task_id" : i}) + if existing_doc: + print "Updating: " + i + else: + print "New material: " + i + d = {"analysis": {}, "error": [], "warning": []} + d["ndocs"] = ndocs + o = tasks.find_one({"task_id" : i}, + {"pretty_formula" : 1, "spacegroup" : 1, + "snl" : 1, "snl_final" : 1, "run_tags" : 1}) + if not o: + raise ValueError("Cannot find original task id") + if o: + print("o exists") + d["strain_tasks"] = {} + volume_values = [] + energy_values = [] + for k in tasks.find({"original_task_id": i}, {"strainfactor":1, "calculations.output":1, "state":1, "task_id":1}): + kenerg_atom = k['calculations'][0]['output']['final_energy_per_atom'] + kvol = k['calculations'][0]['output']['crystal']['lattice']['volume'] + ksites = k['calculations'][0]['output']['crystal']['sites'] + knatoms = len(ksites) + kenerg = kenerg_atom * knatoms + energy_values.append(kenerg) + volume_values.append(kvol) + d["snl"] = o["snl"] + if "run_tags" in o.keys(): + d["run_tags"] = o["run_tags"] + for tag in o["run_tags"]: + if isinstance(tag, dict): + if "input_id" in tag.keys(): + d["input_mp_id"] = tag["input_id"] + d["snl_final"] = o["snl_final"] + d["pretty_formula"] = o["pretty_formula"] + + # Old input mp-id style + if o["snl"]["about"].get("_mp_id"): + d["material_id"] = o["snl"]["about"]["_mp_id"] + + # New style + elif "input_mp_id" in d: + d["material_id"] = d["input_mp_id"] + else: + d["material_id"] = None + d["relaxation_task_id"] = i + + calc_struct = Structure.from_dict(o["snl_final"]) + d["spacegroup"]=o.get("spacegroup", "Unknown") + + + if ndocs >= 21: + eos_thermal_dict = {} + # Perform thermal equation of state fitting and analysis + eos_thermal_properties_inst = eos_thermal_properties() + eos_thermal_dict = eos_thermal_properties_inst.eos_thermal_run(calc_struct, volume_values, energy_values, poissonratio=poisson_val) + + # Add equation of state results to dict + d["temperature"] = eos_thermal_dict["temperature"] + d["pressure"] = eos_thermal_dict["pressure"] + d["Thermal_conductivity_temp_list"] = eos_thermal_dict["thermal_conductivity"] + d["Debye_temperature_temp_list"] = eos_thermal_dict["Debye_temperature"] + d["Gruneisen_parameter_temp_list"] = eos_thermal_dict["Gruneisen_parameter"] + d["Heat_capacity_Cv_temp_list"] = eos_thermal_dict["Heat_capacity_Cv"] + d["Heat_capacity_Cp_temp_list"] = eos_thermal_dict["Heat_capacity_Cp"] + d["Volume_temp_list"] = eos_thermal_dict["Volume"] + d["Bulk_modulus_temp_list"] = eos_thermal_dict["Bulk_modulus"] + d["BM_coeffs"] = eos_thermal_dict["BM_coeffs"] + jtdbest = eos_thermal_dict["Best_fit_temperature"] + + # Add values at specific temperatures (Debye temperature and Gruneisen parameter best fit, values at 300K) + d["Debye_temperature"] = d["Debye_temperature_temp_list"][eos_thermal_dict["Best_fit_temperature"]] + d["Gruneisen_parameter"] = d["Gruneisen_parameter_temp_list"][eos_thermal_dict["Best_fit_temperature"]] + d["Thermal_conductivity_300K"] = d["Thermal_conductivity_temp_list"][eos_thermal_dict["300K_point"]] + d["Heat_capacity_Cv_300K"] = d["Heat_capacity_Cv_temp_list"][eos_thermal_dict["300K_point"]] + d["Heat_capacity_Cp_300K"] = d["Heat_capacity_Cp_temp_list"][eos_thermal_dict["300K_point"]] + d["Bulk_modulus_300K"] = d["Bulk_modulus_temp_list"][eos_thermal_dict["300K_point"]] + + + + else: + d['state'] = "Fewer than 21 successful tasks completed" + print("Fewer than 21 successful tasks completed") + print("ndocs = ", ndocs) + print("mp_id = ", i) + return FWAction() + + if o["snl"]["about"].get("_kpoint_density"): + d["kpoint_density"]= o["snl"]["about"].get("_kpoint_density") + + if d["error"]: + raise ValueError("Thermal equation of state analysis failed: {}".format(d["error"])) + elif d["analysis"]["filter_pass"]: + d["state"] = "successful" + else: + d["state"] = "filter_failed" + eos_thermal.update({"relaxation_task_id": d["relaxation_task_id"]}, + d, upsert=True) + return FWAction() diff --git a/mpworks/firetasks/snl_tasks.py b/mpworks/firetasks/snl_tasks.py index 69fb2db4..66ba9beb 100644 --- a/mpworks/firetasks/snl_tasks.py +++ b/mpworks/firetasks/snl_tasks.py @@ -21,7 +21,7 @@ class AddSNLTask(FireTaskBase, FWSerializable): def run_task(self, fw_spec): sma = SNLMongoAdapter.auto_load() - snl = StructureNL.from_dict(fw_spec['snl']) + snl = fw_spec['snl'] mpsnl, snlgroup_id, spec_group = sma.add_snl(snl) mod_spec = [{"_push": {"run_tags": "species_group={}".format(spec_group)}}] if spec_group else None diff --git a/mpworks/firetasks/vasp_io_tasks.py b/mpworks/firetasks/vasp_io_tasks.py index 03a59bd5..596fd136 100644 --- a/mpworks/firetasks/vasp_io_tasks.py +++ b/mpworks/firetasks/vasp_io_tasks.py @@ -23,7 +23,7 @@ from mpworks.workflows.wf_settings import QA_VASP, QA_DB, MOVE_TO_GARDEN_PROD, MOVE_TO_GARDEN_DEV from mpworks.workflows.wf_utils import last_relax, get_loc, move_to_garden from pymatgen import Composition -from pymatgen.io.vaspio.vasp_input import Incar, Poscar, Potcar, Kpoints +from pymatgen.io.vasp.inputs import Incar, Poscar, Potcar, Kpoints from pymatgen.matproj.snl import StructureNL __author__ = 'Anubhav Jain' @@ -42,10 +42,10 @@ class VaspWriterTask(FireTaskBase, FWSerializable): _fw_name = "Vasp Writer Task" def run_task(self, fw_spec): - Incar.from_dict(fw_spec['vasp']['incar']).write_file('INCAR') - Poscar.from_dict(fw_spec['vasp']['poscar']).write_file('POSCAR') - Potcar.from_dict(fw_spec['vasp']['potcar']).write_file('POTCAR') - Kpoints.from_dict(fw_spec['vasp']['kpoints']).write_file('KPOINTS') + fw_spec['vasp']['incar'].write_file('INCAR') + fw_spec['vasp']['poscar'].write_file('POSCAR') + fw_spec['vasp']['potcar'].write_file('POTCAR') + fw_spec['vasp']['kpoints'].write_file('KPOINTS') class VaspCopyTask(FireTaskBase, FWSerializable): @@ -159,16 +159,14 @@ def run_task(self, fw_spec): sh = logging.StreamHandler(stream=sys.stdout) sh.setLevel(getattr(logging, 'INFO')) logger.addHandler(sh) - with open(db_path) as f: db_creds = json.load(f) - drone = MPVaspDrone( - host=db_creds['host'], port=db_creds['port'], - database=db_creds['database'], user=db_creds['admin_user'], - password=db_creds['admin_password'], - collection=db_creds['collection'], parse_dos=parse_dos, - additional_fields=self.additional_fields, - update_duplicates=self.update_duplicates) + drone = MPVaspDrone(host=db_creds['host'], port=db_creds['port'], + database=db_creds['database'], user=db_creds['admin_user'], + password=db_creds['admin_password'], + collection=db_creds['collection'], parse_dos=parse_dos, + additional_fields=self.additional_fields, + update_duplicates=self.update_duplicates) t_id, d = drone.assimilate(prev_dir, launches_coll=LaunchPad.auto_load().launches) mpsnl = d['snl_final'] if 'snl_final' in d else d['snl'] @@ -180,6 +178,9 @@ def run_task(self, fw_spec): if d['state'] == 'successful': update_spec['analysis'] = d['analysis'] update_spec['output'] = d['output'] + update_spec['vasp']={'incar':d['calculations'][-1]['input']['incar'], + 'kpoints':d['calculations'][-1]['input']['kpoints']} + update_spec["task_id"]=t_id return FWAction(stored_data=stored_data, update_spec=update_spec) # not successful - first test to see if UnconvergedHandler is needed @@ -187,6 +188,7 @@ def run_task(self, fw_spec): unconverged_tag = 'unconverged_handler--{}'.format(fw_spec['prev_task_type']) output_dir = last_relax(os.path.join(prev_dir, 'vasprun.xml')) ueh = UnconvergedErrorHandler(output_filename=output_dir) + # TODO: make this a little more flexible if ueh.check() and unconverged_tag not in fw_spec['run_tags']: print 'Unconverged run! Creating dynamic FW...' @@ -198,7 +200,10 @@ def run_task(self, fw_spec): 'parameters': fw_spec.get('parameters'), '_dupefinder': DupeFinderVasp().to_dict(), '_priority': fw_spec['_priority']} - + # Pass elastic tensor spec + if 'deformation_matrix' in fw_spec.keys(): + spec['deformation_matrix'] = fw_spec['deformation_matrix'] + spec['original_task_id'] = fw_spec['original_task_id'] snl = StructureNL.from_dict(spec['mpsnl']) spec['run_tags'].append(unconverged_tag) spec['_queueadapter'] = QA_VASP @@ -218,6 +223,9 @@ def run_task(self, fw_spec): spec = {'task_type': 'VASP db insertion', '_allow_fizzled_parents': True, '_priority': fw_spec['_priority'], '_queueadapter': QA_DB, 'run_tags': list(fw_spec['run_tags'])} + if 'deformation_matrix' in fw_spec.keys(): + spec['deformation_matrix'] = fw_spec['deformation_matrix'] + spec['original_task_id'] = fw_spec['original_task_id'] spec['run_tags'].append(unconverged_tag) fws.append( Firework([VaspToDBTask()], spec, name=get_slug(f + '--' + spec['task_type']), @@ -229,4 +237,5 @@ def run_task(self, fw_spec): return FWAction(detours=wf) # not successful and not due to convergence problem - FIZZLE - raise ValueError("DB insertion successful, but don't know how to fix this Firework! Can't continue with workflow...") \ No newline at end of file + raise ValueError("DB insertion successful, but don't know how to \ + fix this Firework! Can't continue with workflow...") diff --git a/mpworks/firetasks/vasp_setup_tasks.py b/mpworks/firetasks/vasp_setup_tasks.py index 5d813cc8..c93f3486 100644 --- a/mpworks/firetasks/vasp_setup_tasks.py +++ b/mpworks/firetasks/vasp_setup_tasks.py @@ -3,11 +3,11 @@ from custodian.vasp.handlers import UnconvergedErrorHandler from fireworks.utilities.fw_serializers import FWSerializable from fireworks.core.firework import FireTaskBase, FWAction -from pymatgen.io.vaspio.vasp_output import Vasprun, Outcar -from pymatgen.io.vaspio.vasp_input import VaspInput, Incar, Poscar, Kpoints, Potcar -from pymatgen.io.vaspio_set import MPVaspInputSet, MPStaticVaspInputSet, \ - MPNonSCFVaspInputSet +from pymatgen.io.vasp.outputs import Vasprun, Outcar +from pymatgen.io.vasp.inputs import VaspInput, Incar, Poscar, Kpoints, Potcar +from pymatgen.io.vasp.sets import MPRelaxSet, MPStaticSet, MPNonSCFSet from pymatgen.symmetry.bandstructure import HighSymmKpath +from pymatgen.symmetry.analyzer import SpacegroupAnalyzer __author__ = 'Wei Chen, Anubhav Jain' __copyright__ = 'Copyright 2013, The Materials Project' @@ -39,17 +39,21 @@ def __init__(self, parameters=None): def run_task(self, fw_spec): self.user_incar_settings.update({"NPAR": 2}) - - MPStaticVaspInputSet.from_previous_vasp_run(os.getcwd(), - user_incar_settings=self.user_incar_settings, kpoints_density=self.kpoints_density) - structure = MPStaticVaspInputSet.get_structure(Vasprun(zpath("vasprun.xml")), Outcar(zpath("OUTCAR")), - initial_structure=False, - additional_info=True) - - return FWAction(stored_data={'refined_structure': structure[1][0].as_dict(), - 'conventional_standard_structure': structure[1][1].as_dict(), - 'symmetry_dataset': structure[1][2], - 'symmetry_operations': [x.as_dict() for x in structure[1][3]]}) + # Get kpoint density per vol + vol = Poscar.from_file("POSCAR").structure.volume + kppra_vol = self.kpoints_density / vol + new_set = MPStaticSet.from_prev_calc( + os.getcwd(), + user_incar_settings=self.user_incar_settings, + reciprocal_density=kppra_vol) + new_set.write_input('.') + structure = new_set.structure + sga = SpacegroupAnalyzer(structure, 0.1) + return FWAction(stored_data={ + 'refined_structure': sga.get_refined_structure().as_dict(), + 'conventional_standard_structure': sga.get_conventional_standard_structure().as_dict(), + 'symmetry_dataset': sga.get_symmetry_dataset(), + 'symmetry_operations': [x.as_dict() for x in sga.get_symmetry_operations()]}) class SetupUnconvergedHandlerTask(FireTaskBase, FWSerializable): @@ -86,15 +90,22 @@ def __init__(self, parameters=None): def run_task(self, fw_spec): user_incar_settings= {"NPAR": 2} + vol = Poscar.from_file("POSCAR").structure.volume + kppra_vol = self.kpoints_density / vol if self.line: - MPNonSCFVaspInputSet.from_previous_vasp_run(os.getcwd(), mode="Line", copy_chgcar=False, - user_incar_settings=user_incar_settings, kpoints_line_density=self.kpoints_line_density) + MPNonSCFSet.from_prev_calc( + os.getcwd(), mode="Line", copy_chgcar=False, + user_incar_settings=user_incar_settings, + kpoints_line_density=self.kpoints_line_density).write_input('.') kpath = HighSymmKpath(Poscar.from_file("POSCAR").structure) + return FWAction(stored_data={"kpath": kpath.kpath, "kpath_name": kpath.name}) else: - MPNonSCFVaspInputSet.from_previous_vasp_run(os.getcwd(), mode="Uniform", copy_chgcar=False, - user_incar_settings=user_incar_settings, kpoints_density=self.kpoints_density) + MPNonSCFSet.from_prev_calc( + os.getcwd(), mode="Uniform", copy_chgcar=False, + user_incar_settings=user_incar_settings, + reciprocal_density=kppra_vol).write_input('.') return FWAction() @@ -114,8 +125,8 @@ def run_task(self, fw_spec): # figure out what GGA+U values to use and override them # LDAU values to use - mpvis = MPVaspInputSet() - ggau_incar = mpvis.get_incar(poscar.structure).as_dict() + mpvis = MPRelaxSet(poscar.structure) + ggau_incar = mpvis.incar.as_dict() incar_updates = {k: ggau_incar[k] for k in ggau_incar.keys() if 'LDAU' in k} for k in ggau_incar: @@ -133,4 +144,4 @@ def run_task(self, fw_spec): # write back the new INCAR to the current directory incar.write_file('INCAR') - return FWAction(stored_data={'chgcar_start': chgcar_start}) \ No newline at end of file + return FWAction(stored_data={'chgcar_start': chgcar_start}) diff --git a/mpworks/firetasks_staging/__init__.py b/mpworks/firetasks_staging/__init__.py new file mode 100755 index 00000000..ca35517e --- /dev/null +++ b/mpworks/firetasks_staging/__init__.py @@ -0,0 +1 @@ +__author__ = 'ajain' diff --git a/mpworks/firetasks_staging/surface_tasks.py b/mpworks/firetasks_staging/surface_tasks.py new file mode 100755 index 00000000..7ea82491 --- /dev/null +++ b/mpworks/firetasks_staging/surface_tasks.py @@ -0,0 +1,465 @@ +## for Surface Energy Calculation +from __future__ import division, unicode_literals + +__author__ = "Zihan XU" +__version__ = "0.1" +__email__ = "vivid0036@gmail.com" +__date__ = "6/2/15" + +import os +import numpy as np + +from fireworks.core.firework import FireTaskBase, FWAction, Firework, Workflow +from fireworks import explicit_serialize +from pymatgen.io.vasp.outputs import Vasprun, Poscar, Incar, Outcar +from custodian.custodian import Custodian +from custodian.vasp.jobs import VaspJob +from matgendb.creator import VaspToDbTaskDrone + +from pymatgen.core.surface import SlabGenerator +from monty.json import MontyDecoder +from pymatgen.io.vaspio_metal_slabs import MPSlabVaspInputSet +from matgendb import QueryEngine + +""" +Firework tasks +""" + + +@explicit_serialize +class VaspSlabDBInsertTask(FireTaskBase): + + """ + Inserts a single vasp calculation in a folder into a + DB. Also inserts useful information pertaining + to slabs and oriented unit cells. + """ + + required_params = ["host", "port", "user", "password", + "database", "collection", "struct_type", "loc", + "cwd", "miller_index"] + optional_params = ["surface_area", "shift", "vsize", "ssize"] + + def run_task(self, fw_spec): + + """ + Required Parameters: + host (str): See SurfaceWorkflowManager in surface_wf.py + port (int): See SurfaceWorkflowManager in surface_wf.py + user (str): See SurfaceWorkflowManager in surface_wf.py + password (str): See SurfaceWorkflowManager in surface_wf.py + database (str): See SurfaceWorkflowManager in surface_wf.py + collection (str): See SurfaceWorkflowManager in surface_wf.py + struct_type (str): either oriented_unit_cell or slab_cell + miller_index (list): Miller Index of the oriented + unit cell or slab + loc (str path): Location of the outputs of + the vasp calculations + Optional Parameters: + surface_area (float): surface area of the slab, obtained + from slab object before relaxation + shift (float): A shift value in Angstrom that determines how + much a slab should be shifted. For determining number of + terminations, obtained from slab object before relaxation + vsize (float): Size of vacuum layer of slab in Angstroms, + obtained from slab object before relaxation + ssize (float): Size of slab layer of slab in Angstroms, + obtained from slab object before relaxation + """ + + dec = MontyDecoder() + struct_type = dec.process_decoded(self.get("struct_type")) + loc = dec.process_decoded(self.get("loc")) + cwd = dec.process_decoded(self.get("cwd")) + surface_area = dec.process_decoded(self.get("surface_area", None)) + shift = dec.process_decoded(self.get("shift", None)) + vsize = dec.process_decoded(self.get("vsize", None)) + ssize = dec.process_decoded(self.get("ssize", None)) + miller_index = dec.process_decoded(self.get("miller_index")) + + # Sets default for DB parameters + if not self["host"]: + self["host"] = "127.0.0.1" + + if not self["port"]: + self["port"] = 27017 + + if not self["database"]: + self["database"] = "vasp" + + if not self["collection"]: + self["collection"] = "tasks" + + # Addtional info relating to slabs + additional_fields={"author": os.environ.get("USER"), + "structure_type": struct_type, + "miller_index": miller_index, + "surface_area": surface_area, "shift": shift, + "vac_size": vsize, "slab_size": ssize} + + drone = VaspToDbTaskDrone(host=self["host"], port=self["port"], + user=self["user"], + password=self["password"], + database=self["database"], + use_full_uri=False, + additional_fields=additional_fields, + collection=self["collection"]) + drone.assimilate(cwd+loc) + # print loc + # print self["collection"] + # print additional_fields['vsize'] + # print additional_fields['miller_index'] + + +@explicit_serialize +class WriteUCVaspInputs(FireTaskBase): + """ + Writes VASP inputs for an oriented unit cell + """ + + required_params = ["oriented_ucell", "folder", "cwd"] + optional_params = ["angle_tolerance", "user_incar_settings", + "k_product", "potcar_functional", "symprec"] + + def run_task(self, fw_spec): + + """ + Required Parameters: + oriented_unit_cell (Structure): Generated with surface.py + folder (str path): Location where vasp inputs + are to be written + Optional Parameters: + angle_tolerance (int): See SpaceGroupAnalyzer in analyzer.py + user_incar_settings (dict): See launch_workflow() method in + CreateSurfaceWorkflow class + k_product (dict): See launch_workflow() method in + CreateSurfaceWorkflow class + potcar_functional (dict): See launch_workflow() method in + CreateSurfaceWorkflow class + symprec (float): See SpaceGroupAnalyzer in analyzer.py + """ + dec = MontyDecoder() + oriented_ucell = dec.process_decoded(self.get("oriented_ucell")) + folder = dec.process_decoded(self.get("folder")) + cwd = dec.process_decoded(self.get("cwd")) + symprec = dec.process_decoded(self.get("symprec", 0.001)) + angle_tolerance = dec.process_decoded(self.get("angle_tolerance", 5)) + + user_incar_settings = \ + dec.process_decoded(self.get("user_incar_settings", + MPSlabVaspInputSet().incar_settings)) + k_product = \ + dec.process_decoded(self.get("k_product", 50)) + potcar_functional = \ + dec.process_decoded(self.get("potcar_fuctional", 'PBE')) + + # Will continue an incomplete job from a previous contcar file if it exists + print 'cwd is %s' %(os.getcwd()) + print 'the folder is %s' %(folder) + print os.path.join(os.getcwd(), folder) + print cwd+'/'+folder + path = cwd+'/'+folder + + # path = os.path.join(os.getcwd(), folder) + newfolder = os.path.join(path, 'prev_run') + + # print 'check if conditions for continuing calculations have been satisfied' + # print 'check for the following path: %s' %(path) + # print os.path.exists(path) + # print os.path.exists(os.path.join(path, 'CONTCAR.gz')) + # print os.stat(os.path.join(path, 'CONTCAR.gz')).st_size !=0 + + def continue_vasp(contcar): + print folder, 'already exists, will now continue calculation' + print 'making prev_run folder' + os.system('mkdir %s' %(newfolder)) + print 'moving outputs to prev_run' + os.system('mv %s/* %s/prev_run' %(path, path)) + print 'moving outputs as inputs for next calculation' + os.system('cp %s/%s %s/INCAR %s/POTCAR %s/KPOINTS %s' + %(newfolder, contcar, newfolder, newfolder, newfolder, path)) + print 'unzipping new inputs' + os.system('gunzip %s/*' %(path)) + print 'copying contcar as new poscar' + if contcar == 'CONTCAR.relax1.gz': + os.system('mv %s/CONTCAR.relax1 %s/POSCAR' %(path , path)) + else: + os.system('mv %s/CONTCAR %s/POSCAR' %(path , path)) + + + if os.path.exists(path) and \ + os.path.exists(os.path.join(path, 'CONTCAR')) and \ + os.stat(os.path.join(path, 'CONTCAR')).st_size !=0: + continue_vasp('CONTCAR') + elif os.path.exists(path) and \ + os.path.exists(os.path.join(path, 'CONTCAR.gz')) \ + and os.stat(os.path.join(path, 'CONTCAR.gz')).st_size !=0: + continue_vasp('CONTCAR.gz') + elif os.path.exists(path) and \ + os.path.exists(os.path.join(path, 'CONTCAR.relax1.gz')) and \ + os.stat(os.path.join(path, 'CONTCAR.relax1.gz')).st_size !=0: + continue_vasp('CONTCAR.relax1.gz') + + else: + + mplb = MPSlabVaspInputSet(user_incar_settings=user_incar_settings, + k_product=k_product, bulk=True, + potcar_functional=potcar_functional, + ediff_per_atom=False) + mplb.write_input(oriented_ucell, cwd+folder) + + +@explicit_serialize +class WriteSlabVaspInputs(FireTaskBase): + """ + Adds dynamicism to the workflow by creating addition Fireworks for each + termination of a slab or just one slab with shift=0. First the vasp + inputs of a slab is created, then the Firework for that specific slab + is made with a RunCustodianTask and a VaspSlabDBInsertTask + """ + required_params = ["folder", "cwd", "custodian_params", + "vaspdbinsert_parameters", "miller_index"] + optional_params = ["min_slab_size", "min_vacuum_size", + "angle_tolerance", "user_incar_settings", + "k_product","potcar_functional", "symprec", + "terminations"] + + def run_task(self, fw_spec): + + """ + Required Parameters: + folder (str path): Location where vasp inputs + are to be written + custodian_params (dict **kwargs): Contains the job and the + scratch directory for a custodian run + vaspdbinsert_parameters (dict **kwargs): Contains + informations needed to acess a DB, eg, host, + port, password etc. + Optional Parameters: + min_vac_size (float): Size of vacuum layer of slab in Angstroms + min_slab_size (float): Size of slab layer of slab in Angstroms + angle_tolerance (int): See SpaceGroupAnalyzer in analyzer.py + user_incar_settings (dict): See launch_workflow() method in + CreateSurfaceWorkflow class + k_product (dict): See launch_workflow() method in + CreateSurfaceWorkflow class + potcar_functional (dict): See launch_workflow() method in + CreateSurfaceWorkflow class + symprec (float): See SpaceGroupAnalyzer in analyzer.py + terminations (bool): Determines whether or not to consider + different terminations in a slab. If true, each slab with a + specific shift value will have its own Firework and each of the + slab calculations will run in parallel. Defaults to false which + sets the shift value to 0. + """ + dec = MontyDecoder() + folder = dec.process_decoded(self.get("folder")) + cwd = dec.process_decoded(self.get("cwd")) + symprec = dec.process_decoded(self.get("symprec", 0.001)) + angle_tolerance = dec.process_decoded(self.get("angle_tolerance", 5)) + terminations = dec.process_decoded(self.get("terminations", False)) + custodian_params = dec.process_decoded(self.get("custodian_params")) + vaspdbinsert_parameters = \ + dec.process_decoded(self.get("vaspdbinsert_parameters")) + + user_incar_settings = \ + dec.process_decoded(self.get("user_incar_settings", + MPSlabVaspInputSet().incar_settings)) + k_product = \ + dec.process_decoded(self.get("k_product", 50)) + potcar_functional = \ + dec.process_decoded(self.get("potcar_fuctional", 'PBE')) + min_slab_size = dec.process_decoded(self.get("min_slab_size", 10)) + min_vacuum_size = dec.process_decoded(self.get("min_vacuum_size", 10)) + miller_index = dec.process_decoded(self.get("miller_index")) + + print 'about to make mplb' + + mplb = MPSlabVaspInputSet(user_incar_settings=user_incar_settings, + k_product=k_product, + potcar_functional=potcar_functional, + ediff_per_atom=False) + + # Create slabs from the relaxed oriented unit cell. Since the unit + # cell is already oriented with the miller index, entering (0,0,1) + # into SlabGenerator is the same as obtaining a slab in the + # orienetation of the original miller index. + print 'about to copy contcar' + contcar = Poscar.from_file("%s/CONTCAR.relax2.gz" %(cwd+folder)) + relax_orient_uc = contcar.structure + print 'made relaxed oriented structure' + print relax_orient_uc + print 'making slab' + + slabs = SlabGenerator(relax_orient_uc, (0,0,1), + min_slab_size=min_slab_size, + min_vacuum_size=min_vacuum_size, + max_normal_search=max(miller_index)) + + # Whether or not to create a list of Fireworks + # based on different slab terminations + print 'deciding terminations' + slab_list = slabs.get_slabs() if terminations else [slabs.get_slab()] + + qe = QueryEngine(**vaspdbinsert_parameters) + optional_data = ["state"] + print 'query bulk entry for job completion' + bulk_entry = qe.get_entries({'chemsys': relax_orient_uc.composition.reduced_formula, + 'structure_type': 'oriented_unit_cell', 'miller_index': miller_index}, + optional_data=optional_data) + print 'chemical formula', relax_orient_uc.composition.reduced_formula + print 'fomular data type is ', type(relax_orient_uc.composition.reduced_formula) + print 'checking job completion' + print bulk_entry + for entry in bulk_entry: + print 'for loop' + print entry.data['state'] + if entry.data['state'] != 'successful': + print "%s bulk calculations were incomplete, cancelling FW" \ + %(relax_orient_uc.composition.reduced_formula) + return FWAction() + else: + + print entry.data['state'] + + FWs = [] + for slab in slab_list: + + print slab + + new_folder = folder.replace('bulk', 'slab')+'_shift%s' \ + %(slab.shift) + + # Will continue an incomplete job from a previous contcar file if it exists + print 'cwd is %s' %(os.getcwd()) + print 'the folder is %s' %(new_folder) + print os.path.join(os.getcwd(), new_folder) + print cwd+'/'+new_folder + path = cwd+'/'+new_folder + + # path = os.path.join(os.getcwd(), folder) + newfolder = os.path.join(path, 'prev_run') + + # print 'check if conditions for continuing calculations have been satisfied' + # print 'check for the following path: %s' %(path) + # print os.path.exists(path) + # print os.path.exists(os.path.join(path, 'CONTCAR.gz')) + # print os.stat(os.path.join(path, 'CONTCAR.gz')).st_size !=0 + + def continue_vasp(contcar): + print folder, 'already exists, will now continue calculation' + print 'making prev_run folder' + os.system('mkdir %s' %(newfolder)) + print 'moving outputs to prev_run' + os.system('mv %s/* %s/prev_run' %(path, path)) + print 'moving outputs as inputs for next calculation' + os.system('cp %s/%s %s/INCAR %s/POTCAR %s/KPOINTS %s' + %(newfolder, contcar, newfolder, newfolder, newfolder, path)) + print 'unzipping new inputs' + os.system('gunzip %s/*' %(path)) + print 'copying contcar as new poscar' + if contcar == 'CONTCAR.relax1.gz': + os.system('mv %s/CONTCAR.relax1 %s/POSCAR' %(path , path)) + else: + os.system('mv %s/CONTCAR %s/POSCAR' %(path , path)) + + + if os.path.exists(path) and \ + os.path.exists(os.path.join(path, 'CONTCAR')) and \ + os.stat(os.path.join(path, 'CONTCAR')).st_size !=0: + continue_vasp('CONTCAR') + elif os.path.exists(path) and \ + os.path.exists(os.path.join(path, 'CONTCAR.gz')) \ + and os.stat(os.path.join(path, 'CONTCAR.gz')).st_size !=0: + continue_vasp('CONTCAR.gz') + elif os.path.exists(path) and \ + os.path.exists(os.path.join(path, 'CONTCAR.relax1.gz')) and \ + os.stat(os.path.join(path, 'CONTCAR.relax1.gz')).st_size !=0: + continue_vasp('CONTCAR.relax1.gz') + + else: + mplb.write_input(slab, cwd+new_folder) + + # Writes new INCAR file based on changes made by custodian on the bulk's INCAR. + # Only change in parameters between slab and bulk should be MAGMOM and ISIF + if os.path.exists("%s/INCAR.relax2.gz" %(cwd+folder)): + incar = Incar.from_file(cwd+folder +'/INCAR.relax2.gz') + else: + incar = Incar.from_file(cwd+folder +'/INCAR.relax2') + if os.path.exists("%s/OUTCAR.relax2.gz" %(cwd+folder)): + out = Outcar(cwd+folder+'/OUTCAR.relax2.gz') + else: + out = Outcar(cwd+folder+'/OUTCAR.relax2') + out_mag = out.magnetization + tot_mag = [mag['tot'] for mag in out_mag] + magmom = np.mean(tot_mag) + mag= [magmom for i in slab] + incar.__setitem__('MAGMOM', mag) + incar.__setitem__('ISIF', 2) + incar.__setitem__('AMIN', 0.01) + incar.__setitem__('AMIX', 0.2) + incar.__setitem__('BMIX', 0.001) + incar.__setitem__('NELMIN', 8) + incar.__setitem__('ISTART', 0) + incar.write_file(cwd+new_folder+'/INCAR') + + fw = Firework([RunCustodianTask(dir=new_folder, cwd=cwd, + **custodian_params), + VaspSlabDBInsertTask(struct_type="slab_cell", + loc=new_folder, cwd=cwd, shift=slab.shift, + surface_area=slab.surface_area, + vsize=slabs.min_vac_size, + ssize=slabs.min_slab_size, + miller_index=miller_index, + **vaspdbinsert_parameters)], + name=new_folder) + FWs.append(fw) + + return FWAction(additions=FWs) + + +@explicit_serialize +class RunCustodianTask(FireTaskBase): + """ + Runs Custodian. + """ + + required_params = ["dir", "jobs", "cwd"] + optional_params = ["custodian_params", "handlers", "max_errors"] + + def run_task(self, fw_spec): + + """ + Required Parameters: + dir (str path): directory containing the vasp inputs + jobs (VaspJob): Contains the cmd needed to run vasp + Optional Parameters: + custodian_params (dict **kwargs): Contains the job and the + scratch directory for a custodian run + handlers (list of custodian handlers): Defaults to empty list + """ + + dec = MontyDecoder() + dir = dec.process_decoded(self['dir']) + cwd = dec.process_decoded(self['cwd']) + + # Change to the directory with the vasp inputs to run custodian + os.chdir(cwd+dir) + + handlers = dec.process_decoded(self.get('handlers', [])) + jobs = dec.process_decoded(self['jobs']) + max_errors = dec.process_decoded(self['max_errors']) + + fw_env = fw_spec.get("_fw_env", {}) + cust_params = self.get("custodian_params", {}) + + # Get the scratch directory + if fw_env.get('scratch_root'): + cust_params['scratch_dir'] = os.path.expandvars( + fw_env['scratch_root']) + + c = Custodian(handlers=handlers, jobs=jobs, max_errors=max_errors, gzipped_output=True, **cust_params) + + output = c.run() + + return FWAction(stored_data=output) \ No newline at end of file diff --git a/mpworks/fix_scripts/README.md b/mpworks/fix_scripts/README.md new file mode 100755 index 00000000..054180d4 --- /dev/null +++ b/mpworks/fix_scripts/README.md @@ -0,0 +1,3 @@ +# Fix scripts + +These are one-time use scripts that were used to fix the database at one time or another. It is expected that they will never need to be run again, but they are kept here for reference. There are countless other "fix_scripts" on Anubhav's computer...the criteria for inclusion here is arbitrary. \ No newline at end of file diff --git a/mpworks/fix_scripts/fix_bs_controller_tasks.py b/mpworks/fix_scripts/fix_bs_controller_tasks.py new file mode 100755 index 00000000..33144ef9 --- /dev/null +++ b/mpworks/fix_scripts/fix_bs_controller_tasks.py @@ -0,0 +1,274 @@ +import time, yaml, sys, os +from fireworks.core.launchpad import LaunchPad +from fireworks.core.firework import Firework, Workflow +from mpworks.firetasks.controller_tasks import AddEStructureTask +from fireworks.utilities.fw_utilities import get_slug +from mpworks.snl_utils.snl_mongo import SNLMongoAdapter +from pymongo import MongoClient +from collections import Counter +from datetime import datetime +from fnmatch import fnmatch +from custodian.vasp.handlers import VaspErrorHandler + +cwd = os.getcwd() + +# DONE manually: "mp-987" -> fw_id: 119629 + +lpdb = LaunchPad.from_file('/global/homes/m/matcomp/mp_prod/config/config_Mendel/my_launchpad.yaml') +spec = {'task_type': 'Controller: add Electronic Structure v2', '_priority': 100000} +sma = SNLMongoAdapter.from_file('/global/homes/m/matcomp/mp_prod/config/dbs/snl_db.yaml') +with open('/global/homes/m/matcomp/mp_prod/materials_db_prod.yaml') as f: + creds = yaml.load(f) +client = MongoClient(creds['host'], creds['port']) +db = client[creds['db']] +db.authenticate(creds['username'], creds['password']) +materials = db['materials'] +tasks = db['tasks'] +print materials.count() + +def append_wf(fw_id, parent_fw_id=None): + wf = lpdb.workflows.find_one({'nodes':fw_id}, {'parent_links':1,'links':1,'name':1}) + try: + if parent_fw_id is None: + parent_fw_id = wf['parent_links'][str(fw_id)][-1] + # non-defused AddEStructureTask v2 already in children? + for child_fw_id in wf['links'][str(parent_fw_id)]: + if child_fw_id == parent_fw_id: continue + child_fw = lpdb.fireworks.find_one({'fw_id': child_fw_id}, {'spec.task_type':1, 'state':1}) + if child_fw['spec']['task_type'] == 'Controller: add Electronic Structure v2': + if child_fw['state'] == 'DEFUSED': + lpdb.reignite_fw(child_fw_id) + print 'AddEStructureTask v2', child_fw_id , 'reignited for', fw_id + elif child_fw['state'] == 'FIZZLED': + lpdb.rerun_fw(child_fw_id) + print 'AddEStructureTask v2', child_fw_id , 'marked for rerun for', fw_id + elif child_fw['state'] == 'COMPLETED': + print 'AddEStructureTask v2 already successfully run for', fw_id + sec_child_fw_id = wf['links'][str(child_fw_id)][0] + sec_child_fw = lpdb.fireworks.find_one({'fw_id': sec_child_fw_id}, {'spec.task_type':1, 'state':1}) + if sec_child_fw['state'] == 'FIZZLED': + lpdb.rerun_fw(sec_child_fw_id) + print 'FIZZLED -> marked for rerun:', sec_child_fw_id, sec_child_fw['spec']['task_type'] + else: + print 'AddEStructureTask v2 added but neither DEFUSED, FIZZLED, or COMPLETED for', fw_id + return + f = lpdb.get_wf_summary_dict(fw_id)['name'].replace(' ', '_') + name = get_slug(f + '--' + spec['task_type']) + fw = Firework([AddEStructureTask()], spec, name=name) + lpdb.append_wf(Workflow([fw]), [parent_fw_id]) + print name, 'added for', fw_id + except ValueError: + raise ValueError('could not append controller task to wf', wf['name']) + +if __name__ == "__main__": + nfws = 0 + + #append_wf(42391, parent_fw_id=69272) + #append_wf(51449, parent_fw_id=76078) + #for doc in lpdb.fireworks.find( + # {'spec.task_type': 'Controller: add Electronic Structure v2', 'spec._priority': {'$exists':1}}, + # {'fw_id': 1, 'spec._priority': 1, 'state': 1 } + #): + # if (doc['state'] == 'FIZZLED' or doc['state'] == 'READY') and doc['spec']['_priority'] == 100000: + # print nfws, doc['fw_id'] + # lpdb.defuse_fw(doc['fw_id']) + # nfws += 1 + + #for doc in lpdb.fireworks.find( + # {'spec.task_type': 'Controller: add Electronic Structure', 'state': 'COMPLETED', + # 'spec.analysis': {'$exists':1}, 'fw_id': {'$gte': 155067}}, # new controllers added -> fizzled GGA static reruns + # {'fw_id': 1, 'spec.analysis.bandgap': 1} + #): + # fw_id, bandgap = doc['fw_id'], doc['spec']['analysis']['bandgap'] + # if bandgap > 0 and bandgap < 0.5: + # try: + # append_wf(fw_id) + # time.sleep(.5) + # except ValueError: + # continue + # nfws += 1 + # #if nfws > 10: break + #print 'nfws =', nfws + + mp_ids = [ + 'mp-2123', 'mp-10886', 'mp-582799', 'mp-21477', 'mp-535', 'mp-21293', 'mp-8700', + 'mp-9568', 'mp-973', 'mp-505622', 'mp-20839', 'mp-1940', 'mp-16521', 'mp-30354', + 'mp-568953', 'mp-454', 'mp-1010', 'mp-1416', 'mp-21385', 'mp-27659', 'mp-22481', + 'mp-569529', 'mp-1057', 'mp-1834', 'mp-2336', 'mp-12857', 'mp-21109', 'mp-30387', + 'mp-30599', 'mp-21884', 'mp-11397', 'mp-11814', 'mp-510437', 'mp-12565', 'mp-33032', + 'mp-20885', 'mp-1891', + "mp-987", "mp-1542", "mp-2252", "mp-966", "mp-6945", "mp-1598", + "mp-7547", "mp-554340", "mp-384", "mp-2437", "mp-1167", "mp-571266", + "mp-560338", "mp-27253", "mp-1705", "mp-2131", "mp-676", "mp-2402", "mp-9588", + "mp-2452", "mp-690", "mp-30033", "mp-10155", "mp-9921", "mp-9548", "mp-569857", + "mp-29487", "mp-909", "mp-1536", "mp-28391", "mp-558811", "mp-1033", "mp-1220", + "mp-7817", "mp-30952", "mp-569175", "mp-1683", "mp-27821", "mp-554243", + "mp-557837", "mp-867227", "mp-862871", "mp-861979", "mp-24289", "mp-684690", + "mp-551905", "mp-437", "mp-1806", "mp-556395", "mp-14288", "mp-1944", + "mp-15339", "mp-568208", "mp-28096", "mp-542613", "mp-862983", "mp-864974", + "mp-865966", "mp-20401", "mp-864898", "mp-546711", "mp-8429", "mp-867171", + "mp-862705", "mp-864768", "mp-7984", "mp-864844", "mp-865147", "mp-867909", + "mp-861629", "mp-961673", "mp-753287", "mp-4701", "mp-3532", "mp-864839", + "mp-867333", "mp-12335", "mp-7492", "mp-867149", "mp-862316", "mp-865680", + "mp-867193", "mp-5456", "mp-862699", "mp-866139", "mp-866166", "mp-867788", + "mp-865791", "mp-864630", "mp-862719", "mp-865502", "mp-862721", "mp-862717", + "mp-865050", "mp-867223", "mp-865965", "mp-560547", "mp-4826", "mp-8015", + "mp-865097", "mp-861507", "mp-861901", "mp-866165", "mp-11980", "mp-867113", + "mp-10887", "mp-862672", "mp-19799", "mp-867207", "mp-865989", "mp-865681", + "mp-31055", "mp-861656", "mp-864762", "mp-862297", "mp-10810", "mp-644280", + "mp-18971", "mp-19149", "mp-867781", "mp-865044", "mp-867307", "mp-867169", + "mp-8995", "mp-865912", "mp-863707", "mp-866164", "mp-13312", "mp-754540", + "mp-866117", "mp-28250", "mp-20761", "mp-866229", "mp-10809", "mp-8717", + "mp-862947", "mp-866154", "mp-864933", "mp-861725", "mp-865519", "mp-20999", + "mp-865867", "mp-10608", "mp-867833", "mp-540609", "mp-866105", "mp-862318", + "mp-5229", "mp-865986", "mp-22966", "mp-37514", "mp-7006", "mp-862445", + "mp-862691", "mp-541226", "mp-865659", "mp-867810", "mp-7473", "mp-10140", + "mp-867799", "mp-865130", "mp-866096", "mp-10113", "mp-862486", "mp-8994", + "mp-865433", "mp-29506", "mp-21035", "mp-865010", "mp-8014", "mp-28237", + "mp-4047", "mp-4505", "mp-867881", "mp-861736", "mp-867258", "mp-22120", + "mp-865001", "mp-865278", "mp-864883", "mp-6989", "mp-865963", "mp-19279", + "mp-864654", "mp-608311", "mp-865933", "mp-867280", "mp-866219", "mp-28698", + "mp-14006", "mp-29624", "mp-867761", "mp-22793", "mp-9872", "mp-867926", + "mp-510273", "mp-32526", "mp-862296", "mp-19238", "mp-3530", "mp-3332", + "mp-866106", "mp-867807", "mp-21017", "mp-20325", "mp-557912", "mp-867769", + "mp-29751", "mp-11695", "mp-865603", "mp-865186", "mp-19035", "mp-865929", + "mp-864618", "mp-867872", "mp-8397", "mp-865308", "mp-567636", "mp-867266", + "mp-865183", "mp-510268", "mp-862694", "mp-8013", "mp-867271", "mp-578618", + "mp-865713", "mp-865167", "mp-29009", "mp-865128", "mp-32497", "mp-864684", + "mp-862473", "mp-865280", "mp-3020", "mp-27193", "mp-861937", "mp-867896", + "mp-862374", "mp-28872", "mp-23425", "mp-10417" + ] + mp_ids = [ "mp-134", "mp-127", "mp-58", "mp-135", "mp-70", "mp-1" ] + mp_ids = [doc['task_id'] for doc in materials.find({'has_bandstructure': False}, {'task_id':1})] + print '#mp_ids =', len(mp_ids) + + counter = Counter() + materials_wBS = [] + for matidx, material in enumerate(materials.find({'task_id': {'$in': mp_ids}}, {'task_id': 1, '_id': 0, 'snlgroup_id_final': 1, 'has_bandstructure': 1, 'pretty_formula': 1})): + mp_id, snlgroup_id = material['task_id'], material['snlgroup_id_final'] + url = 'https://materialsproject.org/materials/' + mp_id + if material['has_bandstructure']: + materials_wBS.append((mp_id, material['pretty_formula'])) + counter['has_bandstructure'] += 1 + print matidx, '========', mp_id, snlgroup_id, '=============' + fw_list = list(lpdb.fireworks.find( + {'spec.snlgroup_id': snlgroup_id}, + {'_id': 0, 'state': 1, 'name': 1, 'fw_id': 1, 'spec.snlgroup_id': 1, 'spec.task_type': 1, 'launches': 1} + )) + if len(fw_list) > 0: + has_gga_static = False + for fw in fw_list: + if fw['spec']['task_type'] == 'GGA static v2': + has_gga_static = True + if fw['state'] == 'FIZZLED': + #counter[fw['spec']['task_type']] += 1 + print '--'.join([fw['name'], str(fw['fw_id'])]), fw['state'] + launch_dir = lpdb.launches.find_one({'launch_id': fw['launches'][0]}, {'launch_dir':1})['launch_dir'] + launch_subdir = '/'.join(launch_dir.split('/')[-2:]) + if 'oasis' in launch_dir: + launch_dir = os.path.join('/global/projecta/projectdirs/matgen/scratch/mp_prod', launch_subdir) + if 'scratch2/sd' in launch_dir: + launch_dir = os.path.join('/global/projecta/projectdirs/matgen/scratch/mp_prod', launch_subdir) + try: + os.chdir(launch_dir) + except: + launch_dir = launch_dir.replace('scratch/mp_prod', 'garden/dev') + try: + os.chdir(launch_dir) + except: + launch_dir = launch_dir.replace('garden/dev', 'garden') + try: + os.chdir(launch_dir) + except: + print ' |===> could not find launch directory in usual locations' + lpdb.rerun_fw(fw['fw_id']) + print ' |===> marked for RERUN' + counter['LOCATION_NOT_FOUND'] += 1 + continue + print ' |===>', launch_dir + vaspout = os.path.join(launch_dir, "vasp.out") + if not os.path.exists(vaspout): + vaspout = os.path.join(launch_dir, "vasp.out.gz") + try: + h = VaspErrorHandler(vaspout) + h.check() + d = h.correct() + except: + counter['GGA_static_Handler_Error'] += 1 + if d['errors']: + for err in d['errors']: + counter['GGA_static_' + err] += 1 + if 'brmix' in d['errors']: + #lpdb.rerun_fw(fw['fw_id']) + print ' |===> BRMIX error -> marked for RERUN with alternative strategy' + else: + print ' |===> no vasp error indicated -> TODO' + counter['GGA_STATIC_NO_VASP_ERROR'] += 1 + os.chdir(cwd) + else: + workflow = lpdb.workflows.find_one( + {'nodes': fw['fw_id']}, + {'state': 1, '_id': 0, 'fw_states': 1, 'nodes': 1, 'updated_on': 1, 'parent_links': 1} + ) + if workflow is None: + print ' |==> workflow not found', fw['fw_id'] + counter['WF_NOT_FOUND'] += 1 + continue + is_new = bool(datetime(2016, 1, 1) < workflow['updated_on']) + if workflow['state'] == 'FIZZLED': + for fw_id_fizzled, fw_state in workflow['fw_states'].iteritems(): + if fw_state == 'FIZZLED': + fw_fizzled = lpdb.fireworks.find_one({'fw_id': int(fw_id_fizzled)}, {'_id': 0, 'name': 1, 'fw_id': 1, 'spec.task_type': 1}) + counter[fw_fizzled['spec']['task_type']] += 1 + print url, is_new, material['has_bandstructure'], fw_id_fizzled + print 'http://fireworks.dash.materialsproject.org/wf/'+str(fw['fw_id']), workflow['state'] + print ' |==>', '--'.join([fw_fizzled['name'], fw_id_fizzled]) + if fnmatch(fw_fizzled['spec']['task_type'], '*Boltztrap*'): + print ' |====> marked for RERUN (Boltztrap, physical constants from scipy, missing libmkl_lapack.so, BoltzTrap_TE -> pymatgen)' + #lpdb.rerun_fw(fw_fizzled['fw_id']) + continue + elif fw_fizzled['spec']['task_type'] == 'GGA Uniform v2': + fw_id_rerun = str(fw_fizzled['fw_id']) + while 1: + fw_id_rerun = str(workflow['parent_links'][fw_id_rerun][-1]) + fw_rerun = lpdb.fireworks.find_one({'fw_id': int(fw_id_rerun)}, {'_id': 0, 'spec.task_type': 1}) + if fw_rerun['spec']['task_type'] != 'VASP db insertion': + print 'http://fireworks.dash.materialsproject.org/wf/'+fw_id_rerun + break + #lpdb.rerun_fw(int(fw_id_rerun)) + print ' |====> marked for RERUN (could not get valid results from prev_vasp_dir, GGAstatic vasprun.xml validation error)' + elif fw_fizzled['spec']['task_type'] == 'GGA band structure v2': + print ' |===> marked for RERUN (trial & error)' + #try: + # lpdb.rerun_fw(fw_fizzled['fw_id']) + #except: + # print ' |===> could not rerun firework' + # counter['WF_LOCKED'] += 1 + elif fw_fizzled['spec']['task_type'] == 'VASP db insertion': + print ' |===> marked for RERUN (trial & error)' + #lpdb.rerun_fw(fw_fizzled['fw_id']) + #sys.exit(0) + break + elif workflow['state'] == 'COMPLETED': + print url, is_new, material['has_bandstructure'], workflow['nodes'][0] + if not is_new and not material['has_bandstructure']: + #lpdb.rerun_fw(fw['fw_id']) + print ' |===> marked for RERUN with alternative brmix strategy (WF completed but BS missing)' + counter['WF_COMPLETED_MISSING_BS'] += 1 + #sys.exit(0) + else: + counter['COMPLETED'] += 1 + if not has_gga_static: + print 'ERROR: no GGA static run found!' + print '\n'.join([ + '--'.join([fw['name'], str(fw['fw_id']), fw['state']]) for fw in fw_list + ]) + counter['NO_GGA_STATIC'] += 1 + #break + else: + print 'ERROR: no fireworks found!' + counter['NO_FWS'] += 1 + #break + print '#mp_ids =', len(mp_ids) + print counter + #print materials_wBS diff --git a/mpworks/fix_scripts/fix_mpcomplete.py b/mpworks/fix_scripts/fix_mpcomplete.py new file mode 100755 index 00000000..83f8534a --- /dev/null +++ b/mpworks/fix_scripts/fix_mpcomplete.py @@ -0,0 +1,61 @@ +import os, json +from pymongo import DESCENDING, ASCENDING +from fireworks.fw_config import CONFIG_FILE_DIR, SORT_FWS +from fireworks.core.fworker import FWorker +from fireworks.core.launchpad import LaunchPad +from pymongo import ReturnDocument + +launchpad = LaunchPad.from_file(os.path.join(CONFIG_FILE_DIR, 'my_launchpad.yaml')) +fworker = FWorker.from_file(os.path.join(CONFIG_FILE_DIR, 'my_fworker.yaml')) +#print launchpad._get_a_fw_to_run(query=fworker.query, checkout=False) +m_query = dict(fworker.query) +m_query['state'] = 'READY' +sortby = [("spec._priority", DESCENDING)] +if SORT_FWS.upper() == "FIFO": + sortby.append(("created_on", ASCENDING)) +elif SORT_FWS.upper() == "FILO": + sortby.append(("created_on", DESCENDING)) +#print json.dumps(m_query, indent=4) +projection = { + '_id': 0, 'fw_id': 1, 'spec._fworker': 1, 'spec.task_type': 1, 'spec._queueadapter': 1, + 'spec.mpsnl.about.remarks': 1, 'spec.snl.about.remarks': 1, 'spec.prev_vasp_dir': 1, + 'updated_on': 1, 'state': 1 +} + +fw_ids = [] +for idoc, doc in enumerate(launchpad.fireworks.find(m_query, projection=projection, sort=sortby).limit(100)): + #print doc + if 'walltime' in doc['spec']['_queueadapter']: + walltime = doc['spec']['_queueadapter']['walltime'] + if int(walltime.split(':')[0]) > 48: + launchpad.fireworks.find_one_and_update( + {'fw_id': doc['fw_id']}, + {'$set': {'spec._queueadapter.walltime': '48:00:00'}}, + projection=projection, + return_document=ReturnDocument.AFTER + ) + print doc['fw_id'], '----> walltime updated' + if 'nnodes' in doc['spec']['_queueadapter'] and not 'nodes' in doc['spec']['_queueadapter']: + launchpad.fireworks.find_one_and_update( + {'fw_id': doc['fw_id']}, + {'$rename': {'spec._queueadapter.nnodes': 'spec._queueadapter.nodes'}}, + projection=projection, + return_document=ReturnDocument.AFTER + ) + print doc['fw_id'], '----> nodes key renamed' + if 'pre_rocket' in doc['spec']['_queueadapter']: + launchpad.fireworks.find_one_and_update( + m_query, + {'$unset' : { 'spec._queueadapter.pre_rocket' : 1}}, + projection=projection, + return_document=ReturnDocument.AFTER + ) + print doc['fw_id'], '----> pre_rocket dropped' + if 'prev_vasp_dir' in doc['spec'] and not os.path.exists(doc['spec']['prev_vasp_dir']): + block_dir = doc['spec']['prev_vasp_dir'].split('/')[-2:] + launch_dir = '/'.join('/oasis/projects/nsf/csd436/phuck/garden'.split('/') + block_dir) + if not os.path.exists(launch_dir): + print doc['fw_id'], '---->', '/'.join(block_dir), 'does not exists!' + continue + fw_ids.append(doc['fw_id']) +print 'fixed', fw_ids diff --git a/mpworks/fix_scripts/legacy/__init__.py b/mpworks/fix_scripts/legacy/__init__.py new file mode 100755 index 00000000..9c8764e1 --- /dev/null +++ b/mpworks/fix_scripts/legacy/__init__.py @@ -0,0 +1,6 @@ +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 08, 2013' \ No newline at end of file diff --git a/mpworks/fix_scripts/legacy/actions/__init__.py b/mpworks/fix_scripts/legacy/actions/__init__.py new file mode 100755 index 00000000..44b41985 --- /dev/null +++ b/mpworks/fix_scripts/legacy/actions/__init__.py @@ -0,0 +1,6 @@ +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 08, 2013' diff --git a/mpworks/fix_scripts/legacy/actions/add_snl_final.py b/mpworks/fix_scripts/legacy/actions/add_snl_final.py new file mode 100755 index 00000000..9ce82df0 --- /dev/null +++ b/mpworks/fix_scripts/legacy/actions/add_snl_final.py @@ -0,0 +1,27 @@ +import os +from pymongo import MongoClient +import yaml + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 21, 2013' + +module_dir = os.path.dirname(os.path.abspath(__file__)) +tasks_f = os.path.join(module_dir, 'tasks.yaml') + +with open(tasks_f) as f2: + db_creds = yaml.load(f2) + + mc2 = MongoClient(db_creds['host'], db_creds['port']) + db2 = mc2[db_creds['database']] + db2.authenticate(db_creds['admin_user'], db_creds['admin_password']) + new_tasks = db2['tasks'] + + count = 0 + for d in new_tasks.find({'snlgroup_id_final': {'$exists': False}}, {'task_id': 1, 'snl': 1, 'snlgroup_id': 1, 'snlgroup_changed': 1}): + new_tasks.update({'task_id': d['task_id']}, {'$set': {'snl_final': d['snl'], 'snlgroup_id_final': d['snlgroup_id'], 'snlgroup_changed': False}}) + count+=1 + print count diff --git a/mpworks/fix_scripts/legacy/actions/do_fw_conversion.py b/mpworks/fix_scripts/legacy/actions/do_fw_conversion.py new file mode 100755 index 00000000..38e6aab6 --- /dev/null +++ b/mpworks/fix_scripts/legacy/actions/do_fw_conversion.py @@ -0,0 +1,50 @@ +import os + +from pymongo import MongoClient, ASCENDING +import yaml + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 14, 2013' + +if __name__ == '__main__': + + module_dir = os.path.dirname(os.path.abspath(__file__)) + # lp_f = os.path.join(module_dir, 'my_launchpad.yaml') + tasks_f = os.path.join(module_dir, 'tasks.yaml') + + #with open(lp_f) as f: + # lp = LaunchPad.from_file(lp_f) + # lp.reset(None, require_password=False) + + with open(tasks_f) as f2: + db_creds = yaml.load(f2) + + mc2 = MongoClient(db_creds['host'], db_creds['port']) + db2 = mc2[db_creds['database']] + db2.authenticate(db_creds['admin_user'], db_creds['admin_password']) + new_tasks = db2['tasks'] + + print new_tasks.count() + + new_tasks.ensure_index("task_id", unique=True) + new_tasks.ensure_index("task_id_deprecated", unique=True) + new_tasks.ensure_index("chemsys") + new_tasks.ensure_index("analysis.e_above_hull") + new_tasks.ensure_index("pretty_formula") + new_tasks.ensure_index([("elements", ASCENDING), ("nelements", ASCENDING)]) + new_tasks.ensure_index("state") + new_tasks.ensure_index([("state", ASCENDING), ("task_type", ASCENDING)]) + new_tasks.ensure_index([("state", ASCENDING), ("task_type", ASCENDING), ("submission_id", ASCENDING)]) + new_tasks.ensure_index("is_compatible") + new_tasks.ensure_index("snl.snl_id") + new_tasks.ensure_index("snlgroup_id") + + """ + for task_dict in new_tasks.find({"state":"successful"}, sort=[("task_id", ASCENDING)], timeout=False): + fw_id = task_dict_to_wf(task_dict, lp) + new_tasks.update({"task_id": task_dict["task_id"]}, {"$set": {"fw_id": fw_id}}) + """ diff --git a/mpworks/fix_scripts/legacy/actions/do_icsd_to_snl.py b/mpworks/fix_scripts/legacy/actions/do_icsd_to_snl.py new file mode 100755 index 00000000..a0201334 --- /dev/null +++ b/mpworks/fix_scripts/legacy/actions/do_icsd_to_snl.py @@ -0,0 +1,45 @@ +import os +import traceback + +from pymongo import MongoClient, ASCENDING +import yaml + +from mpworks.maintenance_scripts.icsd2012_to_snl import icsd_dict_to_snl +from mpworks.snl_utils.snl_mongo import SNLMongoAdapter + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 12, 2013' + +if __name__ == '__main__': + + module_dir = os.path.dirname(os.path.abspath(__file__)) + icsd_f = os.path.join(module_dir, 'mg_core_dev.yaml') + snl_f = os.path.join(module_dir, 'snl.yaml') + + with open(icsd_f) as f: + y = yaml.load(f) + + mc = MongoClient(y['host'], y['port']) + db = mc[y['db']] + + db.authenticate(y['username'], y['password']) + + snldb = SNLMongoAdapter.from_file(snl_f) + + # query = {"icsd_id": {"$gte": 170623}} + query = {} + + for icsd_dict in db.icsd_2012_crystals.find(query, sort=[("icsd_id", ASCENDING)], timeout=False): + try: + snl = icsd_dict_to_snl(icsd_dict) + if snl: + snldb.add_snl(snl) + except: + traceback.print_exc() + print 'ERROR - icsd id:', icsd_dict['icsd_id'] + + print 'DONE' diff --git a/mpworks/fix_scripts/legacy/actions/do_mps_to_snl.py b/mpworks/fix_scripts/legacy/actions/do_mps_to_snl.py new file mode 100755 index 00000000..8b4d4223 --- /dev/null +++ b/mpworks/fix_scripts/legacy/actions/do_mps_to_snl.py @@ -0,0 +1,61 @@ +import os +import traceback +import time + +from pymongo import MongoClient +import yaml + +from mpworks.fix_scripts.legacy.mps_to_snl import mps_dict_to_snl +from mpworks.snl_utils.snl_mongo import SNLMongoAdapter + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 08, 2013' + + +RESET = False + +if __name__ == '__main__': + + module_dir = os.path.dirname(os.path.abspath(__file__)) + automation_f = os.path.join(module_dir, 'automation.yaml') + snl_f = os.path.join(module_dir, 'snl.yaml') + + with open(automation_f) as f: + y = yaml.load(f) + + mc = MongoClient(y['host'], y['port']) + db = mc[y['db']] + + db.authenticate(y['username'], y['password']) + + snldb = SNLMongoAdapter.from_file(snl_f) + + prev_ids = [] # MPS ids that we already took care of + + print 'INITIALIZING' + if RESET: + snldb._reset() + time.sleep(10) # makes me sleep better at night + + else: + for mps in snldb.snl.find({}, {"about._materialsproject.deprecated.mps_ids": 1}): + prev_ids.extend(mps['about']['_materialsproject']['deprecated']['mps_ids']) + + print 'PROCESSING' + for mps in db.mps.find(timeout=False): + try: + if not mps['mps_id'] in prev_ids: + snl = mps_dict_to_snl(mps) + if snl: + snldb.add_snl(snl) + else: + print 'SKIPPING', mps['mps_id'] + except: + traceback.print_exc() + print 'ERROR - mps id:', mps['mps_id'] + + print 'DONE' diff --git a/mpworks/fix_scripts/legacy/actions/do_task_conversion.py b/mpworks/fix_scripts/legacy/actions/do_task_conversion.py new file mode 100755 index 00000000..9bc0670e --- /dev/null +++ b/mpworks/fix_scripts/legacy/actions/do_task_conversion.py @@ -0,0 +1,93 @@ +from argparse import ArgumentParser +import json +import logging +import multiprocessing +import os +import traceback + +from pymongo import MongoClient +import yaml + +from mpworks.fix_scripts.legacy import MPVaspDrone_CONVERSION + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 13, 2013' + +class OldTaskBuilder(): + + @classmethod + def setup(cls): + module_dir = os.path.dirname(os.path.abspath(__file__)) + tasks_f = os.path.join(module_dir, 'mg_core_dev.yaml') + + with open(tasks_f) as f: + y = yaml.load(f) + + mc = MongoClient(y['host'], y['port']) + db = mc[y['db']] + db.authenticate(y['username'], y['password']) + + cls.old_tasks = db['tasks_dbv2'] + + db_dir = os.environ['DB_LOC'] + db_path = os.path.join(db_dir, 'tasks_db.json') + with open(db_path) as f2: + db_creds = json.load(f2) + + mc2 = MongoClient(db_creds['host'], db_creds['port']) + db2 = mc2[db_creds['database']] + db2.authenticate(db_creds['admin_user'], db_creds['admin_password']) + + cls.new_tasks = db2['tasks'] + + cls.drone = MPVaspDrone_CONVERSION( + host=db_creds['host'], port=db_creds['port'], + database=db_creds['database'], user=db_creds['admin_user'], + password=db_creds['admin_password'], + collection=db_creds['collection'], parse_dos=False, + additional_fields={}, + update_duplicates=False) + + def process_task(self, task_id): + # get the directory containing the db file + if not self.new_tasks.find_one({'task_id': 'mp-{}'.format(task_id)}): + t = self.old_tasks.find_one({'task_id': task_id}) + try: + t_id, d = self.drone.assimilate(t) + print 'ENTERED', t_id + except: + print 'ERROR entering', t['task_id'] + traceback.print_exc() + else: + print 'skip' + + +def _analyze(task_id): + b = OldTaskBuilder() + return b.process_task(task_id) + + +def parallel_build(min, max): + tasks_old = OldTaskBuilder.old_tasks + task_ids = [] + for i in tasks_old.find({'task_id': {'$gte': min, '$lt': max}}, {'task_id': 1}): + task_ids.append(i['task_id']) + + print 'GOT all tasks...' + pool = multiprocessing.Pool(16) + pool.map(_analyze, task_ids) + print 'DONE' + +if __name__ == '__main__': + o = OldTaskBuilder() + o.setup() + logging.basicConfig(level=logging.INFO) + parser = ArgumentParser() + parser.add_argument('min', help='min', type=int) + parser.add_argument('max', help='max', type=int) + args = parser.parse_args() + parallel_build(args.min, args.max) diff --git a/mpworks/fix_scripts/legacy/actions/do_task_conversion_fixes.py b/mpworks/fix_scripts/legacy/actions/do_task_conversion_fixes.py new file mode 100755 index 00000000..b01dc0bb --- /dev/null +++ b/mpworks/fix_scripts/legacy/actions/do_task_conversion_fixes.py @@ -0,0 +1,87 @@ +import json +import logging +import os +import traceback + +from pymongo import MongoClient +import yaml + +from mpworks.fix_scripts.legacy import MPVaspDrone_CONVERSION + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 13, 2013' + +class OldTaskBuilder(): + + @classmethod + def setup(cls): + module_dir = os.path.dirname(os.path.abspath(__file__)) + tasks_f = os.path.join(module_dir, 'mg_core_dev.yaml') + + with open(tasks_f) as f: + y = yaml.load(f) + + mc = MongoClient(y['host'], y['port']) + db = mc[y['db']] + db.authenticate(y['username'], y['password']) + + cls.old_tasks = db['tasks_dbv2'] + + db_dir = os.environ['DB_LOC'] + db_path = os.path.join(db_dir, 'tasks_db.json') + with open(db_path) as f2: + db_creds = json.load(f2) + + mc2 = MongoClient(db_creds['host'], db_creds['port']) + db2 = mc2[db_creds['database']] + db2.authenticate(db_creds['admin_user'], db_creds['admin_password']) + + cls.new_tasks = db2['tasks'] + + cls.drone = MPVaspDrone_CONVERSION( + host=db_creds['host'], port=db_creds['port'], + database=db_creds['database'], user=db_creds['admin_user'], + password=db_creds['admin_password'], + collection=db_creds['collection'], parse_dos=False, + additional_fields={}, + update_duplicates=True) + + def process_task(self, task_id): + t = self.old_tasks.find_one({'task_id': task_id}) + try: + t_id, d = self.drone.assimilate(t) + print 'ENTERED', t_id + except: + print 'ERROR entering', t['task_id'] + traceback.print_exc() + + +if __name__ == '__main__': + o = OldTaskBuilder() + o.setup() + logging.basicConfig(level=logging.INFO) + """ + tasks_old = OldTaskBuilder.old_tasks + for i in tasks_old.find({'dir_name':{'$regex':'cathode_'}}, {'task_id': 1, 'dir_name': 1}): + task_id = i['task_id'] + dir_name = i['dir_name'] + print 'FIXING', task_id + # cut off the last part of the dir_name + cutoff_path = os.path.dirname(dir_name) + final_path = cutoff_path.replace('cathode_block', 'block') + o.old_tasks.find_and_modify({'task_id': task_id}, {'$set': {'dir_name': final_path}}) + # o.process_task(task_id) + """ + with open('to_fix.txt') as f: + for line in f: + old_task_id = int(line.split(' ')[1]) + new_task_id = 'mp-'+str(old_task_id) + t = o.new_tasks.find_one({"task_id": new_task_id}, {"state": 1}) + if t: + o.new_tasks.remove({'task_id': new_task_id}) + print 'REPARSING', old_task_id + o.process_task(old_task_id) diff --git a/mpworks/fix_scripts/legacy/mps_to_snl.py b/mpworks/fix_scripts/legacy/mps_to_snl.py new file mode 100755 index 00000000..1257a53f --- /dev/null +++ b/mpworks/fix_scripts/legacy/mps_to_snl.py @@ -0,0 +1,93 @@ +import datetime +import traceback +from pymongo import MongoClient +from pymatgen.core.structure import Structure +from pymatgen.matproj.snl import StructureNL + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 08, 2013' + + +def mps_dict_to_snl(mps_dict): + m = mps_dict + + if 'deprecated' in m['about']['metadata']['info'] and m['about']['metadata']['info']['deprecated']: + return None + + if 'Carbon Capture Storage Initiative (CCSI)' in m['about']['metadata']['project_names']: + print 'rejected old CCSI' + return None + + mps_ids = [m['mps_id']] + + remarks = [] + for remark in m['about']['metadata']['remarks']: + if 'This entry replaces deprecated mps_id' in remark: + mps_ids.append(int(remark.split()[-1])) # add the deprecated MPS id to this SNL + else: + remarks.append(remark) + for remark in m['about']['metadata']['keywords']: + remarks.append(remark) + + projects = [] + for project in m['about']['metadata']['project_names']: + projects.append(project) + + data = {'_materialsproject': {'deprecated': {'mps_ids': mps_ids}}, '_icsd': {}} + for k, v in m['about']['metadata']['info'].iteritems(): + if k == 'icsd_comments': + data['_icsd']['comments'] = v + elif k == 'icsd_id': + data['_icsd']['icsd_id'] = v + elif k == 'remark': + data['_materialsproject']['ordering_remark'] = v + elif 'deprecated' in k or 'original_structure' in k: + data['_materialsproject']['deprecated'][k] = v + elif 'assert' in k or 'universe' in k or 'mp_duplicates' in k: + pass + else: + data['_materialsproject'][k] = v + + authors = [] + for a in m['about']['authors']: + authors.append({'name': a['name'], 'email': a['email']}) + for a in m['about']['acknowledgements']: + authors.append({'name': a['name'], 'email': a['email']}) + + cites = [m['about']['please_cite']['bibtex']] + if m['about']['supporting_info']: + cites.append(m['about']['supporting_info']['bibtex']) + references = '\n'.join(cites) + + history = [] + for h in m['about']['links']: + if 'direct_copy' in h['description']: + del h['description']['direct_copy'] + history.append({'name': h['name'], 'url': h['url'], 'description': h['description']}) + + struct = Structure.from_dict(m) + + created_at = datetime.datetime.strptime(m['about']['created_at'], "%Y-%m-%d %H:%M:%S") + + return StructureNL(struct, authors, projects, references, remarks, data, history, created_at) + + + + + + + + + + + + + + + + + diff --git a/mpworks/fix_scripts/legacy/old_task_drone.py b/mpworks/fix_scripts/legacy/old_task_drone.py new file mode 100755 index 00000000..3611924f --- /dev/null +++ b/mpworks/fix_scripts/legacy/old_task_drone.py @@ -0,0 +1,236 @@ +import json +import os +import datetime +import logging + +from pymongo import MongoClient +import gridfs + +from matgendb.creator import VaspToDbTaskDrone +from mpworks.drones.signals import VASPInputsExistSignal, \ + VASPOutputsExistSignal, VASPOutSignal, HitAMemberSignal, SegFaultSignal, \ + VASPStartedCompletedSignal, WallTimeSignal, DiskSpaceExceededSignal, \ + SignalDetectorList +from mpworks.fix_scripts.legacy.mps_to_snl import mps_dict_to_snl +from mpworks.snl_utils.snl_mongo import SNLMongoAdapter +from mpworks.workflows.wf_utils import get_block_part +from pymatgen.core.structure import Structure +from pymatgen.matproj.snl import StructureNL +from pymatgen.serializers.json_coders import MontyEncoder + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'Mar 26, 2013' + +logger = logging.getLogger(__name__) + + +def is_valid_vasp_dir(mydir): + # note that the OUTCAR and POSCAR are known to be empty in some + # situations + files = ["OUTCAR", "POSCAR", "INCAR", "KPOINTS"] + for f in files: + m_file = os.path.join(mydir, f) + if not (os.path.exists(m_file) and os.stat(m_file).st_size > 0): + return False + return True + + +class MPVaspDrone_CONVERSION(VaspToDbTaskDrone): + # AJ: give old_task, not dir_name + def assimilate(self, old_task): + """ + Parses vasp runs. Then insert the result into the db. and return the + task_id or doc of the insertion. + + Returns: + If in simulate_mode, the entire doc is returned for debugging + purposes. Else, only the task_id of the inserted doc is returned. + """ + + path = old_task['dir_name'] # AJ: get dir name from task + d = self.get_task_doc(path, self.parse_dos, + self.additional_fields) + d["dir_name_full"] = d["dir_name"].split(":")[1] + d["dir_name"] = get_block_part(d["dir_name_full"]) + if not self.simulate: + # Perform actual insertion into db. Because db connections cannot + # be pickled, every insertion needs to create a new connection + # to the db. + conn = MongoClient(self.host, self.port) + db = conn[self.database] + if self.user: + db.authenticate(self.user, self.password) + coll = db[self.collection] + + # Insert dos data into gridfs and then remove it from the dict. + # DOS data tends to be above the 4Mb limit for mongo docs. A ref + # to the dos file is in the dos_fs_id. + result = coll.find_one({"dir_name": d["dir_name"]}) + if result is None or self.update_duplicates: + if self.parse_dos and "calculations" in d: + for calc in d["calculations"]: + if "dos" in calc: + dos = json.dumps(calc["dos"], cls=MontyEncoder) + fs = gridfs.GridFS(db, "dos_fs") + dosid = fs.put(dos) + calc["dos_fs_id"] = dosid + del calc["dos"] + + d["last_updated"] = datetime.datetime.today() + if result is None: + d["task_id"] = "mp-{}".format(old_task['task_id']) # AJ: old task_id is new + logger.info("Inserting {} with taskid = {}" + .format(d["dir_name"], d["task_id"])) + elif self.update_duplicates: + d["task_id"] = result["task_id"] + logger.info("Updating {} with taskid = {}" + .format(d["dir_name"], d["task_id"])) + + #Fireworks processing + self.process_fw(old_task, d) + coll.update({"dir_name": d["dir_name"]}, {"$set": d}, + upsert=True) + return d["task_id"], d + else: + logger.info("Skipping duplicate {}".format(d["dir_name"])) + return result["task_id"], result + + else: + d["task_id"] = 0 + logger.info("Simulated insert into database for {} with task_id {}" + .format(d["dir_name"], d["task_id"])) + return 0, d + + # AJ: take in old_task + def process_fw(self, old_task, d): + # AJ - this whole section is different + sma = SNLMongoAdapter.auto_load() + + d['old_engine'] = old_task.get('engine') + if 'fw_id' in old_task: + d['old_fw_id'] = old_task['fw_id'] + + d['fw_id'] = None + d['task_type'] = 'GGA+U optimize structure (2x)' if old_task[ + 'is_hubbard'] else 'GGA optimize structure (2x)' + d['submission_id'] = None + d['vaspinputset_name'] = None + + snl_d = sma.snl.find_one({'about._materialsproject.deprecated.mps_ids': old_task['mps_id']}) + if old_task.get('mps_id', -1) > 0 and snl_d: + # grab the SNL from the SNL db + del snl_d['_id'] + d['snl'] = snl_d + d['snlgroup_id'] = sma.snlgroups.find_one({'all_snl_ids': d['snl']['snl_id']}, {'snlgroup_id': 1})['snlgroup_id'] + + elif 'mps' in old_task and old_task['mps']: + snl = mps_dict_to_snl(old_task['mps']) + mpsnl, snlgroup_id = sma.add_snl(snl) + d['snl'] = mpsnl.as_dict() + d['snlgroup_id'] = snlgroup_id + else: + s = Structure.from_dict(old_task['input']['crystal']) + snl = StructureNL(s, 'Anubhav Jain ', remarks=['origin unknown']) + mpsnl, snlgroup_id = sma.add_snl(snl) + d['snl'] = mpsnl.as_dict() + d['snlgroup_id'] = snlgroup_id + + + if 'optimize structure' in d['task_type'] and 'output' in d: + # create a new SNL based on optimized structure + new_s = Structure.from_dict(d['output']['crystal']) + old_snl = StructureNL.from_dict(d['snl']) + history = old_snl.history + history.append( + {'name': 'Materials Project structure optimization', + 'url': 'http://www.materialsproject.org', + 'description': {'task_type': d['task_type'], + 'fw_id': d['fw_id'], + 'task_id': d['task_id']}}) + new_snl = StructureNL(new_s, old_snl.authors, old_snl.projects, + old_snl.references, old_snl.remarks, + old_snl.data, history) + + # add snl + mpsnl, snlgroup_id = sma.add_snl(new_snl, snlgroup_guess=d['snlgroup_id']) + + d['snl_final'] = mpsnl.as_dict() + d['snlgroup_id_final'] = snlgroup_id + d['snlgroup_changed'] = (d['snlgroup_id'] != + d['snlgroup_id_final']) + + # custom processing for detecting errors + dir_name = old_task['dir_name'] + new_style = os.path.exists(os.path.join(dir_name, 'FW.json')) + vasp_signals = {} + critical_errors = ["INPUTS_DONT_EXIST", + "OUTPUTS_DONT_EXIST", "INCOHERENT_POTCARS", + "VASP_HASNT_STARTED", "VASP_HASNT_COMPLETED", + "CHARGE_UNCONVERGED", "NETWORK_QUIESCED", + "HARD_KILLED", "WALLTIME_EXCEEDED", + "ATOMS_TOO_CLOSE", "DISK_SPACE_EXCEEDED"] + + last_relax_dir = dir_name + + if not new_style: + # get the last relaxation dir + # the order is relax2, current dir, then relax1. This is because + # after completing relax1, the job happens in the current dir. + # Finally, it gets moved to relax2. + # There are some weird cases where both the current dir and relax2 + # contain data. The relax2 is good, but the current dir is bad. + if is_valid_vasp_dir(os.path.join(dir_name, "relax2")): + last_relax_dir = os.path.join(dir_name, "relax2") + elif is_valid_vasp_dir(dir_name): + pass + elif is_valid_vasp_dir(os.path.join(dir_name, "relax1")): + last_relax_dir = os.path.join(dir_name, "relax1") + + vasp_signals['last_relax_dir'] = last_relax_dir + ## see what error signals are present + + print "getting signals for dir :{}".format(last_relax_dir) + + sl = SignalDetectorList() + sl.append(VASPInputsExistSignal()) + sl.append(VASPOutputsExistSignal()) + sl.append(VASPOutSignal()) + sl.append(HitAMemberSignal()) + sl.append(SegFaultSignal()) + sl.append(VASPStartedCompletedSignal()) + + signals = sl.detect_all(last_relax_dir) + + signals = signals.union(WallTimeSignal().detect(dir_name)) + if not new_style: + root_dir = os.path.dirname(dir_name) # one level above dir_name + signals = signals.union(WallTimeSignal().detect(root_dir)) + + signals = signals.union(DiskSpaceExceededSignal().detect(dir_name)) + if not new_style: + root_dir = os.path.dirname(dir_name) # one level above dir_name + signals = signals.union(DiskSpaceExceededSignal().detect(root_dir)) + + signals = list(signals) + + critical_signals = [val for val in signals if val in critical_errors] + + vasp_signals['signals'] = signals + vasp_signals['critical_signals'] = critical_signals + + vasp_signals['num_signals'] = len(signals) + vasp_signals['num_critical'] = len(critical_signals) + + if len(critical_signals) > 0 and d['state'] == "successful": + d["state"] = "error" + + d['analysis'] = d.get('analysis', {}) + d['analysis']['errors_MP'] = vasp_signals + + d['run_tags'] = ['PBE'] + d['run_tags'].extend(d['pseudo_potential']['labels']) + d['run_tags'].extend([e+"="+str(d['hubbards'].get(e, 0)) for e in d['elements']]) diff --git a/mpworks/fix_scripts/legacy/submit_snl.py b/mpworks/fix_scripts/legacy/submit_snl.py new file mode 100755 index 00000000..c1081956 --- /dev/null +++ b/mpworks/fix_scripts/legacy/submit_snl.py @@ -0,0 +1,29 @@ +from mpworks.snl_utils.mpsnl import MPStructureNL +from mpworks.snl_utils.snl_mongo import SNLMongoAdapter +from mpworks.submission.submission_mongo import SubmissionMongoAdapter +from mpworks.workflows.wf_utils import NO_POTCARS + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 14, 2013' + + +def submit_all_snl(snldb, sma, snlgroup_constraint=None): + constraints = {'is_ordered': True, 'is_valid': True, 'nsites': {'$lte': 200}, 'canonical_snl.about.projects': {'$ne': 'CederDahn Challenge'}} + constraints['elements'] = {'$nin': NO_POTCARS} + constraints['canonical_snl.about.history.name'] = {"$ne":"Materials Project structure optimization"} + constraints['canonical_snl.about.remarks'] = {"$ne": "DEPRECATED"} + + if snlgroup_constraint: + constraints['snlgroup_id'] = snlgroup_constraint + + for result in snldb.snlgroups.find(constraints, {'canonical_snl': 1, 'snlgroup_id': 1}): + snl = MPStructureNL.from_dict(result['canonical_snl']) + parameters = {'snlgroup_id': result['snlgroup_id']} + sma.submit_snl(snl, 'Anubhav Jain ', parameters=parameters) + +if __name__ == '__main__': + submit_all_snl(0, 0) \ No newline at end of file diff --git a/mpworks/fix_scripts/legacy/task_to_fw.py b/mpworks/fix_scripts/legacy/task_to_fw.py new file mode 100755 index 00000000..0bc3c71f --- /dev/null +++ b/mpworks/fix_scripts/legacy/task_to_fw.py @@ -0,0 +1,62 @@ +import datetime +from fireworks.core.firework import Firework, Launch, FWAction, Workflow +from fireworks.utilities.fw_utilities import get_slug +from mpworks.firetasks.controller_tasks import DummyLegacyTask +from mpworks.snl_utils.mpsnl import get_meta_from_structure +from pymatgen import Composition, Structure + + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 14, 2013' + +# Before the FireWorks code existed, we had already run many jobs. This code was used to convert all those jobs into "fake" FireWorks runs that were entered into the database for legacy and completeness. + +def task_dict_to_wf(task_dict, launchpad): + fw_id = launchpad.get_new_fw_id() + l_id = launchpad.get_new_launch_id() + + spec = {'task_type': task_dict['task_type'], 'run_tags': task_dict['run_tags'], + 'vaspinputset_name': None, 'vasp': None, 'mpsnl': task_dict['snl'], + 'snlgroup_id': task_dict['snlgroup_id']} + tasks = [DummyLegacyTask()] + + launch_dir = task_dict['dir_name_full'] + + stored_data = {'error_list': []} + update_spec = {'prev_vasp_dir': task_dict['dir_name'], + 'prev_task_type': spec['task_type'], + 'mpsnl': spec['mpsnl'], 'snlgroup_id': spec['snlgroup_id'], + 'run_tags': spec['run_tags']} + + fwaction = FWAction(stored_data=stored_data, update_spec=update_spec) + + if task_dict['completed_at']: + complete_date = datetime.datetime.strptime(task_dict['completed_at'], "%Y-%m-%d %H:%M:%S") + state_history = [{"created_on": complete_date, 'state': 'COMPLETED'}] + else: + state_history = [] + + launches = [Launch('COMPLETED', launch_dir, fworker=None, host=None, ip=None, action=fwaction, + state_history=state_history, launch_id=l_id, fw_id=fw_id)] + + f = Composition(task_dict['pretty_formula']).alphabetical_formula + + + fw = Firework(tasks, spec, name=get_slug(f + '--' + spec['task_type']), launches=launches, state='COMPLETED', created_on=None, + fw_id=fw_id) + + wf_meta = get_meta_from_structure(Structure.from_dict(task_dict['snl'])) + wf_meta['run_version'] = 'preproduction (0)' + + wf = Workflow.from_FireWork(fw, name=f, metadata=wf_meta) + + launchpad.add_wf(wf, reassign_all=False) + launchpad._upsert_launch(launches[0]) + + print 'ADDED', fw_id + # return fw_id + return fw_id \ No newline at end of file diff --git a/mpworks/fix_scripts/reparse_old_tasks.py b/mpworks/fix_scripts/reparse_old_tasks.py index 8311f4fa..4adb4d91 100644 --- a/mpworks/fix_scripts/reparse_old_tasks.py +++ b/mpworks/fix_scripts/reparse_old_tasks.py @@ -1,5 +1,5 @@ from monty.os.path import zpath -from pymatgen.io.vaspio import Outcar +from pymatgen.io.vasp import Outcar __author__ = 'Anubhav Jain' __copyright__ = 'Copyright 2013, The Materials Project' @@ -107,4 +107,4 @@ def _analyze(data): print 'GOT all tasks...' pool = multiprocessing.Pool(16) pool.map(_analyze, m_data) - print 'DONE' \ No newline at end of file + print 'DONE' diff --git a/mpworks/fix_scripts/reparse_old_tasks_again.py b/mpworks/fix_scripts/reparse_old_tasks_again.py index 0e274277..232d14cd 100644 --- a/mpworks/fix_scripts/reparse_old_tasks_again.py +++ b/mpworks/fix_scripts/reparse_old_tasks_again.py @@ -1,5 +1,5 @@ from monty.os.path import zpath -from pymatgen.io.vaspio import Outcar +from pymatgen.io.vasp import Outcar __author__ = 'Anubhav Jain' __copyright__ = 'Copyright 2013, The Materials Project' diff --git a/mpworks/fix_scripts/rerun_boltztrap.py b/mpworks/fix_scripts/rerun_boltztrap.py new file mode 100755 index 00000000..67cd9873 --- /dev/null +++ b/mpworks/fix_scripts/rerun_boltztrap.py @@ -0,0 +1,92 @@ +import os, sys +from glob import glob +from dateutil import parser +from datetime import datetime +from fireworks.core.launchpad import LaunchPad +from collections import Counter +from fnmatch import fnmatch + +lpdb = LaunchPad.from_file('/global/homes/m/matcomp/mp_prod/config/config_Mendel/my_launchpad.yaml') + +""" +counter = Counter() +for wf_idx, wf_doc in enumerate(lpdb.workflows.find( + {'updated_on': {'$exists': 1}}, + {'state': 1, 'updated_on': 1, 'nodes': 1} +)): + try: + dt = parser.parse(wf_doc['updated_on']) + except: + dt = wf_doc['updated_on'] + counter['ALL_WFS'] += 1 + if dt > datetime(2016, 1, 1): + counter['ALL_RECENT_WFS'] += 1 + fws_fizzled = [] + for fw_idx, fw_doc in enumerate(lpdb.fireworks.find( + {'fw_id': {'$in': wf_doc['nodes']}, 'updated_on': {'$exists': 1}, 'spec.task_type': {'$ne': 'GGA Boltztrap'}}, + {'fw_id': 1, 'state': 1, 'updated_on': 1, 'launches': 1, 'spec.task_type': 1} + )): + try: + dt = parser.parse(fw_doc['updated_on']) + except: + dt = fw_doc['updated_on'] + if dt > datetime(2016, 1, 1): + counter['ALL_RECENT_FWS'] += 1 + counter['FW_' + fw_doc['state']] += 1 + if fw_doc['state'] == 'FIZZLED': + fws_fizzled.append('_'.join([str(fw_doc['fw_id']), fw_doc['spec']['task_type']])) + if fnmatch(fw_doc['spec']['task_type'], '*GGA optimize structure*'): + lpdb.rerun_fw(fw_doc['fw_id']) + print 'rerunning', fw_doc['fw_id'] + if fws_fizzled: + print '{}:{}> {}'.format(counter['ALL_RECENT_WFS'], wf_idx, fws_fizzled) + if len(fws_fizzled) < 2: + sys.exit(0) +print counter +""" + +counter = Counter() +for fw_doc in lpdb.fireworks.find( + {'updated_on': {'$exists': 1}, 'spec.task_type': 'GGA Boltztrap'}, + {'fw_id': 1, 'state': 1, 'updated_on': 1, 'launches': 1} +): + try: + dt = parser.parse(fw_doc['updated_on']) + except: + dt = fw_doc['updated_on'] + if dt > datetime(2016, 1, 1): + counter['RECENT_BTZ_FWS_ALL'] += 1 + if fw_doc['state'] == 'RUNNING': + launch_dir = lpdb.launches.find_one({'launch_id': fw_doc['launches'][0]}, {'launch_dir':1, '_id':0})['launch_dir'] + with open(glob(os.path.join(launch_dir, '*.error'))[0]) as ferr: + last_line = ferr.readlines()[-1].strip() + if 'TIME LIMIT' in last_line: + lpdb.rerun_fw(fw_doc['fw_id']) + print '[{}] rerun due to TIME LIMIT'.format(fw_doc['fw_id']) + else: + counter['RECENT_BTZ_FWS_' + fw_doc['state']] += 1 + else: + #wf = lpdb.workflows.find_one({'nodes': fw_doc['fw_id']}, {'parent_links':1}) + #parent_fw_id = wf['parent_links'][str(fw_doc['fw_id'])][-1] + #parent_fw = lpdb.fireworks.find_one({'fw_id': parent_fw_id}, {'state':1}) + #if parent_fw['state'] == 'COMPLETED': + counter['RECENT_BTZ_FWS_' + fw_doc['state']] += 1 +print counter + +nfws = 0 +for fw_doc in lpdb.fireworks.find( + {'spec.task_type': 'GGA Boltztrap', 'state': 'FIZZLED'}, + {'fw_id': 1, 'launches': 1, 'state': 1 } +): + wf = lpdb.workflows.find_one({'nodes': fw_doc['fw_id']}, {'parent_links':1}) + launch_dir = lpdb.launches.find_one({'launch_id': fw_doc['launches'][0]}, {'launch_dir':1, '_id':0})['launch_dir'] + with open(glob(os.path.join(launch_dir, '*.error'))[0]) as ferr: + last_line = ferr.readlines()[-1].strip() + if 'parent job unsuccessful' in last_line or 'Could not find task' in last_line: + parent_fw_id = wf['parent_links'][str(fw_doc['fw_id'])][-1] + lpdb.rerun_fw(parent_fw_id) + print '[{}] {} --> marked parent {} for rerun'.format(nfws, fw_doc['fw_id'], parent_fw_id) + else: + #lpdb.rerun_fw(fw_doc['fw_id']) + print '[{}] {} --> {}'.format(nfws, fw_doc['fw_id'], last_line) + nfws += 1 diff --git a/mpworks/go_submissions b/mpworks/go_submissions new file mode 100755 index 00000000..c6ad2ee0 --- /dev/null +++ b/mpworks/go_submissions @@ -0,0 +1,4 @@ +from mpworks.scripts.submissions_run import go_submissions + +if __name__ == '__main__': + go_submissions() \ No newline at end of file diff --git a/mpworks/go_testing b/mpworks/go_testing new file mode 100755 index 00000000..10e88004 --- /dev/null +++ b/mpworks/go_testing @@ -0,0 +1,4 @@ +from mpworks.scripts.submit_canonical_run import go_testing + +if __name__ == '__main__': + go_testing() \ No newline at end of file diff --git a/mpworks/maintenance_scripts/README.md b/mpworks/maintenance_scripts/README.md new file mode 100755 index 00000000..46edb7ec --- /dev/null +++ b/mpworks/maintenance_scripts/README.md @@ -0,0 +1,3 @@ +# Maintenance scripts + +These are scripts that are not needed to run the production workflow, but might be helpful for maintaining runs or in future coding. \ No newline at end of file diff --git a/mpworks/maintenance_scripts/__init__.py b/mpworks/maintenance_scripts/__init__.py new file mode 100755 index 00000000..ca35517e --- /dev/null +++ b/mpworks/maintenance_scripts/__init__.py @@ -0,0 +1 @@ +__author__ = 'ajain' diff --git a/mpworks/maintenance_scripts/classify_fizzled.py b/mpworks/maintenance_scripts/classify_fizzled.py new file mode 100755 index 00000000..2f644071 --- /dev/null +++ b/mpworks/maintenance_scripts/classify_fizzled.py @@ -0,0 +1,116 @@ +from collections import defaultdict +import os +from pymongo import MongoClient +import yaml +from fireworks.core.launchpad import LaunchPad + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'Nov 11, 2013' + +# This script tries to examine the FIZZLED FWS and classify them into groups +# This can be used to identify the greatest causes of failure and fix those first +# The types of failure groups will need to be updated + +def get_parent_launch_locs(fw_id, lpdb): + parent_fw_id = lpdb.workflows.find_one({"nodes": fw_id}, {"parent_links":1})['parent_links'][str(fw_id)][0] + launch_ids = lpdb.fireworks.find_one({"fw_id": parent_fw_id},{'launches': 1})['launches'] + locs = [] + ran_fws = [] + for l in launch_ids: + d = lpdb.launches.find_one({"launch_id": l}, {'launch_dir': 1, 'fw_id': 1}) + launch_loc = str(d['launch_dir']) + ran_fws.append(d['fw_id']) + locs.append("/project/projectdirs/matgen/garden/"+launch_loc[launch_loc.find('block_'):]) + + return locs, parent_fw_id, ran_fws + +def get_task_info(fw_id, tdb): + x = tdb.tasks.find_one({"fw_id": fw_id}, {"analysis": 1}) + warnings = x['analysis'].get('warnings', []) + warnings.extend(x['analysis']['errors_MP']['signals']) + errors = x['analysis'].get('errors', []) + errors.extend(x['analysis']['errors_MP']['critical_signals']) + + warnings = set(warnings) + errors = set(errors) + warnings = warnings.difference(errors) + return set(warnings), set(errors) + + +if __name__ == '__main__': + module_dir = os.path.dirname(os.path.abspath(__file__)) + lp_f = os.path.join(module_dir, 'my_launchpad.yaml') + lpdb = LaunchPad.from_file(lp_f) + + tasks_f = os.path.join(module_dir, 'tasks_read.yaml') + creds = {} + with open(tasks_f) as f: + creds = yaml.load(f) + + connection = MongoClient(creds['host'], creds['port']) + tdb = connection[creds['db']] + tdb.authenticate(creds['username'], creds['password']) + + + except_dict = defaultdict(int) + fizzled_fws = [] + + + + for f in lpdb.fireworks.find({"state": "FIZZLED"}, {"fw_id":1}): + fizzled_fws.append(f['fw_id']) + + for l in lpdb.launches.find({"state": "FIZZLED", "action":{"$ne": None}}, {"action":1, 'fw_id': 1, 'time_start': 1, 'launch_dir':1}, timeout=False): + if l['fw_id'] in fizzled_fws: + except_str = l['action']['stored_data'].get('_exception') + if 'Disk quota exceeded' in except_str: + except_dict['DISK_QUOTA_EXCEEDED'] = except_dict['DISK_QUOTA_EXCEEDED']+1 + print l['fw_id'], '*' + lpdb.rerun_fw(l['fw_id']) + elif 'No such file' in except_str: + # this is due to missing CHGCAR from Michael's old runs + except_dict['NO_SUCH_FILE'] = except_dict['NO_SUCH_FILE']+1 + elif 'IMPROPER PARSING' in except_str: + except_dict['IMPROPER_PARSING'] = except_dict['IMPROPER_PARSING']+1 + elif 'get valid results from relaxed run' in except_str: + except_dict['INVALID_RESULTS'] = except_dict['INVALID_RESULTS']+1 + elif 'dir does not exist!' in except_str: + except_dict['MISSING_DIR'] = except_dict['MISSING_DIR']+1 + elif 'Stale NFS file handle' in except_str: + except_dict['STALE_NFS'] = except_dict['STALE_NFS']+1 + elif 'File exists' in except_str: + except_dict['FILE_EXISTS'] = except_dict['FILE_EXISTS']+1 + elif 'MemoryError' in except_str: + except_dict['MEMORY_ERROR'] = except_dict['MEMORY_ERROR']+1 + elif 'DB insertion successful, but don\'t know how to fix' in except_str: + except_dict['NO_FIX'] = except_dict['NO_FIX']+1 + """ + launches, pfw_id, ran_fws = get_parent_launch_locs(l['fw_id'], lpdb) + print '--',l['fw_id'] + for idx, l in enumerate(launches): + print l + print get_task_info(ran_fws[idx], tdb) + """ + + + elif 'Poscar.from_string' in except_str and 'chunks[0]' in except_str: + except_dict['POSCAR_PARSE'] = except_dict['POSCAR_PARSE']+1 + elif 'TypeError: integer argument expected, got float' in except_str: + except_dict['MAXRUN_TYPE'] = except_dict['MAXRUN_TYPE']+1 + elif 'cannot import name DupeFinderDB' in except_str: + except_dict['DUPEFINDER_DB'] = except_dict['DUPEFINDER_DB']+1 + elif 'jinja2' in except_str: + except_dict['JINJA2'] = except_dict['JINJA2']+1 + elif 'run_tags' in except_str: + except_dict['RUN_TAGS'] = except_dict['RUN_TAGS']+1 + else: + except_dict[except_str] = except_dict[except_str]+1 + + print '-----' + for k, v in except_dict.iteritems(): + print {"{}\t{}".format(v, k)} + diff --git a/mpworks/maintenance_scripts/deprecate_snl.py b/mpworks/maintenance_scripts/deprecate_snl.py new file mode 100755 index 00000000..d935bc23 --- /dev/null +++ b/mpworks/maintenance_scripts/deprecate_snl.py @@ -0,0 +1,23 @@ +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2014, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'Oct 13, 2014' + + +def deprecate_snl(snl_db, snl_id, remarks): + remarks.append('DEPRECATED') + # FIX SNL + remarks.extend(snl_db.snl.find_one({'snl_id': snl_id}, {'about.remarks': 1})['about']['remarks']) + remarks = list(set(remarks)) + + # push existing remarks + snl_db.snl.update({'snl_id': snl_id}, {'$set': {"about.remarks": remarks}}) + + # FIX SNLGROUPS + sg = snl_db.snlgroups.find_one({'canonical_snl.snl_id': snl_id}, {'snlgroup_id': 1}) + if sg: + snl_db.snlgroups.update({'snlgroup_id': sg['snlgroup_id']}, {'$set': {"canonical_snl.about.remarks": remarks}}) + + print('FINISHED deprecating {}'.format(snl_id)) \ No newline at end of file diff --git a/mpworks/maintenance_scripts/icsd2012_to_snl.py b/mpworks/maintenance_scripts/icsd2012_to_snl.py new file mode 100755 index 00000000..c7b3c79b --- /dev/null +++ b/mpworks/maintenance_scripts/icsd2012_to_snl.py @@ -0,0 +1,91 @@ +import re +import random +import unicodedata +import datetime +from pymatgen import Structure +from pymatgen.matproj.snl import StructureNL + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 12, 2013' + +# Convert ICSD database (already converted to MongoDB by MIT from an SQL source) into SNL + +def icsd_dict_to_snl(icsd_dict): + if 'structure' not in icsd_dict: + return None + + struct = Structure.from_dict(icsd_dict['structure']) + references = _get_icsd_reference(icsd_dict) + + data = {'_icsd': {}} + excluded_data = ['_id', 'a_len', 'b_len', 'c_len', 'alpha', 'beta', 'gamma', 'compostion', 'composition', 'created_at', 'crystal_id', 'idnum', 'journal', 'tstruct', 'updated_at', 'username'] + for k, v in icsd_dict.iteritems(): + if k not in excluded_data: + if isinstance(v, datetime.datetime): + v = v.strftime(format='%Y-%m-%d %H:%M:%S') + data['_icsd'][k] = v + + projects = None + remarks = None + + history = [{'name': 'Inorganic Crystal Structure Database (ICSD)', 'url': 'http://icsd.fiz-karlsruhe.de/', 'description': {'icsd_id': data['_icsd']['icsd_id']}}, {'name': 'pymatgen', 'url': 'https://pypi.python.org/pypi/pymatgen', 'description': {'comment': 'converted to explicit structure'}}] + + authors = 'William Davidson Richards , Shyue Ping Ong , Stephen Dacek , Anubhav Jain ' + + return StructureNL(struct, authors, projects, references, remarks, data, history) + + +def _get_icsd_reference(icsd_dict): + + if icsd_dict and 'journal' in icsd_dict and icsd_dict['journal']['authors']: + pages = "" + if icsd_dict['journal']['PAGE_FIRST']: + pages = str(icsd_dict['journal']['PAGE_FIRST']) + + if icsd_dict['journal']['PAGE_LAST']: + pages = pages + "--" + str(icsd_dict['journal']['PAGE_LAST']) + + bibtex_str = "@article{" + #author last name as key + m_key = icsd_dict['journal']['authors'][0] + m_key = re.sub(r'\s', '_', m_key) + m_key = m_key[0:m_key.find(',')] + bibtex_str += m_key + #year + random + bibtex_str += str(icsd_dict['journal']['YEAR']) + "_" + str(random.randrange(1, 1000)) + ",\n" + + bibtex_str += "title = {{" + icsd_dict['au_title']+ "}},\n" + + auth_str = "author = {" + " and ".join(icsd_dict['journal']['authors']) + "},\n" + # sanitize authors so there are no parentheses (weird ICSD conversion thing) + regex = re.compile('\(.+?\)') + auth_str = regex.sub('', auth_str) + bibtex_str += auth_str + + if icsd_dict['journal']['YEAR']: + bibtex_str += "year = {" + str(icsd_dict['journal']['YEAR']) + "},\n" + + if icsd_dict['journal']['J_TITLE']: + bibtex_str += "journal = {" + icsd_dict['journal']['J_TITLE'] + "},\n" + + if icsd_dict['journal']['VOLUME']: + bibtex_str += "volume = {" + str(icsd_dict['journal']['VOLUME']) + "},\n" + + if icsd_dict['journal']['ISSUE']: + bibtex_str += "issue = {" + str(icsd_dict['journal']['ISSUE']) + "},\n" + bibtex_str += "pages = {" + pages + "},\n" + + if icsd_dict['journal']['ISSN']: + bibtex_str += "issn = " + icsd_dict['journal']['ISSN'] + "\n" + + bibtex_str += "}" + bibtex_str = unicodedata.normalize('NFKD', bibtex_str).encode('ascii','ignore') + + return bibtex_str + + + return None \ No newline at end of file diff --git a/mpworks/maintenance_scripts/modify_snl.py b/mpworks/maintenance_scripts/modify_snl.py new file mode 100755 index 00000000..08044fb4 --- /dev/null +++ b/mpworks/maintenance_scripts/modify_snl.py @@ -0,0 +1,133 @@ +from collections import namedtuple +import os +from pymongo import MongoClient +import yaml +from fireworks.core.launchpad import LaunchPad +from mpworks.snl_utils.mpsnl import MPStructureNL +from mpworks.snl_utils.snl_mongo import SNLMongoAdapter +from mpworks.submission.submission_mongo import SubmissionMongoAdapter +from pymatgen.matproj.snl import StructureNL +import datetime + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2014, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'Feb 20, 2014' + + +""" +This is used to modify SNL data. An SNL should not just be changed in the SNL collection because +that SNL is referred to in many different databases. + +This code tries to properly update all relevant databases with the SNL +changes, not just the basic SNL collection. + +Note that the lattice and sites of an SNL cannot be changed! This would be a different material +altogether and have affected the runs / duplicate checking. +""" + + +module_dir = os.path.dirname(os.path.abspath(__file__)) +snl_f = os.path.join(module_dir, 'snl.yaml') +fw_f = os.path.join(module_dir, 'my_launchpad.yaml') +tasks_f = os.path.join(module_dir, 'tasks.yaml') + +def get_colls(): + colls = namedtuple('Collections', ['snl', 'snlgroups']) + sma = SNLMongoAdapter.from_file(snl_f) + lp = LaunchPad.from_file(fw_f) + + colls.snl = sma.snl + colls.snlgroups = sma.snlgroups + colls.fireworks = lp.fireworks + colls.launches = lp.launches + + with open(tasks_f) as f2: + task_creds = yaml.load(f2) + + mc = MongoClient(task_creds['host'], task_creds['port']) + db = mc[task_creds['database']] + db.authenticate(task_creds['admin_user'], task_creds['admin_password']) + colls.tasks = db['tasks'] + + return colls + + +def modify_snl(snl_id, new_snl, colls, reject_bad_tasks=False): + # get the old SNL lattice and sites + snl_old = colls.snl.find_one({'snl_id': snl_id}, {'lattice': 1, 'sites': 1, 'snl_timestamp': 1}) + + # enforce the new SNL's lattice/sites to be same as old + snl_d = new_snl.as_dict() + snl_d['lattice'] = snl_old['lattice'] + snl_d['sites'] = snl_old['sites'] + snl_d['snl_timestamp'] = snl_old['snl_timestamp'] + + # insert the new SNL into the snl collection + print 'INSERTING SNL_ID', {'snl_id': snl_id}, snl_d + colls.snl.update({'snl_id': snl_id}, snl_d) + + # update the canonical SNL of the group + for s in colls.snlgroups.find({'canonical_snl.about._materialsproject.snl_id': snl_id}, {'snlgroup_id': 1}): + print 'CHANGING SNLGROUP_ID', s['snlgroup_id'] + colls.snlgroups.find_and_modify({'snlgroup_id': s['snlgroup_id']}, {'$set': {'canonical_snl': snl_d}}) + + # update FWs pt 1 + for f in colls.fireworks.find({'spec.mpsnl.about._materialsproject.snl_id': snl_id}, {'fw_id': 1}): + print 'CHANGING FW_ID', f['fw_id'] + colls.fireworks.find_and_modify({'fw_id': f['fw_id']}, {'$set': {'spec.mpsnl': snl_d}}) + + # update FWs pt 2 + for f in colls.fireworks.find({'spec.force_mpsnl.about._materialsproject.snl_id': snl_id}, {'fw_id': 1}): + print 'CHANGING FW_ID', f['fw_id'] + colls.fireworks.find_and_modify({'fw_id': f['fw_id']}, {'$set': {'spec.force_mpsnl': snl_d}}) + + # update Launches + for l in colls.launches.find({'action.update_spec.mpsnl.about._materialsproject.snl_id': snl_id}, {'launch_id': 1}): + print 'CHANGING LAUNCH_ID', l['launch_id'] + colls.launches.find_and_modify({'launch_id': l['launch_id']}, {'$set': {'action.update_spec.mpsnl': snl_d}}) + + # update tasks initial + for t in colls.tasks.find({'snl.about._materialsproject.snl_id': snl_id}, {'task_id': 1}): + print 'CHANGING init TASK_ID', t['task_id'] + colls.tasks.find_and_modify({'task_id': t['task_id']}, {'$set': {'snl': snl_d}}) + if reject_bad_tasks: + print 'REJECTING TASK_ID', t['task_id'] + colls.tasks.find_and_modify({'task_id': t['task_id']}, {'$set': {'state': 'rejected'}}) + colls.tasks.find_and_modify({'task_id': t['task_id']}, {'$push': {'analysis.errors_MP.critical_signals': 'BAD STRUCTURE SNL'}}) + colls.tasks.find_and_modify({'task_id': t['task_id']}, {'$inc': {'analysis.errors_MP.num_critical': 1}}) + + + # update tasks final + for t in colls.tasks.find({'snl_final.about._materialsproject.snl_id': snl_id}, {'task_id': 1}): + print 'CHANGING final TASK_ID', t['task_id'] + colls.tasks.find_and_modify({'task_id': t['task_id']}, {'$set': {'snl_final': snl_d}}) + if reject_bad_tasks: + print 'REJECTING TASK_ID', t['task_id'] + colls.tasks.find_and_modify({'task_id': t['task_id']}, {'$set': {'state': 'rejected'}}) + colls.tasks.find_and_modify({'task_id': t['task_id']}, {'$push': {'analysis.errors_MP.critical_signals': 'BAD STRUCTURE SNL'}}) + colls.tasks.find_and_modify({'task_id': t['task_id']}, {'$inc': {'analysis.errors_MP.num_critical': 1}}) + + # note: for now we are not fixing submissions in order to keep a record of submissions accurate, and also because the SNL assignment comes after submission + + print 'DONE PROCESSING', snl_id + + +def get_deprecated_snl(snl_id, colls): + snl_old = colls.snl.find_one({'snl_id': snl_id}) + del snl_old['about']['_icsd'] + snl_old['about']['remarks'].append('Record updated (about._icsd deleted) {}'.format(datetime.datetime.now().strftime('%Y-%m-%d'))) + return MPStructureNL.from_dict(snl_old) + + +if __name__ == '__main__': + + colls = get_colls() + snl_id = 1579 + + snl_new = get_deprecated_snl(snl_id, colls) + print snl_new.as_dict() + + modify_snl(snl_id, snl_new, colls, reject_bad_tasks=True) \ No newline at end of file diff --git a/mpworks/maintenance_scripts/reparse_tasks.py b/mpworks/maintenance_scripts/reparse_tasks.py new file mode 100755 index 00000000..f8c7f057 --- /dev/null +++ b/mpworks/maintenance_scripts/reparse_tasks.py @@ -0,0 +1,115 @@ +from argparse import ArgumentParser +import json +import logging +import os +import sys +from pymongo import MongoClient +from fireworks.core.launchpad import LaunchPad +from mpworks.drones.mp_vaspdrone import MPVaspDrone +import multiprocessing +import traceback + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 13, 2013' + +''' +This script re-runs the MPVaspDrone over all the *new-style* tasks. It can be used when the MPVaspDrone is updated. + +A few notes: +* The old-style tasks will be unaffected by this script +* The dos_fs and band_structure_fs collections should be completely deleted before running this script over the database. + +Note - AJ has not run this code since its inception in May 2013. Changes may be needed. +''' + +class TaskBuilder(): + + @classmethod + def setup(cls): + db_dir = os.environ['DB_LOC'] + db_path = os.path.join(db_dir, 'tasks_db.json') + with open(db_path) as f2: + db_creds = json.load(f2) + mc2 = MongoClient(db_creds['host'], db_creds['port']) + db2 = mc2[db_creds['database']] + db2.authenticate(db_creds['admin_user'], db_creds['admin_password']) + + cls.tasks = db2['tasks'] + cls.host = db_creds['host'] + cls.port = db_creds['port'] + cls.database = db_creds['database'] + cls.collection = db_creds['collection'] + cls.admin_user = db_creds['admin_user'] + cls.admin_password = db_creds['admin_password'] + + def process_task(self, data): + + try: + dir_name = data[0] + parse_dos = data[1] + prev_info = self.tasks.find_one({'dir_name_full': dir_name}, {'task_type': 1, 'snl_final': 1, 'snlgroup_id_final': 1, 'snlgroup_changed': 1}) + drone = MPVaspDrone( + host=self.host, port=self.port, + database=self.database, user=self.admin_user, + password=self.admin_password, + collection=self.collection, parse_dos=parse_dos, + additional_fields={}, + update_duplicates=True) + t_id, d = drone.assimilate(dir_name, launches_coll=LaunchPad.auto_load().launches) + + + self.tasks.update({"task_id": t_id}, {"$set": {"snl_final": prev_info['snl_final'], "snlgroup_id_final": prev_info['snlgroup_id_final'], "snlgroup_changed": prev_info['snlgroup_changed']}}) + print 'FINISHED', t_id + except: + print '-----' + print 'ENCOUNTERED AN EXCEPTION!!!', data[0] + traceback.print_exc() + print '-----' + + +def _analyze(data): + b = TaskBuilder() + return b.process_task(data) + + +if __name__ == '__main__': + + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger('MPVaspDrone') + logger.setLevel(logging.INFO) + sh = logging.StreamHandler(stream=sys.stdout) + sh.setLevel(getattr(logging, 'INFO')) + logger.addHandler(sh) + + finished_tasks = [] + module_dir = os.path.dirname(os.path.abspath(__file__)) + if os.path.exists(os.path.join(module_dir, 'finished_tasks.txt')): + with open(os.path.join(module_dir, 'finished_tasks.txt')) as f: + for line in f: + task = line.split()[1].strip() + finished_tasks.append(task) + + o = TaskBuilder() + o.setup() + tasks = TaskBuilder.tasks + m_data = [] + # q = {'submission_id': {'$exists': False}} # these are all new-style tasks + #q = {"task_type":{"$regex":"band structure"}, "state":"successful", "calculations.0.band_structure_fs_id":{"$exists":False}} + + parser = ArgumentParser() + parser.add_argument('min', help='min', type=int) + parser.add_argument('max', help='max', type=int) + args = parser.parse_args() + q = {"task_id_deprecated": {"$lte": args.max, "$gte":args.min}, "is_deprecated": True} + + for d in tasks.find(q, {'dir_name_full': 1, 'task_type': 1, 'task_id': 1}, timeout=False): + if d['task_id'] in finished_tasks: + print 'DUPLICATE', d['task_id'] + else: + o.process_task((d['dir_name_full'], 'Uniform' in d['task_type'])) + # m_data.append((d['dir_name_full'], 'Uniform' in d['task_type'])) + print 'DONE' \ No newline at end of file diff --git a/mpworks/osti_doi/builders.py b/mpworks/osti_doi/builders.py index 275a7484..8c22e4cc 100644 --- a/mpworks/osti_doi/builders.py +++ b/mpworks/osti_doi/builders.py @@ -5,9 +5,10 @@ import plotly.plotly as py from plotly.graph_objs import * -today = datetime.date.today() +now = datetime.datetime.now() dirname = os.path.dirname(os.path.realpath(__file__)) -logfile = os.path.join(dirname, 'logs', 'dois_{}.log'.format(today)) +backupfile = os.path.join(dirname, 'dois.json') +logfile = os.path.join(dirname, 'logs', 'dois_{}.log'.format(now)) _log = logging.getLogger('mg.build') _log.setLevel(logging.INFO) fh = logging.FileHandler(logfile) @@ -36,10 +37,7 @@ def get_items(self, nmats=2, dois=None, materials=None): :param materials: 'materials' collection in 'mg_core_dev/prod' :type materials: QueryEngine """ - self.osti_record = OstiRecord( - n=nmats, doicoll=dois.collection, matcoll=materials.collection - ) - self.osti_record.submit() + self.nmats = nmats self.doi_qe = dois self.mat_qe = materials self.headers = {'Accept': 'text/bibliography; style=bibtex'} @@ -67,6 +65,22 @@ def process_item(self, item): #doi_url = 'http://doi.org/{}'.format(item['doi']) #doi_url = 'http://dx.doi.org/10.1038/nrd842' #r = requests.get(doi_url, headers=self.headers) + if item['doi'] is None: + # try loading doi from backup file, a.k.a reset item['doi'] (fixed manually) + if os.path.exists(backupfile): + with open(backupfile, 'r') as infile: + data = json.load(infile) + for d in data: + if d['_id'] == item['_id'] and d['doi'] is not None: + item['doi'] = d['doi'] + _log.info(self.doi_qe.collection.update( + {'_id': item['_id']}, {'$set': {'doi': item['doi']}} + )) + break + # if mp-id not found in backup (not fixed manually) + if item['doi'] is None: + _log.warning('missing DOI for {}. Fix manually in dois.json and rerun!'.format(item['_id'])) + return 0 osti_id = item['doi'].split('/')[-1] doi_url = 'http://www.osti.gov/dataexplorer/biblio/{}/cite/bibtex'.format(osti_id) try: @@ -102,8 +116,13 @@ def process_item(self, item): )) def finalize(self, errors): - filepath = os.path.join(dirname, 'dois.json') - with open(filepath, 'w') as outfile: + osti_record = OstiRecord( + n=self.nmats, + doicoll=self.doi_qe.collection, + matcoll=self.mat_qe.collection + ) + osti_record.submit() + with open(backupfile, 'w') as outfile: l = list(self.doi_qe.collection.find( fields={'created_at': True, 'doi': True} )) @@ -112,11 +131,11 @@ def finalize(self, errors): counts = [ self.mat_qe.collection.count(), self.doi_qe.collection.count(), - len(self.osti_record.matad.get_all_dois()) + len(osti_record.matad.get_all_dois()) ] for idx,stream_id in enumerate(stream_ids): s = py.Stream(stream_id) s.open() - s.write(dict(x=today, y=counts[idx])) + s.write(dict(x=now, y=counts[idx])) s.close() return True diff --git a/mpworks/osti_doi/dois.json b/mpworks/osti_doi/dois.json index ca5b71c1..2559ed89 100644 --- a/mpworks/osti_doi/dois.json +++ b/mpworks/osti_doi/dois.json @@ -18,5 +18,1170 @@ "created_at": "2015-04-29T20:29:00", "_id": "mp-188", "doi": "10.17188/1178782" + }, + { + "created_at": "2015-05-20T12:10:10.044316", + "_id": "mp-19918", + "doi": "10.17188/1182188" + }, + { + "created_at": "2015-05-21T12:10:32.864793", + "_id": "mp-1659", + "doi": "10.17188/1182267" + }, + { + "created_at": "2015-05-21T12:10:32.864817", + "_id": "mp-12657", + "doi": "10.17188/1182268" + }, + { + "created_at": "2015-05-21T12:10:32.864832", + "_id": "mp-780031", + "doi": "10.17188/1182266" + }, + { + "created_at": "2015-05-21T12:10:32.864846", + "_id": "mp-22452", + "doi": "10.17188/1182265" + }, + { + "created_at": "2015-05-22T12:11:59.410863", + "_id": "mp-943", + "doi": "10.17188/1182422" + }, + { + "created_at": "2015-05-22T12:11:59.410891", + "_id": "mp-12659", + "doi": "10.17188/1182420" + }, + { + "created_at": "2015-05-22T12:11:59.410906", + "_id": "mp-12651", + "doi": "10.17188/1182419" + }, + { + "created_at": "2015-05-22T12:11:59.410920", + "_id": "mp-600216", + "doi": "10.17188/1182421" + }, + { + "created_at": "2015-05-22T12:11:59.410934", + "_id": "mp-2738", + "doi": "10.17188/1182418" + }, + { + "created_at": "2015-05-25T12:12:13.803141", + "_id": "mp-549970", + "doi": "10.17188/1182448" + }, + { + "created_at": "2015-05-25T12:12:13.803172", + "_id": "mp-20112", + "doi": "10.17188/1182445" + }, + { + "created_at": "2015-05-25T12:12:13.803186", + "_id": "mp-569624", + "doi": "10.17188/1182444" + }, + { + "created_at": "2015-05-25T12:12:13.803201", + "_id": "mp-22441", + "doi": "10.17188/1182447" + }, + { + "created_at": "2015-05-25T12:12:13.803215", + "_id": "mp-30848", + "doi": "10.17188/1182446" + }, + { + "created_at": "2015-06-02T12:10:37.421059", + "_id": "mp-567842", + "doi": "10.17188/1183325" + }, + { + "created_at": "2015-06-02T12:10:37.421092", + "_id": "mp-19091", + "doi": "10.17188/1183324" + }, + { + "created_at": "2015-06-02T12:10:37.421108", + "_id": "mp-12671", + "doi": "10.17188/1183327" + }, + { + "created_at": "2015-06-02T12:10:37.421122", + "_id": "mp-12673", + "doi": "10.17188/1183328" + }, + { + "created_at": "2015-06-02T12:10:37.421136", + "_id": "mp-12668", + "doi": "10.17188/1183326" + }, + { + "created_at": "2015-05-26T12:11:50.121898", + "_id": "mp-12660", + "doi": "10.17188/1182585" + }, + { + "created_at": "2015-05-26T12:11:50.121826", + "_id": "mp-19721", + "doi": "10.17188/1182586" + }, + { + "created_at": "2015-05-26T12:11:50.121928", + "_id": "mp-2340", + "doi": "10.17188/1182584" + }, + { + "created_at": "2015-05-26T12:11:50.121914", + "_id": "mp-4709", + "doi": "10.17188/1182587" + }, + { + "created_at": "2015-05-26T12:11:50.121940", + "_id": "mp-552787", + "doi": "10.17188/1182588" + }, + { + "created_at": "2015-06-03T12:11:22.919904", + "_id": "mp-31899", + "doi": "10.17188/1183484" + }, + { + "created_at": "2015-06-03T12:11:22.919919", + "_id": "mp-19990", + "doi": "10.17188/1183480" + }, + { + "created_at": "2015-06-03T12:11:22.919933", + "_id": "mp-2310", + "doi": "10.17188/1183482" + }, + { + "created_at": "2015-06-03T12:11:22.919947", + "_id": "mp-558955", + "doi": "10.17188/1183487" + }, + { + "created_at": "2015-06-03T12:11:22.919961", + "_id": "mp-32306", + "doi": "10.17188/1183486" + }, + { + "created_at": "2015-06-04T12:11:33.410319", + "_id": "mp-12704", + "doi": "10.17188/1183581" + }, + { + "created_at": "2015-06-04T12:11:33.410349", + "_id": "mp-21467", + "doi": "10.17188/1183582" + }, + { + "created_at": "2015-06-04T12:11:33.410365", + "_id": "mp-12648", + "doi": "10.17188/1183580" + }, + { + "created_at": "2015-06-23T15:45:17.074902", + "_id": "mp-10096", + "doi": "10.17188/1186028" + }, + { + "created_at": "2015-06-23T15:45:17.074910", + "_id": "mp-10094", + "doi": "10.17188/1186027" + }, + { + "created_at": "2015-06-23T15:45:17.074917", + "_id": "mp-10092", + "doi": "10.17188/1186025" + }, + { + "created_at": "2015-06-23T15:45:17.074925", + "_id": "mp-10093", + "doi": "10.17188/1186026" + }, + { + "created_at": "2015-06-04T12:11:33.410379", + "_id": "mp-22891", + "doi": "10.17188/1183579" + }, + { + "created_at": "2015-06-04T12:11:33.410393", + "_id": "mp-20847", + "doi": "10.17188/1183578" + }, + { + "created_at": "2015-06-03T12:11:22.919822", + "_id": "mp-12669", + "doi": "10.17188/1183479" + }, + { + "created_at": "2015-06-03T12:11:22.919876", + "_id": "mp-12672", + "doi": "10.17188/1183488" + }, + { + "created_at": "2015-06-03T12:11:22.919976", + "_id": "mp-31756", + "doi": "10.17188/1183485" + }, + { + "created_at": "2015-06-03T12:11:22.919890", + "_id": "mp-472", + "doi": "10.17188/1183483" + }, + { + "created_at": "2015-06-03T12:11:22.919860", + "_id": "mp-561804", + "doi": "10.17188/1183481" + }, + { + "created_at": "2015-06-05T12:14:43.735929", + "_id": "mp-1016", + "doi": "10.17188/1183669" + }, + { + "created_at": "2015-06-05T12:14:43.735994", + "_id": "mp-12670", + "doi": "10.17188/1183668" + }, + { + "created_at": "2015-06-05T12:14:43.736265", + "_id": "mp-1522", + "doi": "10.17188/1183679" + }, + { + "created_at": "2015-06-05T12:14:43.735979", + "_id": "mp-18725", + "doi": "10.17188/1183675" + }, + { + "created_at": "2015-06-05T12:14:43.736250", + "_id": "mp-18759", + "doi": "10.17188/1183671" + }, + { + "created_at": "2015-06-05T12:14:43.736010", + "_id": "mp-20026", + "doi": "10.17188/1183667" + }, + { + "created_at": "2015-06-05T12:14:43.736222", + "_id": "mp-21167", + "doi": "10.17188/1183670" + }, + { + "created_at": "2015-06-05T12:14:43.736024", + "_id": "mp-22205", + "doi": "10.17188/1183678" + }, + { + "created_at": "2015-06-05T12:14:43.736165", + "_id": "mp-2369", + "doi": "10.17188/1183682" + }, + { + "created_at": "2015-06-05T12:14:43.736280", + "_id": "mp-24431", + "doi": "10.17188/1183673" + }, + { + "created_at": "2015-06-05T12:14:43.736151", + "_id": "mp-2790", + "doi": "10.17188/1183666" + }, + { + "created_at": "2015-06-05T12:14:43.736120", + "_id": "mp-3236", + "doi": "10.17188/1183665" + }, + { + "created_at": "2015-06-05T12:14:43.736179", + "_id": "mp-4865", + "doi": "10.17188/1183674" + }, + { + "created_at": "2015-06-05T12:14:43.736136", + "_id": "mp-5152", + "doi": "10.17188/1183676" + }, + { + "created_at": "2015-06-05T12:14:43.736193", + "_id": "mp-5370", + "doi": "10.17188/1183664" + }, + { + "created_at": "2015-06-05T12:14:43.736294", + "_id": "mp-556125", + "doi": "10.17188/1183677" + }, + { + "created_at": "2015-06-05T12:14:43.736039", + "_id": "mp-6574", + "doi": "10.17188/1183672" + }, + { + "created_at": "2015-06-05T12:14:43.736208", + "_id": "mp-6947", + "doi": "10.17188/1183680" + }, + { + "created_at": "2015-06-05T12:14:43.736236", + "_id": "mp-8539", + "doi": "10.17188/1183663" + }, + { + "created_at": "2015-06-08T12:20:41.250598", + "_id": "mp-22630", + "doi": "10.17188/1183756" + }, + { + "created_at": "2015-06-08T12:20:41.250672", + "_id": "mp-557319", + "doi": "10.17188/1183759" + }, + { + "created_at": "2015-06-08T12:20:41.250701", + "_id": "mp-2725", + "doi": "10.17188/1183749" + }, + { + "created_at": "2015-06-08T12:20:41.250762", + "_id": "mp-554483", + "doi": "10.17188/1183757" + }, + { + "created_at": "2015-06-08T12:20:41.250791", + "_id": "mp-600174", + "doi": "10.17188/1183755" + }, + { + "created_at": "2015-06-08T12:20:41.250465", + "_id": "mp-12678", + "doi": "10.17188/1183742" + }, + { + "created_at": "2015-06-08T12:20:41.250508", + "_id": "mp-17597", + "doi": "10.17188/1183753" + }, + { + "created_at": "2015-06-08T12:20:41.250525", + "_id": "mp-600156", + "doi": "10.17188/1183760" + }, + { + "created_at": "2015-06-08T12:20:41.250539", + "_id": "mp-557853", + "doi": "10.17188/1183752" + }, + { + "created_at": "2015-06-08T12:20:41.250553", + "_id": "mp-557562", + "doi": "10.17188/1183754" + }, + { + "created_at": "2015-06-08T12:20:41.250570", + "_id": "mp-613384", + "doi": "10.17188/1183746" + }, + { + "created_at": "2015-06-08T12:20:41.250584", + "_id": "mp-3555", + "doi": "10.17188/1183745" + }, + { + "created_at": "2015-06-08T12:20:41.250614", + "_id": "mp-3472", + "doi": "10.17188/1183758" + }, + { + "created_at": "2015-06-08T12:20:41.250629", + "_id": "mp-636232", + "doi": "10.17188/1183747" + }, + { + "created_at": "2015-06-08T12:20:41.250643", + "_id": "mp-4667", + "doi": "10.17188/1183741" + }, + { + "created_at": "2015-06-08T12:20:41.250657", + "_id": "mp-8368", + "doi": "10.17188/1183748" + }, + { + "created_at": "2015-06-08T12:20:41.250686", + "_id": "mp-196", + "doi": "10.17188/1183743" + }, + { + "created_at": "2015-06-08T12:20:41.250746", + "_id": "mp-22747", + "doi": "10.17188/1183750" + }, + { + "created_at": "2015-06-08T12:20:41.250776", + "_id": "mp-554808", + "doi": "10.17188/1183751" + }, + { + "created_at": "2015-06-05T12:14:43.736054", + "_id": "mp-2472", + "doi": "10.17188/1183681" + }, + { + "created_at": "2015-06-09T12:22:12.805423", + "_id": "mp-554793", + "doi": "10.17188/1183915" + }, + { + "created_at": "2015-06-09T12:22:12.805437", + "_id": "mp-556951", + "doi": "10.17188/1183911" + }, + { + "created_at": "2015-06-09T12:22:12.805451", + "_id": "mp-561270", + "doi": "10.17188/1183921" + }, + { + "created_at": "2015-06-09T12:22:12.805509", + "_id": "mp-570642", + "doi": "10.17188/1183908" + }, + { + "created_at": "2015-06-09T12:22:12.805525", + "_id": "mp-556025", + "doi": "10.17188/1183918" + }, + { + "created_at": "2015-06-09T12:22:12.805539", + "_id": "mp-554058", + "doi": "10.17188/1183914" + }, + { + "created_at": "2015-06-09T12:22:12.805179", + "_id": "mp-557528", + "doi": "10.17188/1183924" + }, + { + "created_at": "2015-06-09T12:22:12.805231", + "_id": "mp-557512", + "doi": "10.17188/1183904" + }, + { + "created_at": "2015-06-09T12:22:12.805246", + "_id": "mp-555083", + "doi": "10.17188/1183905" + }, + { + "created_at": "2015-06-09T12:22:12.805261", + "_id": "mp-1787", + "doi": "10.17188/1183907" + }, + { + "created_at": "2015-06-09T12:22:12.805275", + "_id": "mp-548747", + "doi": "10.17188/1183923" + }, + { + "created_at": "2015-06-09T12:22:12.805290", + "_id": "mp-744919", + "doi": "10.17188/1183912" + }, + { + "created_at": "2015-06-09T12:22:12.805304", + "_id": "mp-555813", + "doi": "10.17188/1183916" + }, + { + "created_at": "2015-06-09T12:22:12.805318", + "_id": "mp-556151", + "doi": "10.17188/1183920" + }, + { + "created_at": "2015-06-09T12:22:12.805332", + "_id": "mp-504701", + "doi": "10.17188/1183910" + }, + { + "created_at": "2015-06-09T12:22:12.805353", + "_id": "mp-569921", + "doi": "10.17188/1183909" + }, + { + "created_at": "2015-06-09T12:22:12.805367", + "_id": "mp-558144", + "doi": "10.17188/1183919" + }, + { + "created_at": "2015-06-09T12:22:12.805381", + "_id": "mp-600193", + "doi": "10.17188/1183917" + }, + { + "created_at": "2015-06-09T12:22:12.805395", + "_id": "mp-566838", + "doi": "10.17188/1183913" + }, + { + "created_at": "2015-06-09T12:22:12.805409", + "_id": "mp-226", + "doi": "10.17188/1183906" + }, + { + "created_at": "2015-06-10T12:19:03.840910", + "_id": "mp-23664", + "doi": "10.17188/1183999" + }, + { + "created_at": "2015-06-10T12:19:03.840980", + "_id": "mp-23682", + "doi": "10.17188/1184003" + }, + { + "created_at": "2015-06-10T12:19:03.840924", + "_id": "mp-24411", + "doi": "10.17188/1184000" + }, + { + "created_at": "2015-06-10T12:19:03.840938", + "_id": "mp-24412", + "doi": "10.17188/1184001" + }, + { + "created_at": "2015-06-10T12:19:03.841022", + "_id": "mp-2688", + "doi": "10.17188/1184009" + }, + { + "created_at": "2015-06-10T12:19:03.840793", + "_id": "mp-3731", + "doi": "10.17188/1184005" + }, + { + "created_at": "2015-06-10T12:19:03.840863", + "_id": "mp-557366", + "doi": "10.17188/1183996" + }, + { + "created_at": "2015-06-10T12:19:03.840966", + "_id": "mp-558863", + "doi": "10.17188/1184014" + }, + { + "created_at": "2015-06-10T12:19:03.840849", + "_id": "mp-559223", + "doi": "10.17188/1183997" + }, + { + "created_at": "2015-06-10T12:19:03.840877", + "_id": "mp-560533", + "doi": "10.17188/1184012" + }, + { + "created_at": "2015-06-10T12:19:03.840762", + "_id": "mp-560730", + "doi": "10.17188/1183998" + }, + { + "created_at": "2015-06-10T12:19:03.841008", + "_id": "mp-565833", + "doi": "10.17188/1184006" + }, + { + "created_at": "2015-06-10T12:19:03.840952", + "_id": "mp-566598", + "doi": "10.17188/1184007" + }, + { + "created_at": "2015-06-10T12:19:03.840835", + "_id": "mp-570319", + "doi": "10.17188/1184010" + }, + { + "created_at": "2015-06-10T12:19:03.840778", + "_id": "mp-600167", + "doi": "10.17188/1184011" + }, + { + "created_at": "2015-06-10T12:19:03.840807", + "_id": "mp-600171", + "doi": "10.17188/1184015" + }, + { + "created_at": "2015-06-10T12:19:03.840822", + "_id": "mp-600220", + "doi": "10.17188/1184004" + }, + { + "created_at": "2015-06-10T12:19:03.840994", + "_id": "mp-600262", + "doi": "10.17188/1184008" + }, + { + "created_at": "2015-06-10T12:19:03.840891", + "_id": "mp-600433", + "doi": "10.17188/1184002" + }, + { + "created_at": "2015-06-10T12:19:03.840701", + "_id": "mp-654937", + "doi": "10.17188/1184013" + }, + { + "created_at": "2015-06-16T12:17:06.310306", + "_id": "mp-567119", + "doi": "10.17188/1184546" + }, + { + "created_at": "2015-06-16T12:17:06.310342", + "_id": "mp-25769", + "doi": "10.17188/1184549" + }, + { + "created_at": "2015-06-16T12:17:06.310349", + "_id": "mp-23045", + "doi": "10.17188/1184548" + }, + { + "created_at": "2015-06-16T12:17:06.310377", + "_id": "mp-22876", + "doi": "10.17188/1184544" + }, + { + "created_at": "2015-06-16T12:17:06.310398", + "_id": "mp-571553", + "doi": "10.17188/1184554" + }, + { + "created_at": "2015-06-16T12:17:06.310419", + "_id": "mp-558867", + "doi": "10.17188/1184542" + }, + { + "created_at": "2015-06-16T12:17:06.310277", + "_id": "mp-558003", + "doi": "10.17188/1184552" + }, + { + "created_at": "2015-06-16T12:17:06.310297", + "_id": "mp-600154", + "doi": "10.17188/1184536" + }, + { + "created_at": "2015-06-16T12:17:06.310313", + "_id": "mp-566863", + "doi": "10.17188/1184538" + }, + { + "created_at": "2015-06-16T12:17:06.310320", + "_id": "mp-600494", + "doi": "10.17188/1184551" + }, + { + "created_at": "2015-06-16T12:17:06.310328", + "_id": "mp-566045", + "doi": "10.17188/1184540" + }, + { + "created_at": "2015-06-16T12:17:06.310335", + "_id": "mp-600159", + "doi": "10.17188/1184537" + }, + { + "created_at": "2015-06-16T12:17:06.310356", + "_id": "mp-556018", + "doi": "10.17188/1184543" + }, + { + "created_at": "2015-06-16T12:17:06.310363", + "_id": "mp-553984", + "doi": "10.17188/1184547" + }, + { + "created_at": "2015-06-16T12:17:06.310370", + "_id": "mp-24413", + "doi": "10.17188/1184539" + }, + { + "created_at": "2015-06-16T12:17:06.310384", + "_id": "mp-570535", + "doi": "10.17188/1184541" + }, + { + "created_at": "2015-06-16T12:17:06.310391", + "_id": "mp-24995", + "doi": "10.17188/1184550" + }, + { + "created_at": "2015-06-16T12:17:06.310405", + "_id": "mp-600278", + "doi": "10.17188/1184545" + }, + { + "created_at": "2015-06-16T12:17:06.310412", + "_id": "mp-23705", + "doi": "10.17188/1184535" + }, + { + "created_at": "2015-06-16T12:17:06.310426", + "_id": "mp-583589", + "doi": "10.17188/1184553" + }, + { + "created_at": "2015-06-17T12:15:21.044647", + "_id": "mp-600172", + "doi": "10.17188/1184645" + }, + { + "created_at": "2015-06-17T12:15:21.044654", + "_id": "mp-561300", + "doi": "10.17188/1184637" + }, + { + "created_at": "2015-06-17T12:15:21.044661", + "_id": "mp-600273", + "doi": "10.17188/1184650" + }, + { + "created_at": "2015-06-17T12:15:21.044668", + "_id": "mp-24990", + "doi": "10.17188/1184642" + }, + { + "created_at": "2015-06-17T12:15:21.044676", + "_id": "mp-559038", + "doi": "10.17188/1184651" + }, + { + "created_at": "2015-06-17T12:15:21.044683", + "_id": "mp-571435", + "doi": "10.17188/1184654" + }, + { + "created_at": "2015-06-17T12:15:21.044535", + "_id": "mp-24951", + "doi": "10.17188/1184652" + }, + { + "created_at": "2015-06-17T12:15:21.044554", + "_id": "mp-568259", + "doi": "10.17188/1184646" + }, + { + "created_at": "2015-06-17T12:15:21.044562", + "_id": "mp-567467", + "doi": "10.17188/1184653" + }, + { + "created_at": "2015-06-17T12:15:21.044570", + "_id": "mp-600177", + "doi": "10.17188/1184647" + }, + { + "created_at": "2015-06-17T12:15:21.044577", + "_id": "mp-554928", + "doi": "10.17188/1184641" + }, + { + "created_at": "2015-06-17T12:15:21.044584", + "_id": "mp-600181", + "doi": "10.17188/1184655" + }, + { + "created_at": "2015-06-17T12:15:21.044591", + "_id": "mp-2657", + "doi": "10.17188/1184648" + }, + { + "created_at": "2015-06-17T12:15:21.044598", + "_id": "mp-570177", + "doi": "10.17188/1184656" + }, + { + "created_at": "2015-06-17T12:15:21.044605", + "_id": "mp-600190", + "doi": "10.17188/1184643" + }, + { + "created_at": "2015-06-17T12:15:21.044613", + "_id": "mp-567645", + "doi": "10.17188/1184640" + }, + { + "created_at": "2015-06-17T12:15:21.044620", + "_id": "mp-558970", + "doi": "10.17188/1184644" + }, + { + "created_at": "2015-06-17T12:15:21.044627", + "_id": "mp-600186", + "doi": "10.17188/1184639" + }, + { + "created_at": "2015-06-17T12:15:21.044634", + "_id": "mp-540832", + "doi": "10.17188/1184649" + }, + { + "created_at": "2015-06-17T12:15:21.044641", + "_id": "mp-24989", + "doi": "10.17188/1184638" + }, + { + "created_at": "2015-06-18T12:16:30.707133", + "_id": "mp-10008", + "doi": "10.17188/1184819" + }, + { + "created_at": "2015-06-18T12:16:30.707140", + "_id": "mp-10014", + "doi": "10.17188/1184824" + }, + { + "created_at": "2015-06-18T12:16:30.707152", + "_id": "mp-10015", + "doi": "10.17188/1184825" + }, + { + "created_at": "2015-06-18T12:16:30.707160", + "_id": "mp-10013", + "doi": "10.17188/1184823" + }, + { + "created_at": "2015-06-18T12:16:30.707167", + "_id": "mp-10010", + "doi": "10.17188/1184822" + }, + { + "created_at": "2015-06-18T12:16:30.707174", + "_id": "mp-10024", + "doi": "10.17188/1184830" + }, + { + "created_at": "2015-06-18T12:16:30.707021", + "_id": "mp-10020", + "doi": "10.17188/1184827" + }, + { + "created_at": "2015-06-18T12:16:30.707040", + "_id": "mp-1000", + "doi": "10.17188/1184814" + }, + { + "created_at": "2015-06-18T12:16:30.707048", + "_id": "mp-1001", + "doi": "10.17188/1184821" + }, + { + "created_at": "2015-06-18T12:16:30.707055", + "_id": "mp-10006", + "doi": "10.17188/1184818" + }, + { + "created_at": "2015-06-18T12:16:30.707062", + "_id": "mp-1", + "doi": "10.17188/1184811" + }, + { + "created_at": "2015-06-18T12:16:30.707070", + "_id": "mp-10", + "doi": "10.17188/1184812" + }, + { + "created_at": "2015-06-18T12:16:30.707077", + "_id": "mp-10021", + "doi": "10.17188/1184828" + }, + { + "created_at": "2015-06-18T12:16:30.707084", + "_id": "mp-10000", + "doi": "10.17188/1184815" + }, + { + "created_at": "2015-06-18T12:16:30.707091", + "_id": "mp-10003", + "doi": "10.17188/1184816" + }, + { + "created_at": "2015-06-18T12:16:30.707098", + "_id": "mp-10023", + "doi": "10.17188/1184829" + }, + { + "created_at": "2015-06-18T12:16:30.707105", + "_id": "mp-10004", + "doi": "10.17188/1184817" + }, + { + "created_at": "2015-06-18T12:16:30.707112", + "_id": "mp-10018", + "doi": "10.17188/1184826" + }, + { + "created_at": "2015-06-18T12:16:30.707119", + "_id": "mp-100", + "doi": "10.17188/1184813" + }, + { + "created_at": "2015-06-18T12:16:30.707126", + "_id": "mp-10009", + "doi": "10.17188/1184820" + }, + { + "created_at": "2015-06-23T14:29:03.312695", + "_id": "mp-10026", + "doi": "10.17188/1185062" + }, + { + "created_at": "2015-06-23T14:29:03.312716", + "_id": "mp-10025", + "doi": "10.17188/1185061" + }, + { + "created_at": "2015-06-23T15:17:28.081929", + "_id": "mp-10030", + "doi": "10.17188/1185065" + }, + { + "created_at": "2015-06-23T15:17:28.081949", + "_id": "mp-10032", + "doi": "10.17188/1185066" + }, + { + "created_at": "2015-06-23T15:17:28.081957", + "_id": "mp-10033", + "doi": "10.17188/1185067" + }, + { + "created_at": "2015-06-23T15:17:28.081965", + "_id": "mp-10027", + "doi": "10.17188/1185063" + }, + { + "created_at": "2015-06-23T15:17:28.081973", + "_id": "mp-1003", + "doi": "10.17188/1185064" + }, + { + "created_at": "2015-06-23T15:23:04.481507", + "_id": "mp-10036", + "doi": "10.17188/1185068" + }, + { + "created_at": "2015-06-23T15:23:04.481526", + "_id": "mp-10037", + "doi": "10.17188/1185069" + }, + { + "created_at": "2015-06-23T15:27:38.247118", + "_id": "mp-1004", + "doi": "10.17188/1185072" + }, + { + "created_at": "2015-06-23T15:27:38.247137", + "_id": "mp-10048", + "doi": "10.17188/1185080" + }, + { + "created_at": "2015-06-23T15:27:38.247145", + "_id": "mp-10045", + "doi": "10.17188/1185077" + }, + { + "created_at": "2015-06-23T15:27:38.247153", + "_id": "mp-10044", + "doi": "10.17188/1185076" + }, + { + "created_at": "2015-06-23T15:27:38.247160", + "_id": "mp-10047", + "doi": "10.17188/1185079" + }, + { + "created_at": "2015-06-23T15:27:38.247168", + "_id": "mp-10046", + "doi": "10.17188/1185078" + }, + { + "created_at": "2015-06-23T15:27:38.247176", + "_id": "mp-10041", + "doi": "10.17188/1185075" + }, + { + "created_at": "2015-06-23T15:27:38.247183", + "_id": "mp-10040", + "doi": "10.17188/1185074" + }, + { + "created_at": "2015-06-23T15:27:38.247190", + "_id": "mp-10038", + "doi": "10.17188/1185070" + }, + { + "created_at": "2015-06-23T15:27:38.247198", + "_id": "mp-10039", + "doi": "10.17188/1185071" + }, + { + "created_at": "2015-06-23T15:37:16.499764", + "_id": "mp-1005", + "doi": "10.17188/1185082" + }, + { + "created_at": "2015-06-23T15:37:16.499784", + "_id": "mp-10058", + "doi": "10.17188/1185089" + }, + { + "created_at": "2015-06-23T15:37:16.499793", + "_id": "mp-10059", + "doi": "10.17188/1185090" + }, + { + "created_at": "2015-06-23T15:37:16.499801", + "_id": "mp-10049", + "doi": "10.17188/1185081" + }, + { + "created_at": "2015-06-23T15:37:16.499809", + "_id": "mp-10052", + "doi": "10.17188/1185084" + }, + { + "created_at": "2015-06-23T15:37:16.499816", + "_id": "mp-10051", + "doi": "10.17188/1185083" + }, + { + "created_at": "2015-06-23T15:37:16.499824", + "_id": "mp-10056", + "doi": "10.17188/1185087" + }, + { + "created_at": "2015-06-23T15:45:17.074966", + "_id": "mp-10078", + "doi": "10.17188/1185103" + }, + { + "created_at": "2015-06-23T15:45:17.074973", + "_id": "mp-10079", + "doi": "10.17188/1185104" + }, + { + "created_at": "2015-06-23T15:45:17.074981", + "_id": "mp-10089", + "doi": "10.17188/1185109" + }, + { + "created_at": "2015-06-23T15:45:17.074988", + "_id": "mp-10080", + "doi": "10.17188/1185106" + }, + { + "created_at": "2015-06-23T15:45:17.075003", + "_id": "mp-10086", + "doi": "10.17188/1185107" + }, + { + "created_at": "2015-06-23T15:37:16.499831", + "_id": "mp-10057", + "doi": "10.17188/1185088" + }, + { + "created_at": "2015-06-23T15:37:16.499839", + "_id": "mp-10054", + "doi": "10.17188/1185085" + }, + { + "created_at": "2015-06-23T15:37:16.499846", + "_id": "mp-10055", + "doi": "10.17188/1185086" + }, + { + "created_at": "2015-06-23T15:45:17.074867", + "_id": "mp-1007", + "doi": "10.17188/1185099" + }, + { + "created_at": "2015-06-23T15:45:17.074886", + "_id": "mp-1008", + "doi": "10.17188/1185105" + }, + { + "created_at": "2015-06-23T15:45:17.074895", + "_id": "mp-1009", + "doi": "10.17188/1185110" + }, + { + "created_at": "2015-06-23T15:45:17.074932", + "_id": "mp-10091", + "doi": "10.17188/1185110" + }, + { + "created_at": "2015-06-23T15:45:17.074939", + "_id": "mp-10074", + "doi": "10.17188/1185101" + }, + { + "created_at": "2015-06-23T15:45:17.074950", + "_id": "mp-10076", + "doi": "10.17188/1185102" + }, + { + "created_at": "2015-06-23T15:45:17.074958", + "_id": "mp-10070", + "doi": "10.17188/1185100" + }, + { + "created_at": "2015-06-23T15:45:17.074995", + "_id": "mp-10087", + "doi": "10.17188/1185108" + }, + { + "created_at": "2015-06-23T15:45:17.075010", + "_id": "mp-10065", + "doi": "10.17188/1185096" + }, + { + "created_at": "2015-06-23T15:45:17.075017", + "_id": "mp-10064", + "doi": "10.17188/1185095" + }, + { + "created_at": "2015-06-23T15:45:17.075025", + "_id": "mp-10063", + "doi": "10.17188/1185094" + }, + { + "created_at": "2015-06-23T15:45:17.075032", + "_id": "mp-10062", + "doi": "10.17188/1185093" + }, + { + "created_at": "2015-06-23T15:45:17.075040", + "_id": "mp-10061", + "doi": "10.17188/1185092" + }, + { + "created_at": "2015-06-23T15:45:17.075047", + "_id": "mp-10060", + "doi": "10.17188/1185091" + }, + { + "created_at": "2015-06-23T15:45:17.075057", + "_id": "mp-10069", + "doi": "10.17188/1185098" + }, + { + "created_at": "2015-06-23T15:45:17.075065", + "_id": "mp-10068", + "doi": "10.17188/1185097" } ] \ No newline at end of file diff --git a/mpworks/osti_doi/osti_record.py b/mpworks/osti_doi/osti_record.py index 96093211..cc9547d3 100644 --- a/mpworks/osti_doi/osti_record.py +++ b/mpworks/osti_doi/osti_record.py @@ -3,6 +3,7 @@ import logging import datetime import json +import sys from pymongo import MongoClient from monty.serialization import loadfn from collections import OrderedDict @@ -73,7 +74,10 @@ def get_all_dois(self): def get_materials_cursor(self, l, n): if l is None: - return self.matcoll.find({'doi': {'$exists': False}}, limit=n) + return self.matcoll.find({ + 'doi': {'$exists': False}, + 'task_id': {'$nin': self.doicoll.find().distinct('_id')} + }, limit=n) else: mp_ids = [ 'mp-{}'.format(el) for el in l ] return self.matcoll.find({'task_id': {'$in': mp_ids}}) @@ -154,11 +158,16 @@ def __init__(self, l=None, n=0, doicoll=None, matcoll=None, def submit(self): """submit generated records to OSTI""" - r = requests.post( - self.endpoint, data=self.records_xml.toxml(), - auth=(os.environ['OSTI_USER'], os.environ['OSTI_PASSWORD']) - ) - logger.info(r.content) + try: + r = requests.post( + self.endpoint, data=self.records_xml.toxml(), + auth=(os.environ['OSTI_USER'], os.environ['OSTI_PASSWORD']) + ) + logger.info(r.content) + except: + exc_type, exc_value, exc_traceback = sys.exc_info() + logger.warning('%r %r', exc_type, exc_value) + return None if r.status_code != 200: logger.warning('Did not request DOIs due to {} error.'.format(r.status_code)) return None @@ -166,7 +175,8 @@ def submit(self): records = [ records ] if not isinstance(records, list) else records dois = {} for ridx,record in enumerate(records): - if record['status'] == 'SUCCESS': + if record['status'] == 'SUCCESS' or \ + record['status_message'] == 'Duplicate URL Found.;': dois[record['product_nos']] = { 'doi': record['doi'], 'updated': bool('osti_id' in self.records[ridx]) diff --git a/mpworks/osti_doi/run.sh b/mpworks/osti_doi/run.sh index e05e3cc8..d6cbc100 100644 --- a/mpworks/osti_doi/run.sh +++ b/mpworks/osti_doi/run.sh @@ -2,6 +2,7 @@ source $HOME/.credentials source $HOME/.virtualenvs/env_mp_osti_doi/bin/activate cd $HOME/MPWorks export PYTHONPATH=`pwd`:$PYTHONPATH -mgbuild run -v mpworks.osti_doi.builders.DoiBuilder nmats=1 dois=dois.json materials=materials.json +mgbuild run -v mpworks.osti_doi.builders.DoiBuilder nmats=25 dois=dois.json materials=materials.json git add mpworks/osti_doi/dois.json git commit -m "osti_doi: new dois backup" +git push origin osti_doi diff --git a/mpworks/processors/README.md b/mpworks/processors/README.md new file mode 100755 index 00000000..d0ffb026 --- /dev/null +++ b/mpworks/processors/README.md @@ -0,0 +1,5 @@ +# Processors package + +This package is used in managing the MPEnv strategy of runs - e.g., insert things into the submissions database, move those to workflows, back-update the submissions with information on the runs, etc... + +It also contains a "canonical" set of test runs for testing changes to MPWorks/MPEnv/etc. \ No newline at end of file diff --git a/mpworks/processors/process_submissions.py b/mpworks/processors/process_submissions.py index dc7485cf..c7ada0b8 100644 --- a/mpworks/processors/process_submissions.py +++ b/mpworks/processors/process_submissions.py @@ -4,6 +4,7 @@ from mpworks.snl_utils.mpsnl import MPStructureNL from mpworks.submission.submission_mongo import SubmissionMongoAdapter from mpworks.workflows.snl_to_wf import snl_to_wf +from mpworks.workflows.snl_to_wf_elastic import snl_to_wf_elastic from mpworks.workflows.wf_utils import NO_POTCARS from pymatgen.matproj.snl import StructureNL @@ -30,7 +31,7 @@ def run(self, sleep_time=None, infinite=False): while True: self.submit_all_new_workflows() print "Updating existing workflows..." - self.update_existing_workflows() + self.update_existing_workflows() # for updating the display if not infinite: break print 'sleeping', sleep_time @@ -75,7 +76,10 @@ def submit_new_workflow(self): snl.data['_materialsproject']['submission_id'] = submission_id # create a workflow - wf = snl_to_wf(snl, job['parameters']) + if "Elasticity" in snl.projects: + wf=snl_to_wf_elastic(snl, job['parameters']) + else: + wf = snl_to_wf(snl, job['parameters']) self.launchpad.add_wf(wf) print 'ADDED WORKFLOW FOR {}'.format(snl.structure.formula) except: @@ -87,6 +91,8 @@ def submit_new_workflow(self): def update_existing_workflows(self): # updates the state of existing workflows by querying the FireWorks database + # this is an optional step that updates the submissions db with jobs info + # it is useful for the frontend display but not needed for workflow execution for submission in self.jobs.find({'state': {'$nin': ['COMPLETED', 'ERROR', 'REJECTED', 'CANCELLED']}}, {'submission_id': 1}): submission_id = submission['submission_id'] diff --git a/mpworks/snl_utils/README.md b/mpworks/snl_utils/README.md new file mode 100755 index 00000000..6d2c6061 --- /dev/null +++ b/mpworks/snl_utils/README.md @@ -0,0 +1,8 @@ +# snl_utils + +This package is poorly-named, but contains MP extensions to SNL that are needed for duplicate checking and database storage of SNL. + +This includes: +- MPSNL, which adds snl_id and spacegroup info to an SNL +- SNLGroup, which represents a "material" and can have several associated SNL +- Routines for adding an SNL into the database, assigning an SNLGroup, etc. \ No newline at end of file diff --git a/mpworks/snl_utils/mpsnl.py b/mpworks/snl_utils/mpsnl.py index 2d577efc..cee62fa7 100644 --- a/mpworks/snl_utils/mpsnl.py +++ b/mpworks/snl_utils/mpsnl.py @@ -32,7 +32,7 @@ def get_meta_from_structure(structure): def has_species_properties(structure): for site in structure: - for species in site: + for species in site.species_and_occu: if hasattr(species, 'spin'): return True diff --git a/mpworks/snl_utils/snl_mongo.py b/mpworks/snl_utils/snl_mongo.py index 84ef81b2..961cf417 100644 --- a/mpworks/snl_utils/snl_mongo.py +++ b/mpworks/snl_utils/snl_mongo.py @@ -171,12 +171,17 @@ def switch_canonical_snl(self, snlgroup_id, canonical_mpsnl): new_group = SNLGroup(snlgroup_id, canonical_mpsnl, all_snl_ids) self.snlgroups.update({'snlgroup_id': snlgroup_id}, new_group.as_dict()) - def lock_db(self): + def lock_db(self, n_tried=0, n_max_tries=10): x = self.id_assigner.find_and_modify(query={}, update={'$set':{'lock': True}}, fields={'lock':1}) if 'lock' in x and x['lock']: - print 'DB is already locked, waiting 30 secs...' - time.sleep(30) - self.lock_db() # DB was already locked by another process in a race condition + # x is original object + if n_tried < n_max_tries: + time.sleep(30) + n_tried += 1 + # DB was already locked by another process in a race condition + self.lock_db(n_tried=n_tried, n_max_tries=n_max_tries) + else: + raise ValueError('DB locked by another process! Could not lock even after {} minutes!'.format(n_max_tries/60)) def release_lock(self): self.id_assigner.find_and_modify(query={}, update={'$set':{'lock': False}}) diff --git a/mpworks/submission/README.md b/mpworks/submission/README.md new file mode 100755 index 00000000..43732893 --- /dev/null +++ b/mpworks/submission/README.md @@ -0,0 +1,3 @@ +# Submission package + +Contains an interface to a submissions database for managing jobs. It is part of the "MPenv" way of running things. \ No newline at end of file diff --git a/mpworks/workflows/README.md b/mpworks/workflows/README.md new file mode 100755 index 00000000..373e7d24 --- /dev/null +++ b/mpworks/workflows/README.md @@ -0,0 +1,3 @@ +# Workflows package + +This package is used by the production workflow. Indeed, it *defines* the production workflows for various types of runs. \ No newline at end of file diff --git a/mpworks/workflows/snl_to_wf.py b/mpworks/workflows/snl_to_wf.py index 278beca4..40f00dcc 100644 --- a/mpworks/workflows/snl_to_wf.py +++ b/mpworks/workflows/snl_to_wf.py @@ -11,8 +11,8 @@ from mpworks.snl_utils.mpsnl import get_meta_from_structure, MPStructureNL from mpworks.workflows.wf_settings import QA_DB, QA_VASP, QA_CONTROL from pymatgen import Composition -from pymatgen.io.cifio import CifParser -from pymatgen.io.vaspio_set import MPVaspInputSet, MPGGAVaspInputSet +from pymatgen.io.cif import CifParser +from pymatgen.io.vasp.sets import MPRelaxSet from pymatgen.matproj.snl import StructureNL __author__ = 'Anubhav Jain' @@ -34,13 +34,15 @@ def _snl_to_spec(snl, enforce_gga=False, parameters=None): structure = snl.structure else: structure = snl.structure.get_primitive_structure() + if enforce_gga: + incar_enforce.update({"LDAU" : False}) + mpvis = MPRelaxSet(snl.structure, + user_incar_settings=incar_enforce) - mpvis = MPGGAVaspInputSet(user_incar_settings=incar_enforce) if enforce_gga else MPVaspInputSet(user_incar_settings=incar_enforce) - - incar = mpvis.get_incar(structure) - poscar = mpvis.get_poscar(structure) - kpoints = mpvis.get_kpoints(structure) - potcar = mpvis.get_potcar(structure) + incar = mpvis.incar + poscar = mpvis.poscar + kpoints = mpvis.kpoints + potcar = mpvis.potcar spec['vasp'] = {} spec['vasp']['incar'] = incar.as_dict() @@ -95,11 +97,16 @@ def snl_to_wf(snl, parameters=None): else: # add the SNL to the SNL DB and figure out duplicate group tasks = [AddSNLTask()] - spec = {'task_type': 'Add to SNL database', 'snl': snl.as_dict(), '_queueadapter': QA_DB, '_priority': snl_priority} - fws.append(Firework(tasks, spec, name=get_slug(f + '--' + spec['task_type']), fw_id=0)) + spec = {'task_type': 'Add to SNL database', + 'snl': snl.as_dict(), '_queueadapter': QA_DB, + '_priority': snl_priority} + fws.append(Firework(tasks, spec, fw_id=0, + name=get_slug(f + '--' + spec['task_type']))) connections[0] = [1] - trackers = [Tracker('FW_job.out'), Tracker('FW_job.error'), Tracker('vasp.out'), Tracker('OUTCAR'), Tracker('OSZICAR'), Tracker('OUTCAR.relax1'), Tracker('OUTCAR.relax2')] + trackers = [Tracker('FW_job.out'), Tracker('FW_job.error'), + Tracker('vasp.out'), Tracker('OUTCAR'), Tracker('OSZICAR'), + Tracker('OUTCAR.relax1'), Tracker('OUTCAR.relax2')] trackers_db = [Tracker('FW_job.out'), Tracker('FW_job.error')] # run GGA structure optimization spec = _snl_to_spec(snl, enforce_gga=True, parameters=parameters) @@ -112,56 +119,60 @@ def snl_to_wf(snl, parameters=None): # insert into DB - GGA structure optimization spec = {'task_type': 'VASP db insertion', '_priority': priority*2, - '_allow_fizzled_parents': True, '_queueadapter': QA_DB, "_dupefinder": DupeFinderDB().to_dict(), '_trackers': trackers_db} - fws.append( - Firework([VaspToDBTask()], spec, name=get_slug(f + '--' + spec['task_type']), fw_id=2)) + '_allow_fizzled_parents': True, '_queueadapter': QA_DB, + "_dupefinder": DupeFinderDB().to_dict(), '_trackers': trackers_db} + fws.append(Firework([VaspToDBTask()], spec, fw_id=2, + name=get_slug(f + '--' + spec['task_type']))) connections[1] = [2] # determine if GGA+U FW is needed - incar = MPVaspInputSet().get_incar(snl.structure).as_dict() + incar = MPRelaxSet(snl.structure).incar.as_dict() ggau_compound = ('LDAU' in incar and incar['LDAU']) - if not parameters.get('skip_bandstructure', False) and (not ggau_compound or parameters.get('force_gga_bandstructure', False)): - spec = {'task_type': 'Controller: add Electronic Structure v2', '_priority': priority, - '_queueadapter': QA_CONTROL} - fws.append( - Firework([AddEStructureTask()], spec, name=get_slug(f + '--' + spec['task_type']), - fw_id=3)) + if not parameters.get('skip_bandstructure', False) and \ + (not ggau_compound or parameters.get('force_gga_bandstructure', False)): + spec = {'task_type': 'Controller: add Electronic Structure v2', + '_priority': priority, '_queueadapter': QA_CONTROL} + fws.append(Firework([AddEStructureTask()], spec, fw_id=3, + name=get_slug(f + '--' + spec['task_type']))) connections[2] = [3] if ggau_compound: spec = _snl_to_spec(snl, enforce_gga=False, parameters=parameters) - del spec['vasp'] # we are stealing all VASP params and such from previous run + del spec['vasp'] # we are stealing all VASP params from previous run spec['_priority'] = priority spec['_queueadapter'] = QA_VASP spec['_trackers'] = trackers fws.append(Firework( - [VaspCopyTask(), SetupGGAUTask(), - get_custodian_task(spec)], spec, name=get_slug(f + '--' + spec['task_type']), - fw_id=10)) + [VaspCopyTask(), SetupGGAUTask(), get_custodian_task(spec)], + spec, name=get_slug(f + '--' + spec['task_type']), fw_id=10)) connections[2].append(10) spec = {'task_type': 'VASP db insertion', '_queueadapter': QA_DB, - '_allow_fizzled_parents': True, '_priority': priority, "_dupefinder": DupeFinderDB().to_dict(), '_trackers': trackers_db} + '_allow_fizzled_parents': True, '_priority': priority, + "_dupefinder": DupeFinderDB().to_dict(), + '_trackers': trackers_db} fws.append( - Firework([VaspToDBTask()], spec, name=get_slug(f + '--' + spec['task_type']), fw_id=11)) + Firework([VaspToDBTask()], spec, + name=get_slug(f + '--' + spec['task_type']), fw_id=11)) connections[10] = [11] if not parameters.get('skip_bandstructure', False): - spec = {'task_type': 'Controller: add Electronic Structure v2', '_priority': priority, - '_queueadapter': QA_CONTROL} + spec = {'task_type': 'Controller: add Electronic Structure v2', + '_priority': priority, '_queueadapter': QA_CONTROL} fws.append( - Firework([AddEStructureTask()], spec, name=get_slug(f + '--' + spec['task_type']), - fw_id=12)) + Firework([AddEStructureTask()], spec, fw_id=12, + name=get_slug(f + '--' + spec['task_type']))) connections[11] = [12] wf_meta = get_meta_from_structure(snl.structure) - wf_meta['run_version'] = 'May 2013 (1)' + wf_meta['run_version'] = 'May 2013 (1)' # not maintained - if '_materialsproject' in snl.data and 'submission_id' in snl.data['_materialsproject']: + if '_materialsproject' in snl.data and \ + 'submission_id' in snl.data['_materialsproject']: wf_meta['submission_id'] = snl.data['_materialsproject']['submission_id'] - return Workflow(fws, connections, name=Composition( - snl.structure.composition.reduced_formula).alphabetical_formula, metadata=wf_meta) + + return Workflow(fws, connections, name=f, metadata=wf_meta) """ @@ -189,7 +200,6 @@ def snl_to_wf_ggau(snl): return Workflow(fws, connections, name=Composition(snl.structure.composition.reduced_formula).alphabetical_formula) """ - if __name__ == '__main__': s1 = CifParser('test_wfs/Si.cif').get_structures()[0] s2 = CifParser('test_wfs/FeO.cif').get_structures()[0] diff --git a/mpworks/workflows/snl_to_wf_elastic.py b/mpworks/workflows/snl_to_wf_elastic.py new file mode 100755 index 00000000..d6b711f9 --- /dev/null +++ b/mpworks/workflows/snl_to_wf_elastic.py @@ -0,0 +1,79 @@ +from pymatgen.io.vasp import Poscar +from mpworks.firetasks.elastic_tasks import SetupElastConstTask, SetupFConvergenceTask, SetupDeformedStructTask + +__author__ = 'weichen' + +from fireworks.core.firework import Firework, Workflow +from fireworks.utilities.fw_utilities import get_slug +from mpworks.firetasks.custodian_task import get_custodian_task +from mpworks.firetasks.snl_tasks import AddSNLTask +from mpworks.firetasks.vasp_io_tasks import VaspCopyTask, VaspWriterTask, \ + VaspToDBTask +from mpworks.firetasks.vasp_setup_tasks import SetupGGAUTask +from mpworks.snl_utils.mpsnl import get_meta_from_structure, MPStructureNL +from mpworks.workflows.wf_settings import QA_DB, QA_VASP, QA_CONTROL +from pymatgen import Composition +from mpworks.workflows import snl_to_wf +from mpworks.firetasks.elastic_tasks import update_spec_force_convergence +from collections import defaultdict + + +def snl_to_wf_elastic(snl, parameters): + # parameters["user_vasp_settings"] specifies user defined incar/kpoints parameters + fws = [] + connections = defaultdict(list) + parameters = parameters if parameters else {} + + snl_priority = parameters.get('priority', 1) + priority = snl_priority * 2 # once we start a job, keep going! + + f = Composition(snl.structure.composition.reduced_formula).alphabetical_formula + + # add the SNL to the SNL DB and figure out duplicate group + tasks = [AddSNLTask()] + spec = {'task_type': 'Add to SNL database', 'snl': snl.as_dict(), + '_queueadapter': QA_DB, '_priority': snl_priority} + if 'snlgroup_id' in parameters and isinstance(snl, MPStructureNL): + spec['force_mpsnl'] = snl.as_dict() + spec['force_snlgroup_id'] = parameters['snlgroup_id'] + del spec['snl'] + fws.append(Firework(tasks, spec, + name=get_slug(f + '--' + spec['task_type']), fw_id=0)) + connections[0] = [1] + + parameters["exact_structure"] = True + # run GGA structure optimization for force convergence + spec = snl_to_wf._snl_to_spec(snl, parameters=parameters) + user_vasp_settings = parameters.get("user_vasp_settings") + spec = update_spec_force_convergence(spec, user_vasp_settings) + spec['run_tags'].append("origin") + spec['_priority'] = priority + spec['_queueadapter'] = QA_VASP + del spec['_dupefinder'] + spec['task_type'] = "Vasp force convergence optimize structure (2x)" + tasks = [VaspWriterTask(), get_custodian_task(spec)] + fws.append(Firework(tasks, spec, + name=get_slug(f + '--' + spec['task_type']), fw_id=1)) + + # insert into DB - GGA structure optimization + spec = {'task_type': 'VASP db insertion', '_priority': priority, + '_allow_fizzled_parents': True, '_queueadapter': QA_DB, + 'clean_task_doc':True, 'elastic_constant':"force_convergence"} + fws.append(Firework([VaspToDBTask()], spec, + name=get_slug(f + '--' + spec['task_type']), fw_id=2)) + connections[1] = [2] + + spec = {'task_type': 'Setup Deformed Struct Task', '_priority': priority, + '_queueadapter': QA_CONTROL} + fws.append(Firework([SetupDeformedStructTask()], spec, + name=get_slug(f + '--' + spec['task_type']),fw_id=3)) + connections[2] = [3] + + wf_meta = get_meta_from_structure(snl.structure) + wf_meta['run_version'] = 'May 2013 (1)' + + if '_materialsproject' in snl.data and 'submission_id' in snl.data['_materialsproject']: + wf_meta['submission_id'] = snl.data['_materialsproject']['submission_id'] + + return Workflow(fws, connections, name=Composition( + snl.structure.composition.reduced_formula).alphabetical_formula, metadata=wf_meta) diff --git a/mpworks/workflows/snl_to_wf_eos_thermal.py b/mpworks/workflows/snl_to_wf_eos_thermal.py new file mode 100644 index 00000000..5b3b99e6 --- /dev/null +++ b/mpworks/workflows/snl_to_wf_eos_thermal.py @@ -0,0 +1,110 @@ +#from pymatgen.io.vaspio import Poscar + +__author__ = 'Cormac Toher' + +from fireworks.core.firework import Firework, Workflow +from fireworks.utilities.fw_utilities import get_slug +from mpworks.firetasks.custodian_task import get_custodian_task +from mpworks.firetasks.snl_tasks import AddSNLTask +from mpworks.firetasks.vasp_io_tasks import VaspCopyTask, VaspWriterTask, \ + VaspToDBTask +from mpworks.firetasks.vasp_setup_tasks import SetupGGAUTask +from mpworks.snl_utils.mpsnl import get_meta_from_structure, MPStructureNL +from mpworks.workflows.wf_settings import QA_DB, QA_VASP, QA_CONTROL +from pymatgen import Composition +from mpworks.workflows import snl_to_wf +from mpworks.firetasks.phonon_tasks import update_spec_force_convergence +from mpworks.firetasks.eos_thermal_tasks import SetupFConvergenceTask, SetupEoSThermalTask, SetupModifiedVolumeStructTask, AddEoSThermalDataToDBTask + + +def snl_to_wf_eos_thermal(snl, parameters=None): + fws = [] + connections = {} + parameters = parameters if parameters else {} + print("Parameters = ", parameters) + poisson_val = parameters["poisson_ratio"] + print("snl_to_wf_eos_thermal: Poisson ratio = ", poisson_val) + + snl_priority = parameters.get('priority', 1) + priority = snl_priority * 2 # once we start a job, keep going! + + f = Composition(snl.structure.composition.reduced_formula).alphabetical_formula + + # add the SNL to the SNL DB and figure out duplicate group + tasks = [AddSNLTask()] + spec = {'task_type': 'Add to SNL database', 'snl': snl.as_dict(), + '_queueadapter': QA_DB, '_priority': snl_priority} + if 'snlgroup_id' in parameters and isinstance(snl, MPStructureNL): + spec['force_mpsnl'] = snl.as_dict() + spec['force_snlgroup_id'] = parameters['snlgroup_id'] + del spec['snl'] + fws.append(Firework(tasks, spec, name=get_slug(f + '--' + spec['task_type']), fw_id=0)) + connections[0] = [1] + + print("Running structure optimization before generating different volumes") + + parameters["exact_structure"] = True + # run GGA structure optimization for force convergence + spec = snl_to_wf._snl_to_spec(snl, parameters=parameters) + user_vasp_settings = parameters.get("user_vasp_settings") + spec = update_spec_force_convergence(spec, user_vasp_settings) + spec['run_tags'].append("origin") + spec['_priority'] = priority + spec['_queueadapter'] = QA_VASP + spec['task_type'] = "Vasp force convergence optimize structure (2x)" + tasks = [VaspWriterTask(), get_custodian_task(spec)] + fws.append(Firework(tasks, spec, name=get_slug(f + '--' + spec['task_type']), fw_id=1)) + + print("Setting up modified volumes") + + # insert into DB - GGA structure optimization + spec = {'task_type': 'VASP db insertion', '_priority': priority, + '_allow_fizzled_parents': True, '_queueadapter': QA_DB, 'clean_task_doc':True, + 'thermal_properties':"force_convergence"} + fws.append(Firework([VaspToDBTask()], spec, name=get_slug(f + '--' + spec['task_type']), fw_id=2)) + connections[1] = [2] + + spec = {'task_type': 'Setup Modified Volume Struct Task', '_priority': priority, + '_queueadapter': QA_CONTROL, 'poisson_ratio': poisson_val} + fws.append(Firework([SetupModifiedVolumeStructTask()], spec, + name=get_slug(f + '--' + spec['task_type']),fw_id=3)) + connections[2] = [3] + + wf_meta = get_meta_from_structure(snl.structure) + wf_meta['run_version'] = 'May 2013 (1)' + + if '_materialsproject' in snl.data and 'submission_id' in snl.data['_materialsproject']: + wf_meta['submission_id'] = snl.data['_materialsproject']['submission_id'] + + return Workflow(fws, connections, name=Composition( + snl.structure.composition.reduced_formula).alphabetical_formula, metadata=wf_meta) + + +def snl_to_wf_eos_thermal_DB(snl, parameters=None): + fws = [] + connections = {} + parameters = parameters if parameters else {} + print("Parameters = ", parameters) + poisson_val = parameters["poisson_ratio"] + print("snl_to_wf_eos_thermal: Poisson ratio = ", poisson_val) + original_task_id_val = parameters["original_task_id"] + + snl_priority = parameters.get('priority', 1) + priority = snl_priority * 2 # once we start a job, keep going! + + f = Composition(snl.structure.composition.reduced_formula).alphabetical_formula + + spec = {'task_type': 'Add EoS Thermal Data to DB Task', '_priority': priority, + '_queueadapter': QA_CONTROL, 'poisson_ratio': poisson_val, 'original_task_id': original_task_id_val} + print("Calling firetask") + fws.append(Firework([AddEoSThermalDataToDBTask()], spec, + name=get_slug(f + '--' + spec['task_type']),fw_id=99)) + + wf_meta = get_meta_from_structure(snl.structure) + wf_meta['run_version'] = 'May 2013 (1)' + + if '_materialsproject' in snl.data and 'submission_id' in snl.data['_materialsproject']: + wf_meta['submission_id'] = snl.data['_materialsproject']['submission_id'] + + return Workflow(fws, connections, name=Composition( + snl.structure.composition.reduced_formula).alphabetical_formula, metadata=wf_meta) diff --git a/mpworks/workflows/surface_wf.py b/mpworks/workflows/surface_wf.py new file mode 100755 index 00000000..51038b19 --- /dev/null +++ b/mpworks/workflows/surface_wf.py @@ -0,0 +1,384 @@ + +## for Surface Energy Calculation +from __future__ import division, unicode_literals +__author__ = "Richard Tran" +__version__ = "0.1" +__email__ = "rit634989@gmail.com" +__date__ = "6/24/15" + + +import os +import uuid + +from mpworks.firetasks.surface_tasks import RunCustodianTask, \ + VaspSlabDBInsertTask, WriteSlabVaspInputs, WriteUCVaspInputs +from custodian.vasp.jobs import VaspJob +from custodian.vasp.handlers import VaspErrorHandler, NonConvergingErrorHandler, \ + UnconvergedErrorHandler, PotimErrorHandler, PositiveEnergyErrorHandler, \ + FrozenJobErrorHandler +from pymatgen.core.surface import generate_all_slabs, SlabGenerator, \ + get_symmetrically_distinct_miller_indices +from pymatgen.core.surface import SlabGenerator, generate_all_slabs +from pymatgen.symmetry.analyzer import SpacegroupAnalyzer +from pymatgen.matproj.rest import MPRester +from pymatgen.io.vasp.outputs import Outcar + +from fireworks.core.firework import Firework, Workflow +from fireworks.core.launchpad import LaunchPad +from matgendb import QueryEngine + +# import socket +# hostname = socket.gethostname() +# if hostname[:3] != 'cvr' or hostname[:6] != 'hopper' or hostname[:6] != 'edison': +# from pymatgen.analysis.wulff_dual import wulff_3d +# else: +# print "working on nersc, turning off wulff_dual" + +class SurfaceWorkflowManager(object): + + """ + Initializes the workflow manager by taking in a list of compounds in their + compositional formula or a dictionary with the formula as the key referring + to a list of miller indices. + """ + + def __init__(self, api_key, list_of_elements=[], indices_dict=None, + slab_size=10, vac_size=10, host=None, port=None, user=None, + password=None, symprec=0.001, angle_tolerance=5, database=None, + collection="Surface_Collection", fail_safe=True, reset=False): + + """ + Args: + api_key (str): A String API key for accessing the MaterialsProject + list_of_elements ([str, ...]): A list of compounds or elements to create + slabs from. Must be a string that can be searched for with MPRester. + Either list_of_elements or indices_dict has to be entered in. + indices_dict ({element(str): [[h,k,l], ...]}): A dictionary of + miller indices corresponding to the composition formula + (key) to transform into a list of slabs. Either list_of_elements + or indices_dict has to be entered in. + host (str): For database insertion + port (int): For database insertion + user (str): For database insertion + password (str): For database insertion + symprec (float): See SpaceGroupAnalyzer in analyzer.py + angle_tolerance (int): See SpaceGroupAnalyzer in analyzer.py + database (str): For database insertion + """ + + unit_cells_dict = {} + vaspdbinsert_params = {'host': host, + 'port': port, 'user': user, + 'password': password, + 'database': database, + 'collection': collection} + + elements = [key for key in indices_dict.keys()] \ + if indices_dict else list_of_elements + + # For loop will eneumerate through all the compositional + # formulas in list_of_elements or indices_dict to get a + # list of relaxed conventional unit cells froom MP. These + # will be used to generate all oriented unit cells and slabs. + + for el in elements: + + """ + element: str, element name of Metal + miller_index: hkl, e.g. [1, 1, 0] + api_key: to get access to MP DB + """ + + # This initializes the REST adaptor. Put your own API key in. + mprest = MPRester(api_key) + #Returns a list of MPIDs with the compositional formular, the + # first MPID IS NOT the lowest energy per atom + entries = mprest.get_entries(el, inc_structure="final") + + e_per_atom = [entry.energy_per_atom for entry in entries] + for entry in entries: + if min(e_per_atom) == entry.energy_per_atom: + prim_unit_cell = entry.structure + + spa = SpacegroupAnalyzer(prim_unit_cell, symprec=symprec, + angle_tolerance=angle_tolerance) + conv_unit_cell = spa.get_conventional_standard_structure() + print conv_unit_cell + unit_cells_dict[el] = [conv_unit_cell, min(e_per_atom)] + print el + + + self.api_key = api_key + self.vaspdbinsert_params = vaspdbinsert_params + self.symprec = symprec + self.angle_tolerance = angle_tolerance + self.unit_cells_dict = unit_cells_dict + self.indices_dict = indices_dict + self.elements = elements + self.ssize = slab_size + self.vsize = vac_size + self.reset = reset + self.fail_safe = fail_safe + + + def from_max_index(self, max_index, max_normal_search=True, + terminations=False, get_bulk_e=True, max_only=False): + + """ + Class method to create a surface workflow with a list of unit cells + based on the max miller index. Used in combination with list_of_elements + + Args: + max_index (int): The maximum miller index to create slabs from + max_normal_search (bool): Whether or not to orthogonalize slabs + and oriented unit cells along the c direction. + terminations (bool): Whether or not to consider the different + possible terminations in a slab. If set to false, only one + slab is calculated per miller index with the shift value + set to 0. + """ + + miller_dict = {} + for el in self.elements: + max_miller = [] + # generate_all_slabs() is very slow, especially for Mn + list_of_indices = \ + get_symmetrically_distinct_miller_indices(self.unit_cells_dict[el][0], + max_index) + + print 'surface ', el + + print '# ', el + + if max_only: + for hkl in list_of_indices: + if abs(min(hkl)) == max_index or abs(max(hkl)) == max_index: + max_miller.append(hkl) + miller_dict[el] = max_only + else: + miller_dict[el] = list_of_indices + + return CreateSurfaceWorkflow(miller_dict, self.unit_cells_dict, + self.vaspdbinsert_params, ssize=self.ssize, + vsize=self.vsize, + max_normal_search=max_normal_search, + terminations=terminations, + fail_safe=self.fail_safe, reset=self.reset, + get_bulk_e=get_bulk_e) + + + def from_list_of_indices(self, list_of_indices, max_normal_search=True, + terminations=False, get_bulk_e = True): + + """ + Class method to create a surface workflow with a + list of unit cells based on a list of miller indices. + + Args: + list_of_indices (list of indices): eg. [[h,k,l], [h,k,l], ...etc] + A list of miller indices to generate slabs from. + Used in combination with list_of_elements. + """ + + miller_dict = {} + for el in self.elements: + miller_dict[el] = list_of_indices + + return CreateSurfaceWorkflow(miller_dict, self.unit_cells_dict, + self.vaspdbinsert_params, + ssize=self.ssize, vsize=self.vsize, + max_normal_search=max_normal_search, + terminations=terminations, + fail_safe=self.fail_safe, reset=self.reset, get_bulk_e=get_bulk_e) + + + def from_indices_dict(self, max_normal_search=True, terminations=False, get_bulk_e=True): + + """ + Class method to create a surface workflow with a dictionary with the keys + being the formula of the unit cells we want to create slabs from which + will refer to a list of miller indices. + eg. indices_dict={'Fe': [[1,1,0]], 'LiFePO4': [[1,1,1], [2,2,1]]} + """ + + return CreateSurfaceWorkflow(self.indices_dict, self.unit_cells_dict, + self.vaspdbinsert_params, + ssize=self.ssize, vsize=self.vsize, + max_normal_search=max_normal_search, + terminations=terminations, + fail_safe=self.fail_safe, reset=self.reset, get_bulk_e=get_bulk_e) + + +class CreateSurfaceWorkflow(object): + + """ + A class for creating surface workflows and creating a dicionary of all + calculated surface energies and wulff shape objects. Don't actually + create an object of this class manually, instead use + SurfaceWorkflowManager to create an object of this class. + """ + + def __init__(self, miller_dict, unit_cells_dict, vaspdbinsert_params, ssize, vsize, + terminations=False, max_normal_search=True, fail_safe=True, reset=False, get_bulk_e=True): + + """ + Args: + miller_dict (ditionary): Each class method from SurfaceWorkflowManager + will create a dictionary similar to indices_dict (see previous doc). + unit_cells_dict (dictionary): A dictionary of unit cells with the + formula of the unit cell being the key reffering to a Structure + object taken from MP, eg. + unit_cells_dict={'Cr': , 'LiCoO2': } + vaspdbinsert_params (dictionary): A kwargs used for the VaspSlabDBInsertTask + containing information pertaining to the database that the vasp + outputs will be inserted into, + ie vaspdbinsert_params = {'host': host,'port': port, 'user': user, + 'password': password, 'database': database} + """ + + self.miller_dict = miller_dict + self.unit_cells_dict = unit_cells_dict + self.vaspdbinsert_params = vaspdbinsert_params + self.max_normal_search = max_normal_search + self.terminations = terminations + self.ssize = ssize + self.vsize = vsize + self.reset = reset + self.fail_safe = fail_safe + self.get_bulk_e = get_bulk_e + + + def launch_workflow(self, launchpad_dir="", k_product=50, job=None, + user_incar_settings=None, potcar_functional='PBE', + additional_handlers=[]): + + """ + Creates a list of Fireworks. Each Firework represents calculations + that will be done on a slab system of a compound in a specific + orientation. Each Firework contains a oriented unit cell relaxation job + and a WriteSlabVaspInputs which creates os. Firework(s) depending + on whether or not Termination=True. Vasp outputs from all slab and + oriented unit cell calculations will then be inserted into a database. + Args: + launchpad_dir (str path): The path to my_launchpad.yaml. Defaults to + the current working directory containing your runs + k_product: kpts[0][0]*a. Decide k density without + kpoint0, default to 50 + cwd: (str path): The curent working directory. Location of where you + want your vasp outputs to be. + job (VaspJob): The command (cmd) entered into VaspJob object. Default + is specifically set for running vasp jobs on Carver at NERSC + (use aprun for Hopper or Edison). + user_incar_settings(dict): A dict specifying additional incar + settings, default to None (ediff_per_atom=False) + potcar_functional (str): default to PBE + """ + + launchpad = LaunchPad.from_file(os.path.join(os.environ["HOME"], + launchpad_dir, + "my_launchpad.yaml")) + if self.reset: + launchpad.reset('', require_password=False) + + # Scratch directory reffered to by custodian. + # May be different on non-Nersc systems. + + if not job: + job = VaspJob(["mpirun", "-n", "64", "vasp"], + auto_npar=False, copy_magmom=True) + + handlers = [VaspErrorHandler(), + NonConvergingErrorHandler(), + UnconvergedErrorHandler(), + PotimErrorHandler(), + PositiveEnergyErrorHandler(), + FrozenJobErrorHandler(timeout=3600)] + if additional_handlers: + handlers.extend(additional_handlers) + + cust_params = {"custodian_params": + {"scratch_dir": + os.path.join("/global/scratch2/sd/", + os.environ["USER"])}, + "jobs": job.double_relaxation_run(job.vasp_cmd, + auto_npar=False), + "handlers": handlers, + "max_errors": 100} # will return a list of jobs + # instead of just being one job + + fws=[] + for key in self.miller_dict.keys(): + # Enumerate through all compounds in the dictionary, + # the key is the compositional formula of the compound + print key + for miller_index in self.miller_dict[key]: + # Enumerates through all miller indices we + # want to create slabs of that compound from + + print str(miller_index) + + max_norm = max(miller_index) if self.max_normal_search else None + # Whether or not we want to use the + # max_normal_search algorithm from surface.py + print 'true or false max norm is ', max_norm, self.max_normal_search + + slab = SlabGenerator(self.unit_cells_dict[key][0], miller_index, + self.ssize, self.vsize, max_normal_search=max_norm) + oriented_uc = slab.oriented_unit_cell + + if self.fail_safe and len(oriented_uc)> 199: + break + # This method only creates the oriented unit cell, the + # slabs are created in the WriteSlabVaspInputs task. + # WriteSlabVaspInputs will create the slabs from + # the contcar of the oriented unit cell calculation + handler = [] + tasks = [] + + folderbulk = '/%s_%s_k%s_s%sv%s_%s%s%s' %(oriented_uc.composition.reduced_formula, + 'bulk', k_product, self.ssize, self.vsize, + str(miller_index[0]), + str(miller_index[1]), + str(miller_index[2])) + cwd = os.getcwd() + if self.get_bulk_e: + tasks.extend([WriteUCVaspInputs(oriented_ucell=oriented_uc, + folder=folderbulk, cwd=cwd, + user_incar_settings=user_incar_settings, + potcar_functional=potcar_functional, + k_product=k_product), + RunCustodianTask(dir=folderbulk, cwd=cwd, + **cust_params), + VaspSlabDBInsertTask(struct_type="oriented_unit_cell", + loc=folderbulk, cwd=cwd, + miller_index=miller_index, + **self.vaspdbinsert_params)]) + + # Slab will inherit average final magnetic moment + # of the bulk from outcar, will have to generalize + # this for systems with different elements later + # element = oriented_uc.species[0] + # out = Outcar(cwd+folderbulk) + # out_mag = out.magnetization + # tot_mag = [mag['tot'] for mag in out_mag] + # magmom = np.mean(tot_mag) + # user_incar_settings['MAGMOM'] = {element: magmom} + + tasks.append(WriteSlabVaspInputs(folder=folderbulk, cwd=cwd, + user_incar_settings=user_incar_settings, + terminations=self.terminations, + custodian_params=cust_params, + vaspdbinsert_parameters= + self.vaspdbinsert_params, + potcar_functional=potcar_functional, + k_product=k_product, + miller_index=miller_index, + min_slab_size=self.ssize, + min_vacuum_size=self.vsize, + ucell=self.unit_cells_dict[key][0])) + + fw = Firework(tasks, name=folderbulk) + + fws.append(fw) + wf = Workflow(fws, name='Surface Calculations') + launchpad.add_wf(wf) diff --git a/mpworks/workflows/test_wfs/scancel/srun_std_err_example.txt b/mpworks/workflows/test_wfs/scancel/srun_std_err_example.txt new file mode 100755 index 00000000..63e2d201 --- /dev/null +++ b/mpworks/workflows/test_wfs/scancel/srun_std_err_example.txt @@ -0,0 +1,72 @@ +srun: defined options for program `srun' +srun: --------------- --------------------- +srun: user : `xhqu' +srun: uid : 58809 +srun: gid : 58809 +srun: cwd : /global/cscratch1/sd/xhqu/tt/software_test/test_vasp/fix_attempt/launcher_2016-07-07-11-10-02-640839 +srun: ntasks : 32 (set) +srun: nodes : 1 (set) +srun: jobid : 2667797 (default) +srun: partition : default +srun: profile : `NotSet' +srun: job name : `vtest' +srun: reservation : `(null)' +srun: burst_buffer : `(null)' +srun: wckey : `(null)' +srun: cpu_freq_min : 4294967294 +srun: cpu_freq_max : 4294967294 +srun: cpu_freq_gov : 4294967294 +srun: switches : -1 +srun: wait-for-switches : -1 +srun: distribution : unknown +srun: cpu_bind : default +srun: mem_bind : default +srun: verbose : 1 +srun: slurmd_debug : 0 +srun: immediate : false +srun: label output : false +srun: unbuffered IO : false +srun: overcommit : false +srun: threads : 60 +srun: checkpoint_dir : /var/slurm/checkpoint +srun: wait : 0 +srun: nice : -2 +srun: account : (null) +srun: comment : (null) +srun: dependency : (null) +srun: exclusive : false +srun: bcast : false +srun: qos : (null) +srun: constraints : mem-per-node=124928M nodelist=nid00024 +srun: geometry : (null) +srun: reboot : yes +srun: rotate : no +srun: preserve_env : false +srun: network : (null) +srun: propagate : NONE +srun: prolog : (null) +srun: epilog : (null) +srun: mail_type : NONE +srun: mail_user : (null) +srun: task_prolog : (null) +srun: task_epilog : (null) +srun: multi_prog : no +srun: sockets-per-node : -2 +srun: cores-per-socket : -2 +srun: threads-per-core : -2 +srun: ntasks-per-node : 32 +srun: ntasks-per-socket : -2 +srun: ntasks-per-core : -2 +srun: plane_size : 4294967294 +srun: core-spec : NA +srun: power : +srun: remote command : `vasp_std' +srun: Consumable Resources (CR) Node Selection plugin loaded with argument 50 +srun: launching 2667797.4 on host nid00024, 32 tasks: [0-31] +srun: route default plugin loaded +srun: Node nid00024, 32 tasks started +srun: Received task exit notification for 32 tasks (status=0x0009). +srun: error: nid00024: tasks 0-31: Killed +srun: Terminating job step 2667797.4 +srun: Force Terminated job step 2667797.4 +srun: Complete job step 2667797.4 received diff --git a/mpworks/workflows/test_wfs/wf_feo_dupes.json b/mpworks/workflows/test_wfs/wf_feo_dupes.json index 115358e7..4e3f4f68 100644 --- a/mpworks/workflows/test_wfs/wf_feo_dupes.json +++ b/mpworks/workflows/test_wfs/wf_feo_dupes.json @@ -1,21 +1,4 @@ { - "updated_on": "2014-01-06T00:52:40.926616", - "metadata": { - "elements": [ - "Fe", - "O" - ], - "is_ordered": true, - "anonymized_formula": "AB", - "chemsystem": "Fe-O", - "reduced_cell_formula_abc": "Fe1 O1", - "nelements": 2, - "reduced_cell_formula": "FeO", - "nsites": 2, - "run_version": "May 2013 (1)", - "is_valid": true, - "formula": "Fe1 O1" - }, "name": "Fe1 O1", "links": { "11": [ @@ -31,20 +14,21 @@ "0": [ 1 ], - "3": [], "2": [ - 3, 10 ] }, + "created_on": "2016-07-19T23:46:34.426230", + "updated_on": "2016-07-19T23:46:34.426234", "fws": [ { + "updated_on": "2016-07-19T23:46:34.410272", "fw_id": 0, "spec": { "_queueadapter": { + "nodes": 1, "nnodes": 1, - "pre_rocket": "#PBS -v DB_LOC,FW_CONFIG_FILE,VENV_LOC\nulimit -v hard\nmodule load python/2.7.3\nsource $VENV_LOC", - "walltime": "12:00:00" + "walltime": "2:00:00" }, "_tasks": [ { @@ -80,7 +64,7 @@ }, "about": { "created_at": { - "string": "2014-01-06 00:52:40.859438", + "string": "2016-07-19 23:46:34.394377", "@class": "datetime", "@module": "datetime" }, @@ -97,7 +81,6 @@ }, "sites": [ { - "label": "Fe", "xyz": [ 0.0, 0.0, @@ -108,16 +91,15 @@ 0.0, 0.0 ], - "properties": {}, "species": [ { "occu": 1.0, "element": "Fe" } - ] + ], + "label": "Fe" }, { - "label": "O", "xyz": [ -3.5247409773959926, 1.2461859923667329, @@ -128,13 +110,13 @@ 0.49999899999999997, 0.500001 ], - "properties": {}, "species": [ { "occu": 1.0, "element": "O" } - ] + ], + "label": "O" } ], "@class": "StructureNL", @@ -143,14 +125,17 @@ "task_type": "Add to SNL database", "_priority": 1 }, - "created_on": "2014-01-06T00:52:40.888095", + "created_on": "2016-07-19T23:46:34.410269", "name": "Fe1_O1--Add_to_SNL_database" }, { + "updated_on": "2016-07-19T23:46:34.418957", "fw_id": 1, "spec": { "_queueadapter": { - "nnodes": 2 + "nodes": 2, + "nnodes": 2, + "walltime": "48:00:00" }, "_tasks": [ { @@ -165,263 +150,8 @@ "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", - "default_vasp_input_set": { - "sort_structure": true, - "hubbard_off": false, - "constrain_total_magmom": false, - "name": "MIT", - "config_dict": { - "INCAR": { - "NELM": 100, - "IBRION": 2, - "LDAUTYPE": 2, - "LWAVE": false, - "SIGMA": 0.05, - "MAGMOM": { - "Mo": 5, - "Ni": 2, - "Mn4+": 3, - "Co": 5, - "Ni3+": 1, - "Fe2+": 4, - "Mn": 5, - "Co3+": 0.6, - "Fe3+": 5, - "Mn2+": 5, - "Fe": 5, - "W": 5, - "Mn3+": 4, - "Ni4+": 0.6, - "Cr": 5, - "V": 5, - "Fe4+": 4, - "Co4+": 1, - "Ta": 5 - }, - "NELMIN": 6, - "LDAUL": { - "S": { - "Mn": 2.5, - "Fe": 2 - }, - "O": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - }, - "F": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - } - }, - "LDAUJ": { - "S": { - "Mn": 0, - "Fe": 0 - }, - "O": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - }, - "F": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - } - }, - "ENCUT": 520, - "ISIF": 3, - "ICHARG": 1, - "LDAUU": { - "S": { - "Mn": 2.5, - "Fe": 1.9 - }, - "O": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - }, - "F": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - } - }, - "EDIFF": 5e-05, - "NSW": 99, - "LREAL": "AUTO", - "ISMEAR": -5, - "NPAR": 1, - "PREC": "Accurate", - "LDAU": true, - "ALGO": "FAST", - "ISPIN": 2, - "LORBIT": "11" - }, - "KPOINTS": { - "grid_density": 1000 - }, - "POTCAR": { - "Ru": "Ru_pv", - "Re": "Re", - "Rb": "Rb_pv", - "Rh": "Rh", - "Be": "Be", - "Ba": "Ba_sv", - "Bi": "Bi", - "Br": "Br", - "H": "H", - "P": "P", - "Os": "Os", - "Ge": "Ge", - "Gd": "Gd", - "Ga": "Ga", - "Pr": "Pr", - "Pt": "Pt", - "Pu": "Pu", - "C": "C", - "Pb": "Pb", - "Pa": "Pa", - "Pd": "Pd", - "Cd": "Cd", - "Pm": "Pm", - "Ho": "Ho_3", - "Hf": "Hf", - "Hg": "Hg", - "He": "He", - "Mg": "Mg", - "K": "K_sv", - "Mn": "Mn", - "O": "O", - "S": "S", - "W": "W_pv", - "Zn": "Zn", - "Eu": "Eu", - "Zr": "Zr", - "Er": "Er_3", - "Ni": "Ni", - "Na": "Na", - "Nb": "Nb_pv", - "Nd": "Nd", - "Ne": "Ne", - "Np": "Np", - "Fe": "Fe", - "B": "B", - "F": "F", - "Sr": "Sr_sv", - "N": "N", - "Kr": "Kr", - "Si": "Si", - "Sn": "Sn_d", - "Sm": "Sm_3", - "V": "V_pv", - "Sc": "Sc_sv", - "Sb": "Sb", - "Se": "Se", - "Co": "Co", - "Cl": "Cl", - "Ca": "Ca_sv", - "Ce": "Ce", - "Xe": "Xe", - "Tm": "Tm_3", - "Cs": "Cs_sv", - "Cr": "Cr", - "Cu": "Cu", - "La": "La", - "Li": "Li", - "Tl": "Tl", - "Lu": "Lu_3", - "Th": "Th", - "Ti": "Ti", - "Te": "Te", - "Tb": "Tb_3", - "Tc": "Tc", - "Ta": "Ta", - "Yb": "Yb", - "Dy": "Dy_3", - "I": "I", - "U": "U", - "Y": "Y_sv", - "Ac": "Ac", - "Ag": "Ag", - "Ir": "Ir", - "Al": "Al", - "As": "As", - "Ar": "Ar", - "Au": "Au", - "In": "In", - "Mo": "Mo_pv" - } - }, - "@module": "pymatgen.io.vaspio_set", - "@class": "DictVaspInputSet" - }, "gamma_vasp_cmd": null, "vasp_cmd": "VASP_EXE", - "gzipped": false, "backup": true, "final": false, "@class": "VaspJob" @@ -431,7 +161,8 @@ { "action": { "_set": { - "ISTART": 1 + "ISTART": 1, + "EDIFFG": -0.05 } }, "dict": "INCAR" @@ -442,7 +173,7 @@ "dest": "POSCAR" } }, - "filename": "CONTCAR" + "file": "CONTCAR" } ], "suffix": ".relax2", @@ -450,263 +181,8 @@ "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", - "default_vasp_input_set": { - "sort_structure": true, - "hubbard_off": false, - "constrain_total_magmom": false, - "name": "MIT", - "config_dict": { - "INCAR": { - "NELM": 100, - "IBRION": 2, - "LDAUTYPE": 2, - "LWAVE": false, - "SIGMA": 0.05, - "MAGMOM": { - "Mo": 5, - "Ni": 2, - "Mn4+": 3, - "Co": 5, - "Ni3+": 1, - "Fe2+": 4, - "Mn": 5, - "Co3+": 0.6, - "Fe3+": 5, - "Mn2+": 5, - "Fe": 5, - "W": 5, - "Mn3+": 4, - "Ni4+": 0.6, - "Cr": 5, - "V": 5, - "Fe4+": 4, - "Co4+": 1, - "Ta": 5 - }, - "NELMIN": 6, - "LDAUL": { - "S": { - "Mn": 2.5, - "Fe": 2 - }, - "O": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - }, - "F": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - } - }, - "LDAUJ": { - "S": { - "Mn": 0, - "Fe": 0 - }, - "O": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - }, - "F": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - } - }, - "ENCUT": 520, - "ISIF": 3, - "ICHARG": 1, - "LDAUU": { - "S": { - "Mn": 2.5, - "Fe": 1.9 - }, - "O": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - }, - "F": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - } - }, - "EDIFF": 5e-05, - "NSW": 99, - "LREAL": "AUTO", - "ISMEAR": -5, - "NPAR": 1, - "PREC": "Accurate", - "LDAU": true, - "ALGO": "FAST", - "ISPIN": 2, - "LORBIT": "11" - }, - "KPOINTS": { - "grid_density": 1000 - }, - "POTCAR": { - "Ru": "Ru_pv", - "Re": "Re", - "Rb": "Rb_pv", - "Rh": "Rh", - "Be": "Be", - "Ba": "Ba_sv", - "Bi": "Bi", - "Br": "Br", - "H": "H", - "P": "P", - "Os": "Os", - "Ge": "Ge", - "Gd": "Gd", - "Ga": "Ga", - "Pr": "Pr", - "Pt": "Pt", - "Pu": "Pu", - "C": "C", - "Pb": "Pb", - "Pa": "Pa", - "Pd": "Pd", - "Cd": "Cd", - "Pm": "Pm", - "Ho": "Ho_3", - "Hf": "Hf", - "Hg": "Hg", - "He": "He", - "Mg": "Mg", - "K": "K_sv", - "Mn": "Mn", - "O": "O", - "S": "S", - "W": "W_pv", - "Zn": "Zn", - "Eu": "Eu", - "Zr": "Zr", - "Er": "Er_3", - "Ni": "Ni", - "Na": "Na", - "Nb": "Nb_pv", - "Nd": "Nd", - "Ne": "Ne", - "Np": "Np", - "Fe": "Fe", - "B": "B", - "F": "F", - "Sr": "Sr_sv", - "N": "N", - "Kr": "Kr", - "Si": "Si", - "Sn": "Sn_d", - "Sm": "Sm_3", - "V": "V_pv", - "Sc": "Sc_sv", - "Sb": "Sb", - "Se": "Se", - "Co": "Co", - "Cl": "Cl", - "Ca": "Ca_sv", - "Ce": "Ce", - "Xe": "Xe", - "Tm": "Tm_3", - "Cs": "Cs_sv", - "Cr": "Cr", - "Cu": "Cu", - "La": "La", - "Li": "Li", - "Tl": "Tl", - "Lu": "Lu_3", - "Th": "Th", - "Ti": "Ti", - "Te": "Te", - "Tb": "Tb_3", - "Tc": "Tc", - "Ta": "Ta", - "Yb": "Yb", - "Dy": "Dy_3", - "I": "I", - "U": "U", - "Y": "Y_sv", - "Ac": "Ac", - "Ag": "Ag", - "Ir": "Ir", - "Al": "Al", - "As": "As", - "Ar": "Ar", - "Au": "Au", - "In": "In", - "Mo": "Mo_pv" - } - }, - "@module": "pymatgen.io.vaspio_set", - "@class": "DictVaspInputSet" - }, "gamma_vasp_cmd": null, "vasp_cmd": "VASP_EXE", - "gzipped": false, "backup": false, "final": true, "@class": "VaspJob" @@ -721,7 +197,7 @@ "@module": "custodian.vasp.handlers" }, { - "timeout": 3600, + "timeout": 21600, "output_filename": "vasp.out", "@class": "FrozenJobErrorHandler", "@module": "custodian.vasp.handlers" @@ -738,42 +214,57 @@ "output_filename": "OSZICAR", "@class": "NonConvergingErrorHandler", "@module": "custodian.vasp.handlers" + }, + { + "output_filename": "OSZICAR", + "@class": "PositiveEnergyErrorHandler", + "@module": "custodian.vasp.handlers" } ] } ], "task_type": "GGA optimize structure (2x)", + "parameters": { + "boltztrap": true + }, "_trackers": [ { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.out" }, { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.error" }, { + "allow_zipped": false, "nlines": 25, "filename": "vasp.out" }, { + "allow_zipped": false, "nlines": 25, "filename": "OUTCAR" }, { + "allow_zipped": false, "nlines": 25, "filename": "OSZICAR" }, { + "allow_zipped": false, "nlines": 25, "filename": "OUTCAR.relax1" }, { + "allow_zipped": false, "nlines": 25, "filename": "OUTCAR.relax2" } ], - "vaspinputset_name": "DictVaspInputSet", + "vaspinputset_name": "MPRelaxSet", "_priority": 2, "run_tags": [ "PBE", @@ -791,49 +282,55 @@ 5, 0.6 ], - "NELM": 100, "ENCUT": 520, - "@module": "pymatgen.io.vaspio.vasp_input", + "NELM": 100, + "NSW": 99, "PREC": "Accurate", + "@module": "pymatgen.io.vasp.inputs", "ISIF": 3, "ICHARG": 1, "IBRION": 2, - "ALGO": "Fast", "LREAL": "Auto", - "ISMEAR": -5, - "ISPIN": 2, "EDIFF": 0.0001, + "ISPIN": 2, + "ISMEAR": -5, "LWAVE": false, "NPAR": 2, - "SIGMA": 0.2, + "SIGMA": 0.05, "LORBIT": 11, "@class": "Incar", - "NSW": 99 + "ALGO": "Fast" }, "kpoints": { - "comment": "pymatgen generated KPOINTS with grid density = 1000 / atom", + "comment": "pymatgen generated KPOINTS with grid density = 1233 / atom", + "tet_weight": 0, "usershift": [ 0, 0, 0 ], + "labels": null, "kpoints": [ [ - 8, - 8, - 8 + 9, + 9, + 9 ] ], - "@module": "pymatgen.io.vaspio.vasp_input", + "tet_number": 0, + "tet_connections": null, + "@module": "pymatgen.io.vasp.inputs", "nkpoints": 0, - "generation_style": "Monkhorst", + "coord_type": null, + "generation_style": "Gamma", + "kpts_weights": null, "@class": "Kpoints" }, "poscar": { "comment": "Fe1 O1", "selective_dynamics": null, "velocities": null, - "@module": "pymatgen.io.vaspio.vasp_input", + "@module": "pymatgen.io.vasp.inputs", "@class": "Poscar", "predictor_corrector": null, "structure": { @@ -865,7 +362,6 @@ }, "sites": [ { - "label": "Fe", "xyz": [ 0.0, 0.0, @@ -876,16 +372,15 @@ 0.0, 0.0 ], - "properties": {}, "species": [ { "occu": 1.0, "element": "Fe" } - ] + ], + "label": "Fe" }, { - "label": "O", "xyz": [ -3.5247409773959926, 1.2461859923667329, @@ -896,13 +391,13 @@ 0.49999899999999997, 0.500001 ], - "properties": {}, "species": [ { "occu": 1.0, "element": "O" } - ] + ], + "label": "O" } ], "@class": "Structure", @@ -917,20 +412,21 @@ ], "functional": "PBE", "@class": "Potcar", - "@module": "pymatgen.io.vaspio.vasp_input" + "@module": "pymatgen.io.vasp.inputs" } } }, - "created_on": "2014-01-06T00:52:40.915528", + "created_on": "2016-07-19T23:46:34.418953", "name": "Fe1_O1--GGA_optimize_structure_(2x)" }, { + "updated_on": "2016-07-19T23:46:34.419006", "fw_id": 2, "spec": { "_queueadapter": { + "nodes": 1, "nnodes": 1, - "pre_rocket": "#PBS -v DB_LOC,FW_CONFIG_FILE,VENV_LOC\nulimit -v hard\nmodule load python/2.7.3\nsource $VENV_LOC", - "walltime": "12:00:00" + "walltime": "2:00:00" }, "_tasks": [ { @@ -940,10 +436,12 @@ "task_type": "VASP db insertion", "_trackers": [ { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.out" }, { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.error" } @@ -954,32 +452,17 @@ "_fw_name": "Dupe Finder DB" } }, - "created_on": "2014-01-06T00:52:40.915581", + "created_on": "2016-07-19T23:46:34.419005", "name": "Fe1_O1--VASP_db_insertion" }, { - "fw_id": 3, - "spec": { - "_queueadapter": { - "nnodes": 1, - "walltime": "00:30:00" - }, - "_tasks": [ - { - "_fw_name": "Add Electronic Structure Task v2" - } - ], - "_priority": 2, - "task_type": "Controller: add Electronic Structure v2" - }, - "created_on": "2014-01-06T00:52:40.915629", - "name": "Fe1_O1--Controller_add_Electronic_Structure_v2" - }, - { + "updated_on": "2016-07-19T23:46:34.425572", "fw_id": 10, "spec": { "_queueadapter": { - "nnodes": 2 + "nodes": 2, + "nnodes": 2, + "walltime": "48:00:00" }, "_tasks": [ { @@ -997,263 +480,8 @@ "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", - "default_vasp_input_set": { - "sort_structure": true, - "hubbard_off": false, - "constrain_total_magmom": false, - "name": "MIT", - "config_dict": { - "INCAR": { - "NELM": 100, - "IBRION": 2, - "LDAUTYPE": 2, - "LWAVE": false, - "SIGMA": 0.05, - "MAGMOM": { - "Mo": 5, - "Ni": 2, - "Mn4+": 3, - "Co": 5, - "Ni3+": 1, - "Fe2+": 4, - "Mn": 5, - "Co3+": 0.6, - "Fe3+": 5, - "Mn2+": 5, - "Fe": 5, - "W": 5, - "Mn3+": 4, - "Ni4+": 0.6, - "Cr": 5, - "V": 5, - "Fe4+": 4, - "Co4+": 1, - "Ta": 5 - }, - "NELMIN": 6, - "LDAUL": { - "S": { - "Mn": 2.5, - "Fe": 2 - }, - "O": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - }, - "F": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - } - }, - "LDAUJ": { - "S": { - "Mn": 0, - "Fe": 0 - }, - "O": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - }, - "F": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - } - }, - "ENCUT": 520, - "ISIF": 3, - "ICHARG": 1, - "LDAUU": { - "S": { - "Mn": 2.5, - "Fe": 1.9 - }, - "O": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - }, - "F": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - } - }, - "EDIFF": 5e-05, - "NSW": 99, - "LREAL": "AUTO", - "ISMEAR": -5, - "NPAR": 1, - "PREC": "Accurate", - "LDAU": true, - "ALGO": "FAST", - "ISPIN": 2, - "LORBIT": "11" - }, - "KPOINTS": { - "grid_density": 1000 - }, - "POTCAR": { - "Ru": "Ru_pv", - "Re": "Re", - "Rb": "Rb_pv", - "Rh": "Rh", - "Be": "Be", - "Ba": "Ba_sv", - "Bi": "Bi", - "Br": "Br", - "H": "H", - "P": "P", - "Os": "Os", - "Ge": "Ge", - "Gd": "Gd", - "Ga": "Ga", - "Pr": "Pr", - "Pt": "Pt", - "Pu": "Pu", - "C": "C", - "Pb": "Pb", - "Pa": "Pa", - "Pd": "Pd", - "Cd": "Cd", - "Pm": "Pm", - "Ho": "Ho_3", - "Hf": "Hf", - "Hg": "Hg", - "He": "He", - "Mg": "Mg", - "K": "K_sv", - "Mn": "Mn", - "O": "O", - "S": "S", - "W": "W_pv", - "Zn": "Zn", - "Eu": "Eu", - "Zr": "Zr", - "Er": "Er_3", - "Ni": "Ni", - "Na": "Na", - "Nb": "Nb_pv", - "Nd": "Nd", - "Ne": "Ne", - "Np": "Np", - "Fe": "Fe", - "B": "B", - "F": "F", - "Sr": "Sr_sv", - "N": "N", - "Kr": "Kr", - "Si": "Si", - "Sn": "Sn_d", - "Sm": "Sm_3", - "V": "V_pv", - "Sc": "Sc_sv", - "Sb": "Sb", - "Se": "Se", - "Co": "Co", - "Cl": "Cl", - "Ca": "Ca_sv", - "Ce": "Ce", - "Xe": "Xe", - "Tm": "Tm_3", - "Cs": "Cs_sv", - "Cr": "Cr", - "Cu": "Cu", - "La": "La", - "Li": "Li", - "Tl": "Tl", - "Lu": "Lu_3", - "Th": "Th", - "Ti": "Ti", - "Te": "Te", - "Tb": "Tb_3", - "Tc": "Tc", - "Ta": "Ta", - "Yb": "Yb", - "Dy": "Dy_3", - "I": "I", - "U": "U", - "Y": "Y_sv", - "Ac": "Ac", - "Ag": "Ag", - "Ir": "Ir", - "Al": "Al", - "As": "As", - "Ar": "Ar", - "Au": "Au", - "In": "In", - "Mo": "Mo_pv" - } - }, - "@module": "pymatgen.io.vaspio_set", - "@class": "DictVaspInputSet" - }, "gamma_vasp_cmd": null, "vasp_cmd": "VASP_EXE", - "gzipped": false, "backup": true, "final": false, "@class": "VaspJob" @@ -1263,7 +491,8 @@ { "action": { "_set": { - "ISTART": 1 + "ISTART": 1, + "EDIFFG": -0.05 } }, "dict": "INCAR" @@ -1274,7 +503,7 @@ "dest": "POSCAR" } }, - "filename": "CONTCAR" + "file": "CONTCAR" } ], "suffix": ".relax2", @@ -1282,263 +511,8 @@ "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", - "default_vasp_input_set": { - "sort_structure": true, - "hubbard_off": false, - "constrain_total_magmom": false, - "name": "MIT", - "config_dict": { - "INCAR": { - "NELM": 100, - "IBRION": 2, - "LDAUTYPE": 2, - "LWAVE": false, - "SIGMA": 0.05, - "MAGMOM": { - "Mo": 5, - "Ni": 2, - "Mn4+": 3, - "Co": 5, - "Ni3+": 1, - "Fe2+": 4, - "Mn": 5, - "Co3+": 0.6, - "Fe3+": 5, - "Mn2+": 5, - "Fe": 5, - "W": 5, - "Mn3+": 4, - "Ni4+": 0.6, - "Cr": 5, - "V": 5, - "Fe4+": 4, - "Co4+": 1, - "Ta": 5 - }, - "NELMIN": 6, - "LDAUL": { - "S": { - "Mn": 2.5, - "Fe": 2 - }, - "O": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - }, - "F": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - } - }, - "LDAUJ": { - "S": { - "Mn": 0, - "Fe": 0 - }, - "O": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - }, - "F": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - } - }, - "ENCUT": 520, - "ISIF": 3, - "ICHARG": 1, - "LDAUU": { - "S": { - "Mn": 2.5, - "Fe": 1.9 - }, - "O": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - }, - "F": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - } - }, - "EDIFF": 5e-05, - "NSW": 99, - "LREAL": "AUTO", - "ISMEAR": -5, - "NPAR": 1, - "PREC": "Accurate", - "LDAU": true, - "ALGO": "FAST", - "ISPIN": 2, - "LORBIT": "11" - }, - "KPOINTS": { - "grid_density": 1000 - }, - "POTCAR": { - "Ru": "Ru_pv", - "Re": "Re", - "Rb": "Rb_pv", - "Rh": "Rh", - "Be": "Be", - "Ba": "Ba_sv", - "Bi": "Bi", - "Br": "Br", - "H": "H", - "P": "P", - "Os": "Os", - "Ge": "Ge", - "Gd": "Gd", - "Ga": "Ga", - "Pr": "Pr", - "Pt": "Pt", - "Pu": "Pu", - "C": "C", - "Pb": "Pb", - "Pa": "Pa", - "Pd": "Pd", - "Cd": "Cd", - "Pm": "Pm", - "Ho": "Ho_3", - "Hf": "Hf", - "Hg": "Hg", - "He": "He", - "Mg": "Mg", - "K": "K_sv", - "Mn": "Mn", - "O": "O", - "S": "S", - "W": "W_pv", - "Zn": "Zn", - "Eu": "Eu", - "Zr": "Zr", - "Er": "Er_3", - "Ni": "Ni", - "Na": "Na", - "Nb": "Nb_pv", - "Nd": "Nd", - "Ne": "Ne", - "Np": "Np", - "Fe": "Fe", - "B": "B", - "F": "F", - "Sr": "Sr_sv", - "N": "N", - "Kr": "Kr", - "Si": "Si", - "Sn": "Sn_d", - "Sm": "Sm_3", - "V": "V_pv", - "Sc": "Sc_sv", - "Sb": "Sb", - "Se": "Se", - "Co": "Co", - "Cl": "Cl", - "Ca": "Ca_sv", - "Ce": "Ce", - "Xe": "Xe", - "Tm": "Tm_3", - "Cs": "Cs_sv", - "Cr": "Cr", - "Cu": "Cu", - "La": "La", - "Li": "Li", - "Tl": "Tl", - "Lu": "Lu_3", - "Th": "Th", - "Ti": "Ti", - "Te": "Te", - "Tb": "Tb_3", - "Tc": "Tc", - "Ta": "Ta", - "Yb": "Yb", - "Dy": "Dy_3", - "I": "I", - "U": "U", - "Y": "Y_sv", - "Ac": "Ac", - "Ag": "Ag", - "Ir": "Ir", - "Al": "Al", - "As": "As", - "Ar": "Ar", - "Au": "Au", - "In": "In", - "Mo": "Mo_pv" - } - }, - "@module": "pymatgen.io.vaspio_set", - "@class": "DictVaspInputSet" - }, "gamma_vasp_cmd": null, "vasp_cmd": "VASP_EXE", - "gzipped": false, "backup": false, "final": true, "@class": "VaspJob" @@ -1553,7 +527,7 @@ "@module": "custodian.vasp.handlers" }, { - "timeout": 3600, + "timeout": 21600, "output_filename": "vasp.out", "@class": "FrozenJobErrorHandler", "@module": "custodian.vasp.handlers" @@ -1570,42 +544,57 @@ "output_filename": "OSZICAR", "@class": "NonConvergingErrorHandler", "@module": "custodian.vasp.handlers" + }, + { + "output_filename": "OSZICAR", + "@class": "PositiveEnergyErrorHandler", + "@module": "custodian.vasp.handlers" } ] } ], "task_type": "GGA+U optimize structure (2x)", + "parameters": { + "boltztrap": true + }, "_trackers": [ { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.out" }, { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.error" }, { + "allow_zipped": false, "nlines": 25, "filename": "vasp.out" }, { + "allow_zipped": false, "nlines": 25, "filename": "OUTCAR" }, { + "allow_zipped": false, "nlines": 25, "filename": "OSZICAR" }, { + "allow_zipped": false, "nlines": 25, "filename": "OUTCAR.relax1" }, { + "allow_zipped": false, "nlines": 25, "filename": "OUTCAR.relax2" } ], - "vaspinputset_name": "DictVaspInputSet", + "vaspinputset_name": "MPRelaxSet", "_priority": 2, "run_tags": [ "PBE", @@ -1618,16 +607,17 @@ "_fw_name": "Dupe Finder Vasp" } }, - "created_on": "2014-01-06T00:52:40.925514", + "created_on": "2016-07-19T23:46:34.425569", "name": "Fe1_O1--GGAU_optimize_structure_(2x)" }, { + "updated_on": "2016-07-19T23:46:34.425624", "fw_id": 11, "spec": { "_queueadapter": { + "nodes": 1, "nnodes": 1, - "pre_rocket": "#PBS -v DB_LOC,FW_CONFIG_FILE,VENV_LOC\nulimit -v hard\nmodule load python/2.7.3\nsource $VENV_LOC", - "walltime": "12:00:00" + "walltime": "2:00:00" }, "_tasks": [ { @@ -1637,10 +627,12 @@ "task_type": "VASP db insertion", "_trackers": [ { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.out" }, { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.error" } @@ -1651,13 +643,15 @@ "_fw_name": "Dupe Finder DB" } }, - "created_on": "2014-01-06T00:52:40.925586", + "created_on": "2016-07-19T23:46:34.425623", "name": "Fe1_O1--VASP_db_insertion" }, { + "updated_on": "2016-07-19T23:46:34.425671", "fw_id": 12, "spec": { "_queueadapter": { + "nodes": 1, "nnodes": 1, "walltime": "00:30:00" }, @@ -1669,8 +663,24 @@ "_priority": 2, "task_type": "Controller: add Electronic Structure v2" }, - "created_on": "2014-01-06T00:52:40.925639", + "created_on": "2016-07-19T23:46:34.425670", "name": "Fe1_O1--Controller_add_Electronic_Structure_v2" } - ] + ], + "metadata": { + "elements": [ + "Fe", + "O" + ], + "is_ordered": true, + "anonymized_formula": "AB", + "chemsystem": "Fe-O", + "reduced_cell_formula_abc": "Fe1 O1", + "nelements": 2, + "reduced_cell_formula": "FeO", + "nsites": 2, + "run_version": "May 2013 (1)", + "is_valid": true, + "formula": "Fe1 O1" + } } \ No newline at end of file diff --git a/mpworks/workflows/test_wfs/wf_si_dupes.json b/mpworks/workflows/test_wfs/wf_si_dupes.json index 65366c28..4865c8f2 100644 --- a/mpworks/workflows/test_wfs/wf_si_dupes.json +++ b/mpworks/workflows/test_wfs/wf_si_dupes.json @@ -1,20 +1,4 @@ { - "updated_on": "2014-01-06T00:52:40.880577", - "metadata": { - "elements": [ - "Si" - ], - "is_ordered": true, - "anonymized_formula": "A", - "chemsystem": "Si", - "reduced_cell_formula_abc": "Si1", - "nelements": 1, - "reduced_cell_formula": "Si", - "nsites": 2, - "run_version": "May 2013 (1)", - "is_valid": true, - "formula": "Si2" - }, "name": "Si1", "links": { "1": [ @@ -28,14 +12,17 @@ 3 ] }, + "created_on": "2016-07-19T23:46:34.404927", + "updated_on": "2016-07-19T23:46:34.404930", "fws": [ { + "updated_on": "2016-07-19T23:46:34.395271", "fw_id": 0, "spec": { "_queueadapter": { + "nodes": 1, "nnodes": 1, - "pre_rocket": "#PBS -v DB_LOC,FW_CONFIG_FILE,VENV_LOC\nulimit -v hard\nmodule load python/2.7.3\nsource $VENV_LOC", - "walltime": "12:00:00" + "walltime": "2:00:00" }, "_tasks": [ { @@ -51,12 +38,12 @@ [ -3.3488982599877457, 0.0, - -1.933487310022217 + -1.9334873100222167 ], [ -3.3488982599877457, 0.0, - 1.9334873099777832 + 1.9334873099777834 ], [ -2.2325988399933285, @@ -71,7 +58,7 @@ }, "about": { "created_at": { - "string": "2014-01-06 00:52:40.859411", + "string": "2016-07-19 23:46:34.394357", "@class": "datetime", "@module": "datetime" }, @@ -88,27 +75,25 @@ }, "sites": [ { - "label": "Si", "xyz": [ - -6.139646809978845, + -6.139646809978846, 2.762700114996515, - 0.9667436549699591 + 0.9667436549699596 ], "abc": [ 0.375, - 0.8750000000000001, + 0.8750000000000002, 0.875 ], - "properties": {}, "species": [ { "occu": 1.0, "element": "Si" } - ] + ], + "label": "Si" }, { - "label": "Si", "xyz": [ -2.7907485499899756, 0.3946714449995022, @@ -119,13 +104,13 @@ 0.12500000000000003, 0.125 ], - "properties": {}, "species": [ { "occu": 1.0, "element": "Si" } - ] + ], + "label": "Si" } ], "@class": "StructureNL", @@ -134,14 +119,17 @@ "task_type": "Add to SNL database", "_priority": 1 }, - "created_on": "2014-01-06T00:52:40.860618", + "created_on": "2016-07-19T23:46:34.395268", "name": "Si1--Add_to_SNL_database" }, { + "updated_on": "2016-07-19T23:46:34.403415", "fw_id": 1, "spec": { "_queueadapter": { - "nnodes": 2 + "nodes": 2, + "nnodes": 2, + "walltime": "48:00:00" }, "_tasks": [ { @@ -156,263 +144,8 @@ "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", - "default_vasp_input_set": { - "sort_structure": true, - "hubbard_off": false, - "constrain_total_magmom": false, - "name": "MIT", - "config_dict": { - "INCAR": { - "NELM": 100, - "IBRION": 2, - "LDAUTYPE": 2, - "LWAVE": false, - "SIGMA": 0.05, - "MAGMOM": { - "Mo": 5, - "Ni": 2, - "Mn4+": 3, - "Co": 5, - "Ni3+": 1, - "Fe2+": 4, - "Mn": 5, - "Co3+": 0.6, - "Fe3+": 5, - "Mn2+": 5, - "Fe": 5, - "W": 5, - "Mn3+": 4, - "Ni4+": 0.6, - "Cr": 5, - "V": 5, - "Fe4+": 4, - "Co4+": 1, - "Ta": 5 - }, - "NELMIN": 6, - "LDAUL": { - "S": { - "Mn": 2.5, - "Fe": 2 - }, - "O": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - }, - "F": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - } - }, - "LDAUJ": { - "S": { - "Mn": 0, - "Fe": 0 - }, - "O": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - }, - "F": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - } - }, - "ENCUT": 520, - "ISIF": 3, - "ICHARG": 1, - "LDAUU": { - "S": { - "Mn": 2.5, - "Fe": 1.9 - }, - "O": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - }, - "F": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - } - }, - "EDIFF": 5e-05, - "NSW": 99, - "LREAL": "AUTO", - "ISMEAR": -5, - "NPAR": 1, - "PREC": "Accurate", - "LDAU": true, - "ALGO": "FAST", - "ISPIN": 2, - "LORBIT": "11" - }, - "KPOINTS": { - "grid_density": 1000 - }, - "POTCAR": { - "Ru": "Ru_pv", - "Re": "Re", - "Rb": "Rb_pv", - "Rh": "Rh", - "Be": "Be", - "Ba": "Ba_sv", - "Bi": "Bi", - "Br": "Br", - "H": "H", - "P": "P", - "Os": "Os", - "Ge": "Ge", - "Gd": "Gd", - "Ga": "Ga", - "Pr": "Pr", - "Pt": "Pt", - "Pu": "Pu", - "C": "C", - "Pb": "Pb", - "Pa": "Pa", - "Pd": "Pd", - "Cd": "Cd", - "Pm": "Pm", - "Ho": "Ho_3", - "Hf": "Hf", - "Hg": "Hg", - "He": "He", - "Mg": "Mg", - "K": "K_sv", - "Mn": "Mn", - "O": "O", - "S": "S", - "W": "W_pv", - "Zn": "Zn", - "Eu": "Eu", - "Zr": "Zr", - "Er": "Er_3", - "Ni": "Ni", - "Na": "Na", - "Nb": "Nb_pv", - "Nd": "Nd", - "Ne": "Ne", - "Np": "Np", - "Fe": "Fe", - "B": "B", - "F": "F", - "Sr": "Sr_sv", - "N": "N", - "Kr": "Kr", - "Si": "Si", - "Sn": "Sn_d", - "Sm": "Sm_3", - "V": "V_pv", - "Sc": "Sc_sv", - "Sb": "Sb", - "Se": "Se", - "Co": "Co", - "Cl": "Cl", - "Ca": "Ca_sv", - "Ce": "Ce", - "Xe": "Xe", - "Tm": "Tm_3", - "Cs": "Cs_sv", - "Cr": "Cr", - "Cu": "Cu", - "La": "La", - "Li": "Li", - "Tl": "Tl", - "Lu": "Lu_3", - "Th": "Th", - "Ti": "Ti", - "Te": "Te", - "Tb": "Tb_3", - "Tc": "Tc", - "Ta": "Ta", - "Yb": "Yb", - "Dy": "Dy_3", - "I": "I", - "U": "U", - "Y": "Y_sv", - "Ac": "Ac", - "Ag": "Ag", - "Ir": "Ir", - "Al": "Al", - "As": "As", - "Ar": "Ar", - "Au": "Au", - "In": "In", - "Mo": "Mo_pv" - } - }, - "@module": "pymatgen.io.vaspio_set", - "@class": "DictVaspInputSet" - }, "gamma_vasp_cmd": null, "vasp_cmd": "VASP_EXE", - "gzipped": false, "backup": true, "final": false, "@class": "VaspJob" @@ -422,7 +155,8 @@ { "action": { "_set": { - "ISTART": 1 + "ISTART": 1, + "EDIFFG": -0.05 } }, "dict": "INCAR" @@ -433,7 +167,7 @@ "dest": "POSCAR" } }, - "filename": "CONTCAR" + "file": "CONTCAR" } ], "suffix": ".relax2", @@ -441,263 +175,8 @@ "output_file": "vasp.out", "auto_npar": false, "@module": "custodian.vasp.jobs", - "default_vasp_input_set": { - "sort_structure": true, - "hubbard_off": false, - "constrain_total_magmom": false, - "name": "MIT", - "config_dict": { - "INCAR": { - "NELM": 100, - "IBRION": 2, - "LDAUTYPE": 2, - "LWAVE": false, - "SIGMA": 0.05, - "MAGMOM": { - "Mo": 5, - "Ni": 2, - "Mn4+": 3, - "Co": 5, - "Ni3+": 1, - "Fe2+": 4, - "Mn": 5, - "Co3+": 0.6, - "Fe3+": 5, - "Mn2+": 5, - "Fe": 5, - "W": 5, - "Mn3+": 4, - "Ni4+": 0.6, - "Cr": 5, - "V": 5, - "Fe4+": 4, - "Co4+": 1, - "Ta": 5 - }, - "NELMIN": 6, - "LDAUL": { - "S": { - "Mn": 2.5, - "Fe": 2 - }, - "O": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - }, - "F": { - "Ni": 2, - "Co": 2, - "Ag": 2, - "Mo": 2, - "Mn": 2, - "Re": 2, - "Fe": 2, - "W": 2, - "V": 2, - "Cr": 2, - "Nb": 2, - "Cu": 2, - "Ta": 2 - } - }, - "LDAUJ": { - "S": { - "Mn": 0, - "Fe": 0 - }, - "O": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - }, - "F": { - "Ni": 0, - "Co": 0, - "Ag": 0, - "Mo": 0, - "Mn": 0, - "Re": 0, - "Fe": 0, - "W": 0, - "V": 0, - "Cr": 0, - "Nb": 0, - "Cu": 0, - "Ta": 0 - } - }, - "ENCUT": 520, - "ISIF": 3, - "ICHARG": 1, - "LDAUU": { - "S": { - "Mn": 2.5, - "Fe": 1.9 - }, - "O": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - }, - "F": { - "Ni": 6, - "Co": 3.4, - "Ag": 1.5, - "Mo": 4.38, - "Mn": 3.9, - "Re": 2, - "Fe": 4.0, - "W": 4.0, - "V": 3.1, - "Cr": 3.5, - "Nb": 1.5, - "Cu": 4, - "Ta": 2 - } - }, - "EDIFF": 5e-05, - "NSW": 99, - "LREAL": "AUTO", - "ISMEAR": -5, - "NPAR": 1, - "PREC": "Accurate", - "LDAU": true, - "ALGO": "FAST", - "ISPIN": 2, - "LORBIT": "11" - }, - "KPOINTS": { - "grid_density": 1000 - }, - "POTCAR": { - "Ru": "Ru_pv", - "Re": "Re", - "Rb": "Rb_pv", - "Rh": "Rh", - "Be": "Be", - "Ba": "Ba_sv", - "Bi": "Bi", - "Br": "Br", - "H": "H", - "P": "P", - "Os": "Os", - "Ge": "Ge", - "Gd": "Gd", - "Ga": "Ga", - "Pr": "Pr", - "Pt": "Pt", - "Pu": "Pu", - "C": "C", - "Pb": "Pb", - "Pa": "Pa", - "Pd": "Pd", - "Cd": "Cd", - "Pm": "Pm", - "Ho": "Ho_3", - "Hf": "Hf", - "Hg": "Hg", - "He": "He", - "Mg": "Mg", - "K": "K_sv", - "Mn": "Mn", - "O": "O", - "S": "S", - "W": "W_pv", - "Zn": "Zn", - "Eu": "Eu", - "Zr": "Zr", - "Er": "Er_3", - "Ni": "Ni", - "Na": "Na", - "Nb": "Nb_pv", - "Nd": "Nd", - "Ne": "Ne", - "Np": "Np", - "Fe": "Fe", - "B": "B", - "F": "F", - "Sr": "Sr_sv", - "N": "N", - "Kr": "Kr", - "Si": "Si", - "Sn": "Sn_d", - "Sm": "Sm_3", - "V": "V_pv", - "Sc": "Sc_sv", - "Sb": "Sb", - "Se": "Se", - "Co": "Co", - "Cl": "Cl", - "Ca": "Ca_sv", - "Ce": "Ce", - "Xe": "Xe", - "Tm": "Tm_3", - "Cs": "Cs_sv", - "Cr": "Cr", - "Cu": "Cu", - "La": "La", - "Li": "Li", - "Tl": "Tl", - "Lu": "Lu_3", - "Th": "Th", - "Ti": "Ti", - "Te": "Te", - "Tb": "Tb_3", - "Tc": "Tc", - "Ta": "Ta", - "Yb": "Yb", - "Dy": "Dy_3", - "I": "I", - "U": "U", - "Y": "Y_sv", - "Ac": "Ac", - "Ag": "Ag", - "Ir": "Ir", - "Al": "Al", - "As": "As", - "Ar": "Ar", - "Au": "Au", - "In": "In", - "Mo": "Mo_pv" - } - }, - "@module": "pymatgen.io.vaspio_set", - "@class": "DictVaspInputSet" - }, "gamma_vasp_cmd": null, "vasp_cmd": "VASP_EXE", - "gzipped": false, "backup": false, "final": true, "@class": "VaspJob" @@ -712,7 +191,7 @@ "@module": "custodian.vasp.handlers" }, { - "timeout": 3600, + "timeout": 21600, "output_filename": "vasp.out", "@class": "FrozenJobErrorHandler", "@module": "custodian.vasp.handlers" @@ -729,42 +208,57 @@ "output_filename": "OSZICAR", "@class": "NonConvergingErrorHandler", "@module": "custodian.vasp.handlers" + }, + { + "output_filename": "OSZICAR", + "@class": "PositiveEnergyErrorHandler", + "@module": "custodian.vasp.handlers" } ] } ], "task_type": "GGA optimize structure (2x)", + "parameters": { + "boltztrap": true + }, "_trackers": [ { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.out" }, { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.error" }, { + "allow_zipped": false, "nlines": 25, "filename": "vasp.out" }, { + "allow_zipped": false, "nlines": 25, "filename": "OUTCAR" }, { + "allow_zipped": false, "nlines": 25, "filename": "OSZICAR" }, { + "allow_zipped": false, "nlines": 25, "filename": "OUTCAR.relax1" }, { + "allow_zipped": false, "nlines": 25, "filename": "OUTCAR.relax2" } ], - "vaspinputset_name": "DictVaspInputSet", + "vaspinputset_name": "MPRelaxSet", "_priority": 2, "run_tags": [ "PBE", @@ -780,32 +274,34 @@ 0.6, 0.6 ], - "NELM": 100, "ENCUT": 520, - "@module": "pymatgen.io.vaspio.vasp_input", + "NELM": 100, + "NSW": 99, "PREC": "Accurate", + "@module": "pymatgen.io.vasp.inputs", "ISIF": 3, "ICHARG": 1, "IBRION": 2, - "ALGO": "Fast", "LREAL": "Auto", - "ISMEAR": -5, - "ISPIN": 2, "EDIFF": 0.0001, + "ISPIN": 2, + "ISMEAR": -5, "LWAVE": false, "NPAR": 2, - "SIGMA": 0.2, + "SIGMA": 0.05, "LORBIT": 11, "@class": "Incar", - "NSW": 99 + "ALGO": "Fast" }, "kpoints": { - "comment": "pymatgen generated KPOINTS with grid density = 1000 / atom", + "comment": "pymatgen generated KPOINTS with grid density = 607 / atom", + "tet_weight": 0, "usershift": [ 0, 0, 0 ], + "labels": null, "kpoints": [ [ 8, @@ -813,16 +309,20 @@ 8 ] ], - "@module": "pymatgen.io.vaspio.vasp_input", + "tet_number": 0, + "tet_connections": null, + "@module": "pymatgen.io.vasp.inputs", "nkpoints": 0, + "coord_type": null, "generation_style": "Monkhorst", + "kpts_weights": null, "@class": "Kpoints" }, "poscar": { "comment": "Si2", "selective_dynamics": null, "velocities": null, - "@module": "pymatgen.io.vaspio.vasp_input", + "@module": "pymatgen.io.vasp.inputs", "@class": "Poscar", "predictor_corrector": null, "structure": { @@ -834,12 +334,12 @@ [ -3.3488982599877457, 0.0, - -1.933487310022217 + -1.9334873100222167 ], [ -3.3488982599877457, 0.0, - 1.9334873099777832 + 1.9334873099777834 ], [ -2.2325988399933285, @@ -854,27 +354,25 @@ }, "sites": [ { - "label": "Si", "xyz": [ - -6.139646809978845, + -6.139646809978846, 2.762700114996515, - 0.9667436549699591 + 0.9667436549699596 ], "abc": [ 0.375, - 0.8750000000000001, + 0.8750000000000002, 0.875 ], - "properties": {}, "species": [ { "occu": 1.0, "element": "Si" } - ] + ], + "label": "Si" }, { - "label": "Si", "xyz": [ -2.7907485499899756, 0.3946714449995022, @@ -885,13 +383,13 @@ 0.12500000000000003, 0.125 ], - "properties": {}, "species": [ { "occu": 1.0, "element": "Si" } - ] + ], + "label": "Si" } ], "@class": "Structure", @@ -905,20 +403,21 @@ ], "functional": "PBE", "@class": "Potcar", - "@module": "pymatgen.io.vaspio.vasp_input" + "@module": "pymatgen.io.vasp.inputs" } } }, - "created_on": "2014-01-06T00:52:40.878351", + "created_on": "2016-07-19T23:46:34.403411", "name": "Si1--GGA_optimize_structure_(2x)" }, { + "updated_on": "2016-07-19T23:46:34.403465", "fw_id": 2, "spec": { "_queueadapter": { + "nodes": 1, "nnodes": 1, - "pre_rocket": "#PBS -v DB_LOC,FW_CONFIG_FILE,VENV_LOC\nulimit -v hard\nmodule load python/2.7.3\nsource $VENV_LOC", - "walltime": "12:00:00" + "walltime": "2:00:00" }, "_tasks": [ { @@ -928,10 +427,12 @@ "task_type": "VASP db insertion", "_trackers": [ { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.out" }, { + "allow_zipped": false, "nlines": 25, "filename": "FW_job.error" } @@ -942,13 +443,15 @@ "_fw_name": "Dupe Finder DB" } }, - "created_on": "2014-01-06T00:52:40.878399", + "created_on": "2016-07-19T23:46:34.403463", "name": "Si1--VASP_db_insertion" }, { + "updated_on": "2016-07-19T23:46:34.404309", "fw_id": 3, "spec": { "_queueadapter": { + "nodes": 1, "nnodes": 1, "walltime": "00:30:00" }, @@ -960,8 +463,23 @@ "_priority": 2, "task_type": "Controller: add Electronic Structure v2" }, - "created_on": "2014-01-06T00:52:40.878433", + "created_on": "2016-07-19T23:46:34.404305", "name": "Si1--Controller_add_Electronic_Structure_v2" } - ] + ], + "metadata": { + "elements": [ + "Si" + ], + "is_ordered": true, + "anonymized_formula": "A", + "chemsystem": "Si", + "reduced_cell_formula_abc": "Si1", + "nelements": 1, + "reduced_cell_formula": "Si", + "nsites": 2, + "run_version": "May 2013 (1)", + "is_valid": true, + "formula": "Si2" + } } \ No newline at end of file diff --git a/mpworks/workflows/tests/__init__.py b/mpworks/workflows/tests/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/mpworks/workflows/tests/test_scancel_job_step_terminator.py b/mpworks/workflows/tests/test_scancel_job_step_terminator.py new file mode 100755 index 00000000..b2eaa3fd --- /dev/null +++ b/mpworks/workflows/tests/test_scancel_job_step_terminator.py @@ -0,0 +1,15 @@ +import os +from unittest import TestCase + +from mpworks.workflows.wf_utils import ScancelJobStepTerminator + +test_dir = os.path.join(os.path.dirname(__file__), "..", + 'test_wfs', "scancel") + +class TestScancelJobStepTerminator(TestCase): + def test_parse_srun_step_number(self): + std_err_file = os.path.join(test_dir, "srun_std_err_example.txt") + terminator = ScancelJobStepTerminator(std_err_file) + step_id = terminator.parse_srun_step_number() + self.assertEqual(step_id, "2667797.4") + diff --git a/mpworks/workflows/wf_settings.py b/mpworks/workflows/wf_settings.py index 60c8d56e..01ea9e39 100644 --- a/mpworks/workflows/wf_settings.py +++ b/mpworks/workflows/wf_settings.py @@ -5,20 +5,25 @@ __email__ = 'ajain@lbl.gov' __date__ = 'May 10, 2013' -QA_VASP = {'nnodes': 2} # don't change nnodes unless other parts of code are also changed -QA_VASP_SMALL = {'nnodes': 2, 'walltime': '72:00:00'} # small walltime jobs -QA_DB = {'nnodes': 1, 'walltime': '2:00:00', - 'pre_rocket': '#PBS -V\nulimit -v hard\nmodule load python/2.7.3\nsource $VENV_LOC'} -QA_CONTROL = {'nnodes': 1, 'walltime': '00:30:00'} +# don't change nnodes unless other parts of code are also changed +# nodes configuration will be ignored on SLURM due to different naming convention (nnodes vs nodes) +QA_VASP = {'nnodes': 2, 'nodes': 2, 'walltime': '48:00:00'} +QA_VASP_SMALL = {'nnodes': 2, 'nodes': 2, 'walltime': '48:00:00'} # small walltime jobs +QA_DB = {'nnodes': 1, 'nodes' : 1, 'walltime': '2:00:00'} +QA_CONTROL = {'nnodes': 1, 'nodes': 1, 'walltime': '00:30:00'} + MOVE_TO_GARDEN_DEV = False MOVE_TO_GARDEN_PROD = False -RUN_LOCS = ['/project/projectdirs/matgen/garden/', - '/project/projectdirs/matgen/garden/dev', +GARDEN = '/project/projectdirs/matgen/garden' + +RUN_LOCS = [GARDEN, GARDEN+'/dev', '/project/projectdirs/matgen/garden/control_blocks', - '/global/scratch/sd/matcomp/', + '/project/projectdirs/matgen/scratch', + '/global/scratch/sd/matcomp/', '/global/homes/m/matcomp', '/scratch/scratchdirs/matcomp/', '/scratch2/scratchdirs/matcomp/', '/global/scratch/sd/matcomp/aj_tests/', '/global/scratch/sd/matcomp/wc_tests/', '/global/scratch/sd/matcomp/aj_prod/', - '/global/scratch2/sd/matcomp/mp_prod'] \ No newline at end of file + '/global/scratch2/sd/matcomp/mp_prod/', + '/global/scratch2/sd/matcomp/mp_prod_hopper/'] diff --git a/mpworks/workflows/wf_utils.py b/mpworks/workflows/wf_utils.py index cd5687e5..79897a9d 100644 --- a/mpworks/workflows/wf_utils.py +++ b/mpworks/workflows/wf_utils.py @@ -1,10 +1,16 @@ import glob +import logging import os +import shlex import shutil import time import traceback + +import subprocess + +import re from monty.os.path import zpath -from mpworks.workflows.wf_settings import RUN_LOCS +from mpworks.workflows.wf_settings import RUN_LOCS, GARDEN __author__ = 'Anubhav Jain' @@ -74,10 +80,7 @@ def get_loc(m_dir): def move_to_garden(m_dir, prod=False): block_part = get_block_part(m_dir) - if prod: - garden_part = '/project/projectdirs/matgen/garden/' - else: - garden_part = '/project/projectdirs/matgen/garden/dev' + garden_part = GARDEN if prod else GARDEN+'/dev' f_dir = os.path.join(garden_part, block_part) if os.path.exists(m_dir) and not os.path.exists(f_dir) and m_dir != f_dir: try: @@ -92,4 +95,37 @@ def move_to_garden(m_dir, prod=False): raise ValueError('Could not move file to GARDEN! {}'.format(traceback.format_exc())) - return f_dir \ No newline at end of file + return f_dir + +class ScancelJobStepTerminator: + """ + A tool to cancel a job step in a SLURM srun job using scancel command. + """ + + def __init__(self, stderr_filename): + """ + + Args: + stderr_filename: The file name of the stderr for srun job step. + """ + self.stderr_filename = stderr_filename + + def cancel_job_step(self): + step_id = self.parse_srun_step_number() + scancel_cmd = shlex.split("scancel --signal=KILL {}".format(step_id)) + logging.info("Terminate the job step using {}".format(' '.join(scancel_cmd))) + subprocess.Popen(scancel_cmd) + + def parse_srun_step_number(self): + step_pat_text = r"srun: launching (?P\d+[.]\d+) on host \w+, \d+ tasks:" + step_pat = re.compile(step_pat_text) + step_id = None + with open(self.stderr_filename) as f: + err_text = f.readlines() + for line in err_text: + m = step_pat.search(line) + if m is not None: + step_id = m.group("step_id") + if step_id is None: + raise ValueError("Can't find SRUN job step number in STDERR file") + return step_id diff --git a/requirements.txt b/requirements.txt index 533a017d..78067974 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ dicttoxml==1.6.4 monty==0.6.4 pybtex==0.18 -pymongo==2.8 PyYAML==3.11 requests==2.6.0 six==1.9.0 xmltodict==0.9.2 +pymatgen>=4.0.0 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100755 index 00000000..bd93e63b --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1,6 @@ +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 06, 2013' \ No newline at end of file diff --git a/scripts/submissions_run.py b/scripts/submissions_run.py new file mode 100755 index 00000000..1c2de509 --- /dev/null +++ b/scripts/submissions_run.py @@ -0,0 +1,36 @@ +from mpworks.processors.process_submissions import SubmissionProcessor + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 06, 2013' + +""" +A runnable script for submissions +""" + +from argparse import ArgumentParser + +__author__ = "Anubhav Jain" +__copyright__ = "Copyright 2013, The Materials Project" +__version__ = "0.1" +__maintainer__ = "Anubhav Jain" +__email__ = "ajain@lbl.gov" +__date__ = "Jan 14, 2013" + + +def go_submissions(): + m_description = 'This program is used to pull jobs from the Submissions database, create FireWorks workflows from those submissions, and then monitor all previous submissions for updates to state (so that the submission database can be updated)' + + parser = ArgumentParser(description=m_description) + parser.add_argument('--sleep', help='sleep time between loops', default=None, type=int) + parser.add_argument('--infinite', help='loop infinite times', action='store_true') + args = parser.parse_args() + + sp = SubmissionProcessor.auto_load() + sp.run(args.sleep, args.infinite) + +if __name__ == '__main__': + go_submissions() \ No newline at end of file diff --git a/scripts/submit_canonical_run.py b/scripts/submit_canonical_run.py new file mode 100755 index 00000000..9011cf58 --- /dev/null +++ b/scripts/submit_canonical_run.py @@ -0,0 +1,44 @@ +from mpworks.processors.submit_canonical import clear_and_submit + +__author__ = 'Anubhav Jain' +__copyright__ = 'Copyright 2013, The Materials Project' +__version__ = '0.1' +__maintainer__ = 'Anubhav Jain' +__email__ = 'ajain@lbl.gov' +__date__ = 'May 06, 2013' + +""" +A runnable script for submitting test jobs +""" + +from argparse import ArgumentParser + +__author__ = "Anubhav Jain" +__copyright__ = "Copyright 2013, The Materials Project" +__version__ = "0.1" +__maintainer__ = "Anubhav Jain" +__email__ = "ajain@lbl.gov" +__date__ = "Jan 14, 2013" + + +def go_testing(): + m_description = 'This program is used to clear and submit jobs from the database' + + parser = ArgumentParser(description=m_description) + parser.add_argument('-c', '--clear', help='clear old databases', action='store_true') + parser.add_argument('-n', '--names', help='csv of compound names', default=None) + parser.add_argument('--noboltztrap', help='do NOT run boltztrap', action='store_true') + parser.add_argument('--exact', help='exact structure', action='store_true') + args = parser.parse_args() + + names = [x.strip() for x in args.names.split(',')] if args.names else None + + params = {} + if args.noboltztrap: + params['boltztrap'] = False + if args.exact: + params['exact_structure'] = True + clear_and_submit(args.clear, names, params) + +if __name__ == '__main__': + go_testing() \ No newline at end of file diff --git a/setup.py b/setup.py index 458cf48a..3eb14f93 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ license='modified BSD', packages=find_packages(), zip_safe=False, - install_requires=["pymatgen>=3.0", "FireWorks>=0.9", "custodian>=0.7"], + install_requires=["pymatgen>=4.0", "FireWorks>=0.9", "custodian>=0.7"], classifiers=["Programming Language :: Python :: 2.7", "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Science/Research", "Intended Audience :: System Administrators", "Intended Audience :: Information Technology",