diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..20d40b6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. \ No newline at end of file diff --git a/README.md b/README.md index a92db7e..5caf80b 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,12 @@ imdb-data-parser Parses the IMDB dumps into CSV and Relational Database insert queries Uses IMDB dumps from: http://www.imdb.com/interfaces +imdb-data-parser is a free software licensed by GPLv3. + + Requirements ================ -Python 3.x +* Python 3.x Configuring ================ @@ -18,9 +21,33 @@ You need to copy this file as `settings.py` and edit this file before running th cp settings.py.example settings.py your_favourite_editor settings.py -Execute -------- +You also need to have dump files at `INPUT_DIR` and you can download dump files from one of the FTP addresses on http://www.imdb.com/interfaces. + +Besides that you can make `imdb-data-parser` dowload dumps for you by giving `-u` argument: + + ~/imdb-data-parser$ ./imdbparser.py -u - ~/imdb-data-parser$ python3 imdbparser.py +Executing +--------- + + ~/imdb-data-parser$ ./imdbparser.py You can use -h parameter to see list of optional arguments + + ~/imdb-data-parser$ ./imdbparser.py -h + +SQL Dumps +--------- +You can use mode parameter to create SQL dumps + + ~/imdb-data-parser$ ./imdbparser.py -h + +Default configuration of MySQL doesn't allow insert data more than 16MB. You need to change your mysql max_allowed_packet size to insert sql dumps. + + max_allowed_packet = 256M + +Our movies data includes series, videos, tv shows for now. You can exclude them by this command: + + grep -v '("\\"' movies.list.sql | grep -v '\\(VG\\)' | grep -v "\\(TV\\)" | grep -v "{" | grep -v "????" | grep -v "(V\\\)" > movies.sql + +Note: SQL dumps tested with only mysql. diff --git a/idp/parser/actorsparser.py b/idp/parser/actorsparser.py new file mode 100644 index 0000000..abf343f --- /dev/null +++ b/idp/parser/actorsparser.py @@ -0,0 +1,109 @@ +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +from .baseparser import * + + +class ActorsParser(BaseParser): + """ + RegExp: /(.*?)\t+((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\s*(\(.*?\))?\s*(\(.*\))?\s*(\[.*\])?\s*(<.*>)?$/gm + pattern: (.*?)\t+((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\s*(\(.*?\))?\s*(\(.*\))?\s*(\[.*\])?\s*(<.*>)?$ + flags: gm + 12 capturing groups: + group 1: (.*?) surname, name + group 2: #TITLE (UNIQUE KEY) + group 3: (.*? \(\S{4,}\)) movie name + year + group 4: (\(\S+\)) type ex:(TV) + group 5: (\{(.*?) ?(\(\S+?\))?\}) series info ex: {Ally Abroad (#3.1)} + group 6: (.*?) episode name ex: Ally Abroad + group 7: (\(\S+?\)) episode number ex: (#3.1) + group 8: (\{\{SUSPENDED\}\}) is suspended? + group 9: (\(.*?\)) info 1 + group 10: (\(.*\)) info 2 + group 11: (\[.*\]) role + group 12: () + """ + + # properties + base_matcher_pattern = '(.*?)\t+((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\s*(\(.*?\))?\s*(\(.*\))?\s*(\[.*\])?\s*(<.*>)?$' + input_file_name = "actors.list" + number_of_lines_to_be_skipped = 239 + db_table_info = { + 'tablename' : 'actors', + 'columns' : [ + {'colname' : 'name', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'surname', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'title', 'colinfo' : DbScriptHelper.keywords['string'] + '(255) NOT NULL'}, + {'colname' : 'info_1', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'info_2', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'role', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'} + ], + 'constraints' : 'PRIMARY KEY(title)' + } + end_of_dump_delimiter = "" + + name = "" + surname = "" + + def __init__(self, preferences_map): + super(ActorsParser, self).__init__(preferences_map) + self.first_one = True + + def parse_into_tsv(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(len(matcher.group(1).strip()) > 0): + namelist = matcher.group(1).split(', ') + if(len(namelist) == 2): + self.name = namelist[1] + self.surname = namelist[0] + else: + self.name = namelist[0] + self.surname = "" + + self.tsv_file.write(self.name + self.seperator + self.surname + self.seperator + self.concat_regex_groups([2,9,10,11], None, matcher) + "\n") + elif(len(matcher.get_last_string()) == 1): + pass + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 + + def parse_into_db(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(len(matcher.group(1).strip()) > 0): + namelist = matcher.group(1).split(', ') + if(len(namelist) == 2): + self.name = namelist[1] + self.surname = namelist[0] + else: + self.name = namelist[0] + self.surname = "" + + if(self.first_one): + self.sql_file.write("(\"" + self.name + "\", \"" + self.surname + "\", " + self.concat_regex_groups([2,9,10,11], [2,3,4,5], matcher) + ")") + self.first_one = False; + else: + self.sql_file.write(",\n(\"" + self.name + "\", \"" + self.surname + "\", " + self.concat_regex_groups([2,9,10,11], [2,3,4,5], matcher) + ")") + + elif(len(matcher.get_last_string()) == 1): + pass + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 diff --git a/idp/parser/actressesparser.py b/idp/parser/actressesparser.py new file mode 100644 index 0000000..3cdb6b4 --- /dev/null +++ b/idp/parser/actressesparser.py @@ -0,0 +1,109 @@ +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +from .baseparser import * + + +class ActressesParser(BaseParser): + """ + RegExp: /(.*?)\t+((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\s*(\(.*?\))?\s*(\(.*\))?\s*(\[.*\])?\s*(<.*>)?$/gm + pattern: (.*?)\t+((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\s*(\(.*?\))?\s*(\(.*\))?\s*(\[.*\])?\s*(<.*>)?$ + flags: gm + 12 capturing groups: + group 1: (.*?) surname, name + group 2: #TITLE (UNIQUE KEY) + group 3: (.*? \(\S{4,}\)) movie name + year + group 4: (\(\S+\)) type ex:(TV) + group 5: (\{(.*?) ?(\(\S+?\))?\}) series info ex: {Ally Abroad (#3.1)} + group 6: (.*?) episode name ex: Ally Abroad + group 7: (\(\S+?\)) episode number ex: (#3.1) + group 8: (\{\{SUSPENDED\}\}) is suspended? + group 9: (\(.*?\)) info 1 + group 10: (\(.*\)) info 2 + group 11: (\[.*\]) role + group 12: () + """ + + # properties + base_matcher_pattern = '(.*?)\t+((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\s*(\(.*?\))?\s*(\(.*\))?\s*(\[.*\])?\s*(<.*>)?$' + input_file_name = "actresses.list" + number_of_lines_to_be_skipped = 241 + db_table_info = { + 'tablename' : 'actresses', + 'columns' : [ + {'colname' : 'name', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'surname', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'title', 'colinfo' : DbScriptHelper.keywords['string'] + '(255) NOT NULL'}, + {'colname' : 'info_1', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'info_2', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'role', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'} + ], + 'constraints' : 'PRIMARY KEY(title)' + } + end_of_dump_delimiter = "" + + name = "" + surname = "" + + def __init__(self, preferences_map): + super(ActressesParser, self).__init__(preferences_map) + self.first_one = True + + def parse_into_tsv(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(len(matcher.group(1).strip()) > 0): + namelist = matcher.group(1).split(', ') + if(len(namelist) == 2): + self.name = namelist[1] + self.surname = namelist[0] + else: + self.name = namelist[0] + self.surname = "" + + self.tsv_file.write(self.name + self.seperator + self.surname + self.seperator + self.concat_regex_groups([2,9,10,11], None, matcher) + "\n") + elif(len(matcher.get_last_string()) == 1): + pass + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 + + def parse_into_db(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(len(matcher.group(1).strip()) > 0): + namelist = matcher.group(1).split(', ') + if(len(namelist) == 2): + self.name = namelist[1] + self.surname = namelist[0] + else: + self.name = namelist[0] + self.surname = "" + + if(self.first_one): + self.sql_file.write("(\"" + self.name + "\", \"" + self.surname + "\", " + self.concat_regex_groups([2,9,10,11], [2,3,4,5], matcher) + ")") + self.first_one = False; + else: + self.sql_file.write(",\n(\"" + self.name + "\", \"" + self.surname + "\", " + self.concat_regex_groups([2,9,10,11], [2,3,4,5], matcher) + ")") + + elif(len(matcher.get_last_string()) == 1): + pass + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 diff --git a/idp/parser/baseparser.py b/idp/parser/baseparser.py index c2468b4..4ea8614 100644 --- a/idp/parser/baseparser.py +++ b/idp/parser/baseparser.py @@ -1,45 +1,148 @@ -from abc import * -from ..utils.filehandler import * - -class BaseParser(metaclass=ABCMeta): - """Common methods for all parser classes""" - - seperator = "\t" - - @abstractmethod - def parse_into_tsv(self): - raise NotImplemented - - @abstractmethod - def parse_into_db(self): - raise NotImplemented - - def start_processing(self): - if(self.preferencesMap["mode"] == "TSV"): - self.parse_into_tsv() - elif(self.preferencesMap["mode"] == "SQL"): - self.parse_into_db() - else: - raise NotImplemented("Mode: " + self.preferencesMap["mode"]) - - def get_input_file(self): - return openfile(get_full_path(self.inputFileName)) - - def get_output_file(self): - return open(get_full_path_for_tsv(self.inputFileName), "w") - - @abstractproperty - def baseMatcherPattern(self): - raise NotImplemented - - @abstractproperty - def inputFileName(self): - raise NotImplemented - - @abstractproperty - def numberOfLinesToBeSkipped(self): - raise NotImplemented - - @abstractproperty - def preferencesMap(self): - raise NotImplemented \ No newline at end of file +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +import re +import logging +from abc import * +from ..utils.filehandler import FileHandler +from ..utils.regexhelper import RegExHelper +from ..utils.decorators import duration_logged +from ..utils.dbscripthelper import DbScriptHelper + + +class BaseParser(metaclass=ABCMeta): + """ + Base class for all parser classes + + This class holds common methods for all parser classes and + must be implemented by any Parser class + + Implementing classes' responsibilities are as follows: + * Implement parse_into_tsv function + * Implement parse_into_db function + * Calculate fuckedUpCount and store in self.fuckedUpCount + * Define following properties: + - baseMatcherPattern + - inputFileName + - numberOfLinesToBeSkipped + - scripts + """ + + seperator = "\t" #TODO: get from settings + + def __init__(self, preferences_map): + self.mode = preferences_map['mode'] + self.filehandler = FileHandler(self.input_file_name, preferences_map) + self.input_file = self.filehandler.get_input_file() + + if (self.mode == "TSV"): + self.tsv_file = self.filehandler.get_tsv_file() + elif (self.mode == "SQL"): + self.sql_file = self.filehandler.get_sql_file() + self.scripthelper = DbScriptHelper(self.db_table_info) + self.sql_file.write(self.scripthelper.scripts['drop']) + self.sql_file.write(self.scripthelper.scripts['create']) + self.sql_file.write(self.scripthelper.scripts['insert']) + + @abstractmethod + def parse_into_tsv(self, matcher): + raise NotImplemented + + @abstractmethod + def parse_into_db(self, matcher): + raise NotImplemented + + @duration_logged + def start_processing(self): + ''' + Actual parsing and generation of scripts (tsv & sql) are done here. + ''' + + self.fucked_up_count = 0 + counter = 0 + number_of_processed_lines = 0 + + for line in self.input_file : #assuming the file is opened in the subclass before here + if(number_of_processed_lines >= self.number_of_lines_to_be_skipped): + #end of data + if(self.end_of_dump_delimiter != "" and self.end_of_dump_delimiter in line): + break + + matcher = RegExHelper(line) + + if(self.mode == "TSV"): + ''' + give the matcher directly to implementing class + and let it decide what to do when regEx is matched and unmatched + ''' + self.parse_into_tsv(matcher) + elif(self.mode == "SQL"): + self.parse_into_db(matcher) + else: + raise NotImplemented("Mode: " + self.mode) + + number_of_processed_lines += 1 + + #print("Processed lines: %d\r" % (number_of_processed_lines), end="") + + self.input_file.close() + + if(self.mode == "SQL"): + self.sql_file.write(";\n COMMIT;") + self.sql_file.close() + + if 'outputFile' in locals(): + self.output_file.flush() + self.output_file.close() + + # fuckedUpCount is calculated in implementing class + logging.info("Finished with " + str(self.fucked_up_count) + " fucked up line") + + def concat_regex_groups(self, group_list, col_list, matcher): + ret_val = "" + if col_list == None: + ret_val = self.seperator.join('%s' % (matcher.group(i)) for i in group_list) + else: + for i in range(len(group_list)): + if DbScriptHelper.keywords['string'] in self.db_table_info['columns'][col_list[i]]['colinfo']: + ret_val += "\"" + re.escape(matcher.group(group_list[i])) + "\", " + else: + ret_val += matcher.group(group_list[i]) + ", " + ret_val = ret_val[:-2] + return ret_val + + ##### Below methods force associated properties to be defined in any derived class ##### + + @abstractproperty + def base_matcher_pattern(self): + raise NotImplemented + + @abstractproperty + def input_file_name(self): + raise NotImplemented + + @abstractproperty + def number_of_lines_to_be_skipped(self): + raise NotImplemented + + @abstractproperty + def db_table_info(self): + raise NotImplemented + + @abstractproperty + def end_of_dump_delimiter(self): + raise NotImplemented diff --git a/idp/parser/directorsparser.py b/idp/parser/directorsparser.py new file mode 100644 index 0000000..6f77ca7 --- /dev/null +++ b/idp/parser/directorsparser.py @@ -0,0 +1,105 @@ +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +from .baseparser import * + + +class DirectorsParser(BaseParser): + """ + RegExp: /(.*?)\t+((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\s*(\(.*\)|EDIT)?\s*(<.*>)?$/gm + pattern: (.*?)\t+((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\s*(\(.*\)|EDIT)?\s*(<.*>)?$ + flags: gm + 10 capturing groups: + group 1: (.*?) surname, name + group 2: #TITLE (UNIQUE KEY) + group 3: (.*? \(\S{4,}\)) movie name + year + group 4: (\(\S+\)) type ex:(TV) + group 5: (\{(.*?) ?(\(\S+?\))?\}) series info ex: {Ally Abroad (#3.1)} + group 6: (.*?) episode name ex: Ally Abroad + group 7: (\(\S+?\)) episode number ex: (#3.1) + group 8: (\{\{SUSPENDED\}\}) is suspended? + group 9: (\(.*\)) info + group 10: () + """ + + # properties + base_matcher_pattern = '(.*?)\t+((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\s*(\(.*\)|EDIT)?\s*(<.*>)?$' + input_file_name = "directors.list" + number_of_lines_to_be_skipped = 235 + db_table_info = { + 'tablename' : 'directors', + 'columns' : [ + {'colname' : 'name', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'surname', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'title', 'colinfo' : DbScriptHelper.keywords['string'] + '(255) NOT NULL'}, + {'colname' : 'info', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'} + ], + 'constraints' : 'PRIMARY KEY(title)' + } + end_of_dump_delimiter = "" + + name = "" + surname = "" + + def __init__(self, preferences_map): + super(DirectorsParser, self).__init__(preferences_map) + self.first_one = True + + def parse_into_tsv(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(len(matcher.group(1).strip()) > 0): + namelist = matcher.group(1).split(', ') + if(len(namelist) == 2): + self.name = namelist[1] + self.surname = namelist[0] + else: + self.name = namelist[0] + self.surname = "" + + self.tsv_file.write(self.name + self.seperator + self.surname + self.seperator + self.concat_regex_groups([2,9], None, matcher) + "\n") + elif(len(matcher.get_last_string()) == 1): + pass + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 + + def parse_into_db(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(len(matcher.group(1).strip()) > 0): + namelist = matcher.group(1).split(', ') + if(len(namelist) == 2): + self.name = namelist[1] + self.surname = namelist[0] + else: + self.name = namelist[0] + self.surname = "" + + if(self.first_one): + self.sql_file.write("(\"" + self.name + "\", \"" + self.surname + "\", " + self.concat_regex_groups([2,9], [2,3], matcher) + ")") + self.first_one = False; + else: + self.sql_file.write(",\n(\"" + self.name + "\", \"" + self.surname + "\", " + self.concat_regex_groups([2,9], [2,3], matcher) + ")") + + elif(len(matcher.get_last_string()) == 1): + pass + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 diff --git a/idp/parser/genresparser.py b/idp/parser/genresparser.py new file mode 100644 index 0000000..cdbe049 --- /dev/null +++ b/idp/parser/genresparser.py @@ -0,0 +1,75 @@ +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +from .baseparser import * + + +class GenresParser(BaseParser): + """ + RegExp: /((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\t+(.*)$/gm + pattern: ((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\t+(.*)$ + flags: gm + 8 capturing groups: + group 1: #TITLE (UNIQUE KEY) + group 2: (.*? \(\S{4,}\)) movie name + year + group 3: (\(\S+\)) type ex:(TV) + group 4: (\{(.*?) ?(\(\S+?\))?\}) series info ex: {Ally Abroad (#3.1)} + group 5: (.*?) episode name ex: Ally Abroad + group 6: ((\(\S+?\)) episode number ex: (#3.1) + group 7: (\{\{SUSPENDED\}\}) is suspended? + group 8: (.*) genre + """ + + # properties + base_matcher_pattern = "((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\t+(.*)$" + input_file_name = "genres.list" + number_of_lines_to_be_skipped = 378 + db_table_info = { + 'tablename' : 'genres', + 'columns' : [ + {'colname' : 'title', 'colinfo' : DbScriptHelper.keywords['string'] + '(255) NOT NULL'}, + {'colname' : 'genre', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'} + ], + 'constraints' : 'PRIMARY KEY(title)' + } + end_of_dump_delimiter = "" + + def __init__(self, preferences_map): + super(GenresParser, self).__init__(preferences_map) + self.first_one = True + + def parse_into_tsv(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + self.tsv_file.write(self.concat_regex_groups([1,8], None, matcher) + "\n") + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 + + def parse_into_db(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(self.first_one): + self.sql_file.write("(" + self.concat_regex_groups([1,8], [0,1], matcher) + ")") + self.first_one = False; + else: + self.sql_file.write(",\n(" + self.concat_regex_groups([1,8], [0,1], matcher) + ")") + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 diff --git a/idp/parser/moviesparser.py b/idp/parser/moviesparser.py index 021fb73..bc488d4 100644 --- a/idp/parser/moviesparser.py +++ b/idp/parser/moviesparser.py @@ -1,65 +1,83 @@ -from .baseparser import BaseParser -from ..utils.regexhelper import * -import logging - -class MoviesParser(BaseParser): - """ - RegExp: /(.*?) (\(\S{4,}\))\s?(\(.+\))?\s?(\{(.*?)(\(.+?\))\})?\s*(\{\{SUSPENDED\}\})?\s*(.*$)/gm - pattern: (.*?) (\(\S{4,}\))\s?(\(.+\))?\s?(\{(.*?)(\(.+?\))\})?\s*(\{\{SUSPENDED\}\})?\s*(.*$) - flags: gm - 6 capturing groups: - group 1: (.*?) title - group 2: (\(\S{4,}\)) year - group 3: (\(.+\)) type ex:(TV) - group 4: (\{(.*?)(\(.+?\))\}) series info ex: {Ally Abroad (#3.1)} - group 5: (.*?) episode name ex: Ally Abroad - group 6: (\(.+?\)) episode number ex: (#3.1) - group 7: (\{\{SUSPENDED\}\}) is suspended? - group 8: (.*$) year - """ - - # properties - baseMatcherPattern = "(.*?) (\(\S{4,}\))\s?(\(.+\))?\s?(\{(.*?)(\(.+?\))\})?\s*(\{\{SUSPENDED\}\})?\s*(.*$)" - inputFileName = "movies.list" - numberOfLinesToBeSkipped = 15 - - def __init__(self, preferencesMap): - self._preferencesMap = preferencesMap - - @property - def preferencesMap(self): - return self._preferencesMap - - def parse_into_tsv(self): - import time - - startTime = time.time() - - inputFile = self.get_input_file() - outputFile = self.get_output_file() - counter = 0 - fuckedUpCount = 0 - numberOfProcessedLines = 0 - - for line in inputFile : - if(numberOfProcessedLines > self.numberOfLinesToBeSkipped): - matcher = RegExHelper(line) - isMatch = matcher.match(self.baseMatcherPattern) - - if(isMatch): - outputFile.write(matcher.group(1) + "," + matcher.group(2) + "," + matcher.group(3) + "," + matcher.group(5) + "," + matcher.group(6) + "," + matcher.group(7) + "," + matcher.group(8) + "\n") - else: - logging.critical("This line is fucked up: " + line) - fuckedUpCount += 1 - numberOfProcessedLines += 1 - - outputFile.flush() - outputFile.close() - inputFile .close() - - logging.info("Finished with " + str(fuckedUpCount) + " fucked up line\n") - logging.info("Duration: " + str(round(time.time() - startTime))) - - def parse_into_db(self): - #TODO - pass +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +from .baseparser import * + + +class MoviesParser(BaseParser): + """ + Parses movies.list dump + + RegExp: /((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\t+(.*)$/gm + pattern: ((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\t+(.*)$ + flags: gm + 8 capturing groups: + group 1: #TITLE (UNIQUE KEY) + group 2: (.*? \(\S{4,}\)) movie name + year + group 3: (\(\S+\)) type ex:(TV) + group 4: (\{(.*?) ?(\(\S+?\))?\}) series info ex: {Ally Abroad (#3.1)} + group 5: (.*?) episode name ex: Ally Abroad + group 6: ((\(\S+?\)) episode number ex: (#3.1) + group 7: (\{\{SUSPENDED\}\}) is suspended? + group 8: (.*) year + """ + + # properties + base_matcher_pattern = "((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)\t+(.*)$" + input_file_name = "movies.list" + #FIXME: zafer: I think using a static number is critical for us. If imdb sends a new file with first 10 line fucked then we're also fucked + number_of_lines_to_be_skipped = 15 + db_table_info = { + 'tablename' : 'movies', + 'columns' : [ + {'colname' : 'title', 'colinfo' : DbScriptHelper.keywords['string'] + '(255) NOT NULL'}, + {'colname' : 'full_name', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'type', 'colinfo' : DbScriptHelper.keywords['string'] + '(20)'}, + {'colname' : 'ep_name', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'ep_num', 'colinfo' : DbScriptHelper.keywords['string'] + '(20)'}, + {'colname' : 'suspended', 'colinfo' : DbScriptHelper.keywords['string'] + '(20)'}, + {'colname' : 'year', 'colinfo' : DbScriptHelper.keywords['string'] + '(20)'} + ], + 'constraints' : 'PRIMARY KEY(title)' + } + end_of_dump_delimiter = "--------------------------------------------------------------------------------" + + def __init__(self, preferences_map): + super(MoviesParser, self).__init__(preferences_map) + self.first_one = True + + def parse_into_tsv(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + self.tsv_file.write(self.concat_regex_groups([1,2,3,5,6,7,8], None, matcher) + "\n") + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 + + def parse_into_db(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(self.first_one): + self.sql_file.write("(" + self.concat_regex_groups([1,2,3,5,6,7,8], [0,1,2,3,4,5,6], matcher) + ")") + self.first_one = False; + else: + self.sql_file.write(",\n(" + self.concat_regex_groups([1,2,3,5,6,7,8], [0,1,2,3,4,5,6], matcher) + ")") + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 diff --git a/idp/parser/parsinghelper.py b/idp/parser/parsinghelper.py index 00be657..6a480b0 100644 --- a/idp/parser/parsinghelper.py +++ b/idp/parser/parsinghelper.py @@ -1,34 +1,74 @@ -from idp import settings +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + import logging +import traceback +from idp import settings + class ParsingHelper(object): - """ParsingHelper manages parsing order""" + """ + ParsingHelper manages parsing order + """ @staticmethod - def parse_all(preferencesMap): + def parse_one(item, preferences_map): - def get_parser_class_for( itemName ): + def get_parser_class_for(item_name): """ Thanks to http://stackoverflow.com/a/452981 """ - kls = "idp.parser." + itemName + "parser." + itemName.title() + "Parser" + kls = "idp.parser." + item_name + "parser." + item_name.title() + "Parser" parts = kls.split('.') module = ".".join(parts[:-1]) m = __import__( module ) for comp in parts[1:]: - m = getattr(m, comp) + m = getattr(m, comp) return m + try: + ParserClass = get_parser_class_for(item) + except Exception as e: + logging.error("No parser found for: " + item + "\n\tException is: " + str(e)) + return 1 + logging.info("___________________") + logging.info("Parsing " + item + "...") + parser = ParserClass(preferences_map) + try: + parser.start_processing() + except Exception as e: + logging.error("Exception occured while parsing item: " + item + "\n\tException is: " + str(e)) + traceback.print_exc() + logging.info("Parsing finished for item: " + item) + + @staticmethod + def parse_all(preferences_map): for item in settings.LISTS: - try: - ParserClass = get_parser_class_for(item) - except Exception as e: - logging.error("No parser found for: " + item + "\n\tException is: " + str(e)) - continue - logging.info("Parsing " + item + "...") - parser = ParserClass(preferencesMap) - try: - parser.start_processing() - except Exception as e: - logging.error("File not found for " + item + "\n\tException is: " + str(e)) - logging.info("Parsing finished.") \ No newline at end of file + ParsingHelper.parse_one(item, preferences_map) + logging.info("All parsing finished.") + +if __name__ == "__main__": + """ + For debugging purposes + """ + print("Parsing only one file for debugging purposes...") + preferences_map = { + "mode":"TSV", + "inputDir": "../../samples/imdb_lists/", + "outputDir": "../../samples/idp_files/" + } + ParsingHelper.parse_one("movies", preferences_map) diff --git a/idp/parser/plotparser.py b/idp/parser/plotparser.py index 39ac5d7..066c20c 100644 --- a/idp/parser/plotparser.py +++ b/idp/parser/plotparser.py @@ -1,77 +1,103 @@ -from .baseparser import BaseParser -from ..utils.regexhelper import * -import logging - -class PlotParser(BaseParser): - """ - RegExp: /(.+?): (.*)/g - pattern: (.+?): (.*) - flags: g - 2 capturing groups: - group 1: (.+?) type of the line - group 2: (.*) if the line-type is PL then this line is plot, not the whole but one line of it - if the line-type is MV then this line is movie - """ - - # properties - baseMatcherPattern = "(.+?): (.*)" - inputFileName = "plot.list" - numberOfLinesToBeSkipped = 15 - - def __init__(self, preferencesMap): - self._preferencesMap = preferencesMap - - @property - def preferencesMap(self): - return self._preferencesMap - - def parse_into_tsv(self): - import time - startTime = time.time() - - inputFile = self.get_input_file() - outputFile = self.get_output_file() - counter = 0 - fuckedUpCount = 0 - - title = "" - plot = "" - - numberOfProcessedLines = 0 - - for line in inputFile : - if(numberOfProcessedLines > self.numberOfLinesToBeSkipped): - matcher = RegExHelper(line) - isMatch = matcher.match(self.baseMatcherPattern) - - if(isMatch): - if(matcher.group(1) == "MV"): #Title - if(title != ""): - outputFile.write(title + self.seperator + plot + "\n") - - plot = "" - title = matcher.group(2) - - elif(matcher.group(1) == "PL"): #Descriptive text - plot += matcher.group(2) - elif(matcher.group(1) == "BY"): - continue - else: - logging.critical("Unhandled abbreviation: " + matcher.group(1) + " in " + line) - #else: - #just ignore this part, useless lines - numberOfProcessedLines += 1 - - # Covers the last item - outputFile.write(title + self.seperator + plot + "\n") - - outputFile.flush() - outputFile.close() - inputFile.close() - - logging.info("Finished with " + str(fuckedUpCount) + " fucked up line\n") - logging.info("Duration: " + str(round(time.time() - startTime))) - - def parse_into_db(self): - #TODO - pass +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +from .baseparser import * + + +class PlotParser(BaseParser): + """ + RegExp: /(.+?): (.*)/g + pattern: (.+?): (.*) + flags: g + 2 capturing groups: + group 1: (.+?) type of the line + group 2: (.*) if the line-type is PL then this line is plot, not the whole but one line of it + if the line-type is MV then this line is movie + """ + + # properties + base_matcher_pattern = "(.+?): (.*)" + input_file_name = "plot.list" + number_of_lines_to_be_skipped = 15 + db_table_info = { + 'tablename' : 'plot', + 'columns' : [ + {'colname' : 'title', 'colinfo' : DbScriptHelper.keywords['string'] + '(255) NOT NULL'}, + {'colname' : 'plot', 'colinfo' : DbScriptHelper.keywords['string'] + '(4000)'} + ], + 'constraints' : 'PRIMARY KEY(title)' + } + end_of_dump_delimiter = "" + + def __init__(self, preferences_map): + super(PlotParser, self).__init__(preferences_map) + self.first_one = True + + # specific to this class + self.title = "" + self.plot = "" + + def parse_into_tsv(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(matcher.group(1) == "MV"): #Title + if(self.title != ""): + self.tsv_file.write(self.title + self.seperator + self.plot + "\n") + + self.plot = "" + self.title = matcher.group(2) + + elif(matcher.group(1) == "PL"): #Descriptive text + self.plot += matcher.group(2) + elif(matcher.group(1) == "BY"): + pass + else: + logging.critical("Unhandled abbreviation: " + matcher.group(1) + " in " + line) + #else: + #just ignore this part, useless lines + + """ + FIXME: this parsing misses the last entry + need to execute following just after looping the input file's lines: + # Covers the last item + outputFile.write(title + self.seperator + plot + "\n") + + consider writing to the file in "BY:" condition + """ + + def parse_into_db(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(matcher.group(1) == "MV"): #Title + if(self.title != ""): + if(self.first_one): + self.sql_file.write("(\"" + self.title + "\", \"" + self.plot + "\")") + self.first_one = False; + else: + self.sql_file.write(",\n(\"" + self.title + "\", \"" + self.plot + "\")") + + self.plot = "" + self.title = matcher.group(2) + + elif(matcher.group(1) == "PL"): #Descriptive text + self.plot += matcher.group(2) + elif(matcher.group(1) == "BY"): + pass + else: + logging.critical("Unhandled abbreviation: " + matcher.group(1) + " in " + line) diff --git a/idp/parser/ratingsparser.py b/idp/parser/ratingsparser.py new file mode 100644 index 0000000..90e4fff --- /dev/null +++ b/idp/parser/ratingsparser.py @@ -0,0 +1,79 @@ +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +from .baseparser import * + + +class RatingsParser(BaseParser): + """ + RegExp: /\s*(\S*)\s*(\S*)\s*(\S*)\s*((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)$/gm + pattern: \s*(\S*)\s*(\S*)\s*(\S*)\s*((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)$ + flags: gm + 10 capturing groups: + group 1: (\S*) distribution + group 2: (\S*) votes + group 3: (\S*) rank + group 4: #TITLE (UNIQUE KEY) + group 5: (.*? \(\S{4,}\)) movie name + year + group 6: (\(.+\)) type ex:(TV) + group 7: (\{(.*?)\s?(\(.+?\))\}) series info ex: {Ally Abroad (#3.1)} + group 8: (.*?) episode name ex: Ally Abroad + group 9: (\(.+?\)) episode number ex: (#3.1) + group 10: (\{\{SUSPENDED\}\}) is suspended? + """ + + # properties + base_matcher_pattern = "\s*(\S*)\s*(\S*)\s*(\S*)\s*((.*? \(\S{4,}\)) ?(\(\S+\))? ?(?!\{\{SUSPENDED\}\})(\{(.*?) ?(\(\S+?\))?\})? ?(\{\{SUSPENDED\}\})?)$" + input_file_name = "ratings.list" + number_of_lines_to_be_skipped = 28 + db_table_info = { + 'tablename' : 'ratings', + 'columns' : [ + {'colname' : 'distribution', 'colinfo' : DbScriptHelper.keywords['string'] + '(127) NOT NULL'}, + {'colname' : 'votes', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'rank', 'colinfo' : DbScriptHelper.keywords['string'] + '(127)'}, + {'colname' : 'title', 'colinfo' : DbScriptHelper.keywords['string'] + '(255) NOT NULL'} + ], + 'constraints' : 'PRIMARY KEY(title)' + } + end_of_dump_delimiter = "" + + def __init__(self, preferences_map): + super(RatingsParser, self).__init__(preferences_map) + self.first_one = True + + def parse_into_tsv(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + self.tsv_file.write(self.concat_regex_groups([1,2,3,4], None, matcher) + "\n") + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 + + def parse_into_db(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(self.first_one): + self.sql_file.write("(" + self.concat_regex_groups([1,2,3,4], [0,1,2,3], matcher) + ")") + self.first_one = False; + else: + self.sql_file.write(",\n(" + self.concat_regex_groups([1,2,3,4], [0,1,2,3], matcher) + ")") + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 diff --git a/idp/parser/triviaparser.py b/idp/parser/triviaparser.py new file mode 100644 index 0000000..a0e8af2 --- /dev/null +++ b/idp/parser/triviaparser.py @@ -0,0 +1,73 @@ +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +from .baseparser import * + + +class TriviaParser(BaseParser): + """ + RegExp: /((.+?) (.*))|\n/g + pattern: ((.+?) (.*))|\n + flags: g + 2 capturing groups: + group 1: (.+?) type of the line + group 2: (.*) if the line-type is - then this line is plot, not the whole but one line of it + if the line-type is # then this line is movie + """ + + # properties + base_matcher_pattern = "((.+?) (.*))|\n" + input_file_name = "trivia.list" + number_of_lines_to_be_skipped = 15 + db_table_info = { + 'tablename' : 'trivia', + 'columns' : [ + { + 'colname' : '', + 'colinfo' : DbScriptHelper.keywords['string'] + '(255) NOT NULL' + } + ], + 'constraints' : '' + } + end_of_dump_delimiter = "" + + title = "" + trivia = "" + + def __init__(self, preferences_map): + super(TriviaParser, self).__init__(preferences_map) + self.first_one = True + + def parse_into_tsv(self, matcher): + is_match = matcher.match(self.base_matcher_pattern) + + if(is_match): + if(matcher.group(2) == "#"): #Title + self.title = matcher.group(3) + elif(matcher.group(2) == "-"): #Descriptive text + self.trivia = matcher.group(3) + elif(matcher.group(2) == " "): + self.trivia += ' ' + matcher.group(3) + else: + self.tsv_file.write(self.title + self.seperator + self.trivia + "\n") + else: + logging.critical("This line is fucked up: " + matcher.get_last_string()) + self.fucked_up_count += 1 + + def parse_into_db(self, matcher): + #TODO + pass diff --git a/idp/settings.py.example b/idp/settings.py.example index 19119b8..395c2b4 100644 --- a/idp/settings.py.example +++ b/idp/settings.py.example @@ -1,5 +1,22 @@ -SOURCE_PATH = "/home/destan/Desktop/" -DESTINATION_PATH = "/home/destan/Desktop/" +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +INPUT_DIR = "/path/to/lists/files/" +OUTPUT_DIR = "/path/to/tsv/outputs/" INTERFACES_SERVER = "ftp.fu-berlin.de" INTERFACES_DIRECTORY = "pub/misc/movies/database/" @@ -8,12 +25,13 @@ INTERFACES_DIRECTORY = "pub/misc/movies/database/" #ftp://ftp.sunet.se/pub/tv+movies/imdb/ LISTS = [ + "directors", + "genres", "movies", "plot", "actors", "actresses", "aka-names", "aka-titles", - "directors", "ratings" -] \ No newline at end of file +] diff --git a/idp/utils/dbscripthelper.py b/idp/utils/dbscripthelper.py new file mode 100644 index 0000000..8cfa4e5 --- /dev/null +++ b/idp/utils/dbscripthelper.py @@ -0,0 +1,42 @@ +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +import os + + +class DbScriptHelper(object): + keywords = { + 'string': "VARCHAR", + 'number': "NUMERIC", + 'date': "DATE" + } + + scripts = { + 'drop': "DROP TABLE ", + 'create': "CREATE TABLE ", + 'insert': "INSERT INTO " + } + + def __init__(self, db_table_info): + self.scripts = { + 'drop': "DROP TABLE ", + 'create': "CREATE TABLE ", + 'insert': "INSERT INTO " + } + self.scripts['drop'] += db_table_info['tablename'] + ";" + os.linesep + self.scripts['create'] += db_table_info['tablename'] + "(" + ', '.join(filter(None, (', '.join('%s %s' % (col['colname'], col['colinfo']) for col in db_table_info['columns']), db_table_info['constraints']))) + ") CHARACTER SET utf8 COLLATE utf8_bin;" + os.linesep + self.scripts['insert'] += db_table_info['tablename'] + "(" + ', '.join(col['colname'] for col in db_table_info['columns']) + ") VALUES" + os.linesep \ No newline at end of file diff --git a/idp/utils/decorators.py b/idp/utils/decorators.py new file mode 100644 index 0000000..46f9561 --- /dev/null +++ b/idp/utils/decorators.py @@ -0,0 +1,33 @@ +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +import datetime +import logging + + +def duration_logged(func): + ''' + As the name suggests, calculates the execution duration of the function which is annotated by this decorator + ''' + def inner(*args, **kwargs): + start_time = datetime.datetime.now() + ret_val = func(*args, **kwargs) + end_time = datetime.datetime.now() + duration = (end_time - start_time).total_seconds() #difference of 2 datetime is a timedelta + logging.info("Parsing took " + str(duration) + " seconds") + return ret_val + return inner \ No newline at end of file diff --git a/idp/utils/filehandler.py b/idp/utils/filehandler.py index bd4bc84..d1c5227 100644 --- a/idp/utils/filehandler.py +++ b/idp/utils/filehandler.py @@ -1,63 +1,95 @@ -import gzip -import os.path -from ..settings import * -import logging - -def get_full_path(filename, isCompressed = False): - """ - constructs a full path for a dump file in the SOURCE_PATH - filename should be without '.list' - """ - if(isCompressed): - return os.path.join(SOURCE_PATH, filename) + ".gz" - else: - return os.path.join(SOURCE_PATH, filename) - -def get_full_path_for_tsv(filename): - return get_full_path(filename) + ".tsv" - -def get_decompressed_file_name(fullpath): - return fullpath[:-3] - -def extract(fullpath): - try: - logging.info('started to extract list: %s', fullpath) - with gzip.open(fullpath, 'rb') as f: - file_content = f.read() - listfile = open(get_decompressed_file_name(fullpath), 'wb') - listfile.write(file_content) - listfile.close() - logging.info(fullpath + ' list extracted successfully') - except Exception as e: - logging.error('error when extracting list: ' + fullpath + "\n\t" + str(e)) - return 1 - return 0 - -def openfile(fullFilePath): - - logging.info("Trying to find file: %s", fullFilePath) - if os.path.isfile(fullFilePath): - logging.info("File found: %s", fullFilePath) - return open(fullFilePath, "r", encoding='iso-8859-1') - - logging.error("File cannot be found: %s", fullFilePath) - -# -#this part removed until python 3.3 becomes available for ubuntu LTS and debian -# -# print("Trying to find file:", fullFilePath) -# if os.path.isfile(fullFilePath): -# print("File found:", fullFilePath) -# return gzip.open(fullFilePath, 'rt') -# print("File cannot be found:", fullFilePath) - - logging.info("Trying to find file: %s", fullFilePath + ".gz") - if os.path.isfile(fullFilePath + ".gz"): - logging.info("File found: %s", fullFilePath + ".gz") - if extract(fullFilePath + ".gz") == 0: - return open(fullFilePath, "r", encoding='iso-8859-1') - else: - raise RuntimeError("Unknown error occured") - logging.error("File cannot be found: %s", fullFilePath + ".gz") - - raise RuntimeError("FileNotFoundError: " + fullFilePath) \ No newline at end of file +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +import gzip +import os.path +import logging +from ..settings import * + + +class FileHandler(object): + def __init__(self, list_name, preferences_map): + self.list_name = list_name + self.preferences_map = preferences_map + + def full_path(self): + return os.path.join(self.preferences_map['input_dir'], self.list_name) + + def tsv_path(self): + return os.path.join(self.preferences_map['output_dir'], self.list_name) + ".tsv" + + def sql_path(self): + return os.path.join(self.preferences_map['output_dir'], self.list_name) + ".sql" + + def get_input_file(self): + full_file_path = self.full_path() + logging.info("Trying to find file: %s", full_file_path) + if os.path.isfile(full_file_path): + logging.info("File found: %s", full_file_path) + return open(full_file_path, "r", encoding='iso-8859-1') + + logging.error("File cannot be found: %s", full_file_path) + + logging.info("Trying to find file: %s", full_file_path + ".gz") + if os.path.isfile(full_file_path + ".gz"): + logging.info("File found: %s", full_file_path + ".gz") + if extract(full_file_path + ".gz") == 0: + return open(full_file_path, "r", encoding='iso-8859-1') + else: + raise RuntimeError("Unknown error occured") + + logging.error("File cannot be found: %s", full_file_path + ".gz") + + raise RuntimeError("FileNotFoundError: %s", full_file_path) + +#this part removed until python 3.3 becomes available for ubuntu LTS and debian +# +# print("Trying to find file:", full_file_path) +# if os.path.isfile(full_file_path): +# print("File found:", full_file_path) +# return gzip.open(full_file_path, 'rt') +# print("File cannot be found:", full_file_path) + + def get_tsv_file(self): + return open(self.tsv_path(), "w", encoding='utf-8') + + def get_sql_file(self): + return open(self.sql_path(), "w", encoding='utf-8') + + def extract(gzip_path): + try: + logging.info("Started to extract list: %s", gzip_path) + with gzip.open(gzip_path, "rb") as f: + file_content = f.read() + list_file = open(gzip_path[:-3], "wb") + list_file.write(file_content) + list_file.close() + logging.info(gzip_path + " list extracted successfully") + except Exception as e: + logging.error("Error when extracting list: " + gzip_path + "\n\t" + str(e)) + return 1 + return 0 + + def get_full_path(filename, isCompressed = False): + """ + constructs a full path for a dump file in the INPUT_DIR + filename should be without '.list' + """ + if(isCompressed): + return os.path.join(INPUT_DIR, filename) + ".gz" + else: + return os.path.join(INPUT_DIR, filename) \ No newline at end of file diff --git a/idp/utils/freebaseagent.py b/idp/utils/freebaseagent.py new file mode 100644 index 0000000..012a34e --- /dev/null +++ b/idp/utils/freebaseagent.py @@ -0,0 +1,91 @@ +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +import json +import urllib + + +class FreebaseAgent(object): + """Helper class to retrieve IMDb ids from freebase. + + Currently only supports movies. TV series support will + hopefully be added if need arises. + """ + + def __init__(self): + super(FreebaseAgent, self).__init__() + self.API_KEY = 'YOUR-API-KEY-GOES-HERE' #TODO read these values from config + self.topic_service_url = 'https://www.googleapis.com/freebase/v1/topic' + self.search_service_url = 'https://www.googleapis.com/freebase/v1/search' + + def get_imdb_id(self, movie_name): + """Returns the IMDb id of a movie, given its title. + + The returned title is the one with the highest + freebase confidence score. + + Returns None if no such movie exists in freebase. + """ + mid = self.get_topic_id(movie_name) + topic = self.get_topic(mid) + return topic + + def get_topic_id(self, name, entity_type='/film/film'): + """Gets the topic id (aka mid, freebase id) of a title. + """ + params = { + 'query': name, + 'type': entity_type, + 'limit': 1 + } + url = self.search_service_url + '?' + urllib.urlencode(params) + response = json.loads(urllib.urlopen(url).read()) + + for result in response.get('result'): + mid = result.get('mid', None) + return mid + return None + + def get_topic(self, mid): + """Gets the IMDb id of a freebase topic. Returns None if no + such thing exists. + """ + params = { + 'filter': '/type/object/key' + } + url = self.topic_service_url + mid + '?' + urllib.urlencode(params) + topic = json.loads(urllib.urlopen(url).read()) + + for property in topic['property']: + for value in topic['property'][property]['values']: + if value['text'].startswith('/authority/imdb/title'): + return value['text'].split('/')[-1] + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Retrieve imdb id from freebase.") + parser.add_argument('movieName', help='The name of the movie') + args = parser.parse_args() + + agent = FreebaseAgent() + mid = agent.getTopicId(args.movieName) + print 'freebase topic id (mid) is', mid + topic = agent.getTopic(mid) + print 'imdb id is', topic, 'so the url is http://www.imdb.com/title/'+topic + print agent.getImdbId(args.movieName) \ No newline at end of file diff --git a/idp/utils/listdownloader.py b/idp/utils/listdownloader.py index 8629741..a2133f5 100644 --- a/idp/utils/listdownloader.py +++ b/idp/utils/listdownloader.py @@ -1,24 +1,45 @@ -from ..settings import * -from ftplib import FTP +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + import gzip -from .filehandler import * import logging +import os +from ftplib import FTP +from .filehandler import FileHandler +from ..settings import * + def download(): - logging.info("lists will downloaded from server:" + INTERFACES_SERVER) + logging.info("Lists will downloaded from server:" + INTERFACES_SERVER) ftp = FTP(INTERFACES_SERVER) ftp.login() + download_count = 0 - for list in LISTS: + + for list_item in LISTS: try: - logging.info("started to download list:" + list) - r = ftp.retrbinary('RETR '+INTERFACES_DIRECTORY+list+'.list.gz', - open(SOURCE_PATH+list+'.list.gz', 'wb').write) - logging.info(list + "list downloaded successfully") - download_count = download_count+1 - extract(get_full_path(list+".list", True)) + logging.info("Started to download list:" + list_item) + r = ftp.retrbinary("RETR " + INTERFACES_DIRECTORY + list_item + ".list.gz", open(os.path.join(INPUT_DIR, list_item + ".list.gz"), "wb").write) + logging.info(list_item + "list downloaded successfully") + download_count = download_count + 1 + FileHandler.extract(FileHandler.get_full_path(list_item + ".list", True)) except Exception as e: - print("ERROR: there is a problem when downloading list " + list + "\n\t" + str(e)) + logging.error("There is a problem when downloading list " + list_item + "\n\t" + str(e)) + logging.info(str(download_count) + " lists are downloaded") - ftp.quit() \ No newline at end of file + ftp.quit() diff --git a/idp/utils/loggerinitializer.py b/idp/utils/loggerinitializer.py new file mode 100644 index 0000000..beaa747 --- /dev/null +++ b/idp/utils/loggerinitializer.py @@ -0,0 +1,45 @@ +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +import logging +import os.path + + +def initialize_logger(preferences_map): + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + # create console handler and set level to info + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + formatter = logging.Formatter("%(levelname)s - %(message)s") + ch.setFormatter(formatter) + logger.addHandler(ch) + + # create error file handler and set level to error + ch = logging.FileHandler(os.path.join(preferences_map['output_dir'], "imdbparserError.log"),"w", encoding=None, delay="true") + ch.setLevel(logging.ERROR) + formatter = logging.Formatter("%(levelname)s - %(message)s") + ch.setFormatter(formatter) + logger.addHandler(ch) + + # create info file handler and set level to info + ch = logging.FileHandler(os.path.join(preferences_map['output_dir'], "imdbparserAll.log"),"w") + ch.setLevel(logging.INFO) + formatter = logging.Formatter("%(levelname)s - %(message)s") + ch.setFormatter(formatter) + logger.addHandler(ch) \ No newline at end of file diff --git a/idp/utils/regexhelper.py b/idp/utils/regexhelper.py index f3d4ca3..a402731 100644 --- a/idp/utils/regexhelper.py +++ b/idp/utils/regexhelper.py @@ -1,5 +1,23 @@ +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + import re + class RegExHelper(object): def __init__(self, matchstring): self.matchstring = matchstring @@ -9,7 +27,13 @@ def match(self,regexp): return bool(self.rematch) def group(self,i): - if self.rematch.group(i) is None : + if self.rematch.group(i) is None: return "" else: - return self.rematch.group(i) \ No newline at end of file + return self.rematch.group(i) + + def get_last_string(self): + """ + returns the last string that is examined + """ + return self.matchstring \ No newline at end of file diff --git a/idp/utils/test/__init__.py b/idp/utils/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/idp/utils/test/filehandler_test.py b/idp/utils/test/filehandler_test.py new file mode 100644 index 0000000..d445ddf --- /dev/null +++ b/idp/utils/test/filehandler_test.py @@ -0,0 +1,14 @@ +import unittest +from ..filehandler import * +from ... import settings + +class FileHandlerTests(unittest.TestCase): + def setUp(self): + self.list = 'movies' + + def test_get_full_path(self): + self.assertEqual(get_full_path(self.list), settings.INPUT_DIR+self.list) + + +if __name__ == '__main__': + unittest.main() diff --git a/imdb parse path.txt b/imdb_parse_path.txt similarity index 67% rename from imdb parse path.txt rename to imdb_parse_path.txt index c85dc23..2f11e7d 100644 --- a/imdb parse path.txt +++ b/imdb_parse_path.txt @@ -1,3 +1,22 @@ +/* +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +*/ + + + /******************************************************************************** * * * -> optional * diff --git a/imdbparser.py b/imdbparser.py index 654ce70..6f1d9b6 100755 --- a/imdbparser.py +++ b/imdbparser.py @@ -1,36 +1,46 @@ #!/usr/bin/env python3 +""" +This file is part of imdb-data-parser. + +imdb-data-parser is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +imdb-data-parser is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with imdb-data-parser. If not, see . +""" + +""" +Dealing with ": No such file or directory" error: +http://stackoverflow.com/a/8735625/878361 +""" + import sys import argparse -import logging -import logging.config +import datetime +from idp.utils.loggerinitializer import * from idp.parser.parsinghelper import ParsingHelper from idp.settings import * + # check python version if sys.version_info.major != 3: sys.exit("Error: wrong version! You need to install python3 to run this application properly.") -logging.config.fileConfig("logging.conf") - parser = argparse.ArgumentParser(description="an IMDB data parser") -parser.add_argument('-m', '--mode', help='Parsing mode, defines output of parsing process. Default: CSV', choices=['TSV', 'SQL', 'DB']) -parser.add_argument('-s', '--source_dir', help='source directory of interface lists') -parser.add_argument('-d', '--destination_dir', help='destination directory for outputs') +parser.add_argument('-m', '--mode', help='Parsing mode, defines output of parsing process. Default: TSV', choices=['TSV', 'SQL']) +parser.add_argument('-i', '--input_dir', help='source directory of interface lists') +parser.add_argument('-o', '--output_dir', help='destination directory for outputs') parser.add_argument('-u', '--update_lists', action='store_true', help='downloads lists from server') args = parser.parse_args() -logging.info("mode:%s", args.mode) -logging.info("source_dir:%s", args.source_dir) -logging.info("destination_dir:%s", args.destination_dir) -logging.info("update_lists:%s", args.update_lists) - -if args.update_lists: - from idp.utils import listdownloader - logging.info("Downloading IMDB dumps, this may take a while depending on your connection speed") - listdownloader.download() - -logging.info("Parsing, please wait. This may take very long time...") # preparing preferences map if args.mode: @@ -38,23 +48,41 @@ else: #default mode = "TSV" -if args.source_dir: - sourcePath = args.source_dir +if args.input_dir: + input_dir = args.input_dir else: - sourcePath = SOURCE_PATH + input_dir = INPUT_DIR -if args.destination_dir: - destinationPath = args.source_dir +postfix = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S") + '_ImdbParserOutput' +if args.input_dir: + output_dir = os.path.join(args.output_dir, postfix) else: - destinationPath = DESTINATION_PATH + output_dir = os.path.join(OUTPUT_DIR, postfix) -preferencesMap = { - "mode":mode, - "destinationDir": args.destination_dir, - "sourcePath": sourcePath, - "destinationPath": destinationPath +if not os.path.exists(output_dir): + os.makedirs(output_dir) + +preferences_map = { + "mode":mode, + "input_dir": input_dir, + "output_dir": output_dir } -ParsingHelper.parse_all(preferencesMap) +initialize_logger(preferences_map) + +logging.info("mode:%s", mode) +logging.info("input_dir:%s", input_dir) +logging.info("output_dir:%s", output_dir) +logging.info("update_lists:%s", args.update_lists) + +if args.update_lists: + from idp.utils import listdownloader + logging.info("Downloading IMDB dumps, this may take a while depending on your connection speed") + listdownloader.download() + +logging.info("Parsing, please wait. This may take very long time...") + +ParsingHelper.parse_all(preferences_map) -print ("All done, enjoy ;)") \ No newline at end of file +logging.info("Check out output folder: %s", output_dir) +print ("All done, enjoy ;)") #don't print this via logger, this is part of the program diff --git a/logging.conf b/logging.conf deleted file mode 100644 index 03d062d..0000000 --- a/logging.conf +++ /dev/null @@ -1,21 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=consoleHandler - -[formatters] -keys=simpleFormatter - -[logger_root] -level=INFO -handlers=consoleHandler - -[handler_consoleHandler] -class=StreamHandler -level=DEBUG -formatter=simpleFormatter -args=(sys.stdout,) - -[formatter_simpleFormatter] -format=%(levelname)s - %(message)s \ No newline at end of file